Java两种方法实现简单爬虫

第一种：利用HttpClient和Jsoup爬取网站数据

package com.company;
import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
public class Main {
public static void main (String[] args) throws Exception {
    // 创建默认的客户端实例
    CloseableHttpClient httpClient = HttpClients.createDefault();

    // 创建get请求实例
    HttpGet httpget = new HttpGet("/");

    System.out.println("executing request " + httpget.getURI());
    String content = "";

    try {
        // 客户端执行get请求返回响应
        CloseableHttpResponse response = httpClient.execute(httpget);

        // 服务器响应状态行
        System.out.println(response.getStatusLine().toString());

        Header[] heads = response.getAllHeaders();
        HttpEntity entity = response.getEntity();

        if (entity != null) {
            content += EntityUtils.toString(entity, "utf-8");
            EntityUtils.consume(entity);// 关闭内容流
        }

        System.out.println(content);
        System.out.println(response.getHeaders("Content-Type"));
        // 打印所有响应头

        for(Header h:heads){
            System.out.println(h.getName()+":"+h.getValue());
        }
    } finally {
        httpClient.close();
    }
}
}

第二种：直接利用Jsoup解析网址

package com.company;

import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import java.util.HashMap;
import java.util.Map;
public class splider {
public void jsoupText() throws Exception{
    Connection con = Jsoup.connect("/");
    Map<string,string> map = new HashMap<string, string="">();
    for (int i=0;i&lt;=10;i++){
        map.put(String.valueOf(i),String.valueOf(i));
    }
    for (Map.Entry<string,string> entry : map.entrySet()) {
        System.out.println(entry.getValue());
    }
    Document doc = con.get();
    for (Element ds:doc.select("body &gt; div:nth-child(9) &gt; div.page_left &gt; ul &gt; li")) {
        System.out.println(ds);
    }
}
</string,string></string,></string,string>

<string,string><string, string=""><string,string> public static void main(String[] args) throws Exception{
splider splider1 = new splider();
splider1.jsoupText();
}
}
</string,string></string,></string,string>

    public void jsoupText() throws Exception{
Document doc = Jsoup.connect("/").userAgent(“Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3800.0 Safari/537.36”).get();
for (Element ds:doc.select(“body > div:nth-child(9) > div.page_left > ul > li”)) {
System.out.println(ds);
}
}

    public static String httpPost(String url, Map<string,string> map, String cookie) throws IOException {
//获取请求连接
Connection con = Jsoup.connect(url);
//遍历生成参数
if(map!=null){
for (Map.Entry<string, string=""> entry : map.entrySet()) {
//添加参数
con.data(entry.getKey(), entry.getValue());
}
}
//插入cookie（头文件形式）
con.header(“Cookie”, cookie);
Document doc = con.post();
System.out.println(doc);
return doc.toString();
}
</string,></string,string>