Java爬虫实现爬取京东上的手机搜索页面 HttpCliient+Jsoup

 
 org.apache.httpcomponents
 httpclient
 4.5.3
 

 
 
 org.jsoup
 jsoup
 1.11.2
 

 
 
 log4j
 log4j
 1.2.17
 
 

log4j.rootLogger=INFO, Console 
 
#Console 
log4j.appender.Console=org.apache.log4j.ConsoleAppender
log4j.appender.Console.layout=org.apache.log4j.PatternLayout
log4j.appender.Console.layout.ConversionPattern=%d [%t] %-5p [%c] - %m%n

import java.io.IOException;
import org.apache.http.HttpEntity;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.apache.log4j.Logger;

public class getHTML {
 //建立日志
 private static Logger log = Logger.getLogger(getHTML.class);
 private static String getH(String url) throws ClientProtocolException, IOException {
 //控制台输出日志，这样每条访问的URL都可以在控制台上看到访问情况
 log.info("正在解析" + url);
 
 /* 
  * 以下内容为HttpClient建立连接的一般用法
  * 使用HttpClient建立客户端
  * 使用get方法访问指定URL
  * 获得应答
  * */
 
 CloseableHttpClient client = HttpClients.createDefault();
 HttpGet get = new HttpGet(url);
 CloseableHttpResponse response = client.execute(get);
 
 /*
  * 以下内容为将HTML内容转化为String
  * 获得应答体
  * 将应答体转为String格式，此处使用了EntityUtils中的toString方法，编码格式设置为"utf-8"
  * 完成后关闭客户端与应答
  * */
 HttpEntity entity = response.getEntity();
 String content;
 if (entity != null) {
  content = EntityUtils.toString(entity, "utf-8");
  client.close();
  response.close();
  return content;
 } else
  return null;
 }
 public static void urlControl(String baseurl, int page) throws ClientProtocolException, IOException {
 //设置当前页count
 int count = 1;
 //如果当前页小于想要爬取的页数则执行
 while (count < page) {
  //实际访问的URL为不变的URL值拼接上URL变化的值
  String u = baseurl + (2 * count - 1) + "&click=0";
  //此处调用ParseHTML类中的方法对URL中的HTML页面进行处理，后面详细介绍该类
  String content = ParseHTML.parse(getH(u)).toString();
  //此处调用WriteTo类中的方法对解析出来的内容写入到本地，后面详细介绍该类
  WriteTo.writeto(content);
  count++;
 }
 }
}

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

public class ParseHTML {
 
 public static StringBuilder parse(String content)
 {
 //使用Jsoup中的parse方法对已经转换为String的HTML内容进行分析，返回值为Document类
 Document doc = Jsoup.parse(content);
 //使用选择器select对需要找的元素进行抓取，例如第一个select中选择的是ul标签中class属性等于gl-warp clearfix的内容
 Elements ele = doc.select("ul[class = gl-warp clearfix]").select("li[class=gl-item]");
 //设置一个容器，用于装各个属性
 StringBuilder sb = new StringBuilder();
 //通过上一个选择器可以获得整个页面中所有符合要求的元素，也即各款手机，下面则需要对每款手机进行遍历，获取其属性
 for (Element e : ele) {
  //此处对各个属性的获取参考了网上一篇爬取京东上内容的文章，应该有其他不同的写法
  String id = e.attr("data-pid");
  String mingzi = e.select("div[class = p-name p-name-type-2]").select("a").select("em").text();
  String jiage = e.select("div[class=p-price]").select("strong").select("i").text();
  String pinglun = e.select("div[class=p-commit]").select("strong").select("a").text();
  //向容器中添加属性
  sb.append(id+"\t");
  sb.append(mingzi+"\t");
  sb.append(jiage+"\t");
  sb.append(pinglun+"\t");
  sb.append("\r\n");
 }
 return sb;
 }
}

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;

public class WriteTo {
 // 设置文件存放的位置
 private static File f = new File("C:\\jingdong.txt");
 public static void writeto(String content) throws IOException {
 //使用续写的方式，以免覆盖前面写入的内容
 BufferedWriter bw = new BufferedWriter(new FileWriter(f, true));
 bw.append(content);
 bw.flush();
 bw.close();
 }
}

import java.io.IOException;
import org.apache.http.client.ClientProtocolException;

public class MainControl {
 public static void main(String[] args) throws ClientProtocolException, IOException {
 // TODO Auto-generated method stub
 String baseurl = "https://search.jd.com/Search?keyword=%E6%89%8B%E6%9C%BA&enc="
  + "utf-8&qrst=1&rt=1&stop=1&vt=2&cid2=653&cid3=655&page=";
 int page = 5;//设置爬取页数
 getHTML.urlControl(baseurl, page);
 }
}