Java 网络爬虫新手入门详解

/**
 * httpclient + 正则表达式 获取虎扑新闻列表页
 * @param url 虎扑新闻列表页url
 */
public void httpClientList(String url){
 try {
  CloseableHttpClient httpclient = HttpClients.createDefault();
  HttpGet httpGet = new HttpGet(url);
  CloseableHttpResponse response = httpclient.execute(httpGet);
  if (response.getStatusLine().getStatusCode() == 200) {
   HttpEntity entity = response.getEntity();
   String body = EntityUtils.toString(entity,"utf-8");
 
   if (body!=null) {
     /*
     * 替换掉换行符、制表符、回车符，去掉这些符号，正则表示写起来更简单一些
     * 只有空格符号和其他正常字体
     */
    Pattern p = Pattern.compile("\t|\r|\n");
    Matcher m = p.matcher(body);
    body = m.replaceAll("");
    /*
     * 提取列表页的正则表达式
     * 去除换行符之后的 li
     * 
         
          与球迷亲切互动！凯尔特人官方晒球队开放训练日照片         
        
     */
    Pattern pattern = Pattern
      .compile("
\\s* 
\\s* (.*?)\\s* 
\\s* 
" );

    Matcher matcher = pattern.matcher(body);
    // 匹配出所有符合正则表达式的数据
    while (matcher.find()){
//      String info = matcher.group(0);
//      System.out.println(info);
     // 提取出链接和标题
     System.out.println("详情页链接："+matcher.group(1)+" ,详情页标题："+matcher.group(2));
    }
   }else {
    System.out.println("处理失败！！！获取正文内容为空");
   }
  } else {
   System.out.println("处理失败！！！返回状态码：" + response.getStatusLine().getStatusCode());
  }
 }catch (Exception e){
  e.printStackTrace();
 }

}

public static void main(String[] args) {
 String url = "https://voice.hupu.com/nba";
 CrawlerBase crawlerBase = new CrawlerBase();
//  crawlerBase.jsoupList(url);
 crawlerBase.httpClientList(url);
}