基于Java HttpClient和Htmlparser实现网络爬虫代码

StringBean sb = new StringBean();
sb.setLinks(false);//设置结果中去点链接
sb.setURL(url);//设置你所需要滤掉网页标签的页面 url 
System.out.println(sb.getStrings());//打印结果 

package com.ie;
 import java.util.Set;
public class Crawler {
  /* 使用种子 url 初始化 URL 队列*/
  private void initCrawlerWithSeeds(String[] seeds)
  {
    for(int i=0;i links=HtmlParserTool.extracLinks(visitUrl,filter);
      //新的未访问的 URL 入队
      for(String link:links)
      {
          LinkDB.addUnvisitedUrl(link);
      }
    }
  }
  //main 方法入口
  public static void main(String[]args)
  {
    Crawler crawler = new Crawler();
    crawler.crawling(new String[]{"http://www.twt.edu.cn"});
  }
}

package com.ie;
import java.util.HashSet;
import java.util.Set;
/**
 * 用来保存已经访问过 Url 和待访问的 Url 的类
 */
public class LinkDB {
  //已访问的 url 集合
  private static Set visitedUrl = new HashSet();
  //待访问的 url 集合
  private static Queue unVisitedUrl = new Queue();
  public static Queue getUnVisitedUrl() {
    return unVisitedUrl;
  }
  public static void addVisitedUrl(String url) {
    visitedUrl.add(url);
  }
  public static void removeVisitedUrl(String url) {
    visitedUrl.remove(url);
  }
  public static String unVisitedUrlDeQueue() {
    return unVisitedUrl.deQueue();
  }
  // 保证每个 url 只被访问一次
  public static void addUnvisitedUrl(String url) {
    if (url != null && !url.trim().equals("")
 && !visitedUrl.contains(url)
        && !unVisitedUrl.contians(url))
      unVisitedUrl.enQueue(url);
  }
  public static int getVisitedUrlNum() {
    return visitedUrl.size();
  }
  public static boolean unVisitedUrlsEmpty() {
    return unVisitedUrl.empty();
  }
}

package com.ie;
import java.util.LinkedList;
/**
 * 数据结构队列
 */
public class Queue {
  private LinkedList queue=new LinkedList();
  public void enQueue(T t)
  {
    queue.addLast(t);
  }
  public T deQueue()
  {
    return queue.removeFirst();
  }
  public boolean isQueueEmpty()
  {
    return queue.isEmpty();
  }
  public boolean contians(T t)
  {
    return queue.contains(t);
  }
  public boolean empty()
  {
    return queue.isEmpty();
  }
} 

package com.ie;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import org.apache.commons.httpclient.DefaultHttpMethodRetryHandler;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.params.HttpMethodParams;
public class FileDownLoader {
  /**根据 url 和网页类型生成需要保存的网页的文件名
   *去除掉 url 中非文件名字符 
   */
  public String getFileNameByUrl(String url,String contentType)
  {
    url=url.substring(7);//remove http://
    if(contentType.indexOf("html")!=-1)//text/html
    {
      url= url.replaceAll("[\\?/:*|<>\"]", "_")+".html";
      return url;
    }
    else//如application/pdf
    {
return url.replaceAll("[\\?/:*|<>\"]", "_")+"."+ \
     contentType.substring(contentType.lastIndexOf("/")+1);
    }  
  }
  /**保存网页字节数组到本地文件
   * filePath 为要保存的文件的相对地址
   */
  private void saveToLocal(byte[] data,String filePath)
  {
    try {
      DataOutputStream out=new DataOutputStream(
new FileOutputStream(new File(filePath)));
      for(int i=0;i

package com.ie;
public interface LinkFilter {
  public boolean accept(String url);
}