java爬取豆瓣电影示例解析

 
  org.json
  json
  20160810
 

 
  com.alibaba
  fastjson
  1.2.47
 

 
  mysql
  mysql-connector-java
  5.1.47
 

 
  org.mybatis
  mybatis
  3.5.1
 

 
  junit
  junit
  4.12
 
 

public class Movie {

 private String id; //电影的id
 private String directors;//导演
 private String title;//标题
 private String cover;//封面
 private String rate;//评分
 private String casts;//演员


 public String getId() {
  return id;
 }

 public void setId(String id) {
  this.id = id;
 }

 public String getDirectors() {
  return directors;
 }

 public void setDirectors(String directors) {
  this.directors = directors;
 }

 public String getTitle() {
  return title;
 }

 public void setTitle(String title) {
  this.title = title;
 }

 public String getCover() {
  return cover;
 }

 public void setCover(String cover) {
  this.cover = cover;
 }

 public String getRate() {
  return rate;
 }

 public void setRate(String rate) {
  this.rate = rate;
 }

 public String getCasts() {
  return casts;
 }

 public void setCasts(String casts) {
  this.casts = casts;
 }
}

public interface MovieMapper {

 void insert(Movie movie);
 
 List findAll();
}

driver=com.mysql.jdbc.Driver
url=jdbc:mysql://localhost:3306/huadi
username=root
password=root

<?xml version="1.0" encoding="UTF-8" ?>


 
 
  
   
   
    
    
    
    
   
  
 
 
  
 

 
  
  
  
  
  
  

 

 
  INSERT INTO movie(id,title,cover,rate,casts,directors)
  VALUES
  (#{id},#{title},#{cover},#{rate},#{casts},#{directors})
 
 
  SELECT * FROM movie
 

public class GetJson {
 public JSONObject getHttpJson(String url, int comefrom) throws Exception {
  try {
   URL realUrl = new URL(url);
   HttpURLConnection connection = (HttpURLConnection) realUrl.openConnection();
   connection.setRequestProperty("accept", "*/*");
   connection.setRequestProperty("connection", "Keep-Alive");
   connection.setRequestProperty("user-agent", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1;SV1)");
   // 建立实际的连接
   connection.connect();
   //请求成功
   if (connection.getResponseCode() == 200) {
    InputStream is = connection.getInputStream();
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    //10MB的缓存
    byte[] buffer = new byte[10485760];
    int len = 0;
    while ((len = is.read(buffer)) != -1) {
     baos.write(buffer, 0, len);
    }
    String jsonString = baos.toString();
    baos.close();
    is.close();
    //转换成json数据处理
    // getHttpJson函数的后面的参数1，表示返回的是json数据，2表示http接口的数据在一个（）中的数据
    JSONObject jsonArray = getJsonString(jsonString, comefrom);
    return jsonArray;
   }
  } catch (MalformedURLException e) {
   e.printStackTrace();
  } catch (IOException ex) {
   ex.printStackTrace();
  }
  return null;
 }

 public JSONObject getJsonString(String str, int comefrom) throws Exception{
  JSONObject jo = null;
  if(comefrom==1){
   return new JSONObject(str);
  }else if(comefrom==2){
   int indexStart = 0;
   //字符处理
   for(int i=0;i

public class Main {
 public static void main(String [] args) {

  String resource = "mybatis-config.xml"; 定义配置文件路径
  InputStream inputStream = null;
  try {
   inputStream = Resources.getResourceAsStream(resource);//读取配置文件
  } catch (IOException e) {
   e.printStackTrace();
  }

  SqlSessionFactory sqlSessionFactory = new SqlSessionFactoryBuilder().build(inputStream);//注册mybatis 工厂

  SqlSession sqlSession = sqlSessionFactory.openSession();//得到连接对象

  MovieMapper movieMapper = sqlSession.getMapper(MovieMapper.class);//从mybatis中得到dao对象

  int start;//每页多少条
  int total = 0;//记录数
  int end = 9979;//总共9979条数据
  for (start = 0; start <= end; start += 20) {
   try {

    String address = "https://Movie.douban.com/j/new_search_subjects?sort=U&range=0,10&tags=&start=" + start;

    JSONObject dayLine = new GetJson().getHttpJson(address, 1);

     System.out.println("start:" + start);
     JSONArray json = dayLine.getJSONArray("data");
     List list = JSON.parseArray(json.toString(), Movie.class);

     if (start <= end){
      System.out.println("已经爬取到底了");
      sqlSession.close();
     }
     for (Movie movie : list) {
      movieMapper.insert(movie);
      sqlSession.commit();
     }
     total += list.size();
     System.out.println("正在爬取中---共抓取:" + total + "条数据");

   } catch (Exception e) {
    e.printStackTrace();
   }

  }
 }

}