# node - 爬虫初体验

// 引入请求库
const rp = require('request-promise');
// 用来把 html 转化为 DOM
const cheerio = require('cheerio');

const opts = {
  url: 'https://movie.douban.com'
};

rp(opts).then(res =>{
  // res 就是目标页面请求回来的 html 代码
  // load 返回的是一个类似 jq 的对象，操作 DOM 也和完全按照 jq 的方式。
  const $ = cheerio.load(res);
  let result = []; // 结果数组
  
  // 定位到 正在热映电影 模块
  $('#screening li.ui-slide-item').each((index, item) => {
        let ele = $(item);
        // 获取 名称|评分|地址|封面|id
        let name = ele.data('title');
        let score = ele.data('rate') || '暂无评分';
        let href = ele.find('.poster a').attr('href');
        let image = ele.find('img').attr('src');
        let id = href && href.match(/(\d+)/)[1];
  
        if (!name || !image || !href) return;
  
        result.push({
          name,
          score,
          href,
          image,
          id
        });
      });
  
  return result;
})

const mongoose = require('mongoose');
/**
*  数据库连接
*/
const config = {
  db: 'mongodb://localhost/movie_douban',
  options: {
    poolSize: 50,
    auto_reconnect: true,
  }
};
const database = app => {
  // mongoose.set('debug', true);
  // 连接数据库
  mongoose.connect(config.db, config.options);
  // 连接中断重连
  mongoose.connection.on('disconnected', (err) => {
    console.error(err);
    mongoose.connect(config.db);
  });
  // 连接错误
  mongoose.connection.on('error', err => {
    console.error(err);
  });
  // 连接成功
  mongoose.connection.on('open', async () => {
    console.log('Connected to mongoose');
  });
};
database();

const mongoose = require('mongoose');
/**
 * 定义电影字段
 */
const Schema = mongoose.Schema;
const MovieSchema = new Schema({
  name: String,
  score: String,
  href: String,
  image: String,
  id: Number
});
MovieSchema.statics = {
  // 获取
  async getMovie (id) {
    const movie = await this.findOne({
      id: id
    }).exec();
    if (movie) console.log('数据库查询到movie', id, movie.name);
    else console.log('数据库查询movie', id, '没有');
    return movie;
  },
  // 保存
  async saveMovie (data) {
    let id = +data.id;
    let movie = await this.getMovie(id);
    if (movie) {
      //  更新
      movie = await this.update({ id }, movie);
    } else {
      // 保存
      movie = await this.create(data);
    }
    return movie;
  },
  async getMovieQuery (query) {
    let movies = await this.find(query).exec();
    if (!movies) return [];
    return movies;
  }
};
// 获取movie的数据模型
const Movie = mongoose.model('Movie', MovieSchema);

const mongoose = require('mongoose');
// 获取movie的数据模型
const Movie = mongoose.model('Movie');
const write = async (movies) => {
  for (let movie of movies) {
    // 通过 saveMovie 保存或者更新电影
    await Movie.saveMovie(movie);
  }
};

const express = require('express');
const mongoose = require('mongoose');
const path = require('path');
const database = require('../src/db');
const app = express();

// 连接数据库
database();
const Movie = mongoose.model('Movie');
// 设置模板引擎
app.set('view engine', 'html');
app.set('views', path.join(__dirname, 'views'));
app.engine('html', require('ejs').__express);

// 首页路由
app.get('/', async (req, res) => {
  let movies = await Movie.getMovieQuery({});
  res.render('index', {
    movies: movies.filter(i => {
      let score = parseFloat(i.score);
      i.score = score;
      if (score >= 9) {
        i.scoreClass = 'c-1';
      } else if (score >= 8) {
        i.scoreClass = 'c-2';
      } else {
        i.scoreClass = 'c-3';
      }
      // 评分大于 7.5 的才返回
      return score && score >= 7.5;
    }).sort((a, b) => b.score - a.score)
  });
});

app.listen(2333);
console.log('app start at http://localhost:2333');

const rp = require('request-promise');
const opts = {
  url: 'https://movie.douban.com',
  headers:{
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.75 Safari/537.36'
  }  
};

rp(opts).then(res=>console.log(res));

const rp = require('request-promise');
const opts = {
  url: 'https://movie.douban.com',
  proxy: 'http://代理服务器ip:端口'
};

rp(opts).then(res=>console.log(res));

const rp = require('request-promise');
const tough = require('tough-cookie');

// Easy creation of the cookie - see tough-cookie docs for details
// 使用 tough-cookie 创建一个类 cookie 对象 
let cookie = new tough.Cookie({
    key: "some_key",
    value: "some_value",
    domain: 'api.mydomain.com',
    httpOnly: true,
    maxAge: 31536000
});

// Put cookie in an jar which can be used across multiple requests
// 把 cookie 放在 rp 的请求配置里，所有的请求都会带上 cookie 
let cookiejar = rp.jar();
cookiejar.setCookie(cookie, 'https://api.mydomain.com');
// ...all requests to https://api.mydomain.com will include the cookie

let options = {
    uri: 'https://api.mydomain.com/...',
    jar: cookiejar // Tells rp to include cookies in jar that match uri
};
rp(options)
    .then(function (body) {
        // Request succeeded...
    })
    .catch(function (err) {
        // Request failed...
    });