#Python爬虫#--Scrapy之Item Pipeline组件(ITEM_PIPELINES)

scrapy genspider -t basic douban https://movie.douban.com/review/best/

# -*- coding: utf-8 -*-
import scrapy
from .. items import DoubandemoItem

class DoubanSpider(scrapy.Spider):
    name = 'douban'
    allowed_domains = ['movie.douban.com']
    start_urls = ['https://movie.douban.com/review/best//']

    def parse(self, response):
        import sys
        reload(sys)
        sys.setdefaultencoding("utf-8")
        for sel in response.xpath("//div[@class='main-bd']"):
            title = sel.xpath("h2/a/text()").extract()
            content = sel.xpath("div[@class='review-short']//div[@class='short-content']//text()").extract()

            dbitem = DoubandemoItem()
            dbitem["title"] = title[0].strip()
            dbitem["content"] = content[0].strip()
            yield  dbitem

# -*- coding: utf-8 -*-
import scrapy

class DoubandemoItem(scrapy.Item):    
    title = scrapy.Field()
    content = scrapy.Field()

# -*- coding: utf-8 -*-

import json
from scrapy.exceptions import DropItem

class DoubandemotojsonPipeline(object):
    def __init__(self):
        self.file = open("db.jl","wb")

    def process_item(self, item, spider):
        line = json.dumps(dict(item))+"\n"
        self.file.write(line)
        return item


class DoubandemotonullPipeline(object):
    def process_item(self, item, spider):
        if item["content"]:
            return item
        else:
            raise DropItem("Duplicate item found: %s" % item)

ITEM_PIPELINES = {
   'DoubanDemo.pipelines.DoubandemotojsonPipeline': 300,
   'DoubanDemo.pipelines.DoubandemotonullPipeline': 400,
}