scrapy + mogoDB 网站爬虫

python3 -m pip install pymongo
brew install mongodb
mongod --config /usr/local/etc/mongod.confCopy

python3 -m pip install ScrapyCopy

scrapy startproject myspiderCopy

cd myspider/myspider/spidersCopy

scrapy genspider -t crawl zgmlxc www.zgmlxc.com.cnCopy

rules = (
    // 定位到 www.zgmlxc.com.cn/node/72.jspx 这个页面
    Rule(LinkExtractor(allow=r'.72\.jspx')),  
    // 在上面规定的页面中，寻找符合下面规则的 url, 爬取里面的内容，并把获取的信息返回给 parse_item（）函数
    Rule(LinkExtractor(allow=r'./info/\d+\.jspx'), callback='parse_item'),
)        Copy

import scrapy

class CrawlspiderItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    title = scrapy.Field()
    content = scrapy.Field()
    piclist = scrapy.Field()
    shortname = scrapy.Field()Copy

def parse_item(self, response):
    yield {
        'title' : ' '.join(response.xpath("//div[@class='head']/h3/text()").get()).strip(),
        'shortname' : ' '.join(response.xpath("//div[@class='body']/p/strong/text()").get()).strip(),
        'piclist' : ' '.join(response.xpath("//div[@class='body']/p/img/@src").getall()).strip(),
        'content' : ' '.join(response.css("div.body").extract()).strip(),
            }Copy

# 建立爬虫与数据库之间的连接关系
ITEM_PIPELINES = {
   'crawlspider.pipelines.MongoDBPipeline': 300,
}

# 设置数据库信息
MONGODB_SERVER = "localhost"
MONGODB_PORT = 27017
MONGODB_DB = 'spider_world'
MONGODB_COLLECTION = 'zgmlxc'

# 设置文明爬虫, 意思是每个请求之间间歇 5 秒, 对站点友好, 也防止被黑名单
```py
DOWNLOAD_DELAY = 5Copy

import pymongo

from scrapy.conf import settings
from scrapy.exceptions import DropItem
from scrapy import log

class MongoDBPipeline(object):
    def __init__(self):
        connection = pymongo.MongoClient(
            settings['MONGODB_SERVER'],
            settings['MONGODB_PORT']
        )
        db = connection[settings['MONGODB_DB']]
        self.collection = db[settings['MONGODB_COLLECTION']]

    def process_item(self, item, spider):
        valid = True
        for data in item:
            if not data:
                valid = False
                raise DropItem("Missing {0}!".format(data))
        if valid:
            self.collection.insert(dict(item))
            log.msg("Question added to MongoDB database!",
                    level=log.DEBUG, spider=spider)
        return itemCopy

scrapy crawl myspiderCopy