Python爬虫Scrapy框架CrawlSpider原理及使用案例

# -*- coding: utf-8 -*-
import scrapy
# 导入CrawlSpider相关模块
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule

# 表示该爬虫程序是基于CrawlSpider类的
class CrawldemoSpider(CrawlSpider):
  name = 'crawlDemo'  #爬虫文件名称
  #allowed_domains = ['www.qiushibaike.com']
  start_urls = ['http://www.qiushibaike.com/']
  
  #连接提取器：会去起始url响应回来的页面中提取指定的url
  link = LinkExtractor(allow=r'/8hr/page/\d+')
  #rules元组中存放的是不同的规则解析器（封装好了某种解析规则)
  rules = (
    #规则解析器：可以将连接提取器提取到的所有连接表示的页面进行指定规则（回调函数）的解析
    Rule(link, callback='parse_item', follow=True),
  )
  # 解析方法
  def parse_item(self, response):
    #print(response.url)
    divs = response.xpath('//div[@id="content-left"]/div')
    for div in divs:
      author = div.xpath('./div[@class="author clearfix"]/a[2]/h2/text()').extract_first()
      print(author)

# -*- coding: utf-8 -*-
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from qiubaiBycrawl.items import QiubaibycrawlItem
import re
class QiubaitestSpider(CrawlSpider):
  name = 'qiubaiTest'
  #起始url
  start_urls = ['http://www.qiushibaike.com/']

  #定义链接提取器，且指定其提取规则
  page_link = LinkExtractor(allow=r'/8hr/page/\d+/')
  
  rules = (
    #定义规则解析器，且指定解析规则通过callback回调函数
    Rule(page_link, callback='parse_item', follow=True),
  )

  #自定义规则解析器的解析规则函数
  def parse_item(self, response):
    div_list = response.xpath('//div[@id="content-left"]/div')
    
    for div in div_list:
      #定义item
      item = QiubaibycrawlItem()
      #根据xpath表达式提取糗百中段子的作者
      item['author'] = div.xpath('./div/a[2]/h2/text()').extract_first().strip('\n')
      #根据xpath表达式提取糗百中段子的内容
      item['content'] = div.xpath('.//div[@class="content"]/span/text()').extract_first().strip('\n')

      yield item #将item提交至管道

# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html

import scrapy


class QiubaibycrawlItem(scrapy.Item):
  # define the fields for your item here like:
  # name = scrapy.Field()
  author = scrapy.Field() #作者
  content = scrapy.Field() #内容

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html

class QiubaibycrawlPipeline(object):
  
  def __init__(self):
    self.fp = None
    
  def open_spider(self,spider):
    print('开始爬虫')
    self.fp = open('./data.txt','w')
    
  def process_item(self, item, spider):
    #将爬虫文件提交的item写入文件进行持久化存储
    self.fp.write(item['author']+':'+item['content']+'\n')
    return item
  
  def close_spider(self,spider):
    print('结束爬虫')
    self.fp.close()