Scrapy框架高级操作

class QiubaiSpider(scrapy.Spider):
    name = ‘qiubai‘
    # allowed_domains = [‘www.qiushibaike.com/text‘]
    start_urls = [‘https://www.qiushibaike.com/text/‘]
    # 设计一个通用url模板
    url = ‘https://www.qiushibaike.com/text/page/%d/‘
    page_num = 1

    def parse(self,response):
        # 建议使用xpath进行解析（框架集成了xpath解析的接口）
        div_list = response.xpath(‘//div[@id="content-left"]/div ‘)
        # 存储解析到的页面数据
        data_list = []
        for div in div_list:
            # xpath解析到的指定内容存储到了Selector对象
            # extract()该方法可以将Selector对象存储中存储的数据值拿到
            author = div.xpath(‘./div/a[2]/h2/text()‘).extract_first()
            # extract_first = extract()[0]
            content = div.xpath(‘.//div[@class="content"]/span/text()‘).extract_first()

            # 将解析到的数据值(author和content)存储到items对象
            item = QiubaibypageseItem()
            item[‘author‘] = author
            item[‘content‘] = content
            # 将item对象提交给管道
            yield item
        # 请求的手动发送
        # 13表示最后一页页码
        if self.page_num <= 13:
            print("爬取到了第%d页的页面数据" % self.page_num)
            self.page_num += 1
            new_url = format(self.url % self.page_num)
            # callback:将去请求获取到的页面数据进行解析
            yield scrapy.Request(url=new_url,callback=self.parse)

class PostdemoSpider(scrapy.Spider):
    name = ‘postDemo‘
    # allowed_domains = [‘www.baidu.com‘]
    start_urls = [‘https://fanyi.baidu.com/sug‘]

    def start_requests(self):
        # 该方法其实是父类中的一个方法:该方法可以对star_urls列表中的元素进行get请求的fas
        # 发送post:
            # 将Request方法中method参数赋值成post
            # FormRequest()可以发起post请求(推荐)
        data = {
            ‘kw‘: ‘dog‘
        }
        print(‘start requests‘)
        for url in self.start_urls:
            yield scrapy.FormRequest(url=url,formdata=data,callback=self.parse)

    def parse(self,response):
        print(response.text)

class DoubanSpider(scrapy.Spider):
    name = ‘douban‘
    start_urls = [‘https://accounts.douban.com/j/mobile/login/basic‘]

    def start_requests(self):
        param_data = {
            ‘ck‘: ‘dFf2‘,‘name‘: ‘18629090745‘,‘password‘: ‘cs1993413‘,‘remember‘: ‘false‘,‘ticket‘: ‘‘
        }

        for url in self.start_urls:
            yield scrapy.FormRequest(url,formdata=param_data,callback=self.parse)

    def parseBySecondPage(self,response):
        #  指定个人页面数据进行解析
        fp = open(‘second.html‘,‘w‘,encoding=‘utf-8‘)
        fp.write(response.text)

    def parse(self,response):
        print(response.text)
        # 获取当前用户的个人主页
        url = ‘https://www.douban.com/people/181322653/‘
        yield scrapy.Request(url=url,callback=self.parseBySecondPage)

# 自定义一个下载中间件的类,在类中事先process_request（处理中间件拦截到的请求）方法
class MyProxy(object):
    def process_request(self,request,spider):
        # 请求ip的更换
        request.meta[‘proxy‘] = "http://183.233.90.6:8080"

DOWNLOADER_MIDDLEWARES = {
       # ‘postPro.middlewares.PostproDownloaderMiddleware‘: 543,
       ‘postPro.middlewares.MyProxy‘: 542,}

# 指定终端输入指定种类的日志信息
LOG_LEVEL = ‘ERROR‘
# 日志输出到指定的文件
LOG_FILE = ‘log.txt‘

class Moive195Spider(scrapy.Spider):
    name = ‘moive195‘
    # allowed_domains = [‘http://www.1905.com‘]
    start_urls = [‘http://www.1905.com/mdb/film/list/year-2018‘]

    # 用于解析二级子页面中的数据值
    def parseBySecondPage(self,response):
        Feature = response.xpath(‘/html/body/div[2]/div/div[1]/section[1]/div/p/text()‘).extract_first()
        director = response.xpath(‘/html/body/div[2]/section/div/div[2]/div[2]/a[1]/div[2]/text()‘).extract_first()
        # 取出Request方法的meta参数传递过来的字典(response.meta)
        item = response.meta[‘item‘]
        item[‘feature‘] = Feature if Feature else ‘‘
        item[‘director‘] = director if director else ‘‘

        # 将item提交给管道
        yield item

    def parse(self,response):
        div_list = response.xpath("/html/body/div[2]/div[1]/ul/li")
        for div in div_list:
            movie_name = div.xpath("./div/p[1]/a/text()").extract_first()
            href = div.xpath("./a/@href").extract_first()
            rank = div.xpath("./div/p[2]/b/text()").extract_first()
            actors = div.xpath("./div/p[3]/a/text()").extract()
            print(actors,‘演员‘)
            movie_type = div.xpath("./div/p[4]/a/text()").extract_first()
            # 创建item对象
            item = ParsemoiveItem()
            item[‘name‘] = movie_name if movie_name else ‘‘
            item[‘movie_type‘] = movie_type if movie_type else ‘‘
            item[‘rank‘] = rank if rank else ‘‘
            item[‘actors‘] = ‘/‘.join(actors) if actors else ‘‘

            # 需要对url发起请求,获取页面数据进行指定的解析
            url = ‘http://www.1905.com/‘ + href
            # meta参数值可以赋值一个字典(将item对象先封装到一个字典中)
            yield scrapy.Request(url=url,callback=self.parseBySecondPage,meta={‘item‘: item})

class ParsemoivePipeline(object):
    fp = None

    def open_spider(self,spider):
        self.fp = open(‘movie.txt‘,encoding=‘utf-8‘)

    def process_item(self,item,spider):
        detail = item[‘name‘] + ‘:‘ + item[‘movie_type‘] + ‘‘ + item[‘rank‘] + ‘‘ + item[‘director‘] + "n" + item[‘actors‘] + "n" + item[‘feature‘] + "nnn"
        self.fp.write(detail)
        return item

    def close_spider(self,spider):
        self.fp.close()

 scrapy genspider -t crawl 爬虫名称 起始url

import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider,Rule


class CrawlchoutiSpider(CrawlSpider):
    name = ‘crawlchouti‘
    # allowed_domains = [‘dig.chouti.com‘]
    start_urls = [‘https://dig.chouti.com/‘]
    # 实例化了一个链接提取器对象
    # 链接提取器:用来提取指定的链接(url)
    # allow参数：赋值一个正则表达式
    # 链接提取器可以根据正则表达式在页面提取指定的链接
    # 提取到的链接会全部交给规则解析器
    link = LinkExtractor(allow=r‘/all/hot/recent/d+‘)
    rules = (
        # 实例化了一个规则解析器对象
        # 规则解析器接受了链接提取器发送的链接后,就会对这些链接发起请求，获取链接对应的页面内容， 就会根据指定的规则对页面内容中指定的数据值进行解析
        # callback:指定一个解析规则(方法/函数)
        # follow:是否将链接提取器继续作用到链接提取器提出的链接所表示的页面数据
        Rule(link,callback=‘parse_item‘,follow=False),)

    def parse_item(self,response):
        print(response)