python – 使用Scrapy将参数传递给回调函数,因此可以在以后接收

作者|专辑|标题|歌词

每个数据都位于不同的网页上,这就是我为什么要使用Scrapy回调函数来实现这一目标的原因.

此外,上述每个项目都在Scrapy items.py下定义为：

import scrapy

class TutorialItem(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
    author = scrapy.Field()
    album = scrapy.Field()
    title = scrapy.Field()
    lyrics = scrapy.Field()

蜘蛛代码从这里开始：

import scrapy
import re
import json

from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider,Rule
from tutorial.items import TutorialItem


# urls class
class DomainSpider(scrapy.Spider):
    name = "domainspider"
    allowed_domains = ['www.domain.com']
    start_urls = [
        'http://www.domain.com',]

    rules = (
        Rule(LinkExtractor(allow='www.domain.com/[A-Z][a-zA-Z_/]+$'),'parse',follow=True,),)

    # Parsing start here
    # crawling and scraping the links from menu list
    def parse(self,response):
        links = response.xpath('//html/body/nav[1]/div/ul/li/div/a/@href')

        for link in links:
            next_page_link = link.extract()
            if next_page_link:
                next_page = response.urljoin(next_page_link)
                yield scrapy.Request(next_page,callback=self.parse_artist_page)

    # crawling and scraping artist names and links
    def parse_artist_page(self,response):
        artist_links = response.xpath('//*/div[contains(@class,"artist-col")]/a/@href')
        author = response.xpath('//*/div[contains(@class,"artist-col")]/a/text()').extract()

        item = TutorialItem(author=author)

        for link in artist_links:
            next_page_link = link.extract()
            if next_page_link:
                next_page = response.urljoin(next_page_link)
                yield scrapy.Request(next_page,callback=self.parse_album_page)

                request.meta['author'] = item
                yield item
                return

    # crawling and scraping album names and links
    def parse_album_page(self,response):
        album_links = response.xpath('//*/div[contains(@id,"listAlbum")]/a/@href')
        album = response.xpath('//*/div[contains(@class,"album")]/b/text()').extract()

        item = TutorialItem(album=album)

        for link in album_links:
            next_page_link = link.extract()
            if next_page_link:
                next_page = response.urljoin(next_page_link)
                yield scrapy.Request(next_page,callback=self.parse_lyrics_page)

                request.meta['album'] = item
                yield item
                return

    # crawling and scraping titles and lyrics
    def parse_lyrics_page(self,response):
        title = response.xpath('//html/body/div[3]/div/div[2]/b/text()').extract()
        lyrics = map(unicode.strip,response.xpath('//html/body/div[3]/div/div[2]/div[6]/text()').extract())

        item = response.meta['author','album']
        item = TutorialItem(author=author,album=album,title=title,lyrics=lyrics)
        yield item

转到回调函数时代码崩溃：

request.meta['author'] = item
yield item
return

有人可以帮忙吗？