使用scrapy实现爬网站例子和实现网络爬虫(蜘蛛)的步骤

#!/usr/bin/env python
# -*- coding: utf-8 -*-
from scrapy.contrib.spiders import CrawlSpider,Rule
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
from scrapy.selector import Selector

from cnbeta.items import CnbetaItem
class CBSpider(CrawlSpider):
    name = 'cnbeta'
    allowed_domains = ['cnbeta.com']
    start_urls = ['http://www.aspzz.cn']

    rules = (
        Rule(SgmlLinkExtractor(allow=('/articles/.*.htm',)),
             callback='parse_page',follow=True),
    )

    def parse_page(self,response):
        item = CnbetaItem()
        sel = Selector(response)
        item['title'] = sel.xpath('//title/text()').extract()
        item['url'] = response.url
        return item

from scrapy.item import Item,Field
class FjsenItem(Item):
    # define the fields for your item here like:
    # name = Field()
    title=Field()
    link=Field()
    addtime=Field()

#-*- coding: utf-8 -*-
from scrapy.spider import BaseSpider
from scrapy.selector import HtmlXPathSelector
from fjsen.items import FjsenItem
class FjsenSpider(BaseSpider):
    name="fjsen"
    allowed_domains=["fjsen.com"]
    start_urls=['http://www.fjsen.com/j/node_94962_'+str(x)+'.htm' for x in range(2,11)]+['http://www.fjsen.com/j/node_94962.htm']
    def parse(self,response):
        hxs=HtmlXPathSelector(response)
        sites=hxs.select('//ul/li')
        items=[]
        for site in sites:
            item=FjsenItem()
            item['title']=site.select('a/text()').extract()
            item['link'] = site.select('a/@href').extract()
            item['addtime']=site.select('span/text()').extract()
            items.append(item)
        return items                 

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
from os import path
from scrapy import signals
from scrapy.xlib.pydispatch import dispatcher
class FjsenPipeline(object):

    def __init__(self):
        self.conn=None
        dispatcher.connect(self.initialize,signals.engine_started)
        dispatcher.connect(self.finalize,signals.engine_stopped)
    def process_item(self,item,spider):
        self.conn.execute('insert into fjsen values(?,?,?)',(None,item['title'][0],'http://www.aspzz.cn/'+item['link'][0],item['addtime'][0]))
        return item
    def initialize(self):
        if path.exists(self.filename):
            self.conn=sqlite3.connect(self.filename)
        else:
            self.conn=self.create_table(self.filename)
    def finalize(self):
        if self.conn is not None:
            self.conn.commit()
            self.conn.close()
            self.conn=None
    def create_table(self,filename):
        conn=sqlite3.connect(filename)
        conn.execute("""create table fjsen(id integer primary key autoincrement,title text,link text,addtime text)""")
        conn.commit()
        return conn

ITEM_PIPELINES=['fjsen.pipelines.FjsenPipeline']

scrapy crawl fjsen