python - 如何使用scrapy递归抓取整个网站

Question

我想使用scrapy抓取完整的网站，但现在它只抓取单个页面

import scrapy
from scrapy.http import HtmlResponse
from scrapy.contrib.spiders import CrawlSpider, Rule
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
from scrapy.selector import HtmlXPathSelector
from scrapy.contrib.exporter import JsonItemExporter

class IzodspiderSpider(scrapy.Spider):


name = 'izodspider'
allowed_domains = ['izod.com']
start_urls = ['http://izod.com/']
rules = [Rule(SgmlLinkExtractor(), callback='parse_item', follow=True)]
def parse(self, response):
    hxs = scrapy.Selector(response)
    meta = hxs.xpath('//meta[@name=\'description\']/@content').extract()
    name = hxs.xpath('//div[@id=\'product-details\']/h5').extract()
    desc = hxs.xpath('//div[@id=\'product-details\']/p').extract()

有什么方法可以使用 portia 提取元标记？

Answer 1

规则定义和回调内部存在错误。

由于您使用的 parse 函数是parse_item您必须在回调中调用它而不是parse

您可以在此处的文档中找到有关回调函数的更多信息http://doc.scrapy.org/en/latest/topics/request-response.html?highlight=callback#topics-request-response-ref-request-callback -参数

class IzodspiderSpider(CrawlSpider):
name = "izod"
depth_limit= 0 
bot_name = 'izod'
allowed_domains = ['izod.com']
start_urls = ['http://www.izod.com']
rules = (
     Rule(SgmlLinkExtractor(allow=('')), callback='parse_items',follow= True),
     )

def parse_items(self, response):
    hxs = scrapy.Selector(response)
    meta = hxs.xpath('//meta[@name=\'description\']/@content').extract()
    name = hxs.xpath('//div[@id=\'product-details\']/h5').extract()
    desc = hxs.xpath('//div[@id=\'product-details\']/p').extract()