0

我已经修改了Using Middleware 中的代码以忽略 Scrapy 中的重复项。

from scrapy.exceptions import DropItem
from scrapy import log
import os.path

class IgnoreDuplicates():

    def __init__(self):
        self._cu_file = open("crawled_urls.txt", "a+")
        self._crawled_urls = set([line.strip() for line in self._cu_file.readlines()])

    def process_request(self, request, spider):
        if request.url in self._crawled_urls:
            raise DropItem("Duplicate product scrape caught by IgnoreDuplicates at <%s>" % (url))
        else:
            self._crawled_urls.add(request.url)
            self._cu_file.write(request.url + '\n')
            log.msg("IgnoreDuplicates recorded this url " + request.url, level=log.DEBUG)
            return None

我还在 settings.py 中添加了中间件模块：

SPIDER_MANAGER_CLASS = 'slybot.spidermanager.SlybotSpiderManager'
EXTENSIONS = {'slybot.closespider.SlybotCloseSpider': 1}
ITEM_PIPELINES = {'slybot.dupefilter.DupeFilterPipeline': 1}
SPIDER_MIDDLEWARES = {'slybot.middleware.IgnoreDuplicates': 500, 'slybot.spiderlets.SpiderletsMiddleware': 999}  # as close as possible to spider output
PLUGINS = ['slybot.plugins.scrapely_annotations.Annotations']
SLYDUPEFILTER_ENABLED = True
PROJECT_DIR = 'slybot-project'
FEED_EXPORTERS = {
    'csv': 'slybot.exporter.SlybotCSVItemExporter',
}
CSV_EXPORT_FIELDS = None

try:
    from local_slybot_settings import *
except ImportError:
    pass

不会调用 process_request 函数。我尝试更改 settings.py 中中间件键的值，以便在 SpiderletsMiddleware 之前和之后执行它。但是异常和日志消息没有显示在输出中。

如何确保调用中间件？

4

1 回答 1

0

蜘蛛中间件的回调函数是不同的。我将此片段中的代码用作参考： http ://snipplr.com/view/67018/middleware-to-avoid-revisiting-already-visited-items/

这是我在问题中发布的中间件代码的工作版本。

from scrapy.http import Request
from scrapy import log
import os.path

class IgnoreVisitedItems(object):
    def __init__(self):
        # Load the URLs that have already been crawled
        self._cu_file = open("crawled_urls.txt", "a+")
        self._crawled_urls = set([line.strip() for line in self._cu_file.readlines()])

    def process_spider_output(self, response, result, spider):
        ret = []
        for x in result:
            # Find the URL in the result or response
            url = None
            if isinstance(x, Request):
                url = x.url
            else:
                url = response.request.url

            # Check if the URL has been crawled, and add
            # it to the list of crawled URLs.
            if url in self._crawled_urls:
                log.msg("Ignoring already visited: %s" % url,
                        level=log.INFO, spider=spider)
            else:
                log.msg("Adding %s to list of visited urls" % url,
                        level=log.INFO, spider=spider)
                self._cu_file.write(url + '\n')
                self._crawled_urls.add(url)
                ret.append(x)
        return ret
于 2015-03-24T20:59:47.963 回答