2

嗨，我正在使用 scrapy 来抓取网站

我已经编写了蜘蛛，获取了所有信息并通过 pipeline.py 保存到 csv 文件中

pipeline.py 代码

class Examplepipeline(object):

    def __init__(self):
        dispatcher.connect(self.spider_opened, signal=signals.spider_opened)
        dispatcher.connect(self.spider_closed, signal=signals.spider_closed)

    def spider_opened(self, spider):
        log.msg("opened spider  %s at time %s" % (spider.name,datetime.now().strftime('%H-%M-%S')))
        self.exampledotcomCsv = csv.writer(open("csv's/%s(%s).csv"% (spider.name,datetime.now().strftime("%d/%m/%Y,%H-%M-%S")), "wb"),
                   delimiter=',', quoting=csv.QUOTE_MINIMAL)
        self.exampledotcomCsv.writerow(['field1', 'field2','field3','field4'])           

    def process_item(self, item, spider):
            log.msg("Processsing item " + item['title'], level=log.DEBUG)
            self.brandCategoryCsv.writerow([item['field1'].encode('utf-8'),
                                    [i.encode('utf-8') for i in item['field2']],
                                    item['field3'].encode('utf-8'),
                                    [i.encode('utf-8') for i in item['field4']]
                                    ])
            return item 


    def spider_closed(self, spider):
        log.msg("closed spider %s at %s" % (spider.name,datetime.now().strftime('%H-%M-%S')))

在上面的代码中我可以得到start time and end time of spider，但是在关闭蜘蛛之后我想计算并显示total time蜘蛛所采取的那是difference between start time and end time 那么我该怎么做，我们可以在 spider_closed 方法中编写这个功能？

请让我知道这一点。

4

1 回答 1

1

为什么不：

def spider_opened(self, spider):
    spider.started_on = datetime.now()
    ...

def spider_closed(self, spider):
    work_time = datetime.now() - spider.started_on
    ...
于 2012-07-06T11:19:57.650 回答