根据我在浏览源代码和文档后的理解,-t
选项是指FEED_FORMAT
不能有多个值的设置。此外,FeedExporter
内置扩展 ( source ) 仅适用于单个导出器。
实际上,考虑在Scrapy 问题跟踪器上提出功能请求。
更像是一种解决方法,定义一个管道并使用多个 exporters 开始导出。例如,以下是导出为 CSV 和 JSON 格式的方法:
from collections import defaultdict
from scrapy import signals
from scrapy.exporters import JsonItemExporter, CsvItemExporter
class MyExportPipeline(object):
def __init__(self):
self.files = defaultdict(list)
@classmethod
def from_crawler(cls, crawler):
pipeline = cls()
crawler.signals.connect(pipeline.spider_opened, signals.spider_opened)
crawler.signals.connect(pipeline.spider_closed, signals.spider_closed)
return pipeline
def spider_opened(self, spider):
csv_file = open('%s_products.csv' % spider.name, 'w+b')
json_file = open('%s_products.json' % spider.name, 'w+b')
self.files[spider].append(csv_file)
self.files[spider].append(json_file)
self.exporters = [
JsonItemExporter(json_file),
CsvItemExporter(csv_file)
]
for exporter in self.exporters:
exporter.start_exporting()
def spider_closed(self, spider):
for exporter in self.exporters:
exporter.finish_exporting()
files = self.files.pop(spider)
for file in files:
file.close()
def process_item(self, item, spider):
for exporter in self.exporters:
exporter.export_item(item)
return item