基本上，您有很多工具可供选择：
这些工具有不同的用途，但可以根据任务混合在一起。
Scrapy 是一个强大且非常智能的工具，用于抓取网站、提取数据。但是，当涉及到操作页面时：点击按钮、填写表格——它变得更加复杂：
- 有时，通过直接在scrapy中进行底层表单操作来模拟填写/提交表单很容易
- 有时，您必须使用其他工具来帮助刮擦——比如 mechanize 或 selenium
如果您使您的问题更具体，这将有助于了解您应该使用或选择什么样的工具。
看一个有趣的scrapy&selenium mix的例子。在这里，selenium 任务是点击按钮并为 scrapy 项目提供数据：
import time
from scrapy.item import Item, Field
from selenium import webdriver
from scrapy.spider import BaseSpider
class ElyseAvenueItem(Item):
name = Field()
class ElyseAvenueSpider(BaseSpider):
name = "elyse"
allowed_domains = ["ehealthinsurance.com"]
start_urls = [
'http://www.ehealthinsurance.com/individual-family-health-insurance?action=changeCensus&census.zipCode=48341&census.primary.gender=MALE&census.requestEffectiveDate=06/01/2013&census.primary.month=12&census.primary.day=01&census.primary.year=1971']
def __init__(self):
self.driver = webdriver.Firefox()
def parse(self, response):
self.driver.get(response.url)
el = self.driver.find_element_by_xpath("//input[contains(@class,'btn go-btn')]")
if el:
el.click()
time.sleep(10)
plans = self.driver.find_elements_by_class_name("plan-info")
for plan in plans:
item = ElyseAvenueItem()
item['name'] = plan.find_element_by_class_name('primary').text
yield item
self.driver.close()
更新：
这是有关如何在您的情况下使用scrapy的示例：
from scrapy.http import FormRequest
from scrapy.item import Item, Field
from scrapy.selector import HtmlXPathSelector
from scrapy.spider import BaseSpider
class AcrisItem(Item):
borough = Field()
block = Field()
doc_type_name = Field()
class AcrisSpider(BaseSpider):
name = "acris"
allowed_domains = ["a836-acris.nyc.gov"]
start_urls = ['http://a836-acris.nyc.gov/DS/DocumentSearch/DocumentType']
def parse(self, response):
hxs = HtmlXPathSelector(response)
document_classes = hxs.select('//select[@name="combox_doc_doctype"]/option')
form_token = hxs.select('//input[@name="__RequestVerificationToken"]/@value').extract()[0]
for document_class in document_classes:
if document_class:
doc_type = document_class.select('.//@value').extract()[0]
doc_type_name = document_class.select('.//text()').extract()[0]
formdata = {'__RequestVerificationToken': form_token,
'hid_selectdate': '7',
'hid_doctype': doc_type,
'hid_doctype_name': doc_type_name,
'hid_max_rows': '10',
'hid_ISIntranet': 'N',
'hid_SearchType': 'DOCTYPE',
'hid_page': '1',
'hid_borough': '0',
'hid_borough_name': 'ALL BOROUGHS',
'hid_ReqID': '',
'hid_sort': '',
'hid_datefromm': '',
'hid_datefromd': '',
'hid_datefromy': '',
'hid_datetom': '',
'hid_datetod': '',
'hid_datetoy': '', }
yield FormRequest(url="http://a836-acris.nyc.gov/DS/DocumentSearch/DocumentTypeResult",
method="POST",
formdata=formdata,
callback=self.parse_page,
meta={'doc_type_name': doc_type_name})
def parse_page(self, response):
hxs = HtmlXPathSelector(response)
rows = hxs.select('//form[@name="DATA"]/table/tbody/tr[2]/td/table/tr')
for row in rows:
item = AcrisItem()
borough = row.select('.//td[2]/div/font/text()').extract()
block = row.select('.//td[3]/div/font/text()').extract()
if borough and block:
item['borough'] = borough[0]
item['block'] = block[0]
item['doc_type_name'] = response.meta['doc_type_name']
yield item
保存spider.py并通过运行scrapy runspider spider.py -o output.json，output.json你会看到：
{"doc_type_name": "CONDEMNATION PROCEEDINGS ", "borough": "Borough", "block": "Block"}
{"doc_type_name": "CERTIFICATE OF REDUCTION ", "borough": "Borough", "block": "Block"}
{"doc_type_name": "COLLATERAL MORTGAGE ", "borough": "Borough", "block": "Block"}
{"doc_type_name": "CERTIFIED COPY OF WILL ", "borough": "Borough", "block": "Block"}
{"doc_type_name": "CONFIRMATORY DEED ", "borough": "Borough", "block": "Block"}
{"doc_type_name": "CERT NONATTCHMENT FED TAX LIEN ", "borough": "Borough", "block": "Block"}
...
希望有帮助。