我的蜘蛛抓取 MITOPENCOURSEWARE 网站的教学大纲不起作用。有人可以帮我弄清楚它有什么问题吗?.* 代表所有课程。这是正确的吗?
1 from scrapy.contrib.spiders import CrawlSpider, Rule
2 from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
3 from scrapy.selector import HtmlXPathSelector
4 from opensyllabi.items import OpensyllabiItem
5
6 class MITSpider(CrawlSpider):
7 name = 'mit'
8 allowed_domains = ['ocw.mit.edu']
9 start_urls = ['http://ocw.mit.edu/courses']
10 rules = [Rule(SgmlLinkExtractor(allow=['/.*/.*/syllabus']), 'parse_syllabus')]
11
12 def parse_syllabus(self, response):
13 x = HtmlXPathSelector(response)
14
15 syllabus = OpensyllabiItem()
16 syllabus['url'] = response.url
17 syllabus['body'] = x.select("//div[@id='course_inner_section']").extract()
18 return syllabus