from scrapy.spider import BaseSpider
from scrapy.selector import HtmlXPathSelector
from scrapy.contrib.spiders import CrawlSpider, Rule
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
#scrapy crawl dmoz -o items.json -t json
from scrapy.http import Request
from urlparse import urlparse
from manga.items import MangaItem
class MangaHere(CrawlSpider):
name = "mangahs"
allowed_domains = ["mangahere.com"]
start_urls = ["http://www.mangahere.com/seinen/"]
rules = [Rule(SgmlLinkExtractor(allow = ('//a[@class="next"]')), follow=True,
callback='parse_item'),]
#def parse(self, response):
# get indext depth for every page
# hxs = HtmlXPathSelector(response)
# next_link = hxs.select('//a[@class="next"]')
# index_depth = int(next_link.select('preceding-sibling::a[1]/text()').extract()[0])
#create a request for the first page
# url = urlparse("http://www.mangahere.com/seinen/")
# yield Request(url.geturl(), callback=self.parse_item)
#create a request for each subsequent page in the form "./seinen/x.html"
# for x in xrange(2, index_depth):
# pageURL = "http://www.mangahere.com/seinen/%s.htm" % x
# url = urlparse(pageURL)
# yield Request(url.geturl(), callback=self.parse_item)
def parse_start_url(self, response):
list(self.parse_item(response))
def parse_item(self,response):
hxs = HtmlXPathSelector(response)
sites = hxs.select('//ul/li/div')
items = []
for site in sites:
rating = site.select("p/span/text()").extract()
desc = site.select("p[2]/text()").extract()
for i in rating:
for p in desc:
if float(i) > 4.8 and "ecchi" not in str(p):
item = MangaItem()
item['title'] = site.select("div/a/text()").extract()
item['link'] = site.select("div/a/@href").extract()
item['desc'] = site.select("p[2]/text()").extract()
item['rate'] = site.select("p/span/text()").extract()
items.append(item)
return items
评论的东西是一种在没有爬虫的情况下爬取页面的方法,这里有人帮助我。但是为了知道,我还是想学习如何让爬虫工作。
我没有收到任何错误,但它刮掉了 0 页,我检查了很多线程,听起来我parse_start_url
出于某种原因必须添加一个,但这没有帮助,更改 parse 函数的名称也没有帮助。
什么不工作?我的规则不正确还是我遗漏了什么?