我在教程的帮助下制作了一个网络爬虫，它从给定的 url 获取所有链接，你可以给它传递一个数字，它对应于链接的步骤/深度。现在，当您定义一个scraperOut = scraper(url,3)当前为 3 的数字时，爬虫会深入 3 步并将链接附加到同一个列表。我的问题是如何以及在代码中修改什么，以便我可以选择单独打印每个列表，而不是全部附加在一个列表中，或者例如只打印第二步列表？整个代码如下所示：
import urllib
import re
import time
from threading import Thread
import MySQLdb
import mechanize
import readability
from bs4 import BeautifulSoup
from readability.readability import Document
import urlparse
url = "http://www.adbnews.com/area51/"
def scraper(root,steps):
urls = [root]
visited = [root]
counter = 0
while counter < steps:
step_url = scrapeStep(urls)
urls = []
for u in step_url:
if u not in visited:
urls.append(u)
visited.append(u)
counter +=1
return visited
def scrapeStep(root):
result_urls = []
br = mechanize.Browser()
br.set_handle_robots(False)
br.addheaders = [('User-agent', 'Firefox')]
for url in root:
try:
br.open(url)
for link in br.links():
newurl = urlparse.urljoin(link.base_url, link.url)
result_urls.append(newurl)
except:
print "error"
return result_urls
d = {}
threadlist = []
def getReadableArticle(url):
br = mechanize.Browser()
br.set_handle_robots(False)
br.addheaders = [('User-agent', 'Firefox')]
html = br.open(url).read()
readable_article = Document(html).summary()
readable_title = Document(html).short_title()
soup = BeautifulSoup(readable_article)
final_article = soup.text
links = soup.findAll('img', src=True)
return readable_title
return final_article
def dungalo(urls):
article_text = getReadableArticle(urls)[0]
d[urls] = article_text
def getMultiHtml(urlsList):
for urlsl in urlsList:
try:
t = Thread(target=dungalo, args=(urls1,))
threadlist.append(t)
t.start()
except:
nnn = True
for g in threadlist:
g.join()
return d
scraperOut = scraper(url,3)
for s in scraperOut:
print s
#print scraperOut