我想使用以下代码将 page_data 中的“ /xxxxx/ ”等字符串替换为“ /xxxxx.html ”,但不起作用。page_data 是由爬虫下载的字节类型。
page_data.replace(each, neweach)
只有当我将它们更改为:
page_data = page_data.replace(each, neweach)
page_data 中的字符串(每个)实际上已被替换。
整个代码如下:
import os
import sys
import re
import urllib
import urllib2
class WebGet(object):
base_url = ""
urls_list = []
history_list = []
replace_ch={}
def __init__(self, base_url):
self.base_url = base_url[:-1]
self.urls_list.append('/')
self.replace_ch[">>"] = "%3E%3E"
self.replace_ch["<<"] = "%3C%3C"
self.replace_ch["::"] = "%3A%3A"
def recurseGet(self):
'''Get page data recursively'''
while(len(self.urls_list) != 0):
url_suffix = self.urls_list[0]
self.urls_list.remove(url_suffix)
self.history_list.append(url_suffix)
url_to_get = self.base_url + url_suffix
"Get page data with url"
print "To get",url_to_get
page_data = urllib2.urlopen(url_to_get).read()
page_data_done = self.pageHandle(page_data)
"Write the page data into file"
if url_suffix[-1] == '/':
url_suffix = url_suffix[:-1]
if url_suffix == '':
url_suffix = "index"
elif url_suffix[0] == '/':
url_suffix = url_suffix[1:]
url_suffix.replace('/','\\')
url_suffix.replace('>>','%3E%3E')
url_suffix.replace('<<','%3C%3C')
url_suffix.replace('::','%3A%3A')
file_str = "e:\\reference\\"+url_suffix
if file_str.rfind("\\") != 12:
new_dir = file_str[:file_str.rfind("\\")]
if os.path.isdir(file_str) == False:
os.mkdir(file_str)
file_str = file_str.strip()+".html"
print "write file",file_str
f_page = open(file_str, "wb")
f_page.write(page_data_done)
f_page.close
def pageHandle(self, page_data):
page_data.replace("http://www.cplusplus.com/","/") #here the replace works
re_rule = '<a href="/reference(/\S{2,40}/)\">'
list_page_urls = re.findall(re_rule, page_data)
for each in list_page_urls:
neweach = each
neweach = neweach[:-1]+".html"
#page_data = page_data.replace(each, neweach)
page_data.replace(each, neweach)
if each in page_data:
print "fail replace"
if each in self.history_list:
continue
elif each in self.urls_list:
continue
elif each == '/':
continue
self.urls_list.append(each)
return page_data
def main():
url = "http://www.cplusplus.com/reference/"
fc = WebGet(url)
fc.recurseGet()
if __name__ == "__main__":
main()
为什么会这样?