0

我正在尝试使用我正在抓取的多个 url 中的数据创建一个数据框。该代码有效，但是我无法将数据递归地存储在一个 DataFrame 中。DataFrame（称为帧）每次都被新的 url 数据替换，而不是将新数据连接到同一帧。谢谢你，我非常感谢你的帮助！

import urllib
import re
import json
import pandas
import pylab
import numpy
import matplotlib.pyplot
from pandas import *
from pylab import *
from threading import Thread
import sqlite3

urls = ['http://www.ratemyprofessors.com/ShowRatings.jsp?tid=1176131' , 'http://www.ratemyprofessors.com/ShowRatings.jsp?tid=795226', 'http://www.ratemyprofessors.com/ShowRatings.jsp?tid=1176131' , 'http://www.ratemyprofessors.com/ShowRatings.jsp?tid=1807944', 'http://www.ratemyprofessors.com/ShowRatings.jsp?tid=277459' , 'http://www.ratemyprofessors.com/ShowRatings.jsp?tid=1076779' , 'http://www.ratemyprofessors.com/ShowRatings.jsp?tid=971546']

i=0
regex = '<p class="commentText">(.+?)</p>'
regex2 = '<strong>Easiness</strong><span>(.+?)</span></p>'
regex3 = 'Helpfulness</strong><span>(.+?)</span></p>'
regex4 = 'Clarity</strong><span>(.+?)</span></p>'
regex5 = 'Rater Interest</strong><span>(.+?)</span></p>'
regex6 = '<div class="date">(.+?)</div>'
regex7 = '<div class="class"><p style="word-wrap:break-word;">(.+?)</p>'
regex8 = '<meta name="prof_name" content="(.+?)"/>'

pattern = re.compile(regex)
easiness = re.compile(regex2)
helpfulness = re.compile(regex3)
clarity = re.compile(regex4)
interest = re.compile(regex5)
date = re.compile(regex6)
mathclass = re.compile(regex7)
prof_name = re.compile(regex8)

while i < len(urls):
    htmlfile = urllib.urlopen(urls[i])
    htmltext = htmlfile.read()
    content = re.findall(pattern,htmltext)
    Easiness = re.findall(easiness,htmltext)
    Helpfulness = re.findall(helpfulness, htmltext)
    Clarity = re.findall(clarity, htmltext)
    Interest = re.findall(interest, htmltext)
    Date = re.findall(date, htmltext)
    Class = re.findall(mathclass, htmltext)
    PROFNAME=re.findall(prof_name, htmltext)
    i+=1

    frame = DataFrame({'Comments': content, 'Easiness': Easiness, 'Helpfulness': Helpfulness, 
    'Clarity': Clarity, 'Rater Interest': Interest, 'Class': Class,
    'Date': Date[1:len(Date)], 'Professor': PROFNAME[0]})

    print frame
4

2 回答 2

0

使用pd.concat：

frames = []

while i < len(urls):
    htmlfile = urllib.urlopen(urls[i])
    htmltext = htmlfile.read()
    content = re.findall(pattern,htmltext)
    Easiness = re.findall(easiness,htmltext)
    Helpfulness = re.findall(helpfulness, htmltext)
    Clarity = re.findall(clarity, htmltext)
    Interest = re.findall(interest, htmltext)
    Date = re.findall(date, htmltext)
    Class = re.findall(mathclass, htmltext)
    PROFNAME=re.findall(prof_name, htmltext)
    i+=1

    frames.append(DataFrame({'Comments': content, 'Easiness': Easiness, 'Helpfulness': Helpfulness, 
    'Clarity': Clarity, 'Rater Interest': Interest, 'Class': Class,
    'Date': Date[1:len(Date)], 'Professor': PROFNAME[0]}))

pd.concat(frames)
于 2013-08-16T04:23:31.637 回答
0

您正在用循环的每次迭代覆盖您的框架。正如 Phillip Cloud 建议的那样，您可以列出每个循环附加的帧。我以不同的方式简化了您的代码，但我认为这可以满足您的需求。

import urllib
import re
import pandas as pd

urls = ['http://www.ratemyprofessors.com/ShowRatings.jsp?tid=1176131',
        'http://www.ratemyprofessors.com/ShowRatings.jsp?tid=795226',
        'http://www.ratemyprofessors.com/ShowRatings.jsp?tid=1176131',
        'http://www.ratemyprofessors.com/ShowRatings.jsp?tid=1807944',
        'http://www.ratemyprofessors.com/ShowRatings.jsp?tid=277459', 
        'http://www.ratemyprofessors.com/ShowRatings.jsp?tid=1076779', 
        'http://www.ratemyprofessors.com/ShowRatings.jsp?tid=971546']

regex = {'pattern' : re.compile('<p class="commentText">(.+?)</p>'),
        'easiness' : re.compile('<strong>Easiness</strong><span>(.+?)</span></p>'),
        'helpfulness' : re.compile('Helpfulness</strong><span>(.+?)</span></p>'),
        'clarity' : re.compile('Clarity</strong><span>(.+?)</span></p>'),
        'interest' : re.compile('Rater Interest</strong><span>(.+?)</span></p>'),
        'date' : re.compile('<div class="date">(.+?)</div>'),
        'mathclass' : re.compile('<div class="class"><p style="word-wrap:break-word;">(.+?)</p>'),
        'prof_name' : re.compile('<meta name="prof_name" content="(.+?)"/>')}

# Make a dictionary with empty lists using the same keys
d = {}
for k in regex.keys():
    d[k] = []

# Now fill those lists
for url in urls:
    htmlfile = urllib.urlopen(url)
    htmltext = htmlfile.read()
    for k, v in regex.iteritems():
        d[k].append(re.findall(v, htmltext))
frame = pd.DataFrame(d) # Dump the dict into a DataFrame
print frame
于 2013-08-16T04:43:35.303 回答