我是 python 新手，天真地为以下任务编写了一个 python 脚本：
我想创建一个包含多个对象的单词表示。每个对象基本上都是一对，并且要制作概要的词袋表示。所以对象在最终文档中被转换为。
这是脚本：
import re
import math
import itertools
from nltk.corpus import stopwords
from nltk import PorterStemmer
from collections import defaultdict
from collections import Counter
from itertools import dropwhile
import sys, getopt
inp = "inp_6000.txt" #input file name
out = "bowfilter10" #output file name
with open(inp,'r') as plot_data:
main_dict = Counter()
file1, file2 = itertools.tee(plot_data, 2)
line_one = itertools.islice(file1, 0, None, 4)
line_two = itertools.islice(file2, 2, None, 4)
dictionary = defaultdict(Counter)
doc_count = defaultdict(Counter)
for movie_name, movie_plot in itertools.izip(line_one, line_two):
movie_plot = movie_plot.lower()
words = re.findall(r'\w+', movie_plot, flags = re.UNICODE | re.LOCALE) #split words
elemStopW = filter(lambda x: x not in stopwords.words('english'), words) #remove stop words, python nltk
for word in elemStopW:
word = PorterStemmer().stem_word(word) #use python stemmer class to do stemming
#increment the word count of the movie in the particular movie synopsis
dictionary[movie_name][word] += 1
#increment the count of a partiular word in main dictionary which stores frequency of all documents.
main_dict[word] += 1
#This is done to calculate term frequency inverse document frequency. Takes note of the first occurance of the word in the synopsis and neglect all other.
if doc_count[word]['this_mov']==0:
doc_count[word].update(count=1, this_mov=1);
for word in doc_count:
doc_count[word].update(this_mov=-1)
#print "---------main_dict---------"
#print main_dict
#Remove all the words with frequency less than 5 in whole set of movies
for key, count in dropwhile(lambda key_count: key_count[1] >= 5, main_dict.most_common()):
del main_dict[key]
#print main_dict
.#Write to file
bow_vec = open(out, 'w');
#calculate the the bog vector and write it
m = len(dictionary)
for movie_name in dictionary.keys():
#print movie_name
vector = []
for word in list(main_dict):
#print word, dictionary[movie_name][word]
x = dictionary[movie_name][word] * math.log(m/doc_count[word]['count'], 2)
vector.append(x)
#write to file
bow_vec.write("%s" % movie_name)
for item in vector:
bow_vec.write("%s," % item)
bow_vec.write("\n")
数据文件的格式和有关数据的附加信息： 数据文件具有以下格式：
电影名称。空行。电影简介（上可以假设大小为150字左右） 空行。
注意：<*>用于表示。
输入文件大小：
文件大小约为 200 MB。
截至目前，此脚本在 3 GHz 英特尔处理器上大约需要 10-12 小时。
注意：我正在寻找串行代码的改进。我知道并行化会改善它，但我想稍后再研究它。我想借此机会让这个串行代码更有效率。
任何帮助表示赞赏。