重写它比找出你的代码出了什么问题更容易：
import os.path
import glob
import re
import itertools
from collections import namedtuple, deque
from operator import attrgetter
R_PREFIX_VALUE = re.compile(r'^(?P<prefix>[A-Z]+)(?P<suffix>\d+)\s+(?P<value>\d+)\s*$')
getvalue = attrgetter('value')
def interleave(seq, val):
return itertools.chain.from_iterable(itertools.izip(seq, itertools.repeat(val)))
class Fileline(namedtuple('Fileline', 'filename prefix suffix value')):
@classmethod
def _fromstr(cls, s, filename=None, rematch=R_PREFIX_VALUE.match):
m = rematch(s)
if not m:
raise ValueError('No valid line found in %r' % s)
d = m.groupdict()
d['value'] = int(d['value'])
d['filename'] = filename
return cls(**d)
def _asstr(self):
return '{}{} {}'.format(self.prefix, self.suffix, self.value)
def max_value_with_prefix(lineseq, prefix, getvalue=getvalue):
withprefix = (line for line in lineseq if line.prefix==prefix)
return max_value(withprefix)
def filter_lt_line(lineseq, maxline):
for line in lineseq:
if line.prefix != maxline.prefix or line.value >= maxline.value:
yield line
def extreme_value(fn, lineseq, getvalue=getvalue):
try:
return fn((l for l in lineseq if l is not None), key=getvalue)
except ValueError:
return None
def max_value(lineseq):
return extreme_value(max, lineseq)
def min_value(lineseq):
return extreme_value(min, lineseq)
def read_lines(fn, maker=Fileline._fromstr):
with open(fn, 'rb') as f:
return deque(maker(l, fn) for l in f)
def write_file(fn, lineseq):
lines = (l._asstr() for l in lineseq)
newlines = interleave(lines, '\n')
with open(fn, 'wb') as f:
f.writelines(newlines)
def write_output_file(fn, lineseq):
lines = ("{} {}".format(l.filename, l.value) for l in lineseq)
newlines = interleave(lines, "\n")
with open(fn, 'wb') as f:
f.writelines(newlines)
def filter_max_returning_min(fn, prefix):
lineseq = read_lines(fn)
maxvalue = max_value_with_prefix(lineseq, prefix)
filteredlineseq = deque(filter_lt_line(lineseq, maxvalue))
write_file(fn, filteredlineseq)
minline = min_value(filteredlineseq)
return minline
def main(fileglob, prefix, outputfile):
minlines = []
for fn in glob.iglob(fileglob):
minlines.append(filter_max_returning_min(fn, prefix))
write_output_file(outputfile, minlines)
入口点是main()，称为like main('txtdir', 'ENSG', 'output.txt')。对于每个文件filter_max_returning_min()都会打开并重写文件并返回最小值。无需保留您访问的每个文件的每一行的字典或列表。
（顺便说一句，破坏性覆盖文件似乎是个坏主意！您是否考虑过将它们复制到其他地方？）
当您将单独的关注点隔离到单独的函数中时，为不同的执行行为重新组合它们变得非常容易。例如，通过添加两个小函数在所有文件上并行运行此任务很简单：
def _worker(args):
return filter_max_returning_min(*args)
def multi_main(fileglob, prefix, outputfile, processes):
from multiprocessing import Pool
pool = Pool(processes=processes)
workerargs = ((fn, prefix) for fn in glob.iglob(fileglob))
minlines = pool.imap_unordered(_worker, workerargs, processes)
write_file(outputfile, minlines)
现在您可以启动可配置数量的工作人员，每个工作人员都将在一个文件上工作，并在完成后收集他们的最小值。如果您有非常大的文件或大量文件并且不受 IO 限制，这可能会更快。
只是为了好玩，您还可以轻松地将其转换为 CLI 实用程序：
def _argparse():
import argparse
def positive_int(s):
v = int(s)
if v < 1:
raise argparse.ArgumentTypeError('{:r} must be a positive integer'.format(s))
return v
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description="""Filter text files and write min value.
Performs these operations on the text files in supplied `filedir`:
1. In each file, identify lines starting with the matching `maxprefix`
which do *not* contain the maximum value for that prefix in that file.
2. DESTRUCTIVELY REWRITE each file with lines found in step 1 removed!
3. Write the minimum value (for all lines in all files) to `outputfile`.
""")
parser.add_argument('filedir',
help="Directory containg the text files to process. WILL REWRITE FILES!")
parser.add_argument('maxprefix', nargs="?", default="ENSG",
help="Line prefix which should have values less than max value removed in each file")
parser.add_argument('outputfile', nargs="?", default="output.txt",
help="File in which to write min value found. WILL REWRITE FILES!")
parser.add_argument('-p', '--parallel', metavar="N", nargs="?", type=positive_int, const=10,
help="Process files in parallel, with N workers. Default is to process a file at a time.")
return parser.parse_args()
if __name__ == '__main__':
args = _argparse()
fileglob = os.path.join(args.filedir, '*.txt')
prefix = args.maxprefix
outputfile = args.outputfile
if args.parallel:
multi_main(fileglob, prefix, outputfile, args.parallel)
else:
main(fileglob, prefix, outputfile)
现在您可以从命令行调用它：
$ python ENSG.py txtdir ENSCAFG --parallel=4