5

也许这是一个愚蠢的问题，但我在用 Python 从语料库中提取十个最常用的词时遇到了问题。这就是我到目前为止所得到的。（顺便说一句，我与 NLTK 合作阅读一个语料库，其中包含两个子类别，每个 10 个 .txt 文件）

import re
import string
from nltk.corpus import stopwords
stoplist = stopwords.words('dutch')

from collections import defaultdict
from operator import itemgetter

def toptenwords(mycorpus):
    words = mycorpus.words()
    no_capitals = set([word.lower() for word in words]) 
    filtered = [word for word in no_capitals if word not in stoplist]
    no_punct = [s.translate(None, string.punctuation) for s in filtered] 
    wordcounter = {}
    for word in no_punct:
        if word in wordcounter:
            wordcounter[word] += 1
        else:
            wordcounter[word] = 1
    sorting = sorted(wordcounter.iteritems(), key = itemgetter, reverse = True)
    return sorting

如果我用我的语料库打印这个函数，它会给我一个后面带“1”的所有单词的列表。它给了我一本字典，但我所有的价值观都是一个。而且我知道，例如，“婴儿”这个词在我的语料库中出现了五六次……但它仍然给出了“婴儿：1”……所以它不能按照我想要的方式运作……
有人可以帮忙吗我？

4

4 回答 4

9

如果您仍然使用 NLTK，请尝试使用 FreqDist(samples) 函数首先从给定样本生成频率分布。然后调用 most_common(n) 属性查找样本中最常见的 n 个词，按频率降序排列。就像是：

from nltk.probability import FreqDist
fdist = FreqDist(stoplist)
top_ten = fdist.most_common(10)
于 2014-07-29T10:36:20.980 回答
7

蟒蛇方式：

In [1]: from collections import Counter

In [2]: words = ['hello', 'hell', 'owl', 'hello', 'world', 'war', 'hello', 'war']

In [3]: counter_obj = Counter(words)

In [4]: counter_obj.most_common() #counter_obj.most_common(n=10)
Out[4]: [('hello', 3), ('war', 2), ('hell', 1), ('world', 1), ('owl', 1)]
于 2017-07-07T04:23:09.940 回答
3

问题在于您对set.

集合不包含重复项，因此当您创建一组小写单词时，从那里开始每个单词只有一次出现。

假设你words是：

 ['banana', 'Banana', 'tomato', 'tomato','kiwi']

在你的 lambda 降低所有情况后，你有：

 ['banana', 'banana', 'tomato', 'tomato','kiwi']

但是你这样做：

 set(['banana', 'Banana', 'tomato', 'tomato','kiwi'])

返回：

 ['banana','tomato','kiwi']

由于从那一刻起，您将计算基于no_capitals集合，因此每个单词只会出现一次。不要创建set，您的程序可能会正常工作。

于 2013-01-24T11:26:40.610 回答
1

这是一种解决方案。使用前面的响应中讨论的集合。



def token_words(tokn=10, s1_orig='hello i must be going'):
    # tokn is the number of most common words.
    # s1_orig is the text blob that needs to be checked.

    # logic
    # - clean the text - remove punctuations.
    # - make everything lower case
    # - replace common machine read errors.
    # - create a dictionary with orig words and changed words.
    # - create a list of unique clean words
    # - read the "clean" text and count the number of clean words
    # - sort and print the results

    #print 'Number of tokens:', tokn

    # create a dictionary to make puncuations
    # spaces.
    punct_dict = {  ',':' ', 
                    '-':' ',
                    '.':' ',
                    '\n':' ',
                    '\r':' '
                    }

    # dictionary for machine reading errors
    mach_dict = {'1':'I', '0':'O',
                '6':'b','8':'B' }


    # get rid of punctuations
    s1 = s1_orig
    for k,v in punct_dict.items():
        s1 = s1.replace(k,v)

    # create the original list of words.
    orig_list = set(s1.split())

    # for each word in the original list,
    # see if it has machine errors.
    # add error words to a dict.
    error_words = dict()
    for a_word in orig_list:
        a_w2 = a_word
        for k,v in mach_dict.items():
            a_w2 = a_w2.replace(k,v)

        # lower case the result.
        a_w2 = a_w2.lower()

        # add to error word dict.
        try:
            error_words[a_w2].append(a_word)
        except:
            error_words[a_w2] = [a_word]

    # get rid of machine errors in the full text.
    for k,v in mach_dict.items():
        s1 = s1.replace(k,v)

    # make everything lower case
    s1 = s1.lower()

    # split sentence into list.
    s1_list = s1.split()

    # consider only unqiue words
    s1_set = set(s1_list)

    # count the number of times 
    # the each word occurs in s1 
    res_dict = dict()   
    for a_word in s1_set:
        res_dict[a_word] = s1_list.count(a_word)


    # sort the result dictionary by values
    print '--------------'
    temp = 0
    for key, value in sorted(res_dict.iteritems(), reverse=True, key=lambda (k,v): (v,k)):
        if temp < tokn: 
            # print results for token items
            # get all the words that made up the key
            final_key = ''
            for er in error_words[key]:
                final_key = final_key + er + '|'
            final_key = final_key[0:-1]
            print "%s@%s" % (final_key, value)
        else:
            pass
        temp = temp + 1

    # close the function and return
    return True

#-------------------------------------------------------------    
# main

# read the inputs from command line
num_tokens = raw_input('Number of tokens desired: ')    
raw_file = raw_input('File name: ') 

# read the file
try:
    if num_tokens == '': num_tokens = 10
    n_t = int(num_tokens)
    raw_data = open(raw_file,'r').read()
    token_words(n_t, raw_data)
except:
    print 'Token or file error.  Please try again.'
于 2018-03-09T13:20:10.397 回答