from nltk import word_tokenize, pos_tag
from nltk.corpus import wordnet as wn
def penn_to_wn(tag):
""" Convert between a Penn Treebank tag to a simplified Wordnet tag """
if tag.startswith('N'):
return 'n'
if tag.startswith('V'):
return 'v'
if tag.startswith('J'):
return 'a'
if tag.startswith('R'):
return 'r'
return None
def tagged_to_synset(word, tag):
wn_tag = penn_to_wn(tag)
if wn_tag is None:
return None
try:
return wn.synsets(word, wn_tag)[0]
except:
return None
def sentence_similarity(sentence1, sentence2):
""" compute the sentence similarity using Wordnet """
# Tokenize and tag
sentence1 = pos_tag(word_tokenize(sentence1))
sentence2 = pos_tag(word_tokenize(sentence2))
# Get the synsets for the tagged words
synsets1 = [tagged_to_synset(*tagged_word) for tagged_word in sentence1]
synsets2 = [tagged_to_synset(*tagged_word) for tagged_word in sentence2]
# Filter out the Nones
synsets1 = [ss for ss in synsets1 if ss]
synsets2 = [ss for ss in synsets2 if ss]
score, count = 0.0, 0
# For each word in the first sentence
for synset in synsets1:
# Get the similarity value of the most similar word in the other sentence
**best_score = max([(synset.path_similarity(ss)) for ss in synsets2])**
# Check that the similarity could have been computed
if best_score is not None:
score += best_score
count += 1
# Average the values
score /= count
return score
if __name__ == '__main__':
sentences = [
'Password should not be less than 8 characters.',
'The user should enter valid user name and password.',
'User name should not have special characters.',
'Datta passed out from IIT',
]
focus_sentence = 'The user should enter valid user name and password and password should have greater than or equal to 8 characters.'
for sentence in sentences:
print(sentence_similarity(focus_sentence, sentence))
问问题
618 次
2 回答
1
正如@Chris_Rands 所述,您的问题是该函数path_similarity()
可以返回None
然后max()
调用失败。这是一个验证这种情况何时发生的问题。一种可能的解决方案是创建一个列表,simlist
从. 如果为空,则跳过当前迭代,如果不是,则调用 max() 并继续其余的迭代。None
path_similarity()
simlist
# For each word in the first sentence
for synset in synsets1:
# Get the similarity value of the most similar word in the other sentence
simlist = [synset.path_similarity(ss) for ss in synsets2 if synset.path_similarity(ss) is not None]
if not simlist:
continue;
best_score = max(simlist)
# Check that the similarity could have been computed
score += best_score
count += 1
if count == 0:
return 0
# Average the values
score /= count
return score
于 2017-09-24T02:23:12.250 回答
0
对于第一句话中的每个单词
for synset in synsets1:
Get the similarity value of the most similar word in the other sentence
simlist = [synset.path_similarity(ss) for ss in synsets2 if synset.path_similarity(ss) is not None]
if not simlist:
continue;
best_score = max(simlist)
检查是否可以计算相似度 score += best_score count += 1
if count == 0:
return 0
Average the values
score /= count
return score
它正在工作
于 2018-11-02T11:44:03.593 回答