如何使用 Spacy 创建一个新的名称实体“病例”——在传染病病例数的上下文中，然后提取此病例数与病例基数之间的依赖关系。
例如，在以下文本中，“其中，1995 年 10 月 9 日至 11 月 5 日期间报告了 879 例病例，其中 4 人死亡。” 我们想提取“879”和“cases”
根据 Spacy 的示例文档页面上的“训练其他实体类型”的代码：
https://spacy.io/usage/examples#information-extraction
我使用他们现有的预训练“en_core_web_sm”英文模型，成功训练了一个名为“CASES”的附加实体：
from __future__ import unicode_literals, print_function
import plac
import random
from pathlib import Path
import spacy
from spacy.util import minibatch, compounding
LABEL = "CASES"
TRAIN_DATA = results_ent2[0:400]
def main(model="en_core_web_sm", new_model_name="cases", output_dir='data3', n_iter=30):
random.seed(0)
if model is not None:
nlp = spacy.load(model) # load existing spaCy model
print("Loaded model '%s'" % model)
else:
nlp = spacy.blank("en") # create blank Language class
print("Created blank 'en' model")
# Add entity recognizer to model if it's not in the pipeline
# nlp.create_pipe works for built-ins that are registered with spaCy
if "ner" not in nlp.pipe_names:
ner = nlp.create_pipe("ner")
nlp.add_pipe(ner)
# otherwise, get it, so we can add labels to it
else:
ner = nlp.get_pipe("ner")
ner.add_label(LABEL) # add new entity label to entity recognizer
# Adding extraneous labels shouldn't mess anything up
if model is None:
optimizer = nlp.begin_training()
else:
optimizer = nlp.resume_training()
move_names = list(ner.move_names)
# get names of other pipes to disable them during training
pipe_exceptions = ["ner", "trf_wordpiecer", "trf_tok2vec"]
other_pipes = [pipe for pipe in nlp.pipe_names if pipe not in pipe_exceptions]
with nlp.disable_pipes(*other_pipes): # only train NER
sizes = compounding(1.0, 4.0, 1.001)
# batch up the examples using spaCy's minibatch
for itn in range(n_iter):
random.shuffle(TRAIN_DATA)
batches = minibatch(TRAIN_DATA, size=sizes)
losses = {}
for batch in batches:
texts, annotations = zip(*batch)
nlp.update(texts, annotations, sgd=optimizer, drop=0.35, losses=losses)
print("Losses", losses)
# test the trained model
test_text = "There were 100 confirmed cases?"
doc = nlp(test_text)
print("Entities in '%s'" % test_text)F
for ent in doc.ents:
print(ent.label_, ent.text)
# save model to output directory
if output_dir is not None:
output_dir = Path(output_dir)
if not output_dir.exists():
output_dir.mkdir()
nlp.meta["name"] = new_model_name # rename model
nlp.to_disk(output_dir)
print("Saved model to", output_dir)
# test the saved model
print("Loading from", output_dir)
nlp2 = spacy.load(output_dir)
# Check the classes have loaded back consistently
assert nlp2.get_pipe("ner").move_names == move_names
doc2 = nlp2(test_text)
for ent in doc2.ents:
print(ent.label_, ent.text)
main()
测试输出：
test_text = 'Of these, 879 cases with 4 deaths were reported for the period 9 October to 5 November 1995. John was infected. It cost $500'
doc = nlp(test_text)
print("Entities in '%s'" % test_text)
for ent in doc.ents:
print(ent.label_, ent.text)
我们得到一个结果
Entities in 'Of these, 879 cases with 4 deaths were reported for the period 9 October to 5 November 1995. John was infected. It cost $500'
CARDINAL 879
CASES cases
CARDINAL 4
CARDINAL 9
CARDINAL 5
CARDINAL $500
模型已保存，可以从上述文字中正确识别出 CASES。
我的目标是从新闻文章中提取给定疾病/病毒的病例数，然后再提取死亡人数。
我现在使用这个新创建的模型试图找到 CASES 和 CARDINAL 之间的依赖关系：
再次使用 Spacy 的示例
https://spacy.io/usage/examples#new-entity-type
'训练 spaCy 的依赖解析器'
import plac
import spacy
TEXTS = [
"Net income was $9.4 million compared to the prior year of $2.7 million. I have 100,000 cases",
"Revenue exceeded twelve billion dollars, with a loss of $1b.",
"Of these, 879 cases with 4 deaths were reported for the period 9 October to 5 November 1995. John was infected. It cost $500"
]
def main(model="data3"):
nlp = spacy.load(model)
print("Loaded model '%s'" % model)
print("Processing %d texts" % len(TEXTS))
for text in TEXTS:
doc = nlp(text)
relations = extract_currency_relations(doc)
for r1, r2 in relations:
print("{:<10}\t{}\t{}".format(r1.text, r2.ent_type_, r2.text))
def filter_spans(spans):
# Filter a sequence of spans so they don't contain overlaps
# For spaCy 2.1.4+: this function is available as spacy.util.filter_spans()
get_sort_key = lambda span: (span.end - span.start, -span.start)
sorted_spans = sorted(spans, key=get_sort_key, reverse=True)
result = []
seen_tokens = set()
for span in sorted_spans:
# Check for end - 1 here because boundaries are inclusive
if span.start not in seen_tokens and span.end - 1 not in seen_tokens:
result.append(span)
seen_tokens.update(range(span.start, span.end))
result = sorted(result, key=lambda span: span.start)
return result
def extract_currency_relations(doc):
# Merge entities and noun chunks into one token
spans = list(doc.ents) + list(doc.noun_chunks)
spans = filter_spans(spans)
with doc.retokenize() as retokenizer:
for span in spans:
retokenizer.merge(span)
relations = []
for money in filter(lambda w: w.ent_type_ == "MONEY", doc):
if money.dep_ in ("attr", "dobj"):
subject = [w for w in money.head.lefts if w.dep_ == "nsubj"]
if subject:
subject = subject[0]
relations.append((subject, money))
elif money.dep_ == "pobj" and money.head.dep_ == "prep":
relations.append((money.head.head, money))
return relations
main()
输出如下，没有依赖检测。就好像模型失去了这种能力，同时保留了检测命名实体的能力。或者，也许某种设置已被关闭？
Loaded model 'data3'
Processing 3 texts
如果我使用原始的预训练模型“en_core_web_sm”，结果是：
Processing 3 texts
Net income MONEY $9.4 million
the prior year MONEY $2.7 million
Revenue MONEY twelve billion dollars
a loss MONEY 1b
这与 Spacy 示例页面上模型的输出相同。
有人知道发生了什么，为什么我的新模型（在原始 Spacy 'en_core_web_sm' 上使用迁移学习）现在无法在此示例中找到依赖项？
编辑：
如果我使用更新后的训练模型，它可以检测到新实体“案例”和基数“100,000”，但是它失去了检测金钱和日期的能力。
当我训练模型时，我训练了数千个句子，使用基本模型 en_core_web_sm 本身来检测所有实体并标记它们，以避免模型“忘记”旧实体。
