corpus - 用英语以外的其他语言为 Brill Tagger 训练语料库

Question

有谁知道在英语以外的其他语言中使用 Brill 词性 (POS) 标记器的任何标记语料库或词典？

谢谢！

Answer 1

如果您正在使用NLTK( http://nltk.org/ ) 并与您一起编码，python则可以执行以下操作。您甚至不需要编写自己的 brill-tagger 代码，因为它已经在库中，http: //nltk.org/_modules/nltk/tag/brill.html 。

def train_brill_tagger(train_data):
    # Modules for creating the templates.
    from nltk.tag import UnigramTagger
    from nltk.tag.brill import SymmetricProximateTokensTemplate, ProximateTokensTemplate
    from nltk.tag.brill import ProximateTagsRule, ProximateWordsRule
    # The brill tagger module in NLTK.
    from nltk.tag.brill import FastBrillTaggerTrainer
    unigram_tagger = UnigramTagger(train_data)
    templates = [SymmetricProximateTokensTemplate(ProximateTagsRule, (1,1)),
                            SymmetricProximateTokensTemplate(ProximateTagsRule, (2,2)),
                            SymmetricProximateTokensTemplate(ProximateTagsRule, (1,2)),
                            SymmetricProximateTokensTemplate(ProximateTagsRule, (1,3)),
                            SymmetricProximateTokensTemplate(ProximateWordsRule, (1,1)),
                            SymmetricProximateTokensTemplate(ProximateWordsRule, (2,2)),
                            SymmetricProximateTokensTemplate(ProximateWordsRule, (1,2)),
                            SymmetricProximateTokensTemplate(ProximateWordsRule, (1,3)),
                            ProximateTokensTemplate(ProximateTagsRule, (-1, -1), (1,1)),
                            ProximateTokensTemplate(ProximateWordsRule, (-1, -1), (1,1))]

    trainer = FastBrillTaggerTrainer(initial_tagger=unigram_tagger,
                                   templates=templates, trace=3,
                                   deterministic=True)
    brill_tagger = trainer.train(train_data, max_rules=10)
    print
    return brill_tagger

# To train and test using Brown Corpus.
from nltk.corpus import brown
brown_train = list(brown.tagged_sents(categories='news')[:500])
brown_test = list(brown.tagged_sents(categories='news')[500:600])
brown501 = brown.tagged_sents(categories='news')[501]

bt = train_brill_tagger(brown_train)

# To tag one sentence.
print bt.tag(brown501)
print

# To evaluate tagger.
print 'Accuracy of Brill Tagger:', bt.evaluate(brown_test)

有一个语料库列表，其中语料库阅读器已经预编码在NLTK： http: //nltk.googlecode.com/svn/trunk/nltk_data/index.xml

这是将 brill 标记器应用于荷兰语语料库的示例：

# To train and test using Alpino Corpus (Dutch).
from nltk.corpus import alpino
alpino_tagged_sents = alpino.tagged_sents()
# Split corpus into train/test.
datasize = len(alpino_tagged_sents)
trainsize = int(datasize*90/float(100))
alpino_train = list(alpino_tagged_sents[:trainsize])
alpino_test = list(alpino_tagged_sents[trainsize+1:])
alpinotest1 = [i for i,j in alpino_tagged_sents[trainsize+1]]

bt_nld = train_brill_tagger(alpino_train)
print 'Test sentence:', alpinotest
print bt_nld.tag(alpinotest1)
print
print bt_nld.evaluate(alpino_test)
print

事实上，如果你足够勤奋地阅读到这一点，这里就是通过输入语料库在 NLTK 中训练一个聪明的标注器的技巧 =)

from nltk.corpus import LazyCorpusLoader

def train_brill_with_corpus(nltkcorpus, train_percent = 90/float(100)):
    if not isinstance(nltkcorpus, LazyCorpusLoader):
        raise NameError("Please use a pre-coded corpus from NLTK.")

    tagged_sents = nltkcorpus.tagged_sents()
    if not tagged_sents: # i.e. tagged_sents == []
        raise NameError("This corpus doesn't have POS tags.")
    trainsize = int(train_percent*len(tagged_sents))
    corpus_train = list(tagged_sents[:trainsize])
    corpus_test = list(tagged_sents[trainsize+1:])
    bt = train_brill_tagger(corpus_train)
    return bt, corpus_test  

基本上使用train_brill_tagger()and train_brill_with_corpus()，你可以这样做：

# To train and test using CESS_ESP Corpus (Spanish).
from nltk.corpus import cess_esp
bt_spa, cess_test = train_brill_with_corpus(cess_esp)
cesstest1 = [i for i,j in cess_test[0]]
print 'Test sentence:', cesstest1
print bt_spa.tag(cesstest1); print
print bt_spa.evaluate(cess_test)