我设法获得了第一个在 Microsoft.ML 中运行的文本分析器。我想获得模型确定的 ngram 列表,但我只能在不知道它们指的是什么的情况下获得“计数”出现次数的数值向量。
到目前为止,这是我的工作代码的核心:
var mlContext = new MLContext();
var articles = SampleData.Articles.Select(a => new TextData{ Text=a }).ToArray();
var dataview = mlContext.Data.LoadFromEnumerable(articles);
var options = new TextFeaturizingEstimator.Options() {
OutputTokensColumnName = "OutputTokens",
CaseMode = TextNormalizingEstimator.CaseMode.Lower,
KeepDiacritics = false,
KeepPunctuations = false,
KeepNumbers = false,
Norm = TextFeaturizingEstimator.NormFunction.L2,
StopWordsRemoverOptions = new StopWordsRemovingEstimator.Options() {
Language = TextFeaturizingEstimator.Language.Dutch,
},
WordFeatureExtractor = new WordBagEstimator.Options() {
NgramLength = 4,
SkipLength = 1,
UseAllLengths = true,
MaximumNgramsCount = new int[] { 20, 10, 10, 10 },
Weighting = NgramExtractingEstimator.WeightingCriteria.TfIdf,
},
CharFeatureExtractor = null,
};
var textPipeline = mlContext.Transforms.Text
.FeaturizeText("Features", options, "Text");
var textTransformer = textPipeline.Fit(dataview);
var predictionEngine = mlContext.Model.CreatePredictionEngine<TextData, TransformedTextData>(textTransformer);
foreach (var article in articles)
{
var prediction = predictionEngine.Predict(article);
Console.WriteLine($"Article: {article.Text.Substring(0, 30)}...");
Console.WriteLine($"Number of Features: {prediction.Features.Length}");
Console.WriteLine($"Features: {string.Join(",", prediction.Features.Take(50).Select(f => f.ToString("0.00")))}\n");
}