1

在我的代码示例中，我在 lucene 索引中创建了三个文档。其中两个不存储字段 LASTNAME，但存储了 termvector，一个没有存储。使用 LUKE，我可以遍历该字段（LASTNAME）中的所有术语。在我的代码示例中，遍历 TermFreqVectors，它适用于存储有 TermVectors 的文档。

我怎样才能获得所有这些非存储条款？卢克是怎么做到的？

我最初的问题是，我想用另一个字段扩展一个包含近 100 个字段的大索引（60GB），而无需从头开始重新创建索引，因为使用我们的 db-setup 需要几天的 40 个并行计算服务器。从索引中读取所有数据并将这个新字段添加到所有存储的文档中非常快。

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.TermFreqVector;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.NIOFSDirectory;
import org.apache.lucene.util.LuceneTestCase;

import java.io.File;
import java.io.IOException;
import java.util.Arrays;


public class TestDocTerms extends LuceneTestCase {

public void testDocTerms() throws IOException, ParseException {
    Analyzer analyzer = new MockAnalyzer(random);

    String fieldF = "FIRSTNAME";
    String fieldL = "LASTNAME";

    // To store an index on disk, use this instead:
    Directory directory = NIOFSDirectory.open(new File("/tmp/_index_tester/"));
    RandomIndexWriter iwriter = new RandomIndexWriter(random, directory, analyzer);
    iwriter.w.setInfoStream(VERBOSE ? System.out : null);
    Document doc = new Document();
    doc.add(newField(fieldF, "Alex", Field.Store.YES, Field.Index.ANALYZED));
    doc.add(newField(fieldL, "Miller", Field.Store.NO,Field.Index.ANALYZED,Field.TermVector.YES));
    iwriter.addDocument(doc);
    doc = new Document();
    doc.add(newField(fieldF, "Chris", Field.Store.YES, Field.Index.ANALYZED));
    doc.add(newField(fieldL, "Smith", Field.Store.NO, Field.Index.ANALYZED));
    iwriter.addDocument(doc);
    doc = new Document();
    doc.add(newField(fieldF, "Alex", Field.Store.YES, Field.Index.ANALYZED));
    doc.add(newField(fieldL, "Beatle", Field.Store.NO, Field.Index.ANALYZED,Field.TermVector.YES));
    iwriter.addDocument(doc);
    iwriter.close();

    // Now search the index:
    IndexSearcher isearcher = new IndexSearcher(directory, true); // read-only=true
    QueryParser parser = new QueryParser(TEST_VERSION_CURRENT, fieldF, analyzer);
    Query query = parser.parse(fieldF + ":" + "Alex");
    TopDocs hits = isearcher.search(query, null, 2);
    assertEquals(2, hits.totalHits);
    // Iterate through the results:
    for (int i = 0; i < hits.scoreDocs.length; i++) {
        Document hitDoc = isearcher.doc(hits.scoreDocs[i].doc);
        assertEquals("Alex", hitDoc.get(fieldF));
        System.out.println("query for:" +query.toString()+ " with this results firstN:" + hitDoc.get(fieldF) + " and lastN:" + hitDoc.get(fieldL));
    }
    parser = new QueryParser(TEST_VERSION_CURRENT, fieldL, analyzer);
    query = parser.parse(fieldL + ":" + "Miller");
    hits = isearcher.search(query, null, 2);
    assertEquals(1, hits.totalHits);
    // Iterate through the results:
    for (int i = 0; i < hits.scoreDocs.length; i++) {
        Document hitDoc = isearcher.doc(hits.scoreDocs[i].doc);
        assertEquals("Alex", hitDoc.get(fieldF));
        System.out.println("query for:" + query.toString() + " with this results firstN:" +hitDoc.get(fieldF)+ " and lastN:" +hitDoc.get(fieldL));
    }
    isearcher.close();

    // examine terms
    IndexReader ireader = IndexReader.open(directory, true); // read-only=true
    int numDocs = ireader.numDocs();

    for (int i = 0; i < numDocs; i++) {
        doc = ireader.document(i);
        System.out.println("docNum:" + i + " with:" + doc.toString());
        TermFreqVector t = ireader.getTermFreqVector(i, fieldL);
        if (t != null){
            System.out.println("Field:" + fieldL + " contains terms:" + t.toString());
        }
        TermFreqVector[] termFreqVectors = ireader.getTermFreqVectors(i);
        if (termFreqVectors != null){
            for (TermFreqVector tfv : termFreqVectors){
                String[] terms = tfv.getTerms();
                String field = tfv.getField();
                System.out.println("Field:" +field+ " contains terms:" + Arrays.toString(terms));
            }
        }
    }
    ireader.close();
}


}
4

1 回答 1

2

重建未存储的文档必然是尽最大努力。您通常无法撤消分析器对值所做的更改。

当 TermVectors 不可用时，Luke 会枚举与该字段关联的术语。这可能不尊重术语的顺序或任何格式。不过，这可能既不存在也不存在。我不知道你的newField方法到底是做什么的，但我怀疑它的默认值是 not Field.TermVector.NO。

如果您想了解更多实现细节，我会获取 Luke 源代码，并阅读org.getopt.luke.DocReconstructor

于 2013-07-24T15:41:28.063 回答