Example usage for org.apache.lucene.index IndexWriter commit

List of usage examples for org.apache.lucene.index IndexWriter commit

  1. HOME
  2. Java
  3. org.apache.lucene
  4. org.apache.lucene.index.*
  5. IndexWriter
  6. commit

Introduction

In this page you can find the example usage for org.apache.lucene.index IndexWriter commit.

Prototype

@Override
public final long commit() throws IOException

Source Link

Document

Commits all pending changes (added and deleted documents, segment merges, added indexes, etc.) to the index, and syncs all referenced index files, such that a reader will see the changes and the index updates will survive an OS or machine crash or power loss.

Usage

From source file:io.crate.execution.engine.collect.collectors.LuceneOrderedDocCollectorTest.java

License:Apache License

private Directory createLuceneIndex() throws IOException {
    Path tmpDir = newTempDir();//from   ww w  . j  a v  a  2 s  .c  om
    Directory index = FSDirectory.open(tmpDir);
    StandardAnalyzer analyzer = new StandardAnalyzer();
    IndexWriterConfig cfg = new IndexWriterConfig(analyzer);
    IndexWriter w = new IndexWriter(index, cfg);
    for (Long i = 0L; i < 4; i++) {
        if (i < 2) {
            addDocToLucene(w, i + 1);
        } else {
            addDocToLucene(w, null);
        }
        w.commit();
    }
    w.close();
    return index;
}

From source file:io.crate.execution.engine.collect.collectors.LuceneOrderedDocCollectorTest.java

License:Apache License

@Test
public void testSearchMoreAppliesMinScoreFilter() throws Exception {
    IndexWriter w = new IndexWriter(new RAMDirectory(), new IndexWriterConfig(new KeywordAnalyzer()));
    KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType();
    fieldType.setName("x");
    fieldType.freeze();/*from   www . ja  v a  2  s . com*/

    for (int i = 0; i < 3; i++) {
        addDoc(w, fieldType, "Arthur");
    }
    addDoc(w, fieldType, "Arthurr"); // not "Arthur" to lower score
    w.commit();
    IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(w, true, true));

    List<LuceneCollectorExpression<?>> columnReferences = Collections
            .singletonList(new ScoreCollectorExpression());
    Query query = fieldType.fuzzyQuery("Arthur", Fuzziness.AUTO, 2, 3, true);
    LuceneOrderedDocCollector collector;

    // without minScore filter we get 2 and 2 docs - this is not necessary for the test but is here
    // to make sure the "FuzzyQuery" matches the right documents
    collector = collectorWithMinScore(searcher, columnReferences, query, null);
    assertThat(Iterables.size(collector.collect()), is(2));
    assertThat(Iterables.size(collector.collect()), is(2));

    collector = collectorWithMinScore(searcher, columnReferences, query, 0.30f);
    int count = 0;
    // initialSearch -> 2 rows
    for (Row row : collector.collect()) {
        assertThat((float) row.get(0), Matchers.greaterThanOrEqualTo(0.30f));
        count++;
    }
    assertThat(count, is(2));

    count = 0;
    // searchMore -> 1 row is below minScore
    for (Row row : collector.collect()) {
        assertThat((float) row.get(0), Matchers.greaterThanOrEqualTo(0.30f));
        count++;
    }
    assertThat(count, is(1));
}

From source file:io.crate.execution.engine.collect.collectors.OrderedLuceneBatchIteratorBenchmark.java

License:Apache License

@Setup
public void createLuceneBatchIterator() throws Exception {
    IndexWriter iw = new IndexWriter(new RAMDirectory(), new IndexWriterConfig(new StandardAnalyzer()));
    dummyShardId = new ShardId("dummy", UUIDs.randomBase64UUID(), 1);
    columnName = "x";
    for (int i = 0; i < 10_000_000; i++) {
        Document doc = new Document();
        doc.add(new NumericDocValuesField(columnName, i));
        iw.addDocument(doc);/*from  w  ww.  j a v a  2 s.c  om*/
    }
    iw.commit();
    iw.forceMerge(1, true);
    indexSearcher = new IndexSearcher(DirectoryReader.open(iw, true, true));
    collectorContext = new CollectorContext(mappedFieldType -> null, new CollectorFieldsVisitor(0));
    reference = new Reference(
            new ReferenceIdent(new TableIdent(Schemas.DOC_SCHEMA_NAME, "dummyTable"), columnName),
            RowGranularity.DOC, DataTypes.INTEGER);
    orderBy = new OrderBy(Collections.singletonList(reference), reverseFlags, nullsFirst);
}

From source file:io.crate.execution.engine.collect.collectors.OrderedLuceneBatchIteratorFactoryTest.java

License:Apache License

@Before
public void prepareSearchers() throws Exception {
    IndexWriter iw1 = new IndexWriter(new RAMDirectory(), new IndexWriterConfig(new StandardAnalyzer()));
    IndexWriter iw2 = new IndexWriter(new RAMDirectory(), new IndexWriterConfig(new StandardAnalyzer()));

    expectedResult = LongStream.range(0, 20).mapToObj(i -> new Object[] { i }).collect(Collectors.toList());
    // expect descending order to differentiate between insert order
    expectedResult.sort(Comparator.comparingLong((Object[] o) -> ((long) o[0])).reversed());

    for (int i = 0; i < 20; i++) {
        Document doc = new Document();
        doc.add(new NumericDocValuesField(columnName, i));
        if (i % 2 == 0) {
            iw1.addDocument(doc);/*  w w w  .j  a  v  a 2 s.  c om*/
        } else {
            iw2.addDocument(doc);
        }
    }
    iw1.commit();
    iw2.commit();

    searcher1 = new IndexSearcher(DirectoryReader.open(iw1));
    searcher2 = new IndexSearcher(DirectoryReader.open(iw2));
    orderBy = new OrderBy(Collections.singletonList(reference), reverseFlags, nullsFirst);
}

From source file:io.crate.operation.collect.collectors.LuceneBatchIteratorBenchmark.java

License:Apache License

@Setup
public void createLuceneBatchIterator() throws Exception {
    IndexWriter iw = new IndexWriter(new RAMDirectory(), new IndexWriterConfig(new StandardAnalyzer()));
    String columnName = "x";
    for (int i = 0; i < 10_000_000; i++) {
        Document doc = new Document();
        doc.add(new NumericDocValuesField(columnName, i));
        iw.addDocument(doc);//from w  w w . j  av  a2s  .c  o  m
    }
    iw.commit();
    iw.forceMerge(1, true);
    indexSearcher = new IndexSearcher(DirectoryReader.open(iw));
    IntegerColumnReference columnReference = new IntegerColumnReference(columnName);
    columnRefs = Collections.singletonList(columnReference);

    collectorContext = new CollectorContext(mock(IndexFieldDataService.class), new CollectorFieldsVisitor(0));
}

From source file:io.crate.operation.collect.collectors.OrderedLuceneBatchIteratorBenchmark.java

License:Apache License

@Setup
public void createLuceneBatchIterator() throws Exception {
    IndexWriter iw = new IndexWriter(new RAMDirectory(), new IndexWriterConfig(new StandardAnalyzer()));
    dummyShardId = new ShardId("dummy", UUIDs.randomBase64UUID(), 1);
    columnName = "x";
    for (int i = 0; i < 10_000_000; i++) {
        Document doc = new Document();
        doc.add(new NumericDocValuesField(columnName, i));
        iw.addDocument(doc);/*from  w  w w.j av a 2s. c o  m*/
    }
    iw.commit();
    iw.forceMerge(1, true);
    indexSearcher = new IndexSearcher(DirectoryReader.open(iw, true, true));
    collectorContext = new CollectorContext(mock(IndexFieldDataService.class), new CollectorFieldsVisitor(0));
    fieldTypeLookup = column -> {
        NumberFieldMapper.NumberFieldType fieldType = new NumberFieldMapper.NumberFieldType(
                NumberFieldMapper.NumberType.INTEGER);
        fieldType.setName(column);
        return fieldType;
    };
    reference = new Reference(new ReferenceIdent(new TableIdent(null, "dummyTable"), columnName),
            RowGranularity.DOC, DataTypes.INTEGER);
    orderBy = new OrderBy(Collections.singletonList(reference), reverseFlags, nullsFirst);
}

From source file:io.crate.operation.collect.collectors.OrderedLuceneBatchIteratorFactoryTest.java

License:Apache License

@Before
public void prepareSearchers() throws Exception {
    IndexWriter iw1 = new IndexWriter(new RAMDirectory(), new IndexWriterConfig(new StandardAnalyzer()));
    IndexWriter iw2 = new IndexWriter(new RAMDirectory(), new IndexWriterConfig(new StandardAnalyzer()));

    expectedResult = LongStream.range(0, 20).mapToObj(i -> new Object[] { i }).collect(Collectors.toList());
    // expect descending order to differentiate between insert order
    expectedResult.sort(Comparator.comparingLong((Object[] o) -> ((long) o[0])).reversed());

    for (int i = 0; i < 20; i++) {
        Document doc = new Document();
        doc.add(new NumericDocValuesField(columnName, i));
        if (i % 2 == 0) {
            iw1.addDocument(doc);/* ww  w  . j a  va  2  s .co m*/
        } else {
            iw2.addDocument(doc);
        }
    }
    iw1.commit();
    iw2.commit();

    searcher1 = new IndexSearcher(DirectoryReader.open(iw1));
    searcher2 = new IndexSearcher(DirectoryReader.open(iw2));
    fieldTypeLookup = columnName -> {
        NumberFieldMapper.NumberFieldType fieldType = new NumberFieldMapper.NumberFieldType(
                NumberFieldMapper.NumberType.LONG);
        fieldType.setName(columnName);
        return fieldType;
    };
    orderBy = new OrderBy(Collections.singletonList(reference), reverseFlags, nullsFirst);
}

From source file:io.crate.operation.collect.LuceneDocCollectorTest.java

License:Apache License

private Directory createLuceneIndex() throws IOException {
    File tmpDir = temporaryFolder.newFolder();
    Directory index = FSDirectory.open(tmpDir);
    StandardAnalyzer analyzer = new StandardAnalyzer();
    IndexWriterConfig cfg = new IndexWriterConfig(Version.LATEST, analyzer);
    IndexWriter w = new IndexWriter(index, cfg);
    for (Long i = 0L; i < 4; i++) {
        if (i < 2) {
            addDocToLucene(w, i + 1);//from w w w  .j av a2  s .c  o  m
        } else {
            addDocToLucene(w, null);
        }
        w.commit();
    }
    w.close();
    return index;
}

From source file:io.datalayer.lucene.delete.LuceneDeleteTest.java

License:Apache License

@Test
public void testDelete() throws IOException {

    IndexWriter writer = AosIndexUtil.newIndexWithDocuments();

    Term term = new Term(ID, "1");
    Query query = new TermQuery(term);

    IndexReader reader = DirectoryReader.open(writer, true);
    IndexSearcher indexSearcher = new IndexSearcher(DirectoryReader.open(writer, true));

    TopDocs topDocs = indexSearcher.search(query, 1);
    LOGGER.info("" + topDocs.scoreDocs[0].doc);
    assertNotNull(reader.document(topDocs.scoreDocs[0].doc));

    LOGGER.info("Deleting documents containing " + term);
    writer.deleteDocuments(term);//from  w  ww  . j  a v  a 2  s .  c om
    //        writer.deleteDocuments(query);
    writer.commit();

    indexSearcher = new IndexSearcher(DirectoryReader.open(writer, true));
    topDocs = indexSearcher.search(query, 1);
    assertEquals(0, topDocs.scoreDocs.length);

    reader.close();
    writer.close();

}

From source file:io.datalayer.lucene.helper.AosIndexUtil.java

License:Apache License

public static IndexWriter newIndexWithDocuments() throws IOException {
    IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_46, getAnalyzer());
    IndexWriter writer = new IndexWriter(indexDirectory, config);
    for (int i = 0; i < id.length; i++) {
        Document document = new Document();
        document.add(new Field(ID, id[i], AosFieldType.INDEXED_STORED_TERMVECTORS));
        document.add(new Field(CITY, city[i], AosFieldType.INDEXED_STORED_TERMVECTORS));
        document.add(new Field(COUNTRY, unindexed[i], AosFieldType.INDEXEDNOT_STORED_TERMVECTORSNOT));
        document.add(new Field(CONTENT, unstored[i], AosFieldType.INDEXED_STOREDNOT_TERMVECTORS));
        writer.addDocument(document);//from w w w  . j  av  a 2 s . c  o m
    }
    writer.commit();
    return writer;
}