Example usage for org.apache.lucene.index IndexWriter commit

List of usage examples for org.apache.lucene.index IndexWriter commit

  1. HOME
  2. Java
  3. org.apache.lucene
  4. org.apache.lucene.index.*
  5. IndexWriter
  6. commit

Introduction

In this page you can find the example usage for org.apache.lucene.index IndexWriter commit.

Prototype

@Override
public final long commit() throws IOException

Source Link

Document

Commits all pending changes (added and deleted documents, segment merges, added indexes, etc.) to the index, and syncs all referenced index files, such that a reader will see the changes and the index updates will survive an OS or machine crash or power loss.

Usage

From source file:org.elasticsearch.deps.lucene.SimpleLuceneTests.java

License:Apache License

@Test
public void testAddDocAfterPrepareCommit() throws Exception {
    Directory dir = new RAMDirectory();
    IndexWriter indexWriter = new IndexWriter(dir,
            new IndexWriterConfig(Lucene.VERSION, Lucene.STANDARD_ANALYZER));
    Document document = new Document();
    document.add(new TextField("_id", "1", Field.Store.YES));
    indexWriter.addDocument(document);//w  ww .  j  av  a 2 s  . c  o  m
    DirectoryReader reader = DirectoryReader.open(indexWriter, true);
    assertThat(reader.numDocs(), equalTo(1));

    indexWriter.prepareCommit();
    // Returns null b/c no changes.
    assertThat(DirectoryReader.openIfChanged(reader), equalTo(null));

    document = new Document();
    document.add(new TextField("_id", "2", Field.Store.YES));
    indexWriter.addDocument(document);
    indexWriter.commit();
    reader = DirectoryReader.openIfChanged(reader);
    assertThat(reader.numDocs(), equalTo(2));
}

From source file:org.elasticsearch.index.cache.bitset.BitSetFilterCacheTest.java

License:Apache License

@Test
public void testInvalidateEntries() throws Exception {
    IndexWriter writer = new IndexWriter(new RAMDirectory(),
            new IndexWriterConfig(new StandardAnalyzer()).setMergePolicy(new LogByteSizeMergePolicy()));
    Document document = new Document();
    document.add(new StringField("field", "value", Field.Store.NO));
    writer.addDocument(document);/*from   www  .  j a  v  a  2 s.c o  m*/
    writer.commit();

    document = new Document();
    document.add(new StringField("field", "value", Field.Store.NO));
    writer.addDocument(document);
    writer.commit();

    document = new Document();
    document.add(new StringField("field", "value", Field.Store.NO));
    writer.addDocument(document);
    writer.commit();

    IndexReader reader = DirectoryReader.open(writer, false);
    IndexSearcher searcher = new IndexSearcher(reader);

    BitsetFilterCache cache = new BitsetFilterCache(new Index("test"), ImmutableSettings.EMPTY);
    BitDocIdSetFilter filter = cache
            .getBitDocIdSetFilter(Queries.wrap(new TermQuery(new Term("field", "value"))));
    TopDocs docs = searcher.search(new ConstantScoreQuery(filter), 1);
    assertThat(docs.totalHits, equalTo(3));

    // now cached
    docs = searcher.search(new ConstantScoreQuery(filter), 1);
    assertThat(docs.totalHits, equalTo(3));
    // There are 3 segments
    assertThat(cache.getLoadedFilters().size(), equalTo(3l));

    writer.forceMerge(1);
    reader.close();
    reader = DirectoryReader.open(writer, false);
    searcher = new IndexSearcher(reader);

    docs = searcher.search(new ConstantScoreQuery(filter), 1);
    assertThat(docs.totalHits, equalTo(3));

    // now cached
    docs = searcher.search(new ConstantScoreQuery(filter), 1);
    assertThat(docs.totalHits, equalTo(3));
    // Only one segment now, so the size must be 1
    assertThat(cache.getLoadedFilters().size(), equalTo(1l));

    reader.close();
    writer.close();
    // There is no reference from readers and writer to any segment in the test index, so the size in the fbs cache must be 0
    assertThat(cache.getLoadedFilters().size(), equalTo(0l));
}

From source file:org.elasticsearch.index.cache.bitset.BitSetFilterCacheTests.java

License:Apache License

@Test
public void testInvalidateEntries() throws Exception {
    IndexWriter writer = new IndexWriter(new RAMDirectory(),
            new IndexWriterConfig(new StandardAnalyzer()).setMergePolicy(new LogByteSizeMergePolicy()));
    Document document = new Document();
    document.add(new StringField("field", "value", Field.Store.NO));
    writer.addDocument(document);//from  w  w w  .  ja v a2s  .  c o  m
    writer.commit();

    document = new Document();
    document.add(new StringField("field", "value", Field.Store.NO));
    writer.addDocument(document);
    writer.commit();

    document = new Document();
    document.add(new StringField("field", "value", Field.Store.NO));
    writer.addDocument(document);
    writer.commit();

    DirectoryReader reader = DirectoryReader.open(writer, false);
    reader = ElasticsearchDirectoryReader.wrap(reader, new ShardId(new Index("test"), 0));
    IndexSearcher searcher = new IndexSearcher(reader);

    BitsetFilterCache cache = new BitsetFilterCache(new Index("test"), Settings.EMPTY);
    BitSetProducer filter = cache.getBitSetProducer(new TermQuery(new Term("field", "value")));
    assertThat(matchCount(filter, reader), equalTo(3));

    // now cached
    assertThat(matchCount(filter, reader), equalTo(3));
    // There are 3 segments
    assertThat(cache.getLoadedFilters().size(), equalTo(3l));

    writer.forceMerge(1);
    reader.close();
    reader = DirectoryReader.open(writer, false);
    reader = ElasticsearchDirectoryReader.wrap(reader, new ShardId(new Index("test"), 0));
    searcher = new IndexSearcher(reader);

    assertThat(matchCount(filter, reader), equalTo(3));

    // now cached
    assertThat(matchCount(filter, reader), equalTo(3));
    // Only one segment now, so the size must be 1
    assertThat(cache.getLoadedFilters().size(), equalTo(1l));

    reader.close();
    writer.close();
    // There is no reference from readers and writer to any segment in the test index, so the size in the fbs cache must be 0
    assertThat(cache.getLoadedFilters().size(), equalTo(0l));
}

From source file:org.elasticsearch.index.cache.bitset.BitSetFilterCacheTests.java

License:Apache License

public void testListener() throws IOException {
    IndexWriter writer = new IndexWriter(new RAMDirectory(),
            new IndexWriterConfig(new StandardAnalyzer()).setMergePolicy(new LogByteSizeMergePolicy()));
    Document document = new Document();
    document.add(new StringField("field", "value", Field.Store.NO));
    writer.addDocument(document);/*from  ww w.  j  ava2 s.  co  m*/
    writer.commit();
    final DirectoryReader writerReader = DirectoryReader.open(writer, false);
    final IndexReader reader = ElasticsearchDirectoryReader.wrap(writerReader, new ShardId("test", 0));

    final AtomicLong stats = new AtomicLong();
    final AtomicInteger onCacheCalls = new AtomicInteger();
    final AtomicInteger onRemoveCalls = new AtomicInteger();

    final BitsetFilterCache cache = new BitsetFilterCache(new Index("test"), Settings.EMPTY);
    cache.setListener(new BitsetFilterCache.Listener() {
        @Override
        public void onCache(ShardId shardId, Accountable accountable) {
            onCacheCalls.incrementAndGet();
            stats.addAndGet(accountable.ramBytesUsed());
            if (writerReader != reader) {
                assertNotNull(shardId);
                assertEquals("test", shardId.index().name());
                assertEquals(0, shardId.id());
            } else {
                assertNull(shardId);
            }
        }

        @Override
        public void onRemoval(ShardId shardId, Accountable accountable) {
            onRemoveCalls.incrementAndGet();
            stats.addAndGet(-accountable.ramBytesUsed());
            if (writerReader != reader) {
                assertNotNull(shardId);
                assertEquals("test", shardId.index().name());
                assertEquals(0, shardId.id());
            } else {
                assertNull(shardId);
            }
        }
    });
    BitSetProducer filter = cache.getBitSetProducer(new TermQuery(new Term("field", "value")));
    assertThat(matchCount(filter, reader), equalTo(1));
    assertTrue(stats.get() > 0);
    assertEquals(1, onCacheCalls.get());
    assertEquals(0, onRemoveCalls.get());
    IOUtils.close(reader, writer);
    assertEquals(1, onRemoveCalls.get());
    assertEquals(0, stats.get());
}

From source file:org.elasticsearch.index.cache.fixedbitset.FixedBitSetFilterCacheTest.java

License:Apache License

@Test
public void testInvalidateEntries() throws Exception {
    IndexWriter writer = new IndexWriter(new RAMDirectory(),
            new IndexWriterConfig(Lucene.VERSION, new StandardAnalyzer(Lucene.VERSION))
                    .setMergePolicy(new LogByteSizeMergePolicy()));
    Document document = new Document();
    document.add(new StringField("field", "value", Field.Store.NO));
    writer.addDocument(document);//from   w  w w.j a  v a 2s .  c  om
    writer.commit();

    document = new Document();
    document.add(new StringField("field", "value", Field.Store.NO));
    writer.addDocument(document);
    writer.commit();

    document = new Document();
    document.add(new StringField("field", "value", Field.Store.NO));
    writer.addDocument(document);
    writer.commit();

    IndexReader reader = DirectoryReader.open(writer, false);
    IndexSearcher searcher = new IndexSearcher(reader);

    FixedBitSetFilterCache cache = new FixedBitSetFilterCache(new Index("test"), ImmutableSettings.EMPTY);
    FixedBitSetFilter filter = cache.getFixedBitSetFilter(new TermFilter(new Term("field", "value")));
    TopDocs docs = searcher.search(new XConstantScoreQuery(filter), 1);
    assertThat(docs.totalHits, equalTo(3));

    // now cached
    docs = searcher.search(new XConstantScoreQuery(filter), 1);
    assertThat(docs.totalHits, equalTo(3));
    // There are 3 segments
    assertThat(cache.getLoadedFilters().size(), equalTo(3l));

    writer.forceMerge(1);
    reader.close();
    reader = DirectoryReader.open(writer, false);
    searcher = new IndexSearcher(reader);

    docs = searcher.search(new XConstantScoreQuery(filter), 1);
    assertThat(docs.totalHits, equalTo(3));

    // now cached
    docs = searcher.search(new XConstantScoreQuery(filter), 1);
    assertThat(docs.totalHits, equalTo(3));
    // Only one segment now, so the size must be 1
    assertThat(cache.getLoadedFilters().size(), equalTo(1l));

    reader.close();
    writer.close();
    // There is no reference from readers and writer to any segment in the test index, so the size in the fbs cache must be 0
    assertThat(cache.getLoadedFilters().size(), equalTo(0l));
}

From source file:org.elasticsearch.index.cache.id.SimpleIdCacheTests.java

License:Apache License

@Test
public void testDeletedDocuments() throws Exception {
    SimpleIdCache idCache = createSimpleIdCache(Tuple.tuple("child", "parent"));
    IndexWriter writer = createIndexWriter();
    // Begins with parent, ends with child docs
    final Document parent = doc("parent", "1");
    writer.addDocument(parent);//from  ww  w  . j  a v a  2 s  .  com
    writer.addDocument(childDoc("child", "1", "parent", "1"));
    writer.addDocument(childDoc("child", "2", "parent", "1"));
    writer.addDocument(childDoc("child", "3", "parent", "1"));
    writer.commit();

    final String parentUid = parent.get("_uid");
    assert parentUid != null;
    writer.deleteDocuments(new Term("_uid", parentUid));

    writer.close();
    DirectoryReader topLevelReader = DirectoryReader.open(writer.getDirectory());
    List<AtomicReaderContext> leaves = topLevelReader.getContext().leaves();
    idCache.refresh(leaves);

    assertThat(leaves.size(), equalTo(1));
    IdReaderCache readerCache = idCache.reader(leaves.get(0).reader());
    IdReaderTypeCache typeCache = readerCache.type("parent");
    assertThat(typeCache.idByDoc(0).toUtf8(), equalTo("1"));
}

From source file:org.elasticsearch.index.cache.id.SimpleIdCacheTests.java

License:Apache License

@Test
public void testRefresh() throws Exception {
    SimpleIdCache idCache = createSimpleIdCache(Tuple.tuple("child", "parent"));
    IndexWriter writer = createIndexWriter();
    // Begins with parent, ends with child docs
    writer.addDocument(doc("parent", "1"));
    writer.addDocument(childDoc("child", "1", "parent", "1"));
    writer.addDocument(childDoc("child", "2", "parent", "1"));
    writer.addDocument(childDoc("child", "3", "parent", "1"));
    writer.addDocument(childDoc("child", "4", "parent", "1"));
    writer.commit();

    // Begins with child, ends with parent docs
    writer.addDocument(childDoc("child", "5", "parent", "2"));
    writer.addDocument(doc("parent", "2"));
    writer.addDocument(doc("parent", "3"));
    writer.addDocument(doc("parent", "4"));
    writer.addDocument(doc("parent", "5"));
    writer.commit();// w ww  .  j av  a 2 s. c  o  m

    // Begins with parent, child docs in the middle and ends with parent doc
    writer.addDocument(doc("parent", "6"));
    writer.addDocument(childDoc("child", "6", "parent", "6"));
    writer.addDocument(childDoc("child", "7", "parent", "6"));
    writer.addDocument(childDoc("child", "8", "parent", "5"));
    writer.addDocument(childDoc("child", "9", "parent", "4"));
    writer.addDocument(doc("parent", "7"));
    writer.commit();

    // Garbage segment
    writer.addDocument(doc("zzz", "1"));
    writer.addDocument(doc("xxx", "2"));
    writer.addDocument(doc("aaa", "3"));
    writer.addDocument(doc("ccc", "4"));
    writer.addDocument(doc("parent", "8"));
    writer.commit();

    writer.close();
    DirectoryReader topLevelReader = DirectoryReader.open(writer.getDirectory());
    List<AtomicReaderContext> leaves = topLevelReader.getContext().leaves();
    idCache.refresh(leaves);

    // Verify simple id cache for segment 1
    IdReaderCache readerCache = idCache.reader(leaves.get(0).reader());
    assertThat(readerCache.type("child"), nullValue());
    IdReaderTypeCache typeCache = readerCache.type("parent");
    assertThat(typeCache.idByDoc(0).toUtf8(), equalTo("1"));
    assertThat(typeCache.idByDoc(1), nullValue());
    assertThat(typeCache.idByDoc(2), nullValue());
    assertThat(typeCache.idByDoc(3), nullValue());
    assertThat(typeCache.idByDoc(4), nullValue());

    assertThat(typeCache.parentIdByDoc(0), nullValue());
    assertThat(typeCache.parentIdByDoc(1).toUtf8(), equalTo("1"));
    assertThat(typeCache.parentIdByDoc(2).toUtf8(), equalTo("1"));
    assertThat(typeCache.parentIdByDoc(3).toUtf8(), equalTo("1"));
    assertThat(typeCache.parentIdByDoc(4).toUtf8(), equalTo("1"));

    assertThat(typeCache.docById(new HashedBytesArray(Strings.toUTF8Bytes("1"))), equalTo(0));
    assertThat(typeCache.docById(new HashedBytesArray(Strings.toUTF8Bytes("2"))), equalTo(-1));
    assertThat(typeCache.docById(new HashedBytesArray(Strings.toUTF8Bytes("3"))), equalTo(-1));
    assertThat(typeCache.docById(new HashedBytesArray(Strings.toUTF8Bytes("4"))), equalTo(-1));

    // Verify simple id cache for segment 2
    readerCache = idCache.reader(leaves.get(1).reader());
    assertThat(readerCache.type("child"), nullValue());
    typeCache = readerCache.type("parent");
    assertThat(typeCache.idByDoc(0), nullValue());
    assertThat(typeCache.idByDoc(1).toUtf8(), equalTo("2"));
    assertThat(typeCache.idByDoc(2).toUtf8(), equalTo("3"));
    assertThat(typeCache.idByDoc(3).toUtf8(), equalTo("4"));
    assertThat(typeCache.idByDoc(4).toUtf8(), equalTo("5"));

    assertThat(typeCache.parentIdByDoc(0).toUtf8(), equalTo("2"));
    assertThat(typeCache.parentIdByDoc(1), nullValue());
    assertThat(typeCache.parentIdByDoc(2), nullValue());
    assertThat(typeCache.parentIdByDoc(3), nullValue());
    assertThat(typeCache.parentIdByDoc(4), nullValue());

    assertThat(typeCache.docById(new HashedBytesArray(Strings.toUTF8Bytes("2"))), equalTo(1));
    assertThat(typeCache.docById(new HashedBytesArray(Strings.toUTF8Bytes("3"))), equalTo(2));
    assertThat(typeCache.docById(new HashedBytesArray(Strings.toUTF8Bytes("4"))), equalTo(3));
    assertThat(typeCache.docById(new HashedBytesArray(Strings.toUTF8Bytes("5"))), equalTo(4));

    // Verify simple id cache for segment 3
    readerCache = idCache.reader(leaves.get(2).reader());
    assertThat(readerCache.type("child"), nullValue());
    typeCache = readerCache.type("parent");
    assertThat(typeCache.idByDoc(0).toUtf8(), equalTo("6"));
    assertThat(typeCache.idByDoc(1), nullValue());
    assertThat(typeCache.idByDoc(2), nullValue());
    assertThat(typeCache.idByDoc(3), nullValue());
    assertThat(typeCache.idByDoc(4), nullValue());
    assertThat(typeCache.idByDoc(5).toUtf8(), equalTo("7"));

    assertThat(typeCache.parentIdByDoc(0), nullValue());
    assertThat(typeCache.parentIdByDoc(1).toUtf8(), equalTo("6"));
    assertThat(typeCache.parentIdByDoc(2).toUtf8(), equalTo("6"));
    assertThat(typeCache.parentIdByDoc(3).toUtf8(), equalTo("5"));
    assertThat(typeCache.parentIdByDoc(4).toUtf8(), equalTo("4"));
    assertThat(typeCache.parentIdByDoc(5), nullValue());

    assertThat(typeCache.docById(new HashedBytesArray(Strings.toUTF8Bytes("6"))), equalTo(0));
    assertThat(typeCache.docById(new HashedBytesArray(Strings.toUTF8Bytes("7"))), equalTo(5));

    // Verify simple id cache for segment 4
    readerCache = idCache.reader(leaves.get(3).reader());
    assertThat(readerCache.type("child"), nullValue());
    typeCache = readerCache.type("parent");
    assertThat(typeCache.idByDoc(0), nullValue());
    assertThat(typeCache.idByDoc(1), nullValue());
    assertThat(typeCache.idByDoc(2), nullValue());
    assertThat(typeCache.idByDoc(3), nullValue());
    assertThat(typeCache.idByDoc(4).toUtf8(), equalTo("8"));

    assertThat(typeCache.parentIdByDoc(0), nullValue());
    assertThat(typeCache.parentIdByDoc(1), nullValue());
    assertThat(typeCache.parentIdByDoc(2), nullValue());
    assertThat(typeCache.parentIdByDoc(3), nullValue());
    assertThat(typeCache.parentIdByDoc(4), nullValue());

    assertThat(typeCache.docById(new HashedBytesArray(Strings.toUTF8Bytes("8"))), equalTo(4));
}

From source file:org.elasticsearch.index.cache.id.SimpleIdCacheTests.java

License:Apache License

@Test(expected = AssertionError.class)
public void testRefresh_tripAssert() throws Exception {
    SimpleIdCache idCache = createSimpleIdCache(Tuple.tuple("child", "parent"));
    IndexWriter writer = createIndexWriter();
    // Begins with parent, ends with child docs
    writer.addDocument(doc("parent", "1"));
    writer.addDocument(childDoc("child", "1", "parent", "1"));
    writer.addDocument(childDoc("child", "2", "parent", "1"));
    writer.addDocument(childDoc("child", "3", "parent", "1"));
    writer.addDocument(childDoc("child", "4", "parent", "1"));
    // Doc like this should never end up in the index, just wanna trip an assert here!
    Document document = new Document();
    document.add(new StringField(UidFieldMapper.NAME, "parent", Field.Store.NO));
    writer.addDocument(document);// w ww.  ja  va  2s .c o  m
    writer.commit();

    writer.close();
    DirectoryReader topLevelReader = DirectoryReader.open(writer.getDirectory());
    List<AtomicReaderContext> leaves = topLevelReader.getContext().leaves();
    idCache.refresh(leaves);
}

From source file:org.elasticsearch.index.codec.postingformat.DefaultPostingsFormatTests.java

License:Apache License

@Test
public void testUseDefault() throws IOException {

    Codec codec = new TestCodec();
    Directory d = new RAMDirectory();
    IndexWriterConfig config = new IndexWriterConfig(Lucene.VERSION, new WhitespaceAnalyzer(Lucene.VERSION));
    config.setCodec(codec);//from w  w w  .j a va2s. co m
    IndexWriter writer = new IndexWriter(d, config);
    writer.addDocument(Arrays.asList(new TextField("foo", "bar", Store.YES),
            new TextField(UidFieldMapper.NAME, "1234", Store.YES)));
    writer.commit();
    DirectoryReader reader = DirectoryReader.open(writer, false);
    List<AtomicReaderContext> leaves = reader.leaves();
    assertThat(leaves.size(), equalTo(1));
    AtomicReader ar = leaves.get(0).reader();
    Terms terms = ar.terms("foo");
    Terms uidTerms = ar.terms(UidFieldMapper.NAME);

    assertThat(terms.size(), equalTo(1l));
    assertThat(terms, not(instanceOf(BloomFilterPostingsFormat.BloomFilteredTerms.class)));
    assertThat(uidTerms, instanceOf(BloomFilterPostingsFormat.BloomFilteredTerms.class));

    reader.close();
    writer.close();
    d.close();
}

From source file:org.elasticsearch.index.codec.postingformat.DefaultPostingsFormatTests.java

License:Apache License

@Test
public void testNoUIDField() throws IOException {

    Codec codec = new TestCodec();
    Directory d = new RAMDirectory();
    IndexWriterConfig config = new IndexWriterConfig(Lucene.VERSION, new WhitespaceAnalyzer(Lucene.VERSION));
    config.setCodec(codec);//from  w ww .j  a  va  2  s.c  om
    IndexWriter writer = new IndexWriter(d, config);
    for (int i = 0; i < 100; i++) {
        writer.addDocument(Arrays.asList(new TextField("foo", "foo bar foo bar", Store.YES),
                new TextField("some_other_field", "1234", Store.YES)));
    }
    writer.forceMerge(1);
    writer.commit();

    DirectoryReader reader = DirectoryReader.open(writer, false);
    List<AtomicReaderContext> leaves = reader.leaves();
    assertThat(leaves.size(), equalTo(1));
    AtomicReader ar = leaves.get(0).reader();
    Terms terms = ar.terms("foo");
    Terms some_other_field = ar.terms("some_other_field");

    assertThat(terms.size(), equalTo(2l));
    assertThat(terms, not(instanceOf(BloomFilterPostingsFormat.BloomFilteredTerms.class)));
    assertThat(some_other_field, not(instanceOf(BloomFilterPostingsFormat.BloomFilteredTerms.class)));
    TermsEnum iterator = terms.iterator(null);
    Set<String> expected = new HashSet<String>();
    expected.add("foo");
    expected.add("bar");
    while (iterator.next() != null) {
        expected.remove(iterator.term().utf8ToString());
    }
    assertThat(expected.size(), equalTo(0));
    reader.close();
    writer.close();
    d.close();
}