Example usage for org.apache.lucene.index IndexWriter commit

List of usage examples for org.apache.lucene.index IndexWriter commit

  1. HOME
  2. Java
  3. org.apache.lucene
  4. org.apache.lucene.index.*
  5. IndexWriter
  6. commit

Introduction

In this page you can find the example usage for org.apache.lucene.index IndexWriter commit.

Prototype

@Override
public final long commit() throws IOException

Source Link

Document

Commits all pending changes (added and deleted documents, segment merges, added indexes, etc.) to the index, and syncs all referenced index files, such that a reader will see the changes and the index updates will survive an OS or machine crash or power loss.

Usage

From source file:org.elasticsearch.common.lucene.LuceneTests.java

License:Apache License

public void testCleanIndex() throws IOException {
    MockDirectoryWrapper dir = newMockDirectory();
    dir.setEnableVirusScanner(false);//from   w w  w. j  a  v a2 s.c  om
    IndexWriterConfig iwc = newIndexWriterConfig();
    iwc.setIndexDeletionPolicy(NoDeletionPolicy.INSTANCE);
    iwc.setMergePolicy(NoMergePolicy.INSTANCE);
    iwc.setMaxBufferedDocs(2);
    IndexWriter writer = new IndexWriter(dir, iwc);
    Document doc = new Document();
    doc.add(new TextField("id", "1", random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
    writer.addDocument(doc);
    writer.commit();

    doc = new Document();
    doc.add(new TextField("id", "2", random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
    writer.addDocument(doc);

    doc = new Document();
    doc.add(new TextField("id", "3", random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
    writer.addDocument(doc);

    writer.commit();
    doc = new Document();
    doc.add(new TextField("id", "4", random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
    writer.addDocument(doc);

    writer.deleteDocuments(new Term("id", "2"));
    writer.commit();
    try (DirectoryReader open = DirectoryReader.open(writer, true)) {
        assertEquals(3, open.numDocs());
        assertEquals(1, open.numDeletedDocs());
        assertEquals(4, open.maxDoc());
    }
    writer.close();
    if (random().nextBoolean()) {
        for (String file : dir.listAll()) {
            if (file.startsWith("_1")) {
                // delete a random file
                dir.deleteFile(file);
                break;
            }
        }
    }
    Lucene.cleanLuceneIndex(dir);
    if (dir.listAll().length > 0) {
        for (String file : dir.listAll()) {
            if (file.startsWith("extra") == false) {
                assertEquals(file, "write.lock");
            }
        }
    }
    dir.close();
}

From source file:org.elasticsearch.common.lucene.LuceneTests.java

License:Apache License

public void testPruneUnreferencedFiles() throws IOException {
    MockDirectoryWrapper dir = newMockDirectory();
    dir.setEnableVirusScanner(false);/*  w w  w  .  java 2  s  . c o  m*/
    IndexWriterConfig iwc = newIndexWriterConfig();
    iwc.setIndexDeletionPolicy(NoDeletionPolicy.INSTANCE);
    iwc.setMergePolicy(NoMergePolicy.INSTANCE);
    iwc.setMaxBufferedDocs(2);
    IndexWriter writer = new IndexWriter(dir, iwc);
    Document doc = new Document();
    doc.add(new TextField("id", "1", random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
    writer.addDocument(doc);
    writer.commit();

    doc = new Document();
    doc.add(new TextField("id", "2", random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
    writer.addDocument(doc);

    doc = new Document();
    doc.add(new TextField("id", "3", random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
    writer.addDocument(doc);

    writer.commit();
    SegmentInfos segmentCommitInfos = Lucene.readSegmentInfos(dir);

    doc = new Document();
    doc.add(new TextField("id", "4", random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
    writer.addDocument(doc);

    writer.deleteDocuments(new Term("id", "2"));
    writer.commit();
    DirectoryReader open = DirectoryReader.open(writer, true);
    assertEquals(3, open.numDocs());
    assertEquals(1, open.numDeletedDocs());
    assertEquals(4, open.maxDoc());
    open.close();
    writer.close();
    SegmentInfos si = Lucene.pruneUnreferencedFiles(segmentCommitInfos.getSegmentsFileName(), dir);
    assertEquals(si.getSegmentsFileName(), segmentCommitInfos.getSegmentsFileName());
    open = DirectoryReader.open(dir);
    assertEquals(3, open.numDocs());
    assertEquals(0, open.numDeletedDocs());
    assertEquals(3, open.maxDoc());

    IndexSearcher s = new IndexSearcher(open);
    assertEquals(s.search(new TermQuery(new Term("id", "1")), 1).totalHits, 1);
    assertEquals(s.search(new TermQuery(new Term("id", "2")), 1).totalHits, 1);
    assertEquals(s.search(new TermQuery(new Term("id", "3")), 1).totalHits, 1);
    assertEquals(s.search(new TermQuery(new Term("id", "4")), 1).totalHits, 0);

    for (String file : dir.listAll()) {
        assertFalse("unexpected file: " + file, file.equals("segments_3") || file.startsWith("_2"));
    }
    open.close();
    dir.close();

}

From source file:org.elasticsearch.common.lucene.LuceneTests.java

License:Apache License

public void testFiles() throws IOException {
    MockDirectoryWrapper dir = newMockDirectory();
    dir.setEnableVirusScanner(false);//from   w  ww.  j av  a  2s  . com
    IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
    iwc.setMergePolicy(NoMergePolicy.INSTANCE);
    iwc.setMaxBufferedDocs(2);
    iwc.setUseCompoundFile(true);
    IndexWriter writer = new IndexWriter(dir, iwc);
    Document doc = new Document();
    doc.add(new TextField("id", "1", random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
    writer.addDocument(doc);
    writer.commit();
    Set<String> files = new HashSet<>();
    for (String f : Lucene.files(Lucene.readSegmentInfos(dir))) {
        files.add(f);
    }
    final boolean simpleTextCFS = files.contains("_0.scf");
    assertTrue(files.toString(), files.contains("segments_1"));
    if (simpleTextCFS) {
        assertFalse(files.toString(), files.contains("_0.cfs"));
        assertFalse(files.toString(), files.contains("_0.cfe"));
    } else {
        assertTrue(files.toString(), files.contains("_0.cfs"));
        assertTrue(files.toString(), files.contains("_0.cfe"));
    }
    assertTrue(files.toString(), files.contains("_0.si"));

    doc = new Document();
    doc.add(new TextField("id", "2", random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
    writer.addDocument(doc);

    doc = new Document();
    doc.add(new TextField("id", "3", random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
    writer.addDocument(doc);
    writer.commit();

    files.clear();
    for (String f : Lucene.files(Lucene.readSegmentInfos(dir))) {
        files.add(f);
    }
    assertFalse(files.toString(), files.contains("segments_1"));
    assertTrue(files.toString(), files.contains("segments_2"));
    if (simpleTextCFS) {
        assertFalse(files.toString(), files.contains("_0.cfs"));
        assertFalse(files.toString(), files.contains("_0.cfe"));
    } else {
        assertTrue(files.toString(), files.contains("_0.cfs"));
        assertTrue(files.toString(), files.contains("_0.cfe"));
    }
    assertTrue(files.toString(), files.contains("_0.si"));

    if (simpleTextCFS) {
        assertFalse(files.toString(), files.contains("_1.cfs"));
        assertFalse(files.toString(), files.contains("_1.cfe"));
    } else {
        assertTrue(files.toString(), files.contains("_1.cfs"));
        assertTrue(files.toString(), files.contains("_1.cfe"));
    }
    assertTrue(files.toString(), files.contains("_1.si"));
    writer.close();
    dir.close();
}

From source file:org.elasticsearch.common.lucene.LuceneTests.java

License:Apache License

public void testNumDocs() throws IOException {
    MockDirectoryWrapper dir = newMockDirectory();
    dir.setEnableVirusScanner(false);/*from w w  w  .  j  a v a 2s .  c  o m*/
    IndexWriterConfig iwc = newIndexWriterConfig();
    IndexWriter writer = new IndexWriter(dir, iwc);
    Document doc = new Document();
    doc.add(new TextField("id", "1", random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
    writer.addDocument(doc);
    writer.commit();
    SegmentInfos segmentCommitInfos = Lucene.readSegmentInfos(dir);
    assertEquals(1, Lucene.getNumDocs(segmentCommitInfos));

    doc = new Document();
    doc.add(new TextField("id", "2", random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
    writer.addDocument(doc);

    doc = new Document();
    doc.add(new TextField("id", "3", random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
    writer.addDocument(doc);
    segmentCommitInfos = Lucene.readSegmentInfos(dir);
    assertEquals(1, Lucene.getNumDocs(segmentCommitInfos));
    writer.commit();
    segmentCommitInfos = Lucene.readSegmentInfos(dir);
    assertEquals(3, Lucene.getNumDocs(segmentCommitInfos));
    writer.deleteDocuments(new Term("id", "2"));
    writer.commit();
    segmentCommitInfos = Lucene.readSegmentInfos(dir);
    assertEquals(2, Lucene.getNumDocs(segmentCommitInfos));

    int numDocsToIndex = randomIntBetween(10, 50);
    List<Term> deleteTerms = new ArrayList<>();
    for (int i = 0; i < numDocsToIndex; i++) {
        doc = new Document();
        doc.add(new TextField("id", "extra_" + i, random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
        deleteTerms.add(new Term("id", "extra_" + i));
        writer.addDocument(doc);
    }
    int numDocsToDelete = randomIntBetween(0, numDocsToIndex);
    Collections.shuffle(deleteTerms, random());
    for (int i = 0; i < numDocsToDelete; i++) {
        Term remove = deleteTerms.remove(0);
        writer.deleteDocuments(remove);
    }
    writer.commit();
    segmentCommitInfos = Lucene.readSegmentInfos(dir);
    assertEquals(2 + deleteTerms.size(), Lucene.getNumDocs(segmentCommitInfos));
    writer.close();
    dir.close();
}

From source file:org.elasticsearch.common.lucene.search.morelikethis.MoreLikeThisQueryTests.java

License:Apache License

@Test
public void testSimple() throws Exception {
    Directory dir = new RAMDirectory();
    IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER));
    indexWriter.commit();

    Document document = new Document();
    document.add(new TextField("_id", "1", Field.Store.YES));
    document.add(new TextField("text", "lucene", Field.Store.YES));
    indexWriter.addDocument(document);//from   w w  w.  j  a v  a2  s. co  m

    document = new Document();
    document.add(new TextField("_id", "2", Field.Store.YES));
    document.add(new TextField("text", "lucene release", Field.Store.YES));
    indexWriter.addDocument(document);

    IndexReader reader = DirectoryReader.open(indexWriter, true);
    IndexSearcher searcher = new IndexSearcher(reader);

    MoreLikeThisQuery mltQuery = new MoreLikeThisQuery("lucene", new String[] { "text" },
            Lucene.STANDARD_ANALYZER);
    mltQuery.setLikeText("lucene");
    mltQuery.setMinTermFrequency(1);
    mltQuery.setMinDocFreq(1);
    long count = searcher.count(mltQuery);
    assertThat(count, equalTo(2l));

    reader.close();
    indexWriter.close();
}

From source file:org.elasticsearch.common.lucene.search.MoreLikeThisQueryTests.java

License:Apache License

@Test
public void testSimple() throws Exception {
    Directory dir = new RAMDirectory();
    IndexWriter indexWriter = new IndexWriter(dir,
            new IndexWriterConfig(Lucene.VERSION, Lucene.STANDARD_ANALYZER));
    indexWriter.commit();

    Document document = new Document();
    document.add(new TextField("_id", "1", Field.Store.YES));
    document.add(new TextField("text", "lucene", Field.Store.YES));
    indexWriter.addDocument(document);// w  w w .java 2  s . c  o  m

    document = new Document();
    document.add(new TextField("_id", "2", Field.Store.YES));
    document.add(new TextField("text", "lucene release", Field.Store.YES));
    indexWriter.addDocument(document);

    IndexReader reader = DirectoryReader.open(indexWriter, true);
    IndexSearcher searcher = new IndexSearcher(reader);

    MoreLikeThisQuery mltQuery = new MoreLikeThisQuery("lucene", new String[] { "text" },
            Lucene.STANDARD_ANALYZER);
    mltQuery.setLikeText("lucene");
    mltQuery.setMinTermFrequency(1);
    mltQuery.setMinDocFreq(1);
    long count = Lucene.count(searcher, mltQuery);
    assertThat(count, equalTo(2l));

    reader.close();
    indexWriter.close();
}

From source file:org.elasticsearch.common.lucene.search.TermsFilterTests.java

License:Apache License

@Test
public void testTermFilter() throws Exception {
    String fieldName = "field1";
    Directory rd = new RAMDirectory();
    IndexWriter w = new IndexWriter(rd, new IndexWriterConfig(Lucene.VERSION, new KeywordAnalyzer()));
    for (int i = 0; i < 100; i++) {
        Document doc = new Document();
        int term = i * 10; //terms are units of 10;
        doc.add(new Field(fieldName, "" + term, StringField.TYPE_NOT_STORED));
        doc.add(new Field("all", "xxx", StringField.TYPE_NOT_STORED));
        w.addDocument(doc);/*from   www. ja  va 2s  .c  om*/
        if ((i % 40) == 0) {
            w.commit();
        }
    }
    AtomicReader reader = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(w, true));
    w.close();

    TermFilter tf = new TermFilter(new Term(fieldName, "19"));
    FixedBitSet bits = (FixedBitSet) tf.getDocIdSet(reader.getContext(), reader.getLiveDocs());
    assertThat(bits, nullValue());

    tf = new TermFilter(new Term(fieldName, "20"));
    DocIdSet result = tf.getDocIdSet(reader.getContext(), reader.getLiveDocs());
    bits = DocIdSets.toFixedBitSet(result.iterator(), reader.maxDoc());
    assertThat(bits.cardinality(), equalTo(1));

    tf = new TermFilter(new Term("all", "xxx"));
    result = tf.getDocIdSet(reader.getContext(), reader.getLiveDocs());
    bits = DocIdSets.toFixedBitSet(result.iterator(), reader.maxDoc());
    assertThat(bits.cardinality(), equalTo(100));

    reader.close();
    rd.close();
}

From source file:org.elasticsearch.common.lucene.search.TermsFilterTests.java

License:Apache License

@Test
public void testTermsFilter() throws Exception {
    String fieldName = "field1";
    Directory rd = new RAMDirectory();
    IndexWriter w = new IndexWriter(rd, new IndexWriterConfig(Lucene.VERSION, new KeywordAnalyzer()));
    for (int i = 0; i < 100; i++) {
        Document doc = new Document();
        int term = i * 10; //terms are units of 10;
        doc.add(new Field(fieldName, "" + term, StringField.TYPE_NOT_STORED));
        doc.add(new Field("all", "xxx", StringField.TYPE_NOT_STORED));
        w.addDocument(doc);/* w ww . j  a  va 2  s.c  om*/
        if ((i % 40) == 0) {
            w.commit();
        }
    }
    AtomicReader reader = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(w, true));
    w.close();

    TermsFilter tf = new TermsFilter(new Term[] { new Term(fieldName, "19") });
    FixedBitSet bits = (FixedBitSet) tf.getDocIdSet(reader.getContext(), reader.getLiveDocs());
    assertThat(bits, nullValue());

    tf = new TermsFilter(new Term[] { new Term(fieldName, "19"), new Term(fieldName, "20") });
    bits = (FixedBitSet) tf.getDocIdSet(reader.getContext(), reader.getLiveDocs());
    assertThat(bits.cardinality(), equalTo(1));

    tf = new TermsFilter(
            new Term[] { new Term(fieldName, "19"), new Term(fieldName, "20"), new Term(fieldName, "10") });
    bits = (FixedBitSet) tf.getDocIdSet(reader.getContext(), reader.getLiveDocs());
    assertThat(bits.cardinality(), equalTo(2));

    tf = new TermsFilter(new Term[] { new Term(fieldName, "19"), new Term(fieldName, "20"),
            new Term(fieldName, "10"), new Term(fieldName, "00") });
    bits = (FixedBitSet) tf.getDocIdSet(reader.getContext(), reader.getLiveDocs());
    assertThat(bits.cardinality(), equalTo(2));

    reader.close();
    rd.close();
}

From source file:org.elasticsearch.common.lucene.uid.VersionsTests.java

License:Apache License

@Test
public void testBackwardCompatibility() throws IOException {
    Directory dir = newDirectory();//from  ww w. java  2  s .co  m
    IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(Lucene.VERSION, Lucene.STANDARD_ANALYZER));

    DirectoryReader directoryReader = DirectoryReader.open(writer, true);
    MatcherAssert.assertThat(Versions.loadVersion(directoryReader, new Term(UidFieldMapper.NAME, "1")),
            equalTo(Versions.NOT_FOUND));

    Document doc = new Document();
    UidField uidAndVersion = new UidField("1", 1L);
    doc.add(uidAndVersion);
    writer.addDocument(doc);

    uidAndVersion.uid = "2";
    uidAndVersion.version = 2;
    writer.addDocument(doc);
    writer.commit();

    directoryReader = reopen(directoryReader);
    assertThat(Versions.loadVersion(directoryReader, new Term(UidFieldMapper.NAME, "1")), equalTo(1l));
    assertThat(Versions.loadVersion(directoryReader, new Term(UidFieldMapper.NAME, "2")), equalTo(2l));
    assertThat(Versions.loadVersion(directoryReader, new Term(UidFieldMapper.NAME, "3")),
            equalTo(Versions.NOT_FOUND));
    directoryReader.close();
    writer.close();
    dir.close();
}

From source file:org.elasticsearch.common.lucene.uid.VersionsTests.java

License:Apache License

@Test
public void testMergingOldIndices() throws Exception {
    final IndexWriterConfig iwConf = new IndexWriterConfig(Lucene.VERSION, new KeywordAnalyzer());
    iwConf.setMergePolicy(new IndexUpgraderMergePolicy(iwConf.getMergePolicy()));
    final Directory dir = newDirectory();
    final IndexWriter iw = new IndexWriter(dir, iwConf);

    // 1st segment, no _version
    Document document = new Document();
    // Add a dummy field (enough to trigger #3237)
    document.add(new StringField("a", "b", Store.NO));
    StringField uid = new StringField(UidFieldMapper.NAME, "1", Store.YES);
    document.add(uid);/*from   www .j  a v  a 2 s .c  om*/
    iw.addDocument(document);
    uid.setStringValue("2");
    iw.addDocument(document);
    iw.commit();

    // 2nd segment, old layout
    document = new Document();
    UidField uidAndVersion = new UidField("3", 3L);
    document.add(uidAndVersion);
    iw.addDocument(document);
    uidAndVersion.uid = "4";
    uidAndVersion.version = 4L;
    iw.addDocument(document);
    iw.commit();

    // 3rd segment new layout
    document = new Document();
    uid.setStringValue("5");
    Field version = new NumericDocValuesField(VersionFieldMapper.NAME, 5L);
    document.add(uid);
    document.add(version);
    iw.addDocument(document);
    uid.setStringValue("6");
    version.setLongValue(6L);
    iw.addDocument(document);
    iw.commit();

    final Map<String, Long> expectedVersions = ImmutableMap.<String, Long>builder().put("1", 0L).put("2", 0L)
            .put("3", 0L).put("4", 4L).put("5", 5L).put("6", 6L).build();

    // Force merge and check versions
    iw.forceMerge(1);
    final AtomicReader ir = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(iw.getDirectory()));
    final NumericDocValues versions = ir.getNumericDocValues(VersionFieldMapper.NAME);
    assertThat(versions, notNullValue());
    for (int i = 0; i < ir.maxDoc(); ++i) {
        final String uidValue = ir.document(i).get(UidFieldMapper.NAME);
        final long expectedVersion = expectedVersions.get(uidValue);
        assertThat(versions.get(i), equalTo(expectedVersion));
    }

    iw.close();
    assertThat(IndexWriter.isLocked(iw.getDirectory()), is(false));
    ir.close();
    dir.close();
}