Example usage for org.apache.lucene.index IndexWriter updateDocument

List of usage examples for org.apache.lucene.index IndexWriter updateDocument

  1. HOME
  2. Java
  3. org.apache.lucene
  4. org.apache.lucene.index.*
  5. IndexWriter
  6. updateDocument

Introduction

In this page you can find the example usage for org.apache.lucene.index IndexWriter updateDocument.

Prototype

private long updateDocument(final DocumentsWriterDeleteQueue.Node<?> delNode,
            Iterable<? extends IndexableField> doc) throws IOException

Source Link

Usage

From source file:org.elasticsearch.common.lucene.uid.UidFieldTests.java

License:Apache License

@Test
public void testUidField() throws Exception {
    IndexWriter writer = new IndexWriter(new RAMDirectory(),
            new IndexWriterConfig(Lucene.VERSION, Lucene.STANDARD_ANALYZER));

    IndexReader reader = IndexReader.open(writer, true);
    assertThat(UidField.loadVersion(reader, new Term("_uid", "1")), equalTo(-1l));

    Document doc = new Document();
    doc.add(new Field("_uid", "1", Field.Store.YES, Field.Index.NOT_ANALYZED));
    writer.addDocument(doc);//from  w  w w . ja v  a  2  s. com
    reader = reader.reopen();
    assertThat(UidField.loadVersion(reader, new Term("_uid", "1")), equalTo(-2l));
    assertThat(UidField.loadDocIdAndVersion(reader, new Term("_uid", "1")).version, equalTo(-2l));

    doc = new Document();
    doc.add(new UidField("_uid", "1", 1));
    writer.updateDocument(new Term("_uid", "1"), doc);
    reader = reader.reopen();
    assertThat(UidField.loadVersion(reader, new Term("_uid", "1")), equalTo(1l));
    assertThat(UidField.loadDocIdAndVersion(reader, new Term("_uid", "1")).version, equalTo(1l));

    doc = new Document();
    UidField uid = new UidField("_uid", "1", 2);
    doc.add(uid);
    writer.updateDocument(new Term("_uid", "1"), doc);
    reader = reader.reopen();
    assertThat(UidField.loadVersion(reader, new Term("_uid", "1")), equalTo(2l));
    assertThat(UidField.loadDocIdAndVersion(reader, new Term("_uid", "1")).version, equalTo(2l));

    // test reuse of uid field
    doc = new Document();
    uid.version(3);
    doc.add(uid);
    writer.updateDocument(new Term("_uid", "1"), doc);
    reader = reader.reopen();
    assertThat(UidField.loadVersion(reader, new Term("_uid", "1")), equalTo(3l));
    assertThat(UidField.loadDocIdAndVersion(reader, new Term("_uid", "1")).version, equalTo(3l));

    writer.deleteDocuments(new Term("_uid", "1"));
    reader = reader.reopen();
    assertThat(UidField.loadVersion(reader, new Term("_uid", "1")), equalTo(-1l));
    assertThat(UidField.loadDocIdAndVersion(reader, new Term("_uid", "1")), nullValue());
}

From source file:org.elasticsearch.common.lucene.uid.VersionsTests.java

License:Apache License

@Test
public void testVersions() throws Exception {
    Directory dir = newDirectory();//w w w  . j  a  v a2s.  c  o m
    IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(Lucene.VERSION, Lucene.STANDARD_ANALYZER));
    DirectoryReader directoryReader = DirectoryReader.open(writer, true);
    MatcherAssert.assertThat(Versions.loadVersion(directoryReader, new Term(UidFieldMapper.NAME, "1")),
            equalTo(Versions.NOT_FOUND));

    Document doc = new Document();
    doc.add(new Field(UidFieldMapper.NAME, "1", UidFieldMapper.Defaults.FIELD_TYPE));
    writer.addDocument(doc);
    directoryReader = reopen(directoryReader);
    assertThat(Versions.loadVersion(directoryReader, new Term(UidFieldMapper.NAME, "1")),
            equalTo(Versions.NOT_SET));
    assertThat(Versions.loadDocIdAndVersion(directoryReader, new Term(UidFieldMapper.NAME, "1")).version,
            equalTo(Versions.NOT_SET));

    doc = new Document();
    doc.add(new Field(UidFieldMapper.NAME, "1", UidFieldMapper.Defaults.FIELD_TYPE));
    doc.add(new NumericDocValuesField(VersionFieldMapper.NAME, 1));
    writer.updateDocument(new Term(UidFieldMapper.NAME, "1"), doc);
    directoryReader = reopen(directoryReader);
    assertThat(Versions.loadVersion(directoryReader, new Term(UidFieldMapper.NAME, "1")), equalTo(1l));
    assertThat(Versions.loadDocIdAndVersion(directoryReader, new Term(UidFieldMapper.NAME, "1")).version,
            equalTo(1l));

    doc = new Document();
    Field uid = new Field(UidFieldMapper.NAME, "1", UidFieldMapper.Defaults.FIELD_TYPE);
    Field version = new NumericDocValuesField(VersionFieldMapper.NAME, 2);
    doc.add(uid);
    doc.add(version);
    writer.updateDocument(new Term(UidFieldMapper.NAME, "1"), doc);
    directoryReader = reopen(directoryReader);
    assertThat(Versions.loadVersion(directoryReader, new Term(UidFieldMapper.NAME, "1")), equalTo(2l));
    assertThat(Versions.loadDocIdAndVersion(directoryReader, new Term(UidFieldMapper.NAME, "1")).version,
            equalTo(2l));

    // test reuse of uid field
    doc = new Document();
    version.setLongValue(3);
    doc.add(uid);
    doc.add(version);
    writer.updateDocument(new Term(UidFieldMapper.NAME, "1"), doc);

    directoryReader = reopen(directoryReader);
    assertThat(Versions.loadVersion(directoryReader, new Term(UidFieldMapper.NAME, "1")), equalTo(3l));
    assertThat(Versions.loadDocIdAndVersion(directoryReader, new Term(UidFieldMapper.NAME, "1")).version,
            equalTo(3l));

    writer.deleteDocuments(new Term(UidFieldMapper.NAME, "1"));
    directoryReader = reopen(directoryReader);
    assertThat(Versions.loadVersion(directoryReader, new Term(UidFieldMapper.NAME, "1")),
            equalTo(Versions.NOT_FOUND));
    assertThat(Versions.loadDocIdAndVersion(directoryReader, new Term(UidFieldMapper.NAME, "1")), nullValue());
    directoryReader.close();
    writer.close();
    dir.close();
}

From source file:org.elasticsearch.index.store.StoreTest.java

License:Apache License

@Test
public void testWriteLegacyChecksums() throws IOException {
    final ShardId shardId = new ShardId(new Index("index"), 1);
    DirectoryService directoryService = new LuceneManagedDirectoryService(random());
    Store store = new Store(shardId, ImmutableSettings.EMPTY, directoryService,
            randomDistributor(directoryService), new DummyShardLock(shardId));
    // set default codec - all segments need checksums
    final boolean usesOldCodec = randomBoolean();
    IndexWriter writer = new IndexWriter(store.directory(),
            newIndexWriterConfig(random(), TEST_VERSION_CURRENT, new MockAnalyzer(random()))
                    .setCodec(usesOldCodec ? new OldSIMockingCodec() : actualDefaultCodec()));
    int docs = 1 + random().nextInt(100);

    for (int i = 0; i < docs; i++) {
        Document doc = new Document();
        doc.add(new TextField("id", "" + i, random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
        doc.add(new TextField("body", TestUtil.randomRealisticUnicodeString(random()),
                random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
        doc.add(new SortedDocValuesField("dv", new BytesRef(TestUtil.randomRealisticUnicodeString(random()))));
        writer.addDocument(doc);/*from   w  ww .j a  va 2s  .c  o m*/
    }
    if (random().nextBoolean()) {
        for (int i = 0; i < docs; i++) {
            if (random().nextBoolean()) {
                Document doc = new Document();
                doc.add(new TextField("id", "" + i, random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
                doc.add(new TextField("body", TestUtil.randomRealisticUnicodeString(random()),
                        random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
                writer.updateDocument(new Term("id", "" + i), doc);
            }
        }
    }
    if (random().nextBoolean()) {
        DirectoryReader.open(writer, random().nextBoolean()).close(); // flush
    }
    Store.MetadataSnapshot metadata;
    // check before we committed
    try {
        store.getMetadata();
        fail("no index present - expected exception");
    } catch (IndexNotFoundException ex) {
        // expected
    }
    assertThat(store.getMetadataOrEmpty(), is(Store.MetadataSnapshot.EMPTY)); // nothing committed

    writer.close();
    Store.LegacyChecksums checksums = new Store.LegacyChecksums();
    Map<String, StoreFileMetaData> legacyMeta = new HashMap<>();
    for (String file : store.directory().listAll()) {
        if (file.equals("write.lock") || file.equals(IndexFileNames.SEGMENTS_GEN)) {
            continue;
        }
        BytesRef hash = new BytesRef();
        if (file.startsWith("segments")) {
            hash = Store.MetadataSnapshot.hashFile(store.directory(), file);
        }
        StoreFileMetaData storeFileMetaData = new StoreFileMetaData(file, store.directory().fileLength(file),
                file + "checksum", null, hash);
        legacyMeta.put(file, storeFileMetaData);
        checksums.add(storeFileMetaData);
    }
    checksums.write(store);

    metadata = store.getMetadata();
    Map<String, StoreFileMetaData> stringStoreFileMetaDataMap = metadata.asMap();
    assertThat(legacyMeta.size(), equalTo(stringStoreFileMetaDataMap.size()));
    if (usesOldCodec) {
        for (StoreFileMetaData meta : legacyMeta.values()) {
            assertTrue(meta.toString(), stringStoreFileMetaDataMap.containsKey(meta.name()));
            assertEquals(meta.name() + "checksum", meta.checksum());
            assertTrue(meta + " vs. " + stringStoreFileMetaDataMap.get(meta.name()),
                    stringStoreFileMetaDataMap.get(meta.name()).isSame(meta));
        }
    } else {

        // even if we have a legacy checksum - if we use a new codec we should reuse
        for (StoreFileMetaData meta : legacyMeta.values()) {
            assertTrue(meta.toString(), stringStoreFileMetaDataMap.containsKey(meta.name()));
            assertFalse(meta + " vs. " + stringStoreFileMetaDataMap.get(meta.name()),
                    stringStoreFileMetaDataMap.get(meta.name()).isSame(meta));
            StoreFileMetaData storeFileMetaData = metadata.get(meta.name());
            try (IndexInput input = store.openVerifyingInput(meta.name(), IOContext.DEFAULT,
                    storeFileMetaData)) {
                assertTrue(storeFileMetaData.toString(), input instanceof Store.VerifyingIndexInput);
                input.seek(meta.length());
                Store.verify(input);
            }
        }
    }
    assertDeleteContent(store, directoryService);
    IOUtils.close(store);

}

From source file:org.elasticsearch.index.store.StoreTest.java

License:Apache License

@Test
public void testNewChecksums() throws IOException {
    final ShardId shardId = new ShardId(new Index("index"), 1);
    DirectoryService directoryService = new LuceneManagedDirectoryService(random());
    Store store = new Store(shardId, ImmutableSettings.EMPTY, directoryService,
            randomDistributor(directoryService), new DummyShardLock(shardId));
    // set default codec - all segments need checksums
    IndexWriter writer = new IndexWriter(store.directory(),
            newIndexWriterConfig(random(), TEST_VERSION_CURRENT, new MockAnalyzer(random()))
                    .setCodec(actualDefaultCodec()));
    int docs = 1 + random().nextInt(100);

    for (int i = 0; i < docs; i++) {
        Document doc = new Document();
        doc.add(new TextField("id", "" + i, random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
        doc.add(new TextField("body", TestUtil.randomRealisticUnicodeString(random()),
                random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
        doc.add(new SortedDocValuesField("dv", new BytesRef(TestUtil.randomRealisticUnicodeString(random()))));
        writer.addDocument(doc);//w w w .  j  ava  2  s  .c o m
    }
    if (random().nextBoolean()) {
        for (int i = 0; i < docs; i++) {
            if (random().nextBoolean()) {
                Document doc = new Document();
                doc.add(new TextField("id", "" + i, random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
                doc.add(new TextField("body", TestUtil.randomRealisticUnicodeString(random()),
                        random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
                writer.updateDocument(new Term("id", "" + i), doc);
            }
        }
    }
    if (random().nextBoolean()) {
        DirectoryReader.open(writer, random().nextBoolean()).close(); // flush
    }
    Store.MetadataSnapshot metadata;
    // check before we committed
    try {
        store.getMetadata();
        fail("no index present - expected exception");
    } catch (IndexNotFoundException ex) {
        // expected
    }
    assertThat(store.getMetadataOrEmpty(), is(Store.MetadataSnapshot.EMPTY)); // nothing committed
    writer.commit();
    writer.close();
    metadata = store.getMetadata();
    assertThat(metadata.asMap().isEmpty(), is(false));
    for (StoreFileMetaData meta : metadata) {
        try (IndexInput input = store.directory().openInput(meta.name(), IOContext.DEFAULT)) {
            String checksum = Store.digestToString(CodecUtil.retrieveChecksum(input));
            assertThat("File: " + meta.name() + " has a different checksum", meta.checksum(),
                    equalTo(checksum));
            assertThat(meta.hasLegacyChecksum(), equalTo(false));
            assertThat(meta.writtenBy(), equalTo(TEST_VERSION_CURRENT));
            if (meta.name().endsWith(".si") || meta.name().startsWith("segments_")) {
                assertThat(meta.hash().length, greaterThan(0));
            }
        }
    }
    assertConsistent(store, metadata);

    TestUtil.checkIndex(store.directory());
    assertDeleteContent(store, directoryService);
    IOUtils.close(store);
}

From source file:org.elasticsearch.index.store.StoreTest.java

License:Apache License

@Test
public void testMixedChecksums() throws IOException {
    final ShardId shardId = new ShardId(new Index("index"), 1);
    DirectoryService directoryService = new LuceneManagedDirectoryService(random());
    Store store = new Store(shardId, ImmutableSettings.EMPTY, directoryService,
            randomDistributor(directoryService), new DummyShardLock(shardId));
    // this time random codec....
    IndexWriter writer = new IndexWriter(store.directory(),
            newIndexWriterConfig(random(), TEST_VERSION_CURRENT, new MockAnalyzer(random()))
                    .setCodec(actualDefaultCodec()));
    int docs = 1 + random().nextInt(100);

    for (int i = 0; i < docs; i++) {
        Document doc = new Document();
        doc.add(new TextField("id", "" + i, random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
        doc.add(new TextField("body", TestUtil.randomRealisticUnicodeString(random()),
                random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
        doc.add(new SortedDocValuesField("dv", new BytesRef(TestUtil.randomRealisticUnicodeString(random()))));
        writer.addDocument(doc);//from   www  . ja  v a2 s .c o  m
    }
    if (random().nextBoolean()) {
        for (int i = 0; i < docs; i++) {
            if (random().nextBoolean()) {
                Document doc = new Document();
                doc.add(new TextField("id", "" + i, random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
                doc.add(new TextField("body", TestUtil.randomRealisticUnicodeString(random()),
                        random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
                writer.updateDocument(new Term("id", "" + i), doc);
            }
        }
    }
    if (random().nextBoolean()) {
        DirectoryReader.open(writer, random().nextBoolean()).close(); // flush
    }
    Store.MetadataSnapshot metadata;
    // check before we committed
    try {
        store.getMetadata();
        fail("no index present - expected exception");
    } catch (IndexNotFoundException ex) {
        // expected
    }
    assertThat(store.getMetadataOrEmpty(), is(Store.MetadataSnapshot.EMPTY)); // nothing committed
    writer.commit();
    writer.close();
    Store.LegacyChecksums checksums = new Store.LegacyChecksums();
    metadata = store.getMetadata();
    assertThat(metadata.asMap().isEmpty(), is(false));
    for (StoreFileMetaData meta : metadata) {
        try (IndexInput input = store.directory().openInput(meta.name(), IOContext.DEFAULT)) {
            if (meta.checksum() == null) {
                String checksum = null;
                try {
                    CodecUtil.retrieveChecksum(input);
                    fail("expected a corrupt index - posting format has not checksums");
                } catch (CorruptIndexException ex) {
                    try (ChecksumIndexInput checksumIndexInput = store.directory()
                            .openChecksumInput(meta.name(), IOContext.DEFAULT)) {
                        checksumIndexInput.seek(meta.length());
                        checksum = Store.digestToString(checksumIndexInput.getChecksum());
                    }
                    // fine - it's a postings format without checksums
                    checksums.add(new StoreFileMetaData(meta.name(), meta.length(), checksum, null));
                }
            } else {
                String checksum = Store.digestToString(CodecUtil.retrieveChecksum(input));
                assertThat("File: " + meta.name() + " has a different checksum", meta.checksum(),
                        equalTo(checksum));
                assertThat(meta.hasLegacyChecksum(), equalTo(false));
                assertThat(meta.writtenBy(), equalTo(TEST_VERSION_CURRENT));
            }
        }
    }
    assertConsistent(store, metadata);
    checksums.write(store);
    metadata = store.getMetadata();
    assertThat(metadata.asMap().isEmpty(), is(false));
    for (StoreFileMetaData meta : metadata) {
        assertThat("file: " + meta.name() + " has a null checksum", meta.checksum(), not(nullValue()));
        if (meta.hasLegacyChecksum()) {
            try (ChecksumIndexInput checksumIndexInput = store.directory().openChecksumInput(meta.name(),
                    IOContext.DEFAULT)) {
                checksumIndexInput.seek(meta.length());
                assertThat(meta.checksum(), equalTo(Store.digestToString(checksumIndexInput.getChecksum())));
            }
        } else {
            try (IndexInput input = store.directory().openInput(meta.name(), IOContext.DEFAULT)) {
                String checksum = Store.digestToString(CodecUtil.retrieveChecksum(input));
                assertThat("File: " + meta.name() + " has a different checksum", meta.checksum(),
                        equalTo(checksum));
                assertThat(meta.hasLegacyChecksum(), equalTo(false));
                assertThat(meta.writtenBy(), equalTo(TEST_VERSION_CURRENT));
            }
        }
    }
    assertConsistent(store, metadata);
    TestUtil.checkIndex(store.directory());
    assertDeleteContent(store, directoryService);
    IOUtils.close(store);
}

From source file:org.elasticsearch.index.store.StoreTest.java

License:Apache License

@Test
public void testCleanupFromSnapshot() throws IOException {
    final ShardId shardId = new ShardId(new Index("index"), 1);
    DirectoryService directoryService = new LuceneManagedDirectoryService(random());
    Store store = new Store(shardId, ImmutableSettings.EMPTY, directoryService,
            randomDistributor(directoryService), new DummyShardLock(shardId));
    // this time random codec....
    IndexWriterConfig indexWriterConfig = newIndexWriterConfig(random(), TEST_VERSION_CURRENT,
            new MockAnalyzer(random())).setCodec(actualDefaultCodec());
    // we keep all commits and that allows us clean based on multiple snapshots
    indexWriterConfig.setIndexDeletionPolicy(NoDeletionPolicy.INSTANCE);
    IndexWriter writer = new IndexWriter(store.directory(), indexWriterConfig);
    int docs = 1 + random().nextInt(100);
    int numCommits = 0;
    for (int i = 0; i < docs; i++) {
        if (i > 0 && randomIntBetween(0, 10) == 0) {
            writer.commit();/*from   w  ww.  j a v  a2 s  . co  m*/
            numCommits++;
        }
        Document doc = new Document();
        doc.add(new TextField("id", "" + i, random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
        doc.add(new TextField("body", TestUtil.randomRealisticUnicodeString(random()),
                random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
        doc.add(new SortedDocValuesField("dv", new BytesRef(TestUtil.randomRealisticUnicodeString(random()))));
        writer.addDocument(doc);

    }
    if (numCommits < 1) {
        writer.commit();
        Document doc = new Document();
        doc.add(new TextField("id", "" + docs++, random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
        doc.add(new TextField("body", TestUtil.randomRealisticUnicodeString(random()),
                random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
        doc.add(new SortedDocValuesField("dv", new BytesRef(TestUtil.randomRealisticUnicodeString(random()))));
        writer.addDocument(doc);
    }

    Store.MetadataSnapshot firstMeta = store.getMetadata();

    if (random().nextBoolean()) {
        for (int i = 0; i < docs; i++) {
            if (random().nextBoolean()) {
                Document doc = new Document();
                doc.add(new TextField("id", "" + i, random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
                doc.add(new TextField("body", TestUtil.randomRealisticUnicodeString(random()),
                        random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
                writer.updateDocument(new Term("id", "" + i), doc);
            }
        }
    }
    writer.commit();
    writer.close();

    Store.MetadataSnapshot secondMeta = store.getMetadata();

    Store.LegacyChecksums checksums = new Store.LegacyChecksums();
    Map<String, StoreFileMetaData> legacyMeta = new HashMap<>();
    for (String file : store.directory().listAll()) {
        if (file.equals("write.lock") || file.equals(IndexFileNames.SEGMENTS_GEN)) {
            continue;
        }
        BytesRef hash = new BytesRef();
        if (file.startsWith("segments")) {
            hash = Store.MetadataSnapshot.hashFile(store.directory(), file);
        }
        StoreFileMetaData storeFileMetaData = new StoreFileMetaData(file, store.directory().fileLength(file),
                file + "checksum", null, hash);
        legacyMeta.put(file, storeFileMetaData);
        checksums.add(storeFileMetaData);
    }
    checksums.write(store); // write one checksum file here - we expect it to survive all the cleanups

    if (randomBoolean()) {
        store.cleanupAndVerify("test", firstMeta);
        String[] strings = store.directory().listAll();
        int numChecksums = 0;
        int numNotFound = 0;
        for (String file : strings) {
            assertTrue(firstMeta.contains(file) || Store.isChecksum(file) || file.equals("write.lock"));
            if (Store.isChecksum(file)) {
                numChecksums++;
            } else if (secondMeta.contains(file) == false) {
                numNotFound++;
            }

        }
        assertTrue("at least one file must not be in here since we have two commits?", numNotFound > 0);
        assertEquals("we wrote one checksum but it's gone now? - checksums are supposed to be kept",
                numChecksums, 1);
    } else {
        store.cleanupAndVerify("test", secondMeta);
        String[] strings = store.directory().listAll();
        int numChecksums = 0;
        int numNotFound = 0;
        for (String file : strings) {
            assertTrue(file, secondMeta.contains(file) || Store.isChecksum(file) || file.equals("write.lock"));
            if (Store.isChecksum(file)) {
                numChecksums++;
            } else if (firstMeta.contains(file) == false) {
                numNotFound++;
            }

        }
        assertTrue("at least one file must not be in here since we have two commits?", numNotFound > 0);
        assertEquals("we wrote one checksum but it's gone now? - checksums are supposed to be kept",
                numChecksums, 1);
    }

    deleteContent(store.directory());
    IOUtils.close(store);
}

From source file:org.elasticsearch.index.store.StoreTests.java

License:Apache License

@AwaitsFix(bugUrl = "Fails with seed E1394B038144F6E")
// The test currently fails because the segment infos and the index don't
// agree on the oldest version of a segment. We should fix this test by
// switching to a static bw index
@Test//www  .  j  av a 2s  .  co m
public void testWriteLegacyChecksums() throws IOException {
    final ShardId shardId = new ShardId(new Index("index"), 1);
    DirectoryService directoryService = new LuceneManagedDirectoryService(random());
    Store store = new Store(shardId, Settings.EMPTY, directoryService, new DummyShardLock(shardId));
    // set default codec - all segments need checksums
    final boolean usesOldCodec = randomBoolean();
    IndexWriter writer = new IndexWriter(store.directory(),
            newIndexWriterConfig(random(), new MockAnalyzer(random()))
                    .setCodec(usesOldCodec ? new OldSIMockingCodec() : TestUtil.getDefaultCodec()));
    int docs = 1 + random().nextInt(100);

    for (int i = 0; i < docs; i++) {
        Document doc = new Document();
        doc.add(new TextField("id", "" + i, random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
        doc.add(new TextField("body", TestUtil.randomRealisticUnicodeString(random()),
                random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
        doc.add(new SortedDocValuesField("dv", new BytesRef(TestUtil.randomRealisticUnicodeString(random()))));
        writer.addDocument(doc);
    }
    if (random().nextBoolean()) {
        for (int i = 0; i < docs; i++) {
            if (random().nextBoolean()) {
                Document doc = new Document();
                doc.add(new TextField("id", "" + i, random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
                doc.add(new TextField("body", TestUtil.randomRealisticUnicodeString(random()),
                        random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
                writer.updateDocument(new Term("id", "" + i), doc);
            }
        }
    }
    if (random().nextBoolean()) {
        DirectoryReader.open(writer, random().nextBoolean()).close(); // flush
    }
    Store.MetadataSnapshot metadata;
    // check before we committed
    try {
        store.getMetadata();
        fail("no index present - expected exception");
    } catch (IndexNotFoundException ex) {
        // expected
    }
    assertThat(store.getMetadataOrEmpty(), is(Store.MetadataSnapshot.EMPTY)); // nothing committed

    writer.close();
    Store.LegacyChecksums checksums = new Store.LegacyChecksums();
    Map<String, StoreFileMetaData> legacyMeta = new HashMap<>();
    for (String file : store.directory().listAll()) {
        if (file.equals("write.lock") || file.equals(IndexFileNames.OLD_SEGMENTS_GEN)
                || file.startsWith("extra")) {
            continue;
        }
        BytesRef hash = new BytesRef();
        if (file.startsWith("segments")) {
            hash = Store.MetadataSnapshot.hashFile(store.directory(), file);
        }
        StoreFileMetaData storeFileMetaData = new StoreFileMetaData(file, store.directory().fileLength(file),
                file + "checksum", null, hash);
        legacyMeta.put(file, storeFileMetaData);
        checksums.add(storeFileMetaData);
    }
    checksums.write(store);

    metadata = store.getMetadata();
    Map<String, StoreFileMetaData> stringStoreFileMetaDataMap = metadata.asMap();
    assertThat(legacyMeta.size(), equalTo(stringStoreFileMetaDataMap.size()));
    if (usesOldCodec) {
        for (StoreFileMetaData meta : legacyMeta.values()) {
            assertTrue(meta.toString(), stringStoreFileMetaDataMap.containsKey(meta.name()));
            assertEquals(meta.name() + "checksum", meta.checksum());
            assertTrue(meta + " vs. " + stringStoreFileMetaDataMap.get(meta.name()),
                    stringStoreFileMetaDataMap.get(meta.name()).isSame(meta));
        }
    } else {

        // even if we have a legacy checksum - if we use a new codec we should reuse
        for (StoreFileMetaData meta : legacyMeta.values()) {
            assertTrue(meta.toString(), stringStoreFileMetaDataMap.containsKey(meta.name()));
            assertFalse(meta + " vs. " + stringStoreFileMetaDataMap.get(meta.name()),
                    stringStoreFileMetaDataMap.get(meta.name()).isSame(meta));
            StoreFileMetaData storeFileMetaData = metadata.get(meta.name());
            try (IndexInput input = store.openVerifyingInput(meta.name(), IOContext.DEFAULT,
                    storeFileMetaData)) {
                assertTrue(storeFileMetaData.toString(), input instanceof Store.VerifyingIndexInput);
                input.seek(meta.length());
                Store.verify(input);
            }
        }
    }
    assertDeleteContent(store, directoryService);
    IOUtils.close(store);

}

From source file:org.elasticsearch.index.store.StoreTests.java

License:Apache License

@Test
public void testNewChecksums() throws IOException {
    final ShardId shardId = new ShardId(new Index("index"), 1);
    DirectoryService directoryService = new LuceneManagedDirectoryService(random());
    Store store = new Store(shardId, Settings.EMPTY, directoryService, new DummyShardLock(shardId));
    // set default codec - all segments need checksums
    IndexWriter writer = new IndexWriter(store.directory(),
            newIndexWriterConfig(random(), new MockAnalyzer(random())).setCodec(TestUtil.getDefaultCodec()));
    int docs = 1 + random().nextInt(100);

    for (int i = 0; i < docs; i++) {
        Document doc = new Document();
        doc.add(new TextField("id", "" + i, random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
        doc.add(new TextField("body", TestUtil.randomRealisticUnicodeString(random()),
                random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
        doc.add(new SortedDocValuesField("dv", new BytesRef(TestUtil.randomRealisticUnicodeString(random()))));
        writer.addDocument(doc);// ww w . jav  a  2  s.  c om
    }
    if (random().nextBoolean()) {
        for (int i = 0; i < docs; i++) {
            if (random().nextBoolean()) {
                Document doc = new Document();
                doc.add(new TextField("id", "" + i, random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
                doc.add(new TextField("body", TestUtil.randomRealisticUnicodeString(random()),
                        random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
                writer.updateDocument(new Term("id", "" + i), doc);
            }
        }
    }
    if (random().nextBoolean()) {
        DirectoryReader.open(writer, random().nextBoolean()).close(); // flush
    }
    Store.MetadataSnapshot metadata;
    // check before we committed
    try {
        store.getMetadata();
        fail("no index present - expected exception");
    } catch (IndexNotFoundException ex) {
        // expected
    }
    assertThat(store.getMetadataOrEmpty(), is(Store.MetadataSnapshot.EMPTY)); // nothing committed
    writer.commit();
    writer.close();
    metadata = store.getMetadata();
    assertThat(metadata.asMap().isEmpty(), is(false));
    for (StoreFileMetaData meta : metadata) {
        try (IndexInput input = store.directory().openInput(meta.name(), IOContext.DEFAULT)) {
            String checksum = Store.digestToString(CodecUtil.retrieveChecksum(input));
            assertThat("File: " + meta.name() + " has a different checksum", meta.checksum(),
                    equalTo(checksum));
            assertThat(meta.hasLegacyChecksum(), equalTo(false));
            assertThat(meta.writtenBy(), equalTo(Version.LATEST));
            if (meta.name().endsWith(".si") || meta.name().startsWith("segments_")) {
                assertThat(meta.hash().length, greaterThan(0));
            }
        }
    }
    assertConsistent(store, metadata);

    TestUtil.checkIndex(store.directory());
    assertDeleteContent(store, directoryService);
    IOUtils.close(store);
}

From source file:org.elasticsearch.index.store.StoreTests.java

License:Apache License

@Test
public void testMixedChecksums() throws IOException {
    final ShardId shardId = new ShardId(new Index("index"), 1);
    DirectoryService directoryService = new LuceneManagedDirectoryService(random());
    Store store = new Store(shardId, Settings.EMPTY, directoryService, new DummyShardLock(shardId));
    // this time random codec....
    IndexWriter writer = new IndexWriter(store.directory(),
            newIndexWriterConfig(random(), new MockAnalyzer(random())).setCodec(TestUtil.getDefaultCodec()));
    int docs = 1 + random().nextInt(100);

    for (int i = 0; i < docs; i++) {
        Document doc = new Document();
        doc.add(new TextField("id", "" + i, random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
        doc.add(new TextField("body", TestUtil.randomRealisticUnicodeString(random()),
                random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
        doc.add(new SortedDocValuesField("dv", new BytesRef(TestUtil.randomRealisticUnicodeString(random()))));
        writer.addDocument(doc);// w  w  w  .  j a  v  a 2 s .c  o  m
    }
    if (random().nextBoolean()) {
        for (int i = 0; i < docs; i++) {
            if (random().nextBoolean()) {
                Document doc = new Document();
                doc.add(new TextField("id", "" + i, random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
                doc.add(new TextField("body", TestUtil.randomRealisticUnicodeString(random()),
                        random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
                writer.updateDocument(new Term("id", "" + i), doc);
            }
        }
    }
    if (random().nextBoolean()) {
        DirectoryReader.open(writer, random().nextBoolean()).close(); // flush
    }
    Store.MetadataSnapshot metadata;
    // check before we committed
    try {
        store.getMetadata();
        fail("no index present - expected exception");
    } catch (IndexNotFoundException ex) {
        // expected
    }
    assertThat(store.getMetadataOrEmpty(), is(Store.MetadataSnapshot.EMPTY)); // nothing committed
    writer.commit();
    writer.close();
    Store.LegacyChecksums checksums = new Store.LegacyChecksums();
    metadata = store.getMetadata();
    assertThat(metadata.asMap().isEmpty(), is(false));
    for (StoreFileMetaData meta : metadata) {
        try (IndexInput input = store.directory().openInput(meta.name(), IOContext.DEFAULT)) {
            if (meta.checksum() == null) {
                String checksum = null;
                try {
                    CodecUtil.retrieveChecksum(input);
                    fail("expected a corrupt index - posting format has not checksums");
                } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
                    try (ChecksumIndexInput checksumIndexInput = store.directory()
                            .openChecksumInput(meta.name(), IOContext.DEFAULT)) {
                        checksumIndexInput.seek(meta.length());
                        checksum = Store.digestToString(checksumIndexInput.getChecksum());
                    }
                    // fine - it's a postings format without checksums
                    checksums.add(new StoreFileMetaData(meta.name(), meta.length(), checksum, null));
                }
            } else {
                String checksum = Store.digestToString(CodecUtil.retrieveChecksum(input));
                assertThat("File: " + meta.name() + " has a different checksum", meta.checksum(),
                        equalTo(checksum));
                assertThat(meta.hasLegacyChecksum(), equalTo(false));
                assertThat(meta.writtenBy(), equalTo(Version.LATEST));
            }
        }
    }
    assertConsistent(store, metadata);
    checksums.write(store);
    metadata = store.getMetadata();
    assertThat(metadata.asMap().isEmpty(), is(false));
    for (StoreFileMetaData meta : metadata) {
        assertThat("file: " + meta.name() + " has a null checksum", meta.checksum(), not(nullValue()));
        if (meta.hasLegacyChecksum()) {
            try (ChecksumIndexInput checksumIndexInput = store.directory().openChecksumInput(meta.name(),
                    IOContext.DEFAULT)) {
                checksumIndexInput.seek(meta.length());
                assertThat(meta.checksum(), equalTo(Store.digestToString(checksumIndexInput.getChecksum())));
            }
        } else {
            try (IndexInput input = store.directory().openInput(meta.name(), IOContext.DEFAULT)) {
                String checksum = Store.digestToString(CodecUtil.retrieveChecksum(input));
                assertThat("File: " + meta.name() + " has a different checksum", meta.checksum(),
                        equalTo(checksum));
                assertThat(meta.hasLegacyChecksum(), equalTo(false));
                assertThat(meta.writtenBy(), equalTo(Version.LATEST));
            }
        }
    }
    assertConsistent(store, metadata);
    TestUtil.checkIndex(store.directory());
    assertDeleteContent(store, directoryService);
    IOUtils.close(store);
}

From source file:org.elasticsearch.index.store.StoreTests.java

License:Apache License

@Test
public void testCleanupFromSnapshot() throws IOException {
    final ShardId shardId = new ShardId(new Index("index"), 1);
    DirectoryService directoryService = new LuceneManagedDirectoryService(random());
    Store store = new Store(shardId, Settings.EMPTY, directoryService, new DummyShardLock(shardId));
    // this time random codec....
    IndexWriterConfig indexWriterConfig = newIndexWriterConfig(random(), new MockAnalyzer(random()))
            .setCodec(TestUtil.getDefaultCodec());
    // we keep all commits and that allows us clean based on multiple snapshots
    indexWriterConfig.setIndexDeletionPolicy(NoDeletionPolicy.INSTANCE);
    IndexWriter writer = new IndexWriter(store.directory(), indexWriterConfig);
    int docs = 1 + random().nextInt(100);
    int numCommits = 0;
    for (int i = 0; i < docs; i++) {
        if (i > 0 && randomIntBetween(0, 10) == 0) {
            writer.commit();//from   www  .j a v  a 2 s . com
            numCommits++;
        }
        Document doc = new Document();
        doc.add(new TextField("id", "" + i, random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
        doc.add(new TextField("body", TestUtil.randomRealisticUnicodeString(random()),
                random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
        doc.add(new SortedDocValuesField("dv", new BytesRef(TestUtil.randomRealisticUnicodeString(random()))));
        writer.addDocument(doc);

    }
    if (numCommits < 1) {
        writer.commit();
        Document doc = new Document();
        doc.add(new TextField("id", "" + docs++, random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
        doc.add(new TextField("body", TestUtil.randomRealisticUnicodeString(random()),
                random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
        doc.add(new SortedDocValuesField("dv", new BytesRef(TestUtil.randomRealisticUnicodeString(random()))));
        writer.addDocument(doc);
    }

    Store.MetadataSnapshot firstMeta = store.getMetadata();

    if (random().nextBoolean()) {
        for (int i = 0; i < docs; i++) {
            if (random().nextBoolean()) {
                Document doc = new Document();
                doc.add(new TextField("id", "" + i, random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
                doc.add(new TextField("body", TestUtil.randomRealisticUnicodeString(random()),
                        random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
                writer.updateDocument(new Term("id", "" + i), doc);
            }
        }
    }
    writer.commit();
    writer.close();

    Store.MetadataSnapshot secondMeta = store.getMetadata();

    Store.LegacyChecksums checksums = new Store.LegacyChecksums();
    Map<String, StoreFileMetaData> legacyMeta = new HashMap<>();
    for (String file : store.directory().listAll()) {
        if (file.equals("write.lock") || file.equals(IndexFileNames.OLD_SEGMENTS_GEN)
                || file.startsWith("extra")) {
            continue;
        }
        BytesRef hash = new BytesRef();
        if (file.startsWith("segments")) {
            hash = Store.MetadataSnapshot.hashFile(store.directory(), file);
        }
        StoreFileMetaData storeFileMetaData = new StoreFileMetaData(file, store.directory().fileLength(file),
                file + "checksum", null, hash);
        legacyMeta.put(file, storeFileMetaData);
        checksums.add(storeFileMetaData);
    }
    checksums.write(store); // write one checksum file here - we expect it to survive all the cleanups

    if (randomBoolean()) {
        store.cleanupAndVerify("test", firstMeta);
        String[] strings = store.directory().listAll();
        int numChecksums = 0;
        int numNotFound = 0;
        for (String file : strings) {
            if (file.startsWith("extra")) {
                continue;
            }
            assertTrue(firstMeta.contains(file) || Store.isChecksum(file) || file.equals("write.lock"));
            if (Store.isChecksum(file)) {
                numChecksums++;
            } else if (secondMeta.contains(file) == false) {
                numNotFound++;
            }

        }
        assertTrue("at least one file must not be in here since we have two commits?", numNotFound > 0);
        assertEquals("we wrote one checksum but it's gone now? - checksums are supposed to be kept",
                numChecksums, 1);
    } else {
        store.cleanupAndVerify("test", secondMeta);
        String[] strings = store.directory().listAll();
        int numChecksums = 0;
        int numNotFound = 0;
        for (String file : strings) {
            if (file.startsWith("extra")) {
                continue;
            }
            assertTrue(file, secondMeta.contains(file) || Store.isChecksum(file) || file.equals("write.lock"));
            if (Store.isChecksum(file)) {
                numChecksums++;
            } else if (firstMeta.contains(file) == false) {
                numNotFound++;
            }

        }
        assertTrue("at least one file must not be in here since we have two commits?", numNotFound > 0);
        assertEquals("we wrote one checksum but it's gone now? - checksums are supposed to be kept",
                numChecksums, 1);
    }

    deleteContent(store.directory());
    IOUtils.close(store);
}