Example usage for org.apache.lucene.codecs CodecUtil checksumEntireFile

List of usage examples for org.apache.lucene.codecs CodecUtil checksumEntireFile

  1. HOME
  2. Java
  3. org.apache.lucene
  4. org.apache.lucene.codecs.*
  5. CodecUtil
  6. checksumEntireFile

Introduction

In this page you can find the example usage for org.apache.lucene.codecs CodecUtil checksumEntireFile.

Prototype

public static long checksumEntireFile(IndexInput input) throws IOException

Source Link

Document

Clones the provided input, reads all bytes from the file, and calls #checkFooter

Note that this method may be slow, as it must process the entire file.

Usage

From source file:com.lucure.core.codec.CompressingStoredFieldsReader.java

License:Apache License

@Override
public void checkIntegrity() throws IOException {
    if (version >= VERSION_CHECKSUM) {
        CodecUtil.checksumEntireFile(fieldsStream);
    }//from   w ww.  ja v  a  2 s  .  c o  m
}

From source file:com.rocana.lucene.codec.v1.RocanaBlockTreeTermsReader.java

License:Apache License

@Override
public void checkIntegrity() throws IOException {
    // term dictionary
    CodecUtil.checksumEntireFile(termsIn);

    // postings//from   ww  w . ja  v  a2s .c  o  m
    postingsReader.checkIntegrity();
}

From source file:com.sindicetech.siren.index.codecs.block.BlockIndexInput.java

License:Open Source License

/**
 * Checksum the entire file using/*from   w w w .j a va 2 s . c o m*/
 * {@link org.apache.lucene.codecs.CodecUtil#checksumEntireFile(org.apache.lucene.store.IndexInput)}.
 */
public long checksumEntireFile() throws IOException {
    return CodecUtil.checksumEntireFile(in);
}

From source file:com.sindicetech.siren.index.codecs.siren10.Siren10PostingsReader.java

License:Open Source License

@Override
public void checkIntegrity() throws IOException {
    if (docIn != null) {
        docIn.checksumEntireFile();//from www .j  a  v  a2s.  c o m
    }
    if (nodIn != null) {
        nodIn.checksumEntireFile();
    }
    if (posIn != null) {
        posIn.checksumEntireFile();
    }
    if (skipIn != null) {
        CodecUtil.checksumEntireFile(skipIn);
    }
}

From source file:org.codelibs.elasticsearch.common.lucene.Lucene.java

License:Apache License

public static void checkSegmentInfoIntegrity(final Directory directory) throws IOException {
    new SegmentInfos.FindSegmentsFile(directory) {

        @Override/*from   w w  w  .j  a  v a 2  s. c  o m*/
        protected Object doBody(String segmentFileName) throws IOException {
            try (IndexInput input = directory.openInput(segmentFileName, IOContext.READ)) {
                CodecUtil.checksumEntireFile(input);
            }
            return null;
        }
    }.run();
}

From source file:org.elasticsearch.gateway.local.state.meta.MetaDataStateFormat.java

License:Apache License

/**
 * Reads the state from a given file and compares the expected version against the actual version of
 * the state.//from  www.  j  a  va 2  s. c om
 */
public final T read(File file) throws IOException {
    try (Directory dir = newDirectory(file.getParentFile())) {
        try (final IndexInput indexInput = dir.openInput(file.getName(), IOContext.DEFAULT)) {
            // We checksum the entire file before we even go and parse it. If it's corrupted we barf right here.
            CodecUtil.checksumEntireFile(indexInput);
            CodecUtil.checkHeader(indexInput, STATE_FILE_CODEC, STATE_FILE_VERSION, STATE_FILE_VERSION);
            final XContentType xContentType = XContentType.values()[indexInput.readInt()];
            indexInput.readLong(); // version currently unused
            long filePointer = indexInput.getFilePointer();
            long contentSize = indexInput.length() - CodecUtil.footerLength() - filePointer;
            try (IndexInput slice = indexInput.slice("state_xcontent", filePointer, contentSize)) {
                try (XContentParser parser = XContentFactory.xContent(xContentType)
                        .createParser(new InputStreamIndexInput(slice, contentSize))) {
                    return fromXContent(parser);
                }
            }
        } catch (CorruptIndexException ex) {
            // we trick this into a dedicated exception with the original stacktrace
            throw new CorruptStateException(ex);
        }
    }
}

From source file:org.elasticsearch.gateway.MetaDataStateFormat.java

License:Apache License

/**
 * Reads the state from a given file and compares the expected version against the actual version of
 * the state.//from  w  w w. j a v a 2s  . c o m
 */
public final T read(Path file) throws IOException {
    try (Directory dir = newDirectory(file.getParent())) {
        try (final IndexInput indexInput = dir.openInput(file.getFileName().toString(), IOContext.DEFAULT)) {
            // We checksum the entire file before we even go and parse it. If it's corrupted we barf right here.
            CodecUtil.checksumEntireFile(indexInput);
            CodecUtil.checkHeader(indexInput, STATE_FILE_CODEC, STATE_FILE_VERSION, STATE_FILE_VERSION);
            final XContentType xContentType = XContentType.values()[indexInput.readInt()];
            indexInput.readLong(); // version currently unused
            long filePointer = indexInput.getFilePointer();
            long contentSize = indexInput.length() - CodecUtil.footerLength() - filePointer;
            try (IndexInput slice = indexInput.slice("state_xcontent", filePointer, contentSize)) {
                try (XContentParser parser = XContentFactory.xContent(xContentType)
                        .createParser(new InputStreamIndexInput(slice, contentSize))) {
                    return fromXContent(parser);
                }
            }
        } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
            // we trick this into a dedicated exception with the original stacktrace
            throw new CorruptStateException(ex);
        }
    }
}

From source file:org.elasticsearch.index.store.StoreTest.java

License:Apache License

@Test
public void testRenameFile() throws IOException {
    final ShardId shardId = new ShardId(new Index("index"), 1);
    DirectoryService directoryService = new LuceneManagedDirectoryService(random(), false);
    Store store = new Store(shardId, ImmutableSettings.EMPTY, directoryService,
            randomDistributor(directoryService), new DummyShardLock(shardId));
    {/*from w w w. ja v a2 s . c o  m*/
        IndexOutput output = store.directory().createOutput("foo.bar", IOContext.DEFAULT);
        int iters = scaledRandomIntBetween(10, 100);
        for (int i = 0; i < iters; i++) {
            BytesRef bytesRef = new BytesRef(TestUtil.randomRealisticUnicodeString(random(), 10, 1024));
            output.writeBytes(bytesRef.bytes, bytesRef.offset, bytesRef.length);
        }
        CodecUtil.writeFooter(output);
        output.close();
    }
    store.renameFile("foo.bar", "bar.foo");
    assertThat(store.directory().listAll().length, is(1));
    final long lastChecksum;
    try (IndexInput input = store.directory().openInput("bar.foo", IOContext.DEFAULT)) {
        lastChecksum = CodecUtil.checksumEntireFile(input);
    }

    try {
        store.directory().openInput("foo.bar", IOContext.DEFAULT);
        fail("file was renamed");
    } catch (FileNotFoundException | NoSuchFileException ex) {
        // expected
    }
    {
        IndexOutput output = store.directory().createOutput("foo.bar", IOContext.DEFAULT);
        int iters = scaledRandomIntBetween(10, 100);
        for (int i = 0; i < iters; i++) {
            BytesRef bytesRef = new BytesRef(TestUtil.randomRealisticUnicodeString(random(), 10, 1024));
            output.writeBytes(bytesRef.bytes, bytesRef.offset, bytesRef.length);
        }
        CodecUtil.writeFooter(output);
        output.close();
    }
    DistributorDirectory distributorDirectory = DirectoryUtils.getLeaf(store.directory(),
            DistributorDirectory.class);
    if (distributorDirectory != null
            && distributorDirectory.getDirectory("foo.bar") != distributorDirectory.getDirectory("bar.foo")) {
        try {
            store.renameFile("foo.bar", "bar.foo");
            fail("target file already exists in a different directory");
        } catch (IOException ex) {
            // expected
        }

        try (IndexInput input = store.directory().openInput("bar.foo", IOContext.DEFAULT)) {
            assertThat(lastChecksum, equalTo(CodecUtil.checksumEntireFile(input)));
        }
        assertThat(store.directory().listAll().length, is(2));
        assertDeleteContent(store, directoryService);
        IOUtils.close(store);
    } else {
        store.renameFile("foo.bar", "bar.foo");
        assertThat(store.directory().listAll().length, is(1));
        assertDeleteContent(store, directoryService);
        IOUtils.close(store);
    }
}

From source file:org.elasticsearch.index.store.StoreTests.java

License:Apache License

@Test
public void testRenameFile() throws IOException {
    final ShardId shardId = new ShardId(new Index("index"), 1);
    DirectoryService directoryService = new LuceneManagedDirectoryService(random(), false);
    Store store = new Store(shardId, Settings.EMPTY, directoryService, new DummyShardLock(shardId));
    {/*from w  ww.  j  a v a 2  s  .c  o m*/
        IndexOutput output = store.directory().createOutput("foo.bar", IOContext.DEFAULT);
        int iters = scaledRandomIntBetween(10, 100);
        for (int i = 0; i < iters; i++) {
            BytesRef bytesRef = new BytesRef(TestUtil.randomRealisticUnicodeString(random(), 10, 1024));
            output.writeBytes(bytesRef.bytes, bytesRef.offset, bytesRef.length);
        }
        CodecUtil.writeFooter(output);
        output.close();
    }
    store.renameFile("foo.bar", "bar.foo");
    assertThat(numNonExtraFiles(store), is(1));
    final long lastChecksum;
    try (IndexInput input = store.directory().openInput("bar.foo", IOContext.DEFAULT)) {
        lastChecksum = CodecUtil.checksumEntireFile(input);
    }

    try {
        store.directory().openInput("foo.bar", IOContext.DEFAULT);
        fail("file was renamed");
    } catch (FileNotFoundException | NoSuchFileException ex) {
        // expected
    }
    {
        IndexOutput output = store.directory().createOutput("foo.bar", IOContext.DEFAULT);
        int iters = scaledRandomIntBetween(10, 100);
        for (int i = 0; i < iters; i++) {
            BytesRef bytesRef = new BytesRef(TestUtil.randomRealisticUnicodeString(random(), 10, 1024));
            output.writeBytes(bytesRef.bytes, bytesRef.offset, bytesRef.length);
        }
        CodecUtil.writeFooter(output);
        output.close();
    }
    store.renameFile("foo.bar", "bar.foo");
    assertThat(numNonExtraFiles(store), is(1));
    assertDeleteContent(store, directoryService);
    IOUtils.close(store);
}

From source file:org.elasticsearch.repositories.blobstore.ChecksumBlobStoreFormat.java

License:Apache License

/**
 * Reads blob with specified name without resolving the blobName using using {@link #blobName} method.
 *
 * @param blobContainer blob container//www .  j  a  va2 s .c o m
 * @param blobName blob name
 */
public T readBlob(BlobContainer blobContainer, String blobName) throws IOException {
    try (InputStream inputStream = blobContainer.readBlob(blobName)) {
        byte[] bytes = ByteStreams.toByteArray(inputStream);
        final String resourceDesc = "ChecksumBlobStoreFormat.readBlob(blob=\"" + blobName + "\")";
        try (ByteArrayIndexInput indexInput = new ByteArrayIndexInput(resourceDesc, bytes)) {
            CodecUtil.checksumEntireFile(indexInput);
            CodecUtil.checkHeader(indexInput, codec, VERSION, VERSION);
            long filePointer = indexInput.getFilePointer();
            long contentSize = indexInput.length() - CodecUtil.footerLength() - filePointer;
            BytesReference bytesReference = new BytesArray(bytes, (int) filePointer, (int) contentSize);
            return read(bytesReference);
        } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
            // we trick this into a dedicated exception with the original stacktrace
            throw new CorruptStateException(ex);
        }
    }
}