List of usage examples for org.apache.lucene.store ChecksumIndexInput getChecksum
public abstract long getChecksum() throws IOException;
From source file:com.devwebsphere.wxslucene.GridDirectory.java
License:Open Source License
public static void copy(Directory src, GridDirectory dest, boolean closeDirSrc) throws IOException { final String[] files = src.listAll(); IndexFileNameFilter filter = IndexFileNameFilter.getFilter(); byte[] buf = new byte[COPY_BUFFER_SIZE]; for (int i = 0; i < files.length; i++) { if (!filter.accept(null, files[i])) continue; IndexOutput os = null;/*from ww w. j av a 2 s . c o m*/ ChecksumIndexInput is = null; try { // create file in dest directory os = dest.createOutput(files[i]); // read current file is = new ChecksumIndexInput(src.openInput(files[i])); // and copy to dest directory long len = is.length(); long readCount = 0; while (readCount < len) { int toRead = readCount + COPY_BUFFER_SIZE > len ? (int) (len - readCount) : COPY_BUFFER_SIZE; is.readBytes(buf, 0, toRead); os.writeBytes(buf, toRead); readCount += toRead; } long src_sum = is.getChecksum(); os.flush(); // this code can just compare the new file with the old one // to make sure it's copied correctly ChecksumIndexInput dst_check_stream = new ChecksumIndexInput(dest.openInput(files[i])); len = dst_check_stream.length(); readCount = 0; while (readCount < len) { int toRead = readCount + COPY_BUFFER_SIZE > len ? (int) (len - readCount) : COPY_BUFFER_SIZE; dst_check_stream.readBytes(buf, 0, toRead); readCount += toRead; } long dst_sum = dst_check_stream.getChecksum(); if (dst_sum == src_sum) { logger.log(Level.INFO, "Verify " + files[i] + " was successful"); } else { logger.log(Level.INFO, "Verify " + files[i] + " failed"); throw new IllegalStateException("File " + files[i] + " failed verification"); } } finally { // graceful cleanup try { if (os != null) os.close(); } finally { if (is != null) is.close(); } } } if (closeDirSrc) src.close(); }
From source file:com.zimbra.cs.index.LuceneIndexRepair.java
License:Open Source License
private void convert(ChecksumIndexInput input, ChecksumIndexOutput output) throws IOException { int format = input.readInt(); if (format < 0) { if (format < FORMAT) { throw new CorruptIndexException("Unknown format version: " + format); } else {/*from w w w . j ava 2 s . c o m*/ output.writeInt(format); long version = input.readLong(); version++; // increment output.writeLong(version); output.writeInt(input.readInt()); // counter } } else { // file is in old format without explicit format info output.writeInt(format); } int num = input.readInt(); output.writeInt(num); for (int i = 0; i < num; i++) { if (format <= SegmentInfos.FORMAT_3_1) { output.writeString(input.readString()); // version } String name = input.readString(); output.writeString(name); int count = input.readInt(); output.writeInt(count); long delGen = -1; if (format <= SegmentInfos.FORMAT_LOCKLESS) { delGen = input.readLong(); output.writeLong(delGen); if (format <= SegmentInfos.FORMAT_SHARED_DOC_STORE) { int docStoreOffset = input.readInt(); output.writeInt(docStoreOffset); if (docStoreOffset != -1) { output.writeString(input.readString()); // docStoreSegment output.writeByte(input.readByte()); // docStoreIsCompoundFile } } } if (format <= SegmentInfos.FORMAT_SINGLE_NORM_FILE) { output.writeByte(input.readByte()); // hasSingleNormFile } int numNormGen = input.readInt(); output.writeInt(numNormGen); if (numNormGen > 0) { for (int j = 0; j < numNormGen; j++) { output.writeLong(input.readLong()); // normGen } } output.writeByte(input.readByte()); // isCompoundFile if (format <= SegmentInfos.FORMAT_DEL_COUNT) { int delCount = input.readInt(); if (delCount <= count && delCount == getDelCount(name, delGen)) { output.writeInt(delCount); } else { // del count mismatch // https://issues.apache.org/jira/browse/LUCENE-1474 repaired++; output.writeInt(-1); // clear } } if (format <= SegmentInfos.FORMAT_HAS_PROX) { output.writeByte(input.readByte()); // hasProx } if (format <= SegmentInfos.FORMAT_DIAGNOSTICS) { output.writeStringStringMap(input.readStringStringMap()); // diagnostics } if (format <= SegmentInfos.FORMAT_HAS_VECTORS) { output.writeByte(input.readByte()); // hasVectors } } if (format >= 0) { // in old format the version number may be at the end of the file if (input.getFilePointer() < input.length()) { output.writeLong(input.readLong()); // version } } if (format <= SegmentInfos.FORMAT_USER_DATA) { if (format <= SegmentInfos.FORMAT_DIAGNOSTICS) { output.writeStringStringMap(input.readStringStringMap()); // user data } else { output.writeByte(input.readByte()); output.writeString(input.readString()); // user data } } if (format <= SegmentInfos.FORMAT_CHECKSUM) { final long checksumNow = input.getChecksum(); final long checksumThen = input.readLong(); if (checksumNow != checksumThen) { throw new CorruptIndexException("checksum mismatch in segments file"); } } output.finishCommit(); }
From source file:org.apache.solr.codecs.onsql.ONSQLUtil.java
License:Apache License
public static void checkFooter(ChecksumIndexInput input) throws IOException { BytesRefBuilder scratch = new BytesRefBuilder(); String expectedChecksum = String.format(Locale.ROOT, "%020d", input.getChecksum()); ONSQLUtil.readLine(input, scratch);/*from w ww. j a va 2 s . com*/ if (StringHelper.startsWith(scratch.get(), CHECKSUM) == false) { throw new CorruptIndexException("ONSQL failure: expected checksum line but got " + scratch.get().utf8ToString() + " (resource=" + input + ")"); } String actualChecksum = new BytesRef(scratch.bytes(), CHECKSUM.length, scratch.length() - CHECKSUM.length) .utf8ToString(); if (!expectedChecksum.equals(actualChecksum)) { throw new CorruptIndexException("ONSQL checksum failure: " + actualChecksum + " != " + expectedChecksum + " (resource=" + input + ")"); } if (input.length() != input.getFilePointer()) { throw new CorruptIndexException( "Unexpected stuff at the end of file, please be careful with your text editor! (resource=" + input + ")"); } }
From source file:org.elasticsearch.util.lucene.Directories.java
License:Apache License
/** * Computes the checksum of the content represented by the provided index input. * * <p>Closes the index input once checksum is computed. *///from ww w . j a v a 2 s . co m public static long checksum(IndexInput indexInput) throws IOException { final int BUFFER_SIZE = 16384; byte[] buf = new byte[BUFFER_SIZE]; ChecksumIndexInput cii = new ChecksumIndexInput(indexInput); long len = cii.length(); long readCount = 0; while (readCount < len) { int toRead = readCount + BUFFER_SIZE > len ? (int) (len - readCount) : BUFFER_SIZE; cii.readBytes(buf, 0, toRead); readCount += toRead; } cii.close(); return cii.getChecksum(); }