List of usage examples for org.apache.lucene.codecs CodecUtil retrieveChecksum
public static long retrieveChecksum(IndexInput in) throws IOException
From source file:com.lucure.core.codec.CompressingStoredFieldsReader.java
License:Apache License
/** Sole constructor. */ public CompressingStoredFieldsReader(Directory d, SegmentInfo si, String segmentSuffix, FieldInfos fn, IOContext context, String formatName, CompressionMode compressionMode) throws IOException { this.compressionMode = compressionMode; final String segment = si.name; boolean success = false; fieldInfos = fn;/*from w w w . j a v a 2 s . co m*/ numDocs = si.getDocCount(); ChecksumIndexInput indexStream = null; try { final String indexStreamFN = IndexFileNames.segmentFileName(segment, segmentSuffix, FIELDS_INDEX_EXTENSION); final String fieldsStreamFN = IndexFileNames.segmentFileName(segment, segmentSuffix, FIELDS_EXTENSION); // Load the index into memory indexStream = d.openChecksumInput(indexStreamFN, context); final String codecNameIdx = formatName + CODEC_SFX_IDX; version = CodecUtil.checkHeader(indexStream, codecNameIdx, VERSION_START, VERSION_CURRENT); assert CodecUtil.headerLength(codecNameIdx) == indexStream.getFilePointer(); indexReader = new CompressingStoredFieldsIndexReader(indexStream, si); long maxPointer = -1; if (version >= VERSION_CHECKSUM) { maxPointer = indexStream.readVLong(); CodecUtil.checkFooter(indexStream); } else { CodecUtil.checkEOF(indexStream); } indexStream.close(); indexStream = null; // Open the data file and read metadata fieldsStream = d.openInput(fieldsStreamFN, context); if (version >= VERSION_CHECKSUM) { if (maxPointer + CodecUtil.footerLength() != fieldsStream.length()) { throw new CorruptIndexException("Invalid fieldsStream maxPointer (file truncated?): maxPointer=" + maxPointer + ", length=" + fieldsStream.length()); } } else { maxPointer = fieldsStream.length(); } this.maxPointer = maxPointer; final String codecNameDat = formatName + CODEC_SFX_DAT; final int fieldsVersion = CodecUtil.checkHeader(fieldsStream, codecNameDat, VERSION_START, VERSION_CURRENT); if (version != fieldsVersion) { throw new CorruptIndexException("Version mismatch between stored fields index and data: " + version + " != " + fieldsVersion); } assert CodecUtil.headerLength(codecNameDat) == fieldsStream.getFilePointer(); if (version >= VERSION_BIG_CHUNKS) { chunkSize = fieldsStream.readVInt(); } else { chunkSize = -1; } packedIntsVersion = fieldsStream.readVInt(); decompressor = compressionMode.newDecompressor(); this.bytes = new BytesRef(); if (version >= VERSION_CHECKSUM) { // NOTE: data file is too costly to verify checksum against all the bytes on open, // but for now we at least verify proper structure of the checksum footer: which looks // for FOOTER_MAGIC + algorithmID. This is cheap and can detect some forms of corruption // such as file truncation. CodecUtil.retrieveChecksum(fieldsStream); } success = true; } finally { if (!success) { IOUtils.closeWhileHandlingException(this, indexStream); } } }
From source file:com.rocana.lucene.codec.v1.RocanaBlockTreeTermsReader.java
License:Apache License
/** Sole constructor. */ public RocanaBlockTreeTermsReader(PostingsReaderBase postingsReader, SegmentReadState state) throws IOException { boolean success = false; IndexInput indexIn = null;//from www .j a v a2s. c o m this.postingsReader = postingsReader; this.segment = state.segmentInfo.name; String termsName = IndexFileNames.segmentFileName(segment, state.segmentSuffix, TERMS_EXTENSION); try { termsIn = state.directory.openInput(termsName, state.context); version = CodecUtil.checkIndexHeader(termsIn, TERMS_CODEC_NAME, VERSION_START, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix); if (version < VERSION_AUTO_PREFIX_TERMS) { // Old (pre-5.2.0) index, no auto-prefix terms: this.anyAutoPrefixTerms = false; } else if (version == VERSION_AUTO_PREFIX_TERMS) { // 5.2.x index, might have auto-prefix terms: this.anyAutoPrefixTerms = true; } else { // 5.3.x index, we record up front if we may have written any auto-prefix terms: assert version >= VERSION_AUTO_PREFIX_TERMS_COND; byte b = termsIn.readByte(); if (b == 0) { this.anyAutoPrefixTerms = false; } else if (b == 1) { this.anyAutoPrefixTerms = true; } else { throw new CorruptIndexException("invalid anyAutoPrefixTerms: expected 0 or 1 but got " + b, termsIn); } } String indexName = IndexFileNames.segmentFileName(segment, state.segmentSuffix, TERMS_INDEX_EXTENSION); indexIn = state.directory.openInput(indexName, state.context); CodecUtil.checkIndexHeader(indexIn, TERMS_INDEX_CODEC_NAME, version, version, state.segmentInfo.getId(), state.segmentSuffix); // IMPORTANT: comment out this one line to prevent checksumming the entire file. // This is the reason we have a custom Lucene codec and forked Lucene classes. //CodecUtil.checksumEntireFile(indexIn); // Have PostingsReader init itself postingsReader.init(termsIn, state); // NOTE: data file is too costly to verify checksum against all the bytes on open, // but for now we at least verify proper structure of the checksum footer: which looks // for FOOTER_MAGIC + algorithmID. This is cheap and can detect some forms of corruption // such as file truncation. CodecUtil.retrieveChecksum(termsIn); // Read per-field details seekDir(termsIn, dirOffset); seekDir(indexIn, indexDirOffset); final int numFields = termsIn.readVInt(); if (numFields < 0) { throw new CorruptIndexException("invalid numFields: " + numFields, termsIn); } for (int i = 0; i < numFields; ++i) { final int field = termsIn.readVInt(); final long numTerms = termsIn.readVLong(); if (numTerms <= 0) { throw new CorruptIndexException("Illegal numTerms for field number: " + field, termsIn); } final int numBytes = termsIn.readVInt(); if (numBytes < 0) { throw new CorruptIndexException( "invalid rootCode for field number: " + field + ", numBytes=" + numBytes, termsIn); } final BytesRef rootCode = new BytesRef(new byte[numBytes]); termsIn.readBytes(rootCode.bytes, 0, numBytes); rootCode.length = numBytes; final FieldInfo fieldInfo = state.fieldInfos.fieldInfo(field); if (fieldInfo == null) { throw new CorruptIndexException("invalid field number: " + field, termsIn); } final long sumTotalTermFreq = fieldInfo.getIndexOptions() == IndexOptions.DOCS ? -1 : termsIn.readVLong(); final long sumDocFreq = termsIn.readVLong(); final int docCount = termsIn.readVInt(); final int longsSize = termsIn.readVInt(); if (longsSize < 0) { throw new CorruptIndexException( "invalid longsSize for field: " + fieldInfo.name + ", longsSize=" + longsSize, termsIn); } BytesRef minTerm = readBytesRef(termsIn); BytesRef maxTerm = readBytesRef(termsIn); if (docCount < 0 || docCount > state.segmentInfo.maxDoc()) { // #docs with field must be <= #docs throw new CorruptIndexException( "invalid docCount: " + docCount + " maxDoc: " + state.segmentInfo.maxDoc(), termsIn); } if (sumDocFreq < docCount) { // #postings must be >= #docs with field throw new CorruptIndexException("invalid sumDocFreq: " + sumDocFreq + " docCount: " + docCount, termsIn); } if (sumTotalTermFreq != -1 && sumTotalTermFreq < sumDocFreq) { // #positions must be >= #postings throw new CorruptIndexException( "invalid sumTotalTermFreq: " + sumTotalTermFreq + " sumDocFreq: " + sumDocFreq, termsIn); } final long indexStartFP = indexIn.readVLong(); RocanaFieldReader previous = fields.put(fieldInfo.name, new RocanaFieldReader(this, fieldInfo, numTerms, rootCode, sumTotalTermFreq, sumDocFreq, docCount, indexStartFP, longsSize, indexIn, minTerm, maxTerm)); if (previous != null) { throw new CorruptIndexException("duplicate field: " + fieldInfo.name, termsIn); } } indexIn.close(); success = true; } finally { if (!success) { // this.close() will close in: IOUtils.closeWhileHandlingException(indexIn, this); } } }
From source file:org.apache.solr.handler.IndexFetcher.java
License:Apache License
private CompareResult compareFile(Directory indexDir, String filename, Long backupIndexFileLen, Long backupIndexFileChecksum) { CompareResult compareResult = new CompareResult(); try {//from w ww.j ava 2 s . c om try (final IndexInput indexInput = indexDir.openInput(filename, IOContext.READONCE)) { long indexFileLen = indexInput.length(); long indexFileChecksum = 0; if (backupIndexFileChecksum != null) { try { indexFileChecksum = CodecUtil.retrieveChecksum(indexInput); compareResult.checkSummed = true; } catch (Exception e) { LOG.warn("Could not retrieve checksum from file.", e); } } if (!compareResult.checkSummed) { // we don't have checksums to compare if (indexFileLen == backupIndexFileLen) { compareResult.equal = true; return compareResult; } else { LOG.warn("File {} did not match. expected length is {} and actual length is {}", filename, backupIndexFileLen, indexFileLen); compareResult.equal = false; return compareResult; } } // we have checksums to compare if (indexFileLen == backupIndexFileLen && indexFileChecksum == backupIndexFileChecksum) { compareResult.equal = true; return compareResult; } else { LOG.warn( "File {} did not match. expected checksum is {} and actual is checksum {}. " + "expected length is {} and actual length is {}", filename, backupIndexFileChecksum, indexFileChecksum, backupIndexFileLen, indexFileLen); compareResult.equal = false; return compareResult; } } } catch (NoSuchFileException | FileNotFoundException e) { compareResult.equal = false; return compareResult; } catch (IOException e) { LOG.error("Could not read file " + filename + ". Downloading it again", e); compareResult.equal = false; return compareResult; } }
From source file:org.apache.solr.handler.RestoreCore.java
License:Apache License
public boolean doRestore() throws Exception { URI backupPath = backupRepo.resolve(backupLocation, backupName); SimpleDateFormat dateFormat = new SimpleDateFormat(SnapShooter.DATE_FMT, Locale.ROOT); String restoreIndexName = "restore." + dateFormat.format(new Date()); String restoreIndexPath = core.getDataDir() + restoreIndexName; String indexDirPath = core.getIndexDir(); Directory restoreIndexDir = null;//www . j a v a 2 s . c o m Directory indexDir = null; try { restoreIndexDir = core.getDirectoryFactory().get(restoreIndexPath, DirectoryFactory.DirContext.DEFAULT, core.getSolrConfig().indexConfig.lockType); //Prefer local copy. indexDir = core.getDirectoryFactory().get(indexDirPath, DirectoryFactory.DirContext.DEFAULT, core.getSolrConfig().indexConfig.lockType); //Move all files from backupDir to restoreIndexDir for (String filename : backupRepo.listAll(backupPath)) { checkInterrupted(); log.info("Copying file {} to restore directory ", filename); try (IndexInput indexInput = backupRepo.openInput(backupPath, filename, IOContext.READONCE)) { Long checksum = null; try { checksum = CodecUtil.retrieveChecksum(indexInput); } catch (Exception e) { log.warn("Could not read checksum from index file: " + filename, e); } long length = indexInput.length(); IndexFetcher.CompareResult compareResult = IndexFetcher.compareFile(indexDir, filename, length, checksum); if (!compareResult.equal || (IndexFetcher.filesToAlwaysDownloadIfNoChecksums(filename, length, compareResult))) { backupRepo.copyFileTo(backupPath, filename, restoreIndexDir); } else { //prefer local copy restoreIndexDir.copyFrom(indexDir, filename, filename, IOContext.READONCE); } } catch (Exception e) { log.warn("Exception while restoring the backup index ", e); throw new SolrException(SolrException.ErrorCode.UNKNOWN, "Exception while restoring the backup index", e); } } log.debug("Switching directories"); core.modifyIndexProps(restoreIndexName); boolean success; try { core.getUpdateHandler().newIndexWriter(false); openNewSearcher(); success = true; log.info("Successfully restored to the backup index"); } catch (Exception e) { //Rollback to the old index directory. Delete the restore index directory and mark the restore as failed. log.warn("Could not switch to restored index. Rolling back to the current index", e); Directory dir = null; try { dir = core.getDirectoryFactory().get(core.getDataDir(), DirectoryFactory.DirContext.META_DATA, core.getSolrConfig().indexConfig.lockType); dir.deleteFile(IndexFetcher.INDEX_PROPERTIES); } finally { if (dir != null) { core.getDirectoryFactory().release(dir); } } core.getDirectoryFactory().doneWithDirectory(restoreIndexDir); core.getDirectoryFactory().remove(restoreIndexDir); core.getUpdateHandler().newIndexWriter(false); openNewSearcher(); throw new SolrException(SolrException.ErrorCode.UNKNOWN, "Exception while restoring the backup index", e); } if (success) { core.getDirectoryFactory().doneWithDirectory(indexDir); // Cleanup all index files not associated with any *named* snapshot. core.deleteNonSnapshotIndexFiles(indexDirPath); } return true; } finally { if (restoreIndexDir != null) { core.getDirectoryFactory().release(restoreIndexDir); } if (indexDir != null) { core.getDirectoryFactory().release(indexDir); } } }
From source file:org.elasticsearch.gateway.local.state.meta.MetaDataStateFormatTest.java
License:Apache License
public static void corruptFile(Path file, ESLogger logger) throws IOException { Path fileToCorrupt = file;//from w w w .j av a 2 s . co m try (final SimpleFSDirectory dir = new SimpleFSDirectory(fileToCorrupt.getParent().toFile())) { long checksumBeforeCorruption; try (IndexInput input = dir.openInput(fileToCorrupt.getFileName().toString(), IOContext.DEFAULT)) { checksumBeforeCorruption = CodecUtil.retrieveChecksum(input); } try (FileChannel raf = FileChannel.open(fileToCorrupt, StandardOpenOption.READ, StandardOpenOption.WRITE)) { raf.position(randomIntBetween(0, (int) Math.min(Integer.MAX_VALUE, raf.size() - 1))); long filePointer = raf.position(); ByteBuffer bb = ByteBuffer.wrap(new byte[1]); raf.read(bb); bb.flip(); byte oldValue = bb.get(0); byte newValue = (byte) ~oldValue; bb.put(0, newValue); raf.write(bb, filePointer); logger.debug("Corrupting file {} -- flipping at position {} from {} to {} ", fileToCorrupt.getFileName().toString(), filePointer, Integer.toHexString(oldValue), Integer.toHexString(newValue)); } long checksumAfterCorruption; long actualChecksumAfterCorruption; try (ChecksumIndexInput input = dir.openChecksumInput(fileToCorrupt.getFileName().toString(), IOContext.DEFAULT)) { assertThat(input.getFilePointer(), is(0l)); input.seek(input.length() - 8); // one long is the checksum... 8 bytes checksumAfterCorruption = input.getChecksum(); actualChecksumAfterCorruption = input.readLong(); } StringBuilder msg = new StringBuilder(); msg.append("Checksum before: [").append(checksumBeforeCorruption).append("]"); msg.append(" after: [").append(checksumAfterCorruption).append("]"); msg.append(" checksum value after corruption: ").append(actualChecksumAfterCorruption).append("]"); msg.append(" file: ").append(fileToCorrupt.getFileName().toString()).append(" length: ") .append(dir.fileLength(fileToCorrupt.getFileName().toString())); logger.debug(msg.toString()); assumeTrue("Checksum collision - " + msg.toString(), checksumAfterCorruption != checksumBeforeCorruption // collision || actualChecksumAfterCorruption != checksumBeforeCorruption); // checksum corrupted } }
From source file:org.elasticsearch.gateway.MetaDataStateFormatTest.java
License:Apache License
public static void corruptFile(Path file, ESLogger logger) throws IOException { Path fileToCorrupt = file;/* w w w. j a v a 2 s .c om*/ try (final SimpleFSDirectory dir = new SimpleFSDirectory(fileToCorrupt.getParent())) { long checksumBeforeCorruption; try (IndexInput input = dir.openInput(fileToCorrupt.getFileName().toString(), IOContext.DEFAULT)) { checksumBeforeCorruption = CodecUtil.retrieveChecksum(input); } try (FileChannel raf = FileChannel.open(fileToCorrupt, StandardOpenOption.READ, StandardOpenOption.WRITE)) { raf.position(randomIntBetween(0, (int) Math.min(Integer.MAX_VALUE, raf.size() - 1))); long filePointer = raf.position(); ByteBuffer bb = ByteBuffer.wrap(new byte[1]); raf.read(bb); bb.flip(); byte oldValue = bb.get(0); byte newValue = (byte) ~oldValue; bb.put(0, newValue); raf.write(bb, filePointer); logger.debug("Corrupting file {} -- flipping at position {} from {} to {} ", fileToCorrupt.getFileName().toString(), filePointer, Integer.toHexString(oldValue), Integer.toHexString(newValue)); } long checksumAfterCorruption; long actualChecksumAfterCorruption; try (ChecksumIndexInput input = dir.openChecksumInput(fileToCorrupt.getFileName().toString(), IOContext.DEFAULT)) { assertThat(input.getFilePointer(), is(0l)); input.seek(input.length() - 8); // one long is the checksum... 8 bytes checksumAfterCorruption = input.getChecksum(); actualChecksumAfterCorruption = input.readLong(); } StringBuilder msg = new StringBuilder(); msg.append("Checksum before: [").append(checksumBeforeCorruption).append("]"); msg.append(" after: [").append(checksumAfterCorruption).append("]"); msg.append(" checksum value after corruption: ").append(actualChecksumAfterCorruption).append("]"); msg.append(" file: ").append(fileToCorrupt.getFileName().toString()).append(" length: ") .append(dir.fileLength(fileToCorrupt.getFileName().toString())); logger.debug(msg.toString()); assumeTrue("Checksum collision - " + msg.toString(), checksumAfterCorruption != checksumBeforeCorruption // collision || actualChecksumAfterCorruption != checksumBeforeCorruption); // checksum corrupted } }
From source file:org.elasticsearch.gateway.MetaDataStateFormatTests.java
License:Apache License
public static void corruptFile(Path file, Logger logger) throws IOException { Path fileToCorrupt = file;/* w w w. j a va 2s . c o m*/ try (final SimpleFSDirectory dir = new SimpleFSDirectory(fileToCorrupt.getParent())) { long checksumBeforeCorruption; try (IndexInput input = dir.openInput(fileToCorrupt.getFileName().toString(), IOContext.DEFAULT)) { checksumBeforeCorruption = CodecUtil.retrieveChecksum(input); } try (FileChannel raf = FileChannel.open(fileToCorrupt, StandardOpenOption.READ, StandardOpenOption.WRITE)) { raf.position(randomIntBetween(0, (int) Math.min(Integer.MAX_VALUE, raf.size() - 1))); long filePointer = raf.position(); ByteBuffer bb = ByteBuffer.wrap(new byte[1]); raf.read(bb); bb.flip(); byte oldValue = bb.get(0); byte newValue = (byte) ~oldValue; bb.put(0, newValue); raf.write(bb, filePointer); logger.debug("Corrupting file {} -- flipping at position {} from {} to {} ", fileToCorrupt.getFileName().toString(), filePointer, Integer.toHexString(oldValue), Integer.toHexString(newValue)); } long checksumAfterCorruption; long actualChecksumAfterCorruption; try (ChecksumIndexInput input = dir.openChecksumInput(fileToCorrupt.getFileName().toString(), IOContext.DEFAULT)) { assertThat(input.getFilePointer(), is(0L)); input.seek(input.length() - 8); // one long is the checksum... 8 bytes checksumAfterCorruption = input.getChecksum(); actualChecksumAfterCorruption = input.readLong(); } StringBuilder msg = new StringBuilder(); msg.append("Checksum before: [").append(checksumBeforeCorruption).append("]"); msg.append(" after: [").append(checksumAfterCorruption).append("]"); msg.append(" checksum value after corruption: ").append(actualChecksumAfterCorruption).append("]"); msg.append(" file: ").append(fileToCorrupt.getFileName().toString()).append(" length: ") .append(dir.fileLength(fileToCorrupt.getFileName().toString())); logger.debug("{}", msg.toString()); assumeTrue("Checksum collision - " + msg.toString(), checksumAfterCorruption != checksumBeforeCorruption // collision || actualChecksumAfterCorruption != checksumBeforeCorruption); // checksum corrupted } }
From source file:org.elasticsearch.index.store.CorruptedFileIT.java
License:Apache License
private ShardRouting corruptRandomPrimaryFile(final boolean includePerCommitFiles) throws IOException { ClusterState state = client().admin().cluster().prepareState().get().getState(); GroupShardsIterator shardIterators = state.getRoutingNodes().getRoutingTable() .activePrimaryShardsGrouped(new String[] { "test" }, false); List<ShardIterator> iterators = iterableAsArrayList(shardIterators); ShardIterator shardIterator = RandomPicks.randomFrom(getRandom(), iterators); ShardRouting shardRouting = shardIterator.nextOrNull(); assertNotNull(shardRouting);/*from ww w . ja v a2 s . c o m*/ assertTrue(shardRouting.primary()); assertTrue(shardRouting.assignedToNode()); String nodeId = shardRouting.currentNodeId(); NodesStatsResponse nodeStatses = client().admin().cluster().prepareNodesStats(nodeId).setFs(true).get(); Set<Path> files = new TreeSet<>(); // treeset makes sure iteration order is deterministic for (FsInfo.Path info : nodeStatses.getNodes()[0].getFs()) { String path = info.getPath(); final String relativeDataLocationPath = "indices/test/" + Integer.toString(shardRouting.getId()) + "/index"; Path file = PathUtils.get(path).resolve(relativeDataLocationPath); if (Files.exists(file)) { // multi data path might only have one path in use try (DirectoryStream<Path> stream = Files.newDirectoryStream(file)) { for (Path item : stream) { if (Files.isRegularFile(item) && "write.lock".equals(item.getFileName().toString()) == false) { if (includePerCommitFiles || isPerSegmentFile(item.getFileName().toString())) { files.add(item); } } } } } } pruneOldDeleteGenerations(files); Path fileToCorrupt = null; if (!files.isEmpty()) { fileToCorrupt = RandomPicks.randomFrom(getRandom(), files); try (Directory dir = FSDirectory.open(fileToCorrupt.toAbsolutePath().getParent())) { long checksumBeforeCorruption; try (IndexInput input = dir.openInput(fileToCorrupt.getFileName().toString(), IOContext.DEFAULT)) { checksumBeforeCorruption = CodecUtil.retrieveChecksum(input); } try (FileChannel raf = FileChannel.open(fileToCorrupt, StandardOpenOption.READ, StandardOpenOption.WRITE)) { // read raf.position(randomIntBetween(0, (int) Math.min(Integer.MAX_VALUE, raf.size() - 1))); long filePointer = raf.position(); ByteBuffer bb = ByteBuffer.wrap(new byte[1]); raf.read(bb); bb.flip(); // corrupt byte oldValue = bb.get(0); byte newValue = (byte) (oldValue + 1); bb.put(0, newValue); // rewrite raf.position(filePointer); raf.write(bb); logger.info("Corrupting file for shard {} -- flipping at position {} from {} to {} file: {}", shardRouting, filePointer, Integer.toHexString(oldValue), Integer.toHexString(newValue), fileToCorrupt.getFileName()); } long checksumAfterCorruption; long actualChecksumAfterCorruption; try (ChecksumIndexInput input = dir.openChecksumInput(fileToCorrupt.getFileName().toString(), IOContext.DEFAULT)) { assertThat(input.getFilePointer(), is(0l)); input.seek(input.length() - 8); // one long is the checksum... 8 bytes checksumAfterCorruption = input.getChecksum(); actualChecksumAfterCorruption = input.readLong(); } // we need to add assumptions here that the checksums actually really don't match there is a small chance to get collisions // in the checksum which is ok though.... StringBuilder msg = new StringBuilder(); msg.append("Checksum before: [").append(checksumBeforeCorruption).append("]"); msg.append(" after: [").append(checksumAfterCorruption).append("]"); msg.append(" checksum value after corruption: ").append(actualChecksumAfterCorruption).append("]"); msg.append(" file: ").append(fileToCorrupt.getFileName()).append(" length: ") .append(dir.fileLength(fileToCorrupt.getFileName().toString())); logger.info(msg.toString()); assumeTrue("Checksum collision - " + msg.toString(), checksumAfterCorruption != checksumBeforeCorruption // collision || actualChecksumAfterCorruption != checksumBeforeCorruption); // checksum corrupted } } assertThat("no file corrupted", fileToCorrupt, notNullValue()); return shardRouting; }
From source file:org.elasticsearch.index.store.CorruptedFileTest.java
License:Apache License
private ShardRouting corruptRandomPrimaryFile(final boolean includePerCommitFiles) throws IOException { ClusterState state = client().admin().cluster().prepareState().get().getState(); GroupShardsIterator shardIterators = state.getRoutingNodes().getRoutingTable() .activePrimaryShardsGrouped(new String[] { "test" }, false); List<ShardIterator> iterators = Lists.newArrayList(shardIterators); ShardIterator shardIterator = RandomPicks.randomFrom(getRandom(), iterators); ShardRouting shardRouting = shardIterator.nextOrNull(); assertNotNull(shardRouting);/*from w w w .j a va 2 s. c om*/ assertTrue(shardRouting.primary()); assertTrue(shardRouting.assignedToNode()); String nodeId = shardRouting.currentNodeId(); NodesStatsResponse nodeStatses = client().admin().cluster().prepareNodesStats(nodeId).setFs(true).get(); Set<File> files = new TreeSet<>(); // treeset makes sure iteration order is deterministic for (FsStats.Info info : nodeStatses.getNodes()[0].getFs()) { String path = info.getPath(); final String relativeDataLocationPath = "indices/test/" + Integer.toString(shardRouting.getId()) + "/index"; File file = new File(path, relativeDataLocationPath); files.addAll(Arrays.asList(file.listFiles(new FileFilter() { @Override public boolean accept(File pathname) { if (pathname.isFile() && "write.lock".equals(pathname.getName()) == false) { return (includePerCommitFiles || isPerSegmentFile(pathname.getName())); } return false; // no dirs no write.locks } }))); } pruneOldDeleteGenerations(files); File fileToCorrupt = null; if (!files.isEmpty()) { fileToCorrupt = RandomPicks.randomFrom(getRandom(), files); try (Directory dir = FSDirectory.open(fileToCorrupt.getParentFile())) { long checksumBeforeCorruption; try (IndexInput input = dir.openInput(fileToCorrupt.getName(), IOContext.DEFAULT)) { checksumBeforeCorruption = CodecUtil.retrieveChecksum(input); } try (RandomAccessFile raf = new RandomAccessFile(fileToCorrupt, "rw")) { raf.seek(randomIntBetween(0, (int) Math.min(Integer.MAX_VALUE, raf.length() - 1))); long filePointer = raf.getFilePointer(); byte b = raf.readByte(); raf.seek(filePointer); int corruptedValue = (b + 1) & 0xff; raf.writeByte(corruptedValue); raf.getFD().sync(); logger.info("Corrupting file for shard {} -- flipping at position {} from {} to {} file: {}", shardRouting, filePointer, Integer.toHexString(b), Integer.toHexString(corruptedValue), fileToCorrupt.getName()); } long checksumAfterCorruption; long actualChecksumAfterCorruption; try (ChecksumIndexInput input = dir.openChecksumInput(fileToCorrupt.getName(), IOContext.DEFAULT)) { assertThat(input.getFilePointer(), is(0l)); input.seek(input.length() - 8); // one long is the checksum... 8 bytes checksumAfterCorruption = input.getChecksum(); actualChecksumAfterCorruption = input.readLong(); } // we need to add assumptions here that the checksums actually really don't match there is a small chance to get collisions // in the checksum which is ok though.... StringBuilder msg = new StringBuilder(); msg.append("Checksum before: [").append(checksumBeforeCorruption).append("]"); msg.append(" after: [").append(checksumAfterCorruption).append("]"); msg.append(" checksum value after corruption: ").append(actualChecksumAfterCorruption).append("]"); msg.append(" file: ").append(fileToCorrupt.getName()).append(" length: ") .append(dir.fileLength(fileToCorrupt.getName())); logger.info(msg.toString()); assumeTrue("Checksum collision - " + msg.toString(), checksumAfterCorruption != checksumBeforeCorruption // collision || actualChecksumAfterCorruption != checksumBeforeCorruption); // checksum corrupted } } assertThat("no file corrupted", fileToCorrupt, notNullValue()); return shardRouting; }
From source file:org.elasticsearch.index.store.StoreTest.java
License:Apache License
@Test public void testVerifyingIndexOutput() throws IOException { Directory dir = newDirectory();//from w w w . ja v a 2s. co m IndexOutput output = dir.createOutput("foo.bar", IOContext.DEFAULT); int iters = scaledRandomIntBetween(10, 100); for (int i = 0; i < iters; i++) { BytesRef bytesRef = new BytesRef(TestUtil.randomRealisticUnicodeString(random(), 10, 1024)); output.writeBytes(bytesRef.bytes, bytesRef.offset, bytesRef.length); } CodecUtil.writeFooter(output); output.close(); IndexInput indexInput = dir.openInput("foo.bar", IOContext.DEFAULT); String checksum = Store.digestToString(CodecUtil.retrieveChecksum(indexInput)); indexInput.seek(0); BytesRef ref = new BytesRef(scaledRandomIntBetween(1, 1024)); long length = indexInput.length(); IndexOutput verifyingOutput = new Store.LuceneVerifyingIndexOutput( new StoreFileMetaData("foo1.bar", length, checksum, TEST_VERSION_CURRENT), dir.createOutput("foo1.bar", IOContext.DEFAULT)); while (length > 0) { if (random().nextInt(10) == 0) { verifyingOutput.writeByte(indexInput.readByte()); length--; } else { int min = (int) Math.min(length, ref.bytes.length); indexInput.readBytes(ref.bytes, ref.offset, min); verifyingOutput.writeBytes(ref.bytes, ref.offset, min); length -= min; } } Store.verify(verifyingOutput); verifyingOutput.writeByte((byte) 0x0); try { Store.verify(verifyingOutput); fail("should be a corrupted index"); } catch (CorruptIndexException ex) { // ok } IOUtils.close(indexInput, verifyingOutput, dir); }