List of usage examples for org.apache.lucene.codecs CodecUtil checkHeader
public static int checkHeader(DataInput in, String codec, int minVersion, int maxVersion) throws IOException
From source file:com.github.cstoku.neologd.unidic.lucene.analysis.ja.dict.BinaryDictionary.java
License:Apache License
protected BinaryDictionary() throws IOException { InputStream mapIS = null, dictIS = null, posIS = null; int[] targetMapOffsets = null, targetMap = null; String[] posDict = null;//from w w w . jav a 2s. c o m String[] inflFormDict = null; String[] inflTypeDict = null; ByteBuffer buffer = null; boolean success = false; try { mapIS = getResource(TARGETMAP_FILENAME_SUFFIX); mapIS = new BufferedInputStream(mapIS); DataInput in = new InputStreamDataInput(mapIS); CodecUtil.checkHeader(in, TARGETMAP_HEADER, VERSION, VERSION); targetMap = new int[in.readVInt()]; targetMapOffsets = new int[in.readVInt()]; int accum = 0, sourceId = 0; for (int ofs = 0; ofs < targetMap.length; ofs++) { final int val = in.readVInt(); if ((val & 0x01) != 0) { targetMapOffsets[sourceId] = ofs; sourceId++; } accum += val >>> 1; targetMap[ofs] = accum; } if (sourceId + 1 != targetMapOffsets.length) throw new IOException("targetMap file format broken"); targetMapOffsets[sourceId] = targetMap.length; mapIS.close(); mapIS = null; posIS = getResource(POSDICT_FILENAME_SUFFIX); posIS = new BufferedInputStream(posIS); in = new InputStreamDataInput(posIS); CodecUtil.checkHeader(in, POSDICT_HEADER, VERSION, VERSION); int posSize = in.readVInt(); posDict = new String[posSize]; inflTypeDict = new String[posSize]; inflFormDict = new String[posSize]; for (int j = 0; j < posSize; j++) { posDict[j] = in.readString(); inflTypeDict[j] = in.readString(); inflFormDict[j] = in.readString(); // this is how we encode null inflections if (inflTypeDict[j].length() == 0) { inflTypeDict[j] = null; } if (inflFormDict[j].length() == 0) { inflFormDict[j] = null; } } posIS.close(); posIS = null; dictIS = getResource(DICT_FILENAME_SUFFIX); // no buffering here, as we load in one large buffer in = new InputStreamDataInput(dictIS); CodecUtil.checkHeader(in, DICT_HEADER, VERSION, VERSION); final int size = in.readVInt(); final ByteBuffer tmpBuffer = ByteBuffer.allocateDirect(size); final ReadableByteChannel channel = Channels.newChannel(dictIS); final int read = channel.read(tmpBuffer); if (read != size) { throw new EOFException("Cannot read whole dictionary"); } dictIS.close(); dictIS = null; buffer = tmpBuffer.asReadOnlyBuffer(); success = true; } finally { if (success) { IOUtils.close(mapIS, posIS, dictIS); } else { IOUtils.closeWhileHandlingException(mapIS, posIS, dictIS); } } this.targetMap = targetMap; this.targetMapOffsets = targetMapOffsets; this.posDict = posDict; this.inflTypeDict = inflTypeDict; this.inflFormDict = inflFormDict; this.buffer = buffer; }
From source file:com.github.cstoku.neologd.unidic.lucene.analysis.ja.dict.CharacterDefinition.java
License:Apache License
private CharacterDefinition() throws IOException { InputStream is = null;/*from w w w . ja v a 2 s . c o m*/ boolean success = false; try { is = BinaryDictionary.getClassResource(getClass(), FILENAME_SUFFIX); is = new BufferedInputStream(is); final DataInput in = new InputStreamDataInput(is); CodecUtil.checkHeader(in, HEADER, VERSION, VERSION); in.readBytes(characterCategoryMap, 0, characterCategoryMap.length); for (int i = 0; i < CLASS_COUNT; i++) { final byte b = in.readByte(); invokeMap[i] = (b & 0x01) != 0; groupMap[i] = (b & 0x02) != 0; } success = true; } finally { if (success) { IOUtils.close(is); } else { IOUtils.closeWhileHandlingException(is); } } }
From source file:com.github.cstoku.neologd.unidic.lucene.analysis.ja.dict.ConnectionCosts.java
License:Apache License
private ConnectionCosts() throws IOException { InputStream is = null;//from ww w . j a v a 2 s. com short[][] costs = null; boolean success = false; try { is = BinaryDictionary.getClassResource(getClass(), FILENAME_SUFFIX); is = new BufferedInputStream(is); final DataInput in = new InputStreamDataInput(is); CodecUtil.checkHeader(in, HEADER, VERSION, VERSION); int forwardSize = in.readVInt(); int backwardSize = in.readVInt(); costs = new short[backwardSize][forwardSize]; int accum = 0; for (int j = 0; j < costs.length; j++) { final short[] a = costs[j]; for (int i = 0; i < a.length; i++) { accum += in.readZInt(); a[i] = (short) accum; } } success = true; } finally { if (success) { IOUtils.close(is); } else { IOUtils.closeWhileHandlingException(is); } } this.costs = costs; }
From source file:com.lucure.core.codec.CompressingStoredFieldsReader.java
License:Apache License
/** Sole constructor. */ public CompressingStoredFieldsReader(Directory d, SegmentInfo si, String segmentSuffix, FieldInfos fn, IOContext context, String formatName, CompressionMode compressionMode) throws IOException { this.compressionMode = compressionMode; final String segment = si.name; boolean success = false; fieldInfos = fn;/* w w w. ja v a2 s . c o m*/ numDocs = si.getDocCount(); ChecksumIndexInput indexStream = null; try { final String indexStreamFN = IndexFileNames.segmentFileName(segment, segmentSuffix, FIELDS_INDEX_EXTENSION); final String fieldsStreamFN = IndexFileNames.segmentFileName(segment, segmentSuffix, FIELDS_EXTENSION); // Load the index into memory indexStream = d.openChecksumInput(indexStreamFN, context); final String codecNameIdx = formatName + CODEC_SFX_IDX; version = CodecUtil.checkHeader(indexStream, codecNameIdx, VERSION_START, VERSION_CURRENT); assert CodecUtil.headerLength(codecNameIdx) == indexStream.getFilePointer(); indexReader = new CompressingStoredFieldsIndexReader(indexStream, si); long maxPointer = -1; if (version >= VERSION_CHECKSUM) { maxPointer = indexStream.readVLong(); CodecUtil.checkFooter(indexStream); } else { CodecUtil.checkEOF(indexStream); } indexStream.close(); indexStream = null; // Open the data file and read metadata fieldsStream = d.openInput(fieldsStreamFN, context); if (version >= VERSION_CHECKSUM) { if (maxPointer + CodecUtil.footerLength() != fieldsStream.length()) { throw new CorruptIndexException("Invalid fieldsStream maxPointer (file truncated?): maxPointer=" + maxPointer + ", length=" + fieldsStream.length()); } } else { maxPointer = fieldsStream.length(); } this.maxPointer = maxPointer; final String codecNameDat = formatName + CODEC_SFX_DAT; final int fieldsVersion = CodecUtil.checkHeader(fieldsStream, codecNameDat, VERSION_START, VERSION_CURRENT); if (version != fieldsVersion) { throw new CorruptIndexException("Version mismatch between stored fields index and data: " + version + " != " + fieldsVersion); } assert CodecUtil.headerLength(codecNameDat) == fieldsStream.getFilePointer(); if (version >= VERSION_BIG_CHUNKS) { chunkSize = fieldsStream.readVInt(); } else { chunkSize = -1; } packedIntsVersion = fieldsStream.readVInt(); decompressor = compressionMode.newDecompressor(); this.bytes = new BytesRef(); if (version >= VERSION_CHECKSUM) { // NOTE: data file is too costly to verify checksum against all the bytes on open, // but for now we at least verify proper structure of the checksum footer: which looks // for FOOTER_MAGIC + algorithmID. This is cheap and can detect some forms of corruption // such as file truncation. CodecUtil.retrieveChecksum(fieldsStream); } success = true; } finally { if (!success) { IOUtils.closeWhileHandlingException(this, indexStream); } } }
From source file:com.sindicetech.siren.index.codecs.siren10.Siren10PostingsReader.java
License:Open Source License
@Override public void init(final IndexInput termsIn) throws IOException { // Make sure we are talking to the matching past writer CodecUtil.checkHeader(termsIn, Siren10PostingsWriter.CODEC, Siren10PostingsWriter.VERSION_START, Siren10PostingsWriter.VERSION_CURRENT); blockSkipInterval = termsIn.readInt(); maxSkipLevels = termsIn.readInt();/*from w w w . ja va2 s. c o m*/ blockSkipMinimum = termsIn.readInt(); maxBlockSize = termsIn.readInt(); }
From source file:org.apache.blur.lucene.codec.Blur022SegmentInfoReader.java
License:Apache License
@Override public SegmentInfo read(Directory dir, String segment, IOContext context) throws IOException { final String fileName = IndexFileNames.segmentFileName(segment, "", Blur022SegmentInfoFormat.SI_EXTENSION); final IndexInput input = dir.openInput(fileName, context); boolean success = false; try {//from w w w .j a va 2 s.c o m CodecUtil.checkHeader(input, Blur022SegmentInfoFormat.CODEC_NAME, Blur022SegmentInfoFormat.VERSION_START, Blur022SegmentInfoFormat.VERSION_CURRENT); final String version = input.readString(); final int docCount = input.readInt(); if (docCount < 0) { throw new CorruptIndexException("invalid docCount: " + docCount + " (resource=" + input + ")"); } final boolean isCompoundFile = input.readByte() == SegmentInfo.YES; final Map<String, String> diagnostics = input.readStringStringMap(); final Map<String, String> attributes = input.readStringStringMap(); final Set<String> files = input.readStringSet(); if (input.getFilePointer() != input.length()) { throw new CorruptIndexException("did not read all bytes from file \"" + fileName + "\": read " + input.getFilePointer() + " vs size " + input.length() + " (resource: " + input + ")"); } final SegmentInfo si = new SegmentInfo(dir, version, segment, docCount, isCompoundFile, null, diagnostics, Collections.unmodifiableMap(attributes)); si.setFiles(files); success = true; return si; } finally { if (!success) { IOUtils.closeWhileHandlingException(input); } else { input.close(); } } }
From source file:org.apache.blur.lucene.codec.DiskDocValuesProducer.java
License:Apache License
DiskDocValuesProducer(SegmentReadState state, String dataCodec, String dataExtension, String metaCodec,
String metaExtension) throws IOException {
String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix,
metaExtension);//from w w w .ja v a 2s . c o m
// read in the entries from the metadata file.
IndexInput in = state.directory.openInput(metaName, state.context);
boolean success = false;
try {
CodecUtil.checkHeader(in, metaCodec, DiskDocValuesFormat.VERSION_START,
DiskDocValuesFormat.VERSION_START);
numerics = new HashMap<Integer, NumericEntry>();
ords = new HashMap<Integer, NumericEntry>();
ordIndexes = new HashMap<Integer, NumericEntry>();
binaries = new HashMap<Integer, BinaryEntry>();
readFields(in, state.fieldInfos);
success = true;
} finally {
if (success) {
IOUtils.close(in);
} else {
IOUtils.closeWhileHandlingException(in);
}
}
String dataName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix,
dataExtension);
data = state.directory.openInput(dataName, state.context);
CodecUtil.checkHeader(data, dataCodec, DiskDocValuesFormat.VERSION_START,
DiskDocValuesFormat.VERSION_START);
}
From source file:org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager.java
License:Apache License
/** * Reads the snapshot meta-data information from the given {@link Directory}. *//*from w ww . j a va2s . c om*/ private synchronized void loadFromSnapshotMetadataFile() throws IOException { log.debug("Loading from snapshot metadata file..."); long genLoaded = -1; IOException ioe = null; List<String> snapshotFiles = new ArrayList<>(); for (String file : dir.listAll()) { if (file.startsWith(SNAPSHOTS_PREFIX)) { long gen = Long.parseLong(file.substring(SNAPSHOTS_PREFIX.length())); if (genLoaded == -1 || gen > genLoaded) { snapshotFiles.add(file); Map<String, SnapshotMetaData> snapshotMetaDataMapping = new HashMap<>(); IndexInput in = dir.openInput(file, IOContext.DEFAULT); try { CodecUtil.checkHeader(in, CODEC_NAME, VERSION_START, VERSION_START); int count = in.readVInt(); for (int i = 0; i < count; i++) { String name = in.readString(); String indexDirPath = in.readString(); long commitGen = in.readVLong(); snapshotMetaDataMapping.put(name, new SnapshotMetaData(name, indexDirPath, commitGen)); } } catch (IOException ioe2) { // Save first exception & throw in the end if (ioe == null) { ioe = ioe2; } } finally { in.close(); } genLoaded = gen; nameToDetailsMapping.clear(); nameToDetailsMapping.putAll(snapshotMetaDataMapping); } } } if (genLoaded == -1) { // Nothing was loaded... if (ioe != null) { // ... not for lack of trying: throw ioe; } } else { if (snapshotFiles.size() > 1) { // Remove any broken / old snapshot files: String curFileName = SNAPSHOTS_PREFIX + genLoaded; for (String file : snapshotFiles) { if (!curFileName.equals(file)) { IOUtils.deleteFilesIgnoringExceptions(dir, file); } } } nextWriteGen = 1 + genLoaded; } }
From source file:org.elasticsearch.common.settings.KeyStoreWrapper.java
License:Apache License
/** * Loads information about the Elasticsearch keystore from the provided config directory. * * {@link #decrypt(char[])} must be called before reading or writing any entries. * Returns {@code null} if no keystore exists. *///from ww w . j ava 2 s . c om public static KeyStoreWrapper load(Path configDir) throws IOException { Path keystoreFile = keystorePath(configDir); if (Files.exists(keystoreFile) == false) { return null; } SimpleFSDirectory directory = new SimpleFSDirectory(configDir); try (IndexInput indexInput = directory.openInput(KEYSTORE_FILENAME, IOContext.READONCE)) { ChecksumIndexInput input = new BufferedChecksumIndexInput(indexInput); CodecUtil.checkHeader(input, KEYSTORE_FILENAME, FORMAT_VERSION, FORMAT_VERSION); byte hasPasswordByte = input.readByte(); boolean hasPassword = hasPasswordByte == 1; if (hasPassword == false && hasPasswordByte != 0) { throw new IllegalStateException( "hasPassword boolean is corrupt: " + String.format(Locale.ROOT, "%02x", hasPasswordByte)); } String type = input.readString(); String secretKeyAlgo = input.readString(); byte[] keystoreBytes = new byte[input.readInt()]; input.readBytes(keystoreBytes, 0, keystoreBytes.length); CodecUtil.checkFooter(input); return new KeyStoreWrapper(hasPassword, type, secretKeyAlgo, keystoreBytes); } }
From source file:org.elasticsearch.gateway.local.state.meta.MetaDataStateFormat.java
License:Apache License
/** * Reads the state from a given file and compares the expected version against the actual version of * the state./*from w w w.jav a 2 s.c o m*/ */ public final T read(File file) throws IOException { try (Directory dir = newDirectory(file.getParentFile())) { try (final IndexInput indexInput = dir.openInput(file.getName(), IOContext.DEFAULT)) { // We checksum the entire file before we even go and parse it. If it's corrupted we barf right here. CodecUtil.checksumEntireFile(indexInput); CodecUtil.checkHeader(indexInput, STATE_FILE_CODEC, STATE_FILE_VERSION, STATE_FILE_VERSION); final XContentType xContentType = XContentType.values()[indexInput.readInt()]; indexInput.readLong(); // version currently unused long filePointer = indexInput.getFilePointer(); long contentSize = indexInput.length() - CodecUtil.footerLength() - filePointer; try (IndexInput slice = indexInput.slice("state_xcontent", filePointer, contentSize)) { try (XContentParser parser = XContentFactory.xContent(xContentType) .createParser(new InputStreamIndexInput(slice, contentSize))) { return fromXContent(parser); } } } catch (CorruptIndexException ex) { // we trick this into a dedicated exception with the original stacktrace throw new CorruptStateException(ex); } } }