List of usage examples for org.apache.lucene.codecs CodecUtil writeFooter
public static void writeFooter(IndexOutput out) throws IOException
From source file:com.lucure.core.codec.CompressingStoredFieldsIndexWriter.java
License:Apache License
void finish(int numDocs, long maxPointer) throws IOException { if (numDocs != totalDocs) { throw new IllegalStateException("Expected " + numDocs + " docs, but got " + totalDocs); }// www . java 2s . c om if (blockChunks > 0) { writeBlock(); } fieldsIndexOut.writeVInt(0); // end marker fieldsIndexOut.writeVLong(maxPointer); CodecUtil.writeFooter(fieldsIndexOut); }
From source file:com.lucure.core.codec.CompressingStoredFieldsWriter.java
License:Apache License
@Override public void finish(FieldInfos fis, int numDocs) throws IOException { if (numBufferedDocs > 0) { flush();/*from ww w . j a v a2 s.c om*/ } else { assert bufferedDocs.length == 0; } if (docBase != numDocs) { throw new RuntimeException("Wrote " + docBase + " docs, finish called with numDocs=" + numDocs); } indexWriter.finish(numDocs, fieldsStream.getFilePointer()); CodecUtil.writeFooter(fieldsStream); assert bufferedDocs.length == 0; }
From source file:com.sindicetech.siren.index.codecs.block.BlockIndexOutput.java
License:Open Source License
/** * Writes a codec footer, which records both a checksum algorithm ID and a checksum, using * {@link org.apache.lucene.codecs.CodecUtil#writeFooter(org.apache.lucene.store.IndexOutput)} * * @throws java.io.IOException If there is an I/O error writing to the underlying medium. *//*from www. ja v a2s .c om*/ public void writeFooter() throws IOException { CodecUtil.writeFooter(out); }
From source file:com.sindicetech.siren.index.codecs.siren10.Siren10PostingsWriter.java
License:Open Source License
@Override public void close() throws IOException { boolean success = false; try {/*from w w w . j a va 2s . com*/ if (docOut != null) { docOut.writeFooter(); } if (skipOut != null) { CodecUtil.writeFooter(skipOut); } if (nodOut != null) { nodOut.writeFooter(); } if (posOut != null) { posOut.writeFooter(); } success = true; } finally { if (success) { IOUtils.close(docOut, skipOut, nodOut, posOut); } else { IOUtils.closeWhileHandlingException(docOut, skipOut, nodOut, posOut); } docOut = null; nodOut = null; posOut = null; skipOut = null; } }
From source file:com.vmware.xenon.services.common.Lucene60FieldInfosFormatWithCache.java
License:Open Source License
@Override public void write(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, FieldInfos infos, IOContext context) throws IOException { final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, EXTENSION); try (IndexOutput output = directory.createOutput(fileName, context)) { CodecUtil.writeIndexHeader(output, Lucene60FieldInfosFormatWithCache.CODEC_NAME, Lucene60FieldInfosFormatWithCache.FORMAT_CURRENT, segmentInfo.getId(), segmentSuffix); output.writeVInt(infos.size());//ww w. j av a2s . c o m for (FieldInfo fi : infos) { fi.checkConsistency(); output.writeString(fi.name); output.writeVInt(fi.number); byte bits = 0x0; if (fi.hasVectors()) { bits |= STORE_TERMVECTOR; } if (fi.omitsNorms()) { bits |= OMIT_NORMS; } if (fi.hasPayloads()) { bits |= STORE_PAYLOADS; } output.writeByte(bits); output.writeByte(indexOptionsByte(fi.getIndexOptions())); // pack the DV type and hasNorms in one byte output.writeByte(docValuesByte(fi.getDocValuesType())); output.writeLong(fi.getDocValuesGen()); output.writeMapOfStrings(fi.attributes()); int pointDimensionCount = fi.getPointDimensionCount(); output.writeVInt(pointDimensionCount); if (pointDimensionCount != 0) { output.writeVInt(fi.getPointNumBytes()); } } CodecUtil.writeFooter(output); } }
From source file:org.codelibs.elasticsearch.search.suggest.completion2x.AnalyzingCompletionLookupProvider.java
License:Apache License
@Override public FieldsConsumer consumer(final IndexOutput output) throws IOException { CodecUtil.writeHeader(output, CODEC_NAME, CODEC_VERSION_LATEST); return new FieldsConsumer() { private Map<String, Long> fieldOffsets = new HashMap<>(); @Override//from w ww. ja v a 2 s. c om public void close() throws IOException { try { /* * write the offsets per field such that we know where * we need to load the FSTs from */ long pointer = output.getFilePointer(); output.writeVInt(fieldOffsets.size()); for (Map.Entry<String, Long> entry : fieldOffsets.entrySet()) { output.writeString(entry.getKey()); output.writeVLong(entry.getValue()); } output.writeLong(pointer); CodecUtil.writeFooter(output); } finally { IOUtils.close(output); } } @Override public void write(Fields fields) throws IOException { for (String field : fields) { Terms terms = fields.terms(field); if (terms == null) { continue; } terms.iterator(); new SuggestPayload(); throw new UnsupportedOperationException("QueryBuilders does not support this operation."); // final XAnalyzingSuggester.XBuilder builder = new XAnalyzingSuggester.XBuilder( // maxSurfaceFormsPerAnalyzedForm, hasPayloads, XAnalyzingSuggester.PAYLOAD_SEP); // int docCount = 0; // while (true) { // BytesRef term = termsEnum.next(); // if (term == null) { // break; // } // docsEnum = termsEnum.postings(docsEnum, PostingsEnum.PAYLOADS); // builder.startTerm(term); // int docFreq = 0; // while (docsEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) { // for (int i = 0; i < docsEnum.freq(); i++) { // final int position = docsEnum.nextPosition(); // AnalyzingCompletionLookupProvider.this.parsePayload(docsEnum.getPayload(), spare); // builder.addSurface(spare.surfaceForm.get(), spare.payload.get(), spare.weight); // // multi fields have the same surface form so we sum up here // maxAnalyzedPathsForOneInput = Math.max(maxAnalyzedPathsForOneInput, position + 1); // } // docFreq++; // docCount = Math.max(docCount, docsEnum.docID()+1); // } // builder.finishTerm(docFreq); // } // /* // * Here we are done processing the field and we can // * buid the FST and write it to disk. // */ // FST<Pair<Long, BytesRef>> build = builder.build(); // assert build != null || docCount == 0: "the FST is null but docCount is != 0 actual value: [" + docCount + "]"; // /* // * it's possible that the FST is null if we have 2 segments that get merged // * and all docs that have a value in this field are deleted. This will cause // * a consumer to be created but it doesn't consume any values causing the FSTBuilder // * to return null. // */ // if (build != null) { // fieldOffsets.put(field, output.getFilePointer()); // build.save(output); // /* write some more meta-info */ // output.writeVInt(maxAnalyzedPathsForOneInput); // output.writeVInt(maxSurfaceFormsPerAnalyzedForm); // output.writeInt(maxGraphExpansions); // can be negative // int options = 0; // options |= preserveSep ? SERIALIZE_PRESERVE_SEPARATORS : 0; // options |= hasPayloads ? SERIALIZE_HAS_PAYLOADS : 0; // options |= preservePositionIncrements ? SERIALIZE_PRESERVE_POSITION_INCREMENTS : 0; // output.writeVInt(options); // output.writeVInt(XAnalyzingSuggester.SEP_LABEL); // output.writeVInt(XAnalyzingSuggester.END_BYTE); // output.writeVInt(XAnalyzingSuggester.PAYLOAD_SEP); // output.writeVInt(XAnalyzingSuggester.HOLE_CHARACTER); // } } } }; }
From source file:org.elasticsearch.common.settings.KeyStoreWrapper.java
License:Apache License
/** Write the keystore to the given config directory. */ void save(Path configDir) throws Exception { char[] password = this.keystorePassword.get().getPassword(); SimpleFSDirectory directory = new SimpleFSDirectory(configDir); // write to tmp file first, then overwrite String tmpFile = KEYSTORE_FILENAME + ".tmp"; try (IndexOutput output = directory.createOutput(tmpFile, IOContext.DEFAULT)) { CodecUtil.writeHeader(output, KEYSTORE_FILENAME, FORMAT_VERSION); output.writeByte(password.length == 0 ? (byte) 0 : (byte) 1); output.writeString(type);/*ww w. j av a 2s .c o m*/ output.writeString(secretFactory.getAlgorithm()); ByteArrayOutputStream keystoreBytesStream = new ByteArrayOutputStream(); keystore.get().store(keystoreBytesStream, password); byte[] keystoreBytes = keystoreBytesStream.toByteArray(); output.writeInt(keystoreBytes.length); output.writeBytes(keystoreBytes, keystoreBytes.length); CodecUtil.writeFooter(output); } Path keystoreFile = keystorePath(configDir); Files.move(configDir.resolve(tmpFile), keystoreFile, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); PosixFileAttributeView attrs = Files.getFileAttributeView(keystoreFile, PosixFileAttributeView.class); if (attrs != null) { // don't rely on umask: ensure the keystore has minimal permissions attrs.setPermissions(PosixFilePermissions.fromString("rw-------")); } }
From source file:org.elasticsearch.gateway.local.state.meta.MetaDataStateFormat.java
License:Apache License
/** * Writes the given state to the given directories. The state is written to a * state directory ({@value #STATE_DIR_NAME}) underneath each of the given file locations and is created if it * doesn't exist. The state is serialized to a temporary file in that directory and is then atomically moved to * it's target filename of the pattern <tt>{prefix}{version}.st</tt>. * * @param state the state object to write * @param version the version of the state * @param locations the locations where the state should be written to. * @throws IOException if an IOException occurs *//* w w w . j a v a2s . c o m*/ public final void write(final T state, final long version, final File... locations) throws IOException { Preconditions.checkArgument(locations != null, "Locations must not be null"); Preconditions.checkArgument(locations.length > 0, "One or more locations required"); final long maxStateId = findMaxStateId(prefix, locations) + 1; assert maxStateId >= 0 : "maxStateId must be positive but was: [" + maxStateId + "]"; final String fileName = prefix + maxStateId + STATE_FILE_EXTENSION; Path stateLocation = Paths.get(locations[0].getPath(), STATE_DIR_NAME); Files.createDirectories(stateLocation); final Path tmpStatePath = stateLocation.resolve(fileName + ".tmp"); final Path finalStatePath = stateLocation.resolve(fileName); try { try (OutputStreamIndexOutput out = new OutputStreamIndexOutput(Files.newOutputStream(tmpStatePath), BUFFER_SIZE)) { CodecUtil.writeHeader(out, STATE_FILE_CODEC, STATE_FILE_VERSION); out.writeInt(format.index()); out.writeLong(version); try (XContentBuilder builder = newXContentBuilder(format, new org.elasticsearch.common.lucene.store.OutputStreamIndexOutput(out) { @Override public void close() throws IOException { // this is important since some of the XContentBuilders write bytes on close. // in order to write the footer we need to prevent closing the actual index input. } })) { builder.startObject(); { toXContent(builder, state); } builder.endObject(); } CodecUtil.writeFooter(out); } IOUtils.fsync(tmpStatePath.toFile(), false); // fsync the state file Files.move(tmpStatePath, finalStatePath, StandardCopyOption.ATOMIC_MOVE); IOUtils.fsync(stateLocation.toFile(), true); for (int i = 1; i < locations.length; i++) { stateLocation = Paths.get(locations[i].getPath(), STATE_DIR_NAME); Files.createDirectories(stateLocation); Path tmpPath = stateLocation.resolve(fileName + ".tmp"); Path finalPath = stateLocation.resolve(fileName); try { Files.copy(finalStatePath, tmpPath); Files.move(tmpPath, finalPath, StandardCopyOption.ATOMIC_MOVE); // we are on the same FileSystem / Partition here we can do an atomic move IOUtils.fsync(stateLocation.toFile(), true); // we just fsync the dir here.. } finally { Files.deleteIfExists(tmpPath); } } } finally { Files.deleteIfExists(tmpStatePath); } cleanupOldFiles(prefix, fileName, locations); }
From source file:org.elasticsearch.gateway.MetaDataStateFormat.java
License:Apache License
/** * Writes the given state to the given directories. The state is written to a * state directory ({@value #STATE_DIR_NAME}) underneath each of the given file locations and is created if it * doesn't exist. The state is serialized to a temporary file in that directory and is then atomically moved to * it's target filename of the pattern <tt>{prefix}{version}.st</tt>. * * @param state the state object to write * @param version the version of the state * @param locations the locations where the state should be written to. * @throws IOException if an IOException occurs */// w w w. jav a2 s . co m public final void write(final T state, final long version, final Path... locations) throws IOException { Preconditions.checkArgument(locations != null, "Locations must not be null"); Preconditions.checkArgument(locations.length > 0, "One or more locations required"); final long maxStateId = findMaxStateId(prefix, locations) + 1; assert maxStateId >= 0 : "maxStateId must be positive but was: [" + maxStateId + "]"; final String fileName = prefix + maxStateId + STATE_FILE_EXTENSION; Path stateLocation = locations[0].resolve(STATE_DIR_NAME); Files.createDirectories(stateLocation); final Path tmpStatePath = stateLocation.resolve(fileName + ".tmp"); final Path finalStatePath = stateLocation.resolve(fileName); try { final String resourceDesc = "MetaDataStateFormat.write(path=\"" + tmpStatePath + "\")"; try (OutputStreamIndexOutput out = new OutputStreamIndexOutput(resourceDesc, Files.newOutputStream(tmpStatePath), BUFFER_SIZE)) { CodecUtil.writeHeader(out, STATE_FILE_CODEC, STATE_FILE_VERSION); out.writeInt(format.index()); out.writeLong(version); try (XContentBuilder builder = newXContentBuilder(format, new IndexOutputOutputStream(out) { @Override public void close() throws IOException { // this is important since some of the XContentBuilders write bytes on close. // in order to write the footer we need to prevent closing the actual index input. } })) { builder.startObject(); { toXContent(builder, state); } builder.endObject(); } CodecUtil.writeFooter(out); } IOUtils.fsync(tmpStatePath, false); // fsync the state file Files.move(tmpStatePath, finalStatePath, StandardCopyOption.ATOMIC_MOVE); IOUtils.fsync(stateLocation, true); for (int i = 1; i < locations.length; i++) { stateLocation = locations[i].resolve(STATE_DIR_NAME); Files.createDirectories(stateLocation); Path tmpPath = stateLocation.resolve(fileName + ".tmp"); Path finalPath = stateLocation.resolve(fileName); try { Files.copy(finalStatePath, tmpPath); Files.move(tmpPath, finalPath, StandardCopyOption.ATOMIC_MOVE); // we are on the same FileSystem / Partition here we can do an atomic move IOUtils.fsync(stateLocation, true); // we just fsync the dir here.. } finally { Files.deleteIfExists(tmpPath); } } } finally { Files.deleteIfExists(tmpStatePath); } cleanupOldFiles(prefix, fileName, locations); }
From source file:org.elasticsearch.index.shard.StoreRecoveryTests.java
License:Apache License
public void testStatsDirWrapper() throws IOException { Directory dir = newDirectory();/* w ww . j a v a2 s .com*/ Directory target = newDirectory(); RecoveryState.Index indexStats = new RecoveryState.Index(); StoreRecovery.StatsDirectoryWrapper wrapper = new StoreRecovery.StatsDirectoryWrapper(target, indexStats); try (IndexOutput output = dir.createOutput("foo.bar", IOContext.DEFAULT)) { CodecUtil.writeHeader(output, "foo", 0); int numBytes = randomIntBetween(100, 20000); for (int i = 0; i < numBytes; i++) { output.writeByte((byte) i); } CodecUtil.writeFooter(output); } wrapper.copyFrom(dir, "foo.bar", "bar.foo", IOContext.DEFAULT); assertNotNull(indexStats.getFileDetails("bar.foo")); assertNull(indexStats.getFileDetails("foo.bar")); assertEquals(dir.fileLength("foo.bar"), indexStats.getFileDetails("bar.foo").length()); assertEquals(dir.fileLength("foo.bar"), indexStats.getFileDetails("bar.foo").recovered()); assertFalse(indexStats.getFileDetails("bar.foo").reused()); IOUtils.close(dir, target); }