List of usage examples for org.apache.lucene.index SegmentReader getFieldsReader
@Override
public StoredFieldsReader getFieldsReader()
From source file:com.lucure.core.codec.CompressingStoredFieldsWriter.java
License:Apache License
@Override public int merge(MergeState mergeState) throws IOException { int docCount = 0; int idx = 0;// w w w . j a v a 2 s.c o m AccessFilteredDocsAndPositionsEnum.enableMergeAuthorizations(); for (AtomicReader reader : mergeState.readers) { final SegmentReader matchingSegmentReader = mergeState.matchingSegmentReaders[idx++]; CompressingStoredFieldsReader matchingFieldsReader = null; if (matchingSegmentReader != null) { final StoredFieldsReader fieldsReader = matchingSegmentReader.getFieldsReader(); // we can only bulk-copy if the matching reader is also a CompressingStoredFieldsReader if (fieldsReader != null && fieldsReader instanceof CompressingStoredFieldsReader) { matchingFieldsReader = (CompressingStoredFieldsReader) fieldsReader; } } final int maxDoc = reader.maxDoc(); final Bits liveDocs = reader.getLiveDocs(); if (matchingFieldsReader == null || matchingFieldsReader.getVersion() != VERSION_CURRENT // means reader version is not the same as the writer version || matchingFieldsReader.getCompressionMode() != compressionMode || matchingFieldsReader.getChunkSize() != chunkSize) { // the way data is decompressed depends on the chunk size // naive merge... for (int i = nextLiveDoc(0, liveDocs, maxDoc); i < maxDoc; i = nextLiveDoc(i + 1, liveDocs, maxDoc)) { Document doc = reader.document(i); addDocument(doc, mergeState.fieldInfos); ++docCount; mergeState.checkAbort.work(300); } } else { int docID = nextLiveDoc(0, liveDocs, maxDoc); if (docID < maxDoc) { // not all docs were deleted final CompressingStoredFieldsReader.ChunkIterator it = matchingFieldsReader .chunkIterator(docID); int[] startOffsets = new int[0]; do { // go to the next chunk that contains docID it.next(docID); // transform lengths into offsets if (startOffsets.length < it.chunkDocs) { startOffsets = new int[ArrayUtil.oversize(it.chunkDocs, 4)]; } for (int i = 1; i < it.chunkDocs; ++i) { startOffsets[i] = startOffsets[i - 1] + it.lengths[i - 1]; } // decompress it.decompress(); if (startOffsets[it.chunkDocs - 1] + it.lengths[it.chunkDocs - 1] != it.bytes.length) { throw new CorruptIndexException( "Corrupted: expected chunk size=" + startOffsets[it.chunkDocs - 1] + it.lengths[it.chunkDocs - 1] + ", got " + it.bytes.length); } // copy non-deleted docs for (; docID < it.docBase + it.chunkDocs; docID = nextLiveDoc(docID + 1, liveDocs, maxDoc)) { final int diff = docID - it.docBase; startDocument(); bufferedDocs.writeBytes(it.bytes.bytes, it.bytes.offset + startOffsets[diff], it.lengths[diff]); numStoredFieldsInDoc = it.numStoredFields[diff]; finishDocument(); ++docCount; mergeState.checkAbort.work(300); } } while (docID < maxDoc); it.checkIntegrity(); } } } AccessFilteredDocsAndPositionsEnum.disableMergeAuthorizations(); finish(mergeState.fieldInfos, docCount); return docCount; }
From source file:org.apache.solr.codecs.onsql.ONSQLStoredFieldsWriter.java
License:Apache License
@Override public int merge(MergeState mergeState) throws IOException { log.debug("merge has been called"); // check for our primary key completeness /*// w ww.j av a 2 s .com * instead of copying stored fields we need to update mapping table and remove non-existing entries * several cases: doc may be deleted and non existing in new segment * doc may be deleted, yet similar doc might be found * thing is, these fields might be updated, so using linkup is not so productive * because in fact there can be two diffirent versions of documents present after the update * one, old version, deleted now, and one new version * hmmm, since we're using custom primary key, it means on the update entry with our fields will be overwritten, * also, worth considering, that merge procedure only creates new segment based on previous ones, * deletion of old segments is happening later, via Directory.deletefile API * have to check on this. * * first, since it's merge, we assume entries in kvstore already existing * so, first we search for our segment key: segID-docID->customPK, then copy it * all other fields will be left unchanged * * have to consider the failure case, when merge might be aborted in the middle of it * since merge first copies data to new segment, we are safe here, as in worst case we will lose just the links * for this new segment * */ int docCount = 0; int idx = 0; String new_segment_kvstore_key_part = Base62Converter .fromBase10(mergeState.segmentInfo.name.concat(STORED_FIELDS_EXTENSION).hashCode()); for (AtomicReader reader : mergeState.readers) { final SegmentReader seg_reader = mergeState.matchingSegmentReaders[idx++]; ONSQLStoredFieldsReader fields_reader = null; if (seg_reader != null) { final StoredFieldsReader fieldsReader = seg_reader.getFieldsReader(); // we can do the merge only if the matching reader is also a ONSQLStoredFieldsReader if (fieldsReader != null && fieldsReader instanceof ONSQLStoredFieldsReader) { fields_reader = (ONSQLStoredFieldsReader) fieldsReader; } else throw new IllegalStateException("incorrect fieldsreader class at merge procedure, is " + fieldsReader.getClass().getName() + ", only ONSQLStoredFieldsReader is accepted"); } String current_segment = seg_reader.getSegmentName().concat(STORED_FIELDS_EXTENSION); log.debug("current segment name = " + seg_reader.getSegmentName()); // we assume reader always uses the instance of FSDirectory, so that we can extract directory path String dir = ((FSDirectory) seg_reader.directory()).getDirectory().getAbsolutePath(); final int maxDoc = reader.maxDoc(); final Bits liveDocs = reader.getLiveDocs(); boolean canmerge = ONSQLKVstoreHandler.getInstance().getAllowWriting(this.tdir); for (int i = nextLiveDoc(0, liveDocs, maxDoc); i < maxDoc; i = nextLiveDoc(i + 1, liveDocs, maxDoc)) { ++docCount; if (canmerge) { // retrieve link using our doc id Key doc_key = Key.createKey(Arrays.asList(Base62Converter.fromBase10(dir.hashCode()), Base62Converter.fromBase10(current_segment.hashCode()), Base62Converter.fromBase10(i))); Iterator<Key> kv_it = kvstore.multiGetKeysIterator(Direction.FORWARD, 1, doc_key, null, Depth.PARENT_AND_DESCENDANTS); if (!kv_it.hasNext()) throw new IllegalStateException( "unable to get doc segment key using key id=" + doc_key.toString()); Key entry_key = kv_it.next(); // create link to doc id for new segment Key link_key = Key.createKey(Arrays.asList(Base62Converter.fromBase10(dir.hashCode()), new_segment_kvstore_key_part, Base62Converter.fromBase10(numDocsWritten)), entry_key.getMinorPath()); log.debug("putting link key=" + link_key.toString()); kvstore.put(link_key, Value.EMPTY_VALUE); // next add backref Key backref_key = Key.createKey(entry_key.getMinorPath(), Arrays.asList("_1", Base62Converter.fromBase10(dir.hashCode()), new_segment_kvstore_key_part, Base62Converter.fromBase10(numDocsWritten))); kvstore.put(backref_key, Value.EMPTY_VALUE); log.debug("putting backref key=" + backref_key.toString()); //addDocument(doc, mergeState.fieldInfos); } else log.debug("merging is not allowed, skipping doc with internal id=" + i); ++numDocsWritten; mergeState.checkAbort.work(300); } } finish(mergeState.fieldInfos, docCount); return docCount; }