List of usage examples for org.apache.lucene.index IndexReader leaves
public final List<LeafReaderContext> leaves()
From source file:org.elasticsearch.index.fielddata.IndexFieldDataServiceTests.java
License:Apache License
public void testChangeFieldDataFormat() throws Exception { final IndexFieldDataService ifdService = new IndexFieldDataService(new Index("test"), new DummyCircuitBreakerService()); MapperService mapperService = MapperTestUtils.newMapperService(ifdService.index(), ImmutableSettings.Builder.EMPTY_SETTINGS); ifdService.setIndexService(new StubIndexService(mapperService)); final BuilderContext ctx = new BuilderContext(null, new ContentPath(1)); final StringFieldMapper mapper1 = MapperBuilders.stringField("s").tokenized(false) .fieldDataSettings(ImmutableSettings.builder().put(FieldDataType.FORMAT_KEY, "paged_bytes").build()) .build(ctx);/*from w w w.j av a 2 s. c o m*/ final IndexWriter writer = new IndexWriter(new RAMDirectory(), new IndexWriterConfig(TEST_VERSION_CURRENT, new KeywordAnalyzer())); Document doc = new Document(); doc.add(new StringField("s", "thisisastring", Store.NO)); writer.addDocument(doc); final IndexReader reader1 = DirectoryReader.open(writer, true); IndexFieldData<?> ifd = ifdService.getForField(mapper1); assertThat(ifd, instanceOf(PagedBytesIndexFieldData.class)); Set<AtomicReader> oldSegments = Collections.newSetFromMap(new IdentityHashMap<AtomicReader, Boolean>()); for (AtomicReaderContext arc : reader1.leaves()) { oldSegments.add(arc.reader()); AtomicFieldData<?> afd = ifd.load(arc); assertThat(afd, instanceOf(PagedBytesAtomicFieldData.class)); } // write new segment writer.addDocument(doc); final IndexReader reader2 = DirectoryReader.open(writer, true); final StringFieldMapper mapper2 = MapperBuilders.stringField("s").tokenized(false) .fieldDataSettings(ImmutableSettings.builder().put(FieldDataType.FORMAT_KEY, "fst").build()) .build(ctx); ifdService.onMappingUpdate(); ifd = ifdService.getForField(mapper2); assertThat(ifd, instanceOf(FSTBytesIndexFieldData.class)); for (AtomicReaderContext arc : reader2.leaves()) { AtomicFieldData<?> afd = ifd.load(arc); if (oldSegments.contains(arc.reader())) { assertThat(afd, instanceOf(PagedBytesAtomicFieldData.class)); } else { assertThat(afd, instanceOf(FSTBytesAtomicFieldData.class)); } } reader1.close(); reader2.close(); writer.close(); writer.getDirectory().close(); }
From source file:org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder.java
License:Apache License
/** * Build global ordinals for the provided {@link IndexReader}. *///from w ww. j a v a2s.co m public static IndexOrdinalsFieldData build(final IndexReader indexReader, IndexOrdinalsFieldData indexFieldData, Settings settings, CircuitBreakerService breakerService, ESLogger logger) throws IOException { assert indexReader.leaves().size() > 1; long startTimeNS = System.nanoTime(); final AtomicOrdinalsFieldData[] atomicFD = new AtomicOrdinalsFieldData[indexReader.leaves().size()]; final RandomAccessOrds[] subs = new RandomAccessOrds[indexReader.leaves().size()]; for (int i = 0; i < indexReader.leaves().size(); ++i) { atomicFD[i] = indexFieldData.load(indexReader.leaves().get(i)); subs[i] = atomicFD[i].getOrdinalsValues(); } final OrdinalMap ordinalMap = OrdinalMap.build(null, subs, PackedInts.DEFAULT); final long memorySizeInBytes = ordinalMap.ramBytesUsed(); breakerService.getBreaker(CircuitBreaker.FIELDDATA).addWithoutBreaking(memorySizeInBytes); if (logger.isDebugEnabled()) { logger.debug("Global-ordinals[{}][{}] took {} ms", indexFieldData.getFieldNames().fullName(), ordinalMap.getValueCount(), TimeValue.nsecToMSec(System.nanoTime() - startTimeNS)); } return new InternalGlobalOrdinalsIndexFieldData(indexFieldData.index(), settings, indexFieldData.getFieldNames(), indexFieldData.getFieldDataType(), atomicFD, ordinalMap, memorySizeInBytes); }
From source file:org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder.java
License:Apache License
public static IndexOrdinalsFieldData buildEmpty(Index index, Settings settings, final IndexReader indexReader, IndexOrdinalsFieldData indexFieldData) throws IOException { assert indexReader.leaves().size() > 1; final AtomicOrdinalsFieldData[] atomicFD = new AtomicOrdinalsFieldData[indexReader.leaves().size()]; final RandomAccessOrds[] subs = new RandomAccessOrds[indexReader.leaves().size()]; for (int i = 0; i < indexReader.leaves().size(); ++i) { atomicFD[i] = new AbstractAtomicOrdinalsFieldData() { @Override//from www .j av a 2s .c o m public RandomAccessOrds getOrdinalsValues() { return DocValues.emptySortedSet(); } @Override public long ramBytesUsed() { return 0; } @Override public Collection<Accountable> getChildResources() { return Collections.emptyList(); } @Override public void close() { } }; subs[i] = atomicFD[i].getOrdinalsValues(); } final OrdinalMap ordinalMap = OrdinalMap.build(null, subs, PackedInts.DEFAULT); return new InternalGlobalOrdinalsIndexFieldData(index, settings, indexFieldData.getFieldNames(), indexFieldData.getFieldDataType(), atomicFD, ordinalMap, 0); }
From source file:org.elasticsearch.index.fielddata.ordinals.InternalGlobalOrdinalsBuilder.java
License:Apache License
@Override public IndexFieldData.WithOrdinals build(final IndexReader indexReader, IndexFieldData.WithOrdinals indexFieldData, Settings settings, CircuitBreakerService breakerService) throws IOException { assert indexReader.leaves().size() > 1; long startTime = System.currentTimeMillis(); // It makes sense to make the overhead ratio configurable for the mapping from segment ords to global ords // However, other mappings are never the bottleneck and only used to get the original value from an ord, so // it makes sense to force COMPACT for them final float acceptableOverheadRatio = settings.getAsFloat("acceptable_overhead_ratio", PackedInts.FAST); final AppendingPackedLongBuffer globalOrdToFirstSegment = new AppendingPackedLongBuffer(PackedInts.COMPACT); final MonotonicAppendingLongBuffer globalOrdToFirstSegmentDelta = new MonotonicAppendingLongBuffer( PackedInts.COMPACT);/*from w ww . j a v a2 s.c o m*/ FieldDataType fieldDataType = indexFieldData.getFieldDataType(); int defaultThreshold = settings.getAsInt(ORDINAL_MAPPING_THRESHOLD_INDEX_SETTING_KEY, ORDINAL_MAPPING_THRESHOLD_DEFAULT); int threshold = fieldDataType.getSettings().getAsInt(ORDINAL_MAPPING_THRESHOLD_KEY, defaultThreshold); OrdinalMappingSourceBuilder ordinalMappingBuilder = new OrdinalMappingSourceBuilder( indexReader.leaves().size(), acceptableOverheadRatio, threshold); long currentGlobalOrdinal = 0; final AtomicFieldData.WithOrdinals[] withOrdinals = new AtomicFieldData.WithOrdinals[indexReader.leaves() .size()]; TermIterator termIterator = new TermIterator(indexFieldData, indexReader.leaves(), withOrdinals); for (BytesRef term = termIterator.next(); term != null; term = termIterator.next()) { globalOrdToFirstSegment.add(termIterator.firstReaderIndex()); long globalOrdinalDelta = currentGlobalOrdinal - termIterator.firstLocalOrdinal(); globalOrdToFirstSegmentDelta.add(globalOrdinalDelta); for (TermIterator.LeafSource leafSource : termIterator.competitiveLeafs()) { ordinalMappingBuilder.onOrdinal(leafSource.context.ord, leafSource.tenum.ord(), currentGlobalOrdinal); } currentGlobalOrdinal++; } // ram used for the globalOrd to segmentOrd and segmentOrd to firstReaderIndex lookups long memorySizeInBytesCounter = 0; globalOrdToFirstSegment.freeze(); memorySizeInBytesCounter += globalOrdToFirstSegment.ramBytesUsed(); globalOrdToFirstSegmentDelta.freeze(); memorySizeInBytesCounter += globalOrdToFirstSegmentDelta.ramBytesUsed(); final long maxOrd = currentGlobalOrdinal; OrdinalMappingSource[] segmentOrdToGlobalOrdLookups = ordinalMappingBuilder.build(maxOrd); // add ram used for the main segmentOrd to globalOrd lookups memorySizeInBytesCounter += ordinalMappingBuilder.getMemorySizeInBytes(); final long memorySizeInBytes = memorySizeInBytesCounter; breakerService.getBreaker().addWithoutBreaking(memorySizeInBytes); if (logger.isDebugEnabled()) { // this does include the [] from the array in the impl name String implName = segmentOrdToGlobalOrdLookups.getClass().getSimpleName(); logger.debug("Global-ordinals[{}][{}][{}] took {} ms", implName, indexFieldData.getFieldNames().fullName(), maxOrd, (System.currentTimeMillis() - startTime)); } return new InternalGlobalOrdinalsIndexFieldData(indexFieldData.index(), settings, indexFieldData.getFieldNames(), fieldDataType, withOrdinals, globalOrdToFirstSegment, globalOrdToFirstSegmentDelta, segmentOrdToGlobalOrdLookups, memorySizeInBytes); }
From source file:org.elasticsearch.index.percolator.PercolatorQueryCacheTests.java
License:Apache License
public void testLoadQueries() throws Exception { Directory directory = newDirectory(); IndexWriter indexWriter = new IndexWriter(directory, new IndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(NoMergePolicy.INSTANCE)); boolean legacyFormat = randomBoolean(); Version version = legacyFormat ? Version.V_2_0_0 : Version.CURRENT; IndexShard indexShard = mockIndexShard(version, legacyFormat); storeQuery("0", indexWriter, termQuery("field1", "value1"), true, legacyFormat); storeQuery("1", indexWriter, wildcardQuery("field1", "v*"), true, legacyFormat); storeQuery("2", indexWriter, boolQuery().must(termQuery("field1", "value1")).must(termQuery("field2", "value2")), true, legacyFormat);//w w w . j av a 2s . c om // dymmy docs should be skipped during loading: Document doc = new Document(); doc.add(new StringField("dummy", "value", Field.Store.YES)); indexWriter.addDocument(doc); storeQuery("4", indexWriter, termQuery("field2", "value2"), true, legacyFormat); // only documents that .percolator type should be loaded: storeQuery("5", indexWriter, termQuery("field2", "value2"), false, legacyFormat); storeQuery("6", indexWriter, termQuery("field3", "value3"), true, legacyFormat); indexWriter.forceMerge(1); // also include queries for percolator docs marked as deleted: indexWriter.deleteDocuments(new Term("id", "6")); indexWriter.close(); ShardId shardId = new ShardId("_index", ClusterState.UNKNOWN_UUID, 0); IndexReader indexReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(directory), shardId); assertThat(indexReader.leaves().size(), equalTo(1)); assertThat(indexReader.numDeletedDocs(), equalTo(1)); assertThat(indexReader.maxDoc(), equalTo(7)); initialize("field1", "type=keyword", "field2", "type=keyword", "field3", "type=keyword"); PercolatorQueryCache.QueriesLeaf leaf = cache.loadQueries(indexReader.leaves().get(0), indexShard); assertThat(leaf.queries.size(), equalTo(5)); assertThat(leaf.getQuery(0), equalTo(new TermQuery(new Term("field1", "value1")))); assertThat(leaf.getQuery(1), equalTo(new WildcardQuery(new Term("field1", "v*")))); assertThat(leaf.getQuery(2), equalTo(new BooleanQuery.Builder() .add(new TermQuery(new Term("field1", "value1")), BooleanClause.Occur.MUST) .add(new TermQuery(new Term("field2", "value2")), BooleanClause.Occur.MUST).build())); assertThat(leaf.getQuery(4), equalTo(new TermQuery(new Term("field2", "value2")))); assertThat(leaf.getQuery(6), equalTo(new TermQuery(new Term("field3", "value3")))); indexReader.close(); directory.close(); }
From source file:org.elasticsearch.index.percolator.PercolatorQueryCacheTests.java
License:Apache License
public void testGetQueries() throws Exception { Directory directory = newDirectory(); IndexWriter indexWriter = new IndexWriter(directory, new IndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(NoMergePolicy.INSTANCE)); storeQuery("0", indexWriter, termQuery("a", "0"), true, false); storeQuery("1", indexWriter, termQuery("a", "1"), true, false); storeQuery("2", indexWriter, termQuery("a", "2"), true, false); indexWriter.flush();/*from w w w . j av a 2 s . com*/ storeQuery("3", indexWriter, termQuery("a", "3"), true, false); storeQuery("4", indexWriter, termQuery("a", "4"), true, false); storeQuery("5", indexWriter, termQuery("a", "5"), true, false); indexWriter.flush(); storeQuery("6", indexWriter, termQuery("a", "6"), true, false); storeQuery("7", indexWriter, termQuery("a", "7"), true, false); storeQuery("8", indexWriter, termQuery("a", "8"), true, false); indexWriter.flush(); indexWriter.close(); ShardId shardId = new ShardId("_index", ClusterState.UNKNOWN_UUID, 0); IndexReader indexReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(directory), shardId); assertThat(indexReader.leaves().size(), equalTo(3)); assertThat(indexReader.maxDoc(), equalTo(9)); initialize("a", "type=keyword"); try { cache.getQueries(indexReader.leaves().get(0)); fail("IllegalStateException expected"); } catch (IllegalStateException e) { assertThat(e.getMessage(), equalTo("queries not loaded, queries should be have been preloaded during index warming...")); } IndexShard indexShard = mockIndexShard(Version.CURRENT, false); ThreadPool threadPool = mockThreadPool(); IndexWarmer.Listener listener = cache.createListener(threadPool); listener.warmReader(indexShard, new Engine.Searcher("test", new IndexSearcher(indexReader))); PercolatorQueryCacheStats stats = cache.getStats(shardId); assertThat(stats.getNumQueries(), equalTo(9L)); PercolateQuery.QueryRegistry.Leaf leaf = cache.getQueries(indexReader.leaves().get(0)); assertThat(leaf.getQuery(0), equalTo(new TermQuery(new Term("a", "0")))); assertThat(leaf.getQuery(1), equalTo(new TermQuery(new Term("a", "1")))); assertThat(leaf.getQuery(2), equalTo(new TermQuery(new Term("a", "2")))); leaf = cache.getQueries(indexReader.leaves().get(1)); assertThat(leaf.getQuery(0), equalTo(new TermQuery(new Term("a", "3")))); assertThat(leaf.getQuery(1), equalTo(new TermQuery(new Term("a", "4")))); assertThat(leaf.getQuery(2), equalTo(new TermQuery(new Term("a", "5")))); leaf = cache.getQueries(indexReader.leaves().get(2)); assertThat(leaf.getQuery(0), equalTo(new TermQuery(new Term("a", "6")))); assertThat(leaf.getQuery(1), equalTo(new TermQuery(new Term("a", "7")))); assertThat(leaf.getQuery(2), equalTo(new TermQuery(new Term("a", "8")))); indexReader.close(); directory.close(); }
From source file:org.elasticsearch.index.percolator.PercolatorQueryCacheTests.java
License:Apache License
public void testInvalidateEntries() throws Exception { Directory directory = newDirectory(); IndexWriter indexWriter = new IndexWriter(directory, new IndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(NoMergePolicy.INSTANCE)); storeQuery("0", indexWriter, termQuery("a", "0"), true, false); indexWriter.flush();/*from w w w . j a va 2 s. co m*/ storeQuery("1", indexWriter, termQuery("a", "1"), true, false); indexWriter.flush(); storeQuery("2", indexWriter, termQuery("a", "2"), true, false); indexWriter.flush(); ShardId shardId = new ShardId("_index", ClusterState.UNKNOWN_UUID, 0); IndexReader indexReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(indexWriter), shardId); assertThat(indexReader.leaves().size(), equalTo(3)); assertThat(indexReader.maxDoc(), equalTo(3)); initialize("a", "type=keyword"); IndexShard indexShard = mockIndexShard(Version.CURRENT, false); ThreadPool threadPool = mockThreadPool(); IndexWarmer.Listener listener = cache.createListener(threadPool); listener.warmReader(indexShard, new Engine.Searcher("test", new IndexSearcher(indexReader))); assertThat(cache.getStats(shardId).getNumQueries(), equalTo(3L)); PercolateQuery.QueryRegistry.Leaf leaf = cache.getQueries(indexReader.leaves().get(0)); assertThat(leaf.getQuery(0), equalTo(new TermQuery(new Term("a", "0")))); leaf = cache.getQueries(indexReader.leaves().get(1)); assertThat(leaf.getQuery(0), equalTo(new TermQuery(new Term("a", "1")))); leaf = cache.getQueries(indexReader.leaves().get(2)); assertThat(leaf.getQuery(0), equalTo(new TermQuery(new Term("a", "2")))); // change merge policy, so that merges will actually happen: indexWriter.getConfig().setMergePolicy(new TieredMergePolicy()); indexWriter.deleteDocuments(new Term("id", "1")); indexWriter.forceMergeDeletes(); indexReader.close(); indexReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(indexWriter), shardId); assertThat(indexReader.leaves().size(), equalTo(2)); assertThat(indexReader.maxDoc(), equalTo(2)); listener.warmReader(indexShard, new Engine.Searcher("test", new IndexSearcher(indexReader))); assertThat(cache.getStats(shardId).getNumQueries(), equalTo(2L)); leaf = cache.getQueries(indexReader.leaves().get(0)); assertThat(leaf.getQuery(0), equalTo(new TermQuery(new Term("a", "0")))); leaf = cache.getQueries(indexReader.leaves().get(1)); assertThat(leaf.getQuery(0), equalTo(new TermQuery(new Term("a", "2")))); indexWriter.forceMerge(1); indexReader.close(); indexReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(indexWriter), shardId); assertThat(indexReader.leaves().size(), equalTo(1)); assertThat(indexReader.maxDoc(), equalTo(2)); listener.warmReader(indexShard, new Engine.Searcher("test", new IndexSearcher(indexReader))); assertThat(cache.getStats(shardId).getNumQueries(), equalTo(2L)); leaf = cache.getQueries(indexReader.leaves().get(0)); assertThat(leaf.getQuery(0), equalTo(new TermQuery(new Term("a", "0")))); assertThat(leaf.getQuery(1), equalTo(new TermQuery(new Term("a", "2")))); indexWriter.close(); indexReader.close(); directory.close(); }
From source file:org.elasticsearch.index.search.child.DeleteByQueryWrappingFilter.java
License:Apache License
static boolean contains(IndexReader indexReader, AtomicReaderContext context) { for (AtomicReaderContext atomicReaderContext : indexReader.leaves()) { if (context.reader().getCoreCacheKey().equals(atomicReaderContext.reader().getCoreCacheKey())) { return true; }//from ww w . ja va 2s . c om } return false; }
From source file:org.elasticsearch.join.aggregations.ParentToChildrenAggregator.java
License:Apache License
@Override protected void doPostCollection() throws IOException { IndexReader indexReader = context().searcher().getIndexReader(); for (LeafReaderContext ctx : indexReader.leaves()) { Scorer childDocsScorer = childFilter.scorer(ctx); if (childDocsScorer == null) { continue; }/*w w w.j a v a 2 s . c o m*/ DocIdSetIterator childDocsIter = childDocsScorer.iterator(); final LeafBucketCollector sub = collectableSubAggregators.getLeafCollector(ctx); final SortedSetDocValues globalOrdinals = valuesSource.globalOrdinalsValues(ctx); // Set the scorer, since we now replay only the child docIds sub.setScorer(new ConstantScoreScorer(null, 1f, childDocsIter)); final Bits liveDocs = ctx.reader().getLiveDocs(); for (int docId = childDocsIter.nextDoc(); docId != DocIdSetIterator.NO_MORE_DOCS; docId = childDocsIter .nextDoc()) { if (liveDocs != null && liveDocs.get(docId) == false) { continue; } if (globalOrdinals.advanceExact(docId)) { long globalOrdinal = globalOrdinals.nextOrd(); assert globalOrdinals.nextOrd() == SortedSetDocValues.NO_MORE_ORDS; long bucketOrd = parentOrdToBuckets.get(globalOrdinal); if (bucketOrd != -1) { collectBucket(sub, docId, bucketOrd); if (multipleBucketsPerParentOrd) { long[] otherBucketOrds = parentOrdToOtherBuckets.get(globalOrdinal); if (otherBucketOrds != null) { for (long otherBucketOrd : otherBucketOrds) { collectBucket(sub, docId, otherBucketOrd); } } } } } } } }
From source file:org.elasticsearch.percolator.PercolateContext.java
License:Apache License
public void initialize(Engine.Searcher docSearcher, ParsedDocument parsedDocument) { this.docSearcher = docSearcher; IndexReader indexReader = docSearcher.reader(); AtomicReaderContext atomicReaderContext = indexReader.leaves().get(0); lookup().setNextReader(atomicReaderContext); lookup().setNextDocId(0);/*from w ww .j av a2 s .c o m*/ lookup().source().setNextSource(parsedDocument.source()); Map<String, SearchHitField> fields = new HashMap<String, SearchHitField>(); for (IndexableField field : parsedDocument.rootDoc().getFields()) { fields.put(field.name(), new InternalSearchHitField(field.name(), ImmutableList.of())); } hitContext().reset(new InternalSearchHit(0, "unknown", new StringText(parsedDocument.type()), fields), atomicReaderContext, 0, indexReader, 0, new JustSourceFieldsVisitor()); }