List of usage examples for org.apache.lucene.index SortedNumericDocValues nextValue
public abstract long nextValue() throws IOException;
From source file:org.apache.solr.search.SolrDocumentFetcher.java
License:Apache License
/** * This will fetch and add the docValues fields to a given SolrDocument/SolrInputDocument * * @param doc/*from w w w. j a v a 2 s .c o m*/ * A SolrDocument or SolrInputDocument instance where docValues will be added * @param docid * The lucene docid of the document to be populated * @param fields * The list of docValues fields to be decorated */ public void decorateDocValueFields(@SuppressWarnings("rawtypes") SolrDocumentBase doc, int docid, Set<String> fields) throws IOException { final List<LeafReaderContext> leafContexts = searcher.getLeafContexts(); final int subIndex = ReaderUtil.subIndex(docid, leafContexts); final int localId = docid - leafContexts.get(subIndex).docBase; final LeafReader leafReader = leafContexts.get(subIndex).reader(); for (String fieldName : fields) { final SchemaField schemaField = searcher.getSchema().getFieldOrNull(fieldName); if (schemaField == null || !schemaField.hasDocValues() || doc.containsKey(fieldName)) { log.warn("Couldn't decorate docValues for field: [{}], schemaField: [{}]", fieldName, schemaField); continue; } FieldInfo fi = searcher.getFieldInfos().fieldInfo(fieldName); if (fi == null) { continue; // Searcher doesn't have info about this field, hence ignore it. } final DocValuesType dvType = fi.getDocValuesType(); switch (dvType) { case NUMERIC: final NumericDocValues ndv = leafReader.getNumericDocValues(fieldName); if (ndv == null) { continue; } Long val; if (ndv.advanceExact(localId)) { val = ndv.longValue(); } else { continue; } Object newVal = val; if (schemaField.getType().isPointField()) { // TODO: Maybe merge PointField with TrieFields here NumberType type = schemaField.getType().getNumberType(); switch (type) { case INTEGER: newVal = val.intValue(); break; case LONG: newVal = val.longValue(); break; case FLOAT: newVal = Float.intBitsToFloat(val.intValue()); break; case DOUBLE: newVal = Double.longBitsToDouble(val); break; case DATE: newVal = new Date(val); break; default: throw new AssertionError("Unexpected PointType: " + type); } } else { if (schemaField.getType() instanceof TrieIntField) { newVal = val.intValue(); } else if (schemaField.getType() instanceof TrieFloatField) { newVal = Float.intBitsToFloat(val.intValue()); } else if (schemaField.getType() instanceof TrieDoubleField) { newVal = Double.longBitsToDouble(val); } else if (schemaField.getType() instanceof TrieDateField) { newVal = new Date(val); } else if (schemaField.getType() instanceof EnumField) { newVal = ((EnumField) schemaField.getType()).intValueToStringValue(val.intValue()); } } doc.addField(fieldName, newVal); break; case BINARY: BinaryDocValues bdv = leafReader.getBinaryDocValues(fieldName); if (bdv == null) { continue; } BytesRef value; if (bdv.advanceExact(localId)) { value = BytesRef.deepCopyOf(bdv.binaryValue()); } else { continue; } doc.addField(fieldName, value); break; case SORTED: SortedDocValues sdv = leafReader.getSortedDocValues(fieldName); if (sdv == null) { continue; } if (sdv.advanceExact(localId)) { final BytesRef bRef = sdv.binaryValue(); // Special handling for Boolean fields since they're stored as 'T' and 'F'. if (schemaField.getType() instanceof BoolField) { doc.addField(fieldName, schemaField.getType().toObject(schemaField, bRef)); } else { doc.addField(fieldName, bRef.utf8ToString()); } } break; case SORTED_NUMERIC: final SortedNumericDocValues numericDv = leafReader.getSortedNumericDocValues(fieldName); NumberType type = schemaField.getType().getNumberType(); if (numericDv != null) { if (numericDv.advance(localId) == localId) { final List<Object> outValues = new ArrayList<Object>(numericDv.docValueCount()); for (int i = 0; i < numericDv.docValueCount(); i++) { long number = numericDv.nextValue(); switch (type) { case INTEGER: outValues.add((int) number); break; case LONG: outValues.add(number); break; case FLOAT: outValues.add(NumericUtils.sortableIntToFloat((int) number)); break; case DOUBLE: outValues.add(NumericUtils.sortableLongToDouble(number)); break; case DATE: outValues.add(new Date(number)); break; default: throw new AssertionError("Unexpected PointType: " + type); } } assert outValues.size() > 0; doc.addField(fieldName, outValues); } } case SORTED_SET: final SortedSetDocValues values = leafReader.getSortedSetDocValues(fieldName); if (values != null && values.getValueCount() > 0) { if (values.advance(localId) == localId) { final List<Object> outValues = new LinkedList<>(); for (long ord = values.nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; ord = values .nextOrd()) { value = values.lookupOrd(ord); outValues.add(schemaField.getType().toObject(schemaField, value)); } assert outValues.size() > 0; doc.addField(fieldName, outValues); } } case NONE: break; } } }
From source file:org.elasticsearch.index.shard.ShardSplittingQueryTests.java
License:Apache License
void assertSplit(Directory dir, IndexMetaData metaData, int targetShardId, boolean hasNested) throws IOException { try (IndexReader reader = DirectoryReader.open(dir)) { IndexSearcher searcher = new IndexSearcher(reader); searcher.setQueryCache(null);// w ww .j a va2 s .c om final boolean needsScores = false; final Weight splitWeight = searcher.createNormalizedWeight( new ShardSplittingQuery(metaData, targetShardId, hasNested), needsScores); final List<LeafReaderContext> leaves = reader.leaves(); for (final LeafReaderContext ctx : leaves) { Scorer scorer = splitWeight.scorer(ctx); DocIdSetIterator iterator = scorer.iterator(); SortedNumericDocValues shard_id = ctx.reader().getSortedNumericDocValues("shard_id"); int numExpected = 0; while (shard_id.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) { if (targetShardId == shard_id.nextValue()) { numExpected++; } } if (numExpected == ctx.reader().maxDoc()) { // all docs belong in this shard assertEquals(DocIdSetIterator.NO_MORE_DOCS, iterator.nextDoc()); } else { shard_id = ctx.reader().getSortedNumericDocValues("shard_id"); int doc; int numActual = 0; int lastDoc = 0; while ((doc = iterator.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { lastDoc = doc; while (shard_id.nextDoc() < doc) { long shardID = shard_id.nextValue(); assertEquals(shardID, targetShardId); numActual++; } assertEquals(shard_id.docID(), doc); long shardID = shard_id.nextValue(); BytesRef id = reader.document(doc).getBinaryValue("_id"); String actualId = Uid.decodeId(id.bytes, id.offset, id.length); assertNotEquals(ctx.reader() + " docID: " + doc + " actualID: " + actualId, shardID, targetShardId); } if (lastDoc < ctx.reader().maxDoc()) { // check the last docs in the segment and make sure they all have the right shard id while (shard_id.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) { long shardID = shard_id.nextValue(); assertEquals(shardID, targetShardId); numActual++; } } assertEquals(numExpected, numActual); } } } }
From source file:org.elasticsearch.search.aggregations.bucket.composite.LongValuesSource.java
License:Apache License
@Override LeafBucketCollector getLeafCollector(LeafReaderContext context, LeafBucketCollector next) throws IOException { final SortedNumericDocValues dvs = docValuesFunc.apply(context); return new LeafBucketCollector() { @Override// www .j av a 2s . co m public void collect(int doc, long bucket) throws IOException { if (dvs.advanceExact(doc)) { int num = dvs.docValueCount(); for (int i = 0; i < num; i++) { currentValue = dvs.nextValue(); next.collect(doc, bucket); } } } }; }
From source file:org.elasticsearch.search.aggregations.bucket.composite.RoundingValuesSource.java
License:Apache License
@Override public SortedNumericDocValues longValues(LeafReaderContext context) throws IOException { SortedNumericDocValues values = vs.longValues(context); return new SortedNumericDocValues() { @Override/*from w ww . j av a 2s. c o m*/ public long nextValue() throws IOException { return round(values.nextValue()); } @Override public int docValueCount() { return values.docValueCount(); } @Override public boolean advanceExact(int target) throws IOException { return values.advanceExact(target); } @Override public int docID() { return values.docID(); } @Override public int nextDoc() throws IOException { return values.nextDoc(); } @Override public int advance(int target) throws IOException { return values.advance(target); } @Override public long cost() { return values.cost(); } }; }
From source file:org.elasticsearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregator.java
License:Apache License
@Override public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, final LeafBucketCollector sub) throws IOException { if (valuesSource == null) { return LeafBucketCollector.NO_OP_COLLECTOR; }//from ww w. jav a 2s .co m final SortedNumericDocValues values = valuesSource.longValues(ctx); return new LeafBucketCollectorBase(sub, values) { @Override public void collect(int doc, long bucket) throws IOException { assert bucket == 0; if (values.advanceExact(doc)) { final int valuesCount = values.docValueCount(); long previousRounded = Long.MIN_VALUE; for (int i = 0; i < valuesCount; ++i) { long value = values.nextValue(); long rounded = roundingInfos[roundingIdx].rounding.round(value); assert rounded >= previousRounded; if (rounded == previousRounded) { continue; } long bucketOrd = bucketOrds.add(rounded); if (bucketOrd < 0) { // already seen bucketOrd = -1 - bucketOrd; collectExistingBucket(sub, doc, bucketOrd); } else { collectBucket(sub, doc, bucketOrd); while (roundingIdx < roundingInfos.length - 1 && bucketOrds.size() > (targetBuckets * roundingInfos[roundingIdx].getMaximumInnerInterval())) { increaseRounding(); } } previousRounded = rounded; } } } private void increaseRounding() { try (LongHash oldBucketOrds = bucketOrds) { LongHash newBucketOrds = new LongHash(1, context.bigArrays()); long[] mergeMap = new long[(int) oldBucketOrds.size()]; Rounding newRounding = roundingInfos[++roundingIdx].rounding; for (int i = 0; i < oldBucketOrds.size(); i++) { long oldKey = oldBucketOrds.get(i); long newKey = newRounding.round(oldKey); long newBucketOrd = newBucketOrds.add(newKey); if (newBucketOrd >= 0) { mergeMap[i] = newBucketOrd; } else { mergeMap[i] = -1 - newBucketOrd; } } mergeBuckets(mergeMap, newBucketOrds.size()); if (deferringCollector != null) { deferringCollector.mergeBuckets(mergeMap); } bucketOrds = newBucketOrds; } } }; }
From source file:org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder.java
License:Apache License
DateTimeZone rewriteTimeZone(QueryShardContext context) throws IOException { final DateTimeZone tz = timeZone(); if (field() != null && tz != null && tz.isFixed() == false && field() != null && script() == null) { final MappedFieldType ft = context.fieldMapper(field()); final IndexReader reader = context.getIndexReader(); if (ft != null && reader != null) { Long anyInstant = null; final IndexNumericFieldData fieldData = context.getForField(ft); for (LeafReaderContext ctx : reader.leaves()) { AtomicNumericFieldData leafFD = fieldData.load(ctx); SortedNumericDocValues values = leafFD.getLongValues(); if (values.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) { anyInstant = values.nextValue(); break; }// w w w . j av a 2s .c o m } if (anyInstant != null) { final long prevTransition = tz.previousTransition(anyInstant); final long nextTransition = tz.nextTransition(anyInstant); // We need all not only values but also rounded values to be within // [prevTransition, nextTransition]. final long low; DateTimeUnit intervalAsUnit = getIntervalAsDateTimeUnit(); if (intervalAsUnit != null) { final DateTimeField dateTimeField = intervalAsUnit.field(tz); low = dateTimeField.roundCeiling(prevTransition); } else { final TimeValue intervalAsMillis = getIntervalAsTimeValue(); low = Math.addExact(prevTransition, intervalAsMillis.millis()); } // rounding rounds down, so 'nextTransition' is a good upper bound final long high = nextTransition; if (ft.isFieldWithinQuery(reader, low, high, true, false, DateTimeZone.UTC, EPOCH_MILLIS_PARSER, context) == Relation.WITHIN) { // All values in this reader have the same offset despite daylight saving times. // This is very common for location-based timezones such as Europe/Paris in // combination with time-based indices. return DateTimeZone.forOffsetMillis(tz.getOffset(anyInstant)); } } } } return tz; }
From source file:org.elasticsearch.xpack.core.security.authz.accesscontrol.FieldSubsetReaderTests.java
License:Open Source License
/** * test filtering two sortednumeric dv fields *//*from w w w. ja v a 2 s. c o m*/ public void testSortedNumericDocValues() throws Exception { Directory dir = newDirectory(); IndexWriterConfig iwc = new IndexWriterConfig(null); IndexWriter iw = new IndexWriter(dir, iwc); // add document with 2 fields Document doc = new Document(); doc.add(new SortedNumericDocValuesField("fieldA", 1)); doc.add(new SortedNumericDocValuesField("fieldB", 2)); iw.addDocument(doc); // open reader DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA"))); // see only one field LeafReader segmentReader = ir.leaves().get(0).reader(); SortedNumericDocValues dv = segmentReader.getSortedNumericDocValues("fieldA"); assertNotNull(dv); assertTrue(dv.advanceExact(0)); assertEquals(1, dv.docValueCount()); assertEquals(1, dv.nextValue()); assertNull(segmentReader.getSortedNumericDocValues("fieldB")); TestUtil.checkReader(ir); IOUtils.close(ir, iw, dir); }