Example usage for org.apache.lucene.util.packed PackedInts FAST

Introduction

In this page you can find the example usage for org.apache.lucene.util.packed PackedInts FAST.

Prototype

float FAST

To view the source code for org.apache.lucene.util.packed PackedInts FAST.

Click Source Link

Document

At most 50% memory overhead, always select a reasonably fast implementation.

Usage

From source file:org.apache.solr.uninverting.FieldCacheImpl.java

License:Apache License

public SortedDocValues getTermsIndex(LeafReader reader, String field) throws IOException {
    return getTermsIndex(reader, field, PackedInts.FAST);
}

---

From source file:org.apache.solr.uninverting.FieldCacheImpl.java

License:Apache License

public BinaryDocValues getTerms(LeafReader reader, String field) throws IOException {
    return getTerms(reader, field, PackedInts.FAST);
}

---

From source file:org.elasticsearch.index.fielddata.ordinals.InternalGlobalOrdinalsBuilder.java

License:Apache License

@Override
public IndexFieldData.WithOrdinals build(final IndexReader indexReader,
        IndexFieldData.WithOrdinals indexFieldData, Settings settings, CircuitBreakerService breakerService)
        throws IOException {
    assert indexReader.leaves().size() > 1;
    long startTime = System.currentTimeMillis();

    // It makes sense to make the overhead ratio configurable for the mapping from segment ords to global ords
    // However, other mappings are never the bottleneck and only used to get the original value from an ord, so
    // it makes sense to force COMPACT for them
    final float acceptableOverheadRatio = settings.getAsFloat("acceptable_overhead_ratio", PackedInts.FAST);
    final AppendingPackedLongBuffer globalOrdToFirstSegment = new AppendingPackedLongBuffer(PackedInts.COMPACT);
    final MonotonicAppendingLongBuffer globalOrdToFirstSegmentDelta = new MonotonicAppendingLongBuffer(
            PackedInts.COMPACT);/*from  w w  w  .j  a va 2s. co m*/

    FieldDataType fieldDataType = indexFieldData.getFieldDataType();
    int defaultThreshold = settings.getAsInt(ORDINAL_MAPPING_THRESHOLD_INDEX_SETTING_KEY,
            ORDINAL_MAPPING_THRESHOLD_DEFAULT);
    int threshold = fieldDataType.getSettings().getAsInt(ORDINAL_MAPPING_THRESHOLD_KEY, defaultThreshold);
    OrdinalMappingSourceBuilder ordinalMappingBuilder = new OrdinalMappingSourceBuilder(
            indexReader.leaves().size(), acceptableOverheadRatio, threshold);

    long currentGlobalOrdinal = 0;
    final AtomicFieldData.WithOrdinals[] withOrdinals = new AtomicFieldData.WithOrdinals[indexReader.leaves()
            .size()];
    TermIterator termIterator = new TermIterator(indexFieldData, indexReader.leaves(), withOrdinals);
    for (BytesRef term = termIterator.next(); term != null; term = termIterator.next()) {
        globalOrdToFirstSegment.add(termIterator.firstReaderIndex());
        long globalOrdinalDelta = currentGlobalOrdinal - termIterator.firstLocalOrdinal();
        globalOrdToFirstSegmentDelta.add(globalOrdinalDelta);
        for (TermIterator.LeafSource leafSource : termIterator.competitiveLeafs()) {
            ordinalMappingBuilder.onOrdinal(leafSource.context.ord, leafSource.tenum.ord(),
                    currentGlobalOrdinal);
        }
        currentGlobalOrdinal++;
    }

    // ram used for the globalOrd to segmentOrd and segmentOrd to firstReaderIndex lookups
    long memorySizeInBytesCounter = 0;
    globalOrdToFirstSegment.freeze();
    memorySizeInBytesCounter += globalOrdToFirstSegment.ramBytesUsed();
    globalOrdToFirstSegmentDelta.freeze();
    memorySizeInBytesCounter += globalOrdToFirstSegmentDelta.ramBytesUsed();

    final long maxOrd = currentGlobalOrdinal;
    OrdinalMappingSource[] segmentOrdToGlobalOrdLookups = ordinalMappingBuilder.build(maxOrd);
    // add ram used for the main segmentOrd to globalOrd lookups
    memorySizeInBytesCounter += ordinalMappingBuilder.getMemorySizeInBytes();

    final long memorySizeInBytes = memorySizeInBytesCounter;
    breakerService.getBreaker().addWithoutBreaking(memorySizeInBytes);

    if (logger.isDebugEnabled()) {
        // this does include the [] from the array in the impl name
        String implName = segmentOrdToGlobalOrdLookups.getClass().getSimpleName();
        logger.debug("Global-ordinals[{}][{}][{}] took {} ms", implName,
                indexFieldData.getFieldNames().fullName(), maxOrd, (System.currentTimeMillis() - startTime));
    }
    return new InternalGlobalOrdinalsIndexFieldData(indexFieldData.index(), settings,
            indexFieldData.getFieldNames(), fieldDataType, withOrdinals, globalOrdToFirstSegment,
            globalOrdToFirstSegmentDelta, segmentOrdToGlobalOrdLookups, memorySizeInBytes);
}

---

From source file:org.elasticsearch.index.fielddata.plain.xnumeric.xfloat.XFloatArrayIndexFieldData.java

License:Apache License

@Override
public AtomicXFloatFieldData loadDirect(AtomicReaderContext context) throws Exception {
    AtomicReader reader = context.reader();

    Terms terms = reader.terms(getFieldNames().indexName());
    AtomicXFloatFieldData data = null;/*from   w w w  .  j  a  v a2  s.  co m*/
    // TODO: Use an actual estimator to estimate before loading.
    NonEstimatingEstimator estimator = new NonEstimatingEstimator(
            breakerService.getBreaker(CircuitBreaker.Name.FIELDDATA));
    if (terms == null) {
        data = AtomicXFloatFieldData.empty(reader.maxDoc());
        estimator.afterLoad(null, data.ramBytesUsed());
        return data;
    }

    final long numTerms = terms.size();
    final float acceptableTransientOverheadRatio = PackedInts.FAST;

    TermsEnum termsEnum = terms.iterator(null);
    boolean success = false;

    ObjectArray<Object> values = BigArrays.NON_RECYCLING_INSTANCE.newObjectArray(numTerms);

    try (OrdinalsBuilder builder = new OrdinalsBuilder(numTerms, reader.maxDoc(),
            acceptableTransientOverheadRatio)) {
        DocsEnum docsEnum = null;
        TermToFloatSet termToFloat = new TermToFloatSet();
        for (BytesRef term = termsEnum.next(); term != null; term = termsEnum.next()) {
            long termOrd = builder.nextOrdinal();
            values.set(termOrd, termToFloat.parse(term));
            docsEnum = termsEnum.docs(null, docsEnum, DocsEnum.FLAG_NONE);
            for (int docId = docsEnum.nextDoc(); docId != DocsEnum.NO_MORE_DOCS; docId = docsEnum.nextDoc()) {
                builder.addDoc(docId);
            }
        }

        final Ordinals ordinals = builder.build(fieldDataType.getSettings());

        data = new XFloatArrayAtomicFieldData(values, ordinals);
        success = true;

        return data;
    } finally {
        if (!success) {
            // If something went wrong, unwind any current estimations we've made
            estimator.afterLoad(termsEnum, 0);
        } else {
            // Call .afterLoad() to adjust the breaker now that we have an exact size
            estimator.afterLoad(termsEnum, data.ramBytesUsed());
        }

    }

}

---