List of usage examples for org.apache.lucene.util FixedBitSet getBits
public long[] getBits()
From source file:org.apache.solr.request.SimpleFacets.java
License:Apache License
protected void parseParams(String type, String param) throws SyntaxError, IOException { localParams = QueryParsing.getLocalParams(param, req.getParams()); docs = docsOrig;//from w w w . j av a 2 s.c o m facetValue = param; key = param; threads = -1; if (localParams == null) { params = orig; required = new RequiredSolrParams(params); return; } params = SolrParams.wrapDefaults(localParams, orig); required = new RequiredSolrParams(params); // remove local params unless it's a query if (type != FacetParams.FACET_QUERY) { // TODO Cut over to an Enum here facetValue = localParams.get(CommonParams.VALUE); } // reset set the default key now that localParams have been removed key = facetValue; // allow explicit set of the key key = localParams.get(CommonParams.OUTPUT_KEY, key); String threadStr = localParams.get(CommonParams.THREADS); if (threadStr != null) { threads = Integer.parseInt(threadStr); } // figure out if we need a new base DocSet String excludeStr = localParams.get(CommonParams.EXCLUDE); if (excludeStr == null) return; Map<?, ?> tagMap = (Map<?, ?>) req.getContext().get("tags"); if (tagMap != null && rb != null) { List<String> excludeTagList = StrUtils.splitSmart(excludeStr, ','); IdentityHashMap<Query, Boolean> excludeSet = new IdentityHashMap<Query, Boolean>(); for (String excludeTag : excludeTagList) { Object olst = tagMap.get(excludeTag); // tagMap has entries of List<String,List<QParser>>, but subject to change in the future if (!(olst instanceof Collection)) continue; for (Object o : (Collection<?>) olst) { if (!(o instanceof QParser)) continue; QParser qp = (QParser) o; excludeSet.put(qp.getQuery(), Boolean.TRUE); } } if (excludeSet.size() == 0) return; List<Query> qlist = new ArrayList<Query>(); // add the base query if (!excludeSet.containsKey(rb.getQuery())) { qlist.add(rb.getQuery()); } // add the filters if (rb.getFilters() != null) { for (Query q : rb.getFilters()) { if (!excludeSet.containsKey(q)) { qlist.add(q); } } } // get the new base docset for this facet DocSet base = searcher.getDocSet(qlist); if (rb.grouping() && rb.getGroupingSpec().isTruncateGroups()) { Grouping grouping = new Grouping(searcher, null, rb.getQueryCommand(), false, 0, false); grouping.setGroupSort(rb.getGroupingSpec().getSortWithinGroup()); if (rb.getGroupingSpec().getFields().length > 0) { grouping.addFieldCommand(rb.getGroupingSpec().getFields()[0], req); } else if (rb.getGroupingSpec().getFunctions().length > 0) { grouping.addFunctionCommand(rb.getGroupingSpec().getFunctions()[0], req); } else { this.docs = base; return; } AbstractAllGroupHeadsCollector allGroupHeadsCollector = grouping.getCommands().get(0) .createAllGroupCollector(); searcher.search(new MatchAllDocsQuery(), base.getTopFilter(), allGroupHeadsCollector); int maxDoc = searcher.maxDoc(); FixedBitSet fixedBitSet = allGroupHeadsCollector.retrieveGroupHeads(maxDoc); long[] bits = fixedBitSet.getBits(); this.docs = new BitDocSet(new OpenBitSet(bits, bits.length)); } else { this.docs = base; } } }
From source file:org.apache.solr.search.Grouping.java
License:Apache License
public void execute() throws IOException { if (commands.isEmpty()) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Specify at least one field, function or query to group by."); }//from w w w . j ava2s .c o m DocListAndSet out = new DocListAndSet(); qr.setDocListAndSet(out); SolrIndexSearcher.ProcessedFilter pf = searcher.getProcessedFilter(cmd.getFilter(), cmd.getFilterList()); final Filter luceneFilter = pf.filter; maxDoc = searcher.maxDoc(); needScores = (cmd.getFlags() & SolrIndexSearcher.GET_SCORES) != 0; boolean cacheScores = false; // NOTE: Change this when groupSort can be specified per group if (!needScores && !commands.isEmpty()) { if (commands.get(0).groupSort == null) { cacheScores = true; } else { for (SortField field : commands.get(0).groupSort.getSort()) { if (field.getType() == SortField.Type.SCORE) { cacheScores = true; break; } } } } else if (needScores) { cacheScores = needScores; } getDocSet = (cmd.getFlags() & SolrIndexSearcher.GET_DOCSET) != 0; getDocList = (cmd.getFlags() & SolrIndexSearcher.GET_DOCLIST) != 0; query = QueryUtils.makeQueryable(cmd.getQuery()); for (Command cmd : commands) { cmd.prepare(); } AbstractAllGroupHeadsCollector<?> allGroupHeadsCollector = null; List<Collector> collectors = new ArrayList<Collector>(commands.size()); for (Command cmd : commands) { Collector collector = cmd.createFirstPassCollector(); if (collector != null) { collectors.add(collector); } if (getGroupedDocSet && allGroupHeadsCollector == null) { collectors.add(allGroupHeadsCollector = cmd.createAllGroupCollector()); } } Collector allCollectors = MultiCollector.wrap(collectors.toArray(new Collector[collectors.size()])); DocSetCollector setCollector = null; if (getDocSet && allGroupHeadsCollector == null) { setCollector = new DocSetDelegateCollector(maxDoc >> 6, maxDoc, allCollectors); allCollectors = setCollector; } CachingCollector cachedCollector = null; if (cacheSecondPassSearch && allCollectors != null) { int maxDocsToCache = (int) Math.round(maxDoc * (maxDocsPercentageToCache / 100.0d)); // Only makes sense to cache if we cache more than zero. // Maybe we should have a minimum and a maximum, that defines the window we would like caching for. if (maxDocsToCache > 0) { allCollectors = cachedCollector = CachingCollector.create(allCollectors, cacheScores, maxDocsToCache); } } if (pf.postFilter != null) { pf.postFilter.setLastDelegate(allCollectors); allCollectors = pf.postFilter; } if (allCollectors != null) { searchWithTimeLimiter(luceneFilter, allCollectors); } if (getGroupedDocSet && allGroupHeadsCollector != null) { FixedBitSet fixedBitSet = allGroupHeadsCollector.retrieveGroupHeads(maxDoc); long[] bits = fixedBitSet.getBits(); OpenBitSet openBitSet = new OpenBitSet(bits, bits.length); qr.setDocSet(new BitDocSet(openBitSet)); } else if (getDocSet) { qr.setDocSet(setCollector.getDocSet()); } collectors.clear(); for (Command cmd : commands) { Collector collector = cmd.createSecondPassCollector(); if (collector != null) collectors.add(collector); } if (!collectors.isEmpty()) { Collector secondPhaseCollectors = MultiCollector .wrap(collectors.toArray(new Collector[collectors.size()])); if (collectors.size() > 0) { if (cachedCollector != null) { if (cachedCollector.isCached()) { cachedCollector.replay(secondPhaseCollectors); } else { signalCacheWarning = true; logger.warn(String.format(Locale.ROOT, "The grouping cache is active, but not used because it exceeded the max cache limit of %d percent", maxDocsPercentageToCache)); logger.warn("Please increase cache size or disable group caching."); searchWithTimeLimiter(luceneFilter, secondPhaseCollectors); } } else { if (pf.postFilter != null) { pf.postFilter.setLastDelegate(secondPhaseCollectors); secondPhaseCollectors = pf.postFilter; } searchWithTimeLimiter(luceneFilter, secondPhaseCollectors); } } } for (Command cmd : commands) { cmd.finish(); } qr.groupedResults = grouped; if (getDocList) { int sz = idSet.size(); int[] ids = new int[sz]; int idx = 0; for (int val : idSet) { ids[idx++] = val; } qr.setDocList(new DocSlice(0, sz, ids, null, maxMatches, maxScore)); } }
From source file:org.elasticsearch.index.fielddata.plain.DoubleArrayIndexFieldData.java
License:Apache License
@Override public DoubleArrayAtomicFieldData loadDirect(AtomicReaderContext context) throws Exception { AtomicReader reader = context.reader(); Terms terms = reader.terms(getFieldNames().indexName()); DoubleArrayAtomicFieldData data = null; // TODO: Use an actual estimator to estimate before loading. NonEstimatingEstimator estimator = new NonEstimatingEstimator(breakerService.getBreaker()); if (terms == null) { data = DoubleArrayAtomicFieldData.empty(reader.maxDoc()); estimator.afterLoad(null, data.getMemorySizeInBytes()); return data; }/*from w w w . j ava 2 s. c o m*/ // TODO: how can we guess the number of terms? numerics end up creating more terms per value... final BigDoubleArrayList values = new BigDoubleArrayList(); values.add(0); // first "t" indicates null value final float acceptableTransientOverheadRatio = fieldDataType.getSettings().getAsFloat( "acceptable_transient_overhead_ratio", OrdinalsBuilder.DEFAULT_ACCEPTABLE_OVERHEAD_RATIO); OrdinalsBuilder builder = new OrdinalsBuilder(reader.maxDoc(), acceptableTransientOverheadRatio); boolean success = false; try { final BytesRefIterator iter = builder .buildFromTerms(getNumericType().wrapTermsEnum(terms.iterator(null))); BytesRef term; while ((term = iter.next()) != null) { values.add(NumericUtils.sortableLongToDouble(NumericUtils.prefixCodedToLong(term))); } Ordinals build = builder.build(fieldDataType.getSettings()); if (!build.isMultiValued() && CommonSettings.removeOrdsOnSingleValue(fieldDataType)) { Docs ordinals = build.ordinals(); final FixedBitSet set = builder.buildDocsWithValuesSet(); // there's sweet spot where due to low unique value count, using ordinals will consume less memory long singleValuesArraySize = reader.maxDoc() * RamUsageEstimator.NUM_BYTES_DOUBLE + (set == null ? 0 : RamUsageEstimator.sizeOf(set.getBits()) + RamUsageEstimator.NUM_BYTES_INT); long uniqueValuesArraySize = values.sizeInBytes(); long ordinalsSize = build.getMemorySizeInBytes(); if (uniqueValuesArraySize + ordinalsSize < singleValuesArraySize) { data = new DoubleArrayAtomicFieldData.WithOrdinals(values, reader.maxDoc(), build); success = true; return data; } int maxDoc = reader.maxDoc(); BigDoubleArrayList sValues = new BigDoubleArrayList(maxDoc); for (int i = 0; i < maxDoc; i++) { sValues.add(values.get(ordinals.getOrd(i))); } assert sValues.size() == maxDoc; if (set == null) { data = new DoubleArrayAtomicFieldData.Single(sValues, maxDoc, ordinals.getNumOrds()); } else { data = new DoubleArrayAtomicFieldData.SingleFixedSet(sValues, maxDoc, set, ordinals.getNumOrds()); } } else { data = new DoubleArrayAtomicFieldData.WithOrdinals(values, reader.maxDoc(), build); } success = true; return data; } finally { if (success) { estimator.afterLoad(null, data.getMemorySizeInBytes()); } builder.close(); } }
From source file:org.elasticsearch.index.fielddata.plain.FloatArrayIndexFieldData.java
License:Apache License
@Override public FloatArrayAtomicFieldData loadDirect(AtomicReaderContext context) throws Exception { AtomicReader reader = context.reader(); Terms terms = reader.terms(getFieldNames().indexName()); FloatArrayAtomicFieldData data = null; // TODO: Use an actual estimator to estimate before loading. NonEstimatingEstimator estimator = new NonEstimatingEstimator(breakerService.getBreaker()); if (terms == null) { data = FloatArrayAtomicFieldData.empty(reader.maxDoc()); estimator.afterLoad(null, data.getMemorySizeInBytes()); return data; }/* w w w . j a v a 2s . c om*/ // TODO: how can we guess the number of terms? numerics end up creating more terms per value... final BigFloatArrayList values = new BigFloatArrayList(); values.add(0); // first "t" indicates null value final float acceptableTransientOverheadRatio = fieldDataType.getSettings().getAsFloat( "acceptable_transient_overhead_ratio", OrdinalsBuilder.DEFAULT_ACCEPTABLE_OVERHEAD_RATIO); OrdinalsBuilder builder = new OrdinalsBuilder(reader.maxDoc(), acceptableTransientOverheadRatio); boolean success = false; try { BytesRefIterator iter = builder.buildFromTerms(getNumericType().wrapTermsEnum(terms.iterator(null))); BytesRef term; while ((term = iter.next()) != null) { values.add(NumericUtils.sortableIntToFloat(NumericUtils.prefixCodedToInt(term))); } Ordinals build = builder.build(fieldDataType.getSettings()); if (!build.isMultiValued() && CommonSettings.removeOrdsOnSingleValue(fieldDataType)) { Docs ordinals = build.ordinals(); final FixedBitSet set = builder.buildDocsWithValuesSet(); // there's sweet spot where due to low unique value count, using ordinals will consume less memory long singleValuesArraySize = reader.maxDoc() * RamUsageEstimator.NUM_BYTES_FLOAT + (set == null ? 0 : RamUsageEstimator.sizeOf(set.getBits()) + RamUsageEstimator.NUM_BYTES_INT); long uniqueValuesArraySize = values.sizeInBytes(); long ordinalsSize = build.getMemorySizeInBytes(); if (uniqueValuesArraySize + ordinalsSize < singleValuesArraySize) { data = new FloatArrayAtomicFieldData.WithOrdinals(values, reader.maxDoc(), build); success = true; return data; } int maxDoc = reader.maxDoc(); BigFloatArrayList sValues = new BigFloatArrayList(maxDoc); for (int i = 0; i < maxDoc; i++) { sValues.add(values.get(ordinals.getOrd(i))); } assert sValues.size() == maxDoc; if (set == null) { data = new FloatArrayAtomicFieldData.Single(sValues, maxDoc, ordinals.getNumOrds()); } else { data = new FloatArrayAtomicFieldData.SingleFixedSet(sValues, maxDoc, set, ordinals.getNumOrds()); } } else { data = new FloatArrayAtomicFieldData.WithOrdinals(values, reader.maxDoc(), build); } success = true; return data; } finally { if (success) { estimator.afterLoad(null, data.getMemorySizeInBytes()); } builder.close(); } }