List of usage examples for org.apache.lucene.util FixedBitSet cardinality
@Override public int cardinality()
From source file:com.floragunn.searchguard.configuration.DlsFlsFilterLeafReader.java
License:Open Source License
DlsFlsFilterLeafReader(final LeafReader delegate, final Set<String> includes, final Query dlsQuery) { super(delegate); flsEnabled = includes != null && !includes.isEmpty(); dlsEnabled = dlsQuery != null;/*w w w . j a va2s . co m*/ if (flsEnabled) { this.includes = includes.toArray(new String[0]); final FieldInfos infos = delegate.getFieldInfos(); final List<FieldInfo> fi = new ArrayList<FieldInfo>(infos.size()); for (final FieldInfo info : infos) { final String fname = info.name; if ((!WildcardMatcher.containsWildcard(fname) && includes.contains(fname)) || WildcardMatcher.matchAny(this.includes, fname)) { fi.add(info); } } this.flsFieldInfos = new FieldInfos(fi.toArray(new FieldInfo[0])); } else { this.includes = null; this.flsFieldInfos = null; } if (dlsEnabled) { try { //borrowed from Apache Lucene (Copyright Apache Software Foundation (ASF)) final IndexSearcher searcher = new IndexSearcher(this); searcher.setQueryCache(null); final boolean needsScores = false; final Weight preserveWeight = searcher.createNormalizedWeight(dlsQuery, needsScores); final int maxDoc = in.maxDoc(); final FixedBitSet bits = new FixedBitSet(maxDoc); final Scorer preverveScorer = preserveWeight.scorer(this.getContext()); if (preverveScorer != null) { bits.or(preverveScorer.iterator()); } if (in.hasDeletions()) { final Bits oldLiveDocs = in.getLiveDocs(); assert oldLiveDocs != null; final DocIdSetIterator it = new BitSetIterator(bits, 0L); for (int i = it.nextDoc(); i != DocIdSetIterator.NO_MORE_DOCS; i = it.nextDoc()) { if (!oldLiveDocs.get(i)) { bits.clear(i); } } } this.liveDocs = bits; this.numDocs = bits.cardinality(); } catch (Exception e) { throw new RuntimeException(e); } } else { this.liveDocs = null; this.numDocs = -1; } }
From source file:de.unihildesheim.iw.lucene.document.FeedbackQuery.java
License:Open Source License
/** * Tries to get the minimum number of document without {@link * RelaxableQuery#relax() relaxing} the query. If the minimum number of * documents is not reached without relaxing at most the maximum number of * documents is returned while relaxing the query. * * @param searcher Searcher to issue queries * @param query Relaxable query to get matching documents * @param minDocs Minimum number of documents to get. Must be greater than * zero./* w ww. j ava 2s . c o m*/ * @param maxDocCount Maximum number of documents to get. {@code -1} for * unlimited or greater than zero. * @return List of documents matching the (relaxed) query. Ranking order is * not preserved! * @throws IOException Thrown on low-level I/O errors */ public static DocIdSet getMinMax(@NotNull final IndexSearcher searcher, @NotNull final RelaxableQuery query, final int minDocs, final int maxDocCount) throws IOException { final int maxDocs; if (maxDocCount == -1) { maxDocs = Integer.MAX_VALUE; } else if (maxDocCount < 0) { throw new IllegalArgumentException( "Maximum number of documents must " + "be -1 (unlimited) or greater than zero."); } else if (maxDocCount < minDocs) { throw new IllegalArgumentException( "Maximum number of documents must " + "be greater than minimum value."); } else { maxDocs = maxDocCount; } if (minDocs <= 0) { throw new IllegalArgumentException("Minimum number of documents must be" + " greater than zero."); } final int maxRetDocs = getMaxDocs(searcher.getIndexReader(), maxDocs); final FixedBitSet bits = new FixedBitSet(searcher.getIndexReader().maxDoc()); bits.or(BitsUtils.arrayToBits(getDocs(searcher, query.getQueryObj(), maxRetDocs))); // build a log-info string final String logInfo = "Got {} matching feedback documents. " + "Relaxing query to " + (maxDocCount > 0 ? "get additional" : "reach the minimum of") + " {} feedback documents..."; int docsToGet; int bitsCount; while ((bitsCount = bits.cardinality()) < minDocs && query.relax()) { docsToGet = maxRetDocs - bitsCount; LOG.info(logInfo, bitsCount, docsToGet); final int[] docs = getDocs(searcher, query.getQueryObj(), maxRetDocs); int maxAdd = maxDocs - bitsCount; for (int i = docs.length - 1; i >= 0 && maxAdd > 0; i--) { if (!bits.getAndSet(docs[i])) { maxAdd--; } } } LOG.info("Returning {} documents.", bits.cardinality()); return new BitDocIdSet(bits); }
From source file:de.unihildesheim.iw.lucene.query.QueryUtils.java
License:Open Source License
/** * Remove terms from the given collection, if they are not found in the * collection./*w w w . j a va 2s. c o m*/ * * @param dataProv IndexDataProvider * @param terms Collection of terms to check against the collection * @return Passed in terms with non-collection terms removed */ @SuppressFBWarnings("LO_APPENDED_STRING_IN_FORMAT_STRING") private static BytesRefArray removeUnknownTerms(@NotNull final IndexDataProvider dataProv, @NotNull final BytesRefArray terms) { final StringBuilder sb = new StringBuilder("Skipped terms (stopword or not in collection): ["); final FixedBitSet bits = new FixedBitSet(terms.size()); final BytesRefBuilder spare = new BytesRefBuilder(); BytesRef term; if (terms.size() == 0) { return terms; } else { for (int i = terms.size() - 1; i >= 0; i--) { term = terms.get(spare, i); if (dataProv.getTermFrequency(term) <= 0L) { sb.append(term.utf8ToString()).append(' '); bits.set(i); } } if (bits.cardinality() > 0) { LOG.warn(sb.toString().trim() + "]."); final BytesRefArray cleanTerms = new BytesRefArray(Counter.newCounter(false)); for (int i = terms.size() - 1; i >= 0; i--) { if (!bits.get(i)) { term = terms.get(spare, i); cleanTerms.append(term); // copies bytes } } return cleanTerms; } return terms; } }
From source file:de.unihildesheim.iw.lucene.util.BitsUtilsTest.java
License:Open Source License
@SuppressWarnings("ImplicitNumericConversion") @Test// ww w. j av a 2 s . com public void testBits2BitSet() throws Exception { final FixedBitSet fbs = new FixedBitSet(11); fbs.set(1); fbs.set(3); fbs.set(6); fbs.set(7); fbs.set(8); fbs.set(10); final BitSet result = BitsUtils.bits2BitSet(fbs); Assert.assertEquals("Bit count mismatch.", fbs.cardinality(), result.cardinality()); for (int i = 0; i < 11; i++) { Assert.assertEquals("Bits mismatch.", fbs.get(i), result.get(i)); } }
From source file:org.apache.solr.legacy.TestLegacyNumericUtils.java
License:Apache License
/** Note: The neededBounds Iterable must be unsigned (easier understanding what's happening) */ private void assertIntRangeSplit(final int lower, final int upper, int precisionStep, final boolean useBitSet, final Iterable<Integer> expectedBounds, final Iterable<Integer> expectedShifts) { final FixedBitSet bits = useBitSet ? new FixedBitSet(upper - lower + 1) : null; final Iterator<Integer> neededBounds = (expectedBounds == null) ? null : expectedBounds.iterator(); final Iterator<Integer> neededShifts = (expectedShifts == null) ? null : expectedShifts.iterator(); LegacyNumericUtils.splitIntRange(new LegacyNumericUtils.IntRangeBuilder() { @Override/*from w w w . ja va2s .co m*/ public void addRange(int min, int max, int shift) { assertTrue("min, max should be inside bounds", min >= lower && min <= upper && max >= lower && max <= upper); if (useBitSet) for (int i = min; i <= max; i++) { assertFalse("ranges should not overlap", bits.getAndSet(i - lower)); // extra exit condition to prevent overflow on MAX_VALUE if (i == max) break; } if (neededBounds == null) return; // make unsigned ints for easier display and understanding min ^= 0x80000000; max ^= 0x80000000; //System.out.println("0x"+Integer.toHexString(min>>>shift)+",0x"+Integer.toHexString(max>>>shift)+")/*shift="+shift+"*/,"); assertEquals("shift", neededShifts.next().intValue(), shift); assertEquals("inner min bound", neededBounds.next().intValue(), min >>> shift); assertEquals("inner max bound", neededBounds.next().intValue(), max >>> shift); } }, precisionStep, lower, upper); if (useBitSet) { // after flipping all bits in the range, the cardinality should be zero bits.flip(0, upper - lower + 1); assertEquals("The sub-range concenated should match the whole range", 0, bits.cardinality()); } }
From source file:org.apache.solr.search.facet.UniqueSlotAcc.java
License:Apache License
@Override public Object getValue(int slot) throws IOException { if (fcontext.isShard()) { return getShardValue(slot); }//from www . j av a 2s .co m if (counts != null) { // will only be pre-populated if this was used for sorting. return counts[slot]; } FixedBitSet bs = arr[slot]; return bs == null ? 0 : bs.cardinality(); }
From source file:org.apache.solr.search.facet.UniqueSlotAcc.java
License:Apache License
private Object getShardValue(int slot) throws IOException { if (factory != null) return getShardHLL(slot); FixedBitSet ords = arr[slot]; int unique;/*from w ww. j a va2s. co m*/ if (counts != null) { unique = counts[slot]; } else { unique = ords == null ? 0 : ords.cardinality(); } SimpleOrderedMap map = new SimpleOrderedMap(); map.add("unique", unique); map.add("nTerms", nTerms); int maxExplicit = 100; // TODO: make configurable // TODO: share values across buckets if (unique > 0) { List lst = new ArrayList(Math.min(unique, maxExplicit)); long maxOrd = ords.length(); if (ords != null && ords.length() > 0) { for (int ord = 0; lst.size() < maxExplicit;) { ord = ords.nextSetBit(ord); if (ord == DocIdSetIterator.NO_MORE_DOCS) break; BytesRef val = lookupOrd(ord); Object o = field.getType().toObject(field, val); lst.add(o); if (++ord >= maxOrd) break; } } map.add("vals", lst); } return map; }
From source file:org.apache.solr.search.facet.UniqueSlotAcc.java
License:Apache License
public void calcCounts() { counts = new int[arr.length]; for (int i = 0; i < arr.length; i++) { FixedBitSet bs = arr[i]; counts[i] = bs == null ? 0 : bs.cardinality(); }//from w w w . ja v a 2s .c o m }
From source file:org.elasticsearch.action.updatebyquery.TransportShardUpdateByQueryAction.java
License:Apache License
private void doExecuteInternal(ShardUpdateByQueryRequest request, ActionListener<ShardUpdateByQueryResponse> listener) { IndexService indexService = indicesService.indexServiceSafe(request.index()); IndexShard indexShard = indexService.shardSafe(request.shardId()); ShardSearchRequest shardSearchRequest = new ShardSearchLocalRequest(request.types(), request.nowInMillis(), request.filteringAliases()); SearchContext searchContext = new DefaultSearchContext(0, shardSearchRequest, null, indexShard.acquireSearcher("update_by_query"), indexService, indexShard, scriptService, cacheRecycler, pageCacheRecycler, bigArrays, threadPool.estimatedTimeInMillisCounter()); SearchContext.setCurrent(searchContext); try {//from w w w . j a v a2 s . c o m UpdateByQueryContext ubqContext = parseRequestSource(indexService, request, searchContext); searchContext.preProcess(); // TODO: Work per segment. The collector should collect docs per segment instead of one big set of top level ids TopLevelFixedBitSetCollector bitSetCollector = new TopLevelFixedBitSetCollector( searchContext.searcher().getIndexReader().maxDoc()); searchContext.searcher().search(searchContext.query(), searchContext.aliasFilter(), bitSetCollector); FixedBitSet docsToUpdate = bitSetCollector.getBitSet(); int docsToUpdateCount = docsToUpdate.cardinality(); logger.trace("[{}][{}] {} docs to update", request.index(), request.shardId(), docsToUpdateCount); if (docsToUpdateCount == 0) { ShardUpdateByQueryResponse response = new ShardUpdateByQueryResponse(request.shardId()); listener.onResponse(response); searchContext.close(); return; } BatchedShardUpdateByQueryExecutor bulkExecutor = new BatchedShardUpdateByQueryExecutor(listener, docsToUpdate, request, ubqContext); bulkExecutor.executeBulkIndex(); } catch (Throwable t) { // If we end up here then BatchedShardUpdateByQueryExecutor#finalizeBulkActions isn't invoked // so we need to release the search context. searchContext.close(); listener.onFailure(t); } finally { SearchContext.removeCurrent(); } }
From source file:org.elasticsearch.common.lucene.search.AndDocIdSetTests.java
License:Apache License
public void testDuel() throws IOException { for (int iter = 0; iter < 1000; ++iter) { final int numSets = 1 + random().nextInt(5); final int numDocs = 1 + random().nextInt(1000); FixedBitSet anded = new FixedBitSet(numDocs); anded.set(0, numDocs);//from w w w.j a v a 2 s . c o m final DocIdSet[] sets = new DocIdSet[numSets]; for (int i = 0; i < numSets; ++i) { final FixedBitSet randomSet = randomBitSet(numDocs); anded.and(randomSet); if (random().nextBoolean()) { // will be considered 'fast' by AndDocIdSet sets[i] = new BitDocIdSet(randomSet); } else { // will be considered 'slow' by AndDocIdSet sets[i] = new DocValuesDocIdSet(numDocs, null) { @Override protected boolean matchDoc(int doc) { return randomSet.get(doc); } }; } } AndDocIdSet andSet = new AndDocIdSet(sets); Bits andBits = andSet.bits(); if (andBits != null) { for (int i = 0; i < numDocs; ++i) { assertEquals(anded.get(i), andBits.get(i)); } } DocIdSetIterator andIt = andSet.iterator(); if (andIt == null) { assertEquals(0, anded.cardinality()); } else { int previous = -1; for (int doc = andIt.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = andIt.nextDoc()) { for (int j = previous + 1; j < doc; ++j) { assertFalse(anded.get(j)); } assertTrue(anded.get(doc)); previous = doc; } for (int j = previous + 1; j < numDocs; ++j) { assertFalse(anded.get(j)); } } } }