List of usage examples for org.apache.lucene.util FixedBitSet or
public void or(FixedBitSet other)
From source file:DocIdSetBenchmark.java
License:Apache License
public static long scoreBuildFixedBitSet(DocIdSet set, int maxDoc) throws IOException { final long start = System.nanoTime(); int dummy = 0; long score = 0; while (System.nanoTime() - start < SECOND) { final FixedBitSet copy = new FixedBitSet(maxDoc); DocIdSetIterator iterator = set.iterator(); if (iterator != null) { copy.or(iterator); }//from w w w. java 2 s . c o m dummy += copy.hashCode(); ++score; } DUMMY += dummy; return score; }
From source file:com.floragunn.searchguard.configuration.DlsFlsFilterLeafReader.java
License:Open Source License
DlsFlsFilterLeafReader(final LeafReader delegate, final Set<String> includes, final Query dlsQuery) { super(delegate); flsEnabled = includes != null && !includes.isEmpty(); dlsEnabled = dlsQuery != null;//from w w w .j a va 2 s .co m if (flsEnabled) { this.includes = includes.toArray(new String[0]); final FieldInfos infos = delegate.getFieldInfos(); final List<FieldInfo> fi = new ArrayList<FieldInfo>(infos.size()); for (final FieldInfo info : infos) { final String fname = info.name; if ((!WildcardMatcher.containsWildcard(fname) && includes.contains(fname)) || WildcardMatcher.matchAny(this.includes, fname)) { fi.add(info); } } this.flsFieldInfos = new FieldInfos(fi.toArray(new FieldInfo[0])); } else { this.includes = null; this.flsFieldInfos = null; } if (dlsEnabled) { try { //borrowed from Apache Lucene (Copyright Apache Software Foundation (ASF)) final IndexSearcher searcher = new IndexSearcher(this); searcher.setQueryCache(null); final boolean needsScores = false; final Weight preserveWeight = searcher.createNormalizedWeight(dlsQuery, needsScores); final int maxDoc = in.maxDoc(); final FixedBitSet bits = new FixedBitSet(maxDoc); final Scorer preverveScorer = preserveWeight.scorer(this.getContext()); if (preverveScorer != null) { bits.or(preverveScorer.iterator()); } if (in.hasDeletions()) { final Bits oldLiveDocs = in.getLiveDocs(); assert oldLiveDocs != null; final DocIdSetIterator it = new BitSetIterator(bits, 0L); for (int i = it.nextDoc(); i != DocIdSetIterator.NO_MORE_DOCS; i = it.nextDoc()) { if (!oldLiveDocs.get(i)) { bits.clear(i); } } } this.liveDocs = bits; this.numDocs = bits.cardinality(); } catch (Exception e) { throw new RuntimeException(e); } } else { this.liveDocs = null; this.numDocs = -1; } }
From source file:de.unihildesheim.iw.lucene.document.FeedbackQuery.java
License:Open Source License
/** * Tries to get the minimum number of document without {@link * RelaxableQuery#relax() relaxing} the query. If the minimum number of * documents is not reached without relaxing at most the maximum number of * documents is returned while relaxing the query. * * @param searcher Searcher to issue queries * @param query Relaxable query to get matching documents * @param minDocs Minimum number of documents to get. Must be greater than * zero.//from w ww .j av a2 s . c o m * @param maxDocCount Maximum number of documents to get. {@code -1} for * unlimited or greater than zero. * @return List of documents matching the (relaxed) query. Ranking order is * not preserved! * @throws IOException Thrown on low-level I/O errors */ public static DocIdSet getMinMax(@NotNull final IndexSearcher searcher, @NotNull final RelaxableQuery query, final int minDocs, final int maxDocCount) throws IOException { final int maxDocs; if (maxDocCount == -1) { maxDocs = Integer.MAX_VALUE; } else if (maxDocCount < 0) { throw new IllegalArgumentException( "Maximum number of documents must " + "be -1 (unlimited) or greater than zero."); } else if (maxDocCount < minDocs) { throw new IllegalArgumentException( "Maximum number of documents must " + "be greater than minimum value."); } else { maxDocs = maxDocCount; } if (minDocs <= 0) { throw new IllegalArgumentException("Minimum number of documents must be" + " greater than zero."); } final int maxRetDocs = getMaxDocs(searcher.getIndexReader(), maxDocs); final FixedBitSet bits = new FixedBitSet(searcher.getIndexReader().maxDoc()); bits.or(BitsUtils.arrayToBits(getDocs(searcher, query.getQueryObj(), maxRetDocs))); // build a log-info string final String logInfo = "Got {} matching feedback documents. " + "Relaxing query to " + (maxDocCount > 0 ? "get additional" : "reach the minimum of") + " {} feedback documents..."; int docsToGet; int bitsCount; while ((bitsCount = bits.cardinality()) < minDocs && query.relax()) { docsToGet = maxRetDocs - bitsCount; LOG.info(logInfo, bitsCount, docsToGet); final int[] docs = getDocs(searcher, query.getQueryObj(), maxRetDocs); int maxAdd = maxDocs - bitsCount; for (int i = docs.length - 1; i >= 0 && maxAdd > 0; i--) { if (!bits.getAndSet(docs[i])) { maxAdd--; } } } LOG.info("Returning {} documents.", bits.cardinality()); return new BitDocIdSet(bits); }
From source file:net.conquiris.lucene.search.NegatingFilter.java
License:Apache License
@Override public DocIdSet getDocIdSet(IndexReader reader) throws IOException { final int n = reader.maxDoc(); final FixedBitSet bits = new FixedBitSet(reader.maxDoc()); final DocIdSet set = filter.getDocIdSet(reader); if (set == null || set == DocIdSet.EMPTY_DOCIDSET) { bits.set(0, n);/* w w w. j a v a 2s. com*/ } else { DocIdSetIterator i = set.iterator(); if (i == null) { bits.set(0, n); } else { bits.or(i); bits.flip(0, n); } } return bits; }
From source file:org.alfresco.solr.query.BitsFilter.java
License:Open Source License
public void or(BitsFilter bitsFilter) { List<FixedBitSet> andSets = bitsFilter.bitSets; for (int i = 0; i < bitSets.size(); i++) { FixedBitSet a = bitSets.get(i); FixedBitSet b = andSets.get(i);// w ww .j ava2 s .c om a.or(b); } }
From source file:org.elasticsearch.common.lucene.search.XBooleanFilter.java
License:Apache License
/** * Returns the a DocIdSetIterator representing the Boolean composition * of the filters that have been added./* ww w .ja v a 2 s . c om*/ */ @Override public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs) throws IOException { FixedBitSet res = null; final AtomicReader reader = context.reader(); // optimize single case... if (clauses.size() == 1) { FilterClause clause = clauses.get(0); DocIdSet set = clause.getFilter().getDocIdSet(context, acceptDocs); if (clause.getOccur() == Occur.MUST_NOT) { if (DocIdSets.isEmpty(set)) { return new AllDocIdSet(reader.maxDoc()); } else { return new NotDocIdSet(set, reader.maxDoc()); } } // SHOULD or MUST, just return the set... if (DocIdSets.isEmpty(set)) { return null; } return set; } // first, go over and see if we can shortcut the execution // and gather Bits if we need to List<ResultClause> results = new ArrayList<ResultClause>(clauses.size()); boolean hasShouldClauses = false; boolean hasNonEmptyShouldClause = false; boolean hasMustClauses = false; boolean hasMustNotClauses = false; for (int i = 0; i < clauses.size(); i++) { FilterClause clause = clauses.get(i); DocIdSet set = clause.getFilter().getDocIdSet(context, acceptDocs); if (clause.getOccur() == Occur.MUST) { hasMustClauses = true; if (DocIdSets.isEmpty(set)) { return null; } } else if (clause.getOccur() == Occur.SHOULD) { hasShouldClauses = true; if (DocIdSets.isEmpty(set)) { continue; } hasNonEmptyShouldClause = true; } else if (clause.getOccur() == Occur.MUST_NOT) { hasMustNotClauses = true; if (DocIdSets.isEmpty(set)) { // we mark empty ones as null for must_not, handle it in the next run... results.add(new ResultClause(null, null, clause)); continue; } } Bits bits = null; if (!DocIdSets.isFastIterator(set)) { bits = set.bits(); } results.add(new ResultClause(set, bits, clause)); } if (hasShouldClauses && !hasNonEmptyShouldClause) { return null; } // now, go over the clauses and apply the "fast" ones first... hasNonEmptyShouldClause = false; boolean hasBits = false; // But first we need to handle the "fast" should clauses, otherwise a should clause can unset docs // that don't match with a must or must_not clause. List<ResultClause> fastOrClauses = new ArrayList<ResultClause>(); for (int i = 0; i < results.size(); i++) { ResultClause clause = results.get(i); // we apply bits in based ones (slow) in the second run if (clause.bits != null) { hasBits = true; continue; } if (clause.clause.getOccur() == Occur.SHOULD) { if (hasMustClauses || hasMustNotClauses) { fastOrClauses.add(clause); } else if (res == null) { DocIdSetIterator it = clause.docIdSet.iterator(); if (it != null) { hasNonEmptyShouldClause = true; res = new FixedBitSet(reader.maxDoc()); res.or(it); } } else { DocIdSetIterator it = clause.docIdSet.iterator(); if (it != null) { hasNonEmptyShouldClause = true; res.or(it); } } } } // Now we safely handle the "fast" must and must_not clauses. for (int i = 0; i < results.size(); i++) { ResultClause clause = results.get(i); // we apply bits in based ones (slow) in the second run if (clause.bits != null) { hasBits = true; continue; } if (clause.clause.getOccur() == Occur.MUST) { DocIdSetIterator it = clause.docIdSet.iterator(); if (it == null) { return null; } if (res == null) { res = new FixedBitSet(reader.maxDoc()); res.or(it); } else { res.and(it); } } else if (clause.clause.getOccur() == Occur.MUST_NOT) { if (res == null) { res = new FixedBitSet(reader.maxDoc()); res.set(0, reader.maxDoc()); // NOTE: may set bits on deleted docs } if (clause.docIdSet != null) { DocIdSetIterator it = clause.docIdSet.iterator(); if (it != null) { res.andNot(it); } } } } if (!hasBits) { if (!fastOrClauses.isEmpty()) { DocIdSetIterator it = res.iterator(); at_least_one_should_clause_iter: for (int setDoc = it .nextDoc(); setDoc != DocIdSetIterator.NO_MORE_DOCS; setDoc = it.nextDoc()) { for (ResultClause fastOrClause : fastOrClauses) { DocIdSetIterator clauseIterator = fastOrClause.iterator(); if (clauseIterator == null) { continue; } if (iteratorMatch(clauseIterator, setDoc)) { hasNonEmptyShouldClause = true; continue at_least_one_should_clause_iter; } } res.clear(setDoc); } } if (hasShouldClauses && !hasNonEmptyShouldClause) { return null; } else { return res; } } // we have some clauses with bits, apply them... // we let the "res" drive the computation, and check Bits for that List<ResultClause> slowOrClauses = new ArrayList<ResultClause>(); for (int i = 0; i < results.size(); i++) { ResultClause clause = results.get(i); if (clause.bits == null) { continue; } if (clause.clause.getOccur() == Occur.SHOULD) { if (hasMustClauses || hasMustNotClauses) { slowOrClauses.add(clause); } else { if (res == null) { DocIdSetIterator it = clause.docIdSet.iterator(); if (it == null) { continue; } hasNonEmptyShouldClause = true; res = new FixedBitSet(reader.maxDoc()); res.or(it); } else { for (int doc = 0; doc < reader.maxDoc(); doc++) { if (!res.get(doc) && clause.bits.get(doc)) { hasNonEmptyShouldClause = true; res.set(doc); } } } } } else if (clause.clause.getOccur() == Occur.MUST) { if (res == null) { // nothing we can do, just or it... res = new FixedBitSet(reader.maxDoc()); DocIdSetIterator it = clause.docIdSet.iterator(); if (it == null) { return null; } res.or(it); } else { Bits bits = clause.bits; // use the "res" to drive the iteration DocIdSetIterator it = res.iterator(); for (int doc = it.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = it.nextDoc()) { if (!bits.get(doc)) { res.clear(doc); } } } } else if (clause.clause.getOccur() == Occur.MUST_NOT) { if (res == null) { res = new FixedBitSet(reader.maxDoc()); res.set(0, reader.maxDoc()); // NOTE: may set bits on deleted docs DocIdSetIterator it = clause.docIdSet.iterator(); if (it != null) { res.andNot(it); } } else { Bits bits = clause.bits; // let res drive the iteration DocIdSetIterator it = res.iterator(); for (int doc = it.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = it.nextDoc()) { if (bits.get(doc)) { res.clear(doc); } } } } } // From a boolean_logic behavior point of view a should clause doesn't have impact on a bool filter if there // is already a must or must_not clause. However in the current ES bool filter behaviour at least one should // clause must match in order for a doc to be a match. What we do here is checking if matched docs match with // any should filter. TODO: Add an option to have disable minimum_should_match=1 behaviour if (!slowOrClauses.isEmpty() || !fastOrClauses.isEmpty()) { DocIdSetIterator it = res.iterator(); at_least_one_should_clause_iter: for (int setDoc = it .nextDoc(); setDoc != DocIdSetIterator.NO_MORE_DOCS; setDoc = it.nextDoc()) { for (ResultClause fastOrClause : fastOrClauses) { DocIdSetIterator clauseIterator = fastOrClause.iterator(); if (clauseIterator == null) { continue; } if (iteratorMatch(clauseIterator, setDoc)) { hasNonEmptyShouldClause = true; continue at_least_one_should_clause_iter; } } for (ResultClause slowOrClause : slowOrClauses) { if (slowOrClause.bits.get(setDoc)) { hasNonEmptyShouldClause = true; continue at_least_one_should_clause_iter; } } res.clear(setDoc); } } if (hasShouldClauses && !hasNonEmptyShouldClause) { return null; } else { return res; } }
From source file:org.elasticsearch.common.lucene.search.XBooleanFilterTests.java
License:Apache License
@Test public void testWithTwoClausesOfEachOccur_allFixedBitsetFilters() throws Exception { List<XBooleanFilter> booleanFilters = new ArrayList<XBooleanFilter>(); booleanFilters//w ww .ja va 2 s .c o m .add(createBooleanFilter(newFilterClause(0, 'a', MUST, false), newFilterClause(1, 'b', MUST, false), newFilterClause(2, 'c', SHOULD, false), newFilterClause(3, 'd', SHOULD, false), newFilterClause(4, 'e', MUST_NOT, false), newFilterClause(5, 'f', MUST_NOT, false))); booleanFilters.add(createBooleanFilter(newFilterClause(4, 'e', MUST_NOT, false), newFilterClause(5, 'f', MUST_NOT, false), newFilterClause(0, 'a', MUST, false), newFilterClause(1, 'b', MUST, false), newFilterClause(2, 'c', SHOULD, false), newFilterClause(3, 'd', SHOULD, false))); booleanFilters.add( createBooleanFilter(newFilterClause(2, 'c', SHOULD, false), newFilterClause(3, 'd', SHOULD, false), newFilterClause(4, 'e', MUST_NOT, false), newFilterClause(5, 'f', MUST_NOT, false), newFilterClause(0, 'a', MUST, false), newFilterClause(1, 'b', MUST, false))); for (XBooleanFilter booleanFilter : booleanFilters) { FixedBitSet result = new FixedBitSet(reader.maxDoc()); result.or(booleanFilter.getDocIdSet(reader.getContext(), reader.getLiveDocs()).iterator()); assertThat(result.cardinality(), equalTo(2)); assertThat(result.get(0), equalTo(true)); assertThat(result.get(1), equalTo(true)); assertThat(result.get(2), equalTo(false)); } }
From source file:org.elasticsearch.common.lucene.search.XBooleanFilterTests.java
License:Apache License
@Test public void testWithTwoClausesOfEachOccur_allBitsBasedFilters() throws Exception { List<XBooleanFilter> booleanFilters = new ArrayList<XBooleanFilter>(); booleanFilters/*from www . ja v a 2 s. co m*/ .add(createBooleanFilter(newFilterClause(0, 'a', MUST, true), newFilterClause(1, 'b', MUST, true), newFilterClause(2, 'c', SHOULD, true), newFilterClause(3, 'd', SHOULD, true), newFilterClause(4, 'e', MUST_NOT, true), newFilterClause(5, 'f', MUST_NOT, true))); booleanFilters.add(createBooleanFilter(newFilterClause(4, 'e', MUST_NOT, true), newFilterClause(5, 'f', MUST_NOT, true), newFilterClause(0, 'a', MUST, true), newFilterClause(1, 'b', MUST, true), newFilterClause(2, 'c', SHOULD, true), newFilterClause(3, 'd', SHOULD, true))); booleanFilters.add( createBooleanFilter(newFilterClause(2, 'c', SHOULD, true), newFilterClause(3, 'd', SHOULD, true), newFilterClause(4, 'e', MUST_NOT, true), newFilterClause(5, 'f', MUST_NOT, true), newFilterClause(0, 'a', MUST, true), newFilterClause(1, 'b', MUST, true))); for (XBooleanFilter booleanFilter : booleanFilters) { FixedBitSet result = new FixedBitSet(reader.maxDoc()); result.or(booleanFilter.getDocIdSet(reader.getContext(), reader.getLiveDocs()).iterator()); assertThat(result.cardinality(), equalTo(2)); assertThat(result.get(0), equalTo(true)); assertThat(result.get(1), equalTo(true)); assertThat(result.get(2), equalTo(false)); } }
From source file:org.elasticsearch.common.lucene.search.XBooleanFilterTests.java
License:Apache License
@Test public void testWithTwoClausesOfEachOccur_allFilterTypes() throws Exception { List<XBooleanFilter> booleanFilters = new ArrayList<XBooleanFilter>(); booleanFilters/*from w w w .j a va2 s . c o m*/ .add(createBooleanFilter(newFilterClause(0, 'a', MUST, true), newFilterClause(1, 'b', MUST, false), newFilterClause(2, 'c', SHOULD, true), newFilterClause(3, 'd', SHOULD, false), newFilterClause(4, 'e', MUST_NOT, true), newFilterClause(5, 'f', MUST_NOT, false))); booleanFilters.add(createBooleanFilter(newFilterClause(4, 'e', MUST_NOT, true), newFilterClause(5, 'f', MUST_NOT, false), newFilterClause(0, 'a', MUST, true), newFilterClause(1, 'b', MUST, false), newFilterClause(2, 'c', SHOULD, true), newFilterClause(3, 'd', SHOULD, false))); booleanFilters.add( createBooleanFilter(newFilterClause(2, 'c', SHOULD, true), newFilterClause(3, 'd', SHOULD, false), newFilterClause(4, 'e', MUST_NOT, true), newFilterClause(5, 'f', MUST_NOT, false), newFilterClause(0, 'a', MUST, true), newFilterClause(1, 'b', MUST, false))); for (XBooleanFilter booleanFilter : booleanFilters) { FixedBitSet result = new FixedBitSet(reader.maxDoc()); result.or(booleanFilter.getDocIdSet(reader.getContext(), reader.getLiveDocs()).iterator()); assertThat(result.cardinality(), equalTo(2)); assertThat(result.get(0), equalTo(true)); assertThat(result.get(1), equalTo(true)); assertThat(result.get(2), equalTo(false)); } booleanFilters.clear(); booleanFilters .add(createBooleanFilter(newFilterClause(0, 'a', MUST, false), newFilterClause(1, 'b', MUST, true), newFilterClause(2, 'c', SHOULD, false), newFilterClause(3, 'd', SHOULD, true), newFilterClause(4, 'e', MUST_NOT, false), newFilterClause(5, 'f', MUST_NOT, true))); booleanFilters.add(createBooleanFilter(newFilterClause(4, 'e', MUST_NOT, false), newFilterClause(5, 'f', MUST_NOT, true), newFilterClause(0, 'a', MUST, false), newFilterClause(1, 'b', MUST, true), newFilterClause(2, 'c', SHOULD, false), newFilterClause(3, 'd', SHOULD, true))); booleanFilters.add( createBooleanFilter(newFilterClause(2, 'c', SHOULD, false), newFilterClause(3, 'd', SHOULD, true), newFilterClause(4, 'e', MUST_NOT, false), newFilterClause(5, 'f', MUST_NOT, true), newFilterClause(0, 'a', MUST, false), newFilterClause(1, 'b', MUST, true))); for (XBooleanFilter booleanFilter : booleanFilters) { FixedBitSet result = new FixedBitSet(reader.maxDoc()); result.or(booleanFilter.getDocIdSet(reader.getContext(), reader.getLiveDocs()).iterator()); assertThat(result.cardinality(), equalTo(2)); assertThat(result.get(0), equalTo(true)); assertThat(result.get(1), equalTo(true)); assertThat(result.get(2), equalTo(false)); } }
From source file:org.elasticsearch.common.lucene.search.XBooleanFilterTests.java
License:Apache License
@Test public void testWithTwoClausesOfEachOccur_singleClauseOptimisation() throws Exception { List<XBooleanFilter> booleanFilters = new ArrayList<XBooleanFilter>(); booleanFilters.add(createBooleanFilter(newFilterClause(1, 'b', MUST, true))); for (XBooleanFilter booleanFilter : booleanFilters) { FixedBitSet result = new FixedBitSet(reader.maxDoc()); result.or(booleanFilter.getDocIdSet(reader.getContext(), reader.getLiveDocs()).iterator()); assertThat(result.cardinality(), equalTo(2)); assertThat(result.get(0), equalTo(true)); assertThat(result.get(1), equalTo(true)); assertThat(result.get(2), equalTo(false)); }/*from w w w .j a v a2 s .c o m*/ booleanFilters.clear(); booleanFilters.add(createBooleanFilter(newFilterClause(1, 'c', MUST_NOT, true))); for (XBooleanFilter booleanFilter : booleanFilters) { FixedBitSet result = new FixedBitSet(reader.maxDoc()); result.or(booleanFilter.getDocIdSet(reader.getContext(), reader.getLiveDocs()).iterator()); assertThat(result.cardinality(), equalTo(3)); assertThat(result.get(0), equalTo(true)); assertThat(result.get(1), equalTo(true)); assertThat(result.get(2), equalTo(true)); } booleanFilters.clear(); booleanFilters.add(createBooleanFilter(newFilterClause(2, 'c', SHOULD, true))); for (XBooleanFilter booleanFilter : booleanFilters) { FixedBitSet result = new FixedBitSet(reader.maxDoc()); result.or(booleanFilter.getDocIdSet(reader.getContext(), reader.getLiveDocs()).iterator()); assertThat(result.cardinality(), equalTo(2)); assertThat(result.get(0), equalTo(true)); assertThat(result.get(1), equalTo(true)); assertThat(result.get(2), equalTo(false)); } }