Example usage for org.apache.lucene.search DocIdSetIterator nextDoc

Introduction

In this page you can find the example usage for org.apache.lucene.search DocIdSetIterator nextDoc.

Prototype

public abstract int nextDoc() throws IOException;

Source Link

Document

Advances to the next document in the set and returns the doc it is currently on, or #NO_MORE_DOCS if there are no more docs in the set.
NOTE: after the iterator has exhausted you should not call this method, as it may result in unpredicted behavior.

Usage

From source file:org.apache.blur.lucene.security.search.BitSetDocumentVisibilityFilterCacheStrategy.java

License:Apache License

@Override
public Builder createBuilder(String fieldName, BytesRef term, final AtomicReader reader) {
    final OpenBitSet bitSet = new OpenBitSet(reader.maxDoc());
    final Key key = new Key(fieldName, term, reader.getCoreCacheKey());
    LOG.debug("Creating new bitset for key [" + key + "] on index [" + reader + "]");
    return new Builder() {
        @Override/*from  w ww  . j a  v  a 2s .  c  o  m*/
        public void or(DocIdSetIterator it) throws IOException {
            int doc;
            while ((doc = it.nextDoc()) != DocsEnum.NO_MORE_DOCS) {
                bitSet.set(doc);
            }
        }

        @Override
        public DocIdSet getDocIdSet() throws IOException {
            reader.addReaderClosedListener(new ReaderClosedListener() {
                @Override
                public void onClose(IndexReader reader) {
                    LOG.debug("Removing old bitset for key [" + key + "]");
                    DocIdSet docIdSet = _cache.remove(key);
                    if (docIdSet == null) {
                        LOG.warn("DocIdSet was missing for key [" + docIdSet + "]");
                    }
                }
            });
            _cache.put(key, bitSet);
            return bitSet;
        }
    };
}

---

From source file:org.apache.blur.utils.BlurUtil.java

License:Apache License

private static OpenBitSet getMask(DocIdSet docIdSet, int primeDocRowId, int numberOfDocsInRow)
        throws IOException {
    OpenBitSet mask = new OpenBitSet(numberOfDocsInRow);
    DocIdSetIterator iterator = docIdSet.iterator();
    if (iterator == null) {
        return mask;
    }//from ww  w.j  a v a 2 s.c  o  m
    int docId = iterator.advance(primeDocRowId);
    int end = numberOfDocsInRow + primeDocRowId;
    while (docId < end) {
        mask.set(docId - primeDocRowId);
        docId = iterator.nextDoc();
    }
    return mask;
}

---

From source file:org.apache.solr.analytics.AnalyticsDriver.java

License:Apache License

/**
 * Drive the collection of reduction data. This includes overall data as well as faceted data.
 * /*  ww  w  .j  av  a  2 s  . c  om*/
 * @param manager of the request to drive
 * @param searcher the results of the query
 * @param filter that represents the overall query
 * @param queryRequest used for the search request
 * @throws IOException if an error occurs while reading from Solr
 */
public static void drive(AnalyticsRequestManager manager, SolrIndexSearcher searcher, Filter filter,
        SolrQueryRequest queryRequest) throws IOException {
    StreamingInfo streamingInfo = manager.getStreamingFacetInfo();
    Iterable<StreamingFacet> streamingFacets = streamingInfo.streamingFacets;
    ReductionCollectionManager collectionManager = streamingInfo.streamingCollectionManager;

    Iterable<FacetValueQueryExecuter> facetExecuters = manager.getFacetExecuters(filter, queryRequest);

    // Streaming phase (Overall results & Value/Pivot Facets)
    // Loop through all documents and collect reduction data for streaming facets and overall results
    if (collectionManager.needsCollection()) {
        List<LeafReaderContext> contexts = searcher.getTopReaderContext().leaves();
        for (int leafNum = 0; leafNum < contexts.size(); leafNum++) {
            LeafReaderContext context = contexts.get(leafNum);
            DocIdSet dis = filter.getDocIdSet(context, null); // solr docsets already exclude any deleted docs
            if (dis == null) {
                continue;
            }
            DocIdSetIterator disi = dis.iterator();
            if (disi != null) {
                collectionManager.doSetNextReader(context);
                int doc = disi.nextDoc();
                while (doc != DocIdSetIterator.NO_MORE_DOCS) {
                    // Add a document to the statistics being generated
                    collectionManager.collect(doc);
                    streamingFacets.forEach(facet -> facet.addFacetValueCollectionTargets());
                    collectionManager.apply();
                    doc = disi.nextDoc();
                }
            }
        }
    }

    // Executing phase (Query/Range Facets)
    // Send additional Solr Queries to compute facet values
    for (FacetValueQueryExecuter executer : facetExecuters) {
        executer.execute(searcher);
    }
}

---

From source file:org.apache.solr.analytics.function.field.AbstractAnalyticsFieldTest.java

License:Apache License

protected Set<String> collectFieldValues(AnalyticsField testField, Predicate<String> valuesFiller)
        throws IOException {
    StringField idField = new StringField("id");
    Filter filter = new QueryWrapperFilter(new MatchAllDocsQuery());
    Set<String> missing = new HashSet<>();

    List<LeafReaderContext> contexts = searcher.getTopReaderContext().leaves();
    for (int leafNum = 0; leafNum < contexts.size(); leafNum++) {
        LeafReaderContext context = contexts.get(leafNum);
        DocIdSet dis = filter.getDocIdSet(context, null); // solr docsets already exclude any deleted docs
        if (dis == null) {
            continue;
        }//  w  w  w . jav a  2  s  .co  m
        DocIdSetIterator disi = dis.iterator();
        if (disi != null) {
            testField.doSetNextReader(context);
            idField.doSetNextReader(context);
            int doc = disi.nextDoc();
            while (doc != DocIdSetIterator.NO_MORE_DOCS) {
                // Add a document to the statistics being generated
                testField.collect(doc);
                idField.collect(doc);

                String id = idField.getString();
                if (!valuesFiller.test(id)) {
                    missing.add(id);
                }
                doc = disi.nextDoc();
            }
        }
    }
    return missing;
}

---

From source file:org.apache.solr.analytics.request.AnalyticsStats.java

License:Apache License

/**
 * Calculates the analytics requested in the Parameters.
 * //from w  w w  .  j  a v  a 2  s .  c  om
 * @return List of results formated to mirror the input XML.
 * @throws IOException if execution fails
 */
public NamedList<?> execute() throws IOException {
    statsCollector.startRequest();
    NamedList<Object> res = new NamedList<>();
    List<AnalyticsRequest> requests;

    requests = AnalyticsRequestFactory.parse(searcher.getSchema(), params);

    if (requests == null || requests.size() == 0) {
        return res;
    }
    statsCollector.addRequests(requests.size());

    // Get filter to all docs
    Filter filter = docs.getTopFilter();

    // Computing each Analytics Request Seperately
    for (AnalyticsRequest areq : requests) {
        // The Accumulator which will control the statistics generation
        // for the entire analytics request
        ValueAccumulator accumulator;

        // The number of total facet requests
        int facets = areq.getFieldFacets().size() + areq.getRangeFacets().size() + areq.getQueryFacets().size();
        try {
            if (facets == 0) {
                accumulator = BasicAccumulator.create(searcher, docs, areq);
            } else {
                accumulator = FacetingAccumulator.create(searcher, docs, areq, req);
            }
        } catch (IOException e) {
            log.warn("Analytics request '" + areq.getName() + "' failed", e);
            continue;
        }

        statsCollector.addStatsCollected(((BasicAccumulator) accumulator).getNumStatsCollectors());
        statsCollector.addStatsRequests(areq.getExpressions().size());
        statsCollector.addFieldFacets(areq.getFieldFacets().size());
        statsCollector.addRangeFacets(areq.getRangeFacets().size());
        statsCollector.addQueryFacets(areq.getQueryFacets().size());
        statsCollector.addQueries(((BasicAccumulator) accumulator).getNumQueries());

        // Loop through the documents returned by the query and add to accumulator
        List<LeafReaderContext> contexts = searcher.getTopReaderContext().leaves();
        for (int leafNum = 0; leafNum < contexts.size(); leafNum++) {
            LeafReaderContext context = contexts.get(leafNum);
            DocIdSet dis = filter.getDocIdSet(context, null); // solr docsets already exclude any deleted docs
            DocIdSetIterator disi = null;
            if (dis != null) {
                disi = dis.iterator();
            }

            if (disi != null) {
                accumulator.getLeafCollector(context);
                int doc = disi.nextDoc();
                while (doc != DocIdSetIterator.NO_MORE_DOCS) {
                    // Add a document to the statistics being generated
                    accumulator.collect(doc);
                    doc = disi.nextDoc();
                }
            }
        }

        // do some post-processing
        accumulator.postProcess();

        // compute the stats
        accumulator.compute();

        res.add(areq.getName(), accumulator.export());
    }

    statsCollector.endRequest();
    return res;
}

---

From source file:org.apache.solr.handler.ExportWriter.java

License:Apache License

protected void writeDocs(SolrQueryRequest req, IteratorWriter.ItemWriter writer, Sort sort) throws IOException {
    //Write the data.
    List<LeafReaderContext> leaves = req.getSearcher().getTopReaderContext().leaves();
    SortDoc sortDoc = getSortDoc(req.getSearcher(), sort.getSort());
    int count = 0;
    int queueSize = 30000;
    SortQueue queue = new SortQueue(queueSize, sortDoc);
    SortDoc[] outDocs = new SortDoc[queueSize];

    while (count < totalHits) {
        //long begin = System.nanoTime();
        queue.reset();//w  w  w . ja  v  a 2  s . c o  m
        SortDoc top = queue.top();
        for (int i = 0; i < leaves.size(); i++) {
            sortDoc.setNextReader(leaves.get(i));
            DocIdSetIterator it = new BitSetIterator(sets[i], 0); // cost is not useful here
            int docId = -1;
            while ((docId = it.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
                sortDoc.setValues(docId);
                if (top.lessThan(sortDoc)) {
                    top.setValues(sortDoc);
                    top = queue.updateTop();
                }
            }
        }

        int outDocsIndex = -1;

        for (int i = 0; i < queueSize; i++) {
            SortDoc s = queue.pop();
            if (s.docId > -1) {
                outDocs[++outDocsIndex] = s;
            }
        }

        //long end = System.nanoTime();

        count += (outDocsIndex + 1);

        try {
            for (int i = outDocsIndex; i >= 0; --i) {
                SortDoc s = outDocs[i];
                writer.add((MapWriter) ew -> {
                    writeDoc(s, leaves, ew);
                    s.reset();
                });
            }
        } catch (Throwable e) {
            Throwable ex = e;
            e.printStackTrace();
            while (ex != null) {
                String m = ex.getMessage();
                if (m != null && m.contains("Broken pipe")) {
                    throw new IgnoreException();
                }
                ex = ex.getCause();
            }

            if (e instanceof IOException) {
                throw ((IOException) e);
            } else {
                throw new IOException(e);
            }
        }
    }
}

---

From source file:org.apache.solr.request.DocValuesFacets.java

License:Apache License

/** accumulates per-segment single-valued facet counts, mapping to global ordinal space */
// specialized since the single-valued case is different
static void accumSingle(int counts[], int startTermIndex, SortedDocValues si, DocIdSetIterator disi,
        int subIndex, OrdinalMap map) throws IOException {
    int doc;//from  ww  w  .  j  a v a 2s  .c o  m
    while ((doc = disi.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
        int term = si.getOrd(doc);
        if (map != null && term >= 0) {
            term = (int) map.getGlobalOrd(subIndex, term);
        }
        int arrIdx = term - startTermIndex;
        if (arrIdx >= 0 && arrIdx < counts.length)
            counts[arrIdx]++;
    }
}

---

From source file:org.apache.solr.request.DocValuesFacets.java

License:Apache License

/** accumulates per-segment multi-valued facet counts, mapping to global ordinal space */
static void accumMulti(int counts[], int startTermIndex, SortedSetDocValues si, DocIdSetIterator disi,
        int subIndex, OrdinalMap map) throws IOException {
    int doc;/*  w  w  w .  j a  v a  2 s . c o m*/
    while ((doc = disi.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
        si.setDocument(doc);
        // strange do-while to collect the missing count (first ord is NO_MORE_ORDS)
        int term = (int) si.nextOrd();
        if (term < 0) {
            if (startTermIndex == -1) {
                counts[0]++; // missing count
            }
            continue;
        }

        do {
            if (map != null) {
                term = (int) map.getGlobalOrd(subIndex, term);
            }
            int arrIdx = term - startTermIndex;
            if (arrIdx >= 0 && arrIdx < counts.length)
                counts[arrIdx]++;
        } while ((term = (int) si.nextOrd()) >= 0);
    }
}

---

From source file:org.apache.solr.request.DocValuesStats.java

License:Apache License

/** accumulates per-segment single-valued stats */
static int accumSingle(int counts[], int docBase, FieldFacetStats[] facetStats, SortedDocValues si,
        DocIdSetIterator disi, int subIndex, OrdinalMap map) throws IOException {
    final LongValues ordMap = map == null ? null : map.getGlobalOrds(subIndex);
    int missingDocCount = 0;
    int doc;/*from w  ww.ja  v  a  2  s.  c o  m*/
    while ((doc = disi.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
        int term = si.getOrd(doc);
        if (term >= 0) {
            if (map != null) {
                term = (int) ordMap.get(term);
            }
            counts[term]++;
            for (FieldFacetStats f : facetStats) {
                f.facetTermNum(docBase + doc, term);
            }
        } else {
            for (FieldFacetStats f : facetStats) {
                f.facetMissingNum(docBase + doc);
            }

            missingDocCount++;
        }
    }
    return missingDocCount;
}

---

From source file:org.apache.solr.request.DocValuesStats.java

License:Apache License

/** accumulates per-segment multi-valued stats */

static int accumMulti(int counts[], int docBase, FieldFacetStats[] facetStats, SortedSetDocValues si,
        DocIdSetIterator disi, int subIndex, OrdinalMap map) throws IOException {
    final LongValues ordMap = map == null ? null : map.getGlobalOrds(subIndex);
    int missingDocCount = 0;
    int doc;/*  w  ww  .  java  2 s.  c o m*/
    while ((doc = disi.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
        si.setDocument(doc);
        long ord;
        boolean emptyTerm = true;
        while ((ord = si.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
            emptyTerm = false;
            int term = (int) ord;
            if (map != null) {
                term = (int) ordMap.get(term);
            }
            counts[term]++;
            for (FieldFacetStats f : facetStats) {
                f.facetTermNum(docBase + doc, term);
            }
        }
        if (emptyTerm) {
            for (FieldFacetStats f : facetStats) {
                f.facetMissingNum(docBase + doc);
            }

            missingDocCount++;
        }
    }

    return missingDocCount;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• »