List of usage examples for org.apache.lucene.index IndexReaderContext leaves
public abstract List<LeafReaderContext> leaves() throws UnsupportedOperationException;
From source file:com.joliciel.jochre.search.highlight.LuceneQueryHighlighter.java
License:Open Source License
public LuceneQueryHighlighter(JochreQuery jochreQuery, IndexSearcher indexSearcher) { try {/* ww w. j a v a 2 s . co m*/ this.indexSearcher = indexSearcher; this.jochreQuery = jochreQuery; query = rewrite(jochreQuery.getLuceneQuery()); queryTerms = new TreeSet<Term>(); query.extractTerms(queryTerms); if (LOG.isTraceEnabled()) queryTermList = new ArrayList<Term>(queryTerms); final IndexReader reader = indexSearcher.getIndexReader(); // add 1 to doc count to ensure even terms in all docs get a very small weight docCountLog = Math.log(reader.numDocs() + 1); IndexReaderContext readerContext = reader.getContext(); leaves = readerContext.leaves(); // since the same terms might be contained in the query multiple times (e.g. once per field) // we only consider them once each by using a HashSet terms = new HashSet<BytesRef>(); Map<BytesRef, Integer> termFreqs = new HashMap<BytesRef, Integer>(); for (Term term : queryTerms) { terms.add(term.bytes()); termFreqs.put(term.bytes(), 0); } termLogs = new HashMap<BytesRef, Double>(); for (Term term : queryTerms) { int freq = termFreqs.get(term.bytes()); freq += reader.docFreq(term); termFreqs.put(term.bytes(), freq); } for (BytesRef term : terms) { int freq = termFreqs.get(term); termLogs.put(term, Math.log(freq)); } } catch (IOException e) { LogUtils.logError(LOG, e); throw new RuntimeException(e); } }
From source file:com.lucene.MyIndexSearcher.java
License:Apache License
public MyIndexSearcher(IndexReaderContext context) { super(context, null); assert context.isTopLevel : "IndexSearcher's ReaderContext must be topLevel for reader" + context.reader(); reader = context.reader();//from w ww. j a va 2 s . c o m this.readerContext = context; leafContexts = context.leaves(); this.leafSlices = null; }
From source file:com.sindicetech.siren.search.node.NodeTermCollectingRewrite.java
License:Open Source License
final void collectTerms(final IndexReader reader, final MultiNodeTermQuery query, final TermCollector collector) throws IOException { final IndexReaderContext topReaderContext = reader.getContext(); Comparator<BytesRef> lastTermComp = null; for (final AtomicReaderContext context : topReaderContext.leaves()) { final Fields fields = context.reader().fields(); if (fields == null) { // reader has no fields continue; }/*from w w w .j a va 2 s . c om*/ final Terms terms = fields.terms(query.field); if (terms == null) { // field does not exist continue; } final TermsEnum termsEnum = this.getTermsEnum(query, terms, collector.attributes); assert termsEnum != null; if (termsEnum == TermsEnum.EMPTY) continue; // Check comparator compatibility: final Comparator<BytesRef> newTermComp = termsEnum.getComparator(); if (lastTermComp != null && newTermComp != null && newTermComp != lastTermComp) throw new RuntimeException("term comparator should not change between segments: " + lastTermComp + " != " + newTermComp); lastTermComp = newTermComp; collector.setReaderContext(topReaderContext, context); collector.setNextEnum(termsEnum); BytesRef bytes; while ((bytes = termsEnum.next()) != null) { if (!collector.collect(bytes)) return; // interrupt whole term collection, so also don't iterate other subReaders } } }
From source file:com.xiaomi.linden.lucene.search.LindenFieldCacheImpl.java
License:Apache License
public UIDMaps getUIDMaps(IndexReaderContext topReaderContext, String uidField) throws IOException { PerReaderUIDMaps[] uidMapsArray = new PerReaderUIDMaps[topReaderContext.leaves().size()]; for (int i = 0; i < topReaderContext.leaves().size(); ++i) { uidMapsArray[i] = (PerReaderUIDMaps) caches.get(UIDCache.class) .get(topReaderContext.leaves().get(i).reader(), new CacheKey(uidField, null), false); }//from w w w. j av a2 s. c o m return new UIDMaps(uidMapsArray); }
From source file:de.jetsli.lumeo.RawLucene.java
License:Apache License
public Document findById(final long id) { //Check cache IndexOp result = getCurrentRTCache(latestGen).get(id); if (result != null) { if (result.type == IndexOp.Type.DELETE) return null; return result.document; }/* w w w . j av a 2 s. c o m*/ return searchSomething(new SearchExecutor<Document>() { @Override public Document execute(IndexSearcher searcher) throws Exception { // TODO optimize via indexReader.termDocsEnum !? IndexReaderContext trc = searcher.getTopReaderContext(); AtomicReaderContext[] arc = trc.leaves(); for (int i = 0; i < arc.length; i++) { AtomicReader subreader = arc[i].reader(); DocsEnum docs = subreader.terms(UID).iterator(null).docs(subreader.getLiveDocs(), null, false); if (docs != null) { int docID = docs.nextDoc(); if (docID != DocsEnum.NO_MORE_DOCS) { return subreader.document(docID); } } } return null; } }); }
From source file:org.apache.blur.manager.writer.BlurIndex.java
License:Apache License
public long getSegmentCount() throws IOException { IndexSearcherCloseable indexSearcherClosable = getIndexSearcher(); try {/*from ww w . j a va 2 s . com*/ IndexReader indexReader = indexSearcherClosable.getIndexReader(); IndexReaderContext context = indexReader.getContext(); return context.leaves().size(); } finally { indexSearcherClosable.close(); } }
From source file:org.apache.blur.manager.writer.BlurIndexSimpleWriter.java
License:Apache License
@Override public long getSegmentCount() throws IOException { IndexSearcherCloseable indexSearcherClosable = getIndexSearcher(false); try {//from w w w . j a v a 2 s . c om IndexReader indexReader = indexSearcherClosable.getIndexReader(); IndexReaderContext context = indexReader.getContext(); return context.leaves().size(); } finally { indexSearcherClosable.close(); } }
From source file:org.apache.solr.handler.component.HelloHandlerComponent.java
License:Apache License
protected void doFieldSortValues(ResponseBuilder rb, SolrIndexSearcher searcher) throws IOException { SolrQueryRequest req = rb.req;// w w w . j av a2 s .c o m SolrQueryResponse rsp = rb.rsp; final CharsRef spare = new CharsRef(); // The query cache doesn't currently store sort field values, and SolrIndexSearcher doesn't // currently have an option to return sort field values. Because of this, we // take the documents given and re-derive the sort values. boolean fsv = req.getParams().getBool(ResponseBuilder.FIELD_SORT_VALUES, false); if (fsv) { Sort sort = searcher.weightSort(rb.getSortSpec().getSort()); SortField[] sortFields = sort == null ? new SortField[] { SortField.FIELD_SCORE } : sort.getSort(); NamedList<Object[]> sortVals = new NamedList<Object[]>(); // order is important for the sort fields Field field = new StringField("dummy", "", Field.Store.NO); // a dummy Field IndexReaderContext topReaderContext = searcher.getTopReaderContext(); List<AtomicReaderContext> leaves = topReaderContext.leaves(); AtomicReaderContext currentLeaf = null; if (leaves.size() == 1) { // if there is a single segment, use that subReader and avoid looking up each time currentLeaf = leaves.get(0); leaves = null; } DocList docList = rb.getResults().docList; // sort ids from lowest to highest so we can access them in order int nDocs = docList.size(); long[] sortedIds = new long[nDocs]; DocIterator it = rb.getResults().docList.iterator(); for (int i = 0; i < nDocs; i++) { sortedIds[i] = (((long) it.nextDoc()) << 32) | i; } Arrays.sort(sortedIds); for (SortField sortField : sortFields) { SortField.Type type = sortField.getType(); if (type == SortField.Type.SCORE || type == SortField.Type.DOC) continue; FieldComparator comparator = null; String fieldname = sortField.getField(); FieldType ft = fieldname == null ? null : req.getSchema().getFieldTypeNoEx(fieldname); Object[] vals = new Object[nDocs]; int lastIdx = -1; int idx = 0; for (long idAndPos : sortedIds) { int doc = (int) (idAndPos >>> 32); int position = (int) idAndPos; if (leaves != null) { idx = ReaderUtil.subIndex(doc, leaves); currentLeaf = leaves.get(idx); if (idx != lastIdx) { // we switched segments. invalidate comparator. comparator = null; } } if (comparator == null) { comparator = sortField.getComparator(1, 0); comparator = comparator.setNextReader(currentLeaf); } doc -= currentLeaf.docBase; // adjust for what segment this is in comparator.copy(0, doc); Object val = comparator.value(0); // Sortable float, double, int, long types all just use a string // comparator. For these, we need to put the type into a readable // format. One reason for this is that XML can't represent all // string values (or even all unicode code points). // indexedToReadable() should be a no-op and should // thus be harmless anyway (for all current ways anyway) if (val instanceof String) { field.setStringValue((String) val); val = ft.toObject(field); } // Must do the same conversion when sorting by a // String field in Lucene, which returns the terms // data as BytesRef: if (val instanceof BytesRef) { UnicodeUtil.UTF8toUTF16((BytesRef) val, spare); field.setStringValue(spare.toString()); val = ft.toObject(field); } vals[position] = val; } sortVals.add(fieldname, vals); } rsp.add("sort_values", sortVals); } }
From source file:org.apache.solr.handler.component.QueryComponent.java
License:Apache License
protected void doFieldSortValues(ResponseBuilder rb, SolrIndexSearcher searcher) throws IOException { SolrQueryRequest req = rb.req;/* ww w .j a v a 2 s.c o m*/ SolrQueryResponse rsp = rb.rsp; final CharsRef spare = new CharsRef(); // The query cache doesn't currently store sort field values, and SolrIndexSearcher doesn't // currently have an option to return sort field values. Because of this, we // take the documents given and re-derive the sort values. boolean fsv = req.getParams().getBool(ResponseBuilder.FIELD_SORT_VALUES, false); if (fsv) { Sort sort = searcher.weightSort(rb.getSortSpec().getSort()); SortField[] sortFields = sort == null ? new SortField[] { SortField.FIELD_SCORE } : sort.getSort(); NamedList<Object[]> sortVals = new NamedList<Object[]>(); // order is important for the sort fields Field field = new StringField("dummy", "", Field.Store.NO); // a dummy Field IndexReaderContext topReaderContext = searcher.getTopReaderContext(); List<AtomicReaderContext> leaves = topReaderContext.leaves(); AtomicReaderContext currentLeaf = null; if (leaves.size() == 1) { // if there is a single segment, use that subReader and avoid looking up each time currentLeaf = leaves.get(0); leaves = null; } DocList docList = rb.getResults().docList; // sort ids from lowest to highest so we can access them in order int nDocs = docList.size(); long[] sortedIds = new long[nDocs]; DocIterator it = rb.getResults().docList.iterator(); for (int i = 0; i < nDocs; i++) { sortedIds[i] = (((long) it.nextDoc()) << 32) | i; } Arrays.sort(sortedIds); for (SortField sortField : sortFields) { SortField.Type type = sortField.getType(); if (type == SortField.Type.SCORE || type == SortField.Type.DOC) continue; FieldComparator comparator = null; String fieldname = sortField.getField(); FieldType ft = fieldname == null ? null : searcher.getSchema().getFieldTypeNoEx(fieldname); Object[] vals = new Object[nDocs]; int lastIdx = -1; int idx = 0; for (long idAndPos : sortedIds) { int doc = (int) (idAndPos >>> 32); int position = (int) idAndPos; if (leaves != null) { idx = ReaderUtil.subIndex(doc, leaves); currentLeaf = leaves.get(idx); if (idx != lastIdx) { // we switched segments. invalidate comparator. comparator = null; } } if (comparator == null) { comparator = sortField.getComparator(1, 0); comparator = comparator.setNextReader(currentLeaf); } doc -= currentLeaf.docBase; // adjust for what segment this is in comparator.copy(0, doc); Object val = comparator.value(0); // Sortable float, double, int, long types all just use a string // comparator. For these, we need to put the type into a readable // format. One reason for this is that XML can't represent all // string values (or even all unicode code points). // indexedToReadable() should be a no-op and should // thus be harmless anyway (for all current ways anyway) if (val instanceof String) { field.setStringValue((String) val); val = ft.toObject(field); } // Must do the same conversion when sorting by a // String field in Lucene, which returns the terms // data as BytesRef: if (val instanceof BytesRef) { UnicodeUtil.UTF8toUTF16((BytesRef) val, spare); field.setStringValue(spare.toString()); val = ft.toObject(field); } vals[position] = val; } sortVals.add(fieldname, vals); } rsp.add("sort_values", sortVals); } }
From source file:org.apache.solr.search.TestDocSet.java
License:Apache License
public void doFilterTest(IndexReader reader) throws IOException { IndexReaderContext topLevelContext = reader.getContext(); OpenBitSet bs = getRandomSet(reader.maxDoc(), rand.nextInt(reader.maxDoc() + 1)); DocSet a = new BitDocSet(bs); DocSet b = getIntDocSet(bs);/*from w w w .j a v a2 s. c o m*/ Filter fa = a.getTopFilter(); Filter fb = b.getTopFilter(); /*** top level filters are no longer supported // test top-level DocIdSet da = fa.getDocIdSet(topLevelContext); DocIdSet db = fb.getDocIdSet(topLevelContext); doTestIteratorEqual(da, db); ***/ DocIdSet da; DocIdSet db; List<AtomicReaderContext> leaves = topLevelContext.leaves(); // first test in-sequence sub readers for (AtomicReaderContext readerContext : leaves) { da = fa.getDocIdSet(readerContext, null); db = fb.getDocIdSet(readerContext, null); doTestIteratorEqual(da, db); } int nReaders = leaves.size(); // now test out-of-sequence sub readers for (int i = 0; i < nReaders; i++) { AtomicReaderContext readerContext = leaves.get(rand.nextInt(nReaders)); da = fa.getDocIdSet(readerContext, null); db = fb.getDocIdSet(readerContext, null); doTestIteratorEqual(da, db); } }