List of usage examples for org.apache.lucene.search TimeLimitingCollector TimeLimitingCollector
public TimeLimitingCollector(final Collector collector, Counter clock, final long ticksAllowed)
From source file:com.mathworks.xzheng.advsearching.TimeLimitingCollectorTest.java
License:Apache License
public void testTimeLimitingCollector() throws Exception { Directory dir = TestUtil.getBookIndexDirectory(); IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(dir)); Query q = new MatchAllDocsQuery(); int numAllBooks = TestUtil.hitCount(searcher, q); TopScoreDocCollector topDocs = TopScoreDocCollector.create(10, false); Collector collector = new TimeLimitingCollector(topDocs, // #A null, 1000); // #A try {/*from ww w .ja v a 2s . c om*/ searcher.search(q, collector); assertEquals(numAllBooks, topDocs.getTotalHits()); // #B } catch (TimeExceededException tee) { // #C System.out.println("Too much time taken."); // #C } // #C dir.close(); }
From source file:com.xiaomi.linden.core.search.LindenCoreImpl.java
License:Apache License
public LindenResult search(LindenSearchRequest request) throws IOException { SearcherTaxonomyManager.SearcherAndTaxonomy searcherAndTaxonomy = lindenNRTSearcherManager.acquire(); try {//from w ww. ja va 2s .com IndexSearcher indexSearcher = searcherAndTaxonomy.searcher; Filter filter = FilterConstructor.constructFilter(request.getFilter(), config); Sort sort = SortConstructor.constructSort(request, indexSearcher, config); indexSearcher.setSimilarity(config.getSearchSimilarityInstance()); Query query = QueryConstructor.constructQuery(request.getQuery(), config); if (filter != null) { query = new FilteredQuery(query, filter); } int from = request.getOffset(); int size = request.getLength(); LindenResultParser resultParser = new LindenResultParser(config, request, indexSearcher, snippetGenerator, query, filter, sort); // very common search, no group, no facet, no early termination, no search time limit if (!request.isSetGroupParam() && !request.isSetFacet() && !request.isSetEarlyParam() && config.getSearchTimeLimit() <= 0) { TopDocs docs; if (sort != null) { docs = indexSearcher.search(query, from + size, sort); } else { docs = indexSearcher.search(query, from + size); } return resultParser.parse(docs, null, null, null); } // group param will suppress facet, group, early termination and search time limit parameters if (request.isSetGroupParam()) { String groupField = request.getGroupParam().getGroupField(); GroupingSearch groupingSearch = new GroupingSearch(groupField); groupingSearch.setGroupDocsLimit(request.getGroupParam().getGroupInnerLimit()); if (sort != null) { groupingSearch.setGroupSort(sort); groupingSearch.setSortWithinGroup(sort); groupingSearch.setFillSortFields(true); } groupingSearch.setCachingInMB(8.0, true); groupingSearch.setAllGroups(true); TopGroups<TopDocs> topGroupedDocs = groupingSearch.search(indexSearcher, query, 0, from + size); return resultParser.parse(null, topGroupedDocs, null, null); } TopDocsCollector topDocsCollector; if (sort != null) { topDocsCollector = TopFieldCollector.create(sort, from + size, null, true, false, false, false); } else { topDocsCollector = TopScoreDocCollector.create(from + size, false); } LindenDocsCollector lindenDocsCollector; if (request.isSetEarlyParam()) { MergePolicy mergePolicy = indexWriter.getConfig().getMergePolicy(); Sort mergePolicySort = null; if (mergePolicy instanceof SortingMergePolicyDecorator) { mergePolicySort = ((SortingMergePolicyDecorator) mergePolicy).getSort(); } EarlyTerminationCollector earlyTerminationCollector = new EarlyTerminationCollector( topDocsCollector, mergePolicySort, request.getEarlyParam().getMaxNum()); lindenDocsCollector = new LindenDocsCollector(earlyTerminationCollector); } else { lindenDocsCollector = new LindenDocsCollector(topDocsCollector); } Collector collector = lindenDocsCollector; if (config.getSearchTimeLimit() > 0) { collector = new TimeLimitingCollector(lindenDocsCollector, TimeLimitingCollector.getGlobalCounter(), config.getSearchTimeLimit()); } // no facet param if (!request.isSetFacet()) { indexSearcher.search(query, collector); return resultParser.parse(lindenDocsCollector.topDocs(), null, null, null); } // facet search LindenFacet facetRequest = request.getFacet(); FacetsCollector facetsCollector = new FacetsCollector(); lindenDocsCollector.wrap(facetsCollector); Facets facets = null; if (facetRequest.isSetDrillDownDimAndPaths()) { // drillDown or drillSideways DrillDownQuery drillDownQuery = new DrillDownQuery(facetsConfig, query); List<LindenFacetDimAndPath> drillDownDimAndPaths = facetRequest.getDrillDownDimAndPaths(); for (int i = 0; i < drillDownDimAndPaths.size(); ++i) { String fieldName = drillDownDimAndPaths.get(i).dim; if (drillDownDimAndPaths.get(i).path != null) { drillDownQuery.add(fieldName, drillDownDimAndPaths.get(i).path.split("/")); } else { drillDownQuery.add(fieldName); } } // drillSideways if (facetRequest.getFacetDrillingType() == FacetDrillingType.DRILLSIDEWAYS) { DrillSideways dillSideways = new DrillSideways(indexSearcher, facetsConfig, searcherAndTaxonomy.taxonomyReader); DrillSideways.DrillSidewaysResult drillSidewaysResult = dillSideways.search(drillDownQuery, collector); facets = drillSidewaysResult.facets; } else { // drillDown indexSearcher.search(drillDownQuery, collector); facets = new FastTaxonomyFacetCounts(searcherAndTaxonomy.taxonomyReader, facetsConfig, facetsCollector); } } else { indexSearcher.search(query, collector); // Simple facet browsing if (facetRequest.isSetFacetParams()) { facets = new FastTaxonomyFacetCounts(searcherAndTaxonomy.taxonomyReader, facetsConfig, facetsCollector); } } return resultParser.parse(lindenDocsCollector.topDocs(), null, facets, facetsCollector); } catch (Exception e) { throw new IOException(Throwables.getStackTraceAsString(e)); } finally { lindenNRTSearcherManager.release(searcherAndTaxonomy); } }
From source file:org.apache.solr.search.grouping.CommandHandler.java
License:Apache License
/** * Invokes search with the specified filter and collector. * If a time limit has been specified then wrap the collector in the TimeLimitingCollector *//*from w w w .java 2 s . c om*/ private void searchWithTimeLimiter(final Query query, final ProcessedFilter filter, Collector collector) throws IOException { if (queryCommand.getTimeAllowed() > 0) { collector = new TimeLimitingCollector(collector, TimeLimitingCollector.getGlobalCounter(), queryCommand.getTimeAllowed()); } TotalHitCountCollector hitCountCollector = new TotalHitCountCollector(); if (includeHitCount) { collector = MultiCollector.wrap(collector, hitCountCollector); } Filter luceneFilter = filter.filter; if (filter.postFilter != null) { filter.postFilter.setLastDelegate(collector); collector = filter.postFilter; } try { searcher.search(query, luceneFilter, collector); } catch (TimeLimitingCollector.TimeExceededException x) { partialResults = true; logger.warn("Query: " + query + "; " + x.getMessage()); } if (includeHitCount) { totalHitCount = hitCountCollector.getTotalHits(); } }
From source file:org.apache.solr.search.Grouping.java
License:Apache License
/** * Invokes search with the specified filter and collector. * If a time limit has been specified, wrap the collector in a TimeLimitingCollector *//*w w w . jav a 2s . c o m*/ private void searchWithTimeLimiter(final Filter luceneFilter, Collector collector) throws IOException { if (cmd.getTimeAllowed() > 0) { if (timeLimitingCollector == null) { timeLimitingCollector = new TimeLimitingCollector(collector, TimeLimitingCollector.getGlobalCounter(), cmd.getTimeAllowed()); } else { /* * This is so the same timer can be used for grouping's multiple phases. * We don't want to create a new TimeLimitingCollector for each phase because that would * reset the timer for each phase. If time runs out during the first phase, the * second phase should timeout quickly. */ timeLimitingCollector.setCollector(collector); } collector = timeLimitingCollector; } try { searcher.search(query, luceneFilter, collector); } catch (TimeLimitingCollector.TimeExceededException x) { logger.warn("Query: " + query + "; " + x.getMessage()); qr.setPartialResults(true); } }
From source file:org.apache.solr.search.SolrIndexSearcher.java
License:Apache License
private void getDocListNC(QueryResult qr, QueryCommand cmd) throws IOException { final long timeAllowed = cmd.getTimeAllowed(); int len = cmd.getSupersetMaxDoc(); int last = len; if (last < 0 || last > maxDoc()) last = maxDoc();/*from w w w .j a v a2s. co m*/ final int lastDocRequested = last; int nDocsReturned; int totalHits; float maxScore; int[] ids; float[] scores; boolean needScores = (cmd.getFlags() & GET_SCORES) != 0; boolean terminateEarly = (cmd.getFlags() & TERMINATE_EARLY) == TERMINATE_EARLY; Query query = QueryUtils.makeQueryable(cmd.getQuery()); ProcessedFilter pf = getProcessedFilter(cmd.getFilter(), cmd.getFilterList()); final Filter luceneFilter = pf.filter; // handle zero case... if (lastDocRequested <= 0) { final float[] topscore = new float[] { Float.NEGATIVE_INFINITY }; final int[] numHits = new int[1]; Collector collector; if (!needScores) { collector = new Collector() { @Override public void setScorer(Scorer scorer) { } @Override public void collect(int doc) { numHits[0]++; } @Override public void setNextReader(AtomicReaderContext context) { } @Override public boolean acceptsDocsOutOfOrder() { return true; } }; } else { collector = new Collector() { Scorer scorer; @Override public void setScorer(Scorer scorer) { this.scorer = scorer; } @Override public void collect(int doc) throws IOException { numHits[0]++; float score = scorer.score(); if (score > topscore[0]) topscore[0] = score; } @Override public void setNextReader(AtomicReaderContext context) { } @Override public boolean acceptsDocsOutOfOrder() { return true; } }; } if (terminateEarly) { collector = new EarlyTerminatingCollector(collector, cmd.len); } if (timeAllowed > 0) { collector = new TimeLimitingCollector(collector, TimeLimitingCollector.getGlobalCounter(), timeAllowed); } if (pf.postFilter != null) { pf.postFilter.setLastDelegate(collector); collector = pf.postFilter; } try { super.search(query, luceneFilter, collector); if (collector instanceof DelegatingCollector) { ((DelegatingCollector) collector).finish(); } } catch (TimeLimitingCollector.TimeExceededException x) { log.warn("Query: " + query + "; " + x.getMessage()); qr.setPartialResults(true); } nDocsReturned = 0; ids = new int[nDocsReturned]; scores = new float[nDocsReturned]; totalHits = numHits[0]; maxScore = totalHits > 0 ? topscore[0] : 0.0f; } else { TopDocsCollector topCollector; if (cmd.getSort() == null) { if (cmd.getScoreDoc() != null) { topCollector = TopScoreDocCollector.create(len, cmd.getScoreDoc(), true); //create the Collector with InOrderPagingCollector } else { topCollector = TopScoreDocCollector.create(len, true); } } else { topCollector = TopFieldCollector.create(weightSort(cmd.getSort()), len, false, needScores, needScores, true); } Collector collector = topCollector; if (terminateEarly) { collector = new EarlyTerminatingCollector(collector, cmd.len); } if (timeAllowed > 0) { collector = new TimeLimitingCollector(collector, TimeLimitingCollector.getGlobalCounter(), timeAllowed); } if (pf.postFilter != null) { pf.postFilter.setLastDelegate(collector); collector = pf.postFilter; } try { super.search(query, luceneFilter, collector); if (collector instanceof DelegatingCollector) { ((DelegatingCollector) collector).finish(); } } catch (TimeLimitingCollector.TimeExceededException x) { log.warn("Query: " + query + "; " + x.getMessage()); qr.setPartialResults(true); } totalHits = topCollector.getTotalHits(); TopDocs topDocs = topCollector.topDocs(0, len); maxScore = totalHits > 0 ? topDocs.getMaxScore() : 0.0f; nDocsReturned = topDocs.scoreDocs.length; ids = new int[nDocsReturned]; scores = (cmd.getFlags() & GET_SCORES) != 0 ? new float[nDocsReturned] : null; for (int i = 0; i < nDocsReturned; i++) { ScoreDoc scoreDoc = topDocs.scoreDocs[i]; ids[i] = scoreDoc.doc; if (scores != null) scores[i] = scoreDoc.score; } } int sliceLen = Math.min(lastDocRequested, nDocsReturned); if (sliceLen < 0) sliceLen = 0; qr.setDocList(new DocSlice(0, sliceLen, ids, scores, totalHits, maxScore)); }
From source file:org.apache.solr.search.SolrIndexSearcher.java
License:Apache License
private DocSet getDocListAndSetNC(QueryResult qr, QueryCommand cmd) throws IOException { int len = cmd.getSupersetMaxDoc(); int last = len; if (last < 0 || last > maxDoc()) last = maxDoc();/*w w w .ja va2 s.c om*/ final int lastDocRequested = last; int nDocsReturned; int totalHits; float maxScore; int[] ids; float[] scores; DocSet set; boolean needScores = (cmd.getFlags() & GET_SCORES) != 0; boolean terminateEarly = (cmd.getFlags() & TERMINATE_EARLY) == TERMINATE_EARLY; int maxDoc = maxDoc(); int smallSetSize = maxDoc >> 6; ProcessedFilter pf = getProcessedFilter(cmd.getFilter(), cmd.getFilterList()); final Filter luceneFilter = pf.filter; Query query = QueryUtils.makeQueryable(cmd.getQuery()); final long timeAllowed = cmd.getTimeAllowed(); // handle zero case... if (lastDocRequested <= 0) { final float[] topscore = new float[] { Float.NEGATIVE_INFINITY }; Collector collector; DocSetCollector setCollector; if (!needScores) { collector = setCollector = new DocSetCollector(smallSetSize, maxDoc); } else { collector = setCollector = new DocSetDelegateCollector(smallSetSize, maxDoc, new Collector() { Scorer scorer; @Override public void setScorer(Scorer scorer) { this.scorer = scorer; } @Override public void collect(int doc) throws IOException { float score = scorer.score(); if (score > topscore[0]) topscore[0] = score; } @Override public void setNextReader(AtomicReaderContext context) { } @Override public boolean acceptsDocsOutOfOrder() { return false; } }); } if (terminateEarly) { collector = new EarlyTerminatingCollector(collector, cmd.len); } if (timeAllowed > 0) { collector = new TimeLimitingCollector(collector, TimeLimitingCollector.getGlobalCounter(), timeAllowed); } if (pf.postFilter != null) { pf.postFilter.setLastDelegate(collector); collector = pf.postFilter; } try { super.search(query, luceneFilter, collector); if (collector instanceof DelegatingCollector) { ((DelegatingCollector) collector).finish(); } } catch (TimeLimitingCollector.TimeExceededException x) { log.warn("Query: " + query + "; " + x.getMessage()); qr.setPartialResults(true); } set = setCollector.getDocSet(); nDocsReturned = 0; ids = new int[nDocsReturned]; scores = new float[nDocsReturned]; totalHits = set.size(); maxScore = totalHits > 0 ? topscore[0] : 0.0f; } else { TopDocsCollector topCollector; if (cmd.getSort() == null) { topCollector = TopScoreDocCollector.create(len, true); } else { topCollector = TopFieldCollector.create(weightSort(cmd.getSort()), len, false, needScores, needScores, true); } DocSetCollector setCollector = new DocSetDelegateCollector(maxDoc >> 6, maxDoc, topCollector); Collector collector = setCollector; if (terminateEarly) { collector = new EarlyTerminatingCollector(collector, cmd.len); } if (timeAllowed > 0) { collector = new TimeLimitingCollector(collector, TimeLimitingCollector.getGlobalCounter(), timeAllowed); } if (pf.postFilter != null) { pf.postFilter.setLastDelegate(collector); collector = pf.postFilter; } try { super.search(query, luceneFilter, collector); if (collector instanceof DelegatingCollector) { ((DelegatingCollector) collector).finish(); } } catch (TimeLimitingCollector.TimeExceededException x) { log.warn("Query: " + query + "; " + x.getMessage()); qr.setPartialResults(true); } set = setCollector.getDocSet(); totalHits = topCollector.getTotalHits(); assert (totalHits == set.size()); TopDocs topDocs = topCollector.topDocs(0, len); maxScore = totalHits > 0 ? topDocs.getMaxScore() : 0.0f; nDocsReturned = topDocs.scoreDocs.length; ids = new int[nDocsReturned]; scores = (cmd.getFlags() & GET_SCORES) != 0 ? new float[nDocsReturned] : null; for (int i = 0; i < nDocsReturned; i++) { ScoreDoc scoreDoc = topDocs.scoreDocs[i]; ids[i] = scoreDoc.doc; if (scores != null) scores[i] = scoreDoc.score; } } int sliceLen = Math.min(lastDocRequested, nDocsReturned); if (sliceLen < 0) sliceLen = 0; qr.setDocList(new DocSlice(0, sliceLen, ids, scores, totalHits, maxScore)); // TODO: if we collect results before the filter, we just need to intersect with // that filter to generate the DocSet for qr.setDocSet() qr.setDocSet(set); // TODO: currently we don't generate the DocSet for the base query, // but the QueryDocSet == CompleteDocSet if filter==null. return pf.filter == null && pf.postFilter == null ? qr.getDocSet() : null; }
From source file:org.codelibs.elasticsearch.common.lucene.Lucene.java
License:Apache License
/** * Wraps <code>delegate</code> with a time limited collector with a timeout of <code>timeoutInMillis</code> *///from w w w . j a v a2 s .co m public static final TimeLimitingCollector wrapTimeLimitingCollector(final Collector delegate, final Counter counter, long timeoutInMillis) { return new TimeLimitingCollector(delegate, counter, timeoutInMillis); }
From source file:org.elasticsearch.search.internal.ContextIndexSearcher.java
License:Apache License
@Override public void search(List<AtomicReaderContext> leaves, Weight weight, Collector collector) throws IOException { if (searchContext.timeoutInMillis() != -1) { // TODO: change to use our own counter that uses the scheduler in ThreadPool collector = new TimeLimitingCollector(collector, TimeLimitingCollector.getGlobalCounter(), searchContext.timeoutInMillis()); }/*from w ww .j a va 2 s. c o m*/ if (currentState == Stage.MAIN_QUERY) { if (enableMainDocIdSetCollector) { // TODO should we create a cache of segment->docIdSets so we won't create one each time? collector = this.mainDocIdSetCollector = new DocIdSetCollector(searchContext.docSetCache(), collector); } if (searchContext.parsedPostFilter() != null) { // this will only get applied to the actual search collector and not // to any scoped collectors, also, it will only be applied to the main collector // since that is where the filter should only work collector = new FilteredCollector(collector, searchContext.parsedPostFilter().filter()); } if (queryCollectors != null && !queryCollectors.isEmpty()) { collector = new MultiCollector(collector, queryCollectors.toArray(new Collector[queryCollectors.size()])); } // apply the minimum score after multi collector so we filter facets as well if (searchContext.minimumScore() != null) { collector = new MinimumScoreCollector(collector, searchContext.minimumScore()); } } // we only compute the doc id set once since within a context, we execute the same query always... try { if (searchContext.timeoutInMillis() != -1) { try { super.search(leaves, weight, collector); } catch (TimeLimitingCollector.TimeExceededException e) { searchContext.queryResult().searchTimedOut(true); } } else { super.search(leaves, weight, collector); } if (currentState == Stage.MAIN_QUERY) { if (enableMainDocIdSetCollector) { enableMainDocIdSetCollector = false; mainDocIdSetCollector.postCollection(); } if (queryCollectors != null && !queryCollectors.isEmpty()) { for (Collector queryCollector : queryCollectors) { if (queryCollector instanceof XCollector) { ((XCollector) queryCollector).postCollection(); } } } } } finally { searchContext.clearReleasables(); } }
From source file:org.languagetool.dev.index.Searcher.java
License:Open Source License
private PossiblyLimitedTopDocs getTopDocs(Query query, Sort sort) throws IOException { final TopFieldCollector topCollector = TopFieldCollector.create(sort, maxHits, true, false, false, false); final Counter clock = Counter.newCounter(true); final int waitMillis = 1000; // TODO: if we interrupt the whole thread anyway, do we still need the TimeLimitingCollector? final TimeLimitingCollector collector = new TimeLimitingCollector(topCollector, clock, maxSearchTimeMillis / waitMillis); collector.setBaseline(0);//from w ww.ja v a2 s . co m final Thread counterThread = new Thread() { @Override public void run() { final long startTime = System.currentTimeMillis(); while (true) { final long runTimeMillis = System.currentTimeMillis() - startTime; if (runTimeMillis > maxSearchTimeMillis) { // make sure there's no lingering thread for too long return; } clock.addAndGet(1); try { Thread.sleep(waitMillis); } catch (InterruptedException e) { throw new RuntimeException(e); } } } }; counterThread.setName("LuceneSearchTimeoutThread"); counterThread.start(); boolean timeLimitActivated = false; try { indexSearcher.search(query, collector); } catch (TimeLimitingCollector.TimeExceededException e) { timeLimitActivated = true; } return new PossiblyLimitedTopDocs(topCollector.topDocs(), timeLimitActivated); }