List of usage examples for org.apache.lucene.search IndexSearcher getIndexReader
public IndexReader getIndexReader()
From source file:org.apache.maven.index.SearchWithAnEmptyIndexTest.java
License:Apache License
private void createIndex(String filePath, String repoIndex, String contextId) throws Exception { File repo = new File(getBasedir(), filePath); File repoIndexDir = new File(getBasedir(), repoIndex + "/.index"); if (repoIndexDir.exists()) { FileUtils.deleteDirectory(repoIndexDir); }/*from w ww .j a v a 2 s .com*/ repoIndexDir.mkdirs(); System.out.println( "creating Index with id " + contextId + " path : " + filePath + " , indexPath " + repoIndex); IndexingContext indexingContext = nexusIndexer.addIndexingContext(contextId, contextId, repo, repoIndexDir, "http://www.apache.org", "http://www.apache.org/.index", indexCreators); indexingContext.setSearchable(true); nexusIndexer.scan(indexingContext, false); indexingContext.optimize(); File managedRepository = new File(repoIndex); final IndexSearcher indexSearcher = indexingContext.acquireIndexSearcher(); try { final File indexLocation = new File(managedRepository, ".index"); IndexPackingRequest request = new IndexPackingRequest(indexingContext, indexSearcher.getIndexReader(), indexLocation); indexPacker.packIndex(request); } finally { indexingContext.releaseIndexSearcher(indexSearcher); } }
From source file:org.apache.maven.index.updater.AbstractIndexUpdaterTest.java
License:Apache License
protected void packIndex(File targetDir, IndexingContext context) throws IllegalArgumentException, IOException { final IndexSearcher indexSearcher = context.acquireIndexSearcher(); try {/*from w w w . j a v a2 s. c om*/ IndexPackingRequest request = new IndexPackingRequest(context, indexSearcher.getIndexReader(), targetDir); request.setUseTargetProperties(true); packer.packIndex(request); } finally { context.releaseIndexSearcher(indexSearcher); } }
From source file:org.apache.maven.index.updater.IndexDataTest.java
License:Apache License
@Override protected void prepareNexusIndexer(NexusIndexer nexusIndexer) throws Exception { indexDir = new RAMDirectory(); context = nexusIndexer.addIndexingContext("test-default", "test", repo, indexDir, null, null, DEFAULT_CREATORS);//from ww w . j a v a 2 s . co m // assertNull( context.getTimestamp() ); // unknown upon creation nexusIndexer.scan(context); Date timestamp = context.getTimestamp(); assertNotNull(timestamp); // save and restore index to be used by common tests ByteArrayOutputStream bos = new ByteArrayOutputStream(); IndexDataWriter dw = new IndexDataWriter(bos); final IndexSearcher indexSearcher = context.acquireIndexSearcher(); try { dw.write(context, indexSearcher.getIndexReader(), null); } finally { context.releaseIndexSearcher(indexSearcher); } ByteArrayInputStream is = new ByteArrayInputStream(bos.toByteArray()); newDir = new RAMDirectory(); Date newTimestamp = DefaultIndexUpdater.unpackIndexData(is, newDir, context).getTimestamp(); assertEquals(timestamp, newTimestamp); context.replace(newDir); }
From source file:org.apache.maven.index.updater.IndexDataTest.java
License:Apache License
public void testEmptyContext() throws Exception { indexDir = new RAMDirectory(); context = nexusIndexer.addIndexingContext("test-default", "test", repo, indexDir, null, null, DEFAULT_CREATORS);//from ww w . j a v a 2s . c om assertNull(context.getTimestamp()); // unknown upon creation // save and restore index to be used by common tests // the point is that this is virgin context, and timestamp is null, // and it should remain null ByteArrayOutputStream bos = new ByteArrayOutputStream(); IndexDataWriter dw = new IndexDataWriter(bos); final IndexSearcher indexSearcher = context.acquireIndexSearcher(); try { dw.write(context, indexSearcher.getIndexReader(), null); } finally { context.releaseIndexSearcher(indexSearcher); } ByteArrayInputStream is = new ByteArrayInputStream(bos.toByteArray()); newDir = new RAMDirectory(); Date newTimestamp = DefaultIndexUpdater.unpackIndexData(is, newDir, context).getTimestamp(); assertEquals(null, newTimestamp); context.replace(newDir); }
From source file:org.apache.maven.indexer.examples.BasicUsageExample.java
License:Apache License
public void perform() throws IOException, ComponentLookupException, InvalidVersionSpecificationException { // Files where local cache is (if any) and Lucene Index should be located File centralLocalCache = new File("target/central-cache"); File centralIndexDir = new File("target/central-index"); // Creators we want to use (search for fields it defines) List<IndexCreator> indexers = new ArrayList<>(); indexers.add(plexusContainer.lookup(IndexCreator.class, "min")); indexers.add(plexusContainer.lookup(IndexCreator.class, "jarContent")); indexers.add(plexusContainer.lookup(IndexCreator.class, "maven-plugin")); // Create context for central repository index centralContext = indexer.createIndexingContext("central-context", "central", centralLocalCache, centralIndexDir, "http://repo1.maven.org/maven2", null, true, true, indexers); // Update the index (incremental update will happen if this is not 1st run and files are not deleted) // This whole block below should not be executed on every app start, but rather controlled by some configuration // since this block will always emit at least one HTTP GET. Central indexes are updated once a week, but // other index sources might have different index publishing frequency. // Preferred frequency is once a week. if (true) {//from ww w. j ava2s . co m System.out.println("Updating Index..."); System.out.println("This might take a while on first run, so please be patient!"); // Create ResourceFetcher implementation to be used with IndexUpdateRequest // Here, we use Wagon based one as shorthand, but all we need is a ResourceFetcher implementation TransferListener listener = new AbstractTransferListener() { public void transferStarted(TransferEvent transferEvent) { System.out.print(" Downloading " + transferEvent.getResource().getName()); } public void transferProgress(TransferEvent transferEvent, byte[] buffer, int length) { } public void transferCompleted(TransferEvent transferEvent) { System.out.println(" - Done"); } }; ResourceFetcher resourceFetcher = new WagonHelper.WagonFetcher(httpWagon, listener, null, null); Date centralContextCurrentTimestamp = centralContext.getTimestamp(); IndexUpdateRequest updateRequest = new IndexUpdateRequest(centralContext, resourceFetcher); IndexUpdateResult updateResult = indexUpdater.fetchAndUpdateIndex(updateRequest); if (updateResult.isFullUpdate()) { System.out.println("Full update happened!"); } else if (updateResult.getTimestamp().equals(centralContextCurrentTimestamp)) { System.out.println("No update needed, index is up to date!"); } else { System.out.println("Incremental update happened, change covered " + centralContextCurrentTimestamp + " - " + updateResult.getTimestamp() + " period."); } System.out.println(); } System.out.println(); System.out.println("Using index"); System.out.println("==========="); System.out.println(); // ==== // Case: // dump all the GAVs // NOTE: will not actually execute do this below, is too long to do (Central is HUGE), but is here as code // example if (false) { final IndexSearcher searcher = centralContext.acquireIndexSearcher(); try { final IndexReader ir = searcher.getIndexReader(); Bits liveDocs = MultiFields.getLiveDocs(ir); for (int i = 0; i < ir.maxDoc(); i++) { if (liveDocs == null || liveDocs.get(i)) { final Document doc = ir.document(i); final ArtifactInfo ai = IndexUtils.constructArtifactInfo(doc, centralContext); System.out.println(ai.getGroupId() + ":" + ai.getArtifactId() + ":" + ai.getVersion() + ":" + ai.getClassifier() + " (sha1=" + ai.getSha1() + ")"); } } } finally { centralContext.releaseIndexSearcher(searcher); } } // ==== // Case: // Search for all GAVs with known G and A and having version greater than V final GenericVersionScheme versionScheme = new GenericVersionScheme(); final String versionString = "1.5.0"; final Version version = versionScheme.parseVersion(versionString); // construct the query for known GA final Query groupIdQ = indexer.constructQuery(MAVEN.GROUP_ID, new SourcedSearchExpression("org.sonatype.nexus")); final Query artifactIdQ = indexer.constructQuery(MAVEN.ARTIFACT_ID, new SourcedSearchExpression("nexus-api")); final BooleanQuery query = new BooleanQuery(); query.add(groupIdQ, Occur.MUST); query.add(artifactIdQ, Occur.MUST); // we want "jar" artifacts only query.add(indexer.constructQuery(MAVEN.PACKAGING, new SourcedSearchExpression("jar")), Occur.MUST); // we want main artifacts only (no classifier) // Note: this below is unfinished API, needs fixing query.add(indexer.constructQuery(MAVEN.CLASSIFIER, new SourcedSearchExpression(Field.NOT_PRESENT)), Occur.MUST_NOT); // construct the filter to express "V greater than" final ArtifactInfoFilter versionFilter = new ArtifactInfoFilter() { public boolean accepts(final IndexingContext ctx, final ArtifactInfo ai) { try { final Version aiV = versionScheme.parseVersion(ai.getVersion()); // Use ">=" if you are INCLUSIVE return aiV.compareTo(version) > 0; } catch (InvalidVersionSpecificationException e) { // do something here? be safe and include? return true; } } }; System.out.println( "Searching for all GAVs with G=org.sonatype.nexus and nexus-api and having V greater than 1.5.0"); final IteratorSearchRequest request = new IteratorSearchRequest(query, Collections.singletonList(centralContext), versionFilter); final IteratorSearchResponse response = indexer.searchIterator(request); for (ArtifactInfo ai : response) { System.out.println(ai.toString()); } // Case: // Use index // Searching for some artifact Query gidQ = indexer.constructQuery(MAVEN.GROUP_ID, new SourcedSearchExpression("org.apache.maven.indexer")); Query aidQ = indexer.constructQuery(MAVEN.ARTIFACT_ID, new SourcedSearchExpression("indexer-artifact")); BooleanQuery bq = new BooleanQuery(); bq.add(gidQ, Occur.MUST); bq.add(aidQ, Occur.MUST); searchAndDump(indexer, "all artifacts under GA org.apache.maven.indexer:indexer-artifact", bq); // Searching for some main artifact bq = new BooleanQuery(); bq.add(gidQ, Occur.MUST); bq.add(aidQ, Occur.MUST); // bq.add( nexusIndexer.constructQuery( MAVEN.CLASSIFIER, new SourcedSearchExpression( "*" ) ), Occur.MUST_NOT // ); searchAndDump(indexer, "main artifacts under GA org.apache.maven.indexer:indexer-artifact", bq); // doing sha1 search searchAndDump(indexer, "SHA1 7ab67e6b20e5332a7fb4fdf2f019aec4275846c2", indexer.constructQuery(MAVEN.SHA1, new SourcedSearchExpression("7ab67e6b20e5332a7fb4fdf2f019aec4275846c2"))); searchAndDump(indexer, "SHA1 7ab67e6b20 (partial hash)", indexer.constructQuery(MAVEN.SHA1, new UserInputSearchExpression("7ab67e6b20"))); // doing classname search (incomplete classname) searchAndDump(indexer, "classname DefaultNexusIndexer (note: Central does not publish classes in the index)", indexer.constructQuery(MAVEN.CLASSNAMES, new UserInputSearchExpression("DefaultNexusIndexer"))); // doing search for all "canonical" maven plugins latest versions bq = new BooleanQuery(); bq.add(indexer.constructQuery(MAVEN.PACKAGING, new SourcedSearchExpression("maven-plugin")), Occur.MUST); bq.add(indexer.constructQuery(MAVEN.GROUP_ID, new SourcedSearchExpression("org.apache.maven.plugins")), Occur.MUST); searchGroupedAndDump(indexer, "all \"canonical\" maven plugins", bq, new GAGrouping()); // doing search for all archetypes latest versions searchGroupedAndDump(indexer, "all maven archetypes (latest versions)", indexer.constructQuery(MAVEN.PACKAGING, new SourcedSearchExpression("maven-archetype")), new GAGrouping()); // close cleanly indexer.closeIndexingContext(centralContext, false); }
From source file:org.apache.opennlp.corpus_server.impl.LuceneSearchService.java
License:Apache License
@Override public synchronized List<String> search(CorpusStore store, String q) throws IOException { // PERFORMANCE: This method can only be executed by one thread at a time // when there are concurrent search requests this will result // in longer than necessary delays to answer them. IndexSearcher searcher = corpusSearcherMap.get(store.getCorpusId()); // Opening or reopening an index might fail, // in this case every search request fails as well. if (searcher == null) { File indexLocation = getIndexDirectory(store.getCorpusId()); Directory indexDirectory = FSDirectory.open(indexLocation); IndexReader indexReader = IndexReader.open(indexDirectory, false); // Note: Reopening index for every request is slow, // modify code again to keep indexes open! searcher = new IndexSearcher(indexReader); corpusSearcherMap.put(store.getCorpusId(), searcher); }//from w w w . ja v a2 s .c om if (!searcher.getIndexReader().isCurrent()) { IndexReader freshIndexReader = searcher.getIndexReader().reopen(); searcher.close(); searcher = new IndexSearcher(freshIndexReader); corpusSearcherMap.put(store.getCorpusId(), searcher); } QueryParser parser = new QueryParser(Version.LUCENE_29, "text", new StandardAnalyzer(Version.LUCENE_29)); Query query; try { query = parser.parse(q); } catch (ParseException e) { throw new IOException(e); } final List<String> results = new ArrayList<String>(); final IndexSearcher finalSearcher = searcher; // query index ... searcher.search(query, new Collector() { int docBase = Integer.MIN_VALUE; @Override public void setScorer(Scorer scorer) throws IOException { } @Override public void setNextReader(IndexReader reader, int docBase) throws IOException { this.docBase = docBase; } @Override public void collect(int id) throws IOException { Document doc = finalSearcher.doc(docBase + id); String idString = doc.get(LUCENE_ID_FIELD); results.add(idString); } @Override public boolean acceptsDocsOutOfOrder() { return false; } }); searcher.close(); return results; }
From source file:org.apache.solr.ltr.LTRRescorer.java
License:Apache License
/** * rescores the documents:/*ww w .ja va 2 s. c om*/ * * @param searcher * current IndexSearcher * @param firstPassTopDocs * documents to rerank; * @param topN * documents to return; */ @Override public TopDocs rescore(IndexSearcher searcher, TopDocs firstPassTopDocs, int topN) throws IOException { if ((topN == 0) || (firstPassTopDocs.totalHits == 0)) { return firstPassTopDocs; } final ScoreDoc[] hits = firstPassTopDocs.scoreDocs; Arrays.sort(hits, new Comparator<ScoreDoc>() { @Override public int compare(ScoreDoc a, ScoreDoc b) { return a.doc - b.doc; } }); topN = Math.min(topN, firstPassTopDocs.totalHits); final ScoreDoc[] reranked = new ScoreDoc[topN]; final List<LeafReaderContext> leaves = searcher.getIndexReader().leaves(); final LTRScoringQuery.ModelWeight modelWeight = (LTRScoringQuery.ModelWeight) searcher .createNormalizedWeight(scoringQuery, true); final SolrIndexSearcher solrIndexSearch = (SolrIndexSearcher) searcher; scoreFeatures(solrIndexSearch, firstPassTopDocs, topN, modelWeight, hits, leaves, reranked); // Must sort all documents that we reranked, and then select the top Arrays.sort(reranked, new Comparator<ScoreDoc>() { @Override public int compare(ScoreDoc a, ScoreDoc b) { // Sort by score descending, then docID ascending: if (a.score > b.score) { return -1; } else if (a.score < b.score) { return 1; } else { // This subtraction can't overflow int // because docIDs are >= 0: return a.doc - b.doc; } } }); return new TopDocs(firstPassTopDocs.totalHits, reranked, reranked[0].score); }
From source file:org.apache.solr.search.function.TestOrdValues.java
License:Apache License
private void doTestExactScore(String field, boolean inOrder) throws Exception { IndexReader r = DirectoryReader.open(dir); IndexSearcher s = newSearcher(r); ValueSource vs;/* www .j a v a 2s .c o m*/ if (inOrder) { vs = new OrdFieldSource(field); } else { vs = new ReverseOrdFieldSource(field); } Query q = new FunctionQuery(vs); TopDocs td = s.search(q, 1000); assertEquals("All docs should be matched!", N_DOCS, td.totalHits); ScoreDoc sd[] = td.scoreDocs; for (int i = 0; i < sd.length; i++) { float score = sd[i].score; String id = s.getIndexReader().document(sd[i].doc).get(ID_FIELD); log("-------- " + i + ". Explain doc " + id); log(s.explain(q, sd[i].doc)); float expectedScore = N_DOCS - i - 1; assertEquals("score of result " + i + " shuould be " + expectedScore + " != " + score, expectedScore, score, TEST_SCORE_TOLERANCE_DELTA); String expectedId = inOrder ? id2String(N_DOCS - i) // in-order ==> larger values first : id2String(i + 1); // reverse ==> smaller values first assertTrue("id of result " + i + " shuould be " + expectedId + " != " + score, expectedId.equals(id)); } r.close(); }
From source file:org.apache.solr.update.DeleteByQueryWrapper.java
License:Apache License
@Override public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException { final LeafReader wrapped = wrap((LeafReader) searcher.getIndexReader()); final IndexSearcher privateContext = new IndexSearcher(wrapped); final Weight inner = in.createWeight(privateContext, needsScores); return new Weight(DeleteByQueryWrapper.this) { @Override// www .j a v a2 s . co m public Explanation explain(LeafReaderContext context, int doc) throws IOException { throw new UnsupportedOperationException(); } @Override public float getValueForNormalization() throws IOException { return inner.getValueForNormalization(); } @Override public void normalize(float norm, float topLevelBoost) { inner.normalize(norm, topLevelBoost); } @Override public Scorer scorer(LeafReaderContext context, Bits acceptDocs) throws IOException { return inner.scorer(privateContext.getIndexReader().leaves().get(0), acceptDocs); } }; }
From source file:org.apache.tika.eval.tokens.LuceneTokenCounter.java
License:Apache License
public LuceneTokenCounter(Analyzer generalAnalyzer) throws IOException { memoryIndex = new MemoryIndex(); IndexSearcher searcher = memoryIndex.createSearcher(); leafReader = (LeafReader) searcher.getIndexReader(); this.generalAnalyzer = generalAnalyzer; }