List of usage examples for org.apache.lucene.search IndexSearcher getIndexReader
public IndexReader getIndexReader()
From source file:ReadFiles.java
License:Apache License
public static Result doSearch(String path, DIRTYPE type, IndexReader ir) throws IOException { IndexReader reader;//from ww w . j av a 2s.co m Result r = new Result(); long beginTs, endTs; if (ir != null) reader = ir; else { beginTs = System.currentTimeMillis(); switch (type) { default: case MMAP: reader = DirectoryReader.open(MMapDirectory.open(new File(path))); break; case NIO: reader = DirectoryReader.open(NIOFSDirectory.open(new File(path))); break; case SIMPLE: reader = DirectoryReader.open(SimpleFSDirectory.open(new File(path))); break; } endTs = System.currentTimeMillis(); r.initTs += endTs - beginTs; r.initTsNr += 1; } System.out.println("-----Search it------"); IndexSearcher searcher = new IndexSearcher(reader); Query q = NumericRangeQuery.newIntRange("foo", new Integer("100000"), null, false, false); beginTs = System.currentTimeMillis(); ScoreDoc[] hits = searcher.search(q, searcher.getIndexReader().maxDoc()).scoreDocs; endTs = System.currentTimeMillis(); r.searchTs += endTs - beginTs; r.searchTsNr += hits.length; System.out.println("Hits -> " + hits.length); boolean isSeq = true; int lastid = 0; beginTs = System.currentTimeMillis(); for (int i = 0; i < hits.length; i++) { if (hits[i].doc < lastid) isSeq = false; Document doc = searcher.doc(hits[i].doc); doc.get("foo"); doc.get("bar"); //System.out.println("Key: " + doc.get("foo") + ", Value: " + doc.get("bar")); } System.out.println("Search DocID is SEQ? " + isSeq); endTs = System.currentTimeMillis(); r.fetchTs += endTs - beginTs; r.fetchTsNr += hits.length; if (ir == null) { beginTs = System.currentTimeMillis(); reader.close(); endTs = System.currentTimeMillis(); r.closeTs += endTs - beginTs; r.closeTsNr += 1; } return r; }
From source file:aos.lucene.admin.SearcherManager.java
License:Apache License
public synchronized void release(IndexSearcher searcher) throws IOException { searcher.getIndexReader().decRef(); }
From source file:aos.lucene.tools.FastVectorHighlighterSample.java
License:Apache License
static void searchIndex(String filename) throws Exception { QueryParser parser = new QueryParser(Version.LUCENE_46, F, analyzer); Query query = parser.parse(QUERY); FastVectorHighlighter highlighter = getHighlighter(); // #C FieldQuery fieldQuery = highlighter.getFieldQuery(query); // #D IndexSearcher searcher = new IndexSearcher(dir); TopDocs docs = searcher.search(query, 10); FileWriter writer = new FileWriter(filename); writer.write("<html>"); writer.write("<body>"); writer.write("<p>QUERY : " + QUERY + "</p>"); for (ScoreDoc scoreDoc : docs.scoreDocs) { String snippet = highlighter.getBestFragment( // #E fieldQuery, searcher.getIndexReader(), // #E scoreDoc.doc, F, 100); // #E if (snippet != null) { writer.write(scoreDoc.doc + " : " + snippet + "<br/>"); }// w ww .ja v a 2s . c o m } writer.write("</body></html>"); writer.close(); }
From source file:aos.lucene.tools.HighlightTest.java
License:Apache License
public void testHits() throws Exception { IndexSearcher searcher = new IndexSearcher(TestUtil.getBookIndexDirectory()); TermQuery query = new TermQuery(new Term("title", "action")); TopDocs hits = searcher.search(query, 10); QueryScorer scorer = new QueryScorer(query, "title"); Highlighter highlighter = new Highlighter(scorer); highlighter.setTextFragmenter(new SimpleSpanFragmenter(scorer)); Analyzer analyzer = new SimpleAnalyzer(); for (ScoreDoc sd : hits.scoreDocs) { Document doc = searcher.doc(sd.doc); String title = doc.get("title"); TokenStream stream = TokenSources.getAnyTokenStream(searcher.getIndexReader(), sd.doc, "title", doc, analyzer);//from w ww . j a v a2 s. co m String fragment = highlighter.getBestFragment(stream, title); LOGGER.info(fragment); } }
From source file:arena.lucene.LuceneIndexSearcherImpl.java
License:Open Source License
protected TopDocs executeSearch(IndexSearcher searcher, Query query, Filter filter, Sort sort, int collectorLimit) throws IOException { // Decide on how to search based on which elements of the lucene query model are available if (query != null) { // Full scoring search TopDocsCollector<? extends ScoreDoc> collector = null; if (sort == null) { collector = TopScoreDocCollector.create(collectorLimit, true); } else {/*from ww w. j av a 2 s .com*/ SortField sortFields[] = sort.getSort(); if (sortFields != null && sortFields.length > 0 && sortFields[0].getType() == SortField.SCORE && !sortFields[0].getReverse()) { collector = TopScoreDocCollector.create(collectorLimit, true); } else { collector = TopFieldCollector.create(sort, collectorLimit, false, true, true, true); } } searcher.search(query, filter, collector); return collector.topDocs(); } else if (filter != null) { // No query = no need for scoring, just dump the results into a hit collector that runs // off the results in the order we want DocIdSetIterator filterMatchesIterator = filter.getDocIdSet(searcher.getIndexReader()).iterator(); if (sort == null) { // no sort available, so the natural iteration order is fine // if we have an iterator that means sorting is already handled, so just pull off the first n rows into the output ScoreDoc[] scoreDocs = new ScoreDoc[collectorLimit]; int found = 0; int docId; while (found < collectorLimit && (docId = filterMatchesIterator.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { scoreDocs[found++] = new ScoreDoc(docId, 1f); } return new TopDocs(found, found < collectorLimit ? Arrays.copyOf(scoreDocs, found) : scoreDocs, 1f); } else { TopDocsCollector<? extends ScoreDoc> collector = TopFieldCollector.create(sort, collectorLimit, false, true, true, true); int docId; while ((docId = filterMatchesIterator.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { collector.collect(docId); } return collector.topDocs(); } } else if (sort != null) { // no query and no filter so no score but add every doc in the index for non-score sorting TopDocsCollector<? extends ScoreDoc> collector = TopFieldCollector.create(sort, collectorLimit, false, true, true, true); int numDocs = searcher.getIndexReader().numDocs(); for (int n = 0; n < numDocs; n++) { collector.collect(n); } return collector.topDocs(); } else { // no query filter or sort: return the top n docs ScoreDoc[] scoreDocs = new ScoreDoc[Math.min(collectorLimit, searcher.getIndexReader().numDocs())]; for (int n = 0; n < scoreDocs.length; n++) { scoreDocs[n] = new ScoreDoc(n, 1f); } return new TopDocs(scoreDocs.length, scoreDocs, 1f); } }
From source file:au.edu.unimelb.csse.search.complete.AllResults.java
License:Apache License
public Result[] collect(IndexSearcher searcher) throws IOException { expr.init(searcher.getIndexReader()); Result[] r = new Result[numberOfResults]; for (int i = 0; i < numberOfResults; i++) { expr.skipTo(hits[i]);/*from w w w. j ava2 s . c o m*/ Result result = new Result(); byte[] payloadBuffer = new byte[NodeDataBuffer.BUFFER_SIZE * 4]; int[] positionsBuffer = new int[NodeDataBuffer.BUFFER_SIZE]; NodeDataBuffer data = new NodeDataBuffer(); r[i] = expr.allStructureMatch(result, data, payloadBuffer, positionsBuffer); } return r; }
From source file:axiom.db.utils.LuceneManipulator.java
License:Open Source License
public void compress(String dbDir) throws Exception { System.setProperty("org.apache.lucene.FSDirectory.class", "org.apache.lucene.store.TransFSDirectory"); File dbhome = new File(dbDir); String url = getUrl(dbhome);/* www .j ava 2 s .c o m*/ FSDirectory indexDir = FSDirectory.getDirectory(dbhome, false); if (indexDir instanceof TransFSDirectory) { FSDirectory.setDisableLocks(true); TransFSDirectory d = (TransFSDirectory) indexDir; d.setDriverClass(DRIVER_CLASS); d.setUrl(url); d.setUser(null); d.setPassword(null); } File ndbhome = new File(dbhome.getParentFile(), dbhome.getName() + "_tmp"); File olddbhome = new File(dbhome.getParentFile(), dbhome.getName() + "_old"); FSDirectory nindexDir = FSDirectory.getDirectory(ndbhome, true); if (nindexDir instanceof TransFSDirectory) { FSDirectory.setDisableLocks(true); TransFSDirectory d = (TransFSDirectory) nindexDir; d.setDriverClass(DRIVER_CLASS); d.setUrl(url); d.setUser(null); d.setPassword(null); } IndexSearcher searcher = null; IndexWriter writer = null; LuceneManager lmgr = null; try { searcher = new IndexSearcher(indexDir); PerFieldAnalyzerWrapper a = LuceneManager.buildAnalyzer(); writer = IndexWriterManager.getWriter(nindexDir, a, true); final int numDocs = searcher.getIndexReader().numDocs(); HashSet deldocs = new HashSet(); HashMap infos = new HashMap(); for (int i = 0; i < numDocs; i++) { Document doc = searcher.doc(i); String delprop = doc.get(DeletedInfos.DELETED); final String id = doc.get(LuceneManager.ID) + DeletedInfos.KEY_SEPERATOR + doc.get(LuceneManager.LAYER_OF_SAVE); if (delprop != null && "true".equals(delprop)) { deldocs.add(id); } else { Object v; if ((v = infos.get(id)) == null) { infos.put(id, new Integer(i)); } else { final String lmod = doc.get(LuceneManager.LASTMODIFIED); final String lmod_prev = searcher.doc(((Integer) v).intValue()).get("_lastmodified"); if (lmod_prev == null || (lmod != null && lmod.compareTo(lmod_prev) > 0)) { infos.put(id, new Integer(i)); } } } } ArrayList listOfMaps = new ArrayList(); for (int i = 0; i < numDocs; i++) { Document doc = searcher.doc(i); String delprop = doc.get(DeletedInfos.DELETED); String layerStr = doc.get(LuceneManager.LAYER_OF_SAVE); int layer = -1; try { layer = Integer.parseInt(layerStr); } catch (Exception ex) { layer = -1; } final String id = doc.get(LuceneManager.ID) + DeletedInfos.KEY_SEPERATOR + doc.get(LuceneManager.LAYER_OF_SAVE); if (delprop != null && "true".equals(delprop)) { continue; } else if (id != null && deldocs.contains(id)) { continue; } Integer idx = (Integer) infos.get(id); if (idx != null && i != idx.intValue()) { continue; } Document ndoc = convertDocument(doc); if (ndoc != null) { writer.addDocument(ndoc); } } } catch (Exception ex) { ex.printStackTrace(); throw new RuntimeException(ex); } finally { if (searcher != null) { try { searcher.close(); } catch (Exception ex) { } } if (lmgr != null) { lmgr.shutdown(); lmgr = null; } indexDir.close(); SegmentInfos sinfos = IndexObjectsFactory.getFSSegmentInfos(indexDir); sinfos.clear(); IndexObjectsFactory.removeDeletedInfos(indexDir); } Connection conn = null; boolean exceptionOccured = false; try { if (writer != null) { conn = DriverManager.getConnection(url); conn.setAutoCommit(false); writer.close(); writer.flushCache(); LuceneManager.commitSegments(null, conn, dbhome, writer.getDirectory()); writer.finalizeTrans(); } } catch (Exception ex) { ex.printStackTrace(); exceptionOccured = true; throw new RuntimeException(ex); } finally { if (conn != null) { try { if (!conn.getAutoCommit()) { if (!exceptionOccured) { conn.commit(); } else { conn.rollback(); } } conn.close(); } catch (Exception ex) { ex.printStackTrace(); } conn = null; } nindexDir.close(); SegmentInfos sinfos = IndexObjectsFactory.getFSSegmentInfos(nindexDir); sinfos.clear(); IndexObjectsFactory.removeDeletedInfos(nindexDir); } File[] files = dbhome.listFiles(); for (int i = 0; i < files.length; i++) { if (!files[i].isDirectory()) { files[i].delete(); } } files = ndbhome.listFiles(); for (int i = 0; i < files.length; i++) { if (!files[i].isDirectory()) { File nfile = new File(dbhome, files[i].getName()); files[i].renameTo(nfile); } } if (!FileUtils.deleteDir(ndbhome)) { throw new Exception("Could not delete " + ndbhome); } }
From source file:axiom.objectmodel.dom.convert.LuceneConvertor.java
License:Open Source License
public void convert(Application app, File dbhome) throws Exception { FSDirectory indexDir = FSDirectory.getDirectory(dbhome, false); if (indexDir instanceof TransFSDirectory) { FSDirectory.setDisableLocks(true); TransFSDirectory d = (TransFSDirectory) indexDir; TransSource source = app.getTransSource(); d.setDriverClass(source.getDriverClass()); d.setUrl(source.getUrl());/*w ww . j a va 2s . c o m*/ d.setUser(source.getUser()); d.setPassword(source.getPassword()); } File ndbhome = new File(dbhome.getParentFile(), dbhome.getName() + "_tmp"); File olddbhome = new File(dbhome.getParentFile(), dbhome.getName() + "_old"); FSDirectory nindexDir = FSDirectory.getDirectory(ndbhome, true); if (nindexDir instanceof TransFSDirectory) { FSDirectory.setDisableLocks(true); TransFSDirectory d = (TransFSDirectory) nindexDir; TransSource source = app.getTransSource(); d.setDriverClass(source.getDriverClass()); d.setUrl(source.getUrl()); d.setUser(source.getUser()); d.setPassword(source.getPassword()); } IndexSearcher searcher = null; IndexWriter writer = null; LuceneManager lmgr = null; try { searcher = new IndexSearcher(indexDir); PerFieldAnalyzerWrapper a = LuceneManager.buildAnalyzer(); writer = IndexWriterManager.getWriter(nindexDir, a, true); final int numDocs = searcher.getIndexReader().numDocs(); HashSet deldocs = new HashSet(); HashMap infos = new HashMap(); for (int i = 0; i < numDocs; i++) { Document doc = searcher.doc(i); String delprop = doc.get(DeletedInfos.DELETED); String layerStr = doc.get(LuceneManager.LAYER_OF_SAVE); int layer = -1; try { layer = Integer.parseInt(layerStr); } catch (Exception ex) { layer = -1; } final String id = doc.get(LuceneManager.ID) + DeletedInfos.KEY_SEPERATOR + doc.get(LuceneManager.LAYER_OF_SAVE); if (delprop != null && "true".equals(delprop)/* && layer == DbKey.LIVE_LAYER*/) { deldocs.add(id); } else { Object v; if ((v = infos.get(id)) == null) { infos.put(id, new Integer(i)); } else { final String lmod = doc.get(LuceneManager.LASTMODIFIED); final String lmod_prev = searcher.doc(((Integer) v).intValue()).get("_lastmodified"); if (lmod_prev == null || (lmod != null && lmod.compareTo(lmod_prev) > 0)) { infos.put(id, new Integer(i)); } } } } ArrayList listOfMaps = new ArrayList(); for (int i = 0; i < numDocs; i++) { Document doc = searcher.doc(i); String delprop = doc.get(DeletedInfos.DELETED); String layerStr = doc.get(LuceneManager.LAYER_OF_SAVE); int layer = -1; try { layer = Integer.parseInt(layerStr); } catch (Exception ex) { layer = -1; } final String id = doc.get(LuceneManager.ID) + DeletedInfos.KEY_SEPERATOR + doc.get(LuceneManager.LAYER_OF_SAVE); if (delprop != null && "true".equals(delprop)) { continue; } else if (id != null && deldocs.contains(id)/* && layer == DbKey.LIVE_LAYER*/) { continue; } Integer idx = (Integer) infos.get(id); if (idx != null && i != idx.intValue()) { continue; } Document ndoc = convertDocument(doc); if (this.recordNodes) { listOfMaps.add(LuceneManager.luceneDocumentToMap(doc)); } if (ndoc != null) { writer.addDocument(ndoc); } } if (this.recordNodes) { lmgr = new LuceneManager(this.app, false, true); this.allNodes = new HashMap(); final int size = listOfMaps.size(); for (int i = 0; i < size; i++) { HashMap m = (HashMap) listOfMaps.get(i); INode n = lmgr.mapToNode(m); this.allNodes.put(n.getID(), getPath(n)); n = null; } } } catch (Exception ex) { ex.printStackTrace(); throw new RuntimeException(ex); } finally { if (searcher != null) { try { searcher.close(); } catch (Exception ex) { app.logError(ErrorReporter.errorMsg(this.getClass(), "convert"), ex); } } if (lmgr != null) { lmgr.shutdown(); lmgr = null; } indexDir.close(); SegmentInfos sinfos = IndexObjectsFactory.getFSSegmentInfos(indexDir); sinfos.clear(); IndexObjectsFactory.removeDeletedInfos(indexDir); } Connection conn = null; boolean exceptionOccured = false; try { if (writer != null) { TransSource ts = app.getTransSource(); conn = ts.getConnection(); DatabaseMetaData dmd = conn.getMetaData(); ResultSet rs = dmd.getColumns(null, null, "Lucene", "version"); if (!rs.next()) { final String alterTbl = "ALTER TABLE Lucene ADD version INT NOT NULL DEFAULT 1"; PreparedStatement pstmt = null; try { pstmt = conn.prepareStatement(alterTbl); pstmt.execute(); } catch (SQLException sqle) { app.logError(ErrorReporter.errorMsg(this.getClass(), "convert"), sqle); } finally { if (pstmt != null) { pstmt.close(); pstmt = null; } } } rs.close(); rs = null; writer.close(); writer.flushCache();//TODO:writer.writeSegmentsFile(); LuceneManager.commitSegments(conn, app, writer.getDirectory()); writer.finalizeTrans(); this.updateSQL(conn); } } catch (Exception ex) { ex.printStackTrace(); exceptionOccured = true; throw new RuntimeException(ex); } finally { if (conn != null) { try { if (!conn.getAutoCommit()) { if (!exceptionOccured) { conn.commit(); } else { conn.rollback(); } } conn.close(); } catch (Exception ex) { app.logError(ErrorReporter.errorMsg(this.getClass(), "convert"), ex); } conn = null; } nindexDir.close(); SegmentInfos sinfos = IndexObjectsFactory.getFSSegmentInfos(nindexDir); sinfos.clear(); IndexObjectsFactory.removeDeletedInfos(nindexDir); } if (!dbhome.renameTo(olddbhome)) { throw new Exception("Could not move the old version of the db into " + olddbhome); } if (!ndbhome.renameTo(dbhome)) { throw new Exception("Could not move the newer version of the db into " + dbhome); } File oldBlobDir = new File(olddbhome, "blob"); File newBlobDir = new File(ndbhome, "blob"); oldBlobDir.renameTo(newBlobDir); if (!FileUtils.deleteDir(olddbhome)) { throw new Exception("Could not delete the old version of the db at " + olddbhome); } }
From source file:be.ugent.tiwi.sleroux.newsrec.newsreclib.recommend.scorers.DatabaseLuceneScorer.java
License:Apache License
protected Map<String, Double> getTopTerms(String field, String value) throws IOException { manager.maybeRefreshBlocking();//from ww w . j av a 2s. c o m IndexSearcher searcher = manager.acquire(); try (IndexReader reader = searcher.getIndexReader()) { TopScoreDocCollector collector = TopScoreDocCollector.create(1, true); Query q = new TermQuery(new Term(field, value)); searcher.search(q, collector); if (collector.getTotalHits() > 0) { int docNr = collector.topDocs().scoreDocs[0].doc; Document doc = reader.document(docNr); NewsItem nitem = NewsItemLuceneDocConverter.documentToNewsItem(doc); return nitem.getTerms(); } else { logger.warn("Could not find document with " + field + "=" + value); } } manager.release(searcher); return new HashMap<>(); }
From source file:br.bireme.ngrams.NGrams.java
/** * * @param index/* ww w .j a v a 2s.c o m*/ * @param schema * @param inFile * @param inFileEncoding * @param outFile * @param outFileEncoding * @throws IOException * @throws ParseException */ public static void search(final NGIndex index, final NGSchema schema, final String inFile, final String inFileEncoding, final String outFile, final String outFileEncoding) throws IOException, ParseException { if (index == null) { throw new NullPointerException("index"); } if (schema == null) { throw new NullPointerException("schema"); } if (inFile == null) { throw new NullPointerException("inFile"); } if (inFileEncoding == null) { throw new NullPointerException("inFileEncoding"); } if (outFile == null) { throw new NullPointerException("outFile"); } if (outFileEncoding == null) { throw new NullPointerException("outFileEncoding"); } final Charset inCharset = Charset.forName(inFileEncoding); final Charset outCharset = Charset.forName(outFileEncoding); final IndexSearcher searcher = index.getIndexSearcher(); final NGAnalyzer analyzer = (NGAnalyzer) index.getAnalyzer(); final Parameters parameters = schema.getParameters(); final NGramDistance ngDistance = new NGramDistance(analyzer.getNgramSize()); final Set<String> id_id = new HashSet<>(); int cur = 0; try (final BufferedReader reader = Files.newBufferedReader(new File(inFile).toPath(), inCharset); final BufferedWriter writer = Files.newBufferedWriter(new File(outFile).toPath(), outCharset)) { writer.append("rank|similarity|search_doc_id|index_doc_id|" + "ngram_search_text|ngram_index_text|search_source|" + "index_source\n"); final Set<Result> results = new HashSet<>(); while (true) { final String line = reader.readLine(); if (line == null) { break; } if (++cur % 250 == 0) { System.out.println("<<< " + cur); } results.clear(); final String tline = line.replace(':', ' ').trim(); if (!tline.isEmpty()) { final String[] split = tline.split(" *\\| *", Integer.MAX_VALUE); if (split.length != parameters.nameFields.size()) { throw new IOException("invalid number of fields: " + line); } searchRaw(parameters, searcher, analyzer, ngDistance, tline, true, id_id, results); if (!results.isEmpty()) { writeOutput(parameters, results, writer); } } } searcher.getIndexReader().close(); } }