List of usage examples for org.apache.lucene.search IndexSearcher doc
public Document doc(int docID) throws IOException
.getIndexReader().document(docID) From source file:edu.coeia.tasks.ImageLoadingTask.java
License:Open Source License
private Set<ImagePathAndId> loadItemsFast(int from, int size) throws IOException { Set<ImagePathAndId> files = new HashSet<ImagePathAndId>(); int counter = 0; try {//from w ww . j ava 2s . c om Directory directory = FSDirectory.open(new File(this.caseFacade.getCaseIndexFolderLocation())); IndexSearcher searcher = new IndexSearcher(directory); QueryParser parser = new QueryParser(Version.LUCENE_30, IndexingConstant.DOCUMENT_TYPE, new StopAnalyzer(Version.LUCENE_30)); parser.setAllowLeadingWildcard(true); Query query = parser .parse(IndexingConstant.fromDocumentTypeToString(IndexingConstant.DOCUMENT_GENERAL_TYPE.IMAGE)); TopDocs topDocs = searcher.search(query, 500000); for (ScoreDoc scoreDoc : topDocs.scoreDocs) { Document document = searcher.doc(scoreDoc.doc); String imageExtension = document.get(IndexingConstant.FILE_MIME); if (imageExtension != null && !imageExtension.trim().isEmpty() && Arrays.asList(imageExtensions).contains(imageExtension)) { String fullpath = ""; int id = Integer.parseInt(document.get(IndexingConstant.DOCUMENT_ID)); if (IndexingConstant.isImageDocument(document)) { String path = document.get(IndexingConstant.FILE_PATH); if (path.contains(this.aCase.getCaseName() + File.separator + ApplicationConstants.CASE_ARCHIVE_FOLDER)) fullpath = path; else fullpath = this.caseFacade.getFullPath(document.get(IndexingConstant.FILE_PATH)); } if (!fullpath.isEmpty()) { counter++; if (files.size() >= size) break; if (counter >= from) { files.add(new ImagePathAndId(fullpath, Integer.valueOf(id))); } } } } searcher.close(); } catch (ParseException ex) { Logger.getLogger(ChatRefreshTask.class.getName()).log(Level.SEVERE, null, ex); } return files; }
From source file:edu.coeia.tasks.ImageLoadingTask.java
License:Open Source License
private int getNumberOfImagesFast() throws IOException { int numberOfImages = 0; try {//from w ww. java2s . c o m Directory directory = FSDirectory.open(new File(this.caseFacade.getCaseIndexFolderLocation())); IndexSearcher searcher = new IndexSearcher(directory); QueryParser parser = new QueryParser(Version.LUCENE_30, IndexingConstant.DOCUMENT_TYPE, new StopAnalyzer(Version.LUCENE_30)); parser.setAllowLeadingWildcard(true); Query query = parser .parse(IndexingConstant.fromDocumentTypeToString(IndexingConstant.DOCUMENT_GENERAL_TYPE.IMAGE)); TopDocs topDocs = searcher.search(query, 500000); for (ScoreDoc scoreDoc : topDocs.scoreDocs) { Document document = searcher.doc(scoreDoc.doc); String imageExtension = document.get(IndexingConstant.FILE_MIME); if (imageExtension != null && !imageExtension.trim().isEmpty() && Arrays.asList(imageExtensions).contains(imageExtension)) { String fullpath = ""; if (IndexingConstant.isImageDocument(document)) { String path = document.get(IndexingConstant.FILE_PATH); if (path.contains(this.aCase.getCaseName() + File.separator + ApplicationConstants.CASE_ARCHIVE_FOLDER)) fullpath = path; else fullpath = this.caseFacade.getFullPath(document.get(IndexingConstant.FILE_PATH)); } if (!fullpath.isEmpty()) { numberOfImages++; } } } searcher.close(); } catch (ParseException ex) { Logger.getLogger(ChatRefreshTask.class.getName()).log(Level.SEVERE, null, ex); } return numberOfImages; }
From source file:edu.cuhk.hccl.cmd.AppSearchEngine.java
License:Apache License
public static void main(String[] args) throws IOException { // Get parameters CommandLineParser parser = new BasicParser(); Options options = createOptions();/*from w w w . j a v a 2 s . c o m*/ File dataFolder = null; String queryStr = null; int topK = 0; File resultFile = null; String queryType = null; File similarityFile = null; try { CommandLine line = parser.parse(options, args); dataFolder = new File(line.getOptionValue('d')); queryStr = line.getOptionValue('q'); queryType = line.getOptionValue('t'); topK = Integer.parseInt(line.getOptionValue('k')); resultFile = new File(line.getOptionValue('f')); similarityFile = new File(line.getOptionValue('s')); if (line.hasOption('m')) { String modelPath = line.getOptionValue('m'); if (queryType.equalsIgnoreCase("WordVector")) { expander = new WordVectorExpander(modelPath); } else if (queryType.equalsIgnoreCase("WordNet")) { expander = new WordNetExpander(modelPath); } else { System.out.println("Please choose a correct expander: WordNet or WordVector!"); System.exit(-1); } } } catch (ParseException exp) { System.out.println("Error in parameters: \n" + exp.getMessage()); System.exit(-1); } // Create Index StandardAnalyzer analyzer = new StandardAnalyzer(); Directory index = createIndex(dataFolder, analyzer); // Build query Query query = buildQuery(analyzer, queryStr, queryType); // Search index for topK hits IndexReader reader = DirectoryReader.open(index); IndexSearcher searcher = new IndexSearcher(reader); TopScoreDocCollector collector = TopScoreDocCollector.create(topK, true); searcher.search(query, collector); ScoreDoc[] hits = collector.topDocs().scoreDocs; // Show search results System.out.println("\n[INFO] " + hits.length + " hits were returned:"); List<String> hitLines = new ArrayList<String>(); for (int i = 0; i < hits.length; i++) { int docId = hits[i].doc; Document d = searcher.doc(docId); String line = (i + 1) + "\t" + d.get(PATH_FIELD) + "\t" + hits[i].score; System.out.println(line); hitLines.add(line); } // Compute cosine similarity between documents List<String> simLines = new ArrayList<String>(); for (int m = 0; m < hits.length; m++) { int doc1 = hits[m].doc; Terms terms1 = reader.getTermVector(doc1, CONTENT_FIELD); for (int n = m + 1; n < hits.length; n++) { int doc2 = hits[n].doc; Terms terms2 = reader.getTermVector(doc2, CONTENT_FIELD); CosineDocumentSimilarity cosine = new CosineDocumentSimilarity(terms1, terms2); double similarity = cosine.getCosineSimilarity(); String line = searcher.doc(doc1).get(PATH_FIELD) + "\t" + searcher.doc(doc2).get(PATH_FIELD) + "\t" + similarity; simLines.add(line); } } // Release resources reader.close(); if (expander != null) { expander.close(); } // Save search results System.out.println("\n[INFO] Search results are saved in file: " + resultFile.getPath()); FileUtils.writeLines(resultFile, hitLines, false); System.out.println("\n[INFO] Cosine similarities are saved in file: " + similarityFile.getPath()); FileUtils.writeLines(similarityFile, simLines, false); }
From source file:edu.indiana.d2i.htrc.io.SparseVectorsFromLucene.java
License:Apache License
private void generateIDList(String indexLoc, Path idfile, String queryStr) throws IOException { Configuration conf = getConf(); FileSystem fs = FileSystem.get(conf); Path indexPath = new Path(indexLoc); Directory dir = new FileSystemDirectory(fs, indexPath, false, conf); IndexSearcher indexSearcher = new IndexSearcher(dir); Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_31); QueryParser parser = new QueryParser(Version.LUCENE_31, IndexReader.FieldOption.INDEXED_WITH_TERMVECTOR.toString(), analyzer); try {//from ww w .jav a2s. c o m Query query = parser.parse(queryStr); TopDocs tip_docs = indexSearcher.search(query, indexSearcher.maxDoc()); logger.info("total hits: " + tip_docs.totalHits); BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(fs.create(idfile, true))); ScoreDoc[] score_docs = tip_docs.scoreDocs; for (int i = 0; i < score_docs.length; i++) { writer.write(indexSearcher.doc(score_docs[i].doc).get("id") + "\n"); } writer.close(); } catch (ParseException e) { logger.error(e); } finally { indexSearcher.close(); } }
From source file:edu.jhu.pha.vosync.rest.DropboxService.java
License:Apache License
@Path("cont_search") @GET//www .j ava2s . co m @RolesAllowed({ "user", "rwshareuser" }) public Response contSearch(@QueryParam("query") String queryStr) { SciDriveUser user = ((SciDriveUser) security.getUserPrincipal()); try { Directory directory = FSDirectory.open(new File(conf.getString("lucene.index"))); DirectoryReader ireader = DirectoryReader.open(directory); IndexSearcher isearcher = new IndexSearcher(ireader); Analyzer analyzer = new EnglishAnalyzer(Version.LUCENE_41); QueryParser parser = new QueryParser(Version.LUCENE_41, "content", analyzer); String queryFullStr = "owner:\"" + user.getName() + "\" AND " + queryStr; Query query = parser.parse(queryFullStr); ScoreDoc[] hits = isearcher.search(query, null, 100).scoreDocs; StringBuffer buf = new StringBuffer(); buf.append("<p>Results: " + hits.length + "</p>"); for (int i = 0; i < hits.length; i++) { Document hitDoc = isearcher.doc(hits[i].doc); buf.append("<h3>" + hitDoc.get("source") + "</h3>"); if (hitDoc.get("content").length() > 1024) buf.append("<p>" + hitDoc.get("content").substring(0, 1024) + "...</p>"); else buf.append("<p>" + hitDoc.get("content") + "</p>"); } analyzer.close(); ireader.close(); directory.close(); return Response.ok(buf.toString()).build(); } catch (Exception ex) { ex.printStackTrace(); return Response.ok().build(); } }
From source file:edu.ku.brc.specify.dbsupport.cleanuptools.GeoCleanupFuzzySearch.java
License:Open Source License
public static void main(String[] args) throws IOException { //For Debug// ww w. j a v a 2s . c o m String connectStr = "jdbc:mysql://localhost/testfish"; String username = "root"; String password = "root"; DBConnection dbConn; // Debug dbConn = DBConnection.getInstance(); dbConn.setConnectionStr(connectStr); dbConn.setDatabaseName("stats"); dbConn.setUsernamePassword(username, password); dbConn.setDriver("com.mysql.jdbc.Driver"); boolean doBuildIndex = false; //String indexLocation = "/Users/rods/Downloads/lucene/geonames-index"; String indexLocation = "/Users/rods/Documents/Specify/geonames-index"; GeoCleanupFuzzySearch indexer = null; try { indexer = new GeoCleanupFuzzySearch(null); if (doBuildIndex) { indexer.startIndexingProcessSync(1, null); } } catch (Exception ex) { System.out.println("Cannot create index..." + ex.getMessage()); System.exit(-1); } // =================================================== // after adding, we always have to call the // closeIndex, otherwise the index is not created // =================================================== // indexer.closeIndex(); // ========================================================= // Now search // ========================================================= IndexReader reader = DirectoryReader.open(FSDirectory.open(new File(indexLocation))); IndexSearcher searcher = new IndexSearcher(reader); boolean doFuzzy = false; boolean doTerm = false; boolean doParse = true; if (doFuzzy) { System.out.println("-------------------------- Fuzzy -----------------------"); String[] searchStrs = { "Comoro Islands", "Solomon", "united states iowa", "germany brandenburg", "bulgaria sofia", "costa rica alajuela", "costa rica cartago", "costa rica alajuela", "canada newfoundland", "mexico campeche", "australia ashmore and cartier islands", "fiji lau", "fiji lomaiviti", "guam agana", "germany Lower Saxony", "germany Saxony", "germany Sachsen Anhalt", "germany Sachsen-Anhalt", "germany Land Sachsen-Anhalt", "united states iowa,Fayette", "united states iowa Fayette County", "Argentina Buenos Aires", "buenos aires argentina ", }; for (String searchText : searchStrs) { try { Query query = new FuzzyQuery(new Term("name", searchText)); TopDocs docs = searcher.search(query, 10); ScoreDoc[] hits = docs.scoreDocs; System.out.println(searchText + " -> Hits " + hits.length + " hits [" + query + "]"); for (int i = 0; i < hits.length; ++i) { int docId = hits[i].doc; Document d = searcher.doc(docId); System.out.println((i + 1) + ". " + d.get("name") + " score=" + hits[i].score); } } catch (Exception e) { System.out.println("Error searching " + searchText + " : " + e.getMessage()); } } } if (doTerm) { System.out.println("-------------------------- Terms -----------------------"); String[] searchStrs = { "Comoro Islands", "Solomon", "united states,iowa", "germany,brandenburg", "bulgaria,sofia", "costa rica,alajuela", "costa rica,cartago", "costa rica,alajuela", "canada,newfoundland", "mexico,campeche", "australia,ashmore and cartier islands", "fiji,lau", "fiji,lomaiviti", "guam,agana", "germany,Lower Saxony", "germany,Saxony", "germany,Sachsen Anhalt", "germany,Sachsen-Anhalt", "germany,Land Sachsen-Anhalt", "united states,iowa,Fayette", "united states,iowa,Fayette County", "argentina,buenos aires", "Argentina,Buenos Aires", }; for (String searchText : searchStrs) { try { String[] tokens = StringUtils.split(searchText, ','); BooleanQuery query = new BooleanQuery(); TermQuery t1 = new TermQuery(new Term("country", tokens[0])); t1.setBoost(0.2f); query.add(t1, Occur.SHOULD); if (tokens.length > 1) { TermQuery t2 = new TermQuery(new Term("state", tokens[1])); t2.setBoost(0.4f); query.add(t2, Occur.SHOULD); } if (tokens.length > 2) { TermQuery t3 = new TermQuery(new Term("county", tokens[2])); t3.setBoost(0.8f); query.add(t3, Occur.MUST); } TopDocs docs = searcher.search(query, 20); ScoreDoc[] hits = docs.scoreDocs; System.out.println(searchText + " -> Hits " + hits.length + " hits [" + query + "]"); for (int i = 0; i < hits.length; ++i) { int docId = hits[i].doc; Document d = searcher.doc(docId); System.out.println((i + 1) + ". " + d.get("name") + " score=" + hits[i].score); } } catch (Exception e) { System.out.println("Error searching " + searchText + " : " + e.getMessage()); } } } if (doParse) { System.out.println("-------------------------- Parsing -----------------------"); String[] searchStrs = { "Comoro Islands", "Bahamas Elbow Bank", // "Solomon", // "united states iowa", // "germany brandenburg", // "bulgaria sofia", // "costa rica alajuela", // "costa rica cartago", // "costa rica alajuela", // "canada newfoundland", // "mexico campeche", // "australia ashmore and cartier islands", // "fiji lau", // "fiji lomaiviti", // "guam agana", // "germany Lower Saxony", // "germany Saxony", // "germany Sachsen Anhalt", // "germany Sachsen-Anhalt", // "germany Land Sachsen-Anhalt", // "united states iowa,Fayette", // "united states iowa Fayette County", // "Argentina Buenos Aires", // "buenos aires argentina " }; for (String searchText : searchStrs) { try { TopScoreDocCollector collector = TopScoreDocCollector.create(5, true); Query q = new QueryParser(Version.LUCENE_47, "name", analyzer).parse(searchText); searcher.search(q, collector); ScoreDoc[] hits = collector.topDocs().scoreDocs; if (hits != null) { System.out.println(searchText + " -> Hits " + hits.length + " hits."); // System.out.println("For: ["+seatchText+"] Found " + hits.length + " hits."); for (int i = 0; i < hits.length; ++i) { int docId = hits[i].doc; Document d = searcher.doc(docId); if (d != null) { System.out.println((i + 1) + ". " + d.get("name") + " score=" + hits[i].score); } else { System.err.println("Doc was null searching " + searchText); } } } else { System.err.println("Hits was null searching " + searchText); } } catch (Exception e) { e.printStackTrace(); System.err.println("Error searching " + searchText + " : " + e.getMessage()); } } } }
From source file:edu.ku.brc.specify.tools.LocalizerSearchHelper.java
License:Open Source License
public int[] doPropsSearch(final String searchText, final String fieldName) { try {/* w w w . ja v a2 s .com*/ QueryParser parser = new QueryParser(Version.LUCENE_47, fieldName, analyzer); Query query = parser.parse(searchText.toLowerCase()); //System.out.println("Searching for: " + query.toString(fieldName)); IndexSearcher memSearcher = new IndexSearcher(DirectoryReader.open(memIndexer)); TopScoreDocCollector collector = TopScoreDocCollector.create(50000, true); memSearcher.search(query, collector); ScoreDoc[] hits = collector.topDocs().scoreDocs; int[] inxs = new int[hits.length]; int i = 0; for (ScoreDoc doc : hits) { Document d = memSearcher.doc(doc.doc); //System.out.println(doc.doc+" "+doc.score+" "+d.get("index")); inxs[i++] = Integer.parseInt(d.get("index")); } return inxs; } catch (Exception ex) { ex.printStackTrace(); } return null; }
From source file:edu.ku.brc.specify.tools.schemalocale.LocalizerApp.java
License:Open Source License
/** * This demonstrates a typical paging search scenario, where the search engine presents * pages of size n to the user. The user can then go to the next page if interested in * the next hits./*from www.j ava2s.c o m*/ * * When the query is executed for the first time, then only enough results are collected * to fill 5 result pages. If the user wants to page beyond this limit, then the query * is executed another time and all hits are collected. * */ public static void doPagingSearch(BufferedReader in, IndexSearcher searcher, Query query, int hitsPerPage, boolean raw, boolean interactive) throws IOException { // Collect enough docs to show 5 pages TopScoreDocCollector collector = TopScoreDocCollector.create(5 * hitsPerPage, false); searcher.search(query, collector); ScoreDoc[] hits = collector.topDocs().scoreDocs; int numTotalHits = collector.getTotalHits(); System.out.println(numTotalHits + " total matching documents"); int start = 0; int end = Math.min(numTotalHits, hitsPerPage); while (true) { if (end > hits.length) { System.out.println("Only results 1 - " + hits.length + " of " + numTotalHits + " total matching documents collected."); System.out.println("Collect more (y/n) ?"); String line = in.readLine(); if (line.length() == 0 || line.charAt(0) == 'n') { break; } collector = TopScoreDocCollector.create(numTotalHits, false); searcher.search(query, collector); hits = collector.topDocs().scoreDocs; } end = Math.min(hits.length, start + hitsPerPage); for (int i = start; i < end; i++) { if (raw) { // output raw format System.out.println("doc=" + hits[i].doc + " score=" + hits[i].score); continue; } Document doc = searcher.doc(hits[i].doc); String path = doc.get("path"); if (path != null) { System.out.println((i + 1) + ". " + path); String title = doc.get("title"); if (title != null) { System.out.println(" Title: " + doc.get("title")); } } else { System.out.println((i + 1) + ". " + "No path for this document"); } } if (!interactive) { break; } if (numTotalHits >= end) { boolean quit = false; while (true) { System.out.print("Press "); if (start - hitsPerPage >= 0) { System.out.print("(p)revious page, "); } if (start + hitsPerPage < numTotalHits) { System.out.print("(n)ext page, "); } System.out.println("(q)uit or enter number to jump to a page."); String line = in.readLine(); if (line.length() == 0 || line.charAt(0) == 'q') { quit = true; break; } if (line.charAt(0) == 'p') { start = Math.max(0, start - hitsPerPage); break; } else if (line.charAt(0) == 'n') { if (start + hitsPerPage < numTotalHits) { start += hitsPerPage; } break; } else { int page = Integer.parseInt(line); if ((page - 1) * hitsPerPage < numTotalHits) { start = (page - 1) * hitsPerPage; break; } else { System.out.println("No such page"); } } } if (quit) break; end = Math.min(numTotalHits, start + hitsPerPage); } } }
From source file:edu.rpi.tw.linkipedia.search.main.helper.ReadIndex.java
License:Open Source License
public static ArrayList<String> readIndexByTerm(IndexSearcher searcher, String fieldString, String termString, String filter) throws CorruptIndexException, IOException { //System.out.println(fieldString+" "+termString); TermQuery query = new TermQuery(new Term(fieldString, termString)); if (debug)//from w w w . j a v a 2 s. c o m System.out.println("your query |" + query.toString() + "|"); TopDocs topDocs = searcher.search(query, 1); if (topDocs == null) return new ArrayList<String>(); ScoreDoc[] hits = topDocs.scoreDocs; // System.out.println("matching: "+hits.length); ArrayList<String> contents = new ArrayList<String>(); for (int i = 0; i < hits.length; i++) { int docId = hits[i].doc; Document doc = searcher.doc(docId); List<IndexableField> fields = doc.getFields(); for (int j = 0; j < fields.size(); j++) { IndexableField field = fields.get(j); if (field.name().equals(filter)) { // System.out.println(filter+": ("+field.stringValue()+")"); contents.add(field.stringValue()); } // if(!field.name().equals("triple")) // System.out.println(field.name()+" "+field.stringValue()+" "+filter); /* if(field.name().equals("label")||field.name().equals("boost")||field.stringValue().startsWith(filter+"|")){ System.out.println(field.name()+": "+field.stringValue()); } */ } } return contents; /* Term term = new Term(fieldString, termString); TermDocs docs = reader.termDocs(term); if(docs.next()){ int docId = docs.doc(); Document doc = reader.document(docId); return doc; } */ }
From source file:edu.rpi.tw.linkipedia.search.main.helper.ReadIndex.java
License:Open Source License
public static void printIndexByTerm(IndexSearcher searcher, String fieldString, String termString, String filter) throws CorruptIndexException, IOException { TermQuery query = new TermQuery(new Term(fieldString, termString)); System.out.println("your query " + query.toString()); TopDocs topDocs = searcher.search(query, 1); ScoreDoc[] hits = topDocs.scoreDocs; //System.out.println("matching: "+hits.length); //ArrayList<String> contents = new ArrayList<String>(); for (int i = 0; i < hits.length; i++) { int docId = hits[i].doc; Document doc = searcher.doc(docId); List<IndexableField> fields = doc.getFields(); for (int j = 0; j < fields.size(); j++) { IndexableField field = fields.get(j); if (field.name().equals(filter)) { System.out.println(filter + ": (" + field.stringValue() + ")"); //contents.add(field.stringValue()); }//from w w w . jav a 2 s. c om //System.out.println(field.name()+" "+field.stringValue()); /* if(field.name().equals("label")||field.name().equals("boost")||field.stringValue().startsWith(filter+"|")){ System.out.println(field.name()+": "+field.stringValue()); } */ } } //return contents; /* Term term = new Term(fieldString, termString); TermDocs docs = reader.termDocs(term); if(docs.next()){ int docId = docs.doc(); Document doc = reader.document(docId); return doc; } */ }