List of usage examples for org.apache.lucene.index IndexWriter close
@Override public void close() throws IOException
From source file:com.nokia.vikram.lire.LireIndexer.java
License:Open Source License
public static void indexImages(String inputFramesRootDir, String indexDirectory) { try {// w ww . j a va2 s.co m // Getting all images from a directory and its sub directories. ArrayList<String> images = FileUtils.getAllImages(new File(inputFramesRootDir), true); // Creating a CEDD document builder and indexing all files. DocumentBuilder builder = DocumentBuilderFactory.getCEDDDocumentBuilder(); // Creating an Lucene IndexWriter IndexWriterConfig conf = new IndexWriterConfig(LuceneUtils.LUCENE_VERSION, new WhitespaceAnalyzer(LuceneUtils.LUCENE_VERSION)); conf.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); IndexWriter iw = new IndexWriter(FSDirectory.open(new File(indexDirectory)), conf); // Iterating through images building the low level features for (Iterator<String> it = images.iterator(); it.hasNext();) { String imageFilePath = it.next(); System.out.println("Indexing image: " + imageFilePath); try { BufferedImage img = ImageIO.read(new FileInputStream(imageFilePath)); Document document = builder.createDocument(img, imageFilePath); iw.addDocument(document); } catch (Exception e) { System.err.println("ERR: Could not read image or index it!"); e.printStackTrace(); } } // Close the IndexWriter iw.close(); System.out.println("Indexing completed."); } catch (IOException e) { System.out.println("ERR: IOException while attempting to build index!"); e.printStackTrace(); } }
From source file:com.nuvolect.deepdive.lucene.Index.java
public static JSONObject index(final String volumeId, final String searchPath, final boolean forceIndex) { if (m_interrupt[0]) { LogUtil.log(LogUtil.LogType.INDEX, "Index canceled post interrupt"); m_interrupt[0] = false;/*from ww w .j a v a 2 s. c o m*/ return responseInterruptIndexing(); } OmniFile cacheDir = IndexUtil.getCacheDir(volumeId, searchPath); boolean cacheDirCreated = false; try { cacheDirCreated = OmniUtil.forceMkdir(cacheDir); } catch (IOException e) { return responseFolderCreateError(searchPath); } final String luceneDirPath = cacheDir.getAbsolutePath(); boolean cacheDirExists = !cacheDirCreated; boolean indexingOngoing = m_indexThread != null && m_indexThread.isAlive(); boolean indexingRequired = !cacheDirExists || forceIndex; synchronized (m_lock) { if (indexingOngoing) { if (m_fileTreeActive) m_index_state = INDEX_STATE.filetree; else m_index_state = INDEX_STATE.indexing; } else { if (indexingRequired) m_index_state = INDEX_STATE.indexing; else m_index_state = INDEX_STATE.complete; } } if (indexingRequired || indexingOngoing) { if (indexingOngoing) { // Nothing to do, let the background process run. Monitor m_indexedDocs for progress. } else { synchronized (m_lock) { m_index_state = INDEX_STATE.filetree; m_totalDocs[0] = 0; m_indexedDocs[0] = 0; m_error[0] = ""; } m_threadGroup = new ThreadGroup(INDEX_THREAD_GROUP); m_indexThread = new Thread(m_threadGroup, new Runnable() { @Override public void run() { // Analyzer analyzer = new org.apache.lucene.analysis.core.WhitespaceAnalyzer(); // Analyzer analyzer = new org.apache.lucene.analysis.core.KeywordAnalyzer(); // Analyzer analyzer = new org.apache.lucene.analysis.standard.StandardAnalyzer(); Analyzer analyzer = new org.apache.lucene.analysis.core.SimpleAnalyzer(); IndexWriterConfig config = new IndexWriterConfig(analyzer); IndexWriter iwriter = null; try { Directory m_directory = FSDirectory.open(Paths.get(luceneDirPath)); iwriter = new IndexWriter(m_directory, config); iwriter.deleteAll(); iwriter.commit(); } catch (IOException e) { LogUtil.logException(LogUtil.LogType.INDEX, e); m_error[0] = "IndexWriter constructor exception"; } synchronized (m_lock) { m_fileTreeActive = true; m_index_state = INDEX_STATE.filetree; } Collection<OmniFile> files = IndexUtil.getFilePaths(volumeId, searchPath); synchronized (m_lock) { m_index_state = INDEX_STATE.indexing; m_fileTreeActive = false; m_totalDocs[0] = files.size(); m_indexedDocs[0] = 0; } try { for (OmniFile file : files) { if (m_interrupt[0]) { LogUtil.log(LogUtil.LogType.INDEX, "Iterator loop canceled"); break; } String path = file.getPath(); // LogUtil.log(LogUtil.LogType.INDEX, "indexing: " + path);// this is a bit excessive iwriter.addDocument(makeDoc(volumeId, path)); synchronized (m_lock) { ++m_indexedDocs[0]; } } iwriter.commit(); iwriter.close(); synchronized (m_lock) { m_index_state = m_interrupt[0] ? INDEX_STATE.interrupted : INDEX_STATE.complete; m_totalDocs[0] = m_indexedDocs[0]; } } catch (Exception e) { LogUtil.logException(LogUtil.LogType.INDEX, e); m_error[0] = "IndexWriter addDocument exception"; } } }, INDEX_THREAD, STACK_SIZE); m_indexThread.setPriority(Thread.MAX_PRIORITY); m_indexThread.start(); } } else { // Indexing is complete // Get number of documents indexed try { Directory directory = FSDirectory.open(Paths.get(luceneDirPath)); DirectoryReader ireader = DirectoryReader.open(directory); synchronized (m_lock) { m_indexedDocs[0] = ireader.numDocs(); m_totalDocs[0] = m_indexedDocs[0]; m_index_state = INDEX_STATE.complete; } ireader.close(); directory.close(); } catch (IOException e) { LogUtil.logException(LogUtil.LogType.INDEX, e); } } JSONObject result = new JSONObject(); try { synchronized (m_lock) { result.put("index_state", m_index_state.toString()); result.put("error", m_error[0]); result.put("indexed_docs", m_indexedDocs[0]); result.put("total_docs", m_totalDocs[0]); // result.put("full_path", cacheDir.getAbsolutePath()); result.put("search_path", searchPath); } } catch (JSONException e) { e.printStackTrace(); } return result; }
From source file:com.orientechnologies.lucene.test.LuceneNativeFacet.java
License:Apache License
/** Build the example index. */ private void index() throws IOException { IndexWriter indexWriter = new IndexWriter(indexDir, new IndexWriterConfig(Version.LUCENE_47, new WhitespaceAnalyzer(Version.LUCENE_47)) .setOpenMode(OpenMode.CREATE)); // Writes facet ords to a separate directory from the main index DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir); Document doc = new Document(); doc.add(new FacetField("Author", "Bob")); doc.add(new FacetField("Publish Date", "2010", "10", "15")); indexWriter.addDocument(config.build(taxoWriter, doc)); doc = new Document(); doc.add(new FacetField("Author", "Lisa")); doc.add(new FacetField("Publish Date", "2010", "10", "20")); indexWriter.addDocument(config.build(taxoWriter, doc)); doc = new Document(); doc.add(new FacetField("Author", "Lisa")); doc.add(new FacetField("Publish Date", "2012", "1", "1")); indexWriter.addDocument(config.build(taxoWriter, doc)); doc = new Document(); doc.add(new FacetField("Author", "Susan")); doc.add(new FacetField("Publish Date", "2012", "1", "7")); indexWriter.addDocument(config.build(taxoWriter, doc)); doc = new Document(); doc.add(new FacetField("Author", "Frank")); doc.add(new FacetField("Publish Date", "1999", "5", "5")); indexWriter.addDocument(config.build(taxoWriter, doc)); indexWriter.close(); taxoWriter.close();/*from w w w.j a va 2 s .c om*/ }
From source file:com.orientechnologies.spatial.sandbox.LuceneGeoTest.java
License:Apache License
@Test public void geoIntersectTest() throws IOException, ParseException { RecursivePrefixTreeStrategy strategy = new RecursivePrefixTreeStrategy( new GeohashPrefixTree(JtsSpatialContext.GEO, 11), "location"); strategy.setDistErrPct(0);/*from ww w . j a v a2s. c om*/ IndexWriterConfig conf = new IndexWriterConfig(new StandardAnalyzer()); final RAMDirectory directory = new RAMDirectory(); final IndexWriter writer = new IndexWriter(directory, conf); Shape point = JtsSpatialContext.GEO.getWktShapeParser().parse("POINT (9.4714708 47.6819432)"); Shape polygon = JtsSpatialContext.GEO.getWktShapeParser().parse( "POLYGON((9.481201171875 47.64885294675266,9.471416473388672 47.65128140482982,9.462661743164062 47.64781214443791,9.449443817138672 47.656947367880335,9.445838928222656 47.66110972448931,9.455795288085938 47.667352637215,9.469013214111328 47.67255449415724,9.477081298828125 47.679142768657066,9.490299224853516 47.678680460743834,9.506263732910156 47.679258344995326,9.51364517211914 47.68191653011071,9.518795013427734 47.677177931734406,9.526691436767578 47.679489496903706,9.53390121459961 47.67139857075435,9.50918197631836 47.66180341832901,9.50815200805664 47.6529003141482,9.51192855834961 47.64654002455372,9.504375457763672 47.64237650648966,9.49270248413086 47.649662445325035,9.48617935180664 47.65151268066222,9.481201171875 47.64885294675266))"); Document doc = new Document(); Assert.assertNotEquals(point.relate(polygon), SpatialRelation.INTERSECTS); for (IndexableField f : strategy.createIndexableFields(point)) { doc.add(f); } writer.addDocument(doc); writer.commit(); SpatialArgs args = new SpatialArgs(SpatialOperation.Intersects, polygon.getBoundingBox()); Filter filter = strategy.makeFilter(args); IndexReader reader = DirectoryReader.open(directory); IndexSearcher searcher = new IndexSearcher(reader); TopDocs search = searcher.search(new MatchAllDocsQuery(), filter, 1000); Assert.assertEquals(search.totalHits, 0); reader.close(); writer.close(); }
From source file:com.orientechnologies.spatial.sandbox.LuceneGeoTest.java
License:Apache License
@Test public void geoSpeedTest() throws IOException, ParseException { RecursivePrefixTreeStrategy strategy = new RecursivePrefixTreeStrategy( new GeohashPrefixTree(JtsSpatialContext.GEO, 11), "location"); IndexWriterConfig conf = new IndexWriterConfig(new StandardAnalyzer()); final RAMDirectory directory = new RAMDirectory(); final IndexWriter writer = new IndexWriter(directory, conf); Shape multiPolygon = JtsSpatialContext.GEO.getWktShapeParser().parse( "MULTIPOLYGON (((15.520376 38.231155, 15.160243 37.444046, 15.309898 37.134219, 15.099988 36.619987, 14.335229 36.996631, 13.826733 37.104531, 12.431004 37.61295, 12.570944 38.126381, 13.741156 38.034966, 14.761249 38.143874, 15.520376 38.231155)), ((9.210012 41.209991, 9.809975 40.500009, 9.669519 39.177376, 9.214818 39.240473, 8.806936 38.906618, 8.428302 39.171847, 8.388253 40.378311, 8.159998 40.950007, 8.709991 40.899984, 9.210012 41.209991)), ((12.376485 46.767559, 13.806475 46.509306, 13.69811 46.016778, 13.93763 45.591016, 13.141606 45.736692, 12.328581 45.381778, 12.383875 44.885374, 12.261453 44.600482, 12.589237 44.091366, 13.526906 43.587727, 14.029821 42.761008, 15.14257 41.95514, 15.926191 41.961315, 16.169897 41.740295, 15.889346 41.541082, 16.785002 41.179606, 17.519169 40.877143, 18.376687 40.355625, 18.480247 40.168866, 18.293385 39.810774, 17.73838 40.277671, 16.869596 40.442235, 16.448743 39.795401, 17.17149 39.4247, 17.052841 38.902871, 16.635088 38.843572, 16.100961 37.985899, 15.684087 37.908849, 15.687963 38.214593, 15.891981 38.750942, 16.109332 38.964547, 15.718814 39.544072, 15.413613 40.048357, 14.998496 40.172949, 14.703268 40.60455, 14.060672 40.786348, 13.627985 41.188287, 12.888082 41.25309, 12.106683 41.704535, 11.191906 42.355425, 10.511948 42.931463, 10.200029 43.920007, 9.702488 44.036279, 8.888946 44.366336, 8.428561 44.231228, 7.850767 43.767148, 7.435185 43.693845, 7.549596 44.127901, 7.007562 44.254767, 6.749955 45.028518, 7.096652 45.333099, 6.802355 45.70858, 6.843593 45.991147, 7.273851 45.776948, 7.755992 45.82449, 8.31663 46.163642, 8.489952 46.005151, 8.966306 46.036932, 9.182882 46.440215, 9.922837 46.314899, 10.363378 46.483571, 10.442701 46.893546, 11.048556 46.751359, 11.164828 46.941579, 12.153088 47.115393, 12.376485 46.767559)))"); Document doc = new Document(); for (IndexableField f : strategy.createIndexableFields(multiPolygon)) { doc.add(f);/*from ww w . j ava 2 s . c o m*/ } writer.addDocument(doc); writer.commit(); writer.close(); }
From source file:com.orientechnologies.spatial.sandbox.LuceneGeoTest.java
License:Apache License
@Test public void geoSpeedTestInternal() throws IOException, ParseException { RecursivePrefixTreeStrategy strategy = new RecursivePrefixTreeStrategy( new GeohashPrefixTree(JtsSpatialContext.GEO, 11), "location"); IndexWriterConfig conf = new IndexWriterConfig(new StandardAnalyzer()); final RAMDirectory directory = new RAMDirectory(); final IndexWriter writer = new IndexWriter(directory, conf); ODocument entries = loadMultiPolygon(); OMultiPolygonShapeBuilder builder = new OMultiPolygonShapeBuilder(); Shape multiPolygon = builder.fromDoc(entries); Document doc = new Document(); for (IndexableField f : strategy.createIndexableFields(multiPolygon)) { doc.add(f);// w ww . ja va2 s . com } writer.addDocument(doc); writer.commit(); writer.close(); }
From source file:com.paladin.common.LuceneHelper.java
License:Apache License
/** * /*from ww w . ja v a 2s . c om*/ * * @param tables * @throws IOException */ public static void index(final String[] tables, boolean _create) { // true ?? false ? Analyzer analyzer = new IKAnalyzer(false); for (String table : tables) { long begin = System.currentTimeMillis(); IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_33, analyzer); if (_create) iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE); else iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); final String index_dir = Constants.LUCENE_INDEX_ROOT + table; File dir = new File(index_dir); if (!dir.exists()) dir.mkdirs(); try { Directory directory = FSDirectory.open(dir); IndexWriter writer = new IndexWriter(directory, iwc); indexTable(writer, table); writer.close(); } catch (Exception e) { e.printStackTrace(); } log.info(" " + table + " " + (System.currentTimeMillis() - begin) + " milliseconds"); } }
From source file:com.paladin.sys.lucene.IndexFiles.java
License:Apache License
/** * Index all text files under a directory. *//*from www . ja va2 s . c o m*/ public static void main(String[] args) { args = new String[] { "-docs", "D:\\BJ\\ETLWorkspace\\ETL\\src", "-index", "D:\\myData\\luceneIdx" }; final String usage = "java org.apache.lucene.demo.IndexFiles" + " [-index INDEX_PATH] [-docs DOCS_PATH] [-update]\n\n" + "This indexes the documents in DOCS_PATH, creating a Lucene index" + "in INDEX_PATH that can be searched with SearchFiles"; String indexPath = "index"; String docsPath = null; boolean create = true; for (int i = 0; i < args.length; i++) { if ("-index".equals(args[i])) { indexPath = args[i + 1]; i++; } else if ("-docs".equals(args[i])) { docsPath = args[i + 1]; i++; } else if ("-update".equals(args[i])) { create = false; } } if (docsPath == null) { System.err.println("Usage: " + usage); System.exit(1); } final File docDir = new File(docsPath); if (!docDir.exists() || !docDir.canRead()) { out.println("Document directory '" + docDir.getAbsolutePath() + "' does not exist or is not readable, please check the path"); System.exit(1); } Date start = new Date(); try { out.println("Indexing to directory '" + indexPath + "'..."); Directory dir = FSDirectory.open(new File(indexPath)); Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_33); IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_33, analyzer); if (create) // Create a new index in the directory, removing any previously indexed documents: iwc.setOpenMode(OpenMode.CREATE); else // Add new documents to an existing index: iwc.setOpenMode(OpenMode.CREATE_OR_APPEND); // Optional: for better indexing performance, if you are indexing many documents, increase the RAM // buffer. But if you do this, increase the max heap size to the JVM (eg add -Xmx512m or -Xmx1g): // iwc.setRAMBufferSizeMB(256.0); IndexWriter writer = new IndexWriter(dir, iwc); indexDocs(writer, docDir); // NOTE: if you want to maximize search performance, you can optionally call optimize here. This can be // a costly operation, so generally it's only worth it when your index is relatively static (ie you're // done adding documents to it): // writer.optimize(); writer.close(); Date end = new Date(); out.println(end.getTime() - start.getTime() + " total milliseconds"); } catch (IOException e) { out.println(" caught a " + e.getClass() + "\n with message: " + e.getMessage()); } }
From source file:com.parc.uplib.indexing.UpLibQueryParser.java
License:Open Source License
private static void update(File index_file, File doc_root_dir, String[] ids, int start) { ExtractIndexingInfo.DocumentIterator docit; String number;//from w w w. ja v a 2 s.co m remove(index_file, ids, start); try { // Now add the documents to the index IndexWriter writer = new IndexWriter(index_file, new StandardAnalyzer(), !index_file.exists()); if (debug_mode) writer.setInfoStream(System.err); writer.setMaxFieldLength(Integer.MAX_VALUE); try { for (int i = start; i < ids.length; i++) { docit = build_document_iterator(doc_root_dir, ids[i]); int count = 0; while (docit.hasNext()) { writer.addDocument((Document) (docit.next())); count += 1; } System.out.println("Added " + docit.id + " (" + count + " versions)"); System.out.flush(); } } finally { // And close the index System.out.println("Optimizing..."); // See http://www.gossamer-threads.com/lists/lucene/java-dev/47895 about optimize // Can fail if low on disk space writer.optimize(); writer.close(); } } catch (Exception e) { if (debug_mode) { e.printStackTrace(System.err); } else { System.out.println( "* Lucene search engine raised " + e.getClass() + " with message " + e.getMessage()); System.err.println(" 'update' caught a " + e.getClass() + "\n with message: " + e.getMessage()); System.out.flush(); } System.exit(JAVA_EXCEPTION); } System.out.flush(); }
From source file:com.pongasoft.kiwidoc.index.impl.lucene.impl.FSDirectoryFactory.java
License:Apache License
/** * Opens a clean directory. Any data that was stored in the directory * previously is erased./*from w w w .j a v a 2 s . c om*/ * * @return an empty directory * @throws IOException */ private Directory openCleanDirectory() throws IOException { Directory directory = FSDirectory.getDirectory(_directoryPath); IndexWriter writer = new IndexWriter(directory, new StandardAnalyzer(), true, IndexWriter.MaxFieldLength.UNLIMITED); writer.close(); log.info("Created empty FS directory " + _directoryPath); return directory; }