List of usage examples for org.apache.lucene.index IndexWriterConfig IndexWriterConfig
public IndexWriterConfig(Analyzer analyzer)
From source file:com.aurel.track.lucene.index.LuceneIndexer.java
License:Open Source License
/** * Initializes an IndexWriter./*from ww w . j a v a2 s . c om*/ * It will be called from the following places: * - on system startup workItemWriter should be initialized! * created = false if no reindex at startup * created = true if reindex at startup * - before explicit reIndexing: created = true * - after reindexing: create = false; * During the adding/editing/deleting of index data the IndexWriter should be initialized with created = false! * @param created * @param index */ public static IndexWriter initWriter(boolean created, int index) { Directory indexDirectory = LuceneUtil.getIndexDirectory(index); if (indexDirectory == null) { LOGGER.error("Can not find or create the index directory for workitems"); return null; } Analyzer analyzer = LuceneUtil.getAnalyzer(); if (analyzer == null) { LOGGER.error("Analyzer is null"); return null; } IndexWriter indexWriter = getIndexWriter(index); if (indexWriter != null) { try { //release the lock indexWriter.close(); } catch (IOException e) { LOGGER.error("Closing the IndexWriter for index " + index + " failed with " + e.getMessage()); LOGGER.debug(ExceptionUtils.getStackTrace(e)); } } try { IndexWriterConfig indexWriterConfig = new IndexWriterConfig(analyzer); OpenMode openMode = null; if (created) { openMode = OpenMode.CREATE; } else { openMode = OpenMode.APPEND; } indexWriterConfig.setOpenMode(openMode); indexWriter = new IndexWriter(indexDirectory, indexWriterConfig); } catch (OverlappingFileLockException e) { if (LOGGER.isDebugEnabled()) { LOGGER.debug("OverlappingFileLockException " + e.getMessage()); LOGGER.debug(ExceptionUtils.getStackTrace(e)); } if (!created) { //try again this time with created = true try { //open for create IndexWriterConfig indexWriterConfig = new IndexWriterConfig(analyzer); indexWriterConfig.setOpenMode(OpenMode.CREATE); //indexWriter = new IndexWriter(indexDirectory, analyzer, true, IndexWriter.MaxFieldLength.UNLIMITED); indexWriter = new IndexWriter(indexDirectory, indexWriterConfig); //close it in order to reopen it for modifications indexWriter.close(); } catch (IOException e1) { LOGGER.error("Creating the IndexWriter for index " + index + " failed with " + e.getMessage()); LOGGER.debug(ExceptionUtils.getStackTrace(e)); } //try again this time with created = false try { IndexWriterConfig indexWriterConfig = new IndexWriterConfig(analyzer); indexWriterConfig.setOpenMode(OpenMode.APPEND); //indexWriter = new IndexWriter(indexDirectory, analyzer, false, IndexWriter.MaxFieldLength.UNLIMITED); indexWriter = new IndexWriter(indexDirectory, indexWriterConfig); } catch (IOException e1) { LOGGER.error("Creating the IndexWriter for index " + index + " failed with " + e.getMessage()); LOGGER.debug(ExceptionUtils.getStackTrace(e)); } } else { LOGGER.error("Creating the IndexWriter for index " + index + " failed with " + e.getMessage()); LOGGER.debug(ExceptionUtils.getStackTrace(e)); } } catch (IOException e) { //tried probably with created = false, when the index structure doesn't exist yet //it is the case when by startup the useLucene is active but reindexOnStartup not //we should try to open the writers with false (for modifications) in order to not to destroy the possible existing index //but when the index doesn't exists yet the opening of the writer with false fails. And this is the case now. if (!created) { //try again this time with created = true try { //open for create IndexWriterConfig indexWriterConfig = new IndexWriterConfig(analyzer); indexWriterConfig.setOpenMode(OpenMode.CREATE); //indexWriter = new IndexWriter(indexDirectory, analyzer, true, IndexWriter.MaxFieldLength.UNLIMITED); indexWriter = new IndexWriter(indexDirectory, indexWriterConfig); //close it in order to reopen it for modifications indexWriter.close(); } catch (IOException e1) { LOGGER.error("Creating the IndexWriter for index " + index + " failed with " + e.getMessage()); LOGGER.debug(ExceptionUtils.getStackTrace(e)); } //try again this time with created = false try { IndexWriterConfig indexWriterConfig = new IndexWriterConfig(analyzer); indexWriterConfig.setOpenMode(OpenMode.APPEND); //indexWriter = new IndexWriter(indexDirectory, analyzer, false, IndexWriter.MaxFieldLength.UNLIMITED); indexWriter = new IndexWriter(indexDirectory, indexWriterConfig); } catch (IOException e1) { LOGGER.error("Creating the IndexWriter for index " + index + " failed with " + e.getMessage()); LOGGER.debug(ExceptionUtils.getStackTrace(e)); } } else { LOGGER.error("Creating the IndexWriter for index " + index + " failed with " + e.getMessage()); LOGGER.debug(ExceptionUtils.getStackTrace(e)); } } switch (index) { case LuceneUtil.INDEXES.WORKITEM_INDEX: workItemWriter = indexWriter; break; case LuceneUtil.INDEXES.NOT_LOCALIZED_LIST_INDEX: notLocalizedLookupWriter = indexWriter; break; case LuceneUtil.INDEXES.LOCALIZED_LIST_INDEX: localizedLookupWriter = indexWriter; break; case LuceneUtil.INDEXES.EXTERNAL_LOOKUP_WRITER: externalLookupWriter = indexWriter; break; case LuceneUtil.INDEXES.ATTACHMENT_INDEX: attachmentWriter = indexWriter; break; case LuceneUtil.INDEXES.EXPENSE_INDEX: expenseWriter = indexWriter; break; case LuceneUtil.INDEXES.BUDGET_PLAN_INDEX: budgetPlanWriter = indexWriter; break; case LuceneUtil.INDEXES.LINK_INDEX: linkWriter = indexWriter; break; default: return null; } return indexWriter; }
From source file:com.b2international.index.compat.SingleDirectoryIndexImpl.java
License:Apache License
protected void initLucene(final File indexDirectory, final boolean clean) { try {//from www . j ava2s . co m this.directory = Directories.openFile(indexDirectory.toPath()); final Analyzer analyzer = new ComponentTermAnalyzer(); final IndexWriterConfig config = new IndexWriterConfig(analyzer); config.setOpenMode(clean ? OpenMode.CREATE : OpenMode.CREATE_OR_APPEND); config.setIndexDeletionPolicy(new SnapshotDeletionPolicy(config.getIndexDeletionPolicy())); this.writer = new IndexWriter(directory, config); this.writer.commit(); // Create index if it didn't exist this.manager = new SearcherManager(directory, new SearchWarmerFactory()); } catch (final IOException e) { throw new RuntimeException(e.getMessage(), e); } }
From source file:com.b2international.index.lucene.NullIndexSearcher.java
License:Apache License
private static DirectoryReader createRamReader() throws IOException { final RAMDirectory directory = new RAMDirectory(); if (!DirectoryReader.indexExists(directory)) { final IndexWriterConfig conf = new IndexWriterConfig(new WhitespaceAnalyzer()); final IndexWriter writer = new IndexWriter(directory, conf); writer.commit();/* w ww. j ava2 s. co m*/ writer.close(); } return DirectoryReader.open(directory); }
From source file:com.bala.learning.learning.luence.IndexFiles.java
License:Apache License
/** Index all text files under a directory. */ public static void main(String[] args) { String usage = "java org.apache.lucene.demo.IndexFiles" + " [-index INDEX_PATH] [-docs DOCS_PATH] [-update]\n\n" + "This indexes the documents in DOCS_PATH, creating a Lucene index" + "in INDEX_PATH that can be searched with SearchFiles"; String indexPath = "index"; String docsPath = null;/* w ww. j a v a2 s. c o m*/ boolean create = true; for (int i = 0; i < args.length; i++) { if ("-index".equals(args[i])) { indexPath = args[i + 1]; i++; } else if ("-docs".equals(args[i])) { docsPath = args[i + 1]; i++; } else if ("-update".equals(args[i])) { create = false; } } if (docsPath == null) { System.err.println("Usage: " + usage); System.exit(1); } final Path docDir = Paths.get(docsPath); if (!Files.isReadable(docDir)) { System.out.println("Document directory '" + docDir.toAbsolutePath() + "' does not exist or is not readable, please check the path"); System.exit(1); } Date start = new Date(); try { System.out.println("Indexing to directory '" + indexPath + "'..."); Directory dir = FSDirectory.open(Paths.get(indexPath)); Analyzer analyzer = new StandardAnalyzer(); IndexWriterConfig iwc = new IndexWriterConfig(analyzer); if (create) { // Create a new index in the directory, removing any // previously indexed documents: iwc.setOpenMode(OpenMode.CREATE); } else { // Add new documents to an existing index: iwc.setOpenMode(OpenMode.CREATE_OR_APPEND); } // Optional: for better indexing performance, if you // are indexing many documents, increase the RAM // buffer. But if you do this, increase the max heap // size to the JVM (eg add -Xmx512m or -Xmx1g): // // iwc.setRAMBufferSizeMB(256.0); IndexWriter writer = new IndexWriter(dir, iwc); indexDocs(writer, docDir); // NOTE: if you want to maximize search performance, // you can optionally call forceMerge here. This can be // a terribly costly operation, so generally it's only // worth it when your index is relatively static (ie // you're done adding documents to it): // // writer.forceMerge(1); writer.close(); Date end = new Date(); System.out.println(end.getTime() - start.getTime() + " total milliseconds"); } catch (IOException e) { System.out.println(" caught a " + e.getClass() + "\n with message: " + e.getMessage()); } }
From source file:com.basistech.lucene.tools.LuceneQueryToolTest.java
License:Apache License
@BeforeClass public static void oneTimeSetup() throws IOException, ParseException { LuceneQueryToolTest.showOutput = false; // for debugging tests Directory dir = new RAMDirectory(); Analyzer analyzer = new StandardAnalyzer(); IndexWriterConfig config = new IndexWriterConfig(analyzer); IndexWriter writer = new IndexWriter(dir, config); Document doc = new Document(); doc.add(new Field("longest-mention", "Bill Clinton", StringField.TYPE_STORED)); doc.add(new Field("context", "Hillary Clinton Arkansas", TextField.TYPE_NOT_STORED)); writer.addDocument(doc);//from w ww . j a v a2 s . c o m doc = new Document(); doc.add(new Field("longest-mention", "George W. Bush", StringField.TYPE_STORED)); doc.add(new Field("context", "Texas Laura Bush", TextField.TYPE_NOT_STORED)); writer.addDocument(doc); doc = new Document(); doc.add(new Field("longest-mention", "George H. W. Bush", StringField.TYPE_STORED)); doc.add(new Field("context", "Barbara Bush Texas", TextField.TYPE_NOT_STORED)); writer.addDocument(doc); doc = new Document(); doc.add(new Field("bbb", "foo", StringField.TYPE_STORED)); doc.add(new Field("bbb", "bar", StringField.TYPE_STORED)); doc.add(new Field("aaa", "foo", StringField.TYPE_STORED)); FieldType typeUnindexed = new FieldType(StringField.TYPE_STORED); typeUnindexed.setIndexOptions(IndexOptions.NONE); doc.add(new Field("zzz", "foo", typeUnindexed)); writer.addDocument(doc); writer.close(); reader = DirectoryReader.open(dir); }
From source file:com.basistech.lucene.tools.LuceneQueryToolTest.java
License:Apache License
@Test public void testBinaryField() throws IOException, ParseException { Directory dir = new RAMDirectory(); Analyzer analyzer = new StandardAnalyzer(); IndexWriterConfig config = new IndexWriterConfig(analyzer); IndexWriter writer = new IndexWriter(dir, config); Document doc = new Document(); doc.add(new Field("id", "1", StringField.TYPE_STORED)); doc.add(new Field("binary-field", "ABC".getBytes(Charsets.UTF_8), StoredField.TYPE)); writer.addDocument(doc);/*from w w w .j a v a 2s . c o m*/ writer.close(); reader = DirectoryReader.open(dir); ByteArrayOutputStream bytes = new ByteArrayOutputStream(); PrintStream out = new PrintStream(bytes); LuceneQueryTool lqt = new LuceneQueryTool(reader, out); lqt.run(new String[] { "id:1" }); String result = Joiner.on('\n').join(getOutput(bytes)); assertTrue(result.contains("0x414243")); // binary rep of "ABC" }
From source file:com.bdaum.zoom.lal.internal.LireActivator.java
License:Open Source License
public IndexWriter createIndexWriter(Directory dir, boolean create) throws IOException, LockObtainFailedException { IndexWriterConfig config = new IndexWriterConfig(getLuceneAnalyzer()); config.setOpenMode(create ? IndexWriterConfig.OpenMode.CREATE : IndexWriterConfig.OpenMode.APPEND); return new IndexWriter(dir, config); }
From source file:com.bluedragon.search.collection.Collection.java
License:Open Source License
/** * Creates an empty collection to get it up and running */// ww w . j a va2s. com public synchronized void create(boolean _errorOnExists) throws IOException { setDirectory(); if (directory.listAll().length > 2) { if (_errorOnExists) { throw new IOException("directory not empty; possible collection already present"); } else { if (DirectoryReader.indexExists(directory)) { return; } // otherwise an index doesn't exist so allow the creation code to execute } } IndexWriterConfig iwc = new IndexWriterConfig(AnalyzerFactory.get(language)); iwc.setOpenMode(OpenMode.CREATE); indexwriter = new IndexWriter(directory, iwc); indexwriter.commit(); indexwriter.close(); indexwriter = null; // throw an openbd.create file in there so we know when it was created created = System.currentTimeMillis(); File touchFile = new File(collectionpath, "openbd.created"); Writer fw = new FileWriter(touchFile); fw.close(); }
From source file:com.bluedragon.search.collection.Collection.java
License:Open Source License
private void setIndexWriter() throws IOException { if (indexwriter != null) return;/* w w w .j a va 2s . com*/ setDirectory(); IndexWriterConfig iwc = new IndexWriterConfig(AnalyzerFactory.get(language)); iwc.setOpenMode(OpenMode.CREATE_OR_APPEND); indexwriter = new IndexWriter(directory, iwc); }
From source file:com.chenyi.langeasy.lucene.IndexFiles.java
License:Apache License
/** Index all text files under a directory. */ public static void index(String docsPath) { // String indexPath = "index"; String indexPath = "F:/Personal/ws_indigo/lucene/index"; // String docsPath = null; // String docsPath = "E:/langeasy/lucene/podcast"; // boolean create = true; // String docsPath = "E:/langeasy/lucene/tv"; // String docsPath = "E:/langeasy/lucene/srt"; // String docsPath = "E:/langeasy/lucene/podcast/freshair/transcript"; // String docsPath = "E:/langeasy/lucene/podcast/money/transcript"; // String docsPath = // "E:/langeasy/lucene/podcast/allthingsconsidered/transcript"; // String docsPath = // "E:/langeasy/lucene/podcast/morningedition/transcript"; // String docsPath = // "E:/langeasy/lucene/podcast/wait-wait-dont-tell-me/transcript"; // String docsPath = // "E:/langeasy/lucene/podcast/invisibilia/transcript"; // String docsPath = // "E:/langeasy/lucene/podcast/hidden-brain/transcript"; // String docsPath = // "E:/langeasy/lucene/podcast/weekend-edition-saturday/transcript"; // String docsPath = // "E:/langeasy/lucene/podcast/weekend-edition-sunday/transcript"; // String docsPath = "E:/langeasy/lucene/podcast/politics/transcript"; // String docsPath = // "E:/langeasy/lucene/podcast/ask-me-another/transcript"; // String docsPath = "E:/langeasy/lucene/podcast/ted-talks/transcript"; // String docsPath = // "E:/langeasy/lucene/podcast/freakonomics/transcript"; // String docsPath = "E:/langeasy/lucene/podcast/serial/transcript"; // String docsPath = "E:/langeasy/lucene/podcast/ted-ed/transcript"; // String docsPath = "E:/langeasy/lucene/podcast/yale-courses/transcript"; // String docsPath = "E:/langeasy/lucene/youtube/nasa/caption"; // String docsPath = "E:/langeasy/lucene/youtube/movieclipsTRAILERS/caption"; // String docsPath = "E:/langeasy/lucene/youtube/Vsauce/caption"; // String docsPath = "E:/langeasy/lucene/youtube/vice/caption"; // String docsPath = "E:/langeasy/lucene/youtube/DiscoveryNetworks/caption"; // String docsPath = "E:/langeasy/lucene/youtube/collegehumor/caption"; // String docsPath = "E:/langeasy/lucene/youtube/AnimalPlanetTV/caption"; // String docsPath = "E:/langeasy/lucene/youtube/AsapSCIENCE/caption"; // String docsPath = "E:/langeasy/lucene/youtube/latenight/caption"; // String docsPath = "E:/langeasy/lucene/youtube/rhettandlink2/caption"; // String docsPath = "E:/langeasy/lucene/youtube/TheEllenShow/caption"; // String docsPath = "E:/langeasy/lucene/youtube/zoella280390/caption"; // String docsPath = "E:/langeasy/lucene/youtube/cnn-breaking-news/caption"; // String docsPath = "E:/langeasy/lucene/youtube/TEDxTalks"; // String docsPath = "E:/langeasy/lucene/youtube/spotlight"; // String docsPath = "E:/langeasy/lucene/youtube/Howcast"; boolean create = false; final Path docDir = Paths.get(docsPath); if (!Files.isReadable(docDir)) { System.out.println("Document directory '" + docDir.toAbsolutePath() + "' does not exist or is not readable, please check the path"); System.exit(1);/*from w ww.j a va 2s. c o m*/ } Date start = new Date(); try { System.out.println("Indexing to directory '" + indexPath + "'..."); Directory dir = FSDirectory.open(Paths.get(indexPath)); Analyzer analyzer = new StandardAnalyzer(); IndexWriterConfig iwc = new IndexWriterConfig(analyzer); if (create) { // Create a new index in the directory, removing any // previously indexed documents: iwc.setOpenMode(OpenMode.CREATE); } else { // Add new documents to an existing index: iwc.setOpenMode(OpenMode.CREATE_OR_APPEND); } // Optional: for better indexing performance, if you // are indexing many documents, increase the RAM // buffer. But if you do this, increase the max heap // size to the JVM (eg add -Xmx512m or -Xmx1g): // // iwc.setRAMBufferSizeMB(256.0); IndexWriter writer = new IndexWriter(dir, iwc); indexDocs(writer, docDir); // NOTE: if you want to maximize search performance, // you can optionally call forceMerge here. This can be // a terribly costly operation, so generally it's only // worth it when your index is relatively static (ie // you're done adding documents to it): // // writer.forceMerge(1); writer.close(); Date end = new Date(); System.out.println(end.getTime() - start.getTime() + " total milliseconds"); } catch (IOException e) { System.out.println(" caught a " + e.getClass() + "\n with message: " + e.getMessage()); } }