List of usage examples for org.apache.lucene.index IndexWriterConfig setCommitOnClose
public IndexWriterConfig setCommitOnClose(boolean commitOnClose)
From source file:DocIndexer.java
License:Apache License
private RAMDirectory index() throws IOException, UnsupportedEncodingException, FileNotFoundException { RAMDirectory directory = new RAMDirectory(); IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer(CharArraySet.EMPTY_SET)); config.setOpenMode(OpenMode.CREATE); config.setCommitOnClose(true); try (IndexWriter iwriter = new IndexWriter(directory, config)) { for (String inputFile : inputFiles) { File file = new File(inputFile); if (file.length() == 0) { continue; }// www.jav a 2 s . com String title; try (BufferedReader titleReader = new BufferedReader( new InputStreamReader(new FileInputStream(file), "UTF-8"))) { title = titleReader.readLine(); if (title != null && title.startsWith("[[")) { // Generally the first line of the txt is the title. In a few cases the // first line is a "[[tag]]" and the second line is the title. title = titleReader.readLine(); } } Matcher matcher = SECTION_HEADER.matcher(title); if (matcher.matches()) { title = matcher.group(1); } String outputFile = AsciiDoctor.mapInFileToOutFile(inputFile, inExt, outExt); try (FileReader reader = new FileReader(file)) { Document doc = new Document(); doc.add(new TextField(Constants.DOC_FIELD, reader)); doc.add(new StringField(Constants.URL_FIELD, prefix + outputFile, Field.Store.YES)); doc.add(new TextField(Constants.TITLE_FIELD, title, Field.Store.YES)); iwriter.addDocument(doc); } } } return directory; }
From source file:com.spike.text.lucene.util.LuceneTestBookIndexingUtil.java
License:Apache License
public static void main(String[] args) throws IOException { String dataDir = LuceneAppConstants.BOOK_DATA_DIR; String indexDir = LuceneAppConstants.BOOK_INDEX_DIR; List<File> results = new ArrayList<File>(); findFiles(results, new File(dataDir)); System.out.println(results.size() + " books to index"); Directory directory = FSDirectory.open(Paths.get(indexDir)); IndexWriterConfig config = new IndexWriterConfig(new MyStandardAnalyzer()); config.setCommitOnClose(true); IndexWriter indexWriter = new IndexWriter(directory, config); for (File file : results) { Document document = getDocument(dataDir, file); indexWriter.addDocument(document); }//from w w w. ja v a 2 s. c om indexWriter.close(); directory.close(); }
From source file:io.druid.extension.lucene.LuceneDruidSegment.java
License:Apache License
private static IndexWriter buildRamWriter(RAMDirectory dir, Analyzer analyzer, int maxDocsPerSegment) throws IOException { IndexWriterConfig writerConfig = new IndexWriterConfig(analyzer); writerConfig.setOpenMode(OpenMode.CREATE_OR_APPEND); // some arbitrary large numbers writerConfig.setMaxBufferedDocs(maxDocsPerSegment * 2); writerConfig.setRAMBufferSizeMB(5000); writerConfig.setUseCompoundFile(false); writerConfig.setCommitOnClose(true); writerConfig.setIndexDeletionPolicy(NoDeletionPolicy.INSTANCE); writerConfig.setMergePolicy(NoMergePolicy.INSTANCE); writerConfig.setMergeScheduler(NoMergeScheduler.INSTANCE); return new IndexWriter(dir, writerConfig); }
From source file:lsre.utils.LuceneUtils.java
License:Open Source License
/** * Creates an IndexWriter for given index path, with given analyzer. * * @param directory the path to the index directory * @param create set to true if you want to create a new index * @param analyzer gives the analyzer used for the Indexwriter. * @return an IndexWriter/* ww w . j a v a 2 s. c o m*/ * @throws IOException */ public static IndexWriter createIndexWriter(Directory directory, boolean create, AnalyzerType analyzer) throws IOException { // set the analyzer according to the method params Analyzer tmpAnalyzer = null; if (analyzer == AnalyzerType.SimpleAnalyzer) tmpAnalyzer = new SimpleAnalyzer(); // LetterTokenizer with LowerCaseFilter else if (analyzer == AnalyzerType.WhitespaceAnalyzer) tmpAnalyzer = new WhitespaceAnalyzer(); // WhitespaceTokenizer else if (analyzer == AnalyzerType.KeywordAnalyzer) tmpAnalyzer = new KeywordAnalyzer(); // entire string as one token. else if (analyzer == AnalyzerType.StandardAnalyzer) tmpAnalyzer = new StandardAnalyzer(); // The config IndexWriterConfig config = new IndexWriterConfig(tmpAnalyzer); config.setRAMBufferSizeMB(512); config.setCommitOnClose(true); if (create) config.setOpenMode(IndexWriterConfig.OpenMode.CREATE); // overwrite if it exists. else config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); // create new if none is there, append otherwise. config.setCodec(new LsreCustomCodec()); return new IndexWriter(directory, config); }
From source file:net.semanticmetadata.lire.utils.LuceneUtils.java
License:Open Source License
/** * Creates an IndexWriter for given index path, with given analyzer. * * @param directory the path to the index directory * @param create set to true if you want to create a new index * @param analyzer gives the analyzer used for the Indexwriter. * @return an IndexWriter/*from w w w . j a v a2s . c o m*/ * @throws IOException */ public static IndexWriter createIndexWriter(Directory directory, boolean create, AnalyzerType analyzer) throws IOException { // set the analyzer according to the method params Analyzer tmpAnalyzer = null; if (analyzer == AnalyzerType.SimpleAnalyzer) tmpAnalyzer = new SimpleAnalyzer(); // LetterTokenizer with LowerCaseFilter else if (analyzer == AnalyzerType.WhitespaceAnalyzer) tmpAnalyzer = new WhitespaceAnalyzer(); // WhitespaceTokenizer else if (analyzer == AnalyzerType.KeywordAnalyzer) tmpAnalyzer = new KeywordAnalyzer(); // entire string as one token. else if (analyzer == AnalyzerType.StandardAnalyzer) tmpAnalyzer = new StandardAnalyzer(); // The config IndexWriterConfig config = new IndexWriterConfig(tmpAnalyzer); config.setRAMBufferSizeMB(512); config.setCommitOnClose(true); if (create) config.setOpenMode(IndexWriterConfig.OpenMode.CREATE); // overwrite if it exists. else config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); // create new if none is there, append otherwise. config.setCodec(new LireCustomCodec()); return new IndexWriter(directory, config); }
From source file:org.apereo.portal.index.PortalSearchIndexer.java
License:Apache License
/** Called by Quatrz. */ public void updateIndex() { if (!isEnabled()) { return;/*from ww w . j a v a 2 s . c o m*/ } final IndexWriterConfig indexWriterConfig = new IndexWriterConfig(new StandardAnalyzer()); indexWriterConfig.setCommitOnClose(true).setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); try (IndexWriter indexWriter = new IndexWriter(directory, indexWriterConfig)) { final List<IPortletDefinition> portlets = portletRegistry.getAllPortletDefinitions(); portlets.forEach(portlet -> indexPortlet(portlet, indexWriter)); } catch (Exception e) { logger.error("Unable to update index", e); } }
From source file:org.eclipse.dltk.internal.core.index.lucene.IndexContainer.java
License:Open Source License
private IndexWriter createWriter(Path path) throws IOException { Directory indexDir = new IndexDirectory(path, SimpleFSLockFactory.INSTANCE); purgeLocks(path);/*w ww. ja v a2 s .c om*/ IndexWriterConfig config = new IndexWriterConfig(new SimpleAnalyzer()); ConcurrentMergeScheduler mergeScheduler = new ConcurrentMergeScheduler(); mergeScheduler.setDefaultMaxMergesAndThreads(true); config.setMergeScheduler(mergeScheduler); config.setOpenMode(OpenMode.CREATE_OR_APPEND); config.setWriteLockTimeout(WRITE_LOCK_TIMEOUT); config.setCommitOnClose(false); return new IndexWriter(indexDir, config); }
From source file:org.elasticsearch.index.engine.InternalEngine.java
License:Apache License
private IndexWriter createWriter(boolean create) throws IOException { try {//from w w w . j a v a 2s .c o m final IndexWriterConfig iwc = new IndexWriterConfig(engineConfig.getAnalyzer()); iwc.setCommitOnClose(false); // we by default don't commit on close iwc.setOpenMode(create ? IndexWriterConfig.OpenMode.CREATE : IndexWriterConfig.OpenMode.APPEND); iwc.setIndexDeletionPolicy(deletionPolicy); // with tests.verbose, lucene sets this up: plumb to align with filesystem stream boolean verbose = false; try { verbose = Boolean.parseBoolean(System.getProperty("tests.verbose")); } catch (Throwable ignore) { } iwc.setInfoStream(verbose ? InfoStream.getDefault() : new LoggerInfoStream(logger)); iwc.setMergeScheduler(mergeScheduler); MergePolicy mergePolicy = config().getMergePolicy(); // Give us the opportunity to upgrade old segments while performing // background merges mergePolicy = new ElasticsearchMergePolicy(mergePolicy); iwc.setMergePolicy(mergePolicy); iwc.setSimilarity(engineConfig.getSimilarity()); iwc.setRAMBufferSizeMB(engineConfig.getIndexingBufferSize().mbFrac()); iwc.setCodec(engineConfig.getCodec()); /* We set this timeout to a highish value to work around * the default poll interval in the Lucene lock that is * 1000ms by default. We might need to poll multiple times * here but with 1s poll this is only executed twice at most * in combination with the default writelock timeout*/ iwc.setWriteLockTimeout(5000); iwc.setUseCompoundFile(this.engineConfig.isCompoundOnFlush()); // Warm-up hook for newly-merged segments. Warming up segments here is better since it will be performed at the end // of the merge operation and won't slow down _refresh iwc.setMergedSegmentWarmer(new IndexReaderWarmer() { @Override public void warm(LeafReader reader) throws IOException { try { LeafReader esLeafReader = new ElasticsearchLeafReader(reader, shardId); assert isMergedSegment(esLeafReader); if (warmer != null) { final Engine.Searcher searcher = new Searcher("warmer", searcherFactory.newSearcher(esLeafReader, null)); final IndicesWarmer.WarmerContext context = new IndicesWarmer.WarmerContext(shardId, searcher); warmer.warmNewReaders(context); } } catch (Throwable t) { // Don't fail a merge if the warm-up failed if (isClosed.get() == false) { logger.warn("Warm-up failed", t); } if (t instanceof Error) { // assertion/out-of-memory error, don't ignore those throw (Error) t; } } } }); return new IndexWriter(store.directory(), iwc); } catch (LockObtainFailedException ex) { boolean isLocked = IndexWriter.isLocked(store.directory()); logger.warn("Could not lock IndexWriter isLocked [{}]", ex, isLocked); throw ex; } }
From source file:org.modeshape.jcr.index.lucene.LuceneConfig.java
License:Apache License
private IndexWriterConfig newIndexWriterConfig() { IndexWriterConfig writerConfig = new IndexWriterConfig(analyzer); writerConfig.setCommitOnClose(true); writerConfig.setCodec(codec);//from ww w . j ava 2 s. c om return writerConfig; }
From source file:org.python.pydev.shared_core.index.IndexApi.java
License:Open Source License
public void init(boolean applyAllDeletes) throws IOException { this.analyzer = new CodeAnalyzer(); IndexWriterConfig config = new IndexWriterConfig(analyzer); config.setCommitOnClose(true); config.setOpenMode(OpenMode.CREATE_OR_APPEND); try {// ww w. j a v a 2 s. c om writer = new IndexWriter(this.indexDir, config); } catch (IOException e) { config.setOpenMode(OpenMode.CREATE); writer = new IndexWriter(this.indexDir, config); } searcherFactory = new SearcherFactory(); searchManager = new SearcherManager(writer, applyAllDeletes, searcherFactory); }