List of usage examples for org.apache.lucene.index IndexWriterConfig setMergePolicy
@Override
public IndexWriterConfig setMergePolicy(MergePolicy mergePolicy)
From source file:dk.defxws.fgslucene.IndexWriterCache.java
License:Open Source License
/** * get IndexWriter for given indexPath and write it into cache. * /* w ww. java2 s . com*/ * @param indexName * name of index to open. * @param config * gsearch config-Object. * @throws GenericSearchException * e */ private IndexWriter getIndexWriter(final String indexName, final boolean create, final Config config) throws GenericSearchException { if (indexWriters.get(indexName) == null) { IndexWriter iw = null; try { IndexWriterConfig indexWriterConfig = new IndexWriterConfig(Constants.LUCENE_VERSION, getAnalyzer(config.getAnalyzer(indexName))); if (create) { indexWriterConfig.setOpenMode(OpenMode.CREATE); } else { indexWriterConfig.setOpenMode(OpenMode.CREATE_OR_APPEND); } if (config.getMaxBufferedDocs(indexName) > 1) { indexWriterConfig.setMaxBufferedDocs(config.getMaxBufferedDocs(indexName)); } if (config.getRamBufferSizeMb(indexName) > 1) { indexWriterConfig.setRAMBufferSizeMB(config.getRamBufferSizeMb(indexName)); } if (config.getMergeFactor(indexName) > 1 || config.getMaxMergeDocs(indexName) > 1 || config.getMaxMergeMb(indexName) > 1) { LogByteSizeMergePolicy logMergePolicy = new LogByteSizeMergePolicy(); if (config.getMergeFactor(indexName) > 1) { logMergePolicy.setMergeFactor(config.getMergeFactor(indexName)); } if (config.getMaxMergeDocs(indexName) > 1) { logMergePolicy.setMaxMergeDocs(config.getMaxMergeDocs(indexName)); } if (config.getMaxMergeMb(indexName) > 1) { logMergePolicy.setMaxMergeMB(config.getMaxMergeMb(indexName)); } indexWriterConfig.setMergePolicy(logMergePolicy); } if (config.getDefaultWriteLockTimeout(indexName) > 1) { indexWriterConfig.setWriteLockTimeout(config.getDefaultWriteLockTimeout(indexName)); } if (config.getLuceneDirectoryImplementation(indexName) != null) { // Initialize IndexWriter with configured FSDirectory FSDirectory directory = getDirectoryImplementation( config.getLuceneDirectoryImplementation(indexName), new File(config.getIndexDir(indexName))); iw = new IndexWriter(directory, indexWriterConfig); } else { // Initialize IndexWriter with default FSDirectory iw = new IndexWriter(FSDirectory.open(new File(config.getIndexDir(indexName))), indexWriterConfig); } if (config.getMaxChunkSize(indexName) > 1) { if (iw.getDirectory() instanceof MMapDirectory) { ((MMapDirectory) iw.getDirectory()).setMaxChunkSize(config.getMaxChunkSize(indexName)); } } } catch (Exception e) { iw = null; throw new GenericSearchException( "IndexWriter new error, creating index indexName=" + indexName + " :\n", e); } indexWriters.put(indexName, iw); if (logger.isDebugEnabled()) logger.debug("getIndexWriter put to map " + iw); return iw; } return indexWriters.get(indexName); }
From source file:dk.defxws.fgslucene.OperationsImpl.java
License:Open Source License
private void getIndexWriter(String indexName) throws GenericSearchException { if (iw == null) { Directory dir;/*w w w .j a va2s.c o m*/ try { dir = new SimpleFSDirectory(new File(config.getIndexDir(indexName))); } catch (Exception e) { throw new GenericSearchException("IndexWriter new error indexName=" + indexName + " :\n", e); } IndexWriterConfig iwconfig = new IndexWriterConfig(Version.LUCENE_36, getQueryAnalyzer(indexName)); int maxBufferedDocs = config.getMaxBufferedDocs(indexName); if (maxBufferedDocs > 0) { iwconfig.setMaxBufferedDocs(maxBufferedDocs); } int mergeFactor = config.getMergeFactor(indexName); if (mergeFactor > 0) { LogDocMergePolicy ldmp = new LogDocMergePolicy(); ldmp.setMergeFactor(mergeFactor); iwconfig.setMergePolicy(ldmp); } long defaultWriteLockTimeout = config.getDefaultWriteLockTimeout(indexName); if (defaultWriteLockTimeout > 0) { IndexWriterConfig.setDefaultWriteLockTimeout(defaultWriteLockTimeout); } try { iw = new IndexWriter(dir, iwconfig); } catch (Exception e) { throw new GenericSearchException("IndexWriter new error indexName=" + indexName + " :\n", e); } } try { docCount = iw.numDocs(); } catch (Exception e) { closeIndexWriter(indexName); throw new GenericSearchException("IndexWriter numDocs error indexName=" + indexName + " :\n", e); } if (logger.isDebugEnabled()) logger.debug("getIndexWriter indexName=" + indexName + " docCount=" + docCount); }
From source file:edu.cmu.geolocator.io.GetWriter.java
License:Apache License
public static IndexWriter getIndexWriter(String indexdirectory, double buffersize) throws IOException { Directory dir;//from www . jav a 2 s . com if (OSUtil.isWindows()) dir = FSDirectory.open(new File(indexdirectory)); else dir = NIOFSDirectory.open(new File(indexdirectory)); Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_45); IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_45, analyzer); config.setOpenMode(OpenMode.CREATE_OR_APPEND); config.setRAMBufferSizeMB(buffersize); LogDocMergePolicy mergePolicy = new LogDocMergePolicy(); mergePolicy.setMergeFactor(3); config.setMergePolicy(mergePolicy); IndexWriter writer = new IndexWriter(dir, config); return writer; }
From source file:indexer.LuceneIndexer.java
/** * Indexing the files. This method checks for the directories and then * finishes out after the indexing is complete. * @param global This is for reference to the global class variables * and methods.//from w ww. j a va 2 s. c o m * @param createIndex If true a new index will be created from scratch * and the old index will be destroyed. * @param indexPanel If true it will also print the console printout lines * to the main panel. */ public static void IndexFiles(Global global, Boolean createIndex) { String dataDir = global.dataDir; String indexDir = global.indexDir; //Verifies that the data directory exists if (dataDir == null) { System.err.println("Data Directory Is not accessable, Unable to Index files."); } //Verifies that the data directory is readable and writeable final Path docDir = Paths.get(dataDir); if (!Files.isReadable(docDir)) { System.out.println("Document directory '" + docDir.toAbsolutePath() + "' does not exist or is not readable, please check the path"); } startTime = new Date(); try { System.out.println("Indexing to directory '" + indexDir + "'..."); //Setups the analyzer Analyzer analyzer; try (Directory dir = FSDirectory.open(Paths.get(indexDir))) { analyzer = new StandardAnalyzer(); IndexWriterConfig iwc = new IndexWriterConfig(analyzer); if (createIndex) { // Create a new index in the directory, removing any // previously indexed documents: iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE); } else { // Add new documents to an existing index: iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); } iwc.setRAMBufferSizeMB(global.RAM_BUFFER_SIZE); iwc.setMaxBufferedDocs(global.MAX_BUFFERED_DOCS); LogDocMergePolicy ldmp = new LogDocMergePolicy(); ldmp.setMergeFactor(global.MERGE_FACTOR); iwc.setMergePolicy(ldmp); try (IndexWriter writer = new IndexWriter(dir, iwc)) { hm.clear(); indexDocs(writer, docDir, global); //This is a costly operation, we scheduled the time to apply it if (global.merge) { System.out.println("Starting Merge"); writer.forceMerge(1); global.merge = false; } writer.close(); } finishTime = new Date(); long millis = finishTime.getTime() - startTime.getTime(); totalTime = String.format("%02dhr %02dmin %02dsec", TimeUnit.MILLISECONDS.toHours(millis), TimeUnit.MILLISECONDS.toMinutes(millis) - TimeUnit.HOURS.toMinutes(TimeUnit.MILLISECONDS.toHours(millis)), // The change is in this line TimeUnit.MILLISECONDS.toSeconds(millis) - TimeUnit.MINUTES.toSeconds(TimeUnit.MILLISECONDS.toMinutes(millis))); System.out.println(""); System.out.println(""); System.out.println("Start Time: " + global.sdf.format(startTime.getTime())); System.out.println("Building List Time: " + listBuildTime); System.out.println("Indexing Time: " + indexingTime); System.out.println("Total Time: " + totalTime); System.out.println("Number of Documents: " + amountOfDocuments); System.out.println("Finish Time: " + global.sdf.format(finishTime.getTime())); System.out.println(""); } analyzer.close(); } catch (IOException e) { System.out.println(" caught a " + e.getClass() + "\n with message: " + e.getMessage()); log.fatal(" caught a " + e.getClass() + "\n with message: " + e.getMessage()); } }
From source file:io.druid.extension.lucene.LuceneDruidSegment.java
License:Apache License
private static IndexWriter buildRamWriter(RAMDirectory dir, Analyzer analyzer, int maxDocsPerSegment) throws IOException { IndexWriterConfig writerConfig = new IndexWriterConfig(analyzer); writerConfig.setOpenMode(OpenMode.CREATE_OR_APPEND); // some arbitrary large numbers writerConfig.setMaxBufferedDocs(maxDocsPerSegment * 2); writerConfig.setRAMBufferSizeMB(5000); writerConfig.setUseCompoundFile(false); writerConfig.setCommitOnClose(true); writerConfig.setIndexDeletionPolicy(NoDeletionPolicy.INSTANCE); writerConfig.setMergePolicy(NoMergePolicy.INSTANCE); writerConfig.setMergeScheduler(NoMergeScheduler.INSTANCE); return new IndexWriter(dir, writerConfig); }
From source file:io.druid.extension.lucene.LuceneDruidSegment.java
License:Apache License
private static IndexWriter buildPersistWriter(Directory dir) throws IOException { IndexWriterConfig writerConfig = new IndexWriterConfig(null); writerConfig.setUseCompoundFile(false); writerConfig.setOpenMode(OpenMode.CREATE_OR_APPEND); writerConfig.setMergePolicy(NoMergePolicy.INSTANCE); writerConfig.setMergeScheduler(NoMergeScheduler.INSTANCE); return new IndexWriter(dir, writerConfig); }
From source file:io.puntanegra.fhir.index.lucene.LuceneService.java
License:Apache License
/** * Builds a new {@link FSIndex}./* w w w. j a va2 s . c o m*/ * * @param name * the index name * @param mbeanName * the JMX MBean object name * @param path * the directory path * @param analyzer * the index writer analyzer * @param refresh * the index reader refresh frequency in seconds * @param ramBufferMB * the index writer RAM buffer size in MB * @param maxMergeMB * the directory max merge size in MB * @param maxCachedMB * the directory max cache size in MB * @param refreshTask * action to be done during refresh */ public void init(String name, String mbeanName, Path path, Analyzer analyzer, double refresh, int ramBufferMB, int maxMergeMB, int maxCachedMB, Runnable refreshTask) { try { this.path = path; this.name = name; // Open or create directory FSDirectory fsDirectory = FSDirectory.open(path); this.directory = new NRTCachingDirectory(fsDirectory, maxMergeMB, maxCachedMB); // Setup index writer IndexWriterConfig indexWriterConfig = new IndexWriterConfig(analyzer); indexWriterConfig.setRAMBufferSizeMB(ramBufferMB); indexWriterConfig.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); indexWriterConfig.setUseCompoundFile(true); indexWriterConfig.setMergePolicy(new TieredMergePolicy()); this.indexWriter = new IndexWriter(this.directory, indexWriterConfig); // Setup NRT search SearcherFactory searcherFactory = new SearcherFactory() { @Override public IndexSearcher newSearcher(IndexReader reader, IndexReader previousReader) { if (refreshTask != null) { refreshTask.run(); } IndexSearcher searcher = new IndexSearcher(reader); searcher.setSimilarity(new NoIDFSimilarity()); return searcher; } }; TrackingIndexWriter trackingWriter = new TrackingIndexWriter(this.indexWriter); this.searcherManager = new SearcherManager(this.indexWriter, true, searcherFactory); this.searcherReopener = new ControlledRealTimeReopenThread<>(trackingWriter, this.searcherManager, refresh, refresh); this.searcherReopener.start(); // Register JMX MBean // mbean = new ObjectName(mbeanName); // ManagementFactory.getPlatformMBeanServer().registerMBean(service, // this.mbean); } catch (Exception e) { throw new FhirIndexException(e, "Error while creating index %s", name); } }
From source file:net.ion.craken.node.problem.distribute.LuceneSettings.java
License:Open Source License
public static IndexWriter openWriter(Directory directory, int maxMergeDocs, boolean useSerialMerger) throws CorruptIndexException, LockObtainFailedException, IOException { IndexWriterConfig indexWriterConfig = new IndexWriterConfig(LUCENE_VERSION, analyzer); if (useSerialMerger) { indexWriterConfig.setMergeScheduler(mergeScheduler); }/*from w ww .j a v a2s .c o m*/ LogMergePolicy mergePolicy = new LogByteSizeMergePolicy(); mergePolicy.setMaxMergeDocs(maxMergeDocs); indexWriterConfig.setMergePolicy(mergePolicy); return new IndexWriter(directory, indexWriterConfig); }
From source file:net.sf.logsaw.index.internal.ARunWithIndexWriter.java
License:Open Source License
/** * Opens a Lucene index writer, executes the callback method and then closes the writer. * @param log the log resource, may be <code>null</code> * @param analyzer the Lucene analyzer to set on the index writer * @param matchVersion the Lucene match version * @return any object or <code>null</code> * @throws CoreException if an <strong>expected</strong> error occurred *//*from w ww . j a va 2s.com*/ protected final T runWithIndexWriter(ILogResource log, Analyzer analyzer, Version matchVersion) throws CoreException { logger.info("Opening index writer for '" + log.getName() + "'..."); //$NON-NLS-1$ //$NON-NLS-2$ IndexWriter writer = null; try { Directory dir = FSDirectory.open(IndexPlugin.getDefault().getIndexFile(log)); LogMergePolicy mp = new LogByteSizeMergePolicy(); mp.setMergeFactor(30); IndexWriterConfig cfg = new IndexWriterConfig(matchVersion, analyzer); cfg.setMaxBufferedDocs(1000); cfg.setMergePolicy(mp); writer = new IndexWriter(dir, cfg); try { return doRunWithIndexWriter(writer, log); } finally { logger.info("Closing index writer for '" + log.getName() + "'..."); //$NON-NLS-1$ //$NON-NLS-2$ writer.close(); } } catch (CoreException e) { // Rethrow original CoreException throw e; } catch (Exception e) { // Unexpected exception; wrap with CoreException throw new CoreException(new Status(IStatus.ERROR, IndexPlugin.PLUGIN_ID, NLS.bind(Messages.LuceneIndexService_error_failedToUpdateIndex, new Object[] { log.getName(), e.getLocalizedMessage() }), e)); } }
From source file:org.apache.jackrabbit.core.query.lucene.AbstractIndex.java
License:Apache License
/** * Returns an <code>IndexWriter</code> on this index. * @return an <code>IndexWriter</code> on this index. * @throws IOException if the writer cannot be obtained. *//* w w w . j a v a 2s . c o m*/ protected synchronized IndexWriter getIndexWriter() throws IOException { if (indexReader != null) { indexReader.close(); log.debug("closing IndexReader."); indexReader = null; } if (indexWriter == null) { IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_36, analyzer); config.setSimilarity(similarity); LogMergePolicy mergePolicy = new LogByteSizeMergePolicy(); mergePolicy.setUseCompoundFile(useCompoundFile); mergePolicy.setNoCFSRatio(1.0); config.setMergePolicy(mergePolicy); indexWriter = new IndexWriter(getDirectory(), config); indexWriter.setInfoStream(STREAM_LOGGER); } return indexWriter; }