List of usage examples for org.apache.lucene.index LogByteSizeMergePolicy setMaxMergeMB
public void setMaxMergeMB(double mb)
Determines the largest segment (measured by total byte size of the segment's files, in MB) that may be merged with other segments.
From source file:com.graphhopper.compare.misc.LuceneStorage.java
License:Apache License
public boolean init(boolean forceCreate) { try {/* ww w. j ava2 s . c om*/ File file = new File("osm.lucene.test"); if (forceCreate) Helper.deleteDir(file); // germany.osm => 3.6 GB on disc for nodes only, 1.5 GB memory usage at the end of the nodes Directory dir = FSDirectory.open(file); IndexWriterConfig cfg = new IndexWriterConfig(Version.LUCENE_35, new KeywordAnalyzer()); LogByteSizeMergePolicy mp = new LogByteSizeMergePolicy(); mp.setMaxMergeMB(3000); cfg.setRAMBufferSizeMB(128); cfg.setTermIndexInterval(512); cfg.setMergePolicy(mp); // specify different formats for id fields etc // -> this breaks 16 of our tests!? Lucene Bug? // cfg.setCodec(new Lucene40Codec() { // // @Override public PostingsFormat getPostingsFormatForField(String field) { // return new Pulsing40PostingsFormat(); // } // }); // cfg.setMaxThreadStates(8); boolean create = !IndexReader.indexExists(dir); cfg.setOpenMode(create ? IndexWriterConfig.OpenMode.CREATE : IndexWriterConfig.OpenMode.APPEND); writer = new IndexWriter(dir, cfg); return true; } catch (Exception ex) { logger.error("cannot init lucene storage", ex); return false; } }
From source file:com.tinkerpop.graph.benchmark.index.LuceneKeyToNodeIdIndexImpl.java
License:Apache License
public void init() { try {/*from www . j ava 2 s . c o m*/ hotCache = new LRUCache<String, Long>(getLruCacheSize()); if (showDebug) { System.out.println( "timeTaken,numDocs,bloomReadSaves,hotCacheHits,failedLuceneReads,successfulLuceneReads,luceneAdds"); } bloomFilter = new OpenBitSet(bloomFilterSize); deleteDirectoryContents(new File(path)); dir = FSDirectory.open(new File(path)); writer = new IndexWriter(dir, new WhitespaceAnalyzer(), MaxFieldLength.UNLIMITED); // LogByteSizeMergePolicy mp=new LogByteSizeMergePolicy(writer); LogByteSizeMergePolicy mp = new LogByteSizeMergePolicy(); mp.setMaxMergeMB(getMaxMergeMB()); writer.setMergePolicy(mp); writer.setUseCompoundFile(useCompoundFile); writer.setRAMBufferSizeMB(ramBufferSizeMB); writer.setTermIndexInterval(termIndexIntervalSize); writer.commit(); } catch (Exception e) { throw new RuntimeException(e); } }
From source file:de.jetsli.lumeo.RawLucene.java
License:Apache License
public RawLucene init() { indexLock();//w w w .j ava 2 s . co m try { if (closed) throw new IllegalStateException("Already closed"); if (writer != null) throw new IllegalStateException("Already initialized"); // release locks when started if (IndexWriter.isLocked(dir)) { logger.warn("index is locked + " + name + " -> releasing lock"); IndexWriter.unlock(dir); } IndexWriterConfig cfg = new IndexWriterConfig(VERSION, defaultMapping.getCombinedAnalyzer()); LogByteSizeMergePolicy mp = new LogByteSizeMergePolicy(); mp.setMaxMergeMB(getMaxMergeMB()); cfg.setRAMBufferSizeMB(ramBufferSizeMB); cfg.setTermIndexInterval(termIndexIntervalSize); cfg.setMergePolicy(mp); // TODO specify different formats for id fields etc // -> this breaks 16 of our tests!? Lucene Bug? // cfg.setCodec(new Lucene40Codec() { // // @Override public PostingsFormat getPostingsFormatForField(String field) { // return new Pulsing40PostingsFormat(); // } // }); // cfg.setMaxThreadStates(8); boolean create = !DirectoryReader.indexExists(dir); cfg.setOpenMode(create ? IndexWriterConfig.OpenMode.CREATE : IndexWriterConfig.OpenMode.APPEND); //wrap the writer with a tracking index writer writer = new TrackingIndexWriter(new IndexWriter(dir, cfg)); nrtManager = new NRTManager(writer, new SearcherFactory() { // @Override // public IndexSearcher newSearcher(IndexReader reader) throws IOException { // //TODO do some kind of warming here? // return new IndexSearcher(reader); // } }); getCurrentRTCache(latestGen); int priority = Math.min(Thread.currentThread().getPriority() + 2, Thread.MAX_PRIORITY); flushThread = new FlushThread("flush-thread"); flushThread.setPriority(priority); flushThread.setDaemon(true); flushThread.start(); reopenThread = new NRTManagerReopenThread(nrtManager, ordinaryWaiting, incomingSearchesMaximumWaiting); reopenThread.setName("NRT Reopen Thread"); reopenThread.setPriority(priority); reopenThread.setDaemon(true); reopenThread.start(); return this; } catch (Exception e) { throw new RuntimeException(e); } finally { indexUnlock(); } }
From source file:dk.defxws.fgslucene.IndexWriterCache.java
License:Open Source License
/** * get IndexWriter for given indexPath and write it into cache. * /*from ww w . j ava2 s. c o m*/ * @param indexName * name of index to open. * @param config * gsearch config-Object. * @throws GenericSearchException * e */ private IndexWriter getIndexWriter(final String indexName, final boolean create, final Config config) throws GenericSearchException { if (indexWriters.get(indexName) == null) { IndexWriter iw = null; try { IndexWriterConfig indexWriterConfig = new IndexWriterConfig(Constants.LUCENE_VERSION, getAnalyzer(config.getAnalyzer(indexName))); if (create) { indexWriterConfig.setOpenMode(OpenMode.CREATE); } else { indexWriterConfig.setOpenMode(OpenMode.CREATE_OR_APPEND); } if (config.getMaxBufferedDocs(indexName) > 1) { indexWriterConfig.setMaxBufferedDocs(config.getMaxBufferedDocs(indexName)); } if (config.getRamBufferSizeMb(indexName) > 1) { indexWriterConfig.setRAMBufferSizeMB(config.getRamBufferSizeMb(indexName)); } if (config.getMergeFactor(indexName) > 1 || config.getMaxMergeDocs(indexName) > 1 || config.getMaxMergeMb(indexName) > 1) { LogByteSizeMergePolicy logMergePolicy = new LogByteSizeMergePolicy(); if (config.getMergeFactor(indexName) > 1) { logMergePolicy.setMergeFactor(config.getMergeFactor(indexName)); } if (config.getMaxMergeDocs(indexName) > 1) { logMergePolicy.setMaxMergeDocs(config.getMaxMergeDocs(indexName)); } if (config.getMaxMergeMb(indexName) > 1) { logMergePolicy.setMaxMergeMB(config.getMaxMergeMb(indexName)); } indexWriterConfig.setMergePolicy(logMergePolicy); } if (config.getDefaultWriteLockTimeout(indexName) > 1) { indexWriterConfig.setWriteLockTimeout(config.getDefaultWriteLockTimeout(indexName)); } if (config.getLuceneDirectoryImplementation(indexName) != null) { // Initialize IndexWriter with configured FSDirectory FSDirectory directory = getDirectoryImplementation( config.getLuceneDirectoryImplementation(indexName), new File(config.getIndexDir(indexName))); iw = new IndexWriter(directory, indexWriterConfig); } else { // Initialize IndexWriter with default FSDirectory iw = new IndexWriter(FSDirectory.open(new File(config.getIndexDir(indexName))), indexWriterConfig); } if (config.getMaxChunkSize(indexName) > 1) { if (iw.getDirectory() instanceof MMapDirectory) { ((MMapDirectory) iw.getDirectory()).setMaxChunkSize(config.getMaxChunkSize(indexName)); } } } catch (Exception e) { iw = null; throw new GenericSearchException( "IndexWriter new error, creating index indexName=" + indexName + " :\n", e); } indexWriters.put(indexName, iw); if (logger.isDebugEnabled()) logger.debug("getIndexWriter put to map " + iw); return iw; } return indexWriters.get(indexName); }
From source file:net.dataforte.infinispan.amanuensis.DefaultWriterConfigurator.java
License:Open Source License
@Override public void configure(IndexWriter writer) { LogByteSizeMergePolicy mergePolicy = new LogByteSizeMergePolicy(writer); mergePolicy.setMaxMergeMB(maxMergeMB); mergePolicy.setMinMergeMB(minMergeMB); mergePolicy.setMaxMergeDocs(maxMergeDocs); mergePolicy.setMergeFactor(mergeFactor); mergePolicy.setCalibrateSizeByDeletes(calibrateSizeByDeletes); mergePolicy.setNoCFSRatio(noCFSRatio); mergePolicy.setUseCompoundDocStore(useCompoundDocStore); mergePolicy.setUseCompoundFile(useCompoundFile); writer.setMergePolicy(mergePolicy);//from w w w .ja va2s . c o m }
From source file:org.compass.core.lucene.engine.merge.policy.LogByteSizeMergePolicyProvider.java
License:Apache License
public MergePolicy create(CompassSettings settings) throws SearchEngineException { LogByteSizeMergePolicy mergePolicy = new LogByteSizeMergePolicy(); mergePolicy .setMaxMergeMB(settings.getSettingAsDouble(LuceneEnvironment.MergePolicy.LogByteSize.MAX_MERGE_MB, LogByteSizeMergePolicy.DEFAULT_MAX_MERGE_MB)); mergePolicy/*from w w w. ja v a2s .c om*/ .setMinMergeMB(settings.getSettingAsDouble(LuceneEnvironment.MergePolicy.LogByteSize.MIN_MERGE_MB, LogByteSizeMergePolicy.DEFAULT_MIN_MERGE_MB)); return mergePolicy; }
From source file:org.paxle.se.index.lucene.impl.AFlushableLuceneManager.java
License:Open Source License
@Activate protected void activate(Map<String, Object> props) throws IOException, InterruptedException { // the default analyzer to use this.analyzer = this.stopWordsManager.getDefaultAnalyzer(); // init the converter // TODO: this is bad. we need to remove this Converter.fieldManager = this.fieldManager; // the path were the data should be stored //TO-DO: check props for null this.fullPath = System.getProperty("paxle.data") + File.separatorChar + props.get("dataPath"); this.dataPath = new File(this.fullPath); final File writeLock = new File(this.fullPath, "write.lock"); if (writeLock.exists()) { logger.warn("Lucene index directory is locked, removing lock. " + "Shutdown now if any other lucene-compatible application currently accesses the directory '" + writeLock.getPath());/*from ww w . j av a 2 s . c om*/ Thread.sleep(5000l); writeLock.delete(); } writeLock.deleteOnExit(); // opening index-reader and -writer final Directory dir = FSDirectory.open(this.dataPath); this.writer = new IndexWriter(dir, analyzer, MaxFieldLength.UNLIMITED); this.reader = IndexReader.open(dir, true); // open a read-only index, deletions are performed by the writer // getting the amount of known documents this.docCount = reader.numDocs(); // starting a flush timer this.flushTimer = new Timer("LuceneFlushTimer"); this.flushTimer.scheduleAtFixedRate(new FlushTimerTask(), 60 * 60 * 1000, 60 * 60 * 1000); // set merge policy LogByteSizeMergePolicy policy = new LogByteSizeMergePolicy(this.writer); policy.setMaxMergeMB(2048); this.writer.setMergePolicy(policy); }