List of usage examples for org.apache.lucene.index LogByteSizeMergePolicy LogByteSizeMergePolicy
public LogByteSizeMergePolicy()
From source file:com.flaptor.hounder.Index.java
License:Apache License
private Index(final File path, boolean create) { if (!create) { if (!path.exists()) { throw new IllegalArgumentException("The path passed to the costructor doesn't exist."); }/*from w w w. j a v a 2 s . c om*/ if (!path.isDirectory()) { throw new IllegalArgumentException("The path passed to the constructor is not a directory"); } } else { if (path.exists()) { throw new IllegalArgumentException( "Cannot create index on " + path.getAbsolutePath() + ". Path exists."); } } this.path = path; properties = new Properties(); if (!create) { File propFile = new File(path.getAbsolutePath() + File.separator + "index.properties"); if (!propFile.exists()) { throw new IllegalArgumentException( "Cannot find the properties inside the index. Is the index corrupted?."); } if (!propFile.isFile()) { throw new IllegalArgumentException( "There's no file named index.properties in the index. Maybe it is a directory?"); } InputStream is = null; try { is = new FileInputStream(propFile); properties.load(is); indexDescriptor = new IndexDescriptor(properties.get("indexDescriptor").toString()); } catch (IOException e) { logger.error( "Exception while trying to load index.properties for index at " + path.getAbsolutePath(), e); } catch (NullPointerException e) { logger.error("There is no index descriptor on " + propFile.getName() + ". using default. Exception caused by " + e.getMessage(), e); indexDescriptor = IndexDescriptor.defaultDescriptor(); } finally { com.flaptor.util.Execute.close(is, logger); } } Config config = Config.getConfig("common.properties"); failOnLegacyParameters(config); ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler(); cms.setMaxThreadCount(12); cms.setMergeThreadPriority(Thread.MIN_PRIORITY); this.mergeScheduler = cms; LogByteSizeMergePolicy mp = new LogByteSizeMergePolicy(); smallSegmentSize = config.getFloat("Index.smallSegmentSizeMB"); mp.setMinMergeMB(smallSegmentSize); mp.setMergeFactor(config.getInt("Index.mergeFactor")); mergePolicy = mp; createAnalyzer(); setUpDirectory(create); }
From source file:com.graphhopper.compare.misc.LuceneStorage.java
License:Apache License
public boolean init(boolean forceCreate) { try {/*from w ww . j a v a 2 s . c o m*/ File file = new File("osm.lucene.test"); if (forceCreate) Helper.deleteDir(file); // germany.osm => 3.6 GB on disc for nodes only, 1.5 GB memory usage at the end of the nodes Directory dir = FSDirectory.open(file); IndexWriterConfig cfg = new IndexWriterConfig(Version.LUCENE_35, new KeywordAnalyzer()); LogByteSizeMergePolicy mp = new LogByteSizeMergePolicy(); mp.setMaxMergeMB(3000); cfg.setRAMBufferSizeMB(128); cfg.setTermIndexInterval(512); cfg.setMergePolicy(mp); // specify different formats for id fields etc // -> this breaks 16 of our tests!? Lucene Bug? // cfg.setCodec(new Lucene40Codec() { // // @Override public PostingsFormat getPostingsFormatForField(String field) { // return new Pulsing40PostingsFormat(); // } // }); // cfg.setMaxThreadStates(8); boolean create = !IndexReader.indexExists(dir); cfg.setOpenMode(create ? IndexWriterConfig.OpenMode.CREATE : IndexWriterConfig.OpenMode.APPEND); writer = new IndexWriter(dir, cfg); return true; } catch (Exception ex) { logger.error("cannot init lucene storage", ex); return false; } }
From source file:com.tinkerpop.graph.benchmark.index.LuceneKeyToNodeIdIndexImpl.java
License:Apache License
public void init() { try {// www. j a va 2 s .c o m hotCache = new LRUCache<String, Long>(getLruCacheSize()); if (showDebug) { System.out.println( "timeTaken,numDocs,bloomReadSaves,hotCacheHits,failedLuceneReads,successfulLuceneReads,luceneAdds"); } bloomFilter = new OpenBitSet(bloomFilterSize); deleteDirectoryContents(new File(path)); dir = FSDirectory.open(new File(path)); writer = new IndexWriter(dir, new WhitespaceAnalyzer(), MaxFieldLength.UNLIMITED); // LogByteSizeMergePolicy mp=new LogByteSizeMergePolicy(writer); LogByteSizeMergePolicy mp = new LogByteSizeMergePolicy(); mp.setMaxMergeMB(getMaxMergeMB()); writer.setMergePolicy(mp); writer.setUseCompoundFile(useCompoundFile); writer.setRAMBufferSizeMB(ramBufferSizeMB); writer.setTermIndexInterval(termIndexIntervalSize); writer.commit(); } catch (Exception e) { throw new RuntimeException(e); } }
From source file:com.zimbra.cs.index.LuceneIndex.java
License:Open Source License
private IndexWriterConfig getWriterConfig() { IndexWriterConfig config = new IndexWriterConfig(VERSION, mailbox.index.getAnalyzer()); config.setMergeScheduler(new MergeScheduler()); config.setMaxBufferedDocs(LC.zimbra_index_lucene_max_buffered_docs.intValue()); config.setRAMBufferSizeMB(LC.zimbra_index_lucene_ram_buffer_size_kb.intValue() / 1024.0); if (LC.zimbra_index_lucene_merge_policy.booleanValue()) { LogDocMergePolicy policy = new LogDocMergePolicy(); config.setMergePolicy(policy);/* w ww. j a va 2 s . c o m*/ policy.setUseCompoundFile(LC.zimbra_index_lucene_use_compound_file.booleanValue()); policy.setMergeFactor(LC.zimbra_index_lucene_merge_factor.intValue()); policy.setMinMergeDocs(LC.zimbra_index_lucene_min_merge.intValue()); if (LC.zimbra_index_lucene_max_merge.intValue() != Integer.MAX_VALUE) { policy.setMaxMergeDocs(LC.zimbra_index_lucene_max_merge.intValue()); } } else { LogByteSizeMergePolicy policy = new LogByteSizeMergePolicy(); config.setMergePolicy(policy); policy.setUseCompoundFile(LC.zimbra_index_lucene_use_compound_file.booleanValue()); policy.setMergeFactor(LC.zimbra_index_lucene_merge_factor.intValue()); policy.setMinMergeMB(LC.zimbra_index_lucene_min_merge.intValue() / 1024.0); if (LC.zimbra_index_lucene_max_merge.intValue() != Integer.MAX_VALUE) { policy.setMaxMergeMB(LC.zimbra_index_lucene_max_merge.intValue() / 1024.0); } } return config; }
From source file:de.jetsli.lumeo.LucPerfTest.java
License:Apache License
public void testPerf() { new PerfRunner(1000000, 26f) { @Override/*from www . j a va 2s . com*/ public void reinit() throws Exception { super.reinit(); if (nrtManager != null) { nrtManager.close(); reopenThread.close(); writer.waitForMerges(); writer.close(); dir.close(); } Helper.deleteDir(file); docs = 0; IndexWriterConfig cfg = new IndexWriterConfig(version, keyAna); cfg.setRAMBufferSizeMB(128); // cfg.setCodec(new Lucene40Codec() { // // @Override public PostingsFormat getPostingsFormatForField(String field) { // if ("_id".equals(field)) // return new Pulsing40PostingsFormat(); // else // return new Lucene40PostingsFormat(); // } // }); LogByteSizeMergePolicy mp = new LogByteSizeMergePolicy(); mp.setUseCompoundFile(false); cfg.setMergePolicy(mp); dir = FSDirectory.open(file); cfg.setOpenMode(IndexWriterConfig.OpenMode.CREATE); writer = new IndexWriter(dir, cfg); nrtManager = new NRTManager(writer, new SearcherWarmer() { @Override public void warm(IndexSearcher s) throws IOException { // TODO get some random vertices via getVertices? } }); int priority = Math.min(Thread.currentThread().getPriority() + 2, Thread.MAX_PRIORITY); reopenThread = new NRTManagerReopenThread(nrtManager, 5.0, 0.03); reopenThread.setName("NRT Reopen Thread"); reopenThread.setPriority(priority); reopenThread.setDaemon(true); reopenThread.start(); } final BytesRef bytes = new BytesRef(); @Override public void innerRun(int trial, int i) { long id = i; Document newDoc = new Document(); NumericField idField = new NumericField("_id", 6, NumericField.TYPE_STORED).setLongValue(id); Field uIdField = new Field("_uid", "" + id, StringField.TYPE_STORED); Field typeField = new Field("_type", "test", StringField.TYPE_STORED); newDoc.add(idField); newDoc.add(uIdField); newDoc.add(typeField); // Analyzer ana = anas.get(newDoc.get("_type")); try { NumericUtils.longToPrefixCoded(id, 0, bytes); latestGen = nrtManager.updateDocument(new Term("_id", bytes), newDoc, keyAna); docs++; } catch (IOException ex) { logger.error("Cannot update " + i, ex); } } @Override protected void finalAssert() throws Exception { // logger.info("wait for " + latestGen + ", current:" + nrtManager.getCurrentSearchingGen(true)); nrtManager.waitForGeneration(latestGen, true); // writer.commit(); // writer.waitForMerges(); SearcherManager mng = nrtManager.getSearcherManager(true); // mng.maybeReopen(); IndexSearcher searcher = mng.acquire(); try { TotalHitCountCollector coll = new TotalHitCountCollector(); searcher.search(new MatchAllDocsQuery(), coll); long total = coll.getTotalHits(); if (docs != total) throw new IllegalStateException(total + " vs. " + docs); } finally { nrtManager.getSearcherManager(true).release(searcher); } } }.run(); }
From source file:de.jetsli.lumeo.RawLucene.java
License:Apache License
public RawLucene init() { indexLock();//from w w w .ja v a 2 s.com try { if (closed) throw new IllegalStateException("Already closed"); if (writer != null) throw new IllegalStateException("Already initialized"); // release locks when started if (IndexWriter.isLocked(dir)) { logger.warn("index is locked + " + name + " -> releasing lock"); IndexWriter.unlock(dir); } IndexWriterConfig cfg = new IndexWriterConfig(VERSION, defaultMapping.getCombinedAnalyzer()); LogByteSizeMergePolicy mp = new LogByteSizeMergePolicy(); mp.setMaxMergeMB(getMaxMergeMB()); cfg.setRAMBufferSizeMB(ramBufferSizeMB); cfg.setTermIndexInterval(termIndexIntervalSize); cfg.setMergePolicy(mp); // TODO specify different formats for id fields etc // -> this breaks 16 of our tests!? Lucene Bug? // cfg.setCodec(new Lucene40Codec() { // // @Override public PostingsFormat getPostingsFormatForField(String field) { // return new Pulsing40PostingsFormat(); // } // }); // cfg.setMaxThreadStates(8); boolean create = !DirectoryReader.indexExists(dir); cfg.setOpenMode(create ? IndexWriterConfig.OpenMode.CREATE : IndexWriterConfig.OpenMode.APPEND); //wrap the writer with a tracking index writer writer = new TrackingIndexWriter(new IndexWriter(dir, cfg)); nrtManager = new NRTManager(writer, new SearcherFactory() { // @Override // public IndexSearcher newSearcher(IndexReader reader) throws IOException { // //TODO do some kind of warming here? // return new IndexSearcher(reader); // } }); getCurrentRTCache(latestGen); int priority = Math.min(Thread.currentThread().getPriority() + 2, Thread.MAX_PRIORITY); flushThread = new FlushThread("flush-thread"); flushThread.setPriority(priority); flushThread.setDaemon(true); flushThread.start(); reopenThread = new NRTManagerReopenThread(nrtManager, ordinaryWaiting, incomingSearchesMaximumWaiting); reopenThread.setName("NRT Reopen Thread"); reopenThread.setPriority(priority); reopenThread.setDaemon(true); reopenThread.start(); return this; } catch (Exception e) { throw new RuntimeException(e); } finally { indexUnlock(); } }
From source file:dk.defxws.fgslucene.IndexWriterCache.java
License:Open Source License
/** * get IndexWriter for given indexPath and write it into cache. * /*from w w w. j av a2 s . c om*/ * @param indexName * name of index to open. * @param config * gsearch config-Object. * @throws GenericSearchException * e */ private IndexWriter getIndexWriter(final String indexName, final boolean create, final Config config) throws GenericSearchException { if (indexWriters.get(indexName) == null) { IndexWriter iw = null; try { IndexWriterConfig indexWriterConfig = new IndexWriterConfig(Constants.LUCENE_VERSION, getAnalyzer(config.getAnalyzer(indexName))); if (create) { indexWriterConfig.setOpenMode(OpenMode.CREATE); } else { indexWriterConfig.setOpenMode(OpenMode.CREATE_OR_APPEND); } if (config.getMaxBufferedDocs(indexName) > 1) { indexWriterConfig.setMaxBufferedDocs(config.getMaxBufferedDocs(indexName)); } if (config.getRamBufferSizeMb(indexName) > 1) { indexWriterConfig.setRAMBufferSizeMB(config.getRamBufferSizeMb(indexName)); } if (config.getMergeFactor(indexName) > 1 || config.getMaxMergeDocs(indexName) > 1 || config.getMaxMergeMb(indexName) > 1) { LogByteSizeMergePolicy logMergePolicy = new LogByteSizeMergePolicy(); if (config.getMergeFactor(indexName) > 1) { logMergePolicy.setMergeFactor(config.getMergeFactor(indexName)); } if (config.getMaxMergeDocs(indexName) > 1) { logMergePolicy.setMaxMergeDocs(config.getMaxMergeDocs(indexName)); } if (config.getMaxMergeMb(indexName) > 1) { logMergePolicy.setMaxMergeMB(config.getMaxMergeMb(indexName)); } indexWriterConfig.setMergePolicy(logMergePolicy); } if (config.getDefaultWriteLockTimeout(indexName) > 1) { indexWriterConfig.setWriteLockTimeout(config.getDefaultWriteLockTimeout(indexName)); } if (config.getLuceneDirectoryImplementation(indexName) != null) { // Initialize IndexWriter with configured FSDirectory FSDirectory directory = getDirectoryImplementation( config.getLuceneDirectoryImplementation(indexName), new File(config.getIndexDir(indexName))); iw = new IndexWriter(directory, indexWriterConfig); } else { // Initialize IndexWriter with default FSDirectory iw = new IndexWriter(FSDirectory.open(new File(config.getIndexDir(indexName))), indexWriterConfig); } if (config.getMaxChunkSize(indexName) > 1) { if (iw.getDirectory() instanceof MMapDirectory) { ((MMapDirectory) iw.getDirectory()).setMaxChunkSize(config.getMaxChunkSize(indexName)); } } } catch (Exception e) { iw = null; throw new GenericSearchException( "IndexWriter new error, creating index indexName=" + indexName + " :\n", e); } indexWriters.put(indexName, iw); if (logger.isDebugEnabled()) logger.debug("getIndexWriter put to map " + iw); return iw; } return indexWriters.get(indexName); }
From source file:io.crate.expression.reference.doc.DocLevelExpressionsTest.java
License:Apache License
@Before public void prepare() throws Exception { Settings settings = Settings.builder().put("index.fielddata.cache", "none").build(); IndexService indexService = createIndex("test", settings); ifd = indexService.fieldData();/*from ww w . ja v a2 s .co m*/ writer = new IndexWriter(new RAMDirectory(), new IndexWriterConfig(new StandardAnalyzer()).setMergePolicy(new LogByteSizeMergePolicy())); insertValues(writer); DirectoryReader directoryReader = DirectoryReader.open(writer, true, true); readerContext = directoryReader.leaves().get(0); ctx = new CollectorContext(ifd::getForField, null); }
From source file:io.crate.operation.reference.doc.DocLevelExpressionsTest.java
License:Apache License
@Before public void prepare() throws Exception { Settings settings = ImmutableSettings.builder().put("index.fielddata.cache", "none").build(); IndexService indexService = createIndex("test", settings); ifd = indexService.fieldData();/* w ww.j a v a 2 s. c om*/ MapperService mapperService = mock(MapperService.class); FieldMapper fieldMapper = mock(FieldMapper.class); when(fieldMapper.names()).thenReturn(fieldName()); when(fieldMapper.fieldDataType()).thenReturn(fieldType()); when(mapperService.smartNameFieldMapper(anyString(), Matchers.<String[]>any())).thenReturn(fieldMapper); IndexFieldData<?> fieldData = ifd.getForField(fieldMapper); writer = new IndexWriter(new RAMDirectory(), new IndexWriterConfig(Lucene.VERSION, new StandardAnalyzer()) .setMergePolicy(new LogByteSizeMergePolicy())); insertValues(writer); DirectoryReader directoryReader = DirectoryReader.open(writer, true); readerContext = directoryReader.leaves().get(0); fieldData.load(readerContext); ctx = new CollectorContext(mapperService, ifd, null); }
From source file:net.ion.craken.node.problem.distribute.LuceneSettings.java
License:Open Source License
public static IndexWriter openWriter(Directory directory, int maxMergeDocs, boolean useSerialMerger) throws CorruptIndexException, LockObtainFailedException, IOException { IndexWriterConfig indexWriterConfig = new IndexWriterConfig(LUCENE_VERSION, analyzer); if (useSerialMerger) { indexWriterConfig.setMergeScheduler(mergeScheduler); }//from w w w . j a v a 2 s.com LogMergePolicy mergePolicy = new LogByteSizeMergePolicy(); mergePolicy.setMaxMergeDocs(maxMergeDocs); indexWriterConfig.setMergePolicy(mergePolicy); return new IndexWriter(directory, indexWriterConfig); }