List of usage examples for org.apache.lucene.index IndexWriterConfig setRAMBufferSizeMB
@Override
public IndexWriterConfig setRAMBufferSizeMB(double ramBufferSizeMB)
From source file:com.baidu.rigel.biplatform.tesseract.isservice.index.service.IndexWriterFactory.java
License:Open Source License
public static synchronized IndexWriter getIndexWriterWithSingleSlot(String idxPath) throws IOException { LOGGER.info(String.format(LogInfoConstants.INFO_PATTERN_FUNCTION_BEGIN, "getIndexWriter", "[idxPath:" + idxPath + "]")); IndexWriter indexWriter = null;//from w ww . j a v a 2s. c o m Integer maxSlot = 0; if (INSTANCE.idxMaps.containsKey(idxPath)) { maxSlot = INSTANCE.idxMaps.get(idxPath); maxSlot++; } File indexFile = new File(idxPath + File.separator + maxSlot); Directory directory = FSDirectory.open(indexFile); IndexWriterConfig indexWriterConfig = new IndexWriterConfig(Version.LUCENE_4_10_1, new StandardAnalyzer()); indexWriterConfig.setOpenMode(OpenMode.CREATE_OR_APPEND); indexWriterConfig.setRAMBufferSizeMB(48.0); indexWriter = new IndexWriter(directory, indexWriterConfig); INSTANCE.idxMaps.put(idxPath, maxSlot); LOGGER.info(String.format(LogInfoConstants.INFO_PATTERN_FUNCTION_END, "getIndexWriter", "[idxPath:" + idxPath + "]")); return indexWriter; }
From source file:com.bugull.mongo.lucene.cache.IndexWriterCache.java
License:Apache License
public IndexWriter get(String name) { IndexWriter writer = cache.get(name); if (writer != null) { return writer; }// www . j av a 2 s . c o m synchronized (this) { writer = cache.get(name); if (writer == null) { BuguIndex index = BuguIndex.getInstance(); IndexWriterConfig conf = new IndexWriterConfig(index.getVersion(), index.getAnalyzer()); double bufferSizeMB = index.getBufferSizeMB(); conf.setRAMBufferSizeMB(bufferSizeMB); try { String path = index.getDirectoryPath(); Directory directory = FSDirectory.open(new File(path + "/" + name)); writer = new IndexWriter(directory, conf); } catch (IOException ex) { logger.error("Something is wrong when create IndexWriter for " + name, ex); } cache.put(name, writer); } } return writer; }
From source file:com.bugull.mongo.lucene.holder.IndexWriterHolder.java
License:Apache License
public IndexWriter get(String name) { IndexWriter writer = map.get(name);/*from w w w. ja va2s . c o m*/ if (writer != null) { return writer; } synchronized (this) { writer = map.get(name); if (writer == null) { BuguIndex index = BuguIndex.getInstance(); IndexWriterConfig conf = new IndexWriterConfig(index.getVersion(), index.getAnalyzer()); double bufferSizeMB = index.getBufferSizeMB(); conf.setRAMBufferSizeMB(bufferSizeMB); try { String path = index.getDirectoryPath(); Directory directory = FSDirectory.open(new File(path + "/" + name)); writer = new IndexWriter(directory, conf); } catch (IOException ex) { logger.error("Something is wrong when create IndexWriter for " + name, ex); } map.put(name, writer); } } return writer; }
From source file:com.github.rnewson.couchdb.lucene.DatabaseIndexer.java
License:Apache License
private IndexWriter newWriter(final Directory dir) throws IOException { final IndexWriterConfig config = new IndexWriterConfig(Constants.VERSION, Constants.ANALYZER); config.setUseCompoundFile(ini.getBoolean("lucene.useCompoundFile", false)); config.setRAMBufferSizeMB( ini.getDouble("lucene.ramBufferSizeMB", IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB)); return new IndexWriter(dir, config); }
From source file:com.graphhopper.compare.misc.LuceneStorage.java
License:Apache License
public boolean init(boolean forceCreate) { try {//ww w . j av a 2 s.c o m File file = new File("osm.lucene.test"); if (forceCreate) Helper.deleteDir(file); // germany.osm => 3.6 GB on disc for nodes only, 1.5 GB memory usage at the end of the nodes Directory dir = FSDirectory.open(file); IndexWriterConfig cfg = new IndexWriterConfig(Version.LUCENE_35, new KeywordAnalyzer()); LogByteSizeMergePolicy mp = new LogByteSizeMergePolicy(); mp.setMaxMergeMB(3000); cfg.setRAMBufferSizeMB(128); cfg.setTermIndexInterval(512); cfg.setMergePolicy(mp); // specify different formats for id fields etc // -> this breaks 16 of our tests!? Lucene Bug? // cfg.setCodec(new Lucene40Codec() { // // @Override public PostingsFormat getPostingsFormatForField(String field) { // return new Pulsing40PostingsFormat(); // } // }); // cfg.setMaxThreadStates(8); boolean create = !IndexReader.indexExists(dir); cfg.setOpenMode(create ? IndexWriterConfig.OpenMode.CREATE : IndexWriterConfig.OpenMode.APPEND); writer = new IndexWriter(dir, cfg); return true; } catch (Exception ex) { logger.error("cannot init lucene storage", ex); return false; } }
From source file:com.liferay.portal.search.lucene.infinispan.IndexAccessorImpl.java
License:Open Source License
private void _initIndexWriter() { try {/*from w ww . ja v a 2s. c om*/ IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_33, LuceneHelperUtil.getAnalyzer()); config.setRAMBufferSizeMB(PropsValues.LUCENE_BUFFER_SIZE); _indexWriter = new IndexWriter(getLuceneDir(), config); } catch (Exception e) { _log.error("Initializing Lucene writer failed for " + _companyId, e); } }
From source file:com.nearinfinity.blur.mapreduce.BlurReducer.java
License:Apache License
protected void setupWriter(Context context) throws IOException { nullCheck(_directory);/*from w ww.j a va2 s . c o m*/ nullCheck(_analyzer); IndexWriterConfig config = new IndexWriterConfig(LUCENE_VERSION, _analyzer); config.setSimilarity(new FairSimilarity()); config.setRAMBufferSizeMB(_blurTask.getRamBufferSizeMB()); TieredMergePolicy mergePolicy = (TieredMergePolicy) config.getMergePolicy(); mergePolicy.setUseCompoundFile(false); _writer = new IndexWriter(_directory, config); }
From source file:com.qwazr.search.bench.LuceneCommonIndex.java
License:Apache License
LuceneCommonIndex(final Path rootDirectory, final String schemaName, final String indexName, final double ramBufferSize, final boolean useCompoundFile) throws IOException { final Path schemaDirectory = Files.createDirectory(rootDirectory.resolve(schemaName)); this.indexDirectory = Files.createDirectory(schemaDirectory.resolve(indexName)); this.luceneDirectory = indexDirectory.resolve("data"); this.dataDirectory = FSDirectory.open(luceneDirectory); final IndexWriterConfig indexWriterConfig = new IndexWriterConfig( new PerFieldAnalyzerWrapper(new StandardAnalyzer())); indexWriterConfig.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); indexWriterConfig.setRAMBufferSizeMB(ramBufferSize); final ConcurrentMergeScheduler mergeScheduler = new ConcurrentMergeScheduler(); mergeScheduler.setMaxMergesAndThreads(MAX_SSD_MERGE_THREADS, MAX_SSD_MERGE_THREADS); indexWriterConfig.setMergeScheduler(mergeScheduler); indexWriterConfig.setUseCompoundFile(useCompoundFile); final TieredMergePolicy mergePolicy = new TieredMergePolicy(); indexWriterConfig.setMergePolicy(mergePolicy); // We use snapshots deletion policy final SnapshotDeletionPolicy snapshotDeletionPolicy = new SnapshotDeletionPolicy( indexWriterConfig.getIndexDeletionPolicy()); indexWriterConfig.setIndexDeletionPolicy(snapshotDeletionPolicy); this.indexWriter = new IndexWriter(this.dataDirectory, indexWriterConfig); this.localReplicator = new LocalReplicator(); }
From source file:com.stratio.cassandra.index.LuceneIndex.java
License:Apache License
/** * Initializes this using the specified {@link Sort} for trying to keep the {@link Document}s sorted. * * @param sort The {@link Sort} to be used. *///from w w w.j a v a 2s .c o m public void init(Sort sort) { Log.debug("Initializing index"); try { this.sort = sort; // Get directory file file = new File(path); // Open or create directory FSDirectory fsDirectory = FSDirectory.open(file); directory = new NRTCachingDirectory(fsDirectory, maxMergeMB, maxCachedMB); // Setup index writer IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_48, analyzer); config.setRAMBufferSizeMB(ramBufferMB); config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); config.setUseCompoundFile(true); config.setMergePolicy(new SortingMergePolicy(config.getMergePolicy(), sort)); indexWriter = new IndexWriter(directory, config); // Setup NRT search SearcherFactory searcherFactory = new SearcherFactory() { public IndexSearcher newSearcher(IndexReader reader) throws IOException { IndexSearcher searcher = new IndexSearcher(reader); searcher.setSimilarity(new NoIDFSimilarity()); return searcher; } }; TrackingIndexWriter trackingIndexWriter = new TrackingIndexWriter(indexWriter); searcherManager = new SearcherManager(indexWriter, true, searcherFactory); searcherReopener = new ControlledRealTimeReopenThread<>(trackingIndexWriter, searcherManager, refreshSeconds, refreshSeconds); searcherReopener.start(); // Start the refresher thread } catch (IOException e) { Log.error(e, "Error while initializing index"); throw new RuntimeException(e); } }
From source file:com.stratio.cassandra.lucene.index.FSIndex.java
License:Apache License
/** * Builds a new {@link FSIndex}./* www . ja va2s .c o m*/ * * @param name the index name * @param mbeanName the JMX MBean object name * @param path the directory path * @param analyzer the index writer analyzer * @param refresh the index reader refresh frequency in seconds * @param ramBufferMB the index writer RAM buffer size in MB * @param maxMergeMB the directory max merge size in MB * @param maxCachedMB the directory max cache size in MB * @param refreshTask action to be done during refresh */ public FSIndex(String name, String mbeanName, Path path, Analyzer analyzer, double refresh, int ramBufferMB, int maxMergeMB, int maxCachedMB, Runnable refreshTask) { try { this.path = path; this.name = name; // Open or create directory FSDirectory fsDirectory = FSDirectory.open(path); directory = new NRTCachingDirectory(fsDirectory, maxMergeMB, maxCachedMB); // Setup index writer IndexWriterConfig indexWriterConfig = new IndexWriterConfig(analyzer); indexWriterConfig.setRAMBufferSizeMB(ramBufferMB); indexWriterConfig.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); indexWriterConfig.setUseCompoundFile(true); indexWriterConfig.setMergePolicy(new TieredMergePolicy()); indexWriter = new IndexWriter(directory, indexWriterConfig); // Setup NRT search SearcherFactory searcherFactory = new SearcherFactory() { @Override public IndexSearcher newSearcher(IndexReader reader, IndexReader previousReader) { if (refreshTask != null) { refreshTask.run(); } IndexSearcher searcher = new IndexSearcher(reader); searcher.setSimilarity(new NoIDFSimilarity()); return searcher; } }; TrackingIndexWriter trackingWriter = new TrackingIndexWriter(indexWriter); searcherManager = new SearcherManager(indexWriter, true, searcherFactory); searcherReopener = new ControlledRealTimeReopenThread<>(trackingWriter, searcherManager, refresh, refresh); searcherReopener.start(); // Register JMX MBean mbean = new ObjectName(mbeanName); ManagementFactory.getPlatformMBeanServer().registerMBean(this, this.mbean); } catch (Exception e) { throw new IndexException(logger, e, "Error while creating index %s", name); } }