List of usage examples for org.apache.lucene.search ControlledRealTimeReopenThread ControlledRealTimeReopenThread
public ControlledRealTimeReopenThread(IndexWriter writer, ReferenceManager<T> manager, double targetMaxStaleSec, double targetMinStaleSec)
From source file:com.google.gerrit.lucene.AbstractLuceneIndex.java
License:Apache License
AbstractLuceneIndex(Schema<V> schema, SitePaths sitePaths, Directory dir, String name, String subIndex,
GerritIndexWriterConfig writerConfig, SearcherFactory searcherFactory) throws IOException {
this.schema = schema;
this.sitePaths = sitePaths;
this.dir = dir;
this.name = name;
final String index = Joiner.on('_').skipNulls().join(name, subIndex);
IndexWriter delegateWriter;/*from w ww. j ava 2 s. c om*/
long commitPeriod = writerConfig.getCommitWithinMs();
if (commitPeriod < 0) {
delegateWriter = new AutoCommitWriter(dir, writerConfig.getLuceneConfig());
} else if (commitPeriod == 0) {
delegateWriter = new AutoCommitWriter(dir, writerConfig.getLuceneConfig(), true);
} else {
final AutoCommitWriter autoCommitWriter = new AutoCommitWriter(dir, writerConfig.getLuceneConfig());
delegateWriter = autoCommitWriter;
new ScheduledThreadPoolExecutor(1,
new ThreadFactoryBuilder().setNameFormat("Commit-%d " + index).setDaemon(true).build())
.scheduleAtFixedRate(new Runnable() {
@Override
public void run() {
try {
if (autoCommitWriter.hasUncommittedChanges()) {
autoCommitWriter.manualFlush();
autoCommitWriter.commit();
}
} catch (IOException e) {
log.error("Error committing " + index + " Lucene index", e);
} catch (OutOfMemoryError e) {
log.error("Error committing " + index + " Lucene index", e);
try {
autoCommitWriter.close();
} catch (IOException e2) {
log.error(
"SEVERE: Error closing " + index
+ " Lucene index after OOM; index may be corrupted.",
e);
}
}
}
}, commitPeriod, commitPeriod, MILLISECONDS);
}
writer = new TrackingIndexWriter(delegateWriter);
searcherManager = new WrappableSearcherManager(writer.getIndexWriter(), true, searcherFactory);
notDoneNrtFutures = Sets.newConcurrentHashSet();
reopenThread = new ControlledRealTimeReopenThread<>(writer, searcherManager,
0.500 /* maximum stale age (seconds) */, 0.010 /* minimum stale age (seconds) */);
reopenThread.setName("NRT " + name);
reopenThread.setPriority(Math.min(Thread.currentThread().getPriority() + 2, Thread.MAX_PRIORITY));
reopenThread.setDaemon(true);
// This must be added after the reopen thread is created. The reopen thread
// adds its own listener which copies its internally last-refreshed
// generation to the searching generation. removeIfDone() depends on the
// searching generation being up to date when calling
// reopenThread.waitForGeneration(gen, 0), therefore the reopen thread's
// internal listener needs to be called first.
// TODO(dborowitz): This may have been fixed by
// http://issues.apache.org/jira/browse/LUCENE-5461
searcherManager.addListener(new RefreshListener() {
@Override
public void beforeRefresh() throws IOException {
}
@Override
public void afterRefresh(boolean didRefresh) throws IOException {
for (NrtFuture f : notDoneNrtFutures) {
f.removeIfDone();
}
}
});
reopenThread.start();
}
From source file:com.google.gerrit.lucene.SubIndex.java
License:Apache License
SubIndex(Directory dir, final String dirName, GerritIndexWriterConfig writerConfig, SearcherFactory searcherFactory) throws IOException { this.dir = dir; IndexWriter delegateWriter;//from w w w . j a va 2s. c o m long commitPeriod = writerConfig.getCommitWithinMs(); if (commitPeriod < 0) { delegateWriter = new IndexWriter(dir, writerConfig.getLuceneConfig()); } else if (commitPeriod == 0) { delegateWriter = new AutoCommitWriter(dir, writerConfig.getLuceneConfig(), true); } else { final AutoCommitWriter autoCommitWriter = new AutoCommitWriter(dir, writerConfig.getLuceneConfig(), false); delegateWriter = autoCommitWriter; new ScheduledThreadPoolExecutor(1, new ThreadFactoryBuilder().setNameFormat("Commit-%d " + dirName).setDaemon(true).build()) .scheduleAtFixedRate(new Runnable() { @Override public void run() { try { if (autoCommitWriter.hasUncommittedChanges()) { autoCommitWriter.manualFlush(); autoCommitWriter.commit(); } } catch (IOException e) { log.error("Error committing Lucene index " + dirName, e); } catch (OutOfMemoryError e) { log.error("Error committing Lucene index " + dirName, e); try { autoCommitWriter.close(); } catch (IOException e2) { log.error("SEVERE: Error closing Lucene index " + dirName + " after OOM; index may be corrupted.", e); } } } }, commitPeriod, commitPeriod, MILLISECONDS); } writer = new TrackingIndexWriter(delegateWriter); searcherManager = new WrappableSearcherManager(writer.getIndexWriter(), true, searcherFactory); notDoneNrtFutures = Sets.newConcurrentHashSet(); reopenThread = new ControlledRealTimeReopenThread<>(writer, searcherManager, 0.500 /* maximum stale age (seconds) */, 0.010 /* minimum stale age (seconds) */); reopenThread.setName("NRT " + dirName); reopenThread.setPriority(Math.min(Thread.currentThread().getPriority() + 2, Thread.MAX_PRIORITY)); reopenThread.setDaemon(true); // This must be added after the reopen thread is created. The reopen thread // adds its own listener which copies its internally last-refreshed // generation to the searching generation. removeIfDone() depends on the // searching generation being up to date when calling // reopenThread.waitForGeneration(gen, 0), therefore the reopen thread's // internal listener needs to be called first. // TODO(dborowitz): This may have been fixed by // http://issues.apache.org/jira/browse/LUCENE-5461 searcherManager.addListener(new RefreshListener() { @Override public void beforeRefresh() throws IOException { } @Override public void afterRefresh(boolean didRefresh) throws IOException { for (NrtFuture f : notDoneNrtFutures) { f.removeIfDone(); } } }); reopenThread.start(); }
From source file:com.lithium.flow.filer.lucene.LuceneFiler.java
License:Apache License
public LuceneFiler(@Nonnull Filer delegate, @Nonnull Config config) throws IOException { super(delegate); String path = config.getString("index.path"); maxAge = config.getTime("index.maxAge", "-1"); double maxMergeMb = config.getDouble("index.maxMergeMb", 4); double maxCachedMb = config.getDouble("index.maxCacheMb", 64); long targetMaxStale = config.getTime("index.targetMaxStale", "5s"); long targetMinStale = config.getTime("index.targetMinStale", "1s"); Version version = Version.LATEST;/* ww w. j a v a2 s . c om*/ Directory dir = FSDirectory.open(new File(path)); NRTCachingDirectory cachingDir = new NRTCachingDirectory(dir, maxMergeMb, maxCachedMb); IndexWriterConfig writerConfig = new IndexWriterConfig(version, null); writerConfig.setOpenMode(OpenMode.CREATE_OR_APPEND); writer = new TrackingIndexWriter(new IndexWriter(cachingDir, writerConfig)); manager = new SearcherManager(writer.getIndexWriter(), true, new SearcherFactory()); thread = new ControlledRealTimeReopenThread<>(writer, manager, targetMaxStale, targetMinStale); thread.start(); }
From source file:com.orientechnologies.lucene.engine.OLuceneIndexEngineAbstract.java
License:Apache License
private void open(final ODocument metadata) throws IOException { OLuceneDirectoryFactory directoryFactory = new OLuceneDirectoryFactory(); directory = directoryFactory.createDirectory(getDatabase(), name, metadata); final IndexWriter indexWriter = createIndexWriter(directory); mgrWriter = new TrackingIndexWriter(indexWriter); searcherManager = new SearcherManager(indexWriter, true, null); reopenToken = 0;/*from w ww . ja v a2 s . c o m*/ nrt = new ControlledRealTimeReopenThread(mgrWriter, searcherManager, 60.00, 0.1); nrt.setDaemon(true); nrt.start(); closed.set(false); flush(); }
From source file:com.orientechnologies.lucene.engine.OLuceneStorage.java
License:Apache License
private void reOpen() throws IOException { if (mgrWriter != null) { OLogManager.instance().info(this, "index storage is open don't reopen"); return;/* w w w.j av a 2s. c o m*/ } ODatabaseDocumentInternal database = ODatabaseRecordThreadLocal.INSTANCE.get(); final OAbstractPaginatedStorage storageLocalAbstract = (OAbstractPaginatedStorage) database.getStorage() .getUnderlying(); Directory dir = null; if (storageLocalAbstract instanceof OLocalPaginatedStorage) { String pathname = getIndexPath((OLocalPaginatedStorage) storageLocalAbstract); OLogManager.instance().info(this, "Opening NIOFS Lucene db=%s, path=%s", database.getName(), pathname); dir = NIOFSDirectory.open(new File(pathname).toPath()); } else { OLogManager.instance().info(this, "Opening RAM Lucene index db=%s", database.getName()); dir = new RAMDirectory(); } final IndexWriter indexWriter = createIndexWriter(dir); mgrWriter = new TrackingIndexWriter(indexWriter); searcherManager = new SearcherManager(indexWriter, true, null); if (nrt != null) { nrt.close(); } nrt = new ControlledRealTimeReopenThread(mgrWriter, searcherManager, 60.00, 0.1); nrt.setDaemon(true); nrt.start(); flush(); OLogManager.instance().info(this, "REOPEN DONE"); }
From source file:com.orientechnologies.lucene.manager.OLuceneIndexManagerAbstract.java
License:Apache License
private void reOpen(final ODocument metadata) throws IOException { ODatabaseDocumentInternal database = getDatabase(); final OAbstractPaginatedStorage storageLocalAbstract = (OAbstractPaginatedStorage) database.getStorage() .getUnderlying();/*from ww w . jav a 2 s . c o m*/ Directory dir = null; if (storageLocalAbstract instanceof OLocalPaginatedStorage) { String pathname = getIndexPath((OLocalPaginatedStorage) storageLocalAbstract); OLogManager.instance().debug(this, "Opening NIOFS Lucene db=%s, path=%s", database.getName(), pathname); dir = NIOFSDirectory.open(new File(pathname)); } else { OLogManager.instance().debug(this, "Opening RAM Lucene index db=%s", database.getName()); dir = new RAMDirectory(); } final IndexWriter indexWriter = createIndexWriter(dir, metadata); mgrWriter = new TrackingIndexWriter(indexWriter); searcherManager = new SearcherManager(indexWriter, true, null); if (nrt != null) { nrt.close(); } nrt = new ControlledRealTimeReopenThread(mgrWriter, searcherManager, 60.00, 0.1); nrt.setDaemon(true); nrt.start(); flush(); }
From source file:com.stratio.cassandra.index.LuceneIndex.java
License:Apache License
/** * Initializes this using the specified {@link Sort} for trying to keep the {@link Document}s sorted. * * @param sort The {@link Sort} to be used. *///from w w w . j a v a 2 s . co m public void init(Sort sort) { Log.debug("Initializing index"); try { this.sort = sort; // Get directory file file = new File(path); // Open or create directory FSDirectory fsDirectory = FSDirectory.open(file); directory = new NRTCachingDirectory(fsDirectory, maxMergeMB, maxCachedMB); // Setup index writer IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_48, analyzer); config.setRAMBufferSizeMB(ramBufferMB); config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); config.setUseCompoundFile(true); config.setMergePolicy(new SortingMergePolicy(config.getMergePolicy(), sort)); indexWriter = new IndexWriter(directory, config); // Setup NRT search SearcherFactory searcherFactory = new SearcherFactory() { public IndexSearcher newSearcher(IndexReader reader) throws IOException { IndexSearcher searcher = new IndexSearcher(reader); searcher.setSimilarity(new NoIDFSimilarity()); return searcher; } }; TrackingIndexWriter trackingIndexWriter = new TrackingIndexWriter(indexWriter); searcherManager = new SearcherManager(indexWriter, true, searcherFactory); searcherReopener = new ControlledRealTimeReopenThread<>(trackingIndexWriter, searcherManager, refreshSeconds, refreshSeconds); searcherReopener.start(); // Start the refresher thread } catch (IOException e) { Log.error(e, "Error while initializing index"); throw new RuntimeException(e); } }
From source file:com.stratio.cassandra.lucene.index.FSIndex.java
License:Apache License
/** * Builds a new {@link FSIndex}./*from www. j a va 2s. c o m*/ * * @param name the index name * @param mbeanName the JMX MBean object name * @param path the directory path * @param analyzer the index writer analyzer * @param refresh the index reader refresh frequency in seconds * @param ramBufferMB the index writer RAM buffer size in MB * @param maxMergeMB the directory max merge size in MB * @param maxCachedMB the directory max cache size in MB * @param refreshTask action to be done during refresh */ public FSIndex(String name, String mbeanName, Path path, Analyzer analyzer, double refresh, int ramBufferMB, int maxMergeMB, int maxCachedMB, Runnable refreshTask) { try { this.path = path; this.name = name; // Open or create directory FSDirectory fsDirectory = FSDirectory.open(path); directory = new NRTCachingDirectory(fsDirectory, maxMergeMB, maxCachedMB); // Setup index writer IndexWriterConfig indexWriterConfig = new IndexWriterConfig(analyzer); indexWriterConfig.setRAMBufferSizeMB(ramBufferMB); indexWriterConfig.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); indexWriterConfig.setUseCompoundFile(true); indexWriterConfig.setMergePolicy(new TieredMergePolicy()); indexWriter = new IndexWriter(directory, indexWriterConfig); // Setup NRT search SearcherFactory searcherFactory = new SearcherFactory() { @Override public IndexSearcher newSearcher(IndexReader reader, IndexReader previousReader) { if (refreshTask != null) { refreshTask.run(); } IndexSearcher searcher = new IndexSearcher(reader); searcher.setSimilarity(new NoIDFSimilarity()); return searcher; } }; TrackingIndexWriter trackingWriter = new TrackingIndexWriter(indexWriter); searcherManager = new SearcherManager(indexWriter, true, searcherFactory); searcherReopener = new ControlledRealTimeReopenThread<>(trackingWriter, searcherManager, refresh, refresh); searcherReopener.start(); // Register JMX MBean mbean = new ObjectName(mbeanName); ManagementFactory.getPlatformMBeanServer().registerMBean(this, this.mbean); } catch (Exception e) { throw new IndexException(logger, e, "Error while creating index %s", name); } }
From source file:com.stratio.cassandra.lucene.service.LuceneIndex.java
License:Apache License
/** * Builds a new {@code RowDirectory} using the specified directory path and analyzer. * * @param keyspace The keyspace name. * @param table The table name. * @param name The index name. * @param path The path of the directory in where the Lucene files will be stored. * @param ramBufferMB The index writer buffer size in MB. * @param maxMergeMB NRTCachingDirectory max merge size in MB. * @param maxCachedMB NRTCachingDirectory max cached MB. * @param analyzer The default {@link Analyzer}. * @param refreshSeconds The index readers refresh time in seconds. Writings are not visible until this time. * @param refreshCallback A runnable to be run on index refresh. * @throws IOException If Lucene throws IO errors. *//*from ww w. j a va2s . c o m*/ public LuceneIndex(String keyspace, String table, String name, Path path, Integer ramBufferMB, Integer maxMergeMB, Integer maxCachedMB, Analyzer analyzer, Double refreshSeconds, Runnable refreshCallback) throws IOException { this.path = path; this.refreshCallback = refreshCallback; this.logName = String.format("Lucene index %s.%s.%s", keyspace, table, name); // Open or create directory FSDirectory fsDirectory = FSDirectory.open(path); directory = new NRTCachingDirectory(fsDirectory, maxMergeMB, maxCachedMB); // Setup index writer IndexWriterConfig config = new IndexWriterConfig(analyzer); config.setRAMBufferSizeMB(ramBufferMB); config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); config.setUseCompoundFile(true); config.setMergePolicy(new TieredMergePolicy()); indexWriter = new IndexWriter(directory, config); // Setup NRT search SearcherFactory searcherFactory = new SearcherFactory() { public IndexSearcher newSearcher(IndexReader reader) throws IOException { LuceneIndex.this.refreshCallBack(); IndexSearcher searcher = new IndexSearcher(reader); searcher.setSimilarity(new NoIDFSimilarity()); return searcher; } }; TrackingIndexWriter trackingIndexWriter = new TrackingIndexWriter(indexWriter); searcherManager = new SearcherManager(indexWriter, true, searcherFactory); searcherReopener = new ControlledRealTimeReopenThread<>(trackingIndexWriter, searcherManager, refreshSeconds, refreshSeconds); searcherReopener.start(); // Start the refresher thread // Register JMX MBean try { objectName = new ObjectName( String.format("com.stratio.cassandra.lucene:type=LuceneIndexes,keyspace=%s,table=%s,index=%s", keyspace, table, name)); ManagementFactory.getPlatformMBeanServer().registerMBean(this, objectName); } catch (MBeanException | OperationsException e) { Log.error(e, "Error while registering MBean"); } }
From source file:com.tuplejump.stargate.lucene.NearRealTimeIndexer.java
License:Apache License
private void init(Analyzer analyzer, String keyspaceName, String cfName, String indexName, String vNodeName) throws IOException { this.indexName = indexName; this.keyspaceName = keyspaceName; this.cfName = cfName; this.analyzer = analyzer; this.vNodeName = vNodeName; logger.debug(indexName + " Lucene analyzer -" + analyzer); logger.debug(indexName + " Lucene version -" + Properties.luceneVersion); IndexWriter delegate = getIndexWriter(Properties.luceneVersion); indexWriter = new TrackingIndexWriter(delegate); indexSearcherReferenceManager = new SearcherManager(delegate, true, null); reopenThread = new ControlledRealTimeReopenThread<>(indexWriter, indexSearcherReferenceManager, 1, 0.01); startReopenThread();//from w ww . ja v a2s .co m }