Java tutorial
/* * Licensed to STRATIO (C) under one or more contributor license agreements. * See the NOTICE file distributed with this work for additional information * regarding copyright ownership. The STRATIO (C) licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package com.stratio.cassandra.lucene.index; import com.stratio.cassandra.lucene.IndexException; import org.apache.cassandra.io.util.FileUtils; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Document; import org.apache.lucene.index.*; import org.apache.lucene.search.*; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.NRTCachingDirectory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.management.ObjectName; import java.lang.management.ManagementFactory; import java.nio.file.Path; import java.util.Set; /** * Class wrapping a Lucene file system-based directory and its readers, writers and searchers. * * @author Andres de la Pena {@literal <adelapena@stratio.com>} */ public class FSIndex implements FSIndexMBean { private static final Logger logger = LoggerFactory.getLogger(FSIndex.class); private final Path path; private final String name; private final Directory directory; private final IndexWriter indexWriter; private final SearcherManager searcherManager; private final ControlledRealTimeReopenThread<IndexSearcher> searcherReopener; private final ObjectName mbean; // Disable max boolean query clauses limit static { BooleanQuery.setMaxClauseCount(Integer.MAX_VALUE); } /** * Builds a new {@link FSIndex}. * * @param name the index name * @param mbeanName the JMX MBean object name * @param path the directory path * @param analyzer the index writer analyzer * @param refresh the index reader refresh frequency in seconds * @param ramBufferMB the index writer RAM buffer size in MB * @param maxMergeMB the directory max merge size in MB * @param maxCachedMB the directory max cache size in MB * @param refreshTask action to be done during refresh */ public FSIndex(String name, String mbeanName, Path path, Analyzer analyzer, double refresh, int ramBufferMB, int maxMergeMB, int maxCachedMB, Runnable refreshTask) { try { this.path = path; this.name = name; // Open or create directory FSDirectory fsDirectory = FSDirectory.open(path); directory = new NRTCachingDirectory(fsDirectory, maxMergeMB, maxCachedMB); // Setup index writer IndexWriterConfig indexWriterConfig = new IndexWriterConfig(analyzer); indexWriterConfig.setRAMBufferSizeMB(ramBufferMB); indexWriterConfig.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); indexWriterConfig.setUseCompoundFile(true); indexWriterConfig.setMergePolicy(new TieredMergePolicy()); indexWriter = new IndexWriter(directory, indexWriterConfig); // Setup NRT search SearcherFactory searcherFactory = new SearcherFactory() { @Override public IndexSearcher newSearcher(IndexReader reader, IndexReader previousReader) { if (refreshTask != null) { refreshTask.run(); } IndexSearcher searcher = new IndexSearcher(reader); searcher.setSimilarity(new NoIDFSimilarity()); return searcher; } }; TrackingIndexWriter trackingWriter = new TrackingIndexWriter(indexWriter); searcherManager = new SearcherManager(indexWriter, true, searcherFactory); searcherReopener = new ControlledRealTimeReopenThread<>(trackingWriter, searcherManager, refresh, refresh); searcherReopener.start(); // Register JMX MBean mbean = new ObjectName(mbeanName); ManagementFactory.getPlatformMBeanServer().registerMBean(this, this.mbean); } catch (Exception e) { throw new IndexException(logger, e, "Error while creating index %s", name); } } /** * Upserts the specified {@link Document} by first deleting the documents containing {@code Term} and then adding * the new document. The delete and then add are atomic as seen by a reader on the same index (flush may happen only * after the add). * * @param term the {@link Term} to identify the document(s) to be deleted * @param document the {@link Document} to be added */ public void upsert(Term term, Document document) { logger.debug("Indexing {} with term {} in {}", document, term, name); try { indexWriter.updateDocument(term, document); } catch (Exception e) { throw new IndexException(logger, e, "Error indexing %s with term %s in %s", document, term, name); } } /** * Deletes all the {@link Document}s containing the specified {@link Term}. * * @param term the {@link Term} identifying the documents to be deleted */ public void delete(Term term) { logger.debug("Deleting {} from {}", term, name); try { indexWriter.deleteDocuments(term); } catch (Exception e) { throw new IndexException(logger, e, "Error deleting %s from %s", term, name); } } /** * Deletes all the {@link Document}s satisfying the specified {@link Query}. * * @param query the {@link Query} identifying the documents to be deleted */ public void delete(Query query) { logger.debug("Deleting {} from {}", query, name); try { indexWriter.deleteDocuments(query); } catch (Exception e) { throw new IndexException(logger, e, "Error deleting %s from %s", query, name); } } /** * Deletes all the {@link Document}s. */ public void truncate() { try { indexWriter.deleteAll(); } catch (Exception e) { throw new IndexException(logger, e, "Error truncating %s", name); } logger.info("Truncated {}", name); } /** * Commits the pending changes. */ @Override public void commit() { try { indexWriter.commit(); } catch (Exception e) { throw new IndexException(logger, e, "Error committing %s", name); } logger.debug("Committed {}", name); } /** * Commits all changes to the index, waits for pending merges to complete, and closes all associated resources. */ public void close() { try { searcherReopener.interrupt(); searcherManager.close(); indexWriter.close(); directory.close(); ManagementFactory.getPlatformMBeanServer().unregisterMBean(mbean); } catch (Exception e) { throw new IndexException(logger, e, "Error closing %s", name); } logger.info("Closed {}", name); } /** * Closes the index and removes all its files. */ public void delete() { try { close(); } catch (Exception e) { throw new IndexException(logger, e, "Error deleting %s", name); } finally { FileUtils.deleteRecursive(path.toFile()); } logger.info("Deleted {}", name); } /** * Finds the top {@code count} hits for {@code query} and sorting the hits by {@code sort}. * * @param query the {@link Query} to search for * @param sort the {@link Sort} to be applied * @param after the starting {@link ScoreDoc} * @param count the max number of results to be collected * @param fields the names of the fields to be loaded * @return the found documents, sorted according to the supplied {@link Sort} instance */ public DocumentIterator search(Query query, Sort sort, ScoreDoc after, Integer count, Set<String> fields) { logger.debug("Searching in {}\n" + "count: {}\n" + "after: {}\n" + "query: {}\n" + " sort: {}", name, count, after, query, sort); return new DocumentIterator(searcherManager, query, sort, after, count, fields); } /** * Returns the total number of {@link Document}s in this index. * * @return the number of {@link Document}s */ @Override public long getNumDocs() { logger.debug("Getting {} num docs", name); try { IndexSearcher searcher = searcherManager.acquire(); try { return searcher.getIndexReader().numDocs(); } finally { searcherManager.release(searcher); } } catch (Exception e) { throw new IndexException(logger, e, "Error getting %s num docs", name); } } /** * Returns the total number of deleted {@link Document}s in this index. * * @return the number of deleted {@link Document}s */ @Override public long getNumDeletedDocs() { logger.debug("Getting %s num deleted docs", name); try { IndexSearcher searcher = searcherManager.acquire(); try { return searcher.getIndexReader().numDeletedDocs(); } finally { searcherManager.release(searcher); } } catch (Exception e) { throw new IndexException(logger, e, "Error getting %s num docs", name); } } /** * Optimizes the index forcing merge segments leaving the specified number of segments. This operation may block * until all merging completes. * * @param maxNumSegments the maximum number of segments left in the index after merging finishes * @param doWait {@code true} if the call should block until the operation completes */ @Override public void forceMerge(int maxNumSegments, boolean doWait) { logger.info("Merging {} segments to {}", name, maxNumSegments); try { indexWriter.forceMerge(maxNumSegments, doWait); indexWriter.commit(); } catch (Exception e) { throw new IndexException(logger, e, "Error merging %s segments to %s", name, maxNumSegments); } logger.info("Merged {} segments to {}", name, maxNumSegments); } /** * Optimizes the index forcing merge of all segments that have deleted documents. This operation may block until all * merging completes. * * @param doWait {@code true} if the call should block until the operation completes */ @Override public void forceMergeDeletes(boolean doWait) { logger.info("Merging {} segments with deletions", name); try { indexWriter.forceMergeDeletes(doWait); indexWriter.commit(); } catch (Exception e) { throw new IndexException(logger, e, "Error merging %s segments with deletion", name); } logger.info("Merged {} segments with deletions", name); } /** * Refreshes the index readers. */ @Override public void refresh() { logger.debug("Refreshing {} readers", name); try { commit(); searcherManager.maybeRefreshBlocking(); } catch (Exception e) { throw new IndexException(logger, e, "Error refreshing %s readers", name); } logger.debug("Refreshed {} readers", name); } }