Example usage for org.apache.lucene.index TieredMergePolicy TieredMergePolicy

List of usage examples for org.apache.lucene.index TieredMergePolicy TieredMergePolicy

  1. HOME
  2. Java
  3. org.apache.lucene
  4. org.apache.lucene.index.*
  5. TieredMergePolicy
  6. TieredMergePolicy

Introduction

In this page you can find the example usage for org.apache.lucene.index TieredMergePolicy TieredMergePolicy.

Prototype

public TieredMergePolicy()

Source Link

Document

Sole constructor, setting all settings to their defaults.

Usage

From source file:com.qwazr.search.bench.LuceneCommonIndex.java

License:Apache License

LuceneCommonIndex(final Path rootDirectory, final String schemaName, final String indexName,
        final double ramBufferSize, final boolean useCompoundFile) throws IOException {

    final Path schemaDirectory = Files.createDirectory(rootDirectory.resolve(schemaName));
    this.indexDirectory = Files.createDirectory(schemaDirectory.resolve(indexName));
    this.luceneDirectory = indexDirectory.resolve("data");
    this.dataDirectory = FSDirectory.open(luceneDirectory);
    final IndexWriterConfig indexWriterConfig = new IndexWriterConfig(
            new PerFieldAnalyzerWrapper(new StandardAnalyzer()));
    indexWriterConfig.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
    indexWriterConfig.setRAMBufferSizeMB(ramBufferSize);

    final ConcurrentMergeScheduler mergeScheduler = new ConcurrentMergeScheduler();
    mergeScheduler.setMaxMergesAndThreads(MAX_SSD_MERGE_THREADS, MAX_SSD_MERGE_THREADS);
    indexWriterConfig.setMergeScheduler(mergeScheduler);
    indexWriterConfig.setUseCompoundFile(useCompoundFile);

    final TieredMergePolicy mergePolicy = new TieredMergePolicy();
    indexWriterConfig.setMergePolicy(mergePolicy);

    // We use snapshots deletion policy
    final SnapshotDeletionPolicy snapshotDeletionPolicy = new SnapshotDeletionPolicy(
            indexWriterConfig.getIndexDeletionPolicy());
    indexWriterConfig.setIndexDeletionPolicy(snapshotDeletionPolicy);

    this.indexWriter = new IndexWriter(this.dataDirectory, indexWriterConfig);
    this.localReplicator = new LocalReplicator();
}

From source file:com.stratio.cassandra.lucene.index.FSIndex.java

License:Apache License

/**
 * Builds a new {@link FSIndex}.//from   ww  w .  jav  a  2s  .  c  o m
 *
 * @param name the index name
 * @param mbeanName the JMX MBean object name
 * @param path the directory path
 * @param analyzer the index writer analyzer
 * @param refresh the index reader refresh frequency in seconds
 * @param ramBufferMB the index writer RAM buffer size in MB
 * @param maxMergeMB the directory max merge size in MB
 * @param maxCachedMB the directory max cache size in MB
 * @param refreshTask action to be done during refresh
 */
public FSIndex(String name, String mbeanName, Path path, Analyzer analyzer, double refresh, int ramBufferMB,
        int maxMergeMB, int maxCachedMB, Runnable refreshTask) {
    try {
        this.path = path;
        this.name = name;

        // Open or create directory
        FSDirectory fsDirectory = FSDirectory.open(path);
        directory = new NRTCachingDirectory(fsDirectory, maxMergeMB, maxCachedMB);

        // Setup index writer
        IndexWriterConfig indexWriterConfig = new IndexWriterConfig(analyzer);
        indexWriterConfig.setRAMBufferSizeMB(ramBufferMB);
        indexWriterConfig.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
        indexWriterConfig.setUseCompoundFile(true);
        indexWriterConfig.setMergePolicy(new TieredMergePolicy());
        indexWriter = new IndexWriter(directory, indexWriterConfig);

        // Setup NRT search
        SearcherFactory searcherFactory = new SearcherFactory() {
            @Override
            public IndexSearcher newSearcher(IndexReader reader, IndexReader previousReader) {
                if (refreshTask != null) {
                    refreshTask.run();
                }
                IndexSearcher searcher = new IndexSearcher(reader);
                searcher.setSimilarity(new NoIDFSimilarity());
                return searcher;
            }
        };
        TrackingIndexWriter trackingWriter = new TrackingIndexWriter(indexWriter);
        searcherManager = new SearcherManager(indexWriter, true, searcherFactory);
        searcherReopener = new ControlledRealTimeReopenThread<>(trackingWriter, searcherManager, refresh,
                refresh);
        searcherReopener.start();

        // Register JMX MBean
        mbean = new ObjectName(mbeanName);
        ManagementFactory.getPlatformMBeanServer().registerMBean(this, this.mbean);

    } catch (Exception e) {
        throw new IndexException(logger, e, "Error while creating index %s", name);
    }
}

From source file:com.stratio.cassandra.lucene.service.LuceneIndex.java

License:Apache License

/**
 * Builds a new {@code RowDirectory} using the specified directory path and analyzer.
 *
 * @param keyspace        The keyspace name.
 * @param table           The table name.
 * @param name            The index name.
 * @param path            The path of the directory in where the Lucene files will be stored.
 * @param ramBufferMB     The index writer buffer size in MB.
 * @param maxMergeMB      NRTCachingDirectory max merge size in MB.
 * @param maxCachedMB     NRTCachingDirectory max cached MB.
 * @param analyzer        The default {@link Analyzer}.
 * @param refreshSeconds  The index readers refresh time in seconds. Writings are not visible until this time.
 * @param refreshCallback A runnable to be run on index refresh.
 * @throws IOException If Lucene throws IO errors.
 *///from www.j  a v  a  2 s . c om
public LuceneIndex(String keyspace, String table, String name, Path path, Integer ramBufferMB,
        Integer maxMergeMB, Integer maxCachedMB, Analyzer analyzer, Double refreshSeconds,
        Runnable refreshCallback) throws IOException {
    this.path = path;
    this.refreshCallback = refreshCallback;
    this.logName = String.format("Lucene index %s.%s.%s", keyspace, table, name);

    // Open or create directory
    FSDirectory fsDirectory = FSDirectory.open(path);
    directory = new NRTCachingDirectory(fsDirectory, maxMergeMB, maxCachedMB);

    // Setup index writer
    IndexWriterConfig config = new IndexWriterConfig(analyzer);
    config.setRAMBufferSizeMB(ramBufferMB);
    config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
    config.setUseCompoundFile(true);
    config.setMergePolicy(new TieredMergePolicy());
    indexWriter = new IndexWriter(directory, config);

    // Setup NRT search
    SearcherFactory searcherFactory = new SearcherFactory() {
        public IndexSearcher newSearcher(IndexReader reader) throws IOException {
            LuceneIndex.this.refreshCallBack();
            IndexSearcher searcher = new IndexSearcher(reader);
            searcher.setSimilarity(new NoIDFSimilarity());
            return searcher;
        }
    };
    TrackingIndexWriter trackingIndexWriter = new TrackingIndexWriter(indexWriter);
    searcherManager = new SearcherManager(indexWriter, true, searcherFactory);
    searcherReopener = new ControlledRealTimeReopenThread<>(trackingIndexWriter, searcherManager,
            refreshSeconds, refreshSeconds);
    searcherReopener.start(); // Start the refresher thread

    // Register JMX MBean
    try {
        objectName = new ObjectName(
                String.format("com.stratio.cassandra.lucene:type=LuceneIndexes,keyspace=%s,table=%s,index=%s",
                        keyspace, table, name));
        ManagementFactory.getPlatformMBeanServer().registerMBean(this, objectName);
    } catch (MBeanException | OperationsException e) {
        Log.error(e, "Error while registering MBean");
    }
}

From source file:com.xiaomi.linden.lucene.merge.SortingMergePolicyFactory.java

License:Apache License

@Override
public MergePolicy getInstance(Map<String, String> params) throws IOException {
    String field = params.get(SORT_FIELD);
    SortField.Type sortFieldType = SortField.Type.DOC;
    if (params.containsKey(SORT_FIELD_TYPE)) {
        sortFieldType = SortField.Type.valueOf(params.get(SORT_FIELD_TYPE).toUpperCase());
    }//from   w  w w  . jav  a2 s  . c om

    if (sortFieldType == SortField.Type.DOC) {
        throw new IOException(
                "Relying on internal lucene DocIDs is not guaranteed to work, this is only an implementation detail.");
    }

    boolean desc = true;
    if (params.containsKey(SORT_DESC)) {
        try {
            desc = Boolean.valueOf(params.get(SORT_DESC));
        } catch (Exception e) {
            desc = true;
        }
    }
    SortField sortField = new SortField(field, sortFieldType, desc);
    Sort sort = new Sort(sortField);
    return new SortingMergePolicyDecorator(new TieredMergePolicy(), sort);
}

From source file:com.xiaomi.linden.lucene.merge.TieredMergePolicyFactory.java

License:Apache License

@Override
public MergePolicy getInstance(Map<String, String> config) throws IOException {
    TieredMergePolicy mergePolicy = new TieredMergePolicy();

    if (config.containsKey(SEGMENTS_PER_TIER)) {
        mergePolicy.setSegmentsPerTier(Double.valueOf(config.get(SEGMENTS_PER_TIER)));
    }/*from  w  ww.j a va2  s  .com*/
    if (config.containsKey(MAX_MERGE_AT_ONCE)) {
        mergePolicy.setMaxMergeAtOnce(Integer.valueOf(config.get(MAX_MERGE_AT_ONCE)));
    }
    return mergePolicy;
}

From source file:dk.dbc.opensearch.fedora.search.FieldSearchLucene.java

License:Open Source License

@Override
public void postInitModule() throws ModuleInitializationException {
    log.trace("Initializing module with parameters");

    // Version:/*from   w ww  .j a  v  a2 s.  c  o  m*/
    String version = this.getClass().getPackage().getImplementationVersion();
    log.info("Running FieldSearchLucene version {}", version);

    // doManager
    doManager = (DOManager) getServer().getModule("org.fcrepo.server.storage.DOManager");
    if (null == doManager) {
        String error = "FATAL: DOManager module was required, but apparently has not been loaded.";
        log.error(error);
        throw new ModuleInitializationException(error, getRole());
    }
    log.trace("Retrieved DOManager instance");

    // resultLifeTimeInSeconds
    String resultLifetime = getParameter("resultLifetime");
    if (resultLifetime.equals("")) {
        String error = "FATAL: parameter resultLifetime must be specified.";
        log.error(error);
        throw new ModuleInitializationException(error, getRole());
    }
    this.resultLifeTimeInSeconds = Integer.parseInt(resultLifetime);
    log.debug("resultLifeTimeInSeconds = {}", resultLifetime);

    // luceneWriteLockTimeout
    String writeLockTimeout = getParameter("writeLockTimeout");
    if (writeLockTimeout.equals("")) {
        String error = "FATAL: parameter writeLockTimeout must be specified.";
        log.error(error);
        throw new ModuleInitializationException(error, getRole());
    }
    long luceneWriteLockTimeout = Long.parseLong(writeLockTimeout);
    log.debug("luceneWritelockTimeout = {}", writeLockTimeout);

    // maxThreadStates
    String maxThreadStates = getParameter("maxThreadStates");
    if (maxThreadStates.equals("")) {
        String error = "FATAL: parameter maxThreadStates must be specified.";
        log.error(error);
        throw new ModuleInitializationException(error, getRole());
    }
    int luceneMaxThreadStates = Integer.parseInt(maxThreadStates);
    log.debug("luceneMaxThreadStates = {}", luceneMaxThreadStates);

    // directory
    String sDirectory = getParameter("luceneDirectory");
    if (sDirectory.equals("")) {
        String error = "FATAL: parameter luceneDirectory must be specified.";
        log.error(error);
        throw new ModuleInitializationException(error, getRole());
    }
    Directory directory;
    try {
        directory = initializeDirectoryString(sDirectory);
    } catch (IOException ex) {
        String error = String.format("FATAL: Could not initialize lucene directory '%s': %s", sDirectory,
                ex.getMessage());
        log.error(error);
        throw new ModuleInitializationException(error, getRole(), ex);
    }
    log.debug("LuceneDirectory: {}", sDirectory);

    // PidCollector
    initializePidCollectorSettings();

    // Write Ahead Log
    initializeWriteAheadLogSettings();

    // luceneindexer
    Analyzer analyzer = new WhitespaceAnalyzer(Version.LUCENE_41);

    TieredMergePolicy mergePolicy = new TieredMergePolicy();
    try {
        IndexWriter writer = createIndexWriter(luceneWriteLockTimeout, luceneMaxThreadStates, analyzer,
                directory, new TieredMergePolicy());

        WriteAheadLog wal = (writeAheadLogDir == null) ? null
                : new WriteAheadLog(writer, writeAheadLogDir, writeAheadLogCommitSize,
                        writeAheadLogKeepFileOpen);

        this.luceneindexer = new LuceneFieldIndex(writer, mergePolicy, pidCollectorMaxInMemory,
                pidCollectorTmpDir, wal);
        log.trace("Constructed LuceneIndex instance");
    } catch (IOException ex) {
        String error = "FATAL: Could not initialize lucene indexer";
        log.error(error);
        throw new ModuleInitializationException(error, getRole(), ex);
    }

    fsl = new FieldSearchLuceneImpl(luceneindexer);
    cache = new FieldSearchResultCache(resultLifeTimeInSeconds);
    cache.start();
}

From source file:dk.dbc.opensearch.fedora.search.FieldSearchResultLuceneTest.java

License:Open Source License

@Before
public void setUp() throws Exception {
    TieredMergePolicy tieredMergePolicy = new TieredMergePolicy();
    IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_41, new SimpleAnalyzer(Version.LUCENE_41))
            .setWriteLockTimeout(1000L).setMergePolicy(tieredMergePolicy);
    IndexWriter writer = new IndexWriter(new RAMDirectory(), conf);

    indexer = new LuceneFieldIndex(writer, tieredMergePolicy, PID_COLLECTOR_MAX_IN_MEMORY,
            PID_COLLECTOR_TMP_DIR, null);
}

From source file:dk.dbc.opensearch.fedora.search.LuceneFieldIndexTest.java

License:Open Source License

@Before
public void setup() throws Exception {
    fsdir = FSDirectory.open(new File(indexLocation));
    TieredMergePolicy tieredMergePolicy = new TieredMergePolicy();
    IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_41, new WhitespaceAnalyzer(Version.LUCENE_41))
            .setWriteLockTimeout(1000L).setMergePolicy(tieredMergePolicy);
    IndexWriter writer = new IndexWriter(fsdir, conf);

    instance = new LuceneFieldIndex(writer, tieredMergePolicy, PID_COLLECTOR_MAX_IN_MEMORY,
            PID_COLLECTOR_TMP_DIR, null);
}

From source file:dk.dbc.opensearch.fedora.search.WriteAheadLogTest.java

License:Open Source License

@Before
public void setUp() throws IOException {
    TieredMergePolicy tieredMergePolicy = new TieredMergePolicy();
    IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_41, new SimpleAnalyzer(Version.LUCENE_41))
            .setWriteLockTimeout(1000L).setMergePolicy(tieredMergePolicy);

    writer = new IndexWriter(new RAMDirectory(), conf);
}

From source file:io.puntanegra.fhir.index.lucene.LuceneService.java

License:Apache License

/**
 * Builds a new {@link FSIndex}./*w w  w  .  j a v  a 2  s.  com*/
 *
 * @param name
 *            the index name
 * @param mbeanName
 *            the JMX MBean object name
 * @param path
 *            the directory path
 * @param analyzer
 *            the index writer analyzer
 * @param refresh
 *            the index reader refresh frequency in seconds
 * @param ramBufferMB
 *            the index writer RAM buffer size in MB
 * @param maxMergeMB
 *            the directory max merge size in MB
 * @param maxCachedMB
 *            the directory max cache size in MB
 * @param refreshTask
 *            action to be done during refresh
 */
public void init(String name, String mbeanName, Path path, Analyzer analyzer, double refresh, int ramBufferMB,
        int maxMergeMB, int maxCachedMB, Runnable refreshTask) {
    try {

        this.path = path;
        this.name = name;

        // Open or create directory
        FSDirectory fsDirectory = FSDirectory.open(path);
        this.directory = new NRTCachingDirectory(fsDirectory, maxMergeMB, maxCachedMB);

        // Setup index writer
        IndexWriterConfig indexWriterConfig = new IndexWriterConfig(analyzer);
        indexWriterConfig.setRAMBufferSizeMB(ramBufferMB);
        indexWriterConfig.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
        indexWriterConfig.setUseCompoundFile(true);
        indexWriterConfig.setMergePolicy(new TieredMergePolicy());
        this.indexWriter = new IndexWriter(this.directory, indexWriterConfig);

        // Setup NRT search
        SearcherFactory searcherFactory = new SearcherFactory() {
            @Override
            public IndexSearcher newSearcher(IndexReader reader, IndexReader previousReader) {
                if (refreshTask != null) {
                    refreshTask.run();
                }
                IndexSearcher searcher = new IndexSearcher(reader);
                searcher.setSimilarity(new NoIDFSimilarity());
                return searcher;
            }
        };
        TrackingIndexWriter trackingWriter = new TrackingIndexWriter(this.indexWriter);
        this.searcherManager = new SearcherManager(this.indexWriter, true, searcherFactory);
        this.searcherReopener = new ControlledRealTimeReopenThread<>(trackingWriter, this.searcherManager,
                refresh, refresh);
        this.searcherReopener.start();

        // Register JMX MBean
        // mbean = new ObjectName(mbeanName);
        // ManagementFactory.getPlatformMBeanServer().registerMBean(service,
        // this.mbean);

    } catch (Exception e) {
        throw new FhirIndexException(e, "Error while creating index %s", name);
    }
}