Example usage for org.apache.lucene.index IndexWriter forceMerge

List of usage examples for org.apache.lucene.index IndexWriter forceMerge

  1. HOME
  2. Java
  3. org.apache.lucene
  4. org.apache.lucene.index.*
  5. IndexWriter
  6. forceMerge

Introduction

In this page you can find the example usage for org.apache.lucene.index IndexWriter forceMerge.

Prototype

public void forceMerge(int maxNumSegments, boolean doWait) throws IOException

Source Link

Document

Just like #forceMerge(int) , except you can specify whether the call should block until all merging completes.

Usage

From source file:collene.TestLuceneAssumptions.java

License:Apache License

@Test
public void listAfterEachStep() throws Exception {
    File fdir = TestUtil.getRandomTempDir();
    pleaseDelete.add(fdir);//from   w ww.  ja v  a 2 s  .c  o  m

    Directory dir = FSDirectory.open(fdir);
    Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_4_9);
    IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_4_9, analyzer);
    config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);

    //System.out.println("Before creating writer");
    dump(fdir, dir);

    IndexWriter writer = new IndexWriter(dir, config);
    //System.out.println("After creating writer");
    dump(fdir, dir);

    List<Document> docs = new ArrayList<Document>();
    for (int i = 0; i < 50000; i++) {
        Document doc = new Document();
        for (int f = 0; f < 5; f++) {
            doc.add(new Field("field_" + f, TestUtil.randomString(128), TextField.TYPE_STORED));
        }
        docs.add(doc);
    }
    writer.addDocuments(docs, analyzer);
    docs.clear();

    //System.out.println("After doc add 0");
    dump(fdir, dir);

    for (int i = 0; i < 50000; i++) {
        Document doc = new Document();
        for (int f = 0; f < 5; f++) {
            doc.add(new Field("field_" + f, TestUtil.randomString(128), TextField.TYPE_STORED));
        }
        docs.add(doc);
    }
    writer.addDocuments(docs, analyzer);
    docs.clear();

    //System.out.println("After doc add 1");
    dump(fdir, dir);

    writer.commit();

    //System.out.println("After commit");
    dump(fdir, dir);

    writer.forceMerge(1, true);
    //System.out.println("Right after merge");
    dump(fdir, dir);

    try {
        Thread.currentThread().sleep(5000);
    } catch (Exception ex) {
    }
    //System.out.println("After sleeping after merge");
    dump(fdir, dir);

    writer.close();
    //System.out.println("After writer close");
    dump(fdir, dir);

    dir.close();
    //System.out.println("After dir close");
    dump(fdir, dir);
}

From source file:fr.univ_tours.etu.searcher.LikeThisTest.java

public void writerEntries() throws IOException {
    IndexWriterConfig config = new IndexWriterConfig(analyzer).setOpenMode(IndexWriterConfig.OpenMode.CREATE);
    IndexWriter indexWriter = new IndexWriter(indexDir, config);

    Document doc1 = createDocument("1", "doduck", "prototype your idea");
    Document doc2 = createDocument("2", "doduck", "love programming");
    Document doc3 = createDocument("3", "We do", "prototype");
    Document doc4 = createDocument("4", "We love", "challange");
    indexWriter.addDocument(doc1);/*from w  w w  . j av a 2s.  c o m*/
    indexWriter.addDocument(doc2);
    indexWriter.addDocument(doc3);
    indexWriter.addDocument(doc4);

    indexWriter.commit();
    indexWriter.forceMerge(100, true);
    indexWriter.close();
}

From source file:gate.creole.ir.lucene.LuceneIndexManager.java

License:Open Source License

/** Optimize existing index. */
@Override/*from w  ww. j  a  v  a2  s.c  o  m*/
public void optimizeIndex() throws IndexException {
    if (indexDefinition == null)
        throw new GateRuntimeException("Index definition is null!");
    try {

        IndexWriter writer = new IndexWriter(FSDirectory.open(new File(indexDefinition.getIndexLocation())),
                new IndexWriterConfig(Version.LUCENE_31, new SimpleAnalyzer(Version.LUCENE_30))
                        .setOpenMode(OpenMode.APPEND));

        /*IndexWriter writer = new IndexWriter(
                FSDirectory.open(new File(indexDefinition.getIndexLocation())),
                new SimpleAnalyzer(Version.LUCENE_30), 
                false,
                new IndexWriter.MaxFieldLength(IndexWriter.DEFAULT_MAX_FIELD_LENGTH));*/

        //writer.optimize();
        writer.forceMerge(1, true);

        writer.commit();
        writer.close();
    } catch (java.io.IOException ioe) {
        throw new IndexException(ioe.getMessage());
    }
}

From source file:io.crate.execution.engine.collect.collectors.LuceneBatchIteratorBenchmark.java

License:Apache License

@Setup
public void createLuceneBatchIterator() throws Exception {
    IndexWriter iw = new IndexWriter(new RAMDirectory(), new IndexWriterConfig(new StandardAnalyzer()));
    String columnName = "x";
    for (int i = 0; i < 10_000_000; i++) {
        Document doc = new Document();
        doc.add(new NumericDocValuesField(columnName, i));
        iw.addDocument(doc);//  w  ww .  ja va  2s.co  m
    }
    iw.commit();
    iw.forceMerge(1, true);
    indexSearcher = new IndexSearcher(DirectoryReader.open(iw));
    IntegerColumnReference columnReference = new IntegerColumnReference(columnName);
    columnRefs = Collections.singletonList(columnReference);

    collectorContext = new CollectorContext(mappedFieldType -> null, new CollectorFieldsVisitor(0));
}

From source file:io.crate.execution.engine.collect.collectors.OrderedLuceneBatchIteratorBenchmark.java

License:Apache License

@Setup
public void createLuceneBatchIterator() throws Exception {
    IndexWriter iw = new IndexWriter(new RAMDirectory(), new IndexWriterConfig(new StandardAnalyzer()));
    dummyShardId = new ShardId("dummy", UUIDs.randomBase64UUID(), 1);
    columnName = "x";
    for (int i = 0; i < 10_000_000; i++) {
        Document doc = new Document();
        doc.add(new NumericDocValuesField(columnName, i));
        iw.addDocument(doc);/*w  ww  .  ja v a  2s  . c om*/
    }
    iw.commit();
    iw.forceMerge(1, true);
    indexSearcher = new IndexSearcher(DirectoryReader.open(iw, true, true));
    collectorContext = new CollectorContext(mappedFieldType -> null, new CollectorFieldsVisitor(0));
    reference = new Reference(
            new ReferenceIdent(new TableIdent(Schemas.DOC_SCHEMA_NAME, "dummyTable"), columnName),
            RowGranularity.DOC, DataTypes.INTEGER);
    orderBy = new OrderBy(Collections.singletonList(reference), reverseFlags, nullsFirst);
}

From source file:io.crate.operation.collect.collectors.LuceneBatchIteratorBenchmark.java

License:Apache License

@Setup
public void createLuceneBatchIterator() throws Exception {
    IndexWriter iw = new IndexWriter(new RAMDirectory(), new IndexWriterConfig(new StandardAnalyzer()));
    String columnName = "x";
    for (int i = 0; i < 10_000_000; i++) {
        Document doc = new Document();
        doc.add(new NumericDocValuesField(columnName, i));
        iw.addDocument(doc);/*from w  w  w.  j av  a 2s  .co  m*/
    }
    iw.commit();
    iw.forceMerge(1, true);
    indexSearcher = new IndexSearcher(DirectoryReader.open(iw));
    IntegerColumnReference columnReference = new IntegerColumnReference(columnName);
    columnRefs = Collections.singletonList(columnReference);

    collectorContext = new CollectorContext(mock(IndexFieldDataService.class), new CollectorFieldsVisitor(0));
}

From source file:io.crate.operation.collect.collectors.OrderedLuceneBatchIteratorBenchmark.java

License:Apache License

@Setup
public void createLuceneBatchIterator() throws Exception {
    IndexWriter iw = new IndexWriter(new RAMDirectory(), new IndexWriterConfig(new StandardAnalyzer()));
    dummyShardId = new ShardId("dummy", UUIDs.randomBase64UUID(), 1);
    columnName = "x";
    for (int i = 0; i < 10_000_000; i++) {
        Document doc = new Document();
        doc.add(new NumericDocValuesField(columnName, i));
        iw.addDocument(doc);//  w  w  w  .j a  va  2 s  . c o m
    }
    iw.commit();
    iw.forceMerge(1, true);
    indexSearcher = new IndexSearcher(DirectoryReader.open(iw, true, true));
    collectorContext = new CollectorContext(mock(IndexFieldDataService.class), new CollectorFieldsVisitor(0));
    fieldTypeLookup = column -> {
        NumberFieldMapper.NumberFieldType fieldType = new NumberFieldMapper.NumberFieldType(
                NumberFieldMapper.NumberType.INTEGER);
        fieldType.setName(column);
        return fieldType;
    };
    reference = new Reference(new ReferenceIdent(new TableIdent(null, "dummyTable"), columnName),
            RowGranularity.DOC, DataTypes.INTEGER);
    orderBy = new OrderBy(Collections.singletonList(reference), reverseFlags, nullsFirst);
}

From source file:org.apache.jackrabbit.core.query.lucene.AbstractIndex.java

License:Apache License

/**
 * Commits all pending changes to the underlying <code>Directory</code>.
 *
 * @param optimize if <code>true</code> the index is optimized after the
 *                 commit./*www. j ava2  s .co  m*/
 * @throws IOException if an error occurs while commiting changes.
 */
protected synchronized void commit(boolean optimize) throws IOException {
    if (indexReader != null) {
        log.debug("committing IndexReader.");
        indexReader.flush();
    }
    if (indexWriter != null) {
        log.debug("committing IndexWriter.");
        indexWriter.commit();
    }
    // optimize if requested
    if (optimize) {
        IndexWriter writer = getIndexWriter();
        writer.forceMerge(1, true);
        writer.close();
        indexWriter = null;
    }
}

From source file:org.elasticsearch.index.engine.internal.AsynchronousEngine.java

License:Apache License

@Override
public void optimize(Optimize optimize) throws EngineException {
    if (optimizeMutex.compareAndSet(false, true)) {
        try (InternalLock _ = readLock.acquire()) {
            final IndexWriter writer = currentIndexWriter();

            /*//from ww  w  .jav a  2 s.  com
             * The way we implement upgrades is a bit hackish in the sense that we set an instance
             * variable and that this setting will thus apply to the next forced merge that will be run.
             * This is ok because (1) this is the only place we call forceMerge, (2) we have a single
             * thread for optimize, and the 'optimizeMutex' guarding this code, and (3) ConcurrentMergeScheduler
             * syncs calls to findForcedMerges.
             */
            MergePolicy mp = writer.getConfig().getMergePolicy();
            assert mp instanceof ElasticsearchMergePolicy : "MergePolicy is " + mp.getClass().getName();
            if (optimize.upgrade()) {
                ((ElasticsearchMergePolicy) mp).setUpgradeInProgress(true);
            }

            if (optimize.onlyExpungeDeletes()) {
                writer.forceMergeDeletes(false);
            } else if (optimize.maxNumSegments() <= 0) {
                writer.maybeMerge();
                possibleMergeNeeded = false;
            } else {
                writer.forceMerge(optimize.maxNumSegments(), false);
            }
        } catch (Throwable t) {
            maybeFailEngine(t, "optimize");
            throw new OptimizeFailedEngineException(shardId, t);
        } finally {
            optimizeMutex.set(false);
        }
    }

    // wait for the merges outside of the read lock
    if (optimize.waitForMerge()) {
        waitForMerges(optimize.flush());
    } else if (optimize.flush()) {
        // we only need to monitor merges for async calls if we are going to flush
        threadPool.executor(ThreadPool.Names.OPTIMIZE).execute(new AbstractRunnable() {
            @Override
            public void run() {
                try {
                    waitForMerges(true);
                } catch (Exception e) {
                    logger.error("Exception while waiting for merges asynchronously after optimize", e);
                }
            }
        });
    }
}

From source file:org.elasticsearch.index.merge.Merges.java

License:Apache License

/**
 * See {@link org.apache.lucene.index.IndexWriter#forceMerge(int, boolean)}, with the additional
 * logic of explicitly enabling merges if the scheduler is {@link org.elasticsearch.index.merge.EnableMergeScheduler}.
 *//*from   ww  w.  j av a2s  .  c  o  m*/
public static void forceMerge(IndexWriter writer, int maxNumSegments, boolean doWait) throws IOException {
    MergeScheduler mergeScheduler = writer.getConfig().getMergeScheduler();
    if (mergeScheduler instanceof EnableMergeScheduler) {
        ((EnableMergeScheduler) mergeScheduler).enableMerge();
        try {
            writer.forceMerge(maxNumSegments, doWait);
        } finally {
            ((EnableMergeScheduler) mergeScheduler).disableMerge();
        }
    } else {
        writer.forceMerge(maxNumSegments, doWait);
    }
}