Example usage for org.apache.lucene.index IndexWriter commit

List of usage examples for org.apache.lucene.index IndexWriter commit

  1. HOME
  2. Java
  3. org.apache.lucene
  4. org.apache.lucene.index.*
  5. IndexWriter
  6. commit

Introduction

In this page you can find the example usage for org.apache.lucene.index IndexWriter commit.

Prototype

@Override
public final long commit() throws IOException

Source Link

Document

Commits all pending changes (added and deleted documents, segment merges, added indexes, etc.) to the index, and syncs all referenced index files, such that a reader will see the changes and the index updates will survive an OS or machine crash or power loss.

Usage

From source file:cn.hbu.cs.esearch.index.LuceneIndexDataLoader.java

License:Apache License

private final void purgeDocuments() {
    if (_purgeFilter != null) {
        BaseSearchIndex<R> idx = getSearchIndex();
        IndexWriter writer = null;
        LOGGER.info("purging docs started...");
        int count = 0;
        long start = System.currentTimeMillis();

        try {/*from   www.  ja  v  a2  s.c om*/
            writer = idx.openIndexWriter(null, null);
            ConstantScoreQuery q = new ConstantScoreQuery(_purgeFilter);
            writer.deleteDocuments(q);
            writer.commit();
        } catch (Throwable th) {
            LOGGER.error("problem creating purge filter: " + th.getMessage(), th);
        } finally {
            idx.closeIndexWriter();
        }

        long end = System.currentTimeMillis();
        LOGGER.info("purging docs completed in " + (end - start) + "ms");
        LOGGER.info("total docs purged: " + count);
    }
}

From source file:collene.TestIndexing.java

License:Apache License

@Test
public void test() throws IOException, ParseException {
    Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_4_9);

    // write it out.
    IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_4_9, analyzer);
    config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
    IndexWriter writer = new IndexWriter(directory, config);

    for (int i = 0; i < 100; i++) {
        Collection<Document> documents = new ArrayList<Document>();
        Document doc = new Document();
        doc.add(new Field("key", "aaa_" + i, TextField.TYPE_STORED));
        doc.add(new Field("not", "notaaa", TextField.TYPE_NOT_STORED));
        doc.add(new Field("meta", "aaa_meta_aaa_" + i, TextField.TYPE_STORED));
        documents.add(doc);//from ww  w .j a v a2  s.c  o m

        writer.addDocuments(documents);

        writer.commit();
        writer.forceMerge(1);
        writer.forceMergeDeletes(true);
    }

    // now read it back.
    IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, false));
    QueryParser parser = new QueryParser(Version.LUCENE_4_9, "key", analyzer);

    Query query = parser.parse("aaa_4");
    TopDocs docs = searcher.search(query, 1);
    int idToDelete = docs.scoreDocs[0].doc;
    Assert.assertTrue(docs.totalHits > 0);

    query = parser.parse("fersoius");
    docs = searcher.search(query, 1);
    Assert.assertFalse(docs.totalHits > 0);

    // delete that document.
    DirectoryReader reader = DirectoryReader.open(writer, true);
    writer.tryDeleteDocument(reader, idToDelete);

    reader.close();
    writer.close();

    // list files
    Set<String> files = new HashSet<String>();
    System.out.println("Listing files for " + directory.toString());
    for (String file : directory.listAll()) {
        files.add(file);
        System.out.println(" " + file);
    }

    if (strictFileChecking) {
        System.out.println("String file checking...");
        Sets.SetView<String> difference = Sets.difference(expectedFiles, files);
        Assert.assertEquals(Joiner.on(",").join(difference), 0, difference.size());
    }

    reader = DirectoryReader.open(directory);
    searcher = new IndexSearcher(reader);
    query = parser.parse("aaa_4");
    docs = searcher.search(query, 1);
    reader.close();
    Assert.assertFalse(docs.totalHits > 0);

    directory.close();
}

From source file:collene.TestLuceneAssumptions.java

License:Apache License

@Test
public void listAfterEachStep() throws Exception {
    File fdir = TestUtil.getRandomTempDir();
    pleaseDelete.add(fdir);//from  ww  w.j a  v a  2  s .co m

    Directory dir = FSDirectory.open(fdir);
    Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_4_9);
    IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_4_9, analyzer);
    config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);

    //System.out.println("Before creating writer");
    dump(fdir, dir);

    IndexWriter writer = new IndexWriter(dir, config);
    //System.out.println("After creating writer");
    dump(fdir, dir);

    List<Document> docs = new ArrayList<Document>();
    for (int i = 0; i < 50000; i++) {
        Document doc = new Document();
        for (int f = 0; f < 5; f++) {
            doc.add(new Field("field_" + f, TestUtil.randomString(128), TextField.TYPE_STORED));
        }
        docs.add(doc);
    }
    writer.addDocuments(docs, analyzer);
    docs.clear();

    //System.out.println("After doc add 0");
    dump(fdir, dir);

    for (int i = 0; i < 50000; i++) {
        Document doc = new Document();
        for (int f = 0; f < 5; f++) {
            doc.add(new Field("field_" + f, TestUtil.randomString(128), TextField.TYPE_STORED));
        }
        docs.add(doc);
    }
    writer.addDocuments(docs, analyzer);
    docs.clear();

    //System.out.println("After doc add 1");
    dump(fdir, dir);

    writer.commit();

    //System.out.println("After commit");
    dump(fdir, dir);

    writer.forceMerge(1, true);
    //System.out.println("Right after merge");
    dump(fdir, dir);

    try {
        Thread.currentThread().sleep(5000);
    } catch (Exception ex) {
    }
    //System.out.println("After sleeping after merge");
    dump(fdir, dir);

    writer.close();
    //System.out.println("After writer close");
    dump(fdir, dir);

    dir.close();
    //System.out.println("After dir close");
    dump(fdir, dir);
}

From source file:com.aliasi.lingmed.medline.IndexMedline.java

License:Lingpipe license

/**
 * Run the command.  See class documentation above for details on
 * arguments and behavior.//from w  w  w  .  j a  va  2  s  .  co  m
 */
public void run() {
    System.out.println("start run");
    try {
        File[] files = getLaterFiles(mDistDir);
        System.out.println("Total files to process: " + files.length);
        System.out.println("File names: " + java.util.Arrays.asList(files));
        //            if (mLogger.isDebugEnabled())
        //                mLogger.debug("File names: " + java.util.Arrays.asList(files));
        if (files.length > 0) {
            MedlineParser parser = new MedlineParser(true); // true = save raw XML

            Directory fsDir = FSDirectory.open(mIndex);
            IndexWriterConfig iwConf = new IndexWriterConfig(Version.LUCENE_36, mCodec.getAnalyzer());
            iwConf.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
            iwConf.setRAMBufferSizeMB(RAM_BUF_SIZE);
            if (sIsBaseline) {
                LogDocMergePolicy ldmp = new LogDocMergePolicy();
                ldmp.setMergeFactor(MERGE_FACTOR_HI);
                iwConf.setMergePolicy(ldmp);
            }
            IndexWriter indexWriter = new IndexWriter(fsDir, iwConf);

            for (File file : files) {
                System.out.println("processing file: " + file);
                MedlineIndexer indexer = new MedlineIndexer(indexWriter, mCodec);
                parser.setHandler(indexer);
                parseFile(parser, file);
                indexer.close();
                recordFile(indexWriter, file.getName());
                System.out.println("completed processing file: " + file);
            }
            System.out.println("All files parsed, now optimize index");
            indexWriter.forceMerge(1);
            indexWriter.commit();
            indexWriter.close();
        }
        System.out.println("Processing complete.");
    } catch (Exception e) {
        //            mLogger.warn("Unexpected Exception: "+e.getMessage());
        //            mLogger.warn("stack trace: "+Logging.logStackTrace(e));
        //            mLogger.warn("Aborting this run");
        IllegalStateException e2 = new IllegalStateException(e.getMessage());
        e2.setStackTrace(e.getStackTrace());
        throw e2;
    }
}

From source file:com.aliasi.lingmed.medline.OptimizeMedline.java

License:Lingpipe license

/**
 * Run the command.  See class documentation above for details on
 * arguments and behavior.//from w ww  .j a  v a 2 s . c  o  m
 */
public void run() {
    //        mLogger.info("start run");
    try {
        Directory fsDir = FSDirectory.open(mIndex);
        IndexWriterConfig iwConf = new IndexWriterConfig(Version.LUCENE_36,
                new StandardAnalyzer(Version.LUCENE_36));
        iwConf.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);

        IndexWriter indexWriter = new IndexWriter(fsDir, iwConf);
        indexWriter.forceMerge(1);
        indexWriter.commit();
        //            mLogger.info("Processing complete.");
    } catch (Exception e) {
        //            mLogger.warn("Unexpected Exception: "+e.getMessage());
        //            mLogger.warn("stack trace: "+Logging.logStackTrace(e));
        //            mLogger.warn("Aborting this run");
        IllegalStateException e2 = new IllegalStateException(e.getMessage());
        e2.setStackTrace(e.getStackTrace());
        throw e2;
    }
}

From source file:com.aurel.track.lucene.index.associatedFields.AbstractAssociatedFieldIndexer.java

License:Open Source License

/**
 * Deletes a document by key/*  w  w  w.  jav a  2 s  .co m*/
 * @param objectID
 */
@Override
public void deleteByKey(Integer objectID) {
    if (!ClusterBL.indexInstantly()) {
        LOGGER.debug("Index instantly is false by deleting the " + getLuceneFieldName() + " with entryID "
                + objectID);
        return;
    }
    if (objectID != null) {
        if (LOGGER.isDebugEnabled()) {
            LOGGER.debug("Deleting the " + getLuceneFieldName() + " document by Term " + getObjectIDFieldName()
                    + "=" + objectID.toString() + " from " + LuceneUtil.getDirectoryString(getIndexWriterID()));
        }
        IndexWriter indexWriter = LuceneIndexer.getIndexWriter(getIndexWriterID());
        if (indexWriter == null) {
            LOGGER.error("IndexWriter null by deleting by key");
            return;
        }
        Term keyTerm = new Term(getObjectIDFieldName(), objectID.toString());
        try {
            indexWriter.deleteDocuments(keyTerm);
        } catch (IOException e) {
            LOGGER.error("Deleting the " + getLuceneFieldName() + " with ID " + objectID + " failed with "
                    + e.getMessage());
            LOGGER.debug(ExceptionUtils.getStackTrace(e));
        }
        try {
            indexWriter.commit();
        } catch (IOException e) {
            LOGGER.error("Flushing the indexWriter after removing a " + getLuceneFieldName() + " with ID "
                    + objectID + "  failed with " + e.getMessage());
            LOGGER.debug(ExceptionUtils.getStackTrace(e));
        }
    }
}

From source file:com.aurel.track.lucene.index.associatedFields.AbstractAssociatedFieldIndexer.java

License:Open Source License

/**
 * Removes the indexes for a list of workItems
 * @param workItemIDs/*from   ww  w  .  j a va  2s. c  om*/
 */
@Override
public void deleteByWorkItems(List<Integer> workItemIDs) {
    IndexWriter indexWriter = null;
    try {
        if (!LuceneUtil.isUseLucene() || workItemIDs == null || workItemIDs.isEmpty()) {
            return;
        }
        indexWriter = LuceneIndexer.getIndexWriter(getIndexWriterID());
        if (indexWriter == null) {
            LOGGER.error("IndexWriter null in deleting by workItems");
            return;
        }
        if (LOGGER.isDebugEnabled()) {
            LOGGER.debug("Deleting " + getLuceneFieldName() + " documents for " + workItemIDs.size()
                    + " issues from " + LuceneUtil.getDirectoryString(getIndexWriterID()));
        }
        for (Integer workItemID : workItemIDs) {
            String workItemFieldName = getWorkItemFieldName();
            Term keyTerm = new Term(workItemFieldName, workItemID.toString());
            if (LOGGER.isDebugEnabled()) {
                LOGGER.debug("Deleting the " + getLuceneFieldName() + " document by Term " + workItemFieldName
                        + "=" + workItemID.toString());
            }
            try {
                indexWriter.deleteDocuments(keyTerm);
            } catch (IOException e) {
                LOGGER.error("Removing by workItemID " + workItemID + " failed with " + e.getMessage());
                LOGGER.debug(ExceptionUtils.getStackTrace(e));
            }
            String additionalWorkitemField = getAdditionalFieldName();
            if (additionalWorkitemField != null) {
                keyTerm = new Term(additionalWorkitemField, workItemID.toString());
                if (LOGGER.isDebugEnabled()) {
                    LOGGER.debug("Deleting the " + getLuceneFieldName() + " document by Term "
                            + additionalWorkitemField + "=" + workItemID.toString());
                }
                try {
                    indexWriter.deleteDocuments(keyTerm);
                } catch (IOException e) {
                    LOGGER.error("Removing by workItemID " + workItemID + " failed with " + e.getMessage());
                    LOGGER.debug(ExceptionUtils.getStackTrace(e));
                }
            }
        }
    } catch (Exception ex) {
        LOGGER.error("deleteListFromIndex failed with " + ex.getMessage());
    }
    try {
        indexWriter.commit();
    } catch (IOException e) {
        LOGGER.error("Flushing the indexWriter after removing a list of workItemBeans  failed with "
                + e.getMessage());
        LOGGER.debug(ExceptionUtils.getStackTrace(e));
    }
}

From source file:com.aurel.track.lucene.index.associatedFields.AttachmentIndexer.java

License:Open Source License

/**
 * Adds an attachment to the attachment index
 * Used by attaching a new file to the workItem
 * @param attachFile//from  w ww  .  j  ava2s  . c om
 */
@Override
public void addToIndex(Object object, boolean add) {
    if (!LuceneUtil.isUseLucene() || !LuceneUtil.isIndexAttachments() || object == null) {
        return;
    }
    if (!ClusterBL.indexInstantly()) {
        LOGGER.debug("Index instantly is false");
        return;
    }
    IndexWriter indexWriter = LuceneIndexer.getIndexWriter(getIndexWriterID());
    if (indexWriter == null) {
        LOGGER.error("IndexWriter null by adding an attachment");
        return;
    }
    if (LOGGER.isDebugEnabled()) {
        LOGGER.debug("Save a new " + add + " attachment into attachmentIndex");
    }
    TAttachmentBean attachmentBean = (TAttachmentBean) object;
    if (!add) {
        Integer objectID = attachmentBean.getObjectID();
        if (objectID != null) {
            Term keyTerm = new Term(getObjectIDFieldName(), objectID.toString());
            try {
                indexWriter.deleteDocuments(keyTerm);
                indexWriter.commit();
            } catch (IOException e) {
                LOGGER.error("Removing the attachment " + objectID + " failed with " + e.getMessage());
                LOGGER.debug(ExceptionUtils.getStackTrace(e));
            }
        }
    }
    try {
        Document doc = createDocument(attachmentBean);
        if (doc != null) {
            indexWriter.addDocument(doc);
        }
    } catch (IOException e) {
        LOGGER.error("Adding an attachment to the index failed with " + e.getMessage());
        LOGGER.debug(ExceptionUtils.getStackTrace(e));
    }
    try {
        indexWriter.commit();
    } catch (IOException e) {
        LOGGER.error("Flushing the attachment failed with " + e.getMessage());
        LOGGER.debug(ExceptionUtils.getStackTrace(e));
    }
}

From source file:com.aurel.track.lucene.index.associatedFields.BudgetPlanIndexer.java

License:Open Source License

/**
 * Adds an attachment to the attachment index
 * Used by attaching a new file to the workItem 
 * @param attachFile/* www  .  j  a v  a  2  s  .  c  o  m*/
 */
@Override
public void addToIndex(Object object, boolean add) {
    if (!LuceneUtil.isUseLucene()) {
        return;
    }
    if (!ClusterBL.indexInstantly()) {
        LOGGER.debug("Index instantly is false");
        return;
    }
    IndexWriter indexWriter = LuceneIndexer.getIndexWriter(getIndexWriterID());
    if (indexWriter == null) {
        LOGGER.error("IndexWriter null by adding a budget/plan");
        return;
    }
    if (LOGGER.isDebugEnabled()) {
        LOGGER.debug("Save a new " + add + " " + getLuceneFieldName());
    }
    TBudgetBean budgetBean = (TBudgetBean) object;
    if (!add) {
        Integer objectID = budgetBean.getObjectID();
        if (objectID != null) {
            Term keyTerm = new Term(getObjectIDFieldName(), objectID.toString());
            try {
                indexWriter.deleteDocuments(keyTerm);
                indexWriter.commit();
            } catch (IOException e) {
                LOGGER.error("Removing the entity " + objectID + " failed with " + e.getMessage());
                LOGGER.debug(ExceptionUtils.getStackTrace(e));
            }
        }
    }
    try {
        Document doc = createDocument(budgetBean);
        if (doc != null) {
            indexWriter.addDocument(doc);
        }
    } catch (IOException e) {
        LOGGER.error("Adding a budget/plan to the index failed with " + e.getMessage());
        LOGGER.debug(ExceptionUtils.getStackTrace(e));
    }
    try {
        indexWriter.commit();
    } catch (IOException e) {
        LOGGER.error("Flushing the budget/plan failed with " + e.getMessage());
        LOGGER.debug(ExceptionUtils.getStackTrace(e));
    }
}

From source file:com.aurel.track.lucene.index.associatedFields.ExpenseIndexer.java

License:Open Source License

/**
 * Adds an attachment to the attachment index
 * Used by attaching a new file to the workItem 
 * @param attachFile/*  ww  w  .  j a  v a  2s .co  m*/
 */
@Override
public void addToIndex(Object object, boolean add) {
    if (!LuceneUtil.isUseLucene()) {
        return;
    }
    if (!ClusterBL.indexInstantly()) {
        LOGGER.debug("Index instantly is false");
        return;
    }
    IndexWriter indexWriter = LuceneIndexer.getIndexWriter(getIndexWriterID());
    if (indexWriter == null) {
        LOGGER.error("IndexWriter null by adding an expense");
        return;
    }
    if (LOGGER.isDebugEnabled()) {
        LOGGER.debug("Save a new " + add + " " + getLuceneFieldName());
    }
    TCostBean costBean = (TCostBean) object;
    if (!add) {
        Integer objectID = costBean.getObjectID();
        if (objectID != null) {
            Term keyTerm = new Term(getObjectIDFieldName(), objectID.toString());
            try {
                indexWriter.deleteDocuments(keyTerm);
                indexWriter.commit();
            } catch (IOException e) {
                LOGGER.error("Removing the entity " + objectID + " failed with " + e.getMessage());
                LOGGER.debug(ExceptionUtils.getStackTrace(e));
            }
        }
    }
    try {
        Document doc = createDocument(costBean);
        if (doc != null) {
            indexWriter.addDocument(doc);
        }
    } catch (IOException e) {
        LOGGER.error("Adding an expense to the index failed with " + e.getMessage());
        LOGGER.debug(ExceptionUtils.getStackTrace(e));
    }
    try {
        indexWriter.commit();
    } catch (IOException e) {
        LOGGER.error("Flushing the expense failed with " + e.getMessage());
        LOGGER.debug(ExceptionUtils.getStackTrace(e));
    }
}