Example usage for org.apache.lucene.index IndexWriter commit

Introduction

In this page you can find the example usage for org.apache.lucene.index IndexWriter commit.

Prototype

@Override
public final long commit() throws IOException

Source Link

Document

Commits all pending changes (added and deleted documents, segment merges, added indexes, etc.) to the index, and syncs all referenced index files, such that a reader will see the changes and the index updates will survive an OS or machine crash or power loss.

Usage

From source file:org.niord.core.message.MessageLuceneIndex.java

License:Apache License

/**
 * Deletes the current index//from   w  w w.j  av  a  2s.c o m
 */
private void deleteIndex() throws IOException {
    // Delete the index
    IndexWriter writer = null;
    try {
        writer = getNewWriter();
        writer.deleteAll();
        writer.setCommitData(new HashMap<>());
        writer.commit();
    } finally {
        closeWriter(writer);
    }
}

From source file:org.niord.core.message.MessageLuceneIndex.java

License:Apache License

/**
 * Updates the Lucene index/*www .j ava  2s.  com*/
 *
 * @param maxIndexCount max number of messages to index at a time
 * @return the number of updates
 */
private int updateLuceneIndex(int maxIndexCount) {

    Date lastUpdated = getLastUpdated();

    long t0 = System.currentTimeMillis();
    log.debug(String.format("Indexing at most %d changed messages since %s", maxIndexCount, lastUpdated));

    IndexWriter writer = null;
    try {
        // Find all messages changed since the lastUpdated time stamp
        List<Message> updatedMessages = findUpdatedMessages(lastUpdated, maxIndexCount);
        if (updatedMessages.size() == 0) {
            return 0;
        }

        // Create a new index writer
        writer = getNewWriter();

        // Update the index with the changes
        for (Message message : updatedMessages) {
            indexMessage(writer, message);
            if (message.getUpdated().after(lastUpdated)) {
                lastUpdated = message.getUpdated();
            }
        }

        // Update the last-updated flag
        setLastUpdated(lastUpdated, writer);

        // Commit the changes
        writer.commit();

        // Re-open the reader from the writer
        refreshReader(writer);

        // Check if we need to optimize the index
        optimizeIndexCount += updatedMessages.size();
        if (optimizeIndexCount > LUCENE_OPTIMIZE_INDEX_COUNT) {
            writer.forceMerge(LUCENE_MAX_NUM_SEGMENTS);
            optimizeIndexCount = 0;
        }

        log.info("Indexed " + updatedMessages.size() + " messages in " + (System.currentTimeMillis() - t0)
                + " ms");

        return updatedMessages.size();
    } catch (Exception ex) {
        log.error("Error updating Lucene index: " + ex.getMessage(), ex);
        return 0;
    } finally {
        closeWriter(writer);
    }
}

From source file:org.nuxeo.ecm.core.storage.sql.db.H2Fulltext.java

License:Apache License

/**
 * Searches from the given full text index. The returned result set has a single ID column which holds the keys for
 * the matching rows.//from  w w w. j  ava 2s  . c  o m
 * <p>
 * Usually called through:
 *
 * <pre>
 *   SELECT * FROM NXFT_SEARCH(name, 'text');
 * </pre>
 *
 * @param conn the connection
 * @param indexName the index name
 * @param text the search query
 * @return the result set
 */
public static ResultSet search(Connection conn, String indexName, String text) throws SQLException {
    DatabaseMetaData meta = conn.getMetaData();
    if (indexName == null) {
        indexName = DEFAULT_INDEX_NAME;
    }

    String schema;
    String table;
    String analyzerName;

    // find schema, table and analyzer
    try (PreparedStatement ps = conn
            .prepareStatement("SELECT SCHEMA, TABLE, ANALYZER FROM " + FT_TABLE + " WHERE NAME = ?")) {
        ps.setString(1, indexName);
        try (ResultSet res = ps.executeQuery()) {
            if (!res.next()) {
                throw new SQLException("No such index: " + indexName);
            }
            schema = res.getString(1);
            table = res.getString(2);
            analyzerName = res.getString(3);
        }
    }

    int type = getPrimaryKeyType(meta, schema, table);
    SimpleResultSet rs = new SimpleResultSet();
    rs.addColumn(COL_KEY, type, 0, 0);

    if (meta.getURL().startsWith("jdbc:columnlist:")) {
        // this is just to query the result set columns
        return rs;
    }

    // flush changes
    final IndexWriter writer = getIndexWriter(getIndexName(conn), getIndexPath(conn), analyzerName);
    if (writer.hasUncommittedChanges()) {
        try {
            writer.commit();
        } catch (IOException cause) {
            throw convertException(cause);
        }
    }

    // search index
    try {
        BooleanQuery.Builder queryBuilder = new BooleanQuery.Builder();
        String defaultField = fieldForIndex(indexName);
        Analyzer analyzer = getAnalyzer(analyzerName);
        QueryParser parser = new QueryParser(defaultField, analyzer);
        queryBuilder.add(parser.parse(text), BooleanClause.Occur.MUST);

        try (IndexReader reader = DirectoryReader.open(writer.getDirectory())) {
            IndexSearcher searcher = new IndexSearcher(reader);
            Collector collector = new ResultSetCollector(rs, reader, type);
            searcher.search(queryBuilder.build(), collector);
        }
    } catch (SQLException | ParseException | IOException e) {
        throw convertException(e);
    }
    return rs;
}

From source file:org.ofbiz.content.search.DocumentIndexer.java

License:Apache License

@Override
public void run() {
    IndexWriter indexWriter = null;
    int uncommittedDocs = 0;
    while (true) {
        LuceneDocument ofbizDocument;// w w w  .j  a  v  a 2 s.  c o m
        try {
            // Execution will pause here until the queue receives a LuceneDocument for indexing
            ofbizDocument = documentIndexQueue.take();
        } catch (InterruptedException e) {
            Debug.logError(e, module);
            if (indexWriter != null) {
                try {
                    indexWriter.close();
                    indexWriter = null;
                } catch (IOException ioe) {
                    Debug.logError(ioe, module);
                }
            }
            break;
        }
        Term documentIdentifier = ofbizDocument.getDocumentIdentifier();
        Document document = ofbizDocument.prepareDocument(this.delegator);
        if (indexWriter == null) {
            try {
                indexWriter = new IndexWriter(this.indexDirectory, new IndexWriterConfig(
                        SearchWorker.LUCENE_VERSION, new StandardAnalyzer(SearchWorker.LUCENE_VERSION)));
            } catch (CorruptIndexException e) {
                Debug.logError("Corrupted lucene index: " + e.getMessage(), module);
                break;
            } catch (LockObtainFailedException e) {
                Debug.logError("Could not obtain Lock on lucene index " + e.getMessage(), module);
                // TODO: put the thread to sleep waiting for the locked to be released
                break;
            } catch (IOException e) {
                Debug.logError(e.getMessage(), module);
                break;
            }
        }
        try {
            if (document == null) {
                indexWriter.deleteDocuments(documentIdentifier);
                if (Debug.infoOn())
                    Debug.logInfo(getName() + ": deleted Lucene document: " + ofbizDocument, module);
            } else {
                indexWriter.updateDocument(documentIdentifier, document);
                if (Debug.infoOn())
                    Debug.logInfo(getName() + ": indexed Lucene document: " + ofbizDocument, module);
            }
        } catch (Exception e) {
            Debug.logError(e, getName() + ": error processing Lucene document: " + ofbizDocument, module);
            if (documentIndexQueue.peek() == null) {
                try {
                    indexWriter.close();
                    indexWriter = null;
                } catch (IOException ioe) {
                    Debug.logError(ioe, module);
                }
            }
            continue;
        }
        uncommittedDocs++;
        if (uncommittedDocs == UNCOMMITTED_DOC_LIMIT || documentIndexQueue.peek() == null) {
            // limit reached or queue empty, time to commit
            try {
                indexWriter.commit();
            } catch (IOException e) {
                Debug.logError(e, module);
            }
            uncommittedDocs = 0;
        }
        if (documentIndexQueue.peek() == null) {
            try {
                indexWriter.close();
                indexWriter = null;
            } catch (IOException e) {
                Debug.logError(e, module);
            }
        }
    }
}

From source file:org.ojbc.adapters.analyticaldatastore.util.LuceneUtils.java

License:RPL License

public static void main(String[] args) throws Exception {

    if (args.length != 2) {
        System.err.println("Must provide source and target index directories as command line arguments");
        System.exit(1);/*from   www .j  a  v a 2  s .c o m*/
    }
    Directory sourceDir = FSDirectory.open(new File(args[0]));
    DirectoryReader reader = DirectoryReader.open(sourceDir);

    Directory targetDir = FSDirectory.open(new File(args[1]));
    Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_47);
    IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_47, analyzer);
    config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
    config.setIndexDeletionPolicy(new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()));
    IndexWriter writer = new IndexWriter(targetDir, config);

    Set<String> allowedFields = new HashSet<String>();
    allowedFields.add(IdentifierGenerationStrategy.FIRST_NAME_FIELD);
    allowedFields.add(IdentifierGenerationStrategy.LAST_NAME_FIELD);
    allowedFields.add(IdentifierGenerationStrategy.MIDDLE_NAME_FIELD);
    allowedFields.add(IdentifierGenerationStrategy.BIRTHDATE_FIELD);
    allowedFields.add(IdentifierGenerationStrategy.SEX_FIELD);
    allowedFields.add(IdentifierGenerationStrategy.SSN_FIELD);
    allowedFields.add(IdentifierGenerationStrategy.ID_FIELD);

    try {
        int lastDocumentIndex = reader.maxDoc();
        for (int i = 0; i < lastDocumentIndex; i++) {
            Document d = reader.document(i);
            Document newDoc = new Document();
            List<IndexableField> fields = d.getFields();
            for (IndexableField f : fields) {
                String fieldName = f.name();
                String fieldValue = f.stringValue();
                if (allowedFields.contains(fieldName)) {
                    newDoc.add(new StringField(fieldName, fieldValue, Store.YES));
                }
            }
            writer.addDocument(newDoc);
            writer.commit();
        }
    } finally {
        reader.close();
        writer.close();
    }
}

From source file:org.olat.search.service.indexer.IndexWriterHolder.java

License:Apache License

public synchronized IndexWriter getAndLock() throws IOException {
    if (writerRef == null) {
        long start = System.nanoTime();
        IndexWriter indexWriter = new IndexWriter(indexPath, indexer.newIndexWriterConfig());
        if (!DirectoryReader.indexExists(indexPath)) {
            indexWriter.commit();//make sure it exists
        }//from  w ww  . j  a  va  2  s  . c o  m
        log.info("Opening writer takes (ms): " + CodeHelper.nanoToMilliTime(start));
        writerRef = indexWriter;
    }
    counter.incrementAndGet();
    return writerRef;
}

From source file:org.olat.search.service.indexer.IndexWriterHolder.java

License:Apache License

public synchronized void release(IndexWriter indexWriter) {
    if (indexWriter != null) {
        try {//from  w w w  .  ja  v  a 2  s . co m
            int used = counter.decrementAndGet();
            if (used == 0) {
                long start = System.nanoTime();
                indexWriter.commit();
                indexWriter.close();
                writerRef = null;
                log.info("Close writer takes (ms): " + CodeHelper.nanoToMilliTime(start));
            }
        } catch (Exception e) {
            log.error("", e);
        }
    }
}

From source file:org.olat.search.service.indexer.IndexWriterHolder.java

License:Apache License

public synchronized void close() {
    IndexWriter indexWriter = writerRef;
    if (indexWriter != null) {
        try {//from www  . j  av a2s .c o  m
            indexWriter.commit();
            indexWriter.close();
        } catch (IOException e) {
            log.error("", e);
        }
    }
}

From source file:org.openrdf.sail.lucene.LuceneIndex.java

License:BSD License

/**
 * Indexes the specified Statement./* w w  w  . j  a v  a2s . co  m*/
 */
public synchronized void addStatement(Statement statement) throws IOException {
    // determine stuff to store
    Value object = statement.getObject();
    if (!(object instanceof Literal)) {
        return;
    }

    String field = statement.getPredicate().toString();
    String text = ((Literal) object).getLabel();
    String context = getContextID(statement.getContext());
    boolean updated = false;
    IndexWriter writer = null;

    // fetch the Document representing this Resource
    String resourceId = getResourceID(statement.getSubject());
    String contextId = getContextID(statement.getContext());

    String id = formIdString(resourceId, contextId);
    Term idTerm = new Term(ID_FIELD_NAME, id);
    Document document = getDocument(idTerm);

    if (document == null) {
        // there is no such Document: create one now
        document = new Document();
        addID(id, document);
        addResourceID(resourceId, document);
        // add context
        addContext(context, document);

        addProperty(field, text, document);

        // add it to the index
        writer = getIndexWriter();
        writer.addDocument(document);
        updated = true;
    } else {
        // update this Document when this triple has not been stored already
        if (!hasProperty(field, text, document)) {
            // create a copy of the old document; updating the retrieved
            // Document instance works ok for stored properties but indexed data
            // gets lots when doing an IndexWriter.updateDocument with it
            Document newDocument = new Document();

            // add all existing fields (including id, uri, context, and text)
            for (Object oldFieldObject : document.getFields()) {
                Field oldField = (Field) oldFieldObject;
                newDocument.add(oldField);
            }

            // add the new triple to the cloned document
            addProperty(field, text, newDocument);

            // update the index with the cloned document
            writer = getIndexWriter();
            writer.updateDocument(idTerm, newDocument);
            updated = true;
        }
    }

    if (updated) {
        // make sure that these updates are visible for new
        // IndexReaders/Searchers
        writer.commit();

        // the old IndexReaders/Searchers are not outdated
        invalidateReaders();
    }
}

From source file:org.openrdf.sail.lucene.LuceneIndex.java

License:BSD License

public synchronized void removeStatement(Statement statement) throws IOException {
    Value object = statement.getObject();
    if (!(object instanceof Literal)) {
        return;/* w  ww.  ja va 2  s  . co m*/
    }

    IndexWriter writer = null;
    boolean updated = false;

    // fetch the Document representing this Resource
    String resourceId = getResourceID(statement.getSubject());
    String contextId = getContextID(statement.getContext());
    String id = formIdString(resourceId, contextId);
    Term idTerm = new Term(ID_FIELD_NAME, id);

    Document document = getDocument(idTerm);

    if (document != null) {
        // determine the values used in the index for this triple
        String fieldName = statement.getPredicate().toString();
        String text = ((Literal) object).getLabel();

        // see if this triple occurs in this Document
        if (hasProperty(fieldName, text, document)) {
            // if the Document only has one predicate field, we can remove the
            // document
            int nrProperties = numberOfPropertyFields(document);
            if (nrProperties == 0) {
                logger.info("encountered document with zero properties, should have been deleted: {}",
                        resourceId);
            } else if (nrProperties == 1) {
                writer = getIndexWriter();
                writer.deleteDocuments(idTerm);
                updated = true;
            } else {
                // there are more triples encoded in this Document: remove the
                // document and add a new Document without this triple
                Document newDocument = new Document();
                addID(id, newDocument);
                addResourceID(resourceId, newDocument);
                addContext(contextId, newDocument);

                for (Object oldFieldObject : document.getFields()) {
                    Field oldField = (Field) oldFieldObject;
                    String oldFieldName = oldField.name();
                    String oldValue = oldField.stringValue();

                    if (isPropertyField(oldFieldName)
                            && !(fieldName.equals(oldFieldName) && text.equals(oldValue))) {
                        addProperty(oldFieldName, oldValue, newDocument);
                    }
                }

                writer = getIndexWriter();
                writer.updateDocument(idTerm, newDocument);
                updated = true;
            }
        }
    }

    if (updated) {
        // make sure that these updates are visible for new
        // IndexReaders/Searchers
        writer.commit();

        // the old IndexReaders/Searchers are not outdated
        invalidateReaders();
    }
}