Example usage for org.apache.lucene.index IndexWriter commit

Introduction

In this page you can find the example usage for org.apache.lucene.index IndexWriter commit.

Prototype

@Override
public final long commit() throws IOException

Source Link

Document

Commits all pending changes (added and deleted documents, segment merges, added indexes, etc.) to the index, and syncs all referenced index files, such that a reader will see the changes and the index updates will survive an OS or machine crash or power loss.

Usage

From source file:org.openrdf.sail.lucene.LuceneIndex.java

License:BSD License

/**
 * Add many statements at the same time, remove many statements at the same
 * time. Ordering by resource has to be done inside this method. The passed
 * added/removed sets are disjunct, no statement can be in both
 * /*from  w w w  .  ja va  2s.  com*/
 * @param added
 *        all added statements, can have multiple subjects
 * @param removed
 *        all removed statements, can have multiple subjects
 */
public synchronized void addRemoveStatements(Collection<Statement> added, Collection<Statement> removed)
        throws Exception {
    // Buffer per resource
    MapOfListMaps<Resource, String, Statement> rsAdded = new MapOfListMaps<Resource, String, Statement>();
    MapOfListMaps<Resource, String, Statement> rsRemoved = new MapOfListMaps<Resource, String, Statement>();

    HashSet<Resource> resources = new HashSet<Resource>();
    for (Statement s : added) {
        rsAdded.add(s.getSubject(), getContextID(s.getContext()), s);
        resources.add(s.getSubject());
    }
    for (Statement s : removed) {
        rsRemoved.add(s.getSubject(), getContextID(s.getContext()), s);
        resources.add(s.getSubject());
    }

    logger.debug("Removing " + removed.size() + " statements, adding " + added.size() + " statements");

    IndexWriter writer = getIndexWriter();

    // for each resource, add/remove
    for (Resource resource : resources) {
        Map<String, List<Statement>> stmtsToRemove = rsRemoved.get(resource);
        Map<String, List<Statement>> stmtsToAdd = rsAdded.get(resource);

        Set<String> contextsToUpdate = new HashSet<String>(stmtsToAdd.keySet());
        contextsToUpdate.addAll(stmtsToRemove.keySet());

        Map<String, Document> docsByContext = new HashMap<String, Document>();
        // is the resource in the store?
        // fetch the Document representing this Resource
        String resourceId = getResourceID(resource);
        Term uriTerm = new Term(URI_FIELD_NAME, resourceId);
        List<Document> documents = getDocuments(uriTerm);

        for (Document doc : documents) {
            docsByContext.put(this.getContextID(doc), doc);
        }

        for (String contextId : contextsToUpdate) {
            String id = formIdString(resourceId, contextId);

            Term idTerm = new Term(ID_FIELD_NAME, id);
            Document document = docsByContext.get(contextId);
            if (document == null) {
                // there are no such Documents: create one now
                document = new Document();
                addID(id, document);
                addResourceID(resourceId, document);
                addContext(contextId, document);
                // add all statements, remember the contexts
                // HashSet<Resource> contextsToAdd = new HashSet<Resource>();
                List<Statement> list = stmtsToAdd.get(contextId);
                if (list != null) {
                    for (Statement s : list) {
                        addProperty(s, document);
                    }
                }

                // add it to the index
                writer.addDocument(document);

                // THERE SHOULD BE NO DELETED TRIPLES ON A NEWLY ADDED RESOURCE
                if (stmtsToRemove.containsKey(contextId))
                    logger.info(
                            "Statements are marked to be removed that should not be in the store, for resource {} and context {}. Nothing done.",
                            resource, contextId);
            } else {
                // update the Document

                // create a copy of the old document; updating the retrieved
                // Document instance works ok for stored properties but indexed
                // data
                // gets lots when doing an IndexWriter.updateDocument with it
                Document newDocument = new Document();

                // buffer the removed literal statements
                ListMap<String, String> removedOfResource = null;
                {
                    List<Statement> removedStatements = stmtsToRemove.get(contextId);
                    if (removedStatements != null && !removedStatements.isEmpty()) {
                        removedOfResource = new ListMap<String, String>();
                        for (Statement r : removedStatements) {
                            if (r.getObject() instanceof Literal) {
                                // remove value from both property field and the
                                // corresponding text field
                                String label = ((Literal) r.getObject()).getLabel();
                                removedOfResource.put(r.getPredicate().toString(), label);
                                removedOfResource.put(TEXT_FIELD_NAME, label);
                            }
                        }
                    }
                }

                // add all existing fields (including id, uri, context, and text)
                // but without adding the removed ones
                // keep the predicate/value pairs to ensure that the statement
                // cannot be added twice
                SetMap<String, String> copiedProperties = new SetMap<String, String>();
                for (Object oldFieldObject : document.getFields()) {
                    Field oldField = (Field) oldFieldObject;
                    // do not copy removed statements to the new version of the
                    // document
                    if (removedOfResource != null) {
                        // which fields were removed?
                        List<String> objectsRemoved = removedOfResource.get(oldField.name());
                        if ((objectsRemoved != null) && (objectsRemoved.contains(oldField.stringValue())))
                            continue;
                    }
                    newDocument.add(oldField);
                    copiedProperties.put(oldField.name(), oldField.stringValue());
                }

                // add all statements to this document, except for those which
                // are already there
                {
                    List<Statement> addedToResource = stmtsToAdd.get(contextId);
                    String val;
                    if (addedToResource != null && !addedToResource.isEmpty()) {
                        for (Statement s : addedToResource) {
                            val = getLiteralPropertyValueAsString(s);
                            if (val != null) {
                                if (!copiedProperties.containsKeyValuePair(s.getPredicate().stringValue(),
                                        val)) {
                                    addProperty(s, newDocument);
                                }
                            }
                        }
                    }
                }

                // update the index with the cloned document, if it contains any
                // meaningful non-system properties
                int nrProperties = numberOfPropertyFields(newDocument);
                if (nrProperties > 0) {
                    writer.updateDocument(idTerm, newDocument);
                } else {
                    writer.deleteDocuments(idTerm);
                }
            }
        }
    }
    // make sure that these updates are visible for new
    // IndexReaders/Searchers
    writer.commit();

    // the old IndexReaders/Searchers are not outdated
    invalidateReaders();

}

From source file:org.polymap.core.data.feature.lucene.LuceneCache.java

License:Open Source License

LuceneCache(ILayer layer, FeatureType schema) throws IOException {
    this.schema = schema;

    File luceneCacheDir = new File(Polymap.getCacheDir(), "luceneCache_" + layer.id());
    luceneCacheDir.mkdirs();/*  ww w . j a v a  2  s . com*/

    directory = FSDirectory.open(luceneCacheDir);

    if (directory.listAll().length == 0) {
        log.info("No index found, creating...");
        IndexWriter iwriter = new IndexWriter(directory, analyzer, true, new IndexWriter.MaxFieldLength(25000));
        iwriter.commit();
        iwriter.close();
        isEmpty = true;
    }

    //        log.info( "    creating index reader..." );
    //        indexReader = IndexReader.open( directory, true );
    //        log.info( "    creating index searcher..." );
    //        searcher = new IndexSearcher( indexReader );
}

From source file:org.polymap.core.data.feature.lucene.LuceneCache.java

License:Open Source License

public void putFeatures(List<Feature> features)
        throws CorruptIndexException, IOException, PipelineIncubationException {

    IndexWriter iwriter = null;
    try {/*from ww  w.  ja va2s  .  co  m*/
        long start = System.currentTimeMillis();
        rwLock.writeLock().lock();

        iwriter = new IndexWriter(directory, analyzer, false, new IndexWriter.MaxFieldLength(25000));

        int size = 0;
        int indexed = 0;

        for (Feature feature : features) {
            Document doc = new Document();

            for (Property prop : feature.getProperties()) {
                String propName = prop.getName().getLocalPart();
                //log.debug( "        prop: " + propName );

                // no value
                if (prop.getValue() == null) {
                    continue;
                }
                // Geometry
                else if (Geometry.class.isAssignableFrom(prop.getValue().getClass())) {
                    Geometry geom = (Geometry) prop.getValue();
                    StringWriter out = new StringWriter(1024);
                    jsonCoder.write(geom, out);
                    doc.add(new Field(propName, out.toString(), Field.Store.YES, Field.Index.NO));

                    Envelope envelop = geom.getEnvelopeInternal();
                    doc.add(ValueCoder.encode(propName + FIELD_MAXX, envelop.getMaxX(), Double.class,
                            Field.Store.NO, true));
                    doc.add(ValueCoder.encode(propName + FIELD_MAXY, envelop.getMaxY(), Double.class,
                            Field.Store.NO, true));
                    doc.add(ValueCoder.encode(propName + FIELD_MINX, envelop.getMinX(), Double.class,
                            Field.Store.NO, true));
                    doc.add(ValueCoder.encode(propName + FIELD_MINY, envelop.getMinY(), Double.class,
                            Field.Store.NO, true));
                }
                // other
                else {
                    Class valueType = prop.getType().getBinding();
                    Fieldable field = ValueCoder.encode(propName, prop.getValue(), valueType, Field.Store.YES,
                            true);
                    doc.add(field);
                    //log.debug( "    field: " + field );
                }
                indexed++;
            }

            doc.add(new Field("fid", feature.getIdentifier().getID(), Field.Store.YES,
                    Field.Index.NOT_ANALYZED));
            //log.info( "DOC: " + doc );
            iwriter.addDocument(doc);
            size++;
        }
        iwriter.commit();
        iwriter.close();
        iwriter = null;
        isEmpty = false;
        log.debug("    document: count=" + size + " indexed=" + indexed);

        log.info("Added features: " + features.size() + " (" + (System.currentTimeMillis() - start) + "ms)");

        if (indexReader != null) {
            searcher.close();
            searcher = null;
            indexReader.close();
            indexReader = null;
        }
        //            indexReader = IndexReader.open( directory, true );
        //            searcher = new IndexSearcher( indexReader );
        //            log.info( "Index reloaded." );
    } catch (Exception e) {
        log.error("Fehler beim Indizieren:" + e.getLocalizedMessage());
        log.debug(e.getLocalizedMessage(), e);
    } finally {
        if (iwriter != null) {
            iwriter.close();
        }
        rwLock.writeLock().unlock();
    }
}

From source file:org.punksearch.crawler.IndexOperator.java

License:Open Source License

public static void merge(String targetDir, Set<String> sourceDirs) {
    try {/*w w w  .  j  ava 2s .  co  m*/
        IndexWriter iw = createIndexWriter(targetDir);
        Directory[] dirs = new Directory[sourceDirs.size()];
        int i = 0;
        for (String source : sourceDirs) {
            dirs[i] = LuceneUtils.dir(source);
            i++;
        }
        iw.addIndexes(dirs);
        iw.commit();
        iw.close();
    } catch (IOException ex) {
        log.error("Exception during merging index directories", ex);
        throw new RuntimeException(ex);
    }
}

From source file:org.rapidpm.modul.javafx.searchbox.searchbox.SearchBox.java

License:Apache License

private void addElementToIndex(final IndexWriter writer, final T item) {
    try {//from w  w w .  j  av a2 s  .c o  m
        final Document d = new Document();
        final long autoID = System.nanoTime();
        if (autoGeneratedIDS) {
            d.add(new LongField(ITEM_ID, autoID, YES));
        } else {
            d.add(new LongField(ITEM_ID, item.getID(), YES));
        }
        addDocument(d, item.getValues());
        writer.addDocument(d, analyzer);
        if (autoGeneratedIDS) {
            tablevalues.put(autoID, item);
        } else {
            tablevalues.put(item.getID(), item);
        }
        writer.commit();
        //            writer.close();
    } catch (IOException e) {
        logger.error(e);
    }

}

From source file:org.rapidpm.modul.javafx.searchbox.searchbox.SearchBox.java

License:Apache License

public void addElementToIndex(final T item) {
    try {/* w w w . j  a  v a 2 s.  c om*/
        final IndexWriter writer = new IndexWriter(idx, new IndexWriterConfig(Version.LUCENE_43, analyzer));
        final Document d = new Document();
        final long autoID = System.nanoTime();
        if (autoGeneratedIDS) {
            d.add(new LongField(ITEM_ID, autoID, YES));
        } else {
            d.add(new LongField(ITEM_ID, item.getID(), YES));
        }
        addDocument(d, item.getValues());
        writer.addDocument(d, analyzer);
        if (autoGeneratedIDS) {
            tablevalues.put(System.nanoTime(), item);
        } else {
            tablevalues.put(item.getID(), item);
        }
        writer.commit();
        writer.close();
    } catch (IOException e) {
        logger.error(e);
    }

}

From source file:org.remus.marketplace.indexing.impl.IndexServiceImpl.java

License:Open Source License

@Override
public void addToIndex(Node node) {
    try {/*from   www .  j  a v  a 2 s.  c o  m*/

        Map<Integer, String> listOfCat = new HashMap<Integer, String>();
        Set<Category> categories = node.getCategories();
        for (Category category : categories) {
            if (!listOfCat.containsKey(category.getId())) {
                listOfCat.put(category.getId(), category.getMarket().getId());
            }
        }

        IndexWriter indexWriter = new IndexWriter(getDirectory(), new SimpleAnalyzer(),
                MaxFieldLength.UNLIMITED);
        Set<Integer> keySet = listOfCat.keySet();
        indexWriter.deleteDocuments(new Term(Node.ID, String.valueOf(node.getId())));
        for (Integer integer : keySet) {
            final Document document = new Document();

            Field unitField = new Field(Node.ID, String.valueOf(node.getId()), Field.Store.YES,
                    Field.Index.NOT_ANALYZED);
            document.add(unitField);

            Field titleField = new Field(Node.NAME, node.getName(), Field.Store.YES, Field.Index.ANALYZED);
            document.add(titleField);
            titleField.setBoost(2.0f);

            Field shortDescriptionField = new Field(Node.SHORTDESCRIPTION, node.getShortdescription(),
                    Field.Store.YES, Field.Index.ANALYZED);
            document.add(shortDescriptionField);
            shortDescriptionField.setBoost(1.5f);

            Field bodyField = new Field(Node.BODY, node.getBody(), Field.Store.YES, Field.Index.ANALYZED);
            bodyField.setBoost(1.0f);
            document.add(bodyField);

            Field categoryField = new Field("category", String.valueOf(integer), Field.Store.YES,
                    Field.Index.NOT_ANALYZED);
            document.add(categoryField);

            Field marketField = new Field("market", listOfCat.get(integer), Field.Store.YES,
                    Field.Index.NOT_ANALYZED);
            document.add(marketField);

            indexWriter.addDocument(document);
        }

        indexWriter.commit();
        indexWriter.optimize();
        indexWriter.close();
    } catch (Exception e) {
        throw new RuntimeException(e);
    }

}

From source file:org.remus.marketplace.indexing.impl.IndexServiceImpl.java

License:Open Source License

@Override
public void removeFromIndex(Node node) {
    final Term term = new Term(Node.ID, String.valueOf(node.getId()));
    try {/*  w ww .  j  a  v  a  2  s .  c  om*/
        IndexWriter indexWriter = new IndexWriter(getDirectory(), new SimpleAnalyzer(),
                MaxFieldLength.UNLIMITED);
        indexWriter.deleteDocuments(term);
        indexWriter.commit();
        indexWriter.optimize();
        indexWriter.close();
    } catch (Exception e) {
        throw new RuntimeException(e);
    }

}