Example usage for org.apache.lucene.index IndexWriter commit

List of usage examples for org.apache.lucene.index IndexWriter commit

  1. HOME
  2. Java
  3. org.apache.lucene
  4. org.apache.lucene.index.*
  5. IndexWriter
  6. commit

Introduction

In this page you can find the example usage for org.apache.lucene.index IndexWriter commit.

Prototype

@Override
public final long commit() throws IOException

Source Link

Document

Commits all pending changes (added and deleted documents, segment merges, added indexes, etc.) to the index, and syncs all referenced index files, such that a reader will see the changes and the index updates will survive an OS or machine crash or power loss.

Usage

From source file:com.baidu.rigel.biplatform.tesseract.isservice.netty.service.IndexServerHandler.java

License:Open Source License

public void messageReceived_00(ChannelHandlerContext ctx, Object msg) throws Exception {
    logger.info(String.format(LogInfoConstants.INFO_PATTERN_MESSAGE_RECEIVED_BEGIN, "IndexServerHandler"));
    IndexMessage indexMsg = (IndexMessage) msg;
    // ??/*from w w w.j a v a 2 s. co  m*/
    File idxFile = new File(indexMsg.getIdxPath());
    File idxServiceFile = new File(indexMsg.getIdxServicePath());

    if (indexMsg.getMessageHeader().getAction().equals(NettyAction.NETTY_ACTION_UPDATE)
            || indexMsg.getMessageHeader().getAction().equals(NettyAction.NETTY_ACTION_INITINDEX)) {
        // ??
        // ?
        FileUtils.deleteFile(idxFile);
        if (indexMsg.getMessageHeader().getAction().equals(NettyAction.NETTY_ACTION_UPDATE)
                && idxServiceFile.exists()) {
            // ?
            FileUtils.copyFolder(indexMsg.getIdxServicePath(), indexMsg.getIdxPath());
        }
    }

    IndexWriter idxWriter = IndexWriterFactory.getIndexWriterWithSingleSlot(indexMsg.getIdxPath());

    TesseractResultSet data = indexMsg.getDataBody();
    long currDiskSize = FileUtils.getDiskSize(indexMsg.getIdxPath());
    BigDecimal currMaxId = null;
    // ??
    if (currDiskSize < indexMsg.getBlockSize()) {
        while (data.next() && currDiskSize < indexMsg.getBlockSize()) {
            Document doc = new Document();
            String[] fieldNameArr = data.getFieldNameArray();
            for (String select : fieldNameArr) {
                if (select.equals(indexMsg.getIdName())) {
                    currMaxId = data.getBigDecimal(select);
                }

                doc.add(new StringField(select, data.getString(select), Field.Store.NO));
            }

            idxWriter.addDocument(doc);
        }
        idxWriter.commit();
        idxWriter.close();

    }

    String feedBackIndexServicePath = null;
    String feedBackIndexFilePath = null;

    // ? or ???indexWriter\?
    long totalDiskSize = FileUtils.getDiskSize(indexMsg.getIdxPath());
    if (totalDiskSize > indexMsg.getBlockSize() || indexMsg.isLastPiece()) {
        IndexWriterFactory.destoryWriters(indexMsg.getIdxPath());
        feedBackIndexServicePath = indexMsg.getIdxPath();
        feedBackIndexFilePath = indexMsg.getIdxServicePath();
    } else {
        feedBackIndexServicePath = indexMsg.getIdxServicePath();
        feedBackIndexFilePath = indexMsg.getIdxPath();
    }

    MessageHeader messageHeader = new MessageHeader(NettyAction.NETTY_ACTION_INDEX_FEEDBACK);

    IndexMessage indexFeedbackMsg = new IndexMessage(messageHeader, indexMsg.getDataBody());
    indexFeedbackMsg.setBlockSize(indexMsg.getBlockSize());
    indexFeedbackMsg.setDiskSize(totalDiskSize);
    indexFeedbackMsg.setIdxServicePath(feedBackIndexServicePath);
    indexFeedbackMsg.setIdxPath(feedBackIndexFilePath);
    indexFeedbackMsg.setIdName(indexMsg.getIdName());
    indexFeedbackMsg.setMaxId(currMaxId);
    ctx.writeAndFlush(indexFeedbackMsg);
    ctx.channel().close();
    logger.info(String.format(LogInfoConstants.INFO_PATTERN_MESSAGE_RECEIVED_END, "IndexServerHandler"));
}

From source file:com.baidu.rigel.biplatform.tesseract.isservice.netty.service.IndexServerHandler.java

License:Open Source License

@Override
public void messageReceived(ChannelHandlerContext ctx, Object msg) throws Exception {
    logger.info(String.format(LogInfoConstants.INFO_PATTERN_MESSAGE_RECEIVED_BEGIN, "IndexServerHandler"));
    IndexMessage indexMsg = (IndexMessage) msg;
    // ??//w w  w .j a v  a  2s.c o m
    File idxFile = new File(indexMsg.getIdxPath());
    File idxServiceFile = new File(indexMsg.getIdxServicePath());

    if (indexMsg.getMessageHeader().getAction().equals(NettyAction.NETTY_ACTION_UPDATE)
            || indexMsg.getMessageHeader().getAction().equals(NettyAction.NETTY_ACTION_INITINDEX)) {
        // ??
        // ?
        FileUtils.deleteFile(idxFile);
        if (indexMsg.getMessageHeader().getAction().equals(NettyAction.NETTY_ACTION_UPDATE)
                && idxServiceFile.exists()) {
            // ?
            FileUtils.copyFolder(indexMsg.getIdxServicePath(), indexMsg.getIdxPath());
        }
    }

    IndexWriter idxWriter = IndexWriterFactory.getIndexWriter(indexMsg.getIdxPath());

    TesseractResultSet data = indexMsg.getDataBody();
    long currDiskSize = FileUtils.getDiskSize(indexMsg.getIdxPath());
    BigDecimal currMaxId = null;
    // ??
    if (currDiskSize < indexMsg.getBlockSize()) {
        while (data.next() && currDiskSize < indexMsg.getBlockSize()) {
            Document doc = new Document();
            String[] fieldNameArr = data.getFieldNameArray();
            for (String select : fieldNameArr) {
                if (select.equals(indexMsg.getIdName())) {
                    currMaxId = data.getBigDecimal(select);
                }

                doc.add(new StringField(select, data.getString(select), Field.Store.NO));
            }

            idxWriter.addDocument(doc);

            if ((currDiskSize + idxWriter.ramBytesUsed()) > indexMsg.getBlockSize()) {
                // ??????
                idxWriter.commit();
                // ??
                currDiskSize = FileUtils.getDiskSize(indexMsg.getIdxPath());
            }
        }
        idxWriter.commit();

    }

    String feedBackIndexServicePath = null;
    String feedBackIndexFilePath = null;
    // ? or ???indexWriter\?
    long totalDiskSize = FileUtils.getDiskSize(indexMsg.getIdxPath());
    if (totalDiskSize > indexMsg.getBlockSize() || indexMsg.isLastPiece()) {
        IndexWriterFactory.destoryWriters(indexMsg.getIdxPath());
        feedBackIndexServicePath = indexMsg.getIdxPath();
        feedBackIndexFilePath = indexMsg.getIdxServicePath();
    } else {
        feedBackIndexServicePath = indexMsg.getIdxServicePath();
        feedBackIndexFilePath = indexMsg.getIdxPath();
    }

    MessageHeader messageHeader = new MessageHeader(NettyAction.NETTY_ACTION_INDEX_FEEDBACK);

    IndexMessage indexFeedbackMsg = new IndexMessage(messageHeader, indexMsg.getDataBody());
    indexFeedbackMsg.setBlockSize(indexMsg.getBlockSize());
    indexFeedbackMsg.setDiskSize(totalDiskSize);
    indexFeedbackMsg.setIdxServicePath(feedBackIndexServicePath);
    indexFeedbackMsg.setIdxPath(feedBackIndexFilePath);
    indexFeedbackMsg.setIdName(indexMsg.getIdName());
    indexFeedbackMsg.setMaxId(currMaxId);
    ctx.writeAndFlush(indexFeedbackMsg);
    ctx.channel().close();
    logger.info(String.format(LogInfoConstants.INFO_PATTERN_MESSAGE_RECEIVED_END, "IndexServerHandler"));
}

From source file:com.browseengine.bobo.facets.attribute.AttributesFacetHandlerTest.java

License:Apache License

@Override
protected void setUp() throws Exception {
    directory = new RAMDirectory();
    analyzer = new WhitespaceAnalyzer();
    selectionProperties = new HashMap<String, String>();
    IndexWriter writer = new IndexWriter(directory, analyzer, true, MaxFieldLength.UNLIMITED);

    writer.addDocument(doc("prop1=val1", "prop2=val1", "prop5=val1"));
    writer.addDocument(doc("prop1=val2", "prop3=val1", "prop7=val7"));
    writer.addDocument(doc("prop1=val2", "prop3=val2", "prop3=val3"));
    writer.addDocument(doc("prop1=val1", "prop2=val1"));
    writer.addDocument(doc("prop1=val1", "prop2=val1"));
    writer.addDocument(doc("prop1=val1", "prop2=val1", "prop4=val2", "prop4=val3"));
    writer.commit();

    attributesFacetHandler = new AttributesFacetHandler(AttributeHandlerName, AttributeHandlerName, null, null,
            new HashMap<String, String>());
    facetHandlers.add(attributesFacetHandler);
    IndexReader reader = IndexReader.open(directory, true);
    boboReader = BoboIndexReader.getInstance(reader, facetHandlers);
    attributesFacetHandler.loadFacetData(boboReader);
    browser = new BoboBrowser(boboReader);
}

From source file:com.browseengine.bobo.facets.attribute.AttributesFacetHandlerTest.java

License:Apache License

private void modifiedSetup() throws CorruptIndexException, LockObtainFailedException, IOException {
    directory = new RAMDirectory();
    analyzer = new WhitespaceAnalyzer();
    IndexWriter writer = new IndexWriter(directory, analyzer, true, MaxFieldLength.UNLIMITED);

    writer.addDocument(doc("prop1=val1", "prop2=val1", "prop5=val1"));
    writer.addDocument(doc("prop1=val2", "prop3=val1", "prop7=val7"));
    writer.addDocument(doc("prop1=val2", "prop3=val2", "prop3=val3"));
    writer.addDocument(doc("prop1=val1", "prop2=val1"));
    writer.addDocument(doc("prop1=val1", "prop2=val1"));
    writer.addDocument(doc("prop1=val1", "prop2=val1", "prop4=val2", "prop4=val3"));
    writer.commit();

    HashMap<String, String> facetProps = new HashMap<String, String>();
    facetProps.put(AttributesFacetHandler.MAX_FACETS_PER_KEY_PROP_NAME, "1");
    attributesFacetHandler = new AttributesFacetHandler(AttributeHandlerName, AttributeHandlerName, null, null,
            facetProps);/*from w  w  w .  j a  v  a2 s.c om*/
    facetHandlers.add(attributesFacetHandler);
    IndexReader reader = IndexReader.open(directory, true);
    boboReader = BoboIndexReader.getInstance(reader, facetHandlers);
    attributesFacetHandler.loadFacetData(boboReader);
    browser = new BoboBrowser(boboReader);
}

From source file:com.browseengine.bobo.test.TestPathMultiVal.java

License:Apache License

@Override
protected void setUp() throws Exception {
    directory = new RAMDirectory();
    analyzer = new WhitespaceAnalyzer();
    IndexWriter writer = new IndexWriter(directory, analyzer, true, MaxFieldLength.UNLIMITED);
    Document doc = new Document();
    addMetaDataField(doc, PathHandlerName, new String[] { "/a/b/c", "/a/b/d" });
    writer.addDocument(doc);/*from w w w .j  a  va 2  s  . c o  m*/
    writer.commit();

    PathFacetHandler pathHandler = new PathFacetHandler("path", true);
    facetHandlers.add(pathHandler);
}

From source file:com.bugull.mongo.lucene.BuguIndex.java

License:Apache License

public void close() {
    ThreadUtil.safeClose(executor);/*www.  java2  s. c  o m*/
    ThreadUtil.safeClose(scheduler);
    if (clusterConfig != null) {
        clusterConfig.invalidate();
    }

    Map<String, IndexWriter> map = IndexWriterCache.getInstance().getAll();
    for (IndexWriter writer : map.values()) {
        if (writer != null) {
            Directory dir = writer.getDirectory();
            try {
                writer.commit();
                writer.close(true);
            } catch (CorruptIndexException ex) {
                logger.error("Can not commit and close the lucene index", ex);
            } catch (IOException ex) {
                logger.error("Can not commit and close the lucene index", ex);
            } finally {
                try {
                    if (dir != null && IndexWriter.isLocked(dir)) {
                        IndexWriter.unlock(dir);
                    }
                } catch (IOException ex) {
                    logger.error("Can not unlock the lucene index", ex);
                }
            }
        }
    }
}

From source file:com.chimpler.example.FacetLuceneIndexer.java

License:Apache License

public static void main(String args[]) throws Exception {
    //      if (args.length != 3) {
    //         System.err.println("Parameters: [index directory] [taxonomy directory] [json file]");
    //         System.exit(1);
    //      }/*from w w  w.jav a  2s  .  c o  m*/

    String indexDirectory = "index";
    String taxonomyDirectory = "taxonomy";
    String jsonFileName = "/home/qiuqiang/workspace/facet-lucene-example/books.json";

    IndexWriterConfig writerConfig = new IndexWriterConfig(LUCENE_VERSION,
            new WhitespaceAnalyzer(LUCENE_VERSION));
    writerConfig.setOpenMode(OpenMode.APPEND);
    IndexWriter indexWriter = new IndexWriter(FSDirectory.open(new File(indexDirectory)), writerConfig);

    TaxonomyWriter taxonomyWriter = new DirectoryTaxonomyWriter(MMapDirectory.open(new File(taxonomyDirectory)),
            OpenMode.APPEND);

    TaxonomyReader taxonomyReader = new DirectoryTaxonomyReader(FSDirectory.open(new File(taxonomyDirectory)));

    String content = IOUtils.toString(new FileInputStream(jsonFileName));
    JSONArray bookArray = new JSONArray(content);

    Field idField = new IntField("id", 0, Store.YES);
    Field titleField = new TextField("title", "", Store.YES);
    Field authorsField = new TextField("authors", "", Store.YES);
    Field bookCategoryField = new TextField("book_category", "", Store.YES);

    indexWriter.deleteAll();

    FacetFields facetFields = new FacetFields(taxonomyWriter);

    for (int i = 0; i < bookArray.length(); i++) {
        Document document = new Document();

        JSONObject book = bookArray.getJSONObject(i);
        int id = book.getInt("id");
        String title = book.getString("title");
        String bookCategory = book.getString("book_category");

        List<CategoryPath> categoryPaths = new ArrayList<CategoryPath>();

        String authorsString = "";
        JSONArray authors = book.getJSONArray("authors");
        for (int j = 0; j < authors.length(); j++) {
            String author = authors.getString(j);
            if (j > 0) {
                authorsString += ", ";
            }
            categoryPaths.add(new CategoryPath("author", author));
            authorsString += author;
        }
        categoryPaths.add(new CategoryPath("book_category" + bookCategory, '/'));

        idField.setIntValue(id);
        titleField.setStringValue(title);
        authorsField.setStringValue(authorsString);
        bookCategoryField.setStringValue(bookCategory);

        facetFields.addFields(document, categoryPaths);

        document.add(idField);
        document.add(titleField);
        document.add(authorsField);
        document.add(bookCategoryField);

        indexWriter.addDocument(document);

        System.out.printf("Book: id=%d, title=%s, book_category=%s, authors=%s\n", id, title, bookCategory,
                authors);
    }

    taxonomyWriter.prepareCommit();
    try {
        taxonomyWriter.commit();
    } catch (Exception e) {
        taxonomyWriter.rollback();
    }

    //      taxonomyWriter.close();
    //      
    //      indexWriter.commit();
    //      indexWriter.close();

    String query = "story";

    IndexReader indexReader = DirectoryReader.open(indexWriter, false);
    IndexReader indexReader2 = DirectoryReader.open(indexWriter, false);
    System.out.println(indexReader == indexReader2);

    IndexSearcher indexSearcher = new IndexSearcher(indexReader);

    TaxonomyReader newTaxonomyReader = DirectoryTaxonomyReader.openIfChanged(taxonomyReader);
    if (newTaxonomyReader != null) {
        TaxonomyReader tmp = taxonomyReader;
        taxonomyReader = newTaxonomyReader;
        tmp.close();
    } else {
        System.out.println("null");
    }

    ArrayList<FacetRequest> facetRequests = new ArrayList<FacetRequest>();
    facetRequests.add(new CountFacetRequest(new CategoryPath("author"), 100));
    facetRequests.add(new CountFacetRequest(new CategoryPath("book_category"), 100));

    FacetSearchParams searchParams = new FacetSearchParams(facetRequests);

    ComplexPhraseQueryParser queryParser = new ComplexPhraseQueryParser(LUCENE_VERSION, "title",
            new StandardAnalyzer(LUCENE_VERSION));
    Query luceneQuery = queryParser.parse(query);

    // Collectors to get top results and facets
    TopScoreDocCollector topScoreDocCollector = TopScoreDocCollector.create(10, true);
    FacetsCollector facetsCollector = FacetsCollector.create(searchParams, indexReader, taxonomyReader);
    indexSearcher.search(luceneQuery, MultiCollector.wrap(topScoreDocCollector, facetsCollector));
    System.out.println("Found:");

    for (ScoreDoc scoreDoc : topScoreDocCollector.topDocs().scoreDocs) {
        Document document = indexReader.document(scoreDoc.doc);
        System.out.printf("- book: id=%s, title=%s, book_category=%s, authors=%s, score=%f\n",
                document.get("id"), document.get("title"), document.get("book_category"),
                document.get("authors"), scoreDoc.score);
    }

    System.out.println("Facets:");
    for (FacetResult facetResult : facetsCollector.getFacetResults()) {
        System.out.println("- " + facetResult.getFacetResultNode().label);
        for (FacetResultNode facetResultNode : facetResult.getFacetResultNode().subResults) {
            System.out.printf("    - %s (%f)\n", facetResultNode.label.toString(), facetResultNode.value);
            for (FacetResultNode subFacetResultNode : facetResultNode.subResults) {
                System.out.printf("        - %s (%f)\n", subFacetResultNode.label.toString(),
                        subFacetResultNode.value);
            }
        }
    }
    taxonomyReader.close();
    indexReader.close();

    taxonomyWriter.commit();
    taxonomyWriter.close();

    indexWriter.commit();
    indexWriter.close();

}

From source file:com.codenvy.test.lucene.DeleteFilesWithSameName.java

License:Open Source License

public static void main(String[] args) throws Exception {
    String DOC_DIR_NAME = "files";
    filesDirPath = Paths.get(DOC_DIR_NAME).toAbsolutePath().toString();

    Path indexPath = Paths.get("index");
    Path docDir = Paths.get(DOC_DIR_NAME);

    Path file1 = Paths.get(DOC_DIR_NAME, "File1");
    Path file2 = Paths.get(DOC_DIR_NAME, "File1A");

    Analyzer analyzer = new SimpleAnalyzer();
    IndexWriterConfig iwc = new IndexWriterConfig(analyzer);
    iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE);
    //iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);

    if (!Files.isReadable(docDir)) {
        System.out.println("document folder not found");
        return;// w ww.j  ava  2s  . c  o m
    }

    Directory index = FSDirectory.open(indexPath);

    IndexWriter writer = new IndexWriter(index, iwc);

    //add files to index
    indexDocs(writer, file1);
    indexDocs(writer, file2);
    writer.commit();

    searchAndPrintResult(indexPath);

    //delete files
    System.out.println();
    System.out.println("==================================================================");
    System.out.println("delete by prefix \"" + filesDirPath + "/File1\"");
    Query query = new PrefixQuery(new Term(PATH, filesDirPath + "/File1"));

    writer.deleteDocuments(query);
    writer.close();

    searchAndPrintResult(indexPath);
}

From source file:com.dreamerpartner.codereview.lucene.IndexHelper.java

License:Apache License

/**
 * ?/* w  w w .  j  ava  2  s . c  o  m*/
 * @param module ?
 * @param doc
 * @param isNew
 * @param delTerm del
 * @throws IOException 
 */
@SuppressWarnings("deprecation")
public static void add(String module, Document doc, boolean isNew, Term delTerm) throws IOException {
    long beginTime = System.currentTimeMillis();
    IndexWriter writer = null;
    try {
        Directory dir = FSDirectory.open(new File(LuceneUtil.getIndexPath(module)));
        Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_4_10_0);
        IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_4_10_0, analyzer);
        iwc.setMaxBufferedDocs(100);
        iwc.setOpenMode(OpenMode.CREATE_OR_APPEND);
        //         iwc.setRAMBufferSizeMB(256.0);// ?
        writer = new IndexWriter(dir, iwc);
        if (isNew) {
            writer.addDocument(doc);
        } else {
            writer.updateDocument(delTerm, doc);
        }
        //???
        writer.commit();
    } finally {
        long endTime = System.currentTimeMillis();
        logger.debug("isNew:" + isNew + ", add consume " + (endTime - beginTime) + " milliseconds.");
        if (writer != null)
            writer.close();
    }
}

From source file:com.dreamerpartner.codereview.lucene.IndexHelper.java

License:Apache License

/**
 *  ??/*w w w.j  a va 2 s. com*/
 * @param module ?
 * @param docs
 * @param isNew
 * @param delTerm del
 * @throws IOException 
 */
@SuppressWarnings("deprecation")
public static void adds(String module, List<Document> docs, boolean isNew, Term delTerm) throws IOException {
    long beginTime = System.currentTimeMillis();
    IndexWriter writer = null;
    try {
        Directory dir = FSDirectory.open(new File(LuceneUtil.getIndexPath(module)));
        Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_4_10_0);
        IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_4_10_0, analyzer);
        iwc.setMaxBufferedDocs(100);
        iwc.setOpenMode(OpenMode.CREATE_OR_APPEND);
        //         iwc.setRAMBufferSizeMB(256.0);// ?
        writer = new IndexWriter(dir, iwc);
        if (isNew) {
            writer.addDocuments(docs);
        } else {
            writer.updateDocuments(delTerm, docs);
        }
        writer.commit();
    } finally {
        long endTime = System.currentTimeMillis();
        logger.debug("adds consume " + (endTime - beginTime) + " milliseconds.");
        if (writer != null)
            writer.close();
    }
}