Example usage for org.apache.lucene.index IndexWriterConfig setOpenMode

List of usage examples for org.apache.lucene.index IndexWriterConfig setOpenMode

  1. HOME
  2. Java
  3. org.apache.lucene
  4. org.apache.lucene.index.*
  5. IndexWriterConfig
  6. setOpenMode

Introduction

In this page you can find the example usage for org.apache.lucene.index IndexWriterConfig setOpenMode.

Prototype

public IndexWriterConfig setOpenMode(OpenMode openMode)

Source Link

Document

Specifies OpenMode of the index.

Usage

From source file:com.gitblit.tickets.TicketIndexer.java

License:Apache License

private IndexWriter getWriter() throws IOException {
    if (writer == null) {
        indexStore.create();/*from   w ww  . jav  a  2s  .c o m*/

        Directory directory = FSDirectory.open(indexStore.getPath());
        StandardAnalyzer analyzer = new StandardAnalyzer();
        IndexWriterConfig config = new IndexWriterConfig(analyzer);
        config.setOpenMode(OpenMode.CREATE_OR_APPEND);
        writer = new IndexWriter(directory, config);
    }
    return writer;
}

From source file:com.github.alvanson.xltsearch.IndexTask.java

License:Apache License

@Override
protected Boolean call() {
    IndexWriter iwriter = null;//from w  w w . ja  v a 2  s. com
    boolean result = false;

    updateMessage("started");
    try {
        int count = 0;
        Docket docket;

        IndexWriterConfig iwconfig = new IndexWriterConfig(config.getVersion(), config.getAnalyzer());
        iwconfig.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
        iwconfig.setSimilarity(config.getSimilarity());
        iwriter = new IndexWriter(config.getDirectory(), iwconfig);

        while ((docket = inQueue.take()) != Docket.DONE) {
            count++;
            updateMessage(docket.relPath);
            switch (docket.status) {
            case PARSED:
                // index parsed file
                Document doc = new Document();
                // store relative path  ** must be indexed for updateDocument
                doc.add(new StringField(config.pathField, docket.relPath, Field.Store.YES));
                // index content
                doc.add(new TextField(config.contentField, docket.content.toString(), Field.Store.NO));
                // index standard metadata
                for (Map.Entry<String, Property> e : config.metadataFields.entrySet()) {
                    for (String value : docket.metadata.getValues(e.getValue())) {
                        doc.add(new TextField(e.getKey(), value, Field.Store.YES));
                    }
                }
                // store hashsum
                doc.add(new StringField(config.hashSumField, docket.hashSum, Field.Store.YES));
                // add/update document
                iwriter.updateDocument(new Term(config.pathField, docket.relPath), doc);
                // fall through
            case PASS:
                break;
            case DELETE:
                iwriter.deleteDocuments(new Term(config.pathField, docket.relPath));
                break;
            default:
                logger.error("Unexpected docket state while processing {}: {}", docket.relPath,
                        docket.status.toString());
                cancel(true); // cancel task
            }
            updateProgress(count, count + docket.workLeft);
        }
        // end of queue
        updateMessage("complete");
        updateProgress(count, count + docket.workLeft);
        result = true;
    } catch (IOException ex) {
        updateMessage("I/O exception");
        logger.error("I/O exception while writing to index", ex);
    } catch (InterruptedException ex) {
        if (isCancelled()) {
            updateMessage("cancelled");
        } else {
            updateMessage("interrupted");
            logger.error("Interrupted", ex);
        }
    }
    // close iwriter
    if (iwriter != null) {
        try {
            iwriter.close();
        } catch (IOException ex) {
            logger.warn("I/O exception while closing index writer", ex);
        }
    }
    return result;
}

From source file:com.github.lucene.store.jdbc.AbstractJdbcDirectoryITest.java

License:Apache License

protected void addDocuments(final Directory directory, final OpenMode openMode, final boolean useCompoundFile,
        final Collection<String> docs) throws IOException {
    final IndexWriterConfig config = new IndexWriterConfig(analyzer);
    config.setOpenMode(OpenMode.CREATE);
    config.setUseCompoundFile(useCompoundFile);

    final DirectoryTemplate template = new DirectoryTemplate(directory);
    template.execute(new DirectoryTemplate.DirectoryCallbackWithoutResult() {
        @Override/*  w w  w.  jav a  2 s . co  m*/
        public void doInDirectoryWithoutResult(final Directory dir) throws IOException {
            final IndexWriter writer = new IndexWriter(dir, config);
            for (final Object element : docs) {
                final Document doc = new Document();
                final String word = (String) element;
                // FIXME: review
                // doc.add(new Field("keyword", word, Field.Store.YES,
                // Field.Index.UN_TOKENIZED));
                // doc.add(new Field("unindexed", word, Field.Store.YES,
                // Field.Index.NO));
                // doc.add(new Field("unstored", word, Field.Store.NO,
                // Field.Index.TOKENIZED));
                // doc.add(new Field("text", word, Field.Store.YES,
                // Field.Index.TOKENIZED));
                doc.add(new StringField("keyword", word, Field.Store.YES));
                doc.add(new StringField("unindexed", word, Field.Store.YES));
                doc.add(new StringField("unstored", word, Field.Store.NO));
                doc.add(new StringField("text", word, Field.Store.YES));
                writer.addDocument(doc);
            }

            // FIXME: review
            // writer.optimize();
            writer.close();
        }
    });
}

From source file:com.github.mosuka.apache.lucene.example.cmd.AddCommand.java

License:Apache License

@Override
public void execute(Map<String, Object> attrs) {
    Map<String, Object> responseMap = new LinkedHashMap<String, Object>();

    String responseJSON = null;//from  w w  w . j  ava 2s  .  c  o m
    Directory indexDir = null;
    IndexWriter writer = null;

    try {
        String index = (String) attrs.get("index");
        String uniqueId = (String) attrs.get("unique_id");
        String text = (String) attrs.get("text");

        indexDir = FSDirectory.open(new File(index).toPath());

        Document document = LuceneExampleUtil.createDocument(uniqueId, text);

        IndexWriterConfig config = new IndexWriterConfig(LuceneExampleUtil.createAnalyzerWrapper());
        config.setOpenMode(OpenMode.CREATE_OR_APPEND);

        writer = new IndexWriter(indexDir, config);
        writer.addDocument(document);
        writer.commit();

        responseMap.put("status", 0);
        responseMap.put("message", "OK");
    } catch (IOException e) {
        responseMap.put("status", 1);
        responseMap.put("message", e.getMessage());
    } finally {
        try {
            if (writer != null) {
                writer.close();
            }
        } catch (IOException e) {
            responseMap.put("status", 1);
            responseMap.put("message", e.getMessage());
        }
        try {
            if (indexDir != null) {
                indexDir.close();
            }
        } catch (IOException e) {
            responseMap.put("status", 1);
            responseMap.put("message", e.getMessage());
        }
    }

    try {
        ObjectMapper mapper = new ObjectMapper();
        responseJSON = mapper.writeValueAsString(responseMap);
    } catch (IOException e) {
        responseJSON = String.format("{\"status\":1, \"message\":\"%s\"}", e.getMessage());
    }
    System.out.println(responseJSON);
}

From source file:com.github.mosuka.apache.lucene.example.cmd.DeleteCommand.java

License:Apache License

@Override
public void execute(Map<String, Object> attrs) {
    Map<String, Object> responseMap = new LinkedHashMap<String, Object>();

    String responseJSON = null;// www. j av  a  2s.  c o m
    Directory indexDir = null;
    IndexWriter writer = null;

    try {
        String index = (String) attrs.get("index");
        String uniqueId = (String) attrs.get("unique_id");

        indexDir = FSDirectory.open(new File(index).toPath());

        IndexWriterConfig config = new IndexWriterConfig(LuceneExampleUtil.createAnalyzerWrapper());
        config.setOpenMode(OpenMode.CREATE_OR_APPEND);

        writer = new IndexWriter(indexDir, config);
        writer.deleteDocuments(new Term("id", uniqueId));
        writer.commit();

        responseMap.put("status", 0);
        responseMap.put("message", "OK");
    } catch (IOException e) {
        responseMap.put("status", 1);
        responseMap.put("message", e.getMessage());
    } finally {
        try {
            if (writer != null) {
                writer.close();
            }
        } catch (IOException e) {
            responseMap.put("status", 1);
            responseMap.put("message", e.getMessage());
        }
        try {
            if (indexDir != null) {
                indexDir.close();
            }
        } catch (IOException e) {
            responseMap.put("status", 1);
            responseMap.put("message", e.getMessage());
        }
    }

    try {
        ObjectMapper mapper = new ObjectMapper();
        responseJSON = mapper.writeValueAsString(responseMap);
    } catch (IOException e) {
        responseJSON = String.format("{\"status\":1, \"message\":\"%s\"}", e.getMessage());
    }
    System.out.println(responseJSON);
}

From source file:com.github.mosuka.apache.lucene.example.cmd.UpdateCommand.java

License:Apache License

@Override
public void execute(Map<String, Object> attrs) {
    Map<String, Object> responseMap = new LinkedHashMap<String, Object>();

    String responseJSON = null;/*from ww  w . j a v a 2  s .  co  m*/
    Directory indexDir = null;
    IndexWriter writer = null;

    try {
        String index = (String) attrs.get("index");
        String uniqueId = (String) attrs.get("unique_id");
        String text = (String) attrs.get("text");

        indexDir = FSDirectory.open(new File(index).toPath());

        Document document = LuceneExampleUtil.createDocument(uniqueId, text);

        IndexWriterConfig config = new IndexWriterConfig(LuceneExampleUtil.createAnalyzerWrapper());
        config.setOpenMode(OpenMode.CREATE_OR_APPEND);

        writer = new IndexWriter(indexDir, config);
        writer.updateDocument(new Term("id", document.get("id")), document);
        writer.commit();

        responseMap.put("status", 0);
        responseMap.put("message", "OK");
    } catch (IOException e) {
        responseMap.put("status", -1);
        responseMap.put("message", e.getMessage());
    } finally {
        try {
            if (writer != null) {
                writer.close();
            }
        } catch (IOException e) {
            responseMap.put("status", 1);
            responseMap.put("message", e.getMessage());
        }
        try {
            if (indexDir != null) {
                indexDir.close();
            }
        } catch (IOException e) {
            responseMap.put("status", 1);
            responseMap.put("message", e.getMessage());
        }
    }

    try {
        ObjectMapper mapper = new ObjectMapper();
        responseJSON = mapper.writeValueAsString(responseMap);
    } catch (IOException e) {
        responseJSON = String.format("{\"status\":1, \"message\":\"%s\"}", e.getMessage());
    }
    System.out.println(responseJSON);
}

From source file:com.github.tenorviol.gitsearch.IndexFiles.java

License:Apache License

/** Index all text files under a directory. */
public static void index(CommandLine cl) {
    boolean create = true;

    // TODO: multi files
    final File docDir = new File(cl.commandArgs.get(0));
    if (!docDir.exists() || !docDir.canRead()) {
        System.out.println("Document directory '" + docDir.getAbsolutePath()
                + "' does not exist or is not readable, please check the path");
        System.exit(1);/*  ww w  .  j a v a2  s . c  o  m*/
    }

    Date start = new Date();
    try {
        System.out.println("Indexing to directory '" + cl.indexPath + "'...");

        Directory dir = FSDirectory.open(new File(cl.indexPath));
        Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_31);
        IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_31, analyzer);

        if (create) {
            // Create a new index in the directory, removing any
            // previously indexed documents:
            iwc.setOpenMode(OpenMode.CREATE);
        } else {
            // Add new documents to an existing index:
            iwc.setOpenMode(OpenMode.CREATE_OR_APPEND);
        }

        // Optional: for better indexing performance, if you
        // are indexing many documents, increase the RAM
        // buffer.  But if you do this, increase the max heap
        // size to the JVM (eg add -Xmx512m or -Xmx1g):
        //
        // iwc.setRAMBufferSizeMB(256.0);

        IndexWriter writer = new IndexWriter(dir, iwc);
        indexDocs(writer, docDir);

        // NOTE: if you want to maximize search performance,
        // you can optionally call forceMerge here.  This can be
        // a terribly costly operation, so generally it's only
        // worth it when your index is relatively static (ie
        // you're done adding documents to it):
        //
        // writer.forceMerge(1);

        writer.close();

        Date end = new Date();
        System.out.println(end.getTime() - start.getTime() + " total milliseconds");

    } catch (IOException e) {
        System.out.println(" caught a " + e.getClass() + "\n with message: " + e.getMessage());
    }
}

From source file:com.globalsight.ling.lucene.Index.java

License:Apache License

/**
 * Re-creates the index by batch-loading entries into it.
 * The index must be closed before calling this method.
 * Caller must use//from  www. j a  v a2s.  c om
 * <PRE>
 *   close();
 *   try
 *   {
 *     batchOpen();
 *     ...
 *     batchAddDocument();
 *     ...
 *   }
 *   finally
 *   {
 *     batchDone();
 *   }
 */
public void batchOpen() throws IOException {
    synchronized (m_state) {
        if (m_state != STATE_CLOSED) {
            throw new IOException("index is open");
        }

        m_state = STATE_CREATING;
    }

    // setup RAMDirectory and writer
    m_ramdir = new RAMDirectory();
    IndexWriterConfig config = new IndexWriterConfig(LuceneUtil.VERSION, m_analyzer);
    config.setOpenMode(OpenMode.CREATE_OR_APPEND);
    //config.setSimilarity(m_similarity);

    m_ramIndexWriter = new IndexWriter(m_ramdir, config);
    //m_ramIndexWriter.mergeFactor = 10000;
}

From source file:com.globalsight.ling.lucene.Index.java

License:Apache License

/**
 * Allocates a Lucene IndexWriter object. The IndexWriter must be
 * closed by the caller. If p_create is true, the existing index
 * is cleared on disk./*from ww w  .ja v a  2 s  .c o  m*/
 */
private IndexWriter getIndexWriter(boolean p_create) throws IOException {
    IndexWriterConfig conf = new IndexWriterConfig(LuceneUtil.VERSION, m_analyzer);
    OpenMode om = p_create ? OpenMode.CREATE : OpenMode.CREATE_OR_APPEND;
    conf.setOpenMode(om);

    IndexWriter result = null;
    boolean deleteFiles = false;
    try {
        result = new IndexWriter(m_fsDir, conf);
    } catch (EOFException eofe) {
        deleteFiles = true;
    } catch (IndexFormatTooOldException ie) {
        deleteFiles = true;
    }

    if (deleteFiles) {
        File indexDir = new File(m_directory);
        if (!indexDir.exists()) {
            indexDir.mkdirs();
        }
        // delete too old index
        File[] files = indexDir.listFiles();
        if (files != null && files.length > 0) {
            for (int i = 0; i < files.length; i++) {
                File oneFile = files[i];
                try {
                    oneFile.delete();
                } catch (Exception eee) {
                    // ignore
                }
            }
        }

        result = new IndexWriter(m_fsDir, conf);
    }

    return result;
}

From source file:com.globalsight.ling.tm2.lucene.LuceneIndexWriter.java

License:Apache License

/**
 * The constructor gets a lock on the index directory.  If the
 * directory doesn't exist yet, it is created. When an operation
 * is done, close() method must be called to remove the lock.
 * /*from  w w  w.  java2s.co m*/
 * @param p_tmId TM id
 * @param p_locale locale of the index
 */
public LuceneIndexWriter(long p_tmId, GlobalSightLocale p_locale, boolean p_isFirst) throws Exception {
    m_tmId = p_tmId;
    m_analyzer = new GsPerFieldAnalyzer(p_locale);
    m_isFirst = p_isFirst;

    m_indexDir = LuceneUtil.getGoldTmIndexDirectory(p_tmId, p_locale, true);

    // get the directory. Note that the directory cannot be
    // created before getting a lock. Note2:
    // FSDirectory.getDirectory(dir, true) doesn't really create
    // index files. It just clear out the old index files and lock
    // file.
    m_directory = FSDirectory.open(m_indexDir);

    // get a lock on the directory
    m_lock = m_directory.makeLock(LOCK_NAME);
    if (!m_lock.obtain(180000L)) {
        m_lock = null;
        throw new IOException("Index locked for write: " + m_lock);
    }

    // only after gettting a lock, create the initial index files
    // if it doesn't exist.
    if (!DirectoryReader.indexExists(m_directory)) {
        IndexWriterConfig conf = new IndexWriterConfig(LuceneUtil.VERSION, m_analyzer);
        conf.setOpenMode(OpenMode.CREATE_OR_APPEND);
        boolean initSuccess = false;
        IndexWriter writer = null;
        try {
            writer = new IndexWriter(m_directory, conf);
            initSuccess = true;
        } catch (IndexFormatTooOldException ie) {
            // delete too old index
            File[] files = m_indexDir.listFiles();
            if (files != null && files.length > 0) {
                for (int i = 0; i < files.length; i++) {
                    File oneFile = files[i];
                    if (!LuceneIndexWriter.LOCK_NAME.equals(oneFile.getName())) {
                        oneFile.delete();
                    }
                }
            }

            writer = new IndexWriter(m_directory, conf);
            initSuccess = true;
        } finally {
            if (!initSuccess) {
                m_lock.release();
            }
            IOUtils.closeWhileHandlingException(writer);
        }
    }
}