List of usage examples for org.apache.lucene.index IndexWriterConfig setMaxBufferedDocs
@Override
public IndexWriterConfig setMaxBufferedDocs(int maxBufferedDocs)
From source file:DVBench.java
License:Apache License
static void doBench(int bpv) throws Exception { File file = new File("/data/indices/dvbench"); file.mkdirs();//from w ww.j ava2 s .com Directory dir = FSDirectory.open(file); IndexWriterConfig config = new IndexWriterConfig(null); config.setOpenMode(OpenMode.CREATE); config.setMergeScheduler(new SerialMergeScheduler()); config.setMergePolicy(new LogDocMergePolicy()); config.setMaxBufferedDocs(25000); IndexWriter writer = new IndexWriter(dir, config); MyRandom r = new MyRandom(); int numdocs = 400000; Document doc = new Document(); Field dv = new NumericDocValuesField("dv", 0); Field inv = new LongField("inv", 0, Field.Store.NO); Field boxed = new BinaryDocValuesField("boxed", new BytesRef(8)); Field boxed2 = new BinaryDocValuesField("boxed2", new BytesRef(8)); doc.add(dv); doc.add(inv); doc.add(boxed); doc.add(boxed2); for (int i = 0; i < numdocs; i++) { // defeat blockpackedwriter final long value; if (i % 8192 == 0) { value = bpv == 64 ? Long.MIN_VALUE : 0; } else if (i % 8192 == 1) { value = bpv == 64 ? Long.MAX_VALUE : (1L << bpv) - 1; } else { value = r.nextLong(bpv); } dv.setLongValue(value); inv.setLongValue(value); box(value, boxed.binaryValue()); box(value, boxed2.binaryValue()); boxed2.binaryValue().length = (bpv + 7) / 8; // fixed length writer.addDocument(doc); } writer.close(); // run dv search tests String description = "dv (bpv=" + bpv + ")"; DirectoryReader reader = DirectoryReader.open(dir); IndexSearcher searcher = new IndexSearcher(reader); searcher.setQueryCache(null); // don't bench the cache int hash = 0; // warmup hash += search(description, searcher, "dv", 300, true); hash += search(description, searcher, "dv", 300, false); // Uninverting Map<String, UninvertingReader.Type> mapping = Collections.singletonMap("inv", UninvertingReader.Type.LONG); DirectoryReader uninv = UninvertingReader.wrap(reader, mapping); IndexSearcher searcher2 = new IndexSearcher(uninv); searcher2.setQueryCache(null); // don't bench the cache description = "fc (bpv=" + bpv + ")"; // warmup hash += search(description, searcher2, "inv", 300, true); hash += search(description, searcher2, "inv", 300, false); // Boxed inside binary DirectoryReader boxedReader = new BinaryAsVLongReader(reader); IndexSearcher searcher3 = new IndexSearcher(boxedReader); searcher3.setQueryCache(null); // don't bench the cache description = "boxed (bpv=" + bpv + ")"; // warmup hash += search(description, searcher3, "boxed", 300, true); hash += search(description, searcher3, "boxed", 300, false); description = "boxed fixed-length (bpv=" + bpv + ")"; // warmup hash += search(description, searcher3, "boxed2", 300, true); hash += search(description, searcher3, "boxed2", 300, false); if (hash == 3) { // wont happen System.out.println("hash=" + hash); } reader.close(); dir.close(); }
From source file:com.baidu.rigel.biplatform.tesseract.isservice.index.service.IndexWriterFactory.java
License:Open Source License
/** * /* w w w . jav a 2 s. co m*/ * getIndexWriter * * @param idxPath * * @return IndexWriter * @throws IOException * IO */ public static synchronized IndexWriter getIndexWriter(String idxPath) throws IOException { LOGGER.info(String.format(LogInfoConstants.INFO_PATTERN_FUNCTION_BEGIN, "getIndexWriter", "[idxPath:" + idxPath + "]")); IndexWriter indexWriter = null; if (INSTANCE.idxWriterMaps.containsKey(idxPath)) { indexWriter = INSTANCE.idxWriterMaps.get(idxPath); LOGGER.info(String.format(LogInfoConstants.INFO_PATTERN_FUNCTION_PROCESS_NO_PARAM, "getIndexWriter", "return exist IndexWriter ")); } else { File indexFile = new File(idxPath); Directory directory = FSDirectory.open(indexFile); IndexWriterConfig indexWriterConfig = new IndexWriterConfig(Version.LUCENE_4_10_1, new StandardAnalyzer()); indexWriterConfig.setOpenMode(OpenMode.CREATE_OR_APPEND); indexWriterConfig.setRAMBufferSizeMB(64.0); indexWriterConfig.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH); indexWriter = new IndexWriter(directory, indexWriterConfig); INSTANCE.idxWriterMaps.put(idxPath, indexWriter); LOGGER.info(String.format(LogInfoConstants.INFO_PATTERN_FUNCTION_PROCESS_NO_PARAM, "getIndexWriter", "create new IndexWriter ")); } LOGGER.info(String.format(LogInfoConstants.INFO_PATTERN_FUNCTION_END, "getIndexWriter", "[idxPath:" + idxPath + "]")); return indexWriter; }
From source file:com.dreamerpartner.codereview.lucene.IndexHelper.java
License:Apache License
/** * ?/*w w w . j a v a 2 s . com*/ * @param module ? * @param doc * @param isNew * @param delTerm del * @throws IOException */ @SuppressWarnings("deprecation") public static void add(String module, Document doc, boolean isNew, Term delTerm) throws IOException { long beginTime = System.currentTimeMillis(); IndexWriter writer = null; try { Directory dir = FSDirectory.open(new File(LuceneUtil.getIndexPath(module))); Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_4_10_0); IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_4_10_0, analyzer); iwc.setMaxBufferedDocs(100); iwc.setOpenMode(OpenMode.CREATE_OR_APPEND); // iwc.setRAMBufferSizeMB(256.0);// ? writer = new IndexWriter(dir, iwc); if (isNew) { writer.addDocument(doc); } else { writer.updateDocument(delTerm, doc); } //??? writer.commit(); } finally { long endTime = System.currentTimeMillis(); logger.debug("isNew:" + isNew + ", add consume " + (endTime - beginTime) + " milliseconds."); if (writer != null) writer.close(); } }
From source file:com.dreamerpartner.codereview.lucene.IndexHelper.java
License:Apache License
/** * ??// w ww . ja v a2 s . c o m * @param module ? * @param docs * @param isNew * @param delTerm del * @throws IOException */ @SuppressWarnings("deprecation") public static void adds(String module, List<Document> docs, boolean isNew, Term delTerm) throws IOException { long beginTime = System.currentTimeMillis(); IndexWriter writer = null; try { Directory dir = FSDirectory.open(new File(LuceneUtil.getIndexPath(module))); Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_4_10_0); IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_4_10_0, analyzer); iwc.setMaxBufferedDocs(100); iwc.setOpenMode(OpenMode.CREATE_OR_APPEND); // iwc.setRAMBufferSizeMB(256.0);// ? writer = new IndexWriter(dir, iwc); if (isNew) { writer.addDocuments(docs); } else { writer.updateDocuments(delTerm, docs); } writer.commit(); } finally { long endTime = System.currentTimeMillis(); logger.debug("adds consume " + (endTime - beginTime) + " milliseconds."); if (writer != null) writer.close(); } }
From source file:com.dreamerpartner.codereview.lucene.IndexHelper.java
License:Apache License
/** * ?/*w ww .j a va 2 s. c om*/ * @param module ? * @param term ? * @throws IOException */ @SuppressWarnings("deprecation") public static void delete(String module, Term... term) throws IOException { long beginTime = System.currentTimeMillis(); IndexWriter writer = null; try { Directory dir = FSDirectory.open(new File(LuceneUtil.getIndexPath(module))); Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_4_10_0); IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_4_10_0, analyzer); iwc.setMaxBufferedDocs(100); iwc.setOpenMode(OpenMode.CREATE_OR_APPEND); writer = new IndexWriter(dir, iwc); writer.deleteDocuments(term); writer.commit(); } finally { long endTime = System.currentTimeMillis(); logger.debug(module + " delete " + (endTime - beginTime) + " milliseconds."); if (writer != null) writer.close(); } }
From source file:com.dreamerpartner.codereview.lucene.IndexHelper.java
License:Apache License
/** * ?/* w w w. ja v a 2 s . c o m*/ * @param module ? * @throws IOException */ @SuppressWarnings("deprecation") public static void deleteAll(String module) throws IOException { long beginTime = System.currentTimeMillis(); IndexWriter writer = null; try { Directory dir = FSDirectory.open(new File(LuceneUtil.getIndexPath(module))); Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_4_10_0); IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_4_10_0, analyzer); iwc.setMaxBufferedDocs(100); iwc.setOpenMode(OpenMode.CREATE_OR_APPEND); writer = new IndexWriter(dir, iwc); writer.deleteAll(); writer.commit(); } finally { long endTime = System.currentTimeMillis(); logger.debug(module + " deleteAll " + (endTime - beginTime) + " milliseconds."); if (writer != null) writer.close(); } }
From source file:com.edgenius.wiki.search.lucene.SimpleIndexFactory.java
License:Open Source License
private IndexWriterConfig getIndexWriterConfig() { IndexWriterConfig conf = new IndexWriterConfig(LuceneConfig.VERSION, analyzerProvider.getIndexAnalyzer()); conf.setMaxBufferedDocs(maxBufferedDocs); conf.setTermIndexInterval(termIndexInterval); conf.setWriteLockTimeout(writeLockTimeout); LogMergePolicy mergePolicy = new LogDocMergePolicy(); mergePolicy.setUseCompoundFile(useCompoundFile); mergePolicy.setMaxMergeDocs(maxMergeDocs); mergePolicy.setMergeFactor(mergeFactor); conf.setMergePolicy(mergePolicy);//from www.j a va 2s . com return conf; }
From source file:com.rocana.lucene.codec.v1.TestRocanaPerFieldPostingsFormat2.java
License:Apache License
@Test public void testChangeCodecAndMerge() throws IOException { Directory dir = newDirectory();/*w ww . ja v a 2s .com*/ if (VERBOSE) { System.out.println("TEST: make new index"); } IndexWriterConfig iwconf = newIndexWriterConfig(new MockAnalyzer(random())).setOpenMode(OpenMode.CREATE) .setCodec(new MockCodec()); iwconf.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH); //((LogMergePolicy) iwconf.getMergePolicy()).setMergeFactor(10); IndexWriter writer = newWriter(dir, iwconf); addDocs(writer, 10); writer.commit(); assertQuery(new Term("content", "aaa"), dir, 10); if (VERBOSE) { System.out.println("TEST: addDocs3"); } addDocs3(writer, 10); writer.commit(); writer.close(); assertQuery(new Term("content", "ccc"), dir, 10); assertQuery(new Term("content", "aaa"), dir, 10); Codec codec = iwconf.getCodec(); iwconf = newIndexWriterConfig(new MockAnalyzer(random())).setOpenMode(OpenMode.APPEND).setCodec(codec); //((LogMergePolicy) iwconf.getMergePolicy()).setNoCFSRatio(0.0); //((LogMergePolicy) iwconf.getMergePolicy()).setMergeFactor(10); iwconf.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH); iwconf.setCodec(new MockCodec2()); // uses standard for field content writer = newWriter(dir, iwconf); // swap in new codec for currently written segments if (VERBOSE) { System.out.println("TEST: add docs w/ Standard codec for content field"); } addDocs2(writer, 10); writer.commit(); codec = iwconf.getCodec(); assertEquals(30, writer.maxDoc()); assertQuery(new Term("content", "bbb"), dir, 10); assertQuery(new Term("content", "ccc"), dir, 10); //// assertQuery(new Term("content", "aaa"), dir, 10); if (VERBOSE) { System.out.println("TEST: add more docs w/ new codec"); } addDocs2(writer, 10); writer.commit(); assertQuery(new Term("content", "ccc"), dir, 10); assertQuery(new Term("content", "bbb"), dir, 20); assertQuery(new Term("content", "aaa"), dir, 10); assertEquals(40, writer.maxDoc()); if (VERBOSE) { System.out.println("TEST: now optimize"); } writer.forceMerge(1); assertEquals(40, writer.maxDoc()); writer.close(); assertQuery(new Term("content", "ccc"), dir, 10); assertQuery(new Term("content", "bbb"), dir, 20); assertQuery(new Term("content", "aaa"), dir, 10); dir.close(); }
From source file:com.zimbra.cs.index.LuceneIndex.java
License:Open Source License
private IndexWriterConfig getWriterConfig() { IndexWriterConfig config = new IndexWriterConfig(VERSION, mailbox.index.getAnalyzer()); config.setMergeScheduler(new MergeScheduler()); config.setMaxBufferedDocs(LC.zimbra_index_lucene_max_buffered_docs.intValue()); config.setRAMBufferSizeMB(LC.zimbra_index_lucene_ram_buffer_size_kb.intValue() / 1024.0); if (LC.zimbra_index_lucene_merge_policy.booleanValue()) { LogDocMergePolicy policy = new LogDocMergePolicy(); config.setMergePolicy(policy);/* w w w. ja v a 2 s. com*/ policy.setUseCompoundFile(LC.zimbra_index_lucene_use_compound_file.booleanValue()); policy.setMergeFactor(LC.zimbra_index_lucene_merge_factor.intValue()); policy.setMinMergeDocs(LC.zimbra_index_lucene_min_merge.intValue()); if (LC.zimbra_index_lucene_max_merge.intValue() != Integer.MAX_VALUE) { policy.setMaxMergeDocs(LC.zimbra_index_lucene_max_merge.intValue()); } } else { LogByteSizeMergePolicy policy = new LogByteSizeMergePolicy(); config.setMergePolicy(policy); policy.setUseCompoundFile(LC.zimbra_index_lucene_use_compound_file.booleanValue()); policy.setMergeFactor(LC.zimbra_index_lucene_merge_factor.intValue()); policy.setMinMergeMB(LC.zimbra_index_lucene_min_merge.intValue() / 1024.0); if (LC.zimbra_index_lucene_max_merge.intValue() != Integer.MAX_VALUE) { policy.setMaxMergeMB(LC.zimbra_index_lucene_max_merge.intValue() / 1024.0); } } return config; }
From source file:dk.defxws.fgslucene.IndexWriterCache.java
License:Open Source License
/** * get IndexWriter for given indexPath and write it into cache. * //from w w w . j a v a2s. c om * @param indexName * name of index to open. * @param config * gsearch config-Object. * @throws GenericSearchException * e */ private IndexWriter getIndexWriter(final String indexName, final boolean create, final Config config) throws GenericSearchException { if (indexWriters.get(indexName) == null) { IndexWriter iw = null; try { IndexWriterConfig indexWriterConfig = new IndexWriterConfig(Constants.LUCENE_VERSION, getAnalyzer(config.getAnalyzer(indexName))); if (create) { indexWriterConfig.setOpenMode(OpenMode.CREATE); } else { indexWriterConfig.setOpenMode(OpenMode.CREATE_OR_APPEND); } if (config.getMaxBufferedDocs(indexName) > 1) { indexWriterConfig.setMaxBufferedDocs(config.getMaxBufferedDocs(indexName)); } if (config.getRamBufferSizeMb(indexName) > 1) { indexWriterConfig.setRAMBufferSizeMB(config.getRamBufferSizeMb(indexName)); } if (config.getMergeFactor(indexName) > 1 || config.getMaxMergeDocs(indexName) > 1 || config.getMaxMergeMb(indexName) > 1) { LogByteSizeMergePolicy logMergePolicy = new LogByteSizeMergePolicy(); if (config.getMergeFactor(indexName) > 1) { logMergePolicy.setMergeFactor(config.getMergeFactor(indexName)); } if (config.getMaxMergeDocs(indexName) > 1) { logMergePolicy.setMaxMergeDocs(config.getMaxMergeDocs(indexName)); } if (config.getMaxMergeMb(indexName) > 1) { logMergePolicy.setMaxMergeMB(config.getMaxMergeMb(indexName)); } indexWriterConfig.setMergePolicy(logMergePolicy); } if (config.getDefaultWriteLockTimeout(indexName) > 1) { indexWriterConfig.setWriteLockTimeout(config.getDefaultWriteLockTimeout(indexName)); } if (config.getLuceneDirectoryImplementation(indexName) != null) { // Initialize IndexWriter with configured FSDirectory FSDirectory directory = getDirectoryImplementation( config.getLuceneDirectoryImplementation(indexName), new File(config.getIndexDir(indexName))); iw = new IndexWriter(directory, indexWriterConfig); } else { // Initialize IndexWriter with default FSDirectory iw = new IndexWriter(FSDirectory.open(new File(config.getIndexDir(indexName))), indexWriterConfig); } if (config.getMaxChunkSize(indexName) > 1) { if (iw.getDirectory() instanceof MMapDirectory) { ((MMapDirectory) iw.getDirectory()).setMaxChunkSize(config.getMaxChunkSize(indexName)); } } } catch (Exception e) { iw = null; throw new GenericSearchException( "IndexWriter new error, creating index indexName=" + indexName + " :\n", e); } indexWriters.put(indexName, iw); if (logger.isDebugEnabled()) logger.debug("getIndexWriter put to map " + iw); return iw; } return indexWriters.get(indexName); }