List of usage examples for org.apache.lucene.index IndexWriterConfig setRAMBufferSizeMB
@Override
public IndexWriterConfig setRAMBufferSizeMB(double ramBufferSizeMB)
From source file:de.citec.sc.sentence.preprocessing.lucene.CreateIndex.java
public static void main(String[] args) throws IOException { Analyzer analyzer = null;/*from w w w .j a v a2s .co m*/ List<String> files = new ArrayList<>(); files.add("/Users/swalter/Downloads/german_sentences_reduced.txt"); String indexPath = "/Users/swalter/Index/GermanIndexReduced/"; Language language = Language.DE; Directory dir = FSDirectory.open(Paths.get(indexPath)); //files.add("/home/bettina/CITEC/MATOLL/preprocessSentences/idealSentences/idealSents_mecab_jdepp_rmvPunct_CoNLLU"); //String indexPath = "/home/bettina/CITEC/MATOLL/preprocessSentences/idealSentences/index"; //Language language = Language.JA; //Directory dir = FSDirectory.open(Paths.get(indexPath)); if (language.equals(Language.DE)) analyzer = new GermanAnalyzer(); if (language.equals(Language.ES)) analyzer = new SpanishAnalyzer(); if (language.equals(Language.EN)) analyzer = new EnglishAnalyzer(); if (language.equals(Language.JA)) analyzer = new JapaneseAnalyzer(); IndexWriterConfig iwc = new IndexWriterConfig(analyzer); iwc.setOpenMode(OpenMode.CREATE); iwc.setRAMBufferSizeMB(12000); try (IndexWriter writer = new IndexWriter(dir, iwc)) { files.forEach(f -> { try { indexDocs(writer, Paths.get(f), language); } catch (IOException ex) { Logger.getLogger(CreateIndex.class.getName()).log(Level.SEVERE, null, ex); } }); } }
From source file:de.jetsli.lumeo.LucPerfTest.java
License:Apache License
public void testPerf() { new PerfRunner(1000000, 26f) { @Override/* w w w. j ava2 s.c o m*/ public void reinit() throws Exception { super.reinit(); if (nrtManager != null) { nrtManager.close(); reopenThread.close(); writer.waitForMerges(); writer.close(); dir.close(); } Helper.deleteDir(file); docs = 0; IndexWriterConfig cfg = new IndexWriterConfig(version, keyAna); cfg.setRAMBufferSizeMB(128); // cfg.setCodec(new Lucene40Codec() { // // @Override public PostingsFormat getPostingsFormatForField(String field) { // if ("_id".equals(field)) // return new Pulsing40PostingsFormat(); // else // return new Lucene40PostingsFormat(); // } // }); LogByteSizeMergePolicy mp = new LogByteSizeMergePolicy(); mp.setUseCompoundFile(false); cfg.setMergePolicy(mp); dir = FSDirectory.open(file); cfg.setOpenMode(IndexWriterConfig.OpenMode.CREATE); writer = new IndexWriter(dir, cfg); nrtManager = new NRTManager(writer, new SearcherWarmer() { @Override public void warm(IndexSearcher s) throws IOException { // TODO get some random vertices via getVertices? } }); int priority = Math.min(Thread.currentThread().getPriority() + 2, Thread.MAX_PRIORITY); reopenThread = new NRTManagerReopenThread(nrtManager, 5.0, 0.03); reopenThread.setName("NRT Reopen Thread"); reopenThread.setPriority(priority); reopenThread.setDaemon(true); reopenThread.start(); } final BytesRef bytes = new BytesRef(); @Override public void innerRun(int trial, int i) { long id = i; Document newDoc = new Document(); NumericField idField = new NumericField("_id", 6, NumericField.TYPE_STORED).setLongValue(id); Field uIdField = new Field("_uid", "" + id, StringField.TYPE_STORED); Field typeField = new Field("_type", "test", StringField.TYPE_STORED); newDoc.add(idField); newDoc.add(uIdField); newDoc.add(typeField); // Analyzer ana = anas.get(newDoc.get("_type")); try { NumericUtils.longToPrefixCoded(id, 0, bytes); latestGen = nrtManager.updateDocument(new Term("_id", bytes), newDoc, keyAna); docs++; } catch (IOException ex) { logger.error("Cannot update " + i, ex); } } @Override protected void finalAssert() throws Exception { // logger.info("wait for " + latestGen + ", current:" + nrtManager.getCurrentSearchingGen(true)); nrtManager.waitForGeneration(latestGen, true); // writer.commit(); // writer.waitForMerges(); SearcherManager mng = nrtManager.getSearcherManager(true); // mng.maybeReopen(); IndexSearcher searcher = mng.acquire(); try { TotalHitCountCollector coll = new TotalHitCountCollector(); searcher.search(new MatchAllDocsQuery(), coll); long total = coll.getTotalHits(); if (docs != total) throw new IllegalStateException(total + " vs. " + docs); } finally { nrtManager.getSearcherManager(true).release(searcher); } } }.run(); }
From source file:de.jetsli.lumeo.RawLucene.java
License:Apache License
public RawLucene init() { indexLock();/*from ww w .ja v a 2 s .c o m*/ try { if (closed) throw new IllegalStateException("Already closed"); if (writer != null) throw new IllegalStateException("Already initialized"); // release locks when started if (IndexWriter.isLocked(dir)) { logger.warn("index is locked + " + name + " -> releasing lock"); IndexWriter.unlock(dir); } IndexWriterConfig cfg = new IndexWriterConfig(VERSION, defaultMapping.getCombinedAnalyzer()); LogByteSizeMergePolicy mp = new LogByteSizeMergePolicy(); mp.setMaxMergeMB(getMaxMergeMB()); cfg.setRAMBufferSizeMB(ramBufferSizeMB); cfg.setTermIndexInterval(termIndexIntervalSize); cfg.setMergePolicy(mp); // TODO specify different formats for id fields etc // -> this breaks 16 of our tests!? Lucene Bug? // cfg.setCodec(new Lucene40Codec() { // // @Override public PostingsFormat getPostingsFormatForField(String field) { // return new Pulsing40PostingsFormat(); // } // }); // cfg.setMaxThreadStates(8); boolean create = !DirectoryReader.indexExists(dir); cfg.setOpenMode(create ? IndexWriterConfig.OpenMode.CREATE : IndexWriterConfig.OpenMode.APPEND); //wrap the writer with a tracking index writer writer = new TrackingIndexWriter(new IndexWriter(dir, cfg)); nrtManager = new NRTManager(writer, new SearcherFactory() { // @Override // public IndexSearcher newSearcher(IndexReader reader) throws IOException { // //TODO do some kind of warming here? // return new IndexSearcher(reader); // } }); getCurrentRTCache(latestGen); int priority = Math.min(Thread.currentThread().getPriority() + 2, Thread.MAX_PRIORITY); flushThread = new FlushThread("flush-thread"); flushThread.setPriority(priority); flushThread.setDaemon(true); flushThread.start(); reopenThread = new NRTManagerReopenThread(nrtManager, ordinaryWaiting, incomingSearchesMaximumWaiting); reopenThread.setName("NRT Reopen Thread"); reopenThread.setPriority(priority); reopenThread.setDaemon(true); reopenThread.start(); return this; } catch (Exception e) { throw new RuntimeException(e); } finally { indexUnlock(); } }
From source file:de.mpii.docsimilarity.tasks.qe.wiki.IndexWikipediaDump.java
License:Apache License
public static void constructIndex(String indexPath, String inputPath) throws UnsupportedEncodingException, IOException, ClassNotFoundException, InstantiationException, IllegalAccessException { int threads = 24; WikiClean cleaner = new WikiCleanBuilder().withTitle(true).build(); Directory dir = null;/*from w w w .ja v a 2 s . c om*/ //FSDirectory.open(Paths.get(indexPath)); // the analyzer should be the same with the runtime analyzer Analyzer analyzer = new StandardAnalyzer(new CharArraySet(Arrays.asList(StopWordsFilter.STOPWORDS), true)); IndexWriterConfig iwc = null; //new IndexWriterConfig(analyzer); iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE); iwc.setRAMBufferSizeMB(1024.0); IndexWriter writer = new IndexWriter(dir, iwc); logger.info("Creating index at " + indexPath); logger.info("Indexing with " + threads + " threads"); long startTime = System.currentTimeMillis(); try { WikipediaXMLDumpInputStream stream = new WikipediaXMLDumpInputStream(inputPath); ExecutorService executor = Executors.newFixedThreadPool(threads); int cnt = 0; String page; while ((page = stream.readNext()) != null) { String title = cleaner.getTitle(page); // These are heuristic specifically for filtering out non-articles in enwiki-20120104. if (title.startsWith("Wikipedia:") || title.startsWith("Portal:") || title.startsWith("File:")) { continue; } if (page.contains("#REDIRECT") || page.contains("#redirect") || page.contains("#Redirect")) { continue; } Runnable worker = new AddDocumentRunnable(writer, cleaner, page); executor.execute(worker); cnt++; if (cnt % 10000 == 0) { logger.info(cnt + " articles added"); } } executor.shutdown(); // Wait until all threads are finish while (!executor.isTerminated()) { } logger.info("Total of " + cnt + " articles indexed."); logger.info("Total elapsed time: " + (System.currentTimeMillis() - startTime) + "ms"); } catch (Exception ex) { logger.error("", ex); } finally { writer.close(); dir.close(); } }
From source file:de.mpii.microblogtrack.component.thirdparty.IndexWikipediaDump.java
License:Apache License
public static void constructIndex(String indexPath, String inputPath) throws UnsupportedEncodingException, IOException, ClassNotFoundException, InstantiationException, IllegalAccessException { int threads = 16; WikiClean cleaner = new WikiCleanBuilder().withTitle(true).build(); Directory dir = FSDirectory.open(Paths.get(indexPath)); // the analyzer should be the same with the runtime analyzer IndexWriterConfig iwc = new IndexWriterConfig( (Analyzer) Class.forName(Configuration.LUCENE_ANALYZER).newInstance()); iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE); iwc.setRAMBufferSizeMB(Configuration.LUCENE_MEM_SIZE); IndexWriter writer = new IndexWriter(dir, iwc); logger.info("Creating index at " + indexPath); logger.info("Indexing with " + threads + " threads"); long startTime = System.currentTimeMillis(); try {/*from www. j a v a 2s . c o m*/ WikipediaXMLDumpInputStream stream = new WikipediaXMLDumpInputStream(inputPath); ExecutorService executor = Executors.newFixedThreadPool(threads); int cnt = 0; String page; while ((page = stream.readNext()) != null) { String title = cleaner.getTitle(page); // These are heuristic specifically for filtering out non-articles in enwiki-20120104. if (title.startsWith("Wikipedia:") || title.startsWith("Portal:") || title.startsWith("File:")) { continue; } if (page.contains("#REDIRECT") || page.contains("#redirect") || page.contains("#Redirect")) { continue; } Runnable worker = new AddDocumentRunnable(writer, cleaner, page); executor.execute(worker); cnt++; if (cnt % 10000 == 0) { logger.info(cnt + " articles added"); } } executor.shutdown(); // Wait until all threads are finish while (!executor.isTerminated()) { } logger.info("Total of " + cnt + " articles indexed."); logger.info("Total elapsed time: " + (System.currentTimeMillis() - startTime) + "ms"); } catch (Exception ex) { logger.error("", ex); } finally { writer.close(); dir.close(); } }
From source file:de.tudarmstadt.lt.lm.app.GenerateNgramIndex.java
License:Apache License
public void create_ngram_index(File ngram_joined_counts_file) throws IOException { File index_dir = new File(_index_dir, "ngram"); if (index_dir.exists()) { LOG.info("Ngram index already exists in directory '{}'.", index_dir.getAbsolutePath()); if (_overwrite) { LOG.info("Overwriting index '{}',", index_dir); index_dir.delete();// w ww.j a va2 s.c o m } else return; } index_dir.mkdirs(); Analyzer analyzer = new KeywordAnalyzer(); IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_4_9, analyzer); iwc.setOpenMode(OpenMode.CREATE); // use 80 percent of the available total memory double total_mem_mb = (double) Runtime.getRuntime().maxMemory() / 1e6; double percentage_ram_buffer = Properties.ramBufferPercentage(); if (percentage_ram_buffer > 0) { double percentage_ram_buffer_mb = total_mem_mb * percentage_ram_buffer; LOG.info(String.format("Setting ram buffer size to %.2f MB (%.2f%% from %.2f MB)", percentage_ram_buffer_mb, percentage_ram_buffer * 100, total_mem_mb)); iwc.setRAMBufferSizeMB(percentage_ram_buffer_mb); } Directory directory = new MMapDirectory(index_dir); IndexWriter writer_ngram = new IndexWriter(directory, iwc); InputStream in = new FileInputStream(ngram_joined_counts_file); if (ngram_joined_counts_file.getName().endsWith(".gz")) in = new GZIPInputStream(in); LineIterator iter = new LineIterator(new BufferedReader(new InputStreamReader(in, "UTF-8"))); Document doc = new Document(); Field f_ngram = new StringField("ngram", "", Store.YES); doc.add(f_ngram); Field f_n = new IntField("cardinality", 0, Store.YES); doc.add(f_n); Field f_word = new StringField("word", "", Store.YES); doc.add(f_word); Field f_hist = new StringField("history", "", Store.YES); doc.add(f_hist); Field f_lower = new StringField("lower", "", Store.YES); doc.add(f_lower); Field f_count = new StoredField("num", 0L); doc.add(f_count); Field[] f_follow = new Field[4]; f_follow[0] = new StoredField("nf_s", 0L); doc.add(f_follow[0]); f_follow[1] = new StoredField("nf_N1", 0L); doc.add(f_follow[1]); f_follow[2] = new StoredField("nf_N2", 0L); doc.add(f_follow[2]); f_follow[3] = new StoredField("nf_N3", 0L); doc.add(f_follow[3]); Field[] f_precede = new Field[4]; f_precede[0] = new StoredField("np_s", 0L); doc.add(f_precede[0]); f_precede[1] = new StoredField("np_N1", 0L); doc.add(f_precede[1]); f_precede[2] = new StoredField("np_N2", 0L); doc.add(f_precede[2]); f_precede[3] = new StoredField("np_N3", 0L); doc.add(f_precede[3]); Field[] f_followerprecede = new Field[4]; f_followerprecede[0] = new StoredField("nfp_s", 0L); doc.add(f_followerprecede[0]); f_followerprecede[1] = new StoredField("nfp_N1", 0L); doc.add(f_followerprecede[1]); f_followerprecede[2] = new StoredField("nfp_N2", 0L); doc.add(f_followerprecede[2]); f_followerprecede[3] = new StoredField("nfp_N3", 0L); doc.add(f_followerprecede[3]); Long[][] N = new Long[][] { { 0L, 0L, 0L, 0L, 0L, 0L } }; Long[] S = new Long[] { 0L }; long c = 0; while (iter.hasNext()) { if (++c % 100000 == 0) LOG.info("Adding {}'th ngram.", c); String line = iter.next(); try { String[] splits = de.tudarmstadt.lt.utilities.StringUtils.rtrim(line).split("\t"); String ngram_str = splits[0]; if (de.tudarmstadt.lt.utilities.StringUtils.trim(ngram_str).isEmpty()) { LOG.warn("Ngram is empty, skipping line {}: '{}' (file '{}').", c, line, ngram_joined_counts_file); continue; } List<String> ngram = Arrays.asList(ngram_str.split(" ")); long num = Long.parseLong(splits[1]); int n = ngram.size(); f_ngram.setStringValue(ngram_str); f_n.setIntValue(n); f_word.setStringValue(ngram.get(ngram.size() - 1)); f_hist.setStringValue(StringUtils.join(ngram.subList(0, ngram.size() - 1), " ")); f_lower.setStringValue(StringUtils.join(ngram.subList(1, ngram.size()), " ")); f_count.setLongValue(num); for (int j = 0; j < f_follow.length; j++) { f_follow[j].setLongValue(0L); f_precede[j].setLongValue(0L); f_followerprecede[j].setLongValue(0L); } if (splits.length > 2 && !splits[2].isEmpty()) { // precede or follow or followerprecede String[] splits_ = splits[2].split(":"); String type = splits_[0]; String[] count_values = splits_[1].split(","); if (count_values.length > 0) { if ("n_f".equals(type)) f_follow[0].setLongValue(Long.parseLong(count_values[0])); else if ("n_p".equals(type)) f_precede[0].setLongValue(Long.parseLong(count_values[0])); else if ("n_fp".equals(type)) f_followerprecede[0].setLongValue(Long.parseLong(count_values[0])); } for (int i = 1; i < count_values.length; i++) { if ("n_f".equals(type)) f_follow[i].setLongValue(Long.parseLong(count_values[i])); else if ("n_p".equals(type)) f_precede[i].setLongValue(Long.parseLong(count_values[i])); else if ("n_fp".equals(type)) f_followerprecede[i].setLongValue(Long.parseLong(count_values[i])); } } if (splits.length > 3 && !splits[3].isEmpty()) { // should be follow or followerprecede String[] splits_ = splits[3].split(":"); String type = splits_[0]; String[] count_values = splits_[1].split(","); if (count_values.length > 0) { if ("n_f".equals(type)) f_follow[0].setLongValue(Long.parseLong(count_values[0])); else if ("n_p".equals(type)) f_precede[0].setLongValue(Long.parseLong(count_values[0])); else if ("n_fp".equals(type)) f_followerprecede[0].setLongValue(Long.parseLong(count_values[0])); } for (int i = 1; i < count_values.length; i++) { if ("n_f".equals(type)) f_follow[i].setLongValue(Long.parseLong(count_values[i])); else if ("n_p".equals(type)) f_precede[i].setLongValue(Long.parseLong(count_values[i])); else if ("n_fp".equals(type)) f_followerprecede[i].setLongValue(Long.parseLong(count_values[i])); } } if (splits.length > 4 && !splits[4].isEmpty()) { // should be followerprecede String[] splits_ = splits[4].split(":"); String type = splits_[0]; String[] count_values = splits_[1].split(","); if (count_values.length > 0) { if ("n_f".equals(type)) f_follow[0].setLongValue(Long.parseLong(count_values[0])); else if ("n_p".equals(type)) f_precede[0].setLongValue(Long.parseLong(count_values[0])); else if ("n_fp".equals(type)) f_followerprecede[0].setLongValue(Long.parseLong(count_values[0])); } for (int i = 1; i < count_values.length; i++) { if ("n_f".equals(type)) f_follow[i].setLongValue(Long.parseLong(count_values[i])); else if ("n_p".equals(type)) f_precede[i].setLongValue(Long.parseLong(count_values[i])); else if ("n_fp".equals(type)) f_followerprecede[i].setLongValue(Long.parseLong(count_values[i])); } } writer_ngram.addDocument(doc); while (N.length <= n) { N = ArrayUtils.getConcatinatedArray(N, new Long[][] { { 0L, 0L, 0L, 0L, 0L, 0L } }); S = ArrayUtils.getConcatinatedArray(S, new Long[] { 0L }); } if (num == 1L) N[n][1]++; else if (num == 2L) N[n][2]++; else if (num == 3L) N[n][3]++; else if (num == 4L) N[n][4]++; else N[n][5]++; N[n][0]++; S[n] += num; } catch (Exception e) { LOG.error("Could not process line '{}' in file '{}:{}', malformed line.", line, ngram_joined_counts_file, c, e); } } writer_ngram.forceMergeDeletes(); writer_ngram.commit(); writer_ngram.close(); StringBuilder b = new StringBuilder(String.format( "#%n# Number of times where an ngram occurred: %n# at_least_once, exactly_once, exactly_twice, exactly_three_times, exactly_four_times, five_times_or_more.%n#%nmax_n=%d%nmax_c=6%n", N.length - 1)); for (int n = 1; n < N.length; n++) b.append(String.format("n%d=%s%n", n, StringUtils.join(N[n], ','))); for (int n = 1; n < S.length; n++) b.append(String.format("s%d=%d%n", n, S[n])); FileUtils.writeStringToFile(new File(_index_dir, "__sum_ngrams__"), b.toString()); }
From source file:de.tudarmstadt.lt.lm.app.GenerateNgramIndex.java
License:Apache License
public void create_vocabulary_index(File vocabulary_file) throws IOException { File index_dir = new File(_index_dir, "vocab"); if (index_dir.exists()) { LOG.info("Vocabulary index already exists in directory '{}'.", index_dir.getAbsolutePath()); if (_overwrite) { LOG.info("Overwriting index '{}',", index_dir); index_dir.delete();/* w ww. j a va2s . c o m*/ } else return; } index_dir.mkdirs(); Analyzer analyzer = new KeywordAnalyzer(); IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_4_9, analyzer); iwc.setOpenMode(OpenMode.CREATE); iwc.setRAMBufferSizeMB(1024.0); Directory directory = new MMapDirectory(index_dir); IndexWriter writer_vocab = new IndexWriter(directory, iwc); InputStream in = new FileInputStream(vocabulary_file); if (vocabulary_file.getName().endsWith(".gz")) in = new GZIPInputStream(in); LineIterator iter = new LineIterator(new BufferedReader(new InputStreamReader(in, "UTF-8"))); Document doc = new Document(); Field f_word = new StringField("word", "", Field.Store.YES); doc.add(f_word); long c = 0; while (iter.hasNext()) { if (++c % 10000 == 0) LOG.info("Adding {}'th word.", c); String line = iter.next(); try { String word = line.trim(); f_word.setStringValue(word); writer_vocab.addDocument(doc); } catch (Exception e) { LOG.warn("Could not process line '{}' in file '{}', malformed line.", line, vocabulary_file, e); } } writer_vocab.forceMergeDeletes(); writer_vocab.commit(); writer_vocab.close(); }
From source file:de.uni_koeln.spinfo.maalr.lucene.core.DictionaryCreator.java
License:Apache License
private IndexWriter initIndexWriter() throws IOException { IndexWriterConfig writerConfig = new IndexWriterConfig(LuceneHelper.CURRENT, analyzer); if (!indexAvailable()) { writerConfig.setOpenMode(OpenMode.CREATE); } else {//from ww w . j a v a2 s. co m writerConfig.setOpenMode(OpenMode.CREATE_OR_APPEND); } writerConfig.setRAMBufferSizeMB(512.0); IndexWriter writer = new IndexWriter(indexDirectory, writerConfig); return writer; }
From source file:de.uni_koeln.spinfo.maalr.lucene.core.DictionaryLoader.java
License:Apache License
private IndexWriter initIndexWriter() throws IOException { IndexWriterConfig writerConfig = new IndexWriterConfig(LuceneHelper.CURRENT, LuceneHelper.newAnalyzer()); writerConfig.setOpenMode(OpenMode.APPEND); writerConfig.setRAMBufferSizeMB(512.0); IndexWriter writer = new IndexWriter(ram, writerConfig); return writer; }
From source file:dk.defxws.fgslucene.IndexWriterCache.java
License:Open Source License
/** * get IndexWriter for given indexPath and write it into cache. * /*from w w w . jav a2s.co m*/ * @param indexName * name of index to open. * @param config * gsearch config-Object. * @throws GenericSearchException * e */ private IndexWriter getIndexWriter(final String indexName, final boolean create, final Config config) throws GenericSearchException { if (indexWriters.get(indexName) == null) { IndexWriter iw = null; try { IndexWriterConfig indexWriterConfig = new IndexWriterConfig(Constants.LUCENE_VERSION, getAnalyzer(config.getAnalyzer(indexName))); if (create) { indexWriterConfig.setOpenMode(OpenMode.CREATE); } else { indexWriterConfig.setOpenMode(OpenMode.CREATE_OR_APPEND); } if (config.getMaxBufferedDocs(indexName) > 1) { indexWriterConfig.setMaxBufferedDocs(config.getMaxBufferedDocs(indexName)); } if (config.getRamBufferSizeMb(indexName) > 1) { indexWriterConfig.setRAMBufferSizeMB(config.getRamBufferSizeMb(indexName)); } if (config.getMergeFactor(indexName) > 1 || config.getMaxMergeDocs(indexName) > 1 || config.getMaxMergeMb(indexName) > 1) { LogByteSizeMergePolicy logMergePolicy = new LogByteSizeMergePolicy(); if (config.getMergeFactor(indexName) > 1) { logMergePolicy.setMergeFactor(config.getMergeFactor(indexName)); } if (config.getMaxMergeDocs(indexName) > 1) { logMergePolicy.setMaxMergeDocs(config.getMaxMergeDocs(indexName)); } if (config.getMaxMergeMb(indexName) > 1) { logMergePolicy.setMaxMergeMB(config.getMaxMergeMb(indexName)); } indexWriterConfig.setMergePolicy(logMergePolicy); } if (config.getDefaultWriteLockTimeout(indexName) > 1) { indexWriterConfig.setWriteLockTimeout(config.getDefaultWriteLockTimeout(indexName)); } if (config.getLuceneDirectoryImplementation(indexName) != null) { // Initialize IndexWriter with configured FSDirectory FSDirectory directory = getDirectoryImplementation( config.getLuceneDirectoryImplementation(indexName), new File(config.getIndexDir(indexName))); iw = new IndexWriter(directory, indexWriterConfig); } else { // Initialize IndexWriter with default FSDirectory iw = new IndexWriter(FSDirectory.open(new File(config.getIndexDir(indexName))), indexWriterConfig); } if (config.getMaxChunkSize(indexName) > 1) { if (iw.getDirectory() instanceof MMapDirectory) { ((MMapDirectory) iw.getDirectory()).setMaxChunkSize(config.getMaxChunkSize(indexName)); } } } catch (Exception e) { iw = null; throw new GenericSearchException( "IndexWriter new error, creating index indexName=" + indexName + " :\n", e); } indexWriters.put(indexName, iw); if (logger.isDebugEnabled()) logger.debug("getIndexWriter put to map " + iw); return iw; } return indexWriters.get(indexName); }