List of usage examples for org.apache.lucene.index LogDocMergePolicy LogDocMergePolicy
public LogDocMergePolicy()
From source file:org.apache.solr.index.LogDocMergePolicyFactory.java
License:Apache License
@Override protected MergePolicy getMergePolicyInstance() { return new LogDocMergePolicy(); }
From source file:org.apache.solr.uninverting.TestFieldCache.java
License:Apache License
@BeforeClass public static void beforeClass() throws Exception { NUM_DOCS = atLeast(500);// w w w .j a va2s .c o m NUM_ORDS = atLeast(2); directory = newDirectory(); IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(new LogDocMergePolicy())); long theLong = Long.MAX_VALUE; double theDouble = Double.MAX_VALUE; int theInt = Integer.MAX_VALUE; float theFloat = Float.MAX_VALUE; unicodeStrings = new String[NUM_DOCS]; multiValued = new BytesRef[NUM_DOCS][NUM_ORDS]; if (VERBOSE) { System.out.println("TEST: setUp"); } for (int i = 0; i < NUM_DOCS; i++) { Document doc = new Document(); doc.add(new LongPoint("theLong", theLong--)); doc.add(new DoublePoint("theDouble", theDouble--)); doc.add(new IntPoint("theInt", theInt--)); doc.add(new FloatPoint("theFloat", theFloat--)); if (i % 2 == 0) { doc.add(new IntPoint("sparse", i)); } if (i % 2 == 0) { doc.add(new IntPoint("numInt", i)); } // sometimes skip the field: if (random().nextInt(40) != 17) { unicodeStrings[i] = generateString(i); doc.add(newStringField("theRandomUnicodeString", unicodeStrings[i], Field.Store.YES)); } // sometimes skip the field: if (random().nextInt(10) != 8) { for (int j = 0; j < NUM_ORDS; j++) { String newValue = generateString(i); multiValued[i][j] = new BytesRef(newValue); doc.add(newStringField("theRandomUnicodeMultiValuedField", newValue, Field.Store.YES)); } Arrays.sort(multiValued[i]); } writer.addDocument(doc); } writer.forceMerge(1); // this test relies on one segment and docid order IndexReader r = DirectoryReader.open(writer); assertEquals(1, r.leaves().size()); reader = r.leaves().get(0).reader(); TestUtil.checkReader(reader); writer.close(); }
From source file:org.compass.core.lucene.engine.merge.policy.LogDocMergePolicyProvider.java
License:Apache License
public MergePolicy create(CompassSettings settings) throws SearchEngineException { LogDocMergePolicy mergePolicy = new LogDocMergePolicy(); mergePolicy.setMaxMergeDocs(settings.getSettingAsInt(LuceneEnvironment.MergePolicy.LogDoc.MAX_MERGE_DOCS, LogDocMergePolicy.DEFAULT_MAX_MERGE_DOCS)); mergePolicy.setMinMergeDocs(settings.getSettingAsInt(LuceneEnvironment.MergePolicy.LogDoc.MIN_MERGE_DOCS, LogDocMergePolicy.DEFAULT_MIN_MERGE_DOCS)); return mergePolicy; }
From source file:org.elasticsearch.index.engine.InternalEngineTests.java
License:Apache License
public void testRenewSyncFlush() throws Exception { final int iters = randomIntBetween(2, 5); // run this a couple of times to get some coverage for (int i = 0; i < iters; i++) { try (Store store = createStore(); InternalEngine engine = new InternalEngine(config(defaultSettings, store, createTempDir(), new MergeSchedulerConfig(defaultSettings), new LogDocMergePolicy()), false)) { final String syncId = randomUnicodeOfCodepointLengthBetween(10, 20); ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null);/* w w w.j a v a 2 s . c om*/ Engine.Index doc1 = new Engine.Index(newUid("1"), doc); engine.index(doc1); engine.flush(); Engine.Index doc2 = new Engine.Index(newUid("2"), doc); engine.index(doc2); engine.flush(); final boolean forceMergeFlushes = randomBoolean(); engine.index(new Engine.Index(newUid("3"), doc)); Engine.CommitId commitID = engine.flush(); assertEquals("should succeed to flush commit with right id and no pending doc", engine.syncFlush(syncId, commitID), Engine.SyncedFlushResult.SUCCESS); assertEquals(3, engine.segments(false).size()); engine.forceMerge(forceMergeFlushes, 1, false, false, false); if (forceMergeFlushes == false) { engine.refresh("make all segments visible"); assertEquals(4, engine.segments(false).size()); assertEquals(store.readLastCommittedSegmentsInfo().getUserData().get(Engine.SYNC_COMMIT_ID), syncId); assertEquals(engine.getLastCommittedSegmentInfos().getUserData().get(Engine.SYNC_COMMIT_ID), syncId); assertTrue(engine.tryRenewSyncCommit()); assertEquals(1, engine.segments(false).size()); } else { assertEquals(1, engine.segments(false).size()); } assertEquals(store.readLastCommittedSegmentsInfo().getUserData().get(Engine.SYNC_COMMIT_ID), syncId); assertEquals(engine.getLastCommittedSegmentInfos().getUserData().get(Engine.SYNC_COMMIT_ID), syncId); if (randomBoolean()) { Engine.Index doc4 = new Engine.Index(newUid("4"), doc); engine.index(doc4); } else { Engine.Delete delete = new Engine.Delete(doc1.type(), doc1.id(), doc1.uid()); engine.delete(delete); } assertFalse(engine.tryRenewSyncCommit()); engine.flush(); assertNull(store.readLastCommittedSegmentsInfo().getUserData().get(Engine.SYNC_COMMIT_ID)); assertNull(engine.getLastCommittedSegmentInfos().getUserData().get(Engine.SYNC_COMMIT_ID)); } } }
From source file:org.fracturedatlas.athena.apa.indexing.IndexingApaAdapter.java
License:Open Source License
public void initializeIndex() { analyzer = new WhitespaceLowerCaseAnalyzer(); config = new IndexWriterConfig(Version.LUCENE_32, analyzer); LogMergePolicy mergePolicy = new LogDocMergePolicy(); mergePolicy.setMergeFactor(MERGE_FACTOR); config.setMergePolicy(mergePolicy);//from www. java 2 s . com }
From source file:org.meresco.lucene.numerate.UriEnumerate.java
License:Open Source License
/** * * @param path//from w ww.j av a 2 s . c o m * @param max_cache_size * @param withTransactionLog allows for crash recovery, but slows down UriNumerate considerably because of file system flush. * @throws IOException */ public UriEnumerate(String path, int max_cache_size, boolean withTransactionLog) throws IOException { IndexWriterConfig config = new IndexWriterConfig(null); ConcurrentMergeScheduler ms = (ConcurrentMergeScheduler) config.getMergeScheduler(); ms.setDefaultMaxMergesAndThreads(/* spins= */false); LogDocMergePolicy mp = new LogDocMergePolicy(); mp.setMergeFactor(2); mp.setMinMergeDocs(max_cache_size); config.setMergePolicy(mp); config.setCodec(new Lucene60Codec() { @Override public PostingsFormat getPostingsFormatForField(String field) { return new BloomFilteringPostingsFormat(super.getPostingsFormatForField(field)); } }); config.setUseCompoundFile(false); this.writer = new IndexWriter(FSDirectory.open(FileSystems.getDefault().getPath(path)), config); this.next_ord = writer.numDocs() + 1; this.searcher = new SimpleSearcher(this.writer); this.cache = new Cache(max_cache_size, () -> this.commit()); this.transactionLog = new TransactionLog(withTransactionLog ? path + "/transactionLog" : null); this.transactionLog.maybeRecover(); }
From source file:org.olat.search.service.indexer.JmsIndexer.java
License:Apache License
public LogMergePolicy newLogMergePolicy() { LogMergePolicy logmp = new LogDocMergePolicy(); logmp.setCalibrateSizeByDeletes(true); logmp.setMergeFactor(INDEX_MERGE_FACTOR); return logmp; }
From source file:org.silverpeas.core.index.indexing.model.IndexManager.java
License:Open Source License
/** * * Returns an IndexWriter to the index stored at the given path.The index directory and files are * created if not found .//from ww w . j a v a2s . c o m * * @param path the path to the index root directory * @param language the language of the indexed documents. * @return an IndexWriter or null if the index can't be found or create or read. */ private IndexWriter getIndexWriter(String path, String language) { indexWriters.computeIfPresent(path, (s, w) -> w.isOpen() ? w : null); return indexWriters.computeIfAbsent(path, p -> { try { final File file = new File(path); if (!file.exists()) { file.mkdirs(); } final LogDocMergePolicy policy = new LogDocMergePolicy(); policy.setMergeFactor(mergeFactor); policy.setMaxMergeDocs(maxMergeDocs); final IndexWriterConfig configuration = new IndexWriterConfig(getAnalyzer(language)) .setRAMBufferSizeMB(defaultRamBufferSizeMb).setMergePolicy(policy); return new IndexWriter(FSDirectory.open(file.toPath()), configuration); } catch (IOException e) { SilverLogger.getLogger(this).error("Unknown index file " + path, e); } // The map is not filled return null; }); }
From source file:org.silverpeas.search.indexEngine.model.IndexManager.java
License:Open Source License
/** * * Returns an IndexWriter to the index stored at the given path.The index directory and files are * created if not found ./*from w ww. ja va 2s . co m*/ * * @param path the path to the index root directory * @param language the language of the indexed documents. * @return an IndexWriter or null if the index can't be found or create or read. */ private IndexWriter getIndexWriter(String path, String language) { IndexWriter writer = indexWriters.get(path); if (writer == null) { try { File file = new File(path); if (!file.exists()) { file.mkdirs(); } LogDocMergePolicy policy = new LogDocMergePolicy(); policy.setMergeFactor(mergeFactor); policy.setMaxMergeDocs(maxMergeDocs); IndexWriterConfig configuration = new IndexWriterConfig(LUCENE_36, getAnalyzer(language)) .setRAMBufferSizeMB(RAMBufferSizeMB).setMergePolicy(policy); writer = new IndexWriter(FSDirectory.open(file), configuration); } catch (IOException e) { IOUtils.closeQuietly(writer); writer = null; SilverTrace.error("indexEngine", "IndexManager.getIndexWriter", "indexEngine.MSG_UNKNOWN_INDEX_FILE", path, e); } if (writer != null) { indexWriters.put(path, writer); } } return writer; }
From source file:org.wikimedia.revdiffsearch.IndexMerger.java
License:Open Source License
/** Index all text files under a directory. */ public static void main(String[] args) { if (args.length != 2) { System.out.println("Usage: java -jar IndexMerger.jar " + "merged_index_dir existing_index_dir1 existing_index_dir2 ..."); System.out.println(" merged_index_dir: A directory where the merged " + "index will be stored"); System.out.println(" e.g. merged_indexes"); System.out// ww w. j a v a 2s .c o m .println(" existing_indexes_dir: A directory where the " + "indexes that have to merged exist"); System.out.println(" e.g. indexes/"); System.out.println(" e.g. index1"); System.out.println(" e.g. index2"); System.out.println(" e.g. index3"); System.exit(1); } int ramBufferSizeMB = 1024; int ngram = 3; File INDEX_DIR = new File(args[0]); { String s; if ((s = System.getProperty("ngram")) != null) { ngram = Integer.parseInt(s); } if ((s = System.getProperty("ramBufferSize")) != null) { ramBufferSizeMB = Integer.parseInt(s); } } INDEX_DIR.mkdir(); Date start = new Date(); try { IndexWriterConfig cfg = new IndexWriterConfig(Version.LUCENE_44, new SimpleNGramAnalyzer(ngram)); LogDocMergePolicy lmp = new LogDocMergePolicy(); lmp.setMergeFactor(1000); cfg.setRAMBufferSizeMB(ramBufferSizeMB); cfg.setMergePolicy(lmp); IndexWriter writer = new IndexWriter(FSDirectory.open(INDEX_DIR), cfg); // IndexWriter writer = new IndexWriter(INDEX_DIR, // new StandardAnalyzer(Version.LUCENE_44), // true); // writer.setMergeFactor(1000); // writer.setRAMBufferSizeMB(50); List<Directory> indexes = new ArrayList<Directory>(); for (String indexdir : Arrays.asList(args).subList(1, args.length)) { System.out.println("Adding: " + indexdir); indexes.add(FSDirectory.open(new File(indexdir))); } System.out.print("Merging added indexes..."); writer.addIndexes(indexes.toArray(new Directory[indexes.size()])); System.out.println("done"); writer.close(); System.out.println("done"); Date end = new Date(); System.out.println("It took: " + ((end.getTime() - start.getTime()) / 1000) + "\""); } catch (IOException e) { e.printStackTrace(); } }