List of usage examples for org.apache.lucene.index IndexWriter forceMerge
public void forceMerge(int maxNumSegments) throws IOException
From source file:org.elasticsearch.search.suggest.completion.CompletionPostingsFormatTest.java
License:Apache License
public Lookup buildAnalyzingLookup(final CompletionFieldMapper mapper, String[] terms, String[] surfaces, long[] weights) throws IOException { RAMDirectory dir = new RAMDirectory(); FilterCodec filterCodec = new FilterCodec("filtered", Codec.getDefault()) { public PostingsFormat postingsFormat() { return mapper.postingsFormatProvider().get(); }// w ww.j a v a2 s .c o m }; IndexWriterConfig indexWriterConfig = new IndexWriterConfig(TEST_VERSION_CURRENT, mapper.indexAnalyzer()); indexWriterConfig.setCodec(filterCodec); IndexWriter writer = new IndexWriter(dir, indexWriterConfig); for (int i = 0; i < weights.length; i++) { Document doc = new Document(); BytesRef payload = mapper.buildPayload(new BytesRef(surfaces[i]), weights[i], new BytesRef(Long.toString(weights[i]))); doc.add(mapper.getCompletionField(terms[i], payload)); if (randomBoolean()) { writer.commit(); } writer.addDocument(doc); } writer.commit(); writer.forceMerge(1); writer.commit(); DirectoryReader reader = DirectoryReader.open(writer, true); assertThat(reader.leaves().size(), equalTo(1)); assertThat(reader.leaves().get(0).reader().numDocs(), equalTo(weights.length)); AtomicReaderContext atomicReaderContext = reader.leaves().get(0); Terms luceneTerms = atomicReaderContext.reader().terms(mapper.name()); Lookup lookup = ((Completion090PostingsFormat.CompletionTerms) luceneTerms).getLookup(mapper, new CompletionSuggestionContext(null)); reader.close(); writer.close(); dir.close(); return lookup; }
From source file:org.elasticsearch.test.integration.search.suggest.CompletionPostingsFormatTest.java
License:Apache License
public Lookup buildAnalyzingLookup(final CompletionFieldMapper mapper, String[] terms, String[] surfaces, long[] weights) throws IOException { RAMDirectory dir = new RAMDirectory(); FilterCodec filterCodec = new FilterCodec("filtered", Codec.getDefault()) { public PostingsFormat postingsFormat() { return mapper.postingsFormatProvider().get(); }//from www. j a v a 2 s. c om }; IndexWriterConfig indexWriterConfig = new IndexWriterConfig(TEST_VERSION_CURRENT, mapper.indexAnalyzer()); indexWriterConfig.setCodec(filterCodec); IndexWriter writer = new IndexWriter(dir, indexWriterConfig); for (int i = 0; i < weights.length; i++) { Document doc = new Document(); BytesRef payload = mapper.buildPayload(new BytesRef(surfaces[i]), weights[i], new BytesRef(Long.toString(weights[i]))); doc.add(mapper.getCompletionField(terms[i], payload)); if (randomBoolean()) { writer.commit(); } writer.addDocument(doc); } writer.commit(); writer.forceMerge(1); writer.commit(); DirectoryReader reader = DirectoryReader.open(writer, true); assertThat(reader.leaves().size(), equalTo(1)); assertThat(reader.leaves().get(0).reader().numDocs(), equalTo(weights.length)); AtomicReaderContext atomicReaderContext = reader.leaves().get(0); Terms luceneTerms = atomicReaderContext.reader().terms(mapper.name()); Lookup lookup = ((Completion090PostingsFormat.CompletionTerms) luceneTerms).getLookup(mapper, false); reader.close(); writer.close(); dir.close(); return lookup; }
From source file:org.elasticsearch.test.unit.common.compress.CompressIndexInputOutputTests.java
License:Apache License
private void lucene(Compressor compressor) throws Exception { CompressedDirectory dir = new CompressedDirectory(new RAMDirectory(), compressor, false, "fdt"); IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(Lucene.VERSION, Lucene.STANDARD_ANALYZER)); writer.addDocument(createDoc(1, (int) SizeValue.parseSizeValue("100b").singles())); writer.addDocument(createDoc(2, (int) SizeValue.parseSizeValue("5k").singles())); writer.commit();/* www . j ava2s . c om*/ writer.addDocument(createDoc(3, (int) SizeValue.parseSizeValue("2k").singles())); writer.addDocument(createDoc(4, (int) SizeValue.parseSizeValue("1k").singles())); writer.commit(); verify(writer); writer.forceMerge(1); writer.waitForMerges(); verify(writer); dir.setCompress(false); writer.addDocument(createDoc(5, (int) SizeValue.parseSizeValue("2k").singles())); writer.addDocument(createDoc(6, (int) SizeValue.parseSizeValue("1k").singles())); verify(writer); writer.forceMerge(1); writer.waitForMerges(); verify(writer); writer.close(); }
From source file:org.elasticsearch.test.unit.common.lucene.uid.VersionsTests.java
License:Apache License
@Test public void testMergingOldIndices() throws Exception { final IndexWriterConfig iwConf = new IndexWriterConfig(Lucene.VERSION, new KeywordAnalyzer()); iwConf.setMergePolicy(new IndexUpgraderMergePolicy(iwConf.getMergePolicy())); final Directory dir = newDirectory(); final IndexWriter iw = new IndexWriter(dir, iwConf); // 1st segment, no _version Document document = new Document(); // Add a dummy field (enough to trigger #3237) document.add(new StringField("a", "b", Store.NO)); StringField uid = new StringField(UidFieldMapper.NAME, "1", Store.YES); document.add(uid);/*from w ww.ja v a2s . co m*/ iw.addDocument(document); uid.setStringValue("2"); iw.addDocument(document); iw.commit(); // 2nd segment, old layout document = new Document(); UidField uidAndVersion = new UidField("3", 3L); document.add(uidAndVersion); iw.addDocument(document); uidAndVersion.uid = "4"; uidAndVersion.version = 4L; iw.addDocument(document); iw.commit(); // 3rd segment new layout document = new Document(); uid.setStringValue("5"); Field version = new NumericDocValuesField(UidFieldMapper.VERSION, 5L); document.add(uid); document.add(version); iw.addDocument(document); uid.setStringValue("6"); version.setLongValue(6L); iw.addDocument(document); iw.commit(); final Map<String, Long> expectedVersions = ImmutableMap.<String, Long>builder().put("1", 0L).put("2", 0L) .put("3", 0L).put("4", 4L).put("5", 5L).put("6", 6L).build(); // Force merge and check versions iw.forceMerge(1); final AtomicReader ir = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(iw.getDirectory())); final NumericDocValues versions = ir.getNumericDocValues(UidFieldMapper.VERSION); assertThat(versions, notNullValue()); for (int i = 0; i < ir.maxDoc(); ++i) { final String uidValue = ir.document(i).get(UidFieldMapper.NAME); final long expectedVersion = expectedVersions.get(uidValue); assertThat(versions.get(i), equalTo(expectedVersion)); } iw.close(); assertThat(IndexWriter.isLocked(iw.getDirectory()), is(false)); ir.close(); dir.close(); }
From source file:org.elasticsearch.xpack.core.security.authz.accesscontrol.DocumentSubsetReaderTests.java
License:Open Source License
@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32457") public void testSearch() throws Exception { IndexWriter iw = new IndexWriter(directory, newIndexWriterConfig()); Document document = new Document(); document.add(new StringField("field", "value1", Field.Store.NO)); iw.addDocument(document);//w ww . ja v a 2 s .co m document = new Document(); document.add(new StringField("field", "value2", Field.Store.NO)); iw.addDocument(document); document = new Document(); document.add(new StringField("field", "value3", Field.Store.NO)); iw.addDocument(document); document = new Document(); document.add(new StringField("field", "value4", Field.Store.NO)); iw.addDocument(document); iw.forceMerge(1); iw.deleteDocuments(new Term("field", "value3")); iw.close(); openDirectoryReader(); IndexSearcher indexSearcher = new IndexSearcher(DocumentSubsetReader.wrap(directoryReader, bitsetFilterCache, new TermQuery(new Term("field", "value1")))); assertThat(indexSearcher.getIndexReader().numDocs(), equalTo(1)); TopDocs result = indexSearcher.search(new MatchAllDocsQuery(), 1); assertThat(result.totalHits, equalTo(1L)); assertThat(result.scoreDocs[0].doc, equalTo(0)); indexSearcher = new IndexSearcher(DocumentSubsetReader.wrap(directoryReader, bitsetFilterCache, new TermQuery(new Term("field", "value2")))); assertThat(indexSearcher.getIndexReader().numDocs(), equalTo(1)); result = indexSearcher.search(new MatchAllDocsQuery(), 1); assertThat(result.totalHits, equalTo(1L)); assertThat(result.scoreDocs[0].doc, equalTo(1)); // this doc has been marked as deleted: indexSearcher = new IndexSearcher(DocumentSubsetReader.wrap(directoryReader, bitsetFilterCache, new TermQuery(new Term("field", "value3")))); assertThat(indexSearcher.getIndexReader().numDocs(), equalTo(0)); result = indexSearcher.search(new MatchAllDocsQuery(), 1); assertThat(result.totalHits, equalTo(0L)); indexSearcher = new IndexSearcher(DocumentSubsetReader.wrap(directoryReader, bitsetFilterCache, new TermQuery(new Term("field", "value4")))); assertThat(indexSearcher.getIndexReader().numDocs(), equalTo(1)); result = indexSearcher.search(new MatchAllDocsQuery(), 1); assertThat(result.totalHits, equalTo(1L)); assertThat(result.scoreDocs[0].doc, equalTo(3)); }
From source file:org.elasticsearch.xpack.core.security.authz.accesscontrol.DocumentSubsetReaderTests.java
License:Open Source License
public void testLiveDocs() throws Exception { int numDocs = scaledRandomIntBetween(16, 128); IndexWriter iw = new IndexWriter(directory, new IndexWriterConfig(new StandardAnalyzer()).setMergePolicy(NoMergePolicy.INSTANCE)); for (int i = 0; i < numDocs; i++) { Document document = new Document(); document.add(new StringField("field", "value" + i, Field.Store.NO)); iw.addDocument(document);//from w w w . j a va 2s.c o m } iw.forceMerge(1); iw.close(); openDirectoryReader(); assertThat("should have one segment after force merge", directoryReader.leaves().size(), equalTo(1)); for (int i = 0; i < numDocs; i++) { Query roleQuery = new TermQuery(new Term("field", "value" + i)); DirectoryReader wrappedReader = DocumentSubsetReader.wrap(directoryReader, bitsetFilterCache, roleQuery); LeafReader leafReader = wrappedReader.leaves().get(0).reader(); assertThat(leafReader.hasDeletions(), is(true)); assertThat(leafReader.numDocs(), equalTo(1)); Bits liveDocs = leafReader.getLiveDocs(); assertThat(liveDocs.length(), equalTo(numDocs)); for (int docId = 0; docId < numDocs; docId++) { if (docId == i) { assertThat("docId [" + docId + "] should match", liveDocs.get(docId), is(true)); } else { assertThat("docId [" + docId + "] should not match", liveDocs.get(docId), is(false)); } } } }
From source file:org.elasticsearch.xpack.core.security.authz.accesscontrol.SecurityIndexSearcherWrapperUnitTests.java
License:Open Source License
public void doTestIndexSearcherWrapper(boolean sparse, boolean deletions) throws IOException { Directory dir = newDirectory();//from www . j av a2 s .c o m IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(null)); Document doc = new Document(); StringField allowedField = new StringField("allowed", "yes", Store.NO); doc.add(allowedField); StringField fooField = new StringField("foo", "bar", Store.NO); doc.add(fooField); StringField deleteField = new StringField("delete", "no", Store.NO); doc.add(deleteField); w.addDocument(doc); if (deletions) { // add a document that matches foo:bar but will be deleted deleteField.setStringValue("yes"); w.addDocument(doc); deleteField.setStringValue("no"); } allowedField.setStringValue("no"); w.addDocument(doc); if (sparse) { for (int i = 0; i < 1000; ++i) { w.addDocument(doc); } w.forceMerge(1); } w.deleteDocuments(new Term("delete", "yes")); IndexSettings settings = IndexSettingsModule.newIndexSettings("_index", Settings.EMPTY); BitsetFilterCache.Listener listener = new BitsetFilterCache.Listener() { @Override public void onCache(ShardId shardId, Accountable accountable) { } @Override public void onRemoval(ShardId shardId, Accountable accountable) { } }; DirectoryReader reader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(w), new ShardId(indexSettings.getIndex(), 0)); BitsetFilterCache cache = new BitsetFilterCache(settings, listener); Query roleQuery = new TermQuery(new Term("allowed", "yes")); BitSet bitSet = cache.getBitSetProducer(roleQuery).getBitSet(reader.leaves().get(0)); if (sparse) { assertThat(bitSet, instanceOf(SparseFixedBitSet.class)); } else { assertThat(bitSet, instanceOf(FixedBitSet.class)); } DocumentSubsetDirectoryReader filteredReader = DocumentSubsetReader.wrap(reader, cache, roleQuery); IndexSearcher searcher = new SecurityIndexSearcherWrapper.IndexSearcherWrapper(filteredReader); // Searching a non-existing term will trigger a null scorer assertEquals(0, searcher.count(new TermQuery(new Term("non_existing_field", "non_existing_value")))); assertEquals(1, searcher.count(new TermQuery(new Term("foo", "bar")))); // make sure scorers are created only once, see #1725 assertEquals(1, searcher.count(new CreateScorerOnceQuery(new MatchAllDocsQuery()))); IOUtils.close(reader, w, dir); }
From source file:org.hibernate.search.backend.lucene.work.impl.OptimizeIndexLuceneWork.java
License:LGPL
private CompletableFuture<Long> commitIndex(IndexWriter indexWriter) { try {// ww w .j a va 2 s .c o m indexWriter.forceMerge(1); return CompletableFuture.completedFuture(indexWriter.commit()); } catch (IOException e) { throw log.unableToCommitIndex(getEventContext(), e); } }
From source file:org.niord.core.message.MessageLuceneIndex.java
License:Apache License
/** * Updates the Lucene index/*from w ww . j av a 2 s. co m*/ * * @param maxIndexCount max number of messages to index at a time * @return the number of updates */ private int updateLuceneIndex(int maxIndexCount) { Date lastUpdated = getLastUpdated(); long t0 = System.currentTimeMillis(); log.debug(String.format("Indexing at most %d changed messages since %s", maxIndexCount, lastUpdated)); IndexWriter writer = null; try { // Find all messages changed since the lastUpdated time stamp List<Message> updatedMessages = findUpdatedMessages(lastUpdated, maxIndexCount); if (updatedMessages.size() == 0) { return 0; } // Create a new index writer writer = getNewWriter(); // Update the index with the changes for (Message message : updatedMessages) { indexMessage(writer, message); if (message.getUpdated().after(lastUpdated)) { lastUpdated = message.getUpdated(); } } // Update the last-updated flag setLastUpdated(lastUpdated, writer); // Commit the changes writer.commit(); // Re-open the reader from the writer refreshReader(writer); // Check if we need to optimize the index optimizeIndexCount += updatedMessages.size(); if (optimizeIndexCount > LUCENE_OPTIMIZE_INDEX_COUNT) { writer.forceMerge(LUCENE_MAX_NUM_SEGMENTS); optimizeIndexCount = 0; } log.info("Indexed " + updatedMessages.size() + " messages in " + (System.currentTimeMillis() - t0) + " ms"); return updatedMessages.size(); } catch (Exception ex) { log.error("Error updating Lucene index: " + ex.getMessage(), ex); return 0; } finally { closeWriter(writer); } }
From source file:org.opengrok.indexer.index.IndexDatabase.java
License:Open Source License
/** * Optimize the index database//from w w w. j a v a2s . com * @throws IOException I/O exception */ public void optimize() throws IOException { synchronized (lock) { if (running) { LOGGER.warning("Optimize terminated... Someone else is updating / optimizing it!"); return; } running = true; } IndexWriter wrt = null; IOException writerException = null; try { Statistics elapsed = new Statistics(); String projectDetail = this.project != null ? " for project " + project.getName() : ""; LOGGER.log(Level.INFO, "Optimizing the index{0}", projectDetail); Analyzer analyzer = new StandardAnalyzer(); IndexWriterConfig conf = new IndexWriterConfig(analyzer); conf.setOpenMode(OpenMode.CREATE_OR_APPEND); wrt = new IndexWriter(indexDirectory, conf); wrt.forceMerge(1); // this is deprecated and not needed anymore elapsed.report(LOGGER, String.format("Done optimizing index%s", projectDetail)); synchronized (lock) { if (dirtyFile.exists() && !dirtyFile.delete()) { LOGGER.log(Level.FINE, "Failed to remove \"dirty-file\": {0}", dirtyFile.getAbsolutePath()); } dirty = false; } } catch (IOException e) { writerException = e; LOGGER.log(Level.SEVERE, "ERROR: optimizing index: {0}", e); } finally { if (wrt != null) { try { wrt.close(); } catch (IOException e) { if (writerException == null) { writerException = e; } LOGGER.log(Level.WARNING, "An error occurred while closing writer", e); } } synchronized (lock) { running = false; } } if (writerException != null) { throw writerException; } }