Java tutorial
/* * Licensed to Elasticsearch under one or more contributor * license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright * ownership. Elasticsearch licenses this file to you under * the Apache License, Version 2.0 (the "License"); you may * not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.elasticsearch.index.engine; import com.google.common.collect.ImmutableMap; import org.apache.log4j.AppenderSkeleton; import org.apache.log4j.Level; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.log4j.spi.LoggingEvent; import org.apache.lucene.codecs.Codec; import org.apache.lucene.document.Field; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.TextField; import org.apache.lucene.index.*; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.Directory; import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.TestUtil; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; import org.elasticsearch.action.support.TransportActions; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.Base64; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.index.Index; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.analysis.AnalysisService; import org.elasticsearch.index.codec.CodecService; import org.elasticsearch.index.deletionpolicy.KeepOnlyLastDeletionPolicy; import org.elasticsearch.index.deletionpolicy.SnapshotDeletionPolicy; import org.elasticsearch.index.engine.Engine.Searcher; import org.elasticsearch.index.indexing.ShardIndexingService; import org.elasticsearch.index.mapper.*; import org.elasticsearch.index.mapper.Mapper.BuilderContext; import org.elasticsearch.index.mapper.ParseContext.Document; import org.elasticsearch.index.mapper.internal.SourceFieldMapper; import org.elasticsearch.index.mapper.internal.UidFieldMapper; import org.elasticsearch.index.mapper.object.RootObjectMapper; import org.elasticsearch.index.shard.MergeSchedulerConfig; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.shard.ShardUtils; import org.elasticsearch.index.shard.TranslogRecoveryPerformer; import org.elasticsearch.index.similarity.SimilarityLookupService; import org.elasticsearch.index.store.DirectoryService; import org.elasticsearch.index.store.DirectoryUtils; import org.elasticsearch.index.store.Store; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.index.translog.TranslogConfig; import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.indices.mapper.MapperRegistry; import org.elasticsearch.test.DummyShardLock; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; import org.hamcrest.MatcherAssert; import org.junit.After; import org.junit.Before; import org.junit.Test; import java.io.IOException; import java.io.InputStream; import java.nio.charset.Charset; import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; import java.util.*; import java.util.concurrent.BrokenBarrierException; import java.util.concurrent.CountDownLatch; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.regex.Pattern; import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS; import static org.elasticsearch.index.engine.Engine.Operation.Origin.PRIMARY; import static org.elasticsearch.index.engine.Engine.Operation.Origin.REPLICA; import static org.hamcrest.Matchers.*; public class InternalEngineTests extends ESTestCase { private static final Pattern PARSE_LEGACY_ID_PATTERN = Pattern .compile("^" + Translog.TRANSLOG_FILE_PREFIX + "(\\d+)((\\.recovering))?$"); protected final ShardId shardId = new ShardId(new Index("index"), 1); protected ThreadPool threadPool; private Store store; private Store storeReplica; protected InternalEngine engine; protected InternalEngine replicaEngine; private Settings defaultSettings; private String codecName; private Path primaryTranslogDir; private Path replicaTranslogDir; @Override @Before public void setUp() throws Exception { super.setUp(); CodecService codecService = new CodecService(shardId.index()); String name = Codec.getDefault().getName(); if (Arrays.asList(codecService.availableCodecs()).contains(name)) { // some codecs are read only so we only take the ones that we have in the service and randomly // selected by lucene test case. codecName = name; } else { codecName = "default"; } defaultSettings = Settings.builder().put(EngineConfig.INDEX_COMPOUND_ON_FLUSH, randomBoolean()) .put(EngineConfig.INDEX_GC_DELETES_SETTING, "1h") // make sure this doesn't kick in on us .put(EngineConfig.INDEX_CODEC_SETTING, codecName) .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(); // TODO randomize more settings threadPool = new ThreadPool(getClass().getName()); store = createStore(); storeReplica = createStore(); Lucene.cleanLuceneIndex(store.directory()); Lucene.cleanLuceneIndex(storeReplica.directory()); primaryTranslogDir = createTempDir("translog-primary"); engine = createEngine(store, primaryTranslogDir); LiveIndexWriterConfig currentIndexWriterConfig = engine.getCurrentIndexWriterConfig(); assertEquals(engine.config().getCodec().getName(), codecService.codec(codecName).getName()); assertEquals(currentIndexWriterConfig.getCodec().getName(), codecService.codec(codecName).getName()); if (randomBoolean()) { engine.config().setEnableGcDeletes(false); } replicaTranslogDir = createTempDir("translog-replica"); replicaEngine = createEngine(storeReplica, replicaTranslogDir); currentIndexWriterConfig = replicaEngine.getCurrentIndexWriterConfig(); assertEquals(replicaEngine.config().getCodec().getName(), codecService.codec(codecName).getName()); assertEquals(currentIndexWriterConfig.getCodec().getName(), codecService.codec(codecName).getName()); if (randomBoolean()) { engine.config().setEnableGcDeletes(false); } } @Override @After public void tearDown() throws Exception { super.tearDown(); IOUtils.close(replicaEngine, storeReplica, engine, store); terminate(threadPool); } private Document testDocumentWithTextField() { Document document = testDocument(); document.add(new TextField("value", "test", Field.Store.YES)); return document; } private Document testDocument() { return new Document(); } private ParsedDocument testParsedDocument(String uid, String id, String type, String routing, long timestamp, long ttl, Document document, BytesReference source, Mapping mappingUpdate) { Field uidField = new Field("_uid", uid, UidFieldMapper.Defaults.FIELD_TYPE); Field versionField = new NumericDocValuesField("_version", 0); document.add(uidField); document.add(versionField); return new ParsedDocument(uidField, versionField, id, type, routing, timestamp, ttl, Arrays.asList(document), source, mappingUpdate); } protected Store createStore() throws IOException { return createStore(newDirectory()); } protected Store createStore(final Directory directory) throws IOException { final DirectoryService directoryService = new DirectoryService(shardId, EMPTY_SETTINGS) { @Override public Directory newDirectory() throws IOException { return directory; } @Override public long throttleTimeInNanos() { return 0; } }; return new Store(shardId, EMPTY_SETTINGS, directoryService, new DummyShardLock(shardId)); } protected Translog createTranslog() throws IOException { return createTranslog(primaryTranslogDir); } protected Translog createTranslog(Path translogPath) throws IOException { TranslogConfig translogConfig = new TranslogConfig(shardId, translogPath, EMPTY_SETTINGS, Translog.Durabilty.REQUEST, BigArrays.NON_RECYCLING_INSTANCE, threadPool); return new Translog(translogConfig); } protected IndexDeletionPolicy createIndexDeletionPolicy() { return new KeepOnlyLastDeletionPolicy(shardId, EMPTY_SETTINGS); } protected SnapshotDeletionPolicy createSnapshotDeletionPolicy() { return new SnapshotDeletionPolicy(createIndexDeletionPolicy()); } protected InternalEngine createEngine(Store store, Path translogPath, IndexSearcherWrapper... wrappers) { return createEngine(defaultSettings, store, translogPath, new MergeSchedulerConfig(defaultSettings), newMergePolicy(), wrappers); } protected InternalEngine createEngine(Settings indexSettings, Store store, Path translogPath, MergeSchedulerConfig mergeSchedulerConfig, MergePolicy mergePolicy, IndexSearcherWrapper... wrappers) { return new InternalEngine( config(indexSettings, store, translogPath, mergeSchedulerConfig, mergePolicy, wrappers), false); } public EngineConfig config(Settings indexSettings, Store store, Path translogPath, MergeSchedulerConfig mergeSchedulerConfig, MergePolicy mergePolicy, IndexSearcherWrapper... wrappers) { IndexWriterConfig iwc = newIndexWriterConfig(); TranslogConfig translogConfig = new TranslogConfig(shardId, translogPath, indexSettings, Translog.Durabilty.REQUEST, BigArrays.NON_RECYCLING_INSTANCE, threadPool); EngineConfig config = new EngineConfig(shardId, threadPool, new ShardIndexingService(shardId, indexSettings), indexSettings, null, store, createSnapshotDeletionPolicy(), mergePolicy, mergeSchedulerConfig, iwc.getAnalyzer(), iwc.getSimilarity(), new CodecService(shardId.index()), new Engine.FailedEngineListener() { @Override public void onFailedEngine(ShardId shardId, String reason, @Nullable Throwable t) { // we don't need to notify anybody in this test } }, new TranslogHandler(shardId.index().getName(), logger), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), new IndexSearcherWrappingService(new HashSet<>(Arrays.asList(wrappers))), translogConfig); try { config.setCreate(Lucene.indexExists(store.directory()) == false); } catch (IOException e) { throw new ElasticsearchException("can't find index?", e); } return config; } protected static final BytesReference B_1 = new BytesArray(new byte[] { 1 }); protected static final BytesReference B_2 = new BytesArray(new byte[] { 2 }); protected static final BytesReference B_3 = new BytesArray(new byte[] { 3 }); @Test public void testSegments() throws Exception { try (Store store = createStore(); Engine engine = createEngine(defaultSettings, store, createTempDir(), new MergeSchedulerConfig(defaultSettings), NoMergePolicy.INSTANCE)) { List<Segment> segments = engine.segments(false); assertThat(segments.isEmpty(), equalTo(true)); assertThat(engine.segmentsStats().getCount(), equalTo(0l)); assertThat(engine.segmentsStats().getMemoryInBytes(), equalTo(0l)); final boolean defaultCompound = defaultSettings.getAsBoolean(EngineConfig.INDEX_COMPOUND_ON_FLUSH, true); // create a doc and refresh ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null); engine.create(new Engine.Create(newUid("1"), doc)); ParsedDocument doc2 = testParsedDocument("2", "2", "test", null, -1, -1, testDocumentWithTextField(), B_2, null); engine.create(new Engine.Create(newUid("2"), doc2)); engine.refresh("test"); segments = engine.segments(false); assertThat(segments.size(), equalTo(1)); SegmentsStats stats = engine.segmentsStats(); assertThat(stats.getCount(), equalTo(1l)); assertThat(stats.getTermsMemoryInBytes(), greaterThan(0l)); assertThat(stats.getStoredFieldsMemoryInBytes(), greaterThan(0l)); assertThat(stats.getTermVectorsMemoryInBytes(), equalTo(0l)); assertThat(stats.getNormsMemoryInBytes(), greaterThan(0l)); assertThat(stats.getDocValuesMemoryInBytes(), greaterThan(0l)); assertThat(segments.get(0).isCommitted(), equalTo(false)); assertThat(segments.get(0).isSearch(), equalTo(true)); assertThat(segments.get(0).getNumDocs(), equalTo(2)); assertThat(segments.get(0).getDeletedDocs(), equalTo(0)); assertThat(segments.get(0).isCompound(), equalTo(defaultCompound)); assertThat(segments.get(0).ramTree, nullValue()); engine.flush(); segments = engine.segments(false); assertThat(segments.size(), equalTo(1)); assertThat(engine.segmentsStats().getCount(), equalTo(1l)); assertThat(segments.get(0).isCommitted(), equalTo(true)); assertThat(segments.get(0).isSearch(), equalTo(true)); assertThat(segments.get(0).getNumDocs(), equalTo(2)); assertThat(segments.get(0).getDeletedDocs(), equalTo(0)); assertThat(segments.get(0).isCompound(), equalTo(defaultCompound)); engine.config().setCompoundOnFlush(false); engine.onSettingsChanged(); ParsedDocument doc3 = testParsedDocument("3", "3", "test", null, -1, -1, testDocumentWithTextField(), B_3, null); engine.create(new Engine.Create(newUid("3"), doc3)); engine.refresh("test"); segments = engine.segments(false); assertThat(segments.size(), equalTo(2)); assertThat(engine.segmentsStats().getCount(), equalTo(2l)); assertThat(engine.segmentsStats().getTermsMemoryInBytes(), greaterThan(stats.getTermsMemoryInBytes())); assertThat(engine.segmentsStats().getStoredFieldsMemoryInBytes(), greaterThan(stats.getStoredFieldsMemoryInBytes())); assertThat(engine.segmentsStats().getTermVectorsMemoryInBytes(), equalTo(0l)); assertThat(engine.segmentsStats().getNormsMemoryInBytes(), greaterThan(stats.getNormsMemoryInBytes())); assertThat(engine.segmentsStats().getDocValuesMemoryInBytes(), greaterThan(stats.getDocValuesMemoryInBytes())); assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true)); assertThat(segments.get(0).isCommitted(), equalTo(true)); assertThat(segments.get(0).isSearch(), equalTo(true)); assertThat(segments.get(0).getNumDocs(), equalTo(2)); assertThat(segments.get(0).getDeletedDocs(), equalTo(0)); assertThat(segments.get(0).isCompound(), equalTo(defaultCompound)); assertThat(segments.get(1).isCommitted(), equalTo(false)); assertThat(segments.get(1).isSearch(), equalTo(true)); assertThat(segments.get(1).getNumDocs(), equalTo(1)); assertThat(segments.get(1).getDeletedDocs(), equalTo(0)); assertThat(segments.get(1).isCompound(), equalTo(false)); engine.delete(new Engine.Delete("test", "1", newUid("1"))); engine.refresh("test"); segments = engine.segments(false); assertThat(segments.size(), equalTo(2)); assertThat(engine.segmentsStats().getCount(), equalTo(2l)); assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true)); assertThat(segments.get(0).isCommitted(), equalTo(true)); assertThat(segments.get(0).isSearch(), equalTo(true)); assertThat(segments.get(0).getNumDocs(), equalTo(1)); assertThat(segments.get(0).getDeletedDocs(), equalTo(1)); assertThat(segments.get(0).isCompound(), equalTo(defaultCompound)); assertThat(segments.get(1).isCommitted(), equalTo(false)); assertThat(segments.get(1).isSearch(), equalTo(true)); assertThat(segments.get(1).getNumDocs(), equalTo(1)); assertThat(segments.get(1).getDeletedDocs(), equalTo(0)); assertThat(segments.get(1).isCompound(), equalTo(false)); engine.config().setCompoundOnFlush(true); engine.onSettingsChanged(); ParsedDocument doc4 = testParsedDocument("4", "4", "test", null, -1, -1, testDocumentWithTextField(), B_3, null); engine.create(new Engine.Create(newUid("4"), doc4)); engine.refresh("test"); segments = engine.segments(false); assertThat(segments.size(), equalTo(3)); assertThat(engine.segmentsStats().getCount(), equalTo(3l)); assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true)); assertThat(segments.get(0).isCommitted(), equalTo(true)); assertThat(segments.get(0).isSearch(), equalTo(true)); assertThat(segments.get(0).getNumDocs(), equalTo(1)); assertThat(segments.get(0).getDeletedDocs(), equalTo(1)); assertThat(segments.get(0).isCompound(), equalTo(defaultCompound)); assertThat(segments.get(1).isCommitted(), equalTo(false)); assertThat(segments.get(1).isSearch(), equalTo(true)); assertThat(segments.get(1).getNumDocs(), equalTo(1)); assertThat(segments.get(1).getDeletedDocs(), equalTo(0)); assertThat(segments.get(1).isCompound(), equalTo(false)); assertThat(segments.get(2).isCommitted(), equalTo(false)); assertThat(segments.get(2).isSearch(), equalTo(true)); assertThat(segments.get(2).getNumDocs(), equalTo(1)); assertThat(segments.get(2).getDeletedDocs(), equalTo(0)); assertThat(segments.get(2).isCompound(), equalTo(true)); } } public void testVerboseSegments() throws Exception { try (Store store = createStore(); Engine engine = createEngine(defaultSettings, store, createTempDir(), new MergeSchedulerConfig(defaultSettings), NoMergePolicy.INSTANCE)) { List<Segment> segments = engine.segments(true); assertThat(segments.isEmpty(), equalTo(true)); ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null); engine.create(new Engine.Create(newUid("1"), doc)); engine.refresh("test"); segments = engine.segments(true); assertThat(segments.size(), equalTo(1)); assertThat(segments.get(0).ramTree, notNullValue()); ParsedDocument doc2 = testParsedDocument("2", "2", "test", null, -1, -1, testDocumentWithTextField(), B_2, null); engine.create(new Engine.Create(newUid("2"), doc2)); engine.refresh("test"); ParsedDocument doc3 = testParsedDocument("3", "3", "test", null, -1, -1, testDocumentWithTextField(), B_3, null); engine.create(new Engine.Create(newUid("3"), doc3)); engine.refresh("test"); segments = engine.segments(true); assertThat(segments.size(), equalTo(3)); assertThat(segments.get(0).ramTree, notNullValue()); assertThat(segments.get(1).ramTree, notNullValue()); assertThat(segments.get(2).ramTree, notNullValue()); } } @Test public void testSegmentsWithMergeFlag() throws Exception { try (Store store = createStore(); Engine engine = createEngine(defaultSettings, store, createTempDir(), new MergeSchedulerConfig(defaultSettings), new TieredMergePolicy())) { ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocument(), B_1, null); Engine.Index index = new Engine.Index(newUid("1"), doc); engine.index(index); engine.flush(); assertThat(engine.segments(false).size(), equalTo(1)); index = new Engine.Index(newUid("2"), doc); engine.index(index); engine.flush(); List<Segment> segments = engine.segments(false); assertThat(segments.size(), equalTo(2)); for (Segment segment : segments) { assertThat(segment.getMergeId(), nullValue()); } index = new Engine.Index(newUid("3"), doc); engine.index(index); engine.flush(); segments = engine.segments(false); assertThat(segments.size(), equalTo(3)); for (Segment segment : segments) { assertThat(segment.getMergeId(), nullValue()); } index = new Engine.Index(newUid("4"), doc); engine.index(index); engine.flush(); final long gen1 = store.readLastCommittedSegmentsInfo().getGeneration(); // now, optimize and wait for merges, see that we have no merge flag engine.forceMerge(true); for (Segment segment : engine.segments(false)) { assertThat(segment.getMergeId(), nullValue()); } // we could have multiple underlying merges, so the generation may increase more than once assertTrue(store.readLastCommittedSegmentsInfo().getGeneration() > gen1); final boolean flush = randomBoolean(); final long gen2 = store.readLastCommittedSegmentsInfo().getGeneration(); engine.forceMerge(flush); for (Segment segment : engine.segments(false)) { assertThat(segment.getMergeId(), nullValue()); } if (flush) { // we should have had just 1 merge, so last generation should be exact assertEquals(gen2, store.readLastCommittedSegmentsInfo().getLastGeneration()); } } } public void testCommitStats() { Document document = testDocumentWithTextField(); document.add(new Field(SourceFieldMapper.NAME, B_1.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE)); ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, document, B_1, null); engine.create(new Engine.Create(newUid("1"), doc)); CommitStats stats1 = engine.commitStats(); assertThat(stats1.getGeneration(), greaterThan(0l)); assertThat(stats1.getId(), notNullValue()); assertThat(stats1.getUserData(), hasKey(Translog.TRANSLOG_GENERATION_KEY)); engine.flush(true, true); CommitStats stats2 = engine.commitStats(); assertThat(stats2.getGeneration(), greaterThan(stats1.getGeneration())); assertThat(stats2.getId(), notNullValue()); assertThat(stats2.getId(), not(equalTo(stats1.getId()))); assertThat(stats2.getUserData(), hasKey(Translog.TRANSLOG_GENERATION_KEY)); assertThat(stats2.getUserData(), hasKey(Translog.TRANSLOG_UUID_KEY)); assertThat(stats2.getUserData().get(Translog.TRANSLOG_GENERATION_KEY), not(equalTo(stats1.getUserData().get(Translog.TRANSLOG_GENERATION_KEY)))); assertThat(stats2.getUserData().get(Translog.TRANSLOG_UUID_KEY), equalTo(stats1.getUserData().get(Translog.TRANSLOG_UUID_KEY))); } @Test public void testIndexSearcherWrapper() throws Exception { final AtomicInteger counter = new AtomicInteger(); IndexSearcherWrapper wrapper = new IndexSearcherWrapper() { @Override public DirectoryReader wrap(DirectoryReader reader) { counter.incrementAndGet(); return reader; } @Override public IndexSearcher wrap(EngineConfig engineConfig, IndexSearcher searcher) throws EngineException { counter.incrementAndGet(); return searcher; } }; Store store = createStore(); Path translog = createTempDir("translog-test"); InternalEngine engine = createEngine(store, translog, wrapper); Engine.Searcher searcher = engine.acquireSearcher("test"); assertThat(counter.get(), equalTo(2)); searcher.close(); IOUtils.close(store, engine); } @Test public void testInternalSearcherNotWrapped() throws Exception { IndexSearcherWrapper wrapper = new IndexSearcherWrapper() { @Override public DirectoryReader wrap(DirectoryReader reader) { throw new IllegalStateException("don't wrap internal ops"); } @Override public IndexSearcher wrap(EngineConfig engineConfig, IndexSearcher searcher) throws EngineException { throw new IllegalStateException("don't wrap internal ops"); } }; Store store = createStore(); Path translog = createTempDir("wrapper-test"); InternalEngine engine = createEngine(store, translog, wrapper); try { Engine.Searcher searcher = engine.acquireSearcher("test"); fail("wait what?"); } catch (EngineException ex) { // all well } // create a document Document document = testDocumentWithTextField(); document.add(new Field(SourceFieldMapper.NAME, B_1.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE)); ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, document, B_1, null); engine.create(new Engine.Create(newUid("1"), doc)); engine.flush(); // now do an update document = testDocument(); document.add(new TextField("value", "test1", Field.Store.YES)); document.add(new Field(SourceFieldMapper.NAME, B_2.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE)); doc = testParsedDocument("1", "1", "test", null, -1, -1, document, B_2, null); engine.index(new Engine.Index(newUid("1"), doc)); List<Segment> segments = engine.segments(randomBoolean()); assertTrue(segments.size() >= 1); SegmentsStats segmentsStats = engine.segmentsStats(); assertTrue(segmentsStats.getCount() >= 1); IOUtils.close(store, engine); } @Test /* */ public void testConcurrentGetAndFlush() throws Exception { ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null); engine.create(new Engine.Create(newUid("1"), doc)); final AtomicReference<Engine.GetResult> latestGetResult = new AtomicReference<>(); latestGetResult.set(engine.get(new Engine.Get(true, newUid("1")))); final AtomicBoolean flushFinished = new AtomicBoolean(false); final CyclicBarrier barrier = new CyclicBarrier(2); Thread getThread = new Thread() { @Override public void run() { try { barrier.await(); } catch (InterruptedException | BrokenBarrierException e) { throw new RuntimeException(e); } while (flushFinished.get() == false) { Engine.GetResult previousGetResult = latestGetResult.get(); if (previousGetResult != null) { previousGetResult.release(); } latestGetResult.set(engine.get(new Engine.Get(true, newUid("1")))); if (latestGetResult.get().exists() == false) { break; } } } }; getThread.start(); barrier.await(); engine.flush(); flushFinished.set(true); getThread.join(); assertTrue(latestGetResult.get().exists()); latestGetResult.get().release(); } @Test public void testSimpleOperations() throws Exception { Engine.Searcher searchResult = engine.acquireSearcher("test"); MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0)); searchResult.close(); // create a document Document document = testDocumentWithTextField(); document.add(new Field(SourceFieldMapper.NAME, B_1.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE)); ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, document, B_1, null); engine.create(new Engine.Create(newUid("1"), doc)); // its not there... searchResult = engine.acquireSearcher("test"); MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0)); MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher .engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 0)); searchResult.close(); // but, we can still get it (in realtime) Engine.GetResult getResult = engine.get(new Engine.Get(true, newUid("1"))); assertThat(getResult.exists(), equalTo(true)); assertThat(getResult.source().source.toBytesArray(), equalTo(B_1.toBytesArray())); assertThat(getResult.docIdAndVersion(), nullValue()); getResult.release(); // but, not there non realtime getResult = engine.get(new Engine.Get(false, newUid("1"))); assertThat(getResult.exists(), equalTo(false)); getResult.release(); // refresh and it should be there engine.refresh("test"); // now its there... searchResult = engine.acquireSearcher("test"); MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1)); MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher .engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 1)); searchResult.close(); // also in non realtime getResult = engine.get(new Engine.Get(false, newUid("1"))); assertThat(getResult.exists(), equalTo(true)); assertThat(getResult.docIdAndVersion(), notNullValue()); getResult.release(); // now do an update document = testDocument(); document.add(new TextField("value", "test1", Field.Store.YES)); document.add(new Field(SourceFieldMapper.NAME, B_2.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE)); doc = testParsedDocument("1", "1", "test", null, -1, -1, document, B_2, null); engine.index(new Engine.Index(newUid("1"), doc)); // its not updated yet... searchResult = engine.acquireSearcher("test"); MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1)); MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher .engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 1)); MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher .engineSearcherTotalHits(new TermQuery(new Term("value", "test1")), 0)); searchResult.close(); // but, we can still get it (in realtime) getResult = engine.get(new Engine.Get(true, newUid("1"))); assertThat(getResult.exists(), equalTo(true)); assertThat(getResult.source().source.toBytesArray(), equalTo(B_2.toBytesArray())); assertThat(getResult.docIdAndVersion(), nullValue()); getResult.release(); // refresh and it should be updated engine.refresh("test"); searchResult = engine.acquireSearcher("test"); MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1)); MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher .engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 0)); MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher .engineSearcherTotalHits(new TermQuery(new Term("value", "test1")), 1)); searchResult.close(); // now delete engine.delete(new Engine.Delete("test", "1", newUid("1"))); // its not deleted yet searchResult = engine.acquireSearcher("test"); MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1)); MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher .engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 0)); MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher .engineSearcherTotalHits(new TermQuery(new Term("value", "test1")), 1)); searchResult.close(); // but, get should not see it (in realtime) getResult = engine.get(new Engine.Get(true, newUid("1"))); assertThat(getResult.exists(), equalTo(false)); getResult.release(); // refresh and it should be deleted engine.refresh("test"); searchResult = engine.acquireSearcher("test"); MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0)); MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher .engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 0)); MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher .engineSearcherTotalHits(new TermQuery(new Term("value", "test1")), 0)); searchResult.close(); // add it back document = testDocumentWithTextField(); document.add(new Field(SourceFieldMapper.NAME, B_1.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE)); doc = testParsedDocument("1", "1", "test", null, -1, -1, document, B_1, null); engine.create(new Engine.Create(newUid("1"), doc)); // its not there... searchResult = engine.acquireSearcher("test"); MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0)); MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher .engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 0)); MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher .engineSearcherTotalHits(new TermQuery(new Term("value", "test1")), 0)); searchResult.close(); // refresh and it should be there engine.refresh("test"); // now its there... searchResult = engine.acquireSearcher("test"); MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1)); MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher .engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 1)); MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher .engineSearcherTotalHits(new TermQuery(new Term("value", "test1")), 0)); searchResult.close(); // now flush engine.flush(); // and, verify get (in real time) getResult = engine.get(new Engine.Get(true, newUid("1"))); assertThat(getResult.exists(), equalTo(true)); assertThat(getResult.source(), nullValue()); assertThat(getResult.docIdAndVersion(), notNullValue()); getResult.release(); // make sure we can still work with the engine // now do an update document = testDocument(); document.add(new TextField("value", "test1", Field.Store.YES)); doc = testParsedDocument("1", "1", "test", null, -1, -1, document, B_1, null); engine.index(new Engine.Index(newUid("1"), doc)); // its not updated yet... searchResult = engine.acquireSearcher("test"); MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1)); MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher .engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 1)); MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher .engineSearcherTotalHits(new TermQuery(new Term("value", "test1")), 0)); searchResult.close(); // refresh and it should be updated engine.refresh("test"); searchResult = engine.acquireSearcher("test"); MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1)); MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher .engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 0)); MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher .engineSearcherTotalHits(new TermQuery(new Term("value", "test1")), 1)); searchResult.close(); } @Test public void testSearchResultRelease() throws Exception { Engine.Searcher searchResult = engine.acquireSearcher("test"); MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0)); searchResult.close(); // create a document ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null); engine.create(new Engine.Create(newUid("1"), doc)); // its not there... searchResult = engine.acquireSearcher("test"); MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0)); MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher .engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 0)); searchResult.close(); // refresh and it should be there engine.refresh("test"); // now its there... searchResult = engine.acquireSearcher("test"); MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1)); MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher .engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 1)); // don't release the search result yet... // delete, refresh and do a new search, it should not be there engine.delete(new Engine.Delete("test", "1", newUid("1"))); engine.refresh("test"); Engine.Searcher updateSearchResult = engine.acquireSearcher("test"); MatcherAssert.assertThat(updateSearchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0)); updateSearchResult.close(); // the non release search result should not see the deleted yet... MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1)); MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher .engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 1)); searchResult.close(); } public void testSyncedFlush() throws IOException { try (Store store = createStore(); Engine engine = new InternalEngine(config(defaultSettings, store, createTempDir(), new MergeSchedulerConfig(defaultSettings), new LogByteSizeMergePolicy()), false)) { final String syncId = randomUnicodeOfCodepointLengthBetween(10, 20); ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null); engine.create(new Engine.Create(newUid("1"), doc)); Engine.CommitId commitID = engine.flush(); assertThat(commitID, equalTo(new Engine.CommitId(store.readLastCommittedSegmentsInfo().getId()))); byte[] wrongBytes = Base64.decode(commitID.toString()); wrongBytes[0] = (byte) ~wrongBytes[0]; Engine.CommitId wrongId = new Engine.CommitId(wrongBytes); assertEquals("should fail to sync flush with wrong id (but no docs)", engine.syncFlush(syncId + "1", wrongId), Engine.SyncedFlushResult.COMMIT_MISMATCH); engine.create(new Engine.Create(newUid("2"), doc)); assertEquals("should fail to sync flush with right id but pending doc", engine.syncFlush(syncId + "2", commitID), Engine.SyncedFlushResult.PENDING_OPERATIONS); commitID = engine.flush(); assertEquals("should succeed to flush commit with right id and no pending doc", engine.syncFlush(syncId, commitID), Engine.SyncedFlushResult.SUCCESS); assertEquals(store.readLastCommittedSegmentsInfo().getUserData().get(Engine.SYNC_COMMIT_ID), syncId); assertEquals(engine.getLastCommittedSegmentInfos().getUserData().get(Engine.SYNC_COMMIT_ID), syncId); } } public void testRenewSyncFlush() throws Exception { final int iters = randomIntBetween(2, 5); // run this a couple of times to get some coverage for (int i = 0; i < iters; i++) { try (Store store = createStore(); InternalEngine engine = new InternalEngine(config(defaultSettings, store, createTempDir(), new MergeSchedulerConfig(defaultSettings), new LogDocMergePolicy()), false)) { final String syncId = randomUnicodeOfCodepointLengthBetween(10, 20); ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null); Engine.Index doc1 = new Engine.Index(newUid("1"), doc); engine.index(doc1); engine.flush(); Engine.Index doc2 = new Engine.Index(newUid("2"), doc); engine.index(doc2); engine.flush(); final boolean forceMergeFlushes = randomBoolean(); engine.index(new Engine.Index(newUid("3"), doc)); Engine.CommitId commitID = engine.flush(); assertEquals("should succeed to flush commit with right id and no pending doc", engine.syncFlush(syncId, commitID), Engine.SyncedFlushResult.SUCCESS); assertEquals(3, engine.segments(false).size()); engine.forceMerge(forceMergeFlushes, 1, false, false, false); if (forceMergeFlushes == false) { engine.refresh("make all segments visible"); assertEquals(4, engine.segments(false).size()); assertEquals(store.readLastCommittedSegmentsInfo().getUserData().get(Engine.SYNC_COMMIT_ID), syncId); assertEquals(engine.getLastCommittedSegmentInfos().getUserData().get(Engine.SYNC_COMMIT_ID), syncId); assertTrue(engine.tryRenewSyncCommit()); assertEquals(1, engine.segments(false).size()); } else { assertEquals(1, engine.segments(false).size()); } assertEquals(store.readLastCommittedSegmentsInfo().getUserData().get(Engine.SYNC_COMMIT_ID), syncId); assertEquals(engine.getLastCommittedSegmentInfos().getUserData().get(Engine.SYNC_COMMIT_ID), syncId); if (randomBoolean()) { Engine.Index doc4 = new Engine.Index(newUid("4"), doc); engine.index(doc4); } else { Engine.Delete delete = new Engine.Delete(doc1.type(), doc1.id(), doc1.uid()); engine.delete(delete); } assertFalse(engine.tryRenewSyncCommit()); engine.flush(); assertNull(store.readLastCommittedSegmentsInfo().getUserData().get(Engine.SYNC_COMMIT_ID)); assertNull(engine.getLastCommittedSegmentInfos().getUserData().get(Engine.SYNC_COMMIT_ID)); } } } public void testSycnedFlushSurvivesEngineRestart() throws IOException { final String syncId = randomUnicodeOfCodepointLengthBetween(10, 20); ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null); engine.create(new Engine.Create(newUid("1"), doc)); final Engine.CommitId commitID = engine.flush(); assertEquals("should succeed to flush commit with right id and no pending doc", engine.syncFlush(syncId, commitID), Engine.SyncedFlushResult.SUCCESS); assertEquals(store.readLastCommittedSegmentsInfo().getUserData().get(Engine.SYNC_COMMIT_ID), syncId); assertEquals(engine.getLastCommittedSegmentInfos().getUserData().get(Engine.SYNC_COMMIT_ID), syncId); EngineConfig config = engine.config(); if (randomBoolean()) { engine.close(); } else { engine.flushAndClose(); } engine = new InternalEngine(config, randomBoolean()); assertEquals(engine.getLastCommittedSegmentInfos().getUserData().get(Engine.SYNC_COMMIT_ID), syncId); } public void testSycnedFlushVanishesOnReplay() throws IOException { final String syncId = randomUnicodeOfCodepointLengthBetween(10, 20); ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null); engine.create(new Engine.Create(newUid("1"), doc)); final Engine.CommitId commitID = engine.flush(); assertEquals("should succeed to flush commit with right id and no pending doc", engine.syncFlush(syncId, commitID), Engine.SyncedFlushResult.SUCCESS); assertEquals(store.readLastCommittedSegmentsInfo().getUserData().get(Engine.SYNC_COMMIT_ID), syncId); assertEquals(engine.getLastCommittedSegmentInfos().getUserData().get(Engine.SYNC_COMMIT_ID), syncId); doc = testParsedDocument("2", "2", "test", null, -1, -1, testDocumentWithTextField(), new BytesArray("{}"), null); engine.create(new Engine.Create(newUid("2"), doc)); EngineConfig config = engine.config(); engine.close(); final MockDirectoryWrapper directory = DirectoryUtils.getLeaf(store.directory(), MockDirectoryWrapper.class); if (directory != null) { // since we rollback the IW we are writing the same segment files again after starting IW but MDW prevents // this so we have to disable the check explicitly directory.setPreventDoubleWrite(false); } config.setCreate(false); engine = new InternalEngine(config, false); assertNull("Sync ID must be gone since we have a document to replay", engine.getLastCommittedSegmentInfos().getUserData().get(Engine.SYNC_COMMIT_ID)); } @Test public void testVersioningNewCreate() { ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocument(), B_1, null); Engine.Create create = new Engine.Create(newUid("1"), doc); engine.create(create); assertThat(create.version(), equalTo(1l)); create = new Engine.Create(newUid("1"), doc, create.version(), create.versionType().versionTypeForReplicationAndRecovery(), REPLICA, 0); replicaEngine.create(create); assertThat(create.version(), equalTo(1l)); } @Test public void testExternalVersioningNewCreate() { ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocument(), B_1, null); Engine.Create create = new Engine.Create(newUid("1"), doc, 12, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, 0); engine.create(create); assertThat(create.version(), equalTo(12l)); create = new Engine.Create(newUid("1"), doc, create.version(), create.versionType().versionTypeForReplicationAndRecovery(), REPLICA, 0); replicaEngine.create(create); assertThat(create.version(), equalTo(12l)); } @Test public void testVersioningNewIndex() { ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocument(), B_1, null); Engine.Index index = new Engine.Index(newUid("1"), doc); engine.index(index); assertThat(index.version(), equalTo(1l)); index = new Engine.Index(newUid("1"), doc, index.version(), index.versionType().versionTypeForReplicationAndRecovery(), REPLICA, 0); replicaEngine.index(index); assertThat(index.version(), equalTo(1l)); } @Test public void testExternalVersioningNewIndex() { ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocument(), B_1, null); Engine.Index index = new Engine.Index(newUid("1"), doc, 12, VersionType.EXTERNAL, PRIMARY, 0); engine.index(index); assertThat(index.version(), equalTo(12l)); index = new Engine.Index(newUid("1"), doc, index.version(), index.versionType().versionTypeForReplicationAndRecovery(), REPLICA, 0); replicaEngine.index(index); assertThat(index.version(), equalTo(12l)); } @Test public void testVersioningIndexConflict() { ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocument(), B_1, null); Engine.Index index = new Engine.Index(newUid("1"), doc); engine.index(index); assertThat(index.version(), equalTo(1l)); index = new Engine.Index(newUid("1"), doc); engine.index(index); assertThat(index.version(), equalTo(2l)); index = new Engine.Index(newUid("1"), doc, 1l, VersionType.INTERNAL, Engine.Operation.Origin.PRIMARY, 0); try { engine.index(index); fail(); } catch (VersionConflictEngineException e) { // all is well } // future versions should not work as well index = new Engine.Index(newUid("1"), doc, 3l, VersionType.INTERNAL, PRIMARY, 0); try { engine.index(index); fail(); } catch (VersionConflictEngineException e) { // all is well } } @Test public void testExternalVersioningIndexConflict() { ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocument(), B_1, null); Engine.Index index = new Engine.Index(newUid("1"), doc, 12, VersionType.EXTERNAL, PRIMARY, 0); engine.index(index); assertThat(index.version(), equalTo(12l)); index = new Engine.Index(newUid("1"), doc, 14, VersionType.EXTERNAL, PRIMARY, 0); engine.index(index); assertThat(index.version(), equalTo(14l)); index = new Engine.Index(newUid("1"), doc, 13, VersionType.EXTERNAL, PRIMARY, 0); try { engine.index(index); fail(); } catch (VersionConflictEngineException e) { // all is well } } @Test public void testVersioningIndexConflictWithFlush() { ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocument(), B_1, null); Engine.Index index = new Engine.Index(newUid("1"), doc); engine.index(index); assertThat(index.version(), equalTo(1l)); index = new Engine.Index(newUid("1"), doc); engine.index(index); assertThat(index.version(), equalTo(2l)); engine.flush(); index = new Engine.Index(newUid("1"), doc, 1l, VersionType.INTERNAL, PRIMARY, 0); try { engine.index(index); fail(); } catch (VersionConflictEngineException e) { // all is well } // future versions should not work as well index = new Engine.Index(newUid("1"), doc, 3l, VersionType.INTERNAL, PRIMARY, 0); try { engine.index(index); fail(); } catch (VersionConflictEngineException e) { // all is well } } @Test public void testExternalVersioningIndexConflictWithFlush() { ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocument(), B_1, null); Engine.Index index = new Engine.Index(newUid("1"), doc, 12, VersionType.EXTERNAL, PRIMARY, 0); engine.index(index); assertThat(index.version(), equalTo(12l)); index = new Engine.Index(newUid("1"), doc, 14, VersionType.EXTERNAL, PRIMARY, 0); engine.index(index); assertThat(index.version(), equalTo(14l)); engine.flush(); index = new Engine.Index(newUid("1"), doc, 13, VersionType.EXTERNAL, PRIMARY, 0); try { engine.index(index); fail(); } catch (VersionConflictEngineException e) { // all is well } } public void testForceMerge() throws IOException { try (Store store = createStore(); Engine engine = new InternalEngine(config(defaultSettings, store, createTempDir(), new MergeSchedulerConfig(defaultSettings), new LogByteSizeMergePolicy()), false)) { // use log MP here we test some behavior in ESMP int numDocs = randomIntBetween(10, 100); for (int i = 0; i < numDocs; i++) { ParsedDocument doc = testParsedDocument(Integer.toString(i), Integer.toString(i), "test", null, -1, -1, testDocument(), B_1, null); Engine.Index index = new Engine.Index(newUid(Integer.toString(i)), doc); engine.index(index); engine.refresh("test"); } try (Engine.Searcher test = engine.acquireSearcher("test")) { assertEquals(numDocs, test.reader().numDocs()); } engine.forceMerge(true, 1, false, false, false); assertEquals(engine.segments(true).size(), 1); ParsedDocument doc = testParsedDocument(Integer.toString(0), Integer.toString(0), "test", null, -1, -1, testDocument(), B_1, null); Engine.Index index = new Engine.Index(newUid(Integer.toString(0)), doc); engine.delete(new Engine.Delete(index.type(), index.id(), index.uid())); engine.forceMerge(true, 10, true, false, false); //expunge deletes assertEquals(engine.segments(true).size(), 1); try (Engine.Searcher test = engine.acquireSearcher("test")) { assertEquals(numDocs - 1, test.reader().numDocs()); assertEquals(engine.config().getMergePolicy().toString(), numDocs - 1, test.reader().maxDoc()); } doc = testParsedDocument(Integer.toString(1), Integer.toString(1), "test", null, -1, -1, testDocument(), B_1, null); index = new Engine.Index(newUid(Integer.toString(1)), doc); engine.delete(new Engine.Delete(index.type(), index.id(), index.uid())); engine.forceMerge(true, 10, false, false, false); //expunge deletes assertEquals(engine.segments(true).size(), 1); try (Engine.Searcher test = engine.acquireSearcher("test")) { assertEquals(numDocs - 2, test.reader().numDocs()); assertEquals(numDocs - 1, test.reader().maxDoc()); } } } public void testForceMergeAndClose() throws IOException, InterruptedException { int numIters = randomIntBetween(2, 10); for (int j = 0; j < numIters; j++) { try (Store store = createStore()) { final InternalEngine engine = createEngine(store, createTempDir()); final CountDownLatch startGun = new CountDownLatch(1); final CountDownLatch indexed = new CountDownLatch(1); Thread thread = new Thread() { public void run() { try { try { startGun.await(); } catch (InterruptedException e) { throw new RuntimeException(e); } int i = 0; while (true) { int numDocs = randomIntBetween(1, 20); for (int j = 0; j < numDocs; j++) { i++; ParsedDocument doc = testParsedDocument(Integer.toString(i), Integer.toString(i), "test", null, -1, -1, testDocument(), B_1, null); Engine.Index index = new Engine.Index(newUid(Integer.toString(i)), doc); engine.index(index); } engine.refresh("test"); indexed.countDown(); try { engine.forceMerge(randomBoolean(), 1, false, randomBoolean(), randomBoolean()); } catch (IOException e) { return; } } } catch (AlreadyClosedException | EngineClosedException ex) { // fine } } }; thread.start(); startGun.countDown(); int someIters = randomIntBetween(1, 10); for (int i = 0; i < someIters; i++) { engine.forceMerge(randomBoolean(), 1, false, randomBoolean(), randomBoolean()); } indexed.await(); IOUtils.close(engine); thread.join(); } } } @Test public void testVersioningDeleteConflict() { ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocument(), B_1, null); Engine.Index index = new Engine.Index(newUid("1"), doc); engine.index(index); assertThat(index.version(), equalTo(1l)); index = new Engine.Index(newUid("1"), doc); engine.index(index); assertThat(index.version(), equalTo(2l)); Engine.Delete delete = new Engine.Delete("test", "1", newUid("1"), 1l, VersionType.INTERNAL, PRIMARY, 0, false); try { engine.delete(delete); fail(); } catch (VersionConflictEngineException e) { // all is well } // future versions should not work as well delete = new Engine.Delete("test", "1", newUid("1"), 3l, VersionType.INTERNAL, PRIMARY, 0, false); try { engine.delete(delete); fail(); } catch (VersionConflictEngineException e) { // all is well } // now actually delete delete = new Engine.Delete("test", "1", newUid("1"), 2l, VersionType.INTERNAL, PRIMARY, 0, false); engine.delete(delete); assertThat(delete.version(), equalTo(3l)); // now check if we can index to a delete doc with version index = new Engine.Index(newUid("1"), doc, 2l, VersionType.INTERNAL, PRIMARY, 0); try { engine.index(index); fail(); } catch (VersionConflictEngineException e) { // all is well } // we shouldn't be able to create as well Engine.Create create = new Engine.Create(newUid("1"), doc, 2l, VersionType.INTERNAL, PRIMARY, 0); try { engine.create(create); } catch (VersionConflictEngineException e) { // all is well } } @Test public void testVersioningDeleteConflictWithFlush() { ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocument(), B_1, null); Engine.Index index = new Engine.Index(newUid("1"), doc); engine.index(index); assertThat(index.version(), equalTo(1l)); index = new Engine.Index(newUid("1"), doc); engine.index(index); assertThat(index.version(), equalTo(2l)); engine.flush(); Engine.Delete delete = new Engine.Delete("test", "1", newUid("1"), 1l, VersionType.INTERNAL, PRIMARY, 0, false); try { engine.delete(delete); fail(); } catch (VersionConflictEngineException e) { // all is well } // future versions should not work as well delete = new Engine.Delete("test", "1", newUid("1"), 3l, VersionType.INTERNAL, PRIMARY, 0, false); try { engine.delete(delete); fail(); } catch (VersionConflictEngineException e) { // all is well } engine.flush(); // now actually delete delete = new Engine.Delete("test", "1", newUid("1"), 2l, VersionType.INTERNAL, PRIMARY, 0, false); engine.delete(delete); assertThat(delete.version(), equalTo(3l)); engine.flush(); // now check if we can index to a delete doc with version index = new Engine.Index(newUid("1"), doc, 2l, VersionType.INTERNAL, PRIMARY, 0); try { engine.index(index); fail(); } catch (VersionConflictEngineException e) { // all is well } // we shouldn't be able to create as well Engine.Create create = new Engine.Create(newUid("1"), doc, 2l, VersionType.INTERNAL, PRIMARY, 0); try { engine.create(create); } catch (VersionConflictEngineException e) { // all is well } } @Test public void testVersioningCreateExistsException() { ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocument(), B_1, null); Engine.Create create = new Engine.Create(newUid("1"), doc, Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, 0); engine.create(create); assertThat(create.version(), equalTo(1l)); create = new Engine.Create(newUid("1"), doc, Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, 0); try { engine.create(create); fail(); } catch (DocumentAlreadyExistsException e) { // all is well } } @Test public void testVersioningCreateExistsExceptionWithFlush() { ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocument(), B_1, null); Engine.Create create = new Engine.Create(newUid("1"), doc, Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, 0); engine.create(create); assertThat(create.version(), equalTo(1l)); engine.flush(); create = new Engine.Create(newUid("1"), doc, Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, 0); try { engine.create(create); fail(); } catch (DocumentAlreadyExistsException e) { // all is well } } @Test public void testVersioningReplicaConflict1() { ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocument(), B_1, null); Engine.Index index = new Engine.Index(newUid("1"), doc); engine.index(index); assertThat(index.version(), equalTo(1l)); index = new Engine.Index(newUid("1"), doc); engine.index(index); assertThat(index.version(), equalTo(2l)); // apply the second index to the replica, should work fine index = new Engine.Index(newUid("1"), doc, index.version(), VersionType.INTERNAL.versionTypeForReplicationAndRecovery(), REPLICA, 0); replicaEngine.index(index); assertThat(index.version(), equalTo(2l)); // now, the old one should not work index = new Engine.Index(newUid("1"), doc, 1l, VersionType.INTERNAL.versionTypeForReplicationAndRecovery(), REPLICA, 0); try { replicaEngine.index(index); fail(); } catch (VersionConflictEngineException e) { // all is well } // second version on replica should fail as well try { index = new Engine.Index(newUid("1"), doc, 2l, VersionType.INTERNAL.versionTypeForReplicationAndRecovery(), REPLICA, 0); replicaEngine.index(index); assertThat(index.version(), equalTo(2l)); } catch (VersionConflictEngineException e) { // all is well } } @Test public void testVersioningReplicaConflict2() { ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocument(), B_1, null); Engine.Index index = new Engine.Index(newUid("1"), doc); engine.index(index); assertThat(index.version(), equalTo(1l)); // apply the first index to the replica, should work fine index = new Engine.Index(newUid("1"), doc, 1l, VersionType.INTERNAL.versionTypeForReplicationAndRecovery(), REPLICA, 0); replicaEngine.index(index); assertThat(index.version(), equalTo(1l)); // index it again index = new Engine.Index(newUid("1"), doc); engine.index(index); assertThat(index.version(), equalTo(2l)); // now delete it Engine.Delete delete = new Engine.Delete("test", "1", newUid("1")); engine.delete(delete); assertThat(delete.version(), equalTo(3l)); // apply the delete on the replica (skipping the second index) delete = new Engine.Delete("test", "1", newUid("1"), 3l, VersionType.INTERNAL.versionTypeForReplicationAndRecovery(), REPLICA, 0, false); replicaEngine.delete(delete); assertThat(delete.version(), equalTo(3l)); // second time delete with same version should fail try { delete = new Engine.Delete("test", "1", newUid("1"), 3l, VersionType.INTERNAL.versionTypeForReplicationAndRecovery(), REPLICA, 0, false); replicaEngine.delete(delete); fail("excepted VersionConflictEngineException to be thrown"); } catch (VersionConflictEngineException e) { // all is well } // now do the second index on the replica, it should fail try { index = new Engine.Index(newUid("1"), doc, 2l, VersionType.INTERNAL.versionTypeForReplicationAndRecovery(), REPLICA, 0); replicaEngine.index(index); fail("excepted VersionConflictEngineException to be thrown"); } catch (VersionConflictEngineException e) { // all is well } } @Test public void testBasicCreatedFlag() { ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocument(), B_1, null); Engine.Index index = new Engine.Index(newUid("1"), doc); assertTrue(engine.index(index)); index = new Engine.Index(newUid("1"), doc); assertFalse(engine.index(index)); engine.delete(new Engine.Delete(null, "1", newUid("1"))); index = new Engine.Index(newUid("1"), doc); assertTrue(engine.index(index)); } @Test public void testCreatedFlagAfterFlush() { ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocument(), B_1, null); Engine.Index index = new Engine.Index(newUid("1"), doc); assertTrue(engine.index(index)); engine.delete(new Engine.Delete(null, "1", newUid("1"))); engine.flush(); index = new Engine.Index(newUid("1"), doc); assertTrue(engine.index(index)); } private static class MockAppender extends AppenderSkeleton { public boolean sawIndexWriterMessage; public boolean sawIndexWriterIFDMessage; @Override protected void append(LoggingEvent event) { if (event.getLevel() == Level.TRACE && event.getMessage().toString().contains("[index][1] ")) { if (event.getLoggerName().endsWith("lucene.iw") && event.getMessage().toString().contains("IW: apply all deletes during flush")) { sawIndexWriterMessage = true; } if (event.getLoggerName().endsWith("lucene.iw.ifd")) { sawIndexWriterIFDMessage = true; } } } @Override public boolean requiresLayout() { return false; } @Override public void close() { } } // #5891: make sure IndexWriter's infoStream output is // sent to lucene.iw with log level TRACE: @Test public void testIndexWriterInfoStream() { assumeFalse("who tests the tester?", VERBOSE); MockAppender mockAppender = new MockAppender(); Logger rootLogger = Logger.getRootLogger(); Level savedLevel = rootLogger.getLevel(); rootLogger.addAppender(mockAppender); rootLogger.setLevel(Level.DEBUG); try { // First, with DEBUG, which should NOT log IndexWriter output: ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null); engine.create(new Engine.Create(newUid("1"), doc)); engine.flush(); assertFalse(mockAppender.sawIndexWriterMessage); // Again, with TRACE, which should log IndexWriter output: rootLogger.setLevel(Level.TRACE); engine.create(new Engine.Create(newUid("2"), doc)); engine.flush(); assertTrue(mockAppender.sawIndexWriterMessage); } finally { rootLogger.removeAppender(mockAppender); rootLogger.setLevel(savedLevel); } } // #8603: make sure we can separately log IFD's messages public void testIndexWriterIFDInfoStream() { assumeFalse("who tests the tester?", VERBOSE); MockAppender mockAppender = new MockAppender(); // Works when running this test inside Intellij: Logger iwIFDLogger = LogManager.exists("org.elasticsearch.index.engine.lucene.iw.ifd"); if (iwIFDLogger == null) { // Works when running this test from command line: iwIFDLogger = LogManager.exists("index.engine.lucene.iw.ifd"); assertNotNull(iwIFDLogger); } iwIFDLogger.addAppender(mockAppender); iwIFDLogger.setLevel(Level.DEBUG); try { // First, with DEBUG, which should NOT log IndexWriter output: ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null); engine.create(new Engine.Create(newUid("1"), doc)); engine.flush(); assertFalse(mockAppender.sawIndexWriterMessage); assertFalse(mockAppender.sawIndexWriterIFDMessage); // Again, with TRACE, which should only log IndexWriter IFD output: iwIFDLogger.setLevel(Level.TRACE); engine.create(new Engine.Create(newUid("2"), doc)); engine.flush(); assertFalse(mockAppender.sawIndexWriterMessage); assertTrue(mockAppender.sawIndexWriterIFDMessage); } finally { iwIFDLogger.removeAppender(mockAppender); iwIFDLogger.setLevel(null); } } @Test public void testEnableGcDeletes() throws Exception { try (Store store = createStore(); Engine engine = new InternalEngine(config(defaultSettings, store, createTempDir(), new MergeSchedulerConfig(defaultSettings), newMergePolicy()), false)) { engine.config().setEnableGcDeletes(false); // Add document Document document = testDocument(); document.add(new TextField("value", "test1", Field.Store.YES)); ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, document, B_2, null); engine.index(new Engine.Index(newUid("1"), doc, 1, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), false)); // Delete document we just added: engine.delete(new Engine.Delete("test", "1", newUid("1"), 10, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), false)); // Get should not find the document Engine.GetResult getResult = engine.get(new Engine.Get(true, newUid("1"))); assertThat(getResult.exists(), equalTo(false)); // Give the gc pruning logic a chance to kick in Thread.sleep(1000); if (randomBoolean()) { engine.refresh("test"); } // Delete non-existent document engine.delete(new Engine.Delete("test", "2", newUid("2"), 10, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), false)); // Get should not find the document (we never indexed uid=2): getResult = engine.get(new Engine.Get(true, newUid("2"))); assertThat(getResult.exists(), equalTo(false)); // Try to index uid=1 with a too-old version, should fail: try { engine.index(new Engine.Index(newUid("1"), doc, 2, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime())); fail("did not hit expected exception"); } catch (VersionConflictEngineException vcee) { // expected } // Get should still not find the document getResult = engine.get(new Engine.Get(true, newUid("1"))); assertThat(getResult.exists(), equalTo(false)); // Try to index uid=2 with a too-old version, should fail: try { engine.index(new Engine.Index(newUid("2"), doc, 2, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime())); fail("did not hit expected exception"); } catch (VersionConflictEngineException vcee) { // expected } // Get should not find the document getResult = engine.get(new Engine.Get(true, newUid("2"))); assertThat(getResult.exists(), equalTo(false)); } } protected Term newUid(String id) { return new Term("_uid", id); } @Test public void testExtractShardId() { try (Engine.Searcher test = this.engine.acquireSearcher("test")) { ShardId shardId = ShardUtils.extractShardId(test.getDirectoryReader()); assertNotNull(shardId); assertEquals(shardId, engine.config().getShardId()); } } /** * Random test that throws random exception and ensures all references are * counted down / released and resources are closed. */ @Test public void testFailStart() throws IOException { // this test fails if any reader, searcher or directory is not closed - MDW FTW final int iters = scaledRandomIntBetween(10, 100); for (int i = 0; i < iters; i++) { MockDirectoryWrapper wrapper = newMockDirectory(); wrapper.setFailOnOpenInput(randomBoolean()); wrapper.setAllowRandomFileNotFoundException(randomBoolean()); wrapper.setRandomIOExceptionRate(randomDouble()); wrapper.setRandomIOExceptionRateOnOpen(randomDouble()); final Path translogPath = createTempDir("testFailStart"); try (Store store = createStore(wrapper)) { int refCount = store.refCount(); assertTrue("refCount: " + store.refCount(), store.refCount() > 0); InternalEngine holder; try { holder = createEngine(store, translogPath); } catch (EngineCreationFailureException ex) { assertEquals(store.refCount(), refCount); continue; } assertEquals(store.refCount(), refCount + 1); final int numStarts = scaledRandomIntBetween(1, 5); for (int j = 0; j < numStarts; j++) { try { assertEquals(store.refCount(), refCount + 1); holder.close(); holder = createEngine(store, translogPath); assertEquals(store.refCount(), refCount + 1); } catch (EngineCreationFailureException ex) { // all is fine assertEquals(store.refCount(), refCount); break; } } holder.close(); assertEquals(store.refCount(), refCount); } } } @Test public void testSettings() { CodecService codecService = new CodecService(shardId.index()); LiveIndexWriterConfig currentIndexWriterConfig = engine.getCurrentIndexWriterConfig(); assertEquals(engine.config().getCodec().getName(), codecService.codec(codecName).getName()); assertEquals(currentIndexWriterConfig.getCodec().getName(), codecService.codec(codecName).getName()); } @Test public void testRetryWithAutogeneratedIdWorksAndNoDuplicateDocs() throws IOException { ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocument(), B_1, null); boolean canHaveDuplicates = false; boolean autoGeneratedId = true; Engine.Create index = new Engine.Create(newUid("1"), doc, Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, System.nanoTime(), canHaveDuplicates, autoGeneratedId); engine.create(index); assertThat(index.version(), equalTo(1l)); index = new Engine.Create(newUid("1"), doc, index.version(), index.versionType().versionTypeForReplicationAndRecovery(), REPLICA, System.nanoTime(), canHaveDuplicates, autoGeneratedId); replicaEngine.create(index); assertThat(index.version(), equalTo(1l)); canHaveDuplicates = true; index = new Engine.Create(newUid("1"), doc, Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, System.nanoTime(), canHaveDuplicates, autoGeneratedId); engine.create(index); assertThat(index.version(), equalTo(1l)); engine.refresh("test"); Engine.Searcher searcher = engine.acquireSearcher("test"); TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), 10); assertThat(topDocs.totalHits, equalTo(1)); index = new Engine.Create(newUid("1"), doc, index.version(), index.versionType().versionTypeForReplicationAndRecovery(), REPLICA, System.nanoTime(), canHaveDuplicates, autoGeneratedId); try { replicaEngine.create(index); fail(); } catch (VersionConflictEngineException e) { // we ignore version conflicts on replicas, see TransportReplicationAction.ignoreReplicaException } replicaEngine.refresh("test"); Engine.Searcher replicaSearcher = replicaEngine.acquireSearcher("test"); topDocs = replicaSearcher.searcher().search(new MatchAllDocsQuery(), 10); assertThat(topDocs.totalHits, equalTo(1)); searcher.close(); replicaSearcher.close(); } @Test public void testRetryWithAutogeneratedIdsAndWrongOrderWorksAndNoDuplicateDocs() throws IOException { ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocument(), B_1, null); boolean canHaveDuplicates = true; boolean autoGeneratedId = true; Engine.Create firstIndexRequest = new Engine.Create(newUid("1"), doc, Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, System.nanoTime(), canHaveDuplicates, autoGeneratedId); engine.create(firstIndexRequest); assertThat(firstIndexRequest.version(), equalTo(1l)); Engine.Create firstIndexRequestReplica = new Engine.Create(newUid("1"), doc, firstIndexRequest.version(), firstIndexRequest.versionType().versionTypeForReplicationAndRecovery(), REPLICA, System.nanoTime(), canHaveDuplicates, autoGeneratedId); replicaEngine.create(firstIndexRequestReplica); assertThat(firstIndexRequestReplica.version(), equalTo(1l)); canHaveDuplicates = false; Engine.Create secondIndexRequest = new Engine.Create(newUid("1"), doc, Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, System.nanoTime(), canHaveDuplicates, autoGeneratedId); try { engine.create(secondIndexRequest); fail(); } catch (DocumentAlreadyExistsException e) { // we can ignore the exception. In case this happens because the retry request arrived first then this error will not be sent back anyway. // in any other case this is an actual error } engine.refresh("test"); Engine.Searcher searcher = engine.acquireSearcher("test"); TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), 10); assertThat(topDocs.totalHits, equalTo(1)); Engine.Create secondIndexRequestReplica = new Engine.Create(newUid("1"), doc, firstIndexRequest.version(), firstIndexRequest.versionType().versionTypeForReplicationAndRecovery(), REPLICA, System.nanoTime(), canHaveDuplicates, autoGeneratedId); try { replicaEngine.create(secondIndexRequestReplica); fail(); } catch (VersionConflictEngineException e) { // we ignore version conflicts on replicas, see TransportReplicationAction.ignoreReplicaException. } replicaEngine.refresh("test"); Engine.Searcher replicaSearcher = replicaEngine.acquireSearcher("test"); topDocs = replicaSearcher.searcher().search(new MatchAllDocsQuery(), 10); assertThat(topDocs.totalHits, equalTo(1)); searcher.close(); replicaSearcher.close(); } // #10312 @Test public void testDeletesAloneCanTriggerRefresh() throws Exception { try (Store store = createStore(); Engine engine = new InternalEngine(config(defaultSettings, store, createTempDir(), new MergeSchedulerConfig(defaultSettings), newMergePolicy()), false)) { engine.config().setIndexingBufferSize(new ByteSizeValue(1, ByteSizeUnit.KB)); for (int i = 0; i < 100; i++) { String id = Integer.toString(i); ParsedDocument doc = testParsedDocument(id, id, "test", null, -1, -1, testDocument(), B_1, null); engine.index(new Engine.Index(newUid(id), doc, 2, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime())); } // Force merge so we know all merges are done before we start deleting: engine.forceMerge(true, 1, false, false, false); Searcher s = engine.acquireSearcher("test"); final long version1 = ((DirectoryReader) s.reader()).getVersion(); s.close(); for (int i = 0; i < 100; i++) { String id = Integer.toString(i); engine.delete(new Engine.Delete("test", id, newUid(id), 10, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), false)); } // We must assertBusy because refresh due to version map being full is done in background (REFRESH) thread pool: assertBusy(new Runnable() { @Override public void run() { Searcher s2 = engine.acquireSearcher("test"); long version2 = ((DirectoryReader) s2.reader()).getVersion(); s2.close(); // 100 buffered deletes will easily exceed 25% of our 1 KB indexing buffer so it should have forced a refresh: assertThat(version2, greaterThan(version1)); } }); } } /* public void testMissingTranslog() throws IOException { // test that we can force start the engine , even if the translog is missing. engine.close(); // fake a new translog, causing the engine to point to a missing one. Translog translog = createTranslog(); long id = translog.currentFileGeneration(); translog.close(); IOUtils.rm(translog.location().resolve(Translog.getFilename(id))); try { engine = createEngine(store, primaryTranslogDir); fail("engine shouldn't start without a valid translog id"); } catch (EngineCreationFailureException ex) { // expected } // now it should be OK. Settings indexSettings = Settings.builder().put(defaultSettings).put(EngineConfig.INDEX_FORCE_NEW_TRANSLOG, true).build(); engine = createEngine(indexSettings, store, primaryTranslogDir, new MergeSchedulerConfig(indexSettings), newMergePolicy()); } */ /* public void testTranslogReplayWithFailure() throws IOException { boolean canHaveDuplicates = true; boolean autoGeneratedId = true; final int numDocs = randomIntBetween(1, 10); for (int i = 0; i < numDocs; i++) { ParsedDocument doc = testParsedDocument(Integer.toString(i), Integer.toString(i), "test", null, -1, -1, testDocument(), new BytesArray("{}"), null); Engine.Create firstIndexRequest = new Engine.Create(newUid(Integer.toString(i)), doc, Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, System.nanoTime(), canHaveDuplicates, autoGeneratedId); engine.create(firstIndexRequest); assertThat(firstIndexRequest.version(), equalTo(1l)); } engine.refresh("test"); try (Engine.Searcher searcher = engine.acquireSearcher("test")) { TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), randomIntBetween(numDocs, numDocs + 10)); assertThat(topDocs.totalHits, equalTo(numDocs)); } engine.close(); boolean recoveredButFailed = false; final MockDirectoryWrapper directory = DirectoryUtils.getLeaf(store.directory(), MockDirectoryWrapper.class); if (directory != null) { // since we rollback the IW we are writing the same segment files again after starting IW but MDW prevents // this so we have to disable the check explicitly directory.setPreventDoubleWrite(false); boolean started = false; final int numIters = randomIntBetween(10, 20); for (int i = 0; i < numIters; i++) { directory.setRandomIOExceptionRateOnOpen(randomDouble()); directory.setRandomIOExceptionRate(randomDouble()); directory.setFailOnOpenInput(randomBoolean()); directory.setAllowRandomFileNotFoundException(randomBoolean()); try { engine = createEngine(store, primaryTranslogDir); started = true; break; } catch (EngineCreationFailureException ex) { } } directory.setRandomIOExceptionRateOnOpen(0.0); directory.setRandomIOExceptionRate(0.0); directory.setFailOnOpenInput(false); directory.setAllowRandomFileNotFoundException(false); if (started == false) { engine = createEngine(store, primaryTranslogDir); } } else { // no mock directory, no fun. engine = createEngine(store, primaryTranslogDir); } try (Engine.Searcher searcher = engine.acquireSearcher("test")) { TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), randomIntBetween(numDocs, numDocs + 10)); assertThat(topDocs.totalHits, equalTo(numDocs)); } } */ /* @Test public void testSkipTranslogReplay() throws IOException { boolean canHaveDuplicates = true; boolean autoGeneratedId = true; final int numDocs = randomIntBetween(1, 10); for (int i = 0; i < numDocs; i++) { ParsedDocument doc = testParsedDocument(Integer.toString(i), Integer.toString(i), "test", null, -1, -1, testDocument(), new BytesArray("{}"), null); Engine.Create firstIndexRequest = new Engine.Create(newUid(Integer.toString(i)), doc, Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, System.nanoTime(), canHaveDuplicates, autoGeneratedId); engine.create(firstIndexRequest); assertThat(firstIndexRequest.version(), equalTo(1l)); } engine.refresh("test"); try (Engine.Searcher searcher = engine.acquireSearcher("test")) { TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), randomIntBetween(numDocs, numDocs + 10)); assertThat(topDocs.totalHits, equalTo(numDocs)); } final MockDirectoryWrapper directory = DirectoryUtils.getLeaf(store.directory(), MockDirectoryWrapper.class); if (directory != null) { // since we rollback the IW we are writing the same segment files again after starting IW but MDW prevents // this so we have to disable the check explicitly directory.setPreventDoubleWrite(false); } engine.close(); engine = new InternalEngine(engine.config(), true); try (Engine.Searcher searcher = engine.acquireSearcher("test")) { TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), randomIntBetween(numDocs, numDocs + 10)); assertThat(topDocs.totalHits, equalTo(0)); } } */ private Mapping dynamicUpdate() { BuilderContext context = new BuilderContext(Settings.EMPTY, new ContentPath()); final RootObjectMapper root = MapperBuilders.rootObject("some_type").build(context); return new Mapping(Version.CURRENT, root, new MetadataFieldMapper[0], new Mapping.SourceTransform[0], ImmutableMap.<String, Object>of()); } /* public void testUpgradeOldIndex() throws IOException { List<Path> indexes = new ArrayList<>(); try (DirectoryStream<Path> stream = Files.newDirectoryStream(getBwcIndicesPath(), "index-*.zip")) { for (Path path : stream) { indexes.add(path); } } Collections.shuffle(indexes, random()); for (Path indexFile : indexes.subList(0, scaledRandomIntBetween(1, indexes.size() / 2))) { final String indexName = indexFile.getFileName().toString().replace(".zip", "").toLowerCase(Locale.ROOT); Version version = Version.fromString(indexName.replace("index-", "")); if (version.onOrAfter(Version.V_2_0_0_beta1)) { continue; } Path unzipDir = createTempDir(); Path unzipDataDir = unzipDir.resolve("data"); // decompress the index try (InputStream stream = Files.newInputStream(indexFile)) { TestUtil.unzip(stream, unzipDir); } // check it is unique assertTrue(Files.exists(unzipDataDir)); Path[] list = filterExtraFSFiles(FileSystemUtils.files(unzipDataDir)); if (list.length != 1) { throw new IllegalStateException("Backwards index must contain exactly one cluster but was " + list.length + " " + Arrays.toString(list)); } // the bwc scripts packs the indices under this path Path src = list[0].resolve("nodes/0/indices/" + indexName); Path translog = list[0].resolve("nodes/0/indices/" + indexName).resolve("0").resolve("translog"); assertTrue("[" + indexFile + "] missing index dir: " + src.toString(), Files.exists(src)); assertTrue("[" + indexFile + "] missing translog dir: " + translog.toString(), Files.exists(translog)); Path[] tlogFiles = filterExtraFSFiles(FileSystemUtils.files(translog)); assertEquals(Arrays.toString(tlogFiles), tlogFiles.length, 1); final long size = Files.size(tlogFiles[0]); final long generation = TranslogTests.parseLegacyTranslogFile(tlogFiles[0]); assertTrue(generation >= 1); logger.debug("upgrading index {} file: {} size: {}", indexName, tlogFiles[0].getFileName(), size); Directory directory = newFSDirectory(src.resolve("0").resolve("index")); Store store = createStore(directory); final int iters = randomIntBetween(0, 2); int numDocs = -1; for (int i = 0; i < iters; i++) { // make sure we can restart on an upgraded index try (InternalEngine engine = createEngine(store, translog)) { try (Searcher searcher = engine.acquireSearcher("test")) { if (i > 0) { assertEquals(numDocs, searcher.reader().numDocs()); } TopDocs search = searcher.searcher().search(new MatchAllDocsQuery(), 1); numDocs = searcher.reader().numDocs(); assertTrue(search.totalHits > 1); } CommitStats commitStats = engine.commitStats(); Map<String, String> userData = commitStats.getUserData(); assertTrue("userdata dosn't contain uuid",userData.containsKey(Translog.TRANSLOG_UUID_KEY)); assertTrue("userdata doesn't contain generation key", userData.containsKey(Translog.TRANSLOG_GENERATION_KEY)); assertFalse("userdata contains legacy marker", userData.containsKey("translog_id")); } } try (InternalEngine engine = createEngine(store, translog)) { if (numDocs == -1) { try (Searcher searcher = engine.acquireSearcher("test")) { numDocs = searcher.reader().numDocs(); } } final int numExtraDocs = randomIntBetween(1, 10); for (int i = 0; i < numExtraDocs; i++) { ParsedDocument doc = testParsedDocument("extra" + Integer.toString(i), "extra" + Integer.toString(i), "test", null, -1, -1, testDocument(), new BytesArray("{}"), null); Engine.Create firstIndexRequest = new Engine.Create(newUid(Integer.toString(i)), doc, Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, System.nanoTime(), false, false); engine.create(firstIndexRequest); assertThat(firstIndexRequest.version(), equalTo(1l)); } engine.refresh("test"); try (Engine.Searcher searcher = engine.acquireSearcher("test")) { TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), randomIntBetween(numDocs, numDocs + numExtraDocs)); assertThat(topDocs.totalHits, equalTo(numDocs + numExtraDocs)); } } IOUtils.close(store, directory); } } */ private Path[] filterExtraFSFiles(Path[] files) { List<Path> paths = new ArrayList<>(); for (Path p : files) { if (p.getFileName().toString().startsWith("extra")) { continue; } paths.add(p); } return paths.toArray(new Path[0]); } public void testTranslogReplay() throws IOException { boolean canHaveDuplicates = true; boolean autoGeneratedId = true; final int numDocs = randomIntBetween(1, 10); for (int i = 0; i < numDocs; i++) { ParsedDocument doc = testParsedDocument(Integer.toString(i), Integer.toString(i), "test", null, -1, -1, testDocument(), new BytesArray("{}"), null); Engine.Create firstIndexRequest = new Engine.Create(newUid(Integer.toString(i)), doc, Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, System.nanoTime(), canHaveDuplicates, autoGeneratedId); engine.create(firstIndexRequest); assertThat(firstIndexRequest.version(), equalTo(1l)); } engine.refresh("test"); try (Engine.Searcher searcher = engine.acquireSearcher("test")) { TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), randomIntBetween(numDocs, numDocs + 10)); assertThat(topDocs.totalHits, equalTo(numDocs)); } final MockDirectoryWrapper directory = DirectoryUtils.getLeaf(store.directory(), MockDirectoryWrapper.class); if (directory != null) { // since we rollback the IW we are writing the same segment files again after starting IW but MDW prevents // this so we have to disable the check explicitly directory.setPreventDoubleWrite(false); } TranslogHandler parser = (TranslogHandler) engine.config().getTranslogRecoveryPerformer(); parser.mappingUpdate = dynamicUpdate(); engine.close(); engine.config().setCreate(false); engine = new InternalEngine(engine.config(), false); // we need to reuse the engine config unless the parser.mappingModified won't work try (Engine.Searcher searcher = engine.acquireSearcher("test")) { TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), randomIntBetween(numDocs, numDocs + 10)); assertThat(topDocs.totalHits, equalTo(numDocs)); } parser = (TranslogHandler) engine.config().getTranslogRecoveryPerformer(); assertEquals(numDocs, parser.recoveredOps.get()); if (parser.mappingUpdate != null) { assertEquals(1, parser.getRecoveredTypes().size()); assertTrue(parser.getRecoveredTypes().containsKey("test")); } else { assertEquals(0, parser.getRecoveredTypes().size()); } engine.close(); engine = createEngine(store, primaryTranslogDir); try (Engine.Searcher searcher = engine.acquireSearcher("test")) { TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), randomIntBetween(numDocs, numDocs + 10)); assertThat(topDocs.totalHits, equalTo(numDocs)); } parser = (TranslogHandler) engine.config().getTranslogRecoveryPerformer(); assertEquals(0, parser.recoveredOps.get()); final boolean flush = randomBoolean(); int randomId = randomIntBetween(numDocs + 1, numDocs + 10); String uuidValue = "test#" + Integer.toString(randomId); ParsedDocument doc = testParsedDocument(uuidValue, Integer.toString(randomId), "test", null, -1, -1, testDocument(), new BytesArray("{}"), null); Engine.Create firstIndexRequest = new Engine.Create(newUid(uuidValue), doc, 1, VersionType.EXTERNAL, PRIMARY, System.nanoTime(), canHaveDuplicates, autoGeneratedId); engine.create(firstIndexRequest); assertThat(firstIndexRequest.version(), equalTo(1l)); if (flush) { engine.flush(); } doc = testParsedDocument(uuidValue, Integer.toString(randomId), "test", null, -1, -1, testDocument(), new BytesArray("{}"), null); Engine.Index idxRequest = new Engine.Index(newUid(uuidValue), doc, 2, VersionType.EXTERNAL, PRIMARY, System.nanoTime()); engine.index(idxRequest); engine.refresh("test"); assertThat(idxRequest.version(), equalTo(2l)); try (Engine.Searcher searcher = engine.acquireSearcher("test")) { TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), numDocs + 1); assertThat(topDocs.totalHits, equalTo(numDocs + 1)); } engine.close(); engine = createEngine(store, primaryTranslogDir); try (Engine.Searcher searcher = engine.acquireSearcher("test")) { TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), numDocs + 1); assertThat(topDocs.totalHits, equalTo(numDocs + 1)); } parser = (TranslogHandler) engine.config().getTranslogRecoveryPerformer(); assertEquals(flush ? 1 : 2, parser.recoveredOps.get()); engine.delete(new Engine.Delete("test", Integer.toString(randomId), newUid(uuidValue))); if (randomBoolean()) { engine.refresh("test"); } else { engine.close(); engine = createEngine(store, primaryTranslogDir); } try (Engine.Searcher searcher = engine.acquireSearcher("test")) { TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), numDocs); assertThat(topDocs.totalHits, equalTo(numDocs)); } } public static class TranslogHandler extends TranslogRecoveryPerformer { private final DocumentMapper docMapper; public Mapping mappingUpdate = null; public final AtomicInteger recoveredOps = new AtomicInteger(0); public TranslogHandler(String indexName, ESLogger logger) { super(new ShardId("test", 0), null, null, null, null, logger); Settings settings = Settings.settingsBuilder() .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(); RootObjectMapper.Builder rootBuilder = new RootObjectMapper.Builder("test"); Index index = new Index(indexName); AnalysisService analysisService = new AnalysisService(index, settings); SimilarityLookupService similarityLookupService = new SimilarityLookupService(index, settings); MapperRegistry mapperRegistry = new IndicesModule().getMapperRegistry(); MapperService mapperService = new MapperService(index, settings, analysisService, similarityLookupService, null, mapperRegistry); DocumentMapper.Builder b = new DocumentMapper.Builder(rootBuilder, mapperService); this.docMapper = b.build(mapperService); } @Override protected DocumentMapperForType docMapper(String type) { return new DocumentMapperForType(docMapper, mappingUpdate); } @Override protected void operationProcessed() { recoveredOps.incrementAndGet(); } @Override public void performRecoveryOperation(Engine engine, Translog.Operation operation, boolean allowMappingUpdates) { if (operation.opType() != Translog.Operation.Type.DELETE_BY_QUERY) { // we don't support del by query in this test super.performRecoveryOperation(engine, operation, allowMappingUpdates); } } } /* public void testRecoverFromForeignTranslog() throws IOException { boolean canHaveDuplicates = true; boolean autoGeneratedId = true; final int numDocs = randomIntBetween(1, 10); for (int i = 0; i < numDocs; i++) { ParsedDocument doc = testParsedDocument(Integer.toString(i), Integer.toString(i), "test", null, -1, -1, testDocument(), new BytesArray("{}"), null); Engine.Create firstIndexRequest = new Engine.Create(newUid(Integer.toString(i)), doc, Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, System.nanoTime(), canHaveDuplicates, autoGeneratedId); engine.create(firstIndexRequest); assertThat(firstIndexRequest.version(), equalTo(1l)); } engine.refresh("test"); try (Engine.Searcher searcher = engine.acquireSearcher("test")) { TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), randomIntBetween(numDocs, numDocs + 10)); assertThat(topDocs.totalHits, equalTo(numDocs)); } final MockDirectoryWrapper directory = DirectoryUtils.getLeaf(store.directory(), MockDirectoryWrapper.class); if (directory != null) { // since we rollback the IW we are writing the same segment files again after starting IW but MDW prevents // this so we have to disable the check explicitly directory.setPreventDoubleWrite(false); } Translog.TranslogGeneration generation = engine.getTranslog().getGeneration(); engine.close(); Translog translog = new Translog(new TranslogConfig(shardId, createTempDir(), Settings.EMPTY, Translog.Durabilty.REQUEST, BigArrays.NON_RECYCLING_INSTANCE, threadPool)); translog.add(new Translog.Create("test", "SomeBogusId", "{}".getBytes(Charset.forName("UTF-8")))); assertEquals(generation.translogFileGeneration, translog.currentFileGeneration()); translog.close(); EngineConfig config = engine.config(); // create a TranslogConfig that has been created with a different UUID TranslogConfig translogConfig = new TranslogConfig(shardId, translog.location(), config.getIndexSettings(), Translog.Durabilty.REQUEST, BigArrays.NON_RECYCLING_INSTANCE, threadPool); EngineConfig brokenConfig = new EngineConfig(shardId, threadPool, config.getIndexingService(), config.getIndexSettings() , null, store, createSnapshotDeletionPolicy(), newMergePolicy(), config.getMergeSchedulerConfig(), config.getAnalyzer(), config.getSimilarity(), new CodecService(shardId.index()), config.getFailedEngineListener() , config.getTranslogRecoveryPerformer(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), new IndexSearcherWrappingService(), translogConfig); try { new InternalEngine(brokenConfig, false); fail("translog belongs to a different engine"); } catch (EngineCreationFailureException ex) { } engine = createEngine(store, primaryTranslogDir); // and recover again! try (Engine.Searcher searcher = engine.acquireSearcher("test")) { TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), randomIntBetween(numDocs, numDocs + 10)); assertThat(topDocs.totalHits, equalTo(numDocs)); } } */ public void testShardNotAvailableExceptionWhenEngineClosedConcurrently() throws IOException, InterruptedException { final AtomicReference<Throwable> throwable = new AtomicReference<>(); final String operation = randomFrom("optimize", "refresh", "flush"); Thread mergeThread = new Thread() { @Override public void run() { boolean stop = false; logger.info("try with {}", operation); while (stop == false) { try { switch (operation) { case "optimize": { engine.forceMerge(true, 1, false, false, false); break; } case "refresh": { engine.refresh("test refresh"); break; } case "flush": { engine.flush(true, false); break; } } } catch (Throwable t) { throwable.set(t); stop = true; } } } }; mergeThread.start(); engine.close(); mergeThread.join(); logger.info("exception caught: ", throwable.get()); assertTrue("expected an Exception that signals shard is not available", TransportActions.isShardNotAvailableException(throwable.get())); } }