Java tutorial
/* * Licensed to Elasticsearch under one or more contributor * license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright * ownership. Elasticsearch licenses this file to you under * the Apache License, Version 2.0 (the "License"); you may * not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.elasticsearch.bwcompat; import com.carrotsearch.randomizedtesting.generators.RandomPicks; import org.apache.lucene.index.Fields; import org.apache.lucene.util.English; import org.elasticsearch.ElasticsearchIllegalArgumentException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.Version; import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.elasticsearch.action.count.CountResponse; import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.deletebyquery.DeleteByQueryResponse; import org.elasticsearch.action.deletebyquery.IndexDeleteByQueryResponse; import org.elasticsearch.action.explain.ExplainResponse; import org.elasticsearch.action.get.*; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.termvector.TermVectorResponse; import org.elasticsearch.action.update.UpdateRequestBuilder; import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.internal.FieldNamesFieldMapper; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ElasticsearchBackwardsCompatIntegrationTest; import org.elasticsearch.test.junit.annotations.TestLogging; import org.junit.Test; import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.concurrent.ExecutionException; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.FilterBuilders.existsFilter; import static org.elasticsearch.index.query.FilterBuilders.missingFilter; import static org.elasticsearch.index.query.QueryBuilders.*; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*; import static org.hamcrest.Matchers.*; /** */ public class BasicBackwardsCompatibilityTest extends ElasticsearchBackwardsCompatIntegrationTest { /** * Basic test using Index & Realtime Get with external versioning. This test ensures routing works correctly across versions. */ @Test public void testExternalVersion() throws Exception { createIndex("test"); final boolean routing = randomBoolean(); int numDocs = randomIntBetween(10, 20); for (int i = 0; i < numDocs; i++) { String id = Integer.toString(i); String routingKey = routing ? randomRealisticUnicodeOfLength(10) : null; final long version = randomIntBetween(0, Integer.MAX_VALUE); client().prepareIndex("test", "type1", id).setRouting(routingKey).setVersion(version) .setVersionType(VersionType.EXTERNAL).setSource("field1", English.intToEnglish(i)).get(); GetResponse get = client().prepareGet("test", "type1", id).setRouting(routingKey).setVersion(version) .get(); assertThat("Document with ID " + id + " should exist but doesn't", get.isExists(), is(true)); assertThat(get.getVersion(), equalTo(version)); final long nextVersion = version + randomIntBetween(0, Integer.MAX_VALUE); client().prepareIndex("test", "type1", id).setRouting(routingKey).setVersion(nextVersion) .setVersionType(VersionType.EXTERNAL).setSource("field1", English.intToEnglish(i)).get(); get = client().prepareGet("test", "type1", id).setRouting(routingKey).setVersion(nextVersion).get(); assertThat("Document with ID " + id + " should exist but doesn't", get.isExists(), is(true)); assertThat(get.getVersion(), equalTo(nextVersion)); } } /** * Basic test using Index & Realtime Get with internal versioning. This test ensures routing works correctly across versions. */ @Test public void testInternalVersion() throws Exception { createIndex("test"); final boolean routing = randomBoolean(); int numDocs = randomIntBetween(10, 20); for (int i = 0; i < numDocs; i++) { String routingKey = routing ? randomRealisticUnicodeOfLength(10) : null; String id = Integer.toString(i); assertThat(id, client().prepareIndex("test", "type1", id).setRouting(routingKey) .setSource("field1", English.intToEnglish(i)).get().isCreated(), is(true)); GetResponse get = client().prepareGet("test", "type1", id).setRouting(routingKey).setVersion(1).get(); assertThat("Document with ID " + id + " should exist but doesn't", get.isExists(), is(true)); assertThat(get.getVersion(), equalTo(1l)); client().prepareIndex("test", "type1", id).setRouting(routingKey) .setSource("field1", English.intToEnglish(i)).execute().actionGet(); get = client().prepareGet("test", "type1", id).setRouting(routingKey).setVersion(2).get(); assertThat("Document with ID " + id + " should exist but doesn't", get.isExists(), is(true)); assertThat(get.getVersion(), equalTo(2l)); } assertVersionCreated(compatibilityVersion(), "test"); } /** * Very basic bw compat test with a mixed version cluster random indexing and lookup by ID via term query */ @Test public void testIndexAndSearch() throws Exception { createIndex("test"); int numDocs = randomIntBetween(10, 20); List<IndexRequestBuilder> builder = new ArrayList<>(); for (int i = 0; i < numDocs; i++) { String id = Integer.toString(i); builder.add(client().prepareIndex("test", "type1", id).setSource("field1", English.intToEnglish(i), "the_id", id)); } indexRandom(true, builder); for (int i = 0; i < numDocs; i++) { String id = Integer.toString(i); assertHitCount(client().prepareSearch().setQuery(QueryBuilders.termQuery("the_id", id)).get(), 1); } assertVersionCreated(compatibilityVersion(), "test"); } @Test public void testRecoverFromPreviousVersion() throws ExecutionException, InterruptedException { if (backwardsCluster().numNewDataNodes() == 0) { backwardsCluster().startNewNode(); } assertAcked(prepareCreate("test").setSettings(ImmutableSettings.builder() .put("index.routing.allocation.exclude._name", backwardsCluster().newNodePattern()) .put(indexSettings()))); ensureYellow(); assertAllShardsOnNodes("test", backwardsCluster().backwardsNodePattern()); int numDocs = randomIntBetween(100, 150); ArrayList<String> ids = new ArrayList<>(); logger.info(" --> indexing [{}] docs", numDocs); IndexRequestBuilder[] docs = new IndexRequestBuilder[numDocs]; for (int i = 0; i < numDocs; i++) { String id = randomRealisticUnicodeOfLength(10) + String.valueOf(i); ids.add(id); docs[i] = client().prepareIndex("test", "type1", id).setSource("field1", English.intToEnglish(i)); } indexRandom(true, docs); CountResponse countResponse = client().prepareCount().get(); assertHitCount(countResponse, numDocs); if (randomBoolean()) { logger.info(" --> moving index to new nodes"); backwardsCluster().allowOnlyNewNodes("test"); } else { logger.info(" --> allow index to on all nodes"); backwardsCluster().allowOnAllNodes("test"); } logger.info(" --> indexing [{}] more docs", numDocs); // sometimes index while relocating if (randomBoolean()) { for (int i = 0; i < numDocs; i++) { String id = randomRealisticUnicodeOfLength(10) + String.valueOf(numDocs + i); ids.add(id); docs[i] = client().prepareIndex("test", "type1", id).setSource("field1", English.intToEnglish(numDocs + i)); } indexRandom(true, docs); if (compatibilityVersion().before(Version.V_1_3_0)) { // issue another refresh through a new node to side step issue #6545 assertNoFailures(backwardsCluster().internalCluster().dataNodeClient().admin().indices() .prepareRefresh().setIndicesOptions(IndicesOptions.lenientExpandOpen()).execute().get()); } numDocs *= 2; } logger.info(" --> waiting for relocation to complete", numDocs); ensureYellow("test");// move all shards to the new node (it waits on relocation) final int numIters = randomIntBetween(10, 20); for (int i = 0; i < numIters; i++) { assertSearchHits(client().prepareSearch().setSize(ids.size()).get(), ids.toArray(new String[ids.size()])); } assertVersionCreated(compatibilityVersion(), "test"); } /** * Test that ensures that we will never recover from a newer to an older version (we are not forward compatible) */ @Test public void testNoRecoveryFromNewNodes() throws ExecutionException, InterruptedException { assertAcked(prepareCreate("test").setSettings(ImmutableSettings.builder() .put("index.routing.allocation.exclude._name", backwardsCluster().backwardsNodePattern()) .put(indexSettings()))); if (backwardsCluster().numNewDataNodes() == 0) { backwardsCluster().startNewNode(); } ensureYellow(); assertAllShardsOnNodes("test", backwardsCluster().newNodePattern()); if (randomBoolean()) { backwardsCluster().allowOnAllNodes("test"); } int numDocs = randomIntBetween(100, 150); IndexRequestBuilder[] docs = new IndexRequestBuilder[numDocs]; for (int i = 0; i < numDocs; i++) { docs[i] = client().prepareIndex("test", "type1", randomRealisticUnicodeOfLength(10) + String.valueOf(i)) .setSource("field1", English.intToEnglish(i), "num_int", randomInt(), "num_double", randomDouble()); } indexRandom(true, docs); backwardsCluster().allowOnAllNodes("test"); while (ensureYellow() != ClusterHealthStatus.GREEN) { backwardsCluster().startNewNode(); } assertAllShardsOnNodes("test", backwardsCluster().newNodePattern()); CountResponse countResponse = client().prepareCount().get(); assertHitCount(countResponse, numDocs); final int numIters = randomIntBetween(10, 20); for (int i = 0; i < numIters; i++) { countResponse = client().prepareCount().get(); assertHitCount(countResponse, numDocs); assertSimpleSort("num_double", "num_int"); } assertVersionCreated(compatibilityVersion(), "test"); } public void assertSimpleSort(String... numericFields) { for (String field : numericFields) { SearchResponse searchResponse = client().prepareSearch().addSort(field, SortOrder.ASC).get(); SearchHit[] hits = searchResponse.getHits().getHits(); assertThat(hits.length, greaterThan(0)); Number previous = null; for (SearchHit hit : hits) { assertNotNull(hit.getSource().get(field)); if (previous != null) { assertThat(previous.doubleValue(), lessThanOrEqualTo(((Number) hit.getSource().get(field)).doubleValue())); } previous = (Number) hit.getSource().get(field); } } } public void assertAllShardsOnNodes(String index, String pattern) { ClusterState clusterState = client().admin().cluster().prepareState().execute().actionGet().getState(); for (IndexRoutingTable indexRoutingTable : clusterState.routingTable()) { for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) { for (ShardRouting shardRouting : indexShardRoutingTable) { if (shardRouting.currentNodeId() != null && index.equals(shardRouting.getIndex())) { String name = clusterState.nodes().get(shardRouting.currentNodeId()).name(); assertThat("Allocated on new node: " + name, Regex.simpleMatch(pattern, name), is(true)); } } } } } /** * Upgrades a single node to the current version */ @Test @TestLogging("org.elasticsearch.index.gateway.local:TRACE") public void testIndexUpgradeSingleNode() throws Exception { assertAcked(prepareCreate("test").setSettings(ImmutableSettings.builder() .put("index.routing.allocation.exclude._name", backwardsCluster().newNodePattern()) .put(indexSettings()))); ensureYellow(); int numDocs = randomIntBetween(100, 150); IndexRequestBuilder[] docs = new IndexRequestBuilder[numDocs]; for (int i = 0; i < numDocs; i++) { docs[i] = client().prepareIndex("test", "type1", String.valueOf(i)).setSource("field1", English.intToEnglish(i), "num_int", randomInt(), "num_double", randomDouble()); } indexRandom(true, docs); assertAllShardsOnNodes("test", backwardsCluster().backwardsNodePattern()); client().admin().indices().prepareUpdateSettings("test").setSettings( ImmutableSettings.builder().put(EnableAllocationDecider.INDEX_ROUTING_ALLOCATION_ENABLE, "none")) .get(); backwardsCluster().allowOnAllNodes("test"); CountResponse countResponse = client().prepareCount().get(); assertHitCount(countResponse, numDocs); backwardsCluster().upgradeOneNode(); ensureYellow(); if (randomBoolean()) { for (int i = 0; i < numDocs; i++) { docs[i] = client().prepareIndex("test", "type1", String.valueOf(i)).setSource("field1", English.intToEnglish(i), "num_int", randomInt(), "num_double", randomDouble()); } indexRandom(true, docs); } client().admin().indices().prepareUpdateSettings("test").setSettings( ImmutableSettings.builder().put(EnableAllocationDecider.INDEX_ROUTING_ALLOCATION_ENABLE, "all")) .get(); ensureYellow(); final int numIters = randomIntBetween(1, 20); for (int i = 0; i < numIters; i++) { assertHitCount(client().prepareCount().get(), numDocs); assertSimpleSort("num_double", "num_int"); } assertVersionCreated(compatibilityVersion(), "test"); } /** * Test that allocates an index on one or more old nodes and then do a rolling upgrade * one node after another is shut down and restarted from a newer version and we verify * that all documents are still around after each nodes upgrade. */ @Test @TestLogging("org.elasticsearch.index.gateway.local:TRACE") public void testIndexRollingUpgrade() throws Exception { String[] indices = new String[randomIntBetween(1, 3)]; for (int i = 0; i < indices.length; i++) { indices[i] = "test" + i; assertAcked(prepareCreate(indices[i]).setSettings(ImmutableSettings.builder() .put("index.routing.allocation.exclude._name", backwardsCluster().newNodePattern()) .put(indexSettings()))); } int numDocs = randomIntBetween(100, 150); IndexRequestBuilder[] docs = new IndexRequestBuilder[numDocs]; String[] indexForDoc = new String[docs.length]; for (int i = 0; i < numDocs; i++) { docs[i] = client().prepareIndex(indexForDoc[i] = RandomPicks.randomFrom(getRandom(), indices), "type1", String.valueOf(i)).setSource("field1", English.intToEnglish(i), "num_int", randomInt(), "num_double", randomDouble()); } indexRandom(true, docs); for (String index : indices) { assertAllShardsOnNodes(index, backwardsCluster().backwardsNodePattern()); } client().admin().indices().prepareUpdateSettings(indices).setSettings( ImmutableSettings.builder().put(EnableAllocationDecider.INDEX_ROUTING_ALLOCATION_ENABLE, "none")) .get(); backwardsCluster().allowOnAllNodes(indices); logClusterState(); boolean upgraded; do { logClusterState(); CountResponse countResponse = client().prepareCount().get(); assertHitCount(countResponse, numDocs); assertSimpleSort("num_double", "num_int"); upgraded = backwardsCluster().upgradeOneNode(); ensureYellow(); countResponse = client().prepareCount().get(); assertHitCount(countResponse, numDocs); for (int i = 0; i < numDocs; i++) { docs[i] = client().prepareIndex(indexForDoc[i], "type1", String.valueOf(i)).setSource("field1", English.intToEnglish(i), "num_int", randomInt(), "num_double", randomDouble()); } indexRandom(true, docs); } while (upgraded); client().admin().indices().prepareUpdateSettings(indices).setSettings( ImmutableSettings.builder().put(EnableAllocationDecider.INDEX_ROUTING_ALLOCATION_ENABLE, "all")) .get(); ensureYellow(); CountResponse countResponse = client().prepareCount().get(); assertHitCount(countResponse, numDocs); assertSimpleSort("num_double", "num_int"); String[] newIndices = new String[randomIntBetween(1, 3)]; for (int i = 0; i < newIndices.length; i++) { newIndices[i] = "new_index" + i; createIndex(newIndices[i]); } assertVersionCreated(Version.CURRENT, newIndices); // new indices are all created with the new version assertVersionCreated(compatibilityVersion(), indices); } public void assertVersionCreated(Version version, String... indices) { GetSettingsResponse getSettingsResponse = client().admin().indices().prepareGetSettings(indices).get(); ImmutableOpenMap<String, Settings> indexToSettings = getSettingsResponse.getIndexToSettings(); for (String index : indices) { Settings settings = indexToSettings.get(index); assertThat(settings.getAsVersion(IndexMetaData.SETTING_VERSION_CREATED, null), notNullValue()); assertThat(settings.getAsVersion(IndexMetaData.SETTING_VERSION_CREATED, null), equalTo(version)); } } @Test public void testUnsupportedFeatures() throws IOException { if (compatibilityVersion().before(Version.V_1_3_0)) { XContentBuilder mapping = XContentBuilder.builder(JsonXContent.jsonXContent).startObject() .startObject("type").startObject(FieldNamesFieldMapper.NAME) // by setting randomly index to no we also test the pre-1.3 behavior .field("index", randomFrom("no", "not_analyzed")).field("store", randomFrom("no", "yes")) .endObject().endObject().endObject(); try { assertAcked( prepareCreate("test") .setSettings(ImmutableSettings.builder() .put("index.routing.allocation.exclude._name", backwardsCluster().newNodePattern()) .put(indexSettings())) .addMapping("type", mapping)); } catch (MapperParsingException ex) { if (getMasterVersion().onOrAfter(Version.V_1_3_0)) { assertThat(ex.getCause(), instanceOf(ElasticsearchIllegalArgumentException.class)); assertThat( ExceptionsHelper.detailedMessage(ex).contains( "type=_field_names is not supported on indices created before version 1.3.0"), equalTo(true)); } else { assertThat(ex.getCause(), instanceOf(MapperParsingException.class)); assertThat(ex.getCause().getMessage(), startsWith("Root type mapping not empty after parsing!")); } } } } /** * This filter had a major upgrade in 1.3 where we started to index the field names. Lets see if they still work as expected... * this test is basically copied from SimpleQueryTests... */ @Test @TestLogging("org.elasticsearch.index.gateway.local:TRACE") public void testExistsFilter() throws IOException, ExecutionException, InterruptedException { assumeTrue("this test fails often with 1.0.3 skipping for now....", compatibilityVersion().onOrAfter(Version.V_1_1_0)); int indexId = 0; String indexName; for (;;) { indexName = "test_" + indexId++; createIndex(indexName); ensureYellow(); indexRandom(true, client().prepareIndex(indexName, "type1", "1") .setSource(jsonBuilder().startObject().startObject("obj1").field("obj1_val", "1") .endObject().field("x1", "x_1").field("field1", "value1_1") .field("field2", "value2_1").endObject()), client().prepareIndex(indexName, "type1", "2") .setSource(jsonBuilder().startObject().startObject("obj1").field("obj1_val", "1") .endObject().field("x2", "x_2").field("field1", "value1_2").endObject()), client().prepareIndex(indexName, "type1", "3") .setSource(jsonBuilder().startObject().startObject("obj2").field("obj2_val", "1") .endObject().field("y1", "y_1").field("field2", "value2_3").endObject()), client().prepareIndex(indexName, "type1", "4") .setSource(jsonBuilder().startObject().startObject("obj2").field("obj2_val", "1") .endObject().field("y2", "y_2").field("field3", "value3_4").endObject())); CountResponse countResponse = client().prepareCount() .setQuery(filteredQuery(matchAllQuery(), existsFilter("field1"))).get(); assertHitCount(countResponse, 2l); countResponse = client().prepareCount().setQuery(constantScoreQuery(existsFilter("field1"))).get(); assertHitCount(countResponse, 2l); countResponse = client().prepareCount().setQuery(queryStringQuery("_exists_:field1")).get(); assertHitCount(countResponse, 2l); countResponse = client().prepareCount().setQuery(filteredQuery(matchAllQuery(), existsFilter("field2"))) .get(); assertHitCount(countResponse, 2l); countResponse = client().prepareCount().setQuery(filteredQuery(matchAllQuery(), existsFilter("field3"))) .get(); assertHitCount(countResponse, 1l); // wildcard check countResponse = client().prepareCount().setQuery(filteredQuery(matchAllQuery(), existsFilter("x*"))) .get(); assertHitCount(countResponse, 2l); // object check countResponse = client().prepareCount().setQuery(filteredQuery(matchAllQuery(), existsFilter("obj1"))) .get(); assertHitCount(countResponse, 2l); countResponse = client().prepareCount() .setQuery(filteredQuery(matchAllQuery(), missingFilter("field1"))).get(); assertHitCount(countResponse, 2l); countResponse = client().prepareCount() .setQuery(filteredQuery(matchAllQuery(), missingFilter("field1"))).get(); assertHitCount(countResponse, 2l); countResponse = client().prepareCount().setQuery(constantScoreQuery(missingFilter("field1"))).get(); assertHitCount(countResponse, 2l); countResponse = client().prepareCount().setQuery(queryStringQuery("_missing_:field1")).get(); assertHitCount(countResponse, 2l); // wildcard check countResponse = client().prepareCount().setQuery(filteredQuery(matchAllQuery(), missingFilter("x*"))) .get(); assertHitCount(countResponse, 2l); // object check countResponse = client().prepareCount().setQuery(filteredQuery(matchAllQuery(), missingFilter("obj1"))) .get(); assertHitCount(countResponse, 2l); if (!backwardsCluster().upgradeOneNode()) { break; } ensureYellow(); assertVersionCreated(compatibilityVersion(), indexName); // we had an old node in the cluster so we have to be on the compat version assertAcked(client().admin().indices().prepareDelete(indexName)); } assertVersionCreated(Version.CURRENT, indexName); // after upgrade we have current version } public Version getMasterVersion() { return client().admin().cluster().prepareState().get().getState().nodes().masterNode().getVersion(); } @Test public void testDeleteByQuery() throws ExecutionException, InterruptedException { createIndex("test"); ensureYellow("test"); int numDocs = iterations(10, 50); IndexRequestBuilder[] indexRequestBuilders = new IndexRequestBuilder[numDocs + 1]; for (int i = 0; i < numDocs; i++) { indexRequestBuilders[i] = client().prepareIndex("test", "test", Integer.toString(i)).setSource("field", "value"); } indexRequestBuilders[numDocs] = client().prepareIndex("test", "test", Integer.toString(numDocs)) .setSource("field", "other_value"); indexRandom(true, indexRequestBuilders); SearchResponse searchResponse = client().prepareSearch("test").get(); assertNoFailures(searchResponse); assertThat(searchResponse.getHits().totalHits(), equalTo((long) numDocs + 1)); DeleteByQueryResponse deleteByQueryResponse = client().prepareDeleteByQuery("test") .setQuery(QueryBuilders.termQuery("field", "value")).get(); assertThat(deleteByQueryResponse.getIndices().size(), equalTo(1)); for (IndexDeleteByQueryResponse indexDeleteByQueryResponse : deleteByQueryResponse) { assertThat(indexDeleteByQueryResponse.getIndex(), equalTo("test")); assertThat(indexDeleteByQueryResponse.getFailures().length, equalTo(0)); } refresh(); searchResponse = client().prepareSearch("test").get(); assertNoFailures(searchResponse); assertThat(searchResponse.getHits().totalHits(), equalTo(1l)); } @Test public void testDeleteRoutingRequired() throws ExecutionException, InterruptedException, IOException { createIndexWithAlias(); assertAcked(client().admin().indices().preparePutMapping("test").setType("test") .setSource(XContentFactory.jsonBuilder().startObject().startObject("test").startObject("_routing") .field("required", true).endObject().endObject().endObject())); ensureYellow("test"); int numDocs = iterations(10, 50); IndexRequestBuilder[] indexRequestBuilders = new IndexRequestBuilder[numDocs]; for (int i = 0; i < numDocs - 2; i++) { indexRequestBuilders[i] = client().prepareIndex("test", "test", Integer.toString(i)) .setRouting(randomAsciiOfLength(randomIntBetween(1, 10))).setSource("field", "value"); } String firstDocId = Integer.toString(numDocs - 2); indexRequestBuilders[numDocs - 2] = client().prepareIndex("test", "test", firstDocId).setRouting("routing") .setSource("field", "value"); String secondDocId = Integer.toString(numDocs - 1); String secondRouting = randomAsciiOfLength(randomIntBetween(1, 10)); indexRequestBuilders[numDocs - 1] = client().prepareIndex("test", "test", secondDocId) .setRouting(secondRouting).setSource("field", "value"); indexRandom(true, indexRequestBuilders); SearchResponse searchResponse = client().prepareSearch("test").get(); assertNoFailures(searchResponse); assertThat(searchResponse.getHits().totalHits(), equalTo((long) numDocs)); //use routing DeleteResponse deleteResponse = client().prepareDelete("test", "test", firstDocId).setRouting("routing") .get(); assertThat(deleteResponse.isFound(), equalTo(true)); GetResponse getResponse = client().prepareGet("test", "test", firstDocId).setRouting("routing").get(); assertThat(getResponse.isExists(), equalTo(false)); refresh(); searchResponse = client().prepareSearch("test").get(); assertNoFailures(searchResponse); assertThat(searchResponse.getHits().totalHits(), equalTo((long) numDocs - 1)); //don't use routing and trigger a broadcast delete deleteResponse = client().prepareDelete("test", "test", secondDocId).get(); assertThat(deleteResponse.isFound(), equalTo(true)); getResponse = client().prepareGet("test", "test", secondDocId).setRouting(secondRouting).get(); assertThat(getResponse.isExists(), equalTo(false)); refresh(); searchResponse = client().prepareSearch("test").setSize(numDocs).get(); assertNoFailures(searchResponse); assertThat(searchResponse.getHits().totalHits(), equalTo((long) numDocs - 2)); } @Test public void testIndexGetAndDelete() throws ExecutionException, InterruptedException { createIndexWithAlias(); ensureYellow("test"); int numDocs = iterations(10, 50); for (int i = 0; i < numDocs; i++) { IndexResponse indexResponse = client().prepareIndex(indexOrAlias(), "type", Integer.toString(i)) .setSource("field", "value-" + i).get(); assertThat(indexResponse.isCreated(), equalTo(true)); assertThat(indexResponse.getIndex(), equalTo("test")); assertThat(indexResponse.getType(), equalTo("type")); assertThat(indexResponse.getId(), equalTo(Integer.toString(i))); } refresh(); String docId = Integer.toString(randomIntBetween(0, numDocs - 1)); GetResponse getResponse = client().prepareGet(indexOrAlias(), "type", docId).get(); assertThat(getResponse.isExists(), equalTo(true)); assertThat(getResponse.getIndex(), equalTo("test")); assertThat(getResponse.getType(), equalTo("type")); assertThat(getResponse.getId(), equalTo(docId)); DeleteResponse deleteResponse = client().prepareDelete(indexOrAlias(), "type", docId).get(); assertThat(deleteResponse.isFound(), equalTo(true)); assertThat(deleteResponse.getIndex(), equalTo("test")); assertThat(deleteResponse.getType(), equalTo("type")); assertThat(deleteResponse.getId(), equalTo(docId)); getResponse = client().prepareGet(indexOrAlias(), "type", docId).get(); assertThat(getResponse.isExists(), equalTo(false)); refresh(); SearchResponse searchResponse = client().prepareSearch(indexOrAlias()).get(); assertThat(searchResponse.getHits().totalHits(), equalTo((long) numDocs - 1)); } @Test public void testUpdate() { createIndexWithAlias(); ensureYellow("test"); UpdateRequestBuilder updateRequestBuilder = client().prepareUpdate(indexOrAlias(), "type1", "1") .setUpsert("field1", "value1").setDoc("field2", "value2"); UpdateResponse updateResponse = updateRequestBuilder.get(); assertThat(updateResponse.getIndex(), equalTo("test")); assertThat(updateResponse.getType(), equalTo("type1")); assertThat(updateResponse.getId(), equalTo("1")); assertThat(updateResponse.isCreated(), equalTo(true)); GetResponse getResponse = client().prepareGet("test", "type1", "1").get(); assertThat(getResponse.isExists(), equalTo(true)); assertThat(getResponse.getSourceAsMap().containsKey("field1"), equalTo(true)); assertThat(getResponse.getSourceAsMap().containsKey("field2"), equalTo(false)); updateResponse = updateRequestBuilder.get(); assertThat(updateResponse.getIndex(), equalTo("test")); assertThat(updateResponse.getType(), equalTo("type1")); assertThat(updateResponse.getId(), equalTo("1")); assertThat(updateResponse.isCreated(), equalTo(false)); getResponse = client().prepareGet("test", "type1", "1").get(); assertThat(getResponse.isExists(), equalTo(true)); assertThat(getResponse.getSourceAsMap().containsKey("field1"), equalTo(true)); assertThat(getResponse.getSourceAsMap().containsKey("field2"), equalTo(true)); } @Test public void testAnalyze() { createIndexWithAlias(); assertAcked(client().admin().indices().preparePutMapping("test").setType("test").setSource("field", "type=string,analyzer=keyword")); ensureYellow("test"); AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze("this is a test") .setIndex(indexOrAlias()).setField("field").get(); assertThat(analyzeResponse.getTokens().size(), equalTo(1)); assertThat(analyzeResponse.getTokens().get(0).getTerm(), equalTo("this is a test")); } @Test public void testExplain() { createIndexWithAlias(); ensureYellow("test"); client().prepareIndex(indexOrAlias(), "test", "1").setSource("field", "value1").get(); refresh(); ExplainResponse response = client().prepareExplain(indexOrAlias(), "test", "1") .setQuery(QueryBuilders.termQuery("field", "value1")).get(); assertThat(response.isExists(), equalTo(true)); assertThat(response.isMatch(), equalTo(true)); assertThat(response.getExplanation(), notNullValue()); assertThat(response.getExplanation().isMatch(), equalTo(true)); assertThat(response.getExplanation().getDetails().length, equalTo(1)); } @Test public void testGetTermVector() throws IOException { createIndexWithAlias(); assertAcked(client().admin().indices().preparePutMapping("test").setType("type1") .setSource("field", "type=string,term_vector=with_positions_offsets_payloads").get()); ensureYellow("test"); client().prepareIndex(indexOrAlias(), "type1", "1") .setSource("field", "the quick brown fox jumps over the lazy dog").get(); refresh(); TermVectorResponse termVectorResponse = client().prepareTermVector(indexOrAlias(), "type1", "1").get(); assertThat(termVectorResponse.getIndex(), equalTo("test")); assertThat(termVectorResponse.isExists(), equalTo(true)); Fields fields = termVectorResponse.getFields(); assertThat(fields.size(), equalTo(1)); assertThat(fields.terms("field").size(), equalTo(8l)); } @Test public void testIndicesStats() { createIndex("test"); ensureYellow("test"); IndicesStatsResponse indicesStatsResponse = client().admin().indices().prepareStats().all().get(); assertThat(indicesStatsResponse.getIndices().size(), equalTo(1)); assertThat(indicesStatsResponse.getIndices().containsKey("test"), equalTo(true)); } @Test public void testMultiGet() throws ExecutionException, InterruptedException { createIndexWithAlias(); ensureYellow("test"); int numDocs = iterations(10, 50); IndexRequestBuilder[] indexRequestBuilders = new IndexRequestBuilder[numDocs]; for (int i = 0; i < numDocs; i++) { indexRequestBuilders[i] = client().prepareIndex("test", "type", Integer.toString(i)).setSource("field", "value" + Integer.toString(i)); } indexRandom(false, indexRequestBuilders); int iterations = iterations(1, numDocs); MultiGetRequestBuilder multiGetRequestBuilder = client().prepareMultiGet(); for (int i = 0; i < iterations; i++) { multiGetRequestBuilder.add( new MultiGetRequest.Item(indexOrAlias(), "type", Integer.toString(randomInt(numDocs - 1)))); } MultiGetResponse multiGetResponse = multiGetRequestBuilder.get(); assertThat(multiGetResponse.getResponses().length, equalTo(iterations)); for (int i = 0; i < multiGetResponse.getResponses().length; i++) { MultiGetItemResponse multiGetItemResponse = multiGetResponse.getResponses()[i]; assertThat(multiGetItemResponse.isFailed(), equalTo(false)); assertThat(multiGetItemResponse.getIndex(), equalTo("test")); assertThat(multiGetItemResponse.getType(), equalTo("type")); assertThat(multiGetItemResponse.getId(), equalTo(multiGetRequestBuilder.request().getItems().get(i).id())); assertThat(multiGetItemResponse.getResponse().isExists(), equalTo(true)); assertThat(multiGetItemResponse.getResponse().getIndex(), equalTo("test")); assertThat(multiGetItemResponse.getResponse().getType(), equalTo("type")); assertThat(multiGetItemResponse.getResponse().getId(), equalTo(multiGetRequestBuilder.request().getItems().get(i).id())); } } @Test public void testScroll() throws ExecutionException, InterruptedException { assumeTrue("this test fails often with 1.1.2 skipping for now....", compatibilityVersion().after(Version.V_1_1_2)); createIndex("test"); ensureYellow("test"); int numDocs = iterations(10, 100); IndexRequestBuilder[] indexRequestBuilders = new IndexRequestBuilder[numDocs]; for (int i = 0; i < numDocs; i++) { indexRequestBuilders[i] = client().prepareIndex("test", "type", Integer.toString(i)).setSource("field", "value" + Integer.toString(i)); } indexRandom(true, indexRequestBuilders); int size = randomIntBetween(1, 10); SearchRequestBuilder searchRequestBuilder = client().prepareSearch("test").setScroll("1m").setSize(size); boolean scan = randomBoolean(); if (scan) { searchRequestBuilder.setSearchType(SearchType.SCAN); } SearchResponse searchResponse = searchRequestBuilder.get(); assertThat(searchResponse.getScrollId(), notNullValue()); assertHitCount(searchResponse, numDocs); int hits = 0; if (scan) { assertThat(searchResponse.getHits().getHits().length, equalTo(0)); } else { assertThat(searchResponse.getHits().getHits().length, greaterThan(0)); hits += searchResponse.getHits().getHits().length; } try { do { searchResponse = client().prepareSearchScroll(searchResponse.getScrollId()).setScroll("1m").get(); assertThat(searchResponse.getScrollId(), notNullValue()); assertHitCount(searchResponse, numDocs); hits += searchResponse.getHits().getHits().length; } while (searchResponse.getHits().getHits().length > 0); assertThat(hits, equalTo(numDocs)); } finally { clearScroll(searchResponse.getScrollId()); } } private static String indexOrAlias() { return randomBoolean() ? "test" : "alias"; } private void createIndexWithAlias() { if (compatibilityVersion().onOrAfter(Version.V_1_1_0)) { assertAcked(prepareCreate("test").addAlias(new Alias("alias"))); } else { assertAcked(prepareCreate("test")); assertAcked(client().admin().indices().prepareAliases().addAlias("test", "alias")); } } }