List of usage examples for org.apache.lucene.index IndexWriter MAX_DOCS
int MAX_DOCS
To view the source code for org.apache.lucene.index IndexWriter MAX_DOCS.
Click Source Link
From source file:org.elasticsearch.action.admin.indices.shrink.TransportResizeAction.java
License:Apache License
static CreateIndexClusterStateUpdateRequest prepareCreateIndexRequest(final ResizeRequest resizeRequest, final ClusterState state, final IntFunction<DocsStats> perShardDocStats, String sourceIndexName, String targetIndexName) { final CreateIndexRequest targetIndex = resizeRequest.getTargetIndexRequest(); final IndexMetaData metaData = state.metaData().index(sourceIndexName); if (metaData == null) { throw new IndexNotFoundException(sourceIndexName); }/* w w w . jav a 2s.c o m*/ final Settings targetIndexSettings = Settings.builder().put(targetIndex.settings()) .normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX).build(); final int numShards; if (IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.exists(targetIndexSettings)) { numShards = IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(targetIndexSettings); } else { assert resizeRequest .getResizeType() == ResizeType.SHRINK : "split must specify the number of shards explicitly"; numShards = 1; } for (int i = 0; i < numShards; i++) { if (resizeRequest.getResizeType() == ResizeType.SHRINK) { Set<ShardId> shardIds = IndexMetaData.selectShrinkShards(i, metaData, numShards); long count = 0; for (ShardId id : shardIds) { DocsStats docsStats = perShardDocStats.apply(id.id()); if (docsStats != null) { count += docsStats.getCount(); } if (count > IndexWriter.MAX_DOCS) { throw new IllegalStateException("Can't merge index with more than [" + IndexWriter.MAX_DOCS + "] docs - too many documents in shards " + shardIds); } } } else { Objects.requireNonNull(IndexMetaData.selectSplitShard(i, metaData, numShards)); // we just execute this to ensure we get the right exceptions if the number of shards is wrong or less then etc. } } if (IndexMetaData.INDEX_ROUTING_PARTITION_SIZE_SETTING.exists(targetIndexSettings)) { throw new IllegalArgumentException( "cannot provide a routing partition size value when resizing an index"); } if (IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.exists(targetIndexSettings)) { // if we have a source index with 1 shards it's legal to set this final boolean splitFromSingleShards = resizeRequest.getResizeType() == ResizeType.SPLIT && metaData.getNumberOfShards() == 1; if (splitFromSingleShards == false) { throw new IllegalArgumentException("cannot provide index.number_of_routing_shards on resize"); } } String cause = resizeRequest.getResizeType().name().toLowerCase(Locale.ROOT) + "_index"; targetIndex.cause(cause); Settings.Builder settingsBuilder = Settings.builder().put(targetIndexSettings); settingsBuilder.put("index.number_of_shards", numShards); targetIndex.settings(settingsBuilder); return new CreateIndexClusterStateUpdateRequest(targetIndex, cause, targetIndex.index(), targetIndexName) // mappings are updated on the node when creating in the shards, this prevents race-conditions since all mapping must be // applied once we took the snapshot and if somebody messes things up and switches the index read/write and adds docs we miss // the mappings for everything is corrupted and hard to debug .ackTimeout(targetIndex.timeout()).masterNodeTimeout(targetIndex.masterNodeTimeout()) .settings(targetIndex.settings()).aliases(targetIndex.aliases()).customs(targetIndex.customs()) .waitForActiveShards(targetIndex.waitForActiveShards()).recoverFrom(metaData.getIndex()) .resizeType(resizeRequest.getResizeType()); }
From source file:org.elasticsearch.action.admin.indices.shrink.TransportResizeActionTests.java
License:Apache License
public void testShrinkIndexSettings() { String indexName = randomAlphaOfLength(10); // create one that won't fail ClusterState clusterState = ClusterState .builder(createClusterState(indexName, randomIntBetween(2, 10), 0, Settings.builder().put("index.blocks.write", true).build())) .nodes(DiscoveryNodes.builder().add(newNode("node1"))).build(); AllocationService service = new AllocationService(Settings.builder().build(), new AllocationDeciders(Settings.EMPTY, Collections.singleton(new MaxRetryAllocationDecider(Settings.EMPTY))), new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE); RoutingTable routingTable = service.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); // now we start the shard routingTable = service// ww w . j a va2s. c o m .applyStartedShards(clusterState, routingTable.index(indexName).shardsWithState(ShardRoutingState.INITIALIZING)) .routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); int numSourceShards = clusterState.metaData().index(indexName).getNumberOfShards(); DocsStats stats = new DocsStats(between(0, (IndexWriter.MAX_DOCS) / numSourceShards), between(1, 1000), between(1, 10000)); ResizeRequest target = new ResizeRequest("target", indexName); final ActiveShardCount activeShardCount = randomBoolean() ? ActiveShardCount.ALL : ActiveShardCount.ONE; target.setWaitForActiveShards(activeShardCount); CreateIndexClusterStateUpdateRequest request = TransportResizeAction.prepareCreateIndexRequest(target, clusterState, (i) -> stats, indexName, "target"); assertNotNull(request.recoverFrom()); assertEquals(indexName, request.recoverFrom().getName()); assertEquals("1", request.settings().get("index.number_of_shards")); assertEquals("shrink_index", request.cause()); assertEquals(request.waitForActiveShards(), activeShardCount); }
From source file:org.elasticsearch.action.admin.indices.shrink.TransportShrinkAction.java
License:Apache License
static CreateIndexClusterStateUpdateRequest prepareCreateIndexRequest(final ShrinkRequest shrinkReqeust, final ClusterState state, final IntFunction<DocsStats> perShardDocStats, IndexNameExpressionResolver indexNameExpressionResolver) { final String sourceIndex = indexNameExpressionResolver .resolveDateMathExpression(shrinkReqeust.getSourceIndex()); final CreateIndexRequest targetIndex = shrinkReqeust.getShrinkIndexRequest(); final String targetIndexName = indexNameExpressionResolver.resolveDateMathExpression(targetIndex.index()); final IndexMetaData metaData = state.metaData().index(sourceIndex); final Settings targetIndexSettings = Settings.builder().put(targetIndex.settings()) .normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX).build(); int numShards = 1; if (IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.exists(targetIndexSettings)) { numShards = IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(targetIndexSettings); }/*from www . j a v a2s . com*/ for (int i = 0; i < numShards; i++) { Set<ShardId> shardIds = IndexMetaData.selectShrinkShards(i, metaData, numShards); long count = 0; for (ShardId id : shardIds) { DocsStats docsStats = perShardDocStats.apply(id.id()); if (docsStats != null) { count += docsStats.getCount(); } if (count > IndexWriter.MAX_DOCS) { throw new IllegalStateException("Can't merge index with more than [" + IndexWriter.MAX_DOCS + "] docs - too many documents in shards " + shardIds); } } } targetIndex.cause("shrink_index"); Settings.Builder settingsBuilder = Settings.builder().put(targetIndexSettings); settingsBuilder.put("index.number_of_shards", numShards); targetIndex.settings(settingsBuilder); return new CreateIndexClusterStateUpdateRequest(targetIndex, "shrink_index", targetIndexName, true) // mappings are updated on the node when merging in the shards, this prevents race-conditions since all mapping must be // applied once we took the snapshot and if somebody fucks things up and switches the index read/write and adds docs we miss // the mappings for everything is corrupted and hard to debug .ackTimeout(targetIndex.timeout()).masterNodeTimeout(targetIndex.masterNodeTimeout()) .settings(targetIndex.settings()).aliases(targetIndex.aliases()).customs(targetIndex.customs()) .waitForActiveShards(targetIndex.waitForActiveShards()).shrinkFrom(metaData.getIndex()); }
From source file:org.elasticsearch.action.admin.indices.shrink.TransportShrinkActionTests.java
License:Apache License
public void testShrinkIndexSettings() { String indexName = randomAsciiOfLength(10); // create one that won't fail ClusterState clusterState = ClusterState .builder(createClusterState(indexName, randomIntBetween(2, 10), 0, Settings.builder().put("index.blocks.write", true).build())) .nodes(DiscoveryNodes.builder().add(newNode("node1"))).build(); AllocationService service = new AllocationService(Settings.builder().build(), new AllocationDeciders(Settings.EMPTY, Collections.singleton(new MaxRetryAllocationDecider(Settings.EMPTY))), NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE); RoutingTable routingTable = service.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); // now we start the shard routingTable = service//from w w w. ja va 2 s . co m .applyStartedShards(clusterState, routingTable.index(indexName).shardsWithState(ShardRoutingState.INITIALIZING)) .routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); int numSourceShards = clusterState.metaData().index(indexName).getNumberOfShards(); DocsStats stats = new DocsStats(randomIntBetween(0, (IndexWriter.MAX_DOCS) / numSourceShards), randomIntBetween(1, 1000)); ShrinkRequest target = new ShrinkRequest("target", indexName); final ActiveShardCount activeShardCount = randomBoolean() ? ActiveShardCount.ALL : ActiveShardCount.ONE; target.setWaitForActiveShards(activeShardCount); CreateIndexClusterStateUpdateRequest request = TransportShrinkAction.prepareCreateIndexRequest(target, clusterState, (i) -> stats, new IndexNameExpressionResolver(Settings.EMPTY)); assertNotNull(request.shrinkFrom()); assertEquals(indexName, request.shrinkFrom().getName()); assertEquals("1", request.settings().get("index.number_of_shards")); assertEquals("shrink_index", request.cause()); assertEquals(request.waitForActiveShards(), activeShardCount); }
From source file:org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoHashGridTests.java
License:Apache License
@Override protected InternalGeoHashGrid createTestInstance(String name, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) { int size = randomIntBetween(1, 100); List<InternalGeoHashGrid.Bucket> buckets = new ArrayList<>(size); for (int i = 0; i < size; i++) { long geoHashAsLong = GeoHashUtils.longEncode(randomInt(90), randomInt(90), 4); buckets.add(new InternalGeoHashGrid.Bucket(geoHashAsLong, randomInt(IndexWriter.MAX_DOCS), InternalAggregations.EMPTY)); }/* ww w. j a v a2 s.c o m*/ return new InternalGeoHashGrid(name, size, buckets, pipelineAggregators, metaData); }
From source file:org.elasticsearch.search.aggregations.metrics.tophits.InternalTopHitsTests.java
License:Apache License
@Override protected InternalTopHits createTestInstance(String name, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) { int from = 0; int requestedSize = between(1, 40); int actualSize = between(0, requestedSize); float maxScore = Float.MIN_VALUE; ScoreDoc[] scoreDocs = new ScoreDoc[actualSize]; SearchHit[] hits = new SearchHit[actualSize]; Set<Integer> usedDocIds = new HashSet<>(); for (int i = 0; i < actualSize; i++) { float score = randomFloat(); maxScore = max(maxScore, score); int docId = randomValueOtherThanMany(usedDocIds::contains, () -> between(0, IndexWriter.MAX_DOCS)); usedDocIds.add(docId);//from www. j ava 2 s .c o m Map<String, SearchHitField> searchHitFields = new HashMap<>(); if (testInstancesLookSortedByField) { Object[] fields = new Object[testInstancesSortFields.length]; for (int f = 0; f < testInstancesSortFields.length; f++) { fields[f] = randomOfType(testInstancesSortFields[f].getType()); } scoreDocs[i] = new FieldDoc(docId, score, fields); } else { scoreDocs[i] = new ScoreDoc(docId, score); } hits[i] = new SearchHit(docId, Integer.toString(i), new Text("test"), searchHitFields); hits[i].score(score); } int totalHits = between(actualSize, 500000); SearchHits searchHits = new SearchHits(hits, totalHits, maxScore); TopDocs topDocs; Arrays.sort(scoreDocs, scoreDocComparator()); if (testInstancesLookSortedByField) { topDocs = new TopFieldDocs(totalHits, scoreDocs, testInstancesSortFields, maxScore); } else { topDocs = new TopDocs(totalHits, scoreDocs, maxScore); } return new InternalTopHits(name, from, requestedSize, topDocs, searchHits, pipelineAggregators, metaData); }
From source file:org.elasticsearch.search.aggregations.metrics.tophits.InternalTopHitsTests.java
License:Apache License
private Object randomOfType(SortField.Type type) { switch (type) { case CUSTOM://from ww w . ja va 2 s . c om throw new UnsupportedOperationException(); case DOC: return between(0, IndexWriter.MAX_DOCS); case DOUBLE: return randomDouble(); case FLOAT: return randomFloat(); case INT: return randomInt(); case LONG: return randomLong(); case REWRITEABLE: throw new UnsupportedOperationException(); case SCORE: return randomFloat(); case STRING: return new BytesRef(randomAsciiOfLength(5)); case STRING_VAL: return new BytesRef(randomAsciiOfLength(5)); default: throw new UnsupportedOperationException("Unkown SortField.Type: " + type); } }