List of usage examples for org.apache.lucene.util CollectionUtil timSort
public static <T extends Comparable<? super T>> void timSort(List<T> list)
From source file:org.elasticsearch.action.admin.cluster.snapshots.get.TransportGetSnapshotsAction.java
License:Apache License
@Override protected void masterOperation(final GetSnapshotsRequest request, final ClusterState state, final ActionListener<GetSnapshotsResponse> listener) { try {/*from w ww.j ava2s . com*/ final String repository = request.repository(); final Map<String, SnapshotId> allSnapshotIds = new HashMap<>(); final List<SnapshotInfo> currentSnapshots = new ArrayList<>(); for (SnapshotInfo snapshotInfo : snapshotsService.currentSnapshots(repository)) { SnapshotId snapshotId = snapshotInfo.snapshotId(); allSnapshotIds.put(snapshotId.getName(), snapshotId); currentSnapshots.add(snapshotInfo); } final RepositoryData repositoryData; if (isCurrentSnapshotsOnly(request.snapshots()) == false) { repositoryData = snapshotsService.getRepositoryData(repository); for (SnapshotId snapshotId : repositoryData.getAllSnapshotIds()) { allSnapshotIds.put(snapshotId.getName(), snapshotId); } } else { repositoryData = null; } final Set<SnapshotId> toResolve = new HashSet<>(); if (isAllSnapshots(request.snapshots())) { toResolve.addAll(allSnapshotIds.values()); } else { for (String snapshotOrPattern : request.snapshots()) { if (GetSnapshotsRequest.CURRENT_SNAPSHOT.equalsIgnoreCase(snapshotOrPattern)) { toResolve.addAll(currentSnapshots.stream().map(SnapshotInfo::snapshotId) .collect(Collectors.toList())); } else if (Regex.isSimpleMatchPattern(snapshotOrPattern) == false) { if (allSnapshotIds.containsKey(snapshotOrPattern)) { toResolve.add(allSnapshotIds.get(snapshotOrPattern)); } else if (request.ignoreUnavailable() == false) { throw new SnapshotMissingException(repository, snapshotOrPattern); } } else { for (Map.Entry<String, SnapshotId> entry : allSnapshotIds.entrySet()) { if (Regex.simpleMatch(snapshotOrPattern, entry.getKey())) { toResolve.add(entry.getValue()); } } } } if (toResolve.isEmpty() && request.ignoreUnavailable() == false && isCurrentSnapshotsOnly(request.snapshots()) == false) { throw new SnapshotMissingException(repository, request.snapshots()[0]); } } final List<SnapshotInfo> snapshotInfos; if (request.verbose()) { final Set<SnapshotId> incompatibleSnapshots = repositoryData != null ? new HashSet<>(repositoryData.getIncompatibleSnapshotIds()) : Collections.emptySet(); snapshotInfos = snapshotsService.snapshots(repository, new ArrayList<>(toResolve), incompatibleSnapshots, request.ignoreUnavailable()); } else { if (repositoryData != null) { // want non-current snapshots as well, which are found in the repository data snapshotInfos = buildSimpleSnapshotInfos(toResolve, repositoryData, currentSnapshots); } else { // only want current snapshots snapshotInfos = currentSnapshots.stream().map(SnapshotInfo::basic).collect(Collectors.toList()); CollectionUtil.timSort(snapshotInfos); } } listener.onResponse(new GetSnapshotsResponse(snapshotInfos)); } catch (Exception e) { listener.onFailure(e); } }
From source file:org.elasticsearch.action.admin.cluster.snapshots.get.TransportGetSnapshotsAction.java
License:Apache License
private List<SnapshotInfo> buildSimpleSnapshotInfos(final Set<SnapshotId> toResolve, final RepositoryData repositoryData, final List<SnapshotInfo> currentSnapshots) { List<SnapshotInfo> snapshotInfos = new ArrayList<>(); for (SnapshotInfo snapshotInfo : currentSnapshots) { if (toResolve.remove(snapshotInfo.snapshotId())) { snapshotInfos.add(snapshotInfo.basic()); }//w ww .j ava 2 s .c o m } Map<SnapshotId, List<String>> snapshotsToIndices = new HashMap<>(); for (IndexId indexId : repositoryData.getIndices().values()) { for (SnapshotId snapshotId : repositoryData.getSnapshots(indexId)) { if (toResolve.contains(snapshotId)) { snapshotsToIndices.computeIfAbsent(snapshotId, (k) -> new ArrayList<>()).add(indexId.getName()); } } } for (Map.Entry<SnapshotId, List<String>> entry : snapshotsToIndices.entrySet()) { final List<String> indices = entry.getValue(); CollectionUtil.timSort(indices); final SnapshotId snapshotId = entry.getKey(); snapshotInfos.add(new SnapshotInfo(snapshotId, indices, repositoryData.getSnapshotState(snapshotId))); } CollectionUtil.timSort(snapshotInfos); return Collections.unmodifiableList(snapshotInfos); }
From source file:org.elasticsearch.action.admin.indices.segments.IndicesShardStoreResponseTest.java
License:Apache License
@Test public void testStoreStatusOrdering() throws Exception { DiscoveryNode node1 = new DiscoveryNode("node1", DummyTransportAddress.INSTANCE, Version.CURRENT); List<IndicesShardStoresResponse.StoreStatus> orderedStoreStatuses = new ArrayList<>(); orderedStoreStatuses.add(new IndicesShardStoresResponse.StoreStatus(node1, 2, IndicesShardStoresResponse.StoreStatus.Allocation.PRIMARY, null)); orderedStoreStatuses.add(new IndicesShardStoresResponse.StoreStatus(node1, 1, IndicesShardStoresResponse.StoreStatus.Allocation.PRIMARY, null)); orderedStoreStatuses.add(new IndicesShardStoresResponse.StoreStatus(node1, 1, IndicesShardStoresResponse.StoreStatus.Allocation.REPLICA, null)); orderedStoreStatuses.add(new IndicesShardStoresResponse.StoreStatus(node1, 1, IndicesShardStoresResponse.StoreStatus.Allocation.UNUSED, null)); orderedStoreStatuses.add(new IndicesShardStoresResponse.StoreStatus(node1, 3, IndicesShardStoresResponse.StoreStatus.Allocation.REPLICA, new IOException("corrupted"))); List<IndicesShardStoresResponse.StoreStatus> storeStatuses = new ArrayList<>(orderedStoreStatuses); Collections.shuffle(storeStatuses); CollectionUtil.timSort(storeStatuses); assertThat(storeStatuses, equalTo(orderedStoreStatuses)); }
From source file:org.elasticsearch.action.admin.indices.shards.IndicesShardStoreResponseTests.java
License:Apache License
public void testStoreStatusOrdering() throws Exception { DiscoveryNode node1 = new DiscoveryNode("node1", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT); List<IndicesShardStoresResponse.StoreStatus> orderedStoreStatuses = new ArrayList<>(); orderedStoreStatuses.add(new IndicesShardStoresResponse.StoreStatus(node1, ShardStateMetaData.NO_VERSION, UUIDs.randomBase64UUID(), IndicesShardStoresResponse.StoreStatus.AllocationStatus.PRIMARY, null)); orderedStoreStatuses.add(new IndicesShardStoresResponse.StoreStatus(node1, ShardStateMetaData.NO_VERSION, UUIDs.randomBase64UUID(), IndicesShardStoresResponse.StoreStatus.AllocationStatus.REPLICA, null)); orderedStoreStatuses.add(new IndicesShardStoresResponse.StoreStatus(node1, ShardStateMetaData.NO_VERSION, UUIDs.randomBase64UUID(), IndicesShardStoresResponse.StoreStatus.AllocationStatus.UNUSED, null)); orderedStoreStatuses.add(new IndicesShardStoresResponse.StoreStatus(node1, 2, null, IndicesShardStoresResponse.StoreStatus.AllocationStatus.PRIMARY, null)); orderedStoreStatuses.add(new IndicesShardStoresResponse.StoreStatus(node1, 1, null, IndicesShardStoresResponse.StoreStatus.AllocationStatus.PRIMARY, null)); orderedStoreStatuses.add(new IndicesShardStoresResponse.StoreStatus(node1, 1, null, IndicesShardStoresResponse.StoreStatus.AllocationStatus.REPLICA, null)); orderedStoreStatuses.add(new IndicesShardStoresResponse.StoreStatus(node1, 1, null, IndicesShardStoresResponse.StoreStatus.AllocationStatus.UNUSED, null)); orderedStoreStatuses.add(new IndicesShardStoresResponse.StoreStatus(node1, ShardStateMetaData.NO_VERSION, UUIDs.randomBase64UUID(), IndicesShardStoresResponse.StoreStatus.AllocationStatus.REPLICA, new IOException("corrupted"))); orderedStoreStatuses.add(new IndicesShardStoresResponse.StoreStatus(node1, 3, null, IndicesShardStoresResponse.StoreStatus.AllocationStatus.REPLICA, new IOException("corrupted"))); List<IndicesShardStoresResponse.StoreStatus> storeStatuses = new ArrayList<>(orderedStoreStatuses); Collections.shuffle(storeStatuses, random()); CollectionUtil.timSort(storeStatuses); assertThat(storeStatuses, equalTo(orderedStoreStatuses)); }
From source file:org.elasticsearch.cluster.routing.GroupShardsIterator.java
License:Apache License
/** * Constructs a enw GroupShardsIterator from the given list. *///from w w w .j ava 2s . co m public GroupShardsIterator(List<ShardIterator> iterators) { CollectionUtil.timSort(iterators); this.iterators = iterators; }
From source file:org.elasticsearch.cluster.routing.GroupShardsIteratorTests.java
License:Apache License
public void testIterate() { List<ShardIterator> list = new ArrayList<>(); Index index = new Index("foo", "na"); list.add(new PlainShardIterator(new ShardId(index, 0), Arrays.asList(newRouting(index, 0, true), newRouting(index, 0, true), newRouting(index, 0, true)))); list.add(new PlainShardIterator(new ShardId(index, 1), Collections.emptyList())); list.add(new PlainShardIterator(new ShardId(index, 2), Arrays.asList(newRouting(index, 2, true)))); list.add(new PlainShardIterator(new ShardId(index, 0), Arrays.asList(newRouting(index, 0, true)))); list.add(new PlainShardIterator(new ShardId(index, 1), Arrays.asList(newRouting(index, 1, true)))); index = new Index("foo_2", "na"); list.add(new PlainShardIterator(new ShardId(index, 0), Arrays.asList(newRouting(index, 0, true)))); list.add(new PlainShardIterator(new ShardId(index, 1), Arrays.asList(newRouting(index, 1, true)))); Collections.shuffle(list, random()); ArrayList<ShardIterator> actualIterators = new ArrayList<>(); GroupShardsIterator iter = new GroupShardsIterator(list); for (ShardIterator shardsIterator : iter) { actualIterators.add(shardsIterator); }//w w w . ja v a 2 s . c o m CollectionUtil.timSort(actualIterators); assertEquals(actualIterators, list); }
From source file:org.elasticsearch.index.query.TermsQueryBuilderTests.java
License:Apache License
@Override protected void doAssertLuceneQuery(TermsQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException { if (queryBuilder.termsLookup() == null && (queryBuilder.values() == null || queryBuilder.values().isEmpty())) { assertThat(query, instanceOf(MatchNoDocsQuery.class)); MatchNoDocsQuery matchNoDocsQuery = (MatchNoDocsQuery) query; assertThat(matchNoDocsQuery.toString(), containsString("No terms supplied for \"terms\" query.")); } else if (queryBuilder.termsLookup() != null && randomTerms.size() == 0) { assertThat(query, instanceOf(MatchNoDocsQuery.class)); MatchNoDocsQuery matchNoDocsQuery = (MatchNoDocsQuery) query; assertThat(matchNoDocsQuery.toString(), containsString("No terms supplied for \"terms\" query.")); } else {/*from ww w . j a va 2s . c o m*/ assertThat(query, instanceOf(BooleanQuery.class)); BooleanQuery booleanQuery = (BooleanQuery) query; // we only do the check below for string fields (otherwise we'd have to decode the values) if (queryBuilder.fieldName().equals(INT_FIELD_NAME) || queryBuilder.fieldName().equals(DOUBLE_FIELD_NAME) || queryBuilder.fieldName().equals(BOOLEAN_FIELD_NAME) || queryBuilder.fieldName().equals(DATE_FIELD_NAME)) { return; } // expected returned terms depending on whether we have a terms query or a terms lookup query List<Object> terms; if (queryBuilder.termsLookup() != null) { terms = randomTerms; } else { terms = queryBuilder.values(); } // compare whether we have the expected list of terms returned final List<Term> booleanTerms = new ArrayList<>(); for (BooleanClause booleanClause : booleanQuery) { assertThat(booleanClause.getOccur(), equalTo(BooleanClause.Occur.SHOULD)); assertThat(booleanClause.getQuery(), instanceOf(TermQuery.class)); Term term = ((TermQuery) booleanClause.getQuery()).getTerm(); booleanTerms.add(term); } CollectionUtil.timSort(booleanTerms); List<Term> expectedTerms = new ArrayList<>(); for (Object term : terms) { if (term != null) { // terms lookup filters this out expectedTerms.add(new Term(queryBuilder.fieldName(), term.toString())); } } CollectionUtil.timSort(expectedTerms); assertEquals(expectedTerms + " vs. " + booleanTerms, expectedTerms.size(), booleanTerms.size()); assertEquals(expectedTerms + " vs. " + booleanTerms, expectedTerms, booleanTerms); } }
From source file:org.elasticsearch.indices.IndicesService.java
License:Apache License
/** * Processes all pending deletes for the given index. This method will acquire all locks for the given index and will * process all pending deletes for this index. Pending deletes might occur if the OS doesn't allow deletion of files because * they are used by a different process ie. on Windows where files might still be open by a virus scanner. On a shared * filesystem a replica might not have been closed when the primary is deleted causing problems on delete calls so we * schedule there deletes later.//from w w w . j av a 2s .co m * @param index the index to process the pending deletes for * @param timeout the timeout used for processing pending deletes */ public void processPendingDeletes(Index index, Settings indexSettings, TimeValue timeout) throws IOException, InterruptedException { logger.debug("{} processing pending deletes", index); final long startTimeNS = System.nanoTime(); final List<ShardLock> shardLocks = nodeEnv.lockAllForIndex(index, indexSettings, timeout.millis()); try { Map<ShardId, ShardLock> locks = new HashMap<>(); for (ShardLock lock : shardLocks) { locks.put(lock.getShardId(), lock); } final List<PendingDelete> remove; synchronized (pendingDeletes) { remove = pendingDeletes.remove(index); } if (remove != null && remove.isEmpty() == false) { CollectionUtil.timSort(remove); // make sure we delete indices first final long maxSleepTimeMs = 10 * 1000; // ensure we retry after 10 sec long sleepTime = 10; do { if (remove.isEmpty()) { break; } Iterator<PendingDelete> iterator = remove.iterator(); while (iterator.hasNext()) { PendingDelete delete = iterator.next(); if (delete.deleteIndex) { assert delete.shardId == -1; logger.debug("{} deleting index store reason [{}]", index, "pending delete"); try { nodeEnv.deleteIndexDirectoryUnderLock(index, indexSettings); iterator.remove(); } catch (IOException ex) { logger.debug("{} retry pending delete", ex, index); } } else { assert delete.shardId != -1; ShardLock shardLock = locks.get(new ShardId(delete.index, delete.shardId)); if (shardLock != null) { try { deleteShardStore("pending delete", shardLock, delete.settings); iterator.remove(); } catch (IOException ex) { logger.debug("{} retry pending delete", ex, shardLock.getShardId()); } } else { logger.warn("{} no shard lock for pending delete", delete.shardId); iterator.remove(); } } } if (remove.isEmpty() == false) { logger.warn("{} still pending deletes present for shards {} - retrying", index, remove.toString()); Thread.sleep(sleepTime); sleepTime = Math.min(maxSleepTimeMs, sleepTime * 2); // increase the sleep time gradually logger.debug("{} schedule pending delete retry after {} ms", index, sleepTime); } } while ((System.nanoTime() - startTimeNS) < timeout.nanos()); } } finally { IOUtils.close(shardLocks); } }
From source file:org.elasticsearch.snapshots.SnapshotsService.java
License:Apache License
/** * Returns a list of snapshots from repository sorted by snapshot creation date * * @param repositoryName repository name * @return list of snapshots/*from w w w .jav a 2 s .co m*/ */ public ImmutableList<Snapshot> snapshots(String repositoryName) { ArrayList<Snapshot> snapshotList = newArrayList(); Repository repository = repositoriesService.repository(repositoryName); ImmutableList<SnapshotId> snapshotIds = repository.snapshots(); for (SnapshotId snapshotId : snapshotIds) { snapshotList.add(repository.readSnapshot(snapshotId)); } CollectionUtil.timSort(snapshotList); return ImmutableList.copyOf(snapshotList); }
From source file:org.elasticsearch.transport.AbstractSimpleTransportTestCase.java
License:Apache License
public void testResponseHeadersArePreserved() throws InterruptedException { List<String> executors = new ArrayList<>(ThreadPool.THREAD_POOL_TYPES.keySet()); CollectionUtil.timSort(executors); // makes sure it's reproducible serviceA.registerRequestHandler("action", TestRequest::new, ThreadPool.Names.SAME, (request, channel) -> { threadPool.getThreadContext().putTransient("boom", new Object()); threadPool.getThreadContext().addResponseHeader("foo.bar", "baz"); if ("fail".equals(request.info)) { throw new RuntimeException("boom"); } else {/*from ww w . j a v a2s .c o m*/ channel.sendResponse(TransportResponse.Empty.INSTANCE); } }); CountDownLatch latch = new CountDownLatch(2); TransportResponseHandler<TransportResponse> transportResponseHandler = new TransportResponseHandler<TransportResponse>() { @Override public TransportResponse newInstance() { return TransportResponse.Empty.INSTANCE; } @Override public void handleResponse(TransportResponse response) { try { assertSame(response, TransportResponse.Empty.INSTANCE); assertTrue(threadPool.getThreadContext().getResponseHeaders().containsKey("foo.bar")); assertEquals(1, threadPool.getThreadContext().getResponseHeaders().get("foo.bar").size()); assertEquals("baz", threadPool.getThreadContext().getResponseHeaders().get("foo.bar").get(0)); assertNull(threadPool.getThreadContext().getTransient("boom")); } finally { latch.countDown(); } } @Override public void handleException(TransportException exp) { try { assertTrue(threadPool.getThreadContext().getResponseHeaders().containsKey("foo.bar")); assertEquals(1, threadPool.getThreadContext().getResponseHeaders().get("foo.bar").size()); assertEquals("baz", threadPool.getThreadContext().getResponseHeaders().get("foo.bar").get(0)); assertNull(threadPool.getThreadContext().getTransient("boom")); } finally { latch.countDown(); } } @Override public String executor() { if (1 == 1) return "same"; return randomFrom(executors); } }; serviceB.sendRequest(nodeA, "action", new TestRequest(randomFrom("fail", "pass")), transportResponseHandler); serviceA.sendRequest(nodeA, "action", new TestRequest(randomFrom("fail", "pass")), transportResponseHandler); latch.await(); }