List of usage examples for org.apache.lucene.util CollectionUtil introSort
public static <T> void introSort(List<T> list, Comparator<? super T> comp)
From source file:com.clearclouds.driver.monitor.jvm.HotThreads.java
License:Apache License
private String innerDetect() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("Hot threads at "); sb.append(DATE_TIME_FORMATTER.printer().print(System.currentTimeMillis())); sb.append(", interval="); sb.append(interval);//w w w. ja v a2 s .co m sb.append(", busiestThreads="); sb.append(busiestThreads); sb.append(", ignoreIdleThreads="); sb.append(ignoreIdleThreads); sb.append(":\n"); ThreadMXBean threadBean = ManagementFactory.getThreadMXBean(); boolean enabledCpu = false; try { if (threadBean.isThreadCpuTimeSupported()) { if (!threadBean.isThreadCpuTimeEnabled()) { enabledCpu = true; threadBean.setThreadCpuTimeEnabled(true); } } else { throw new IllegalStateException("MBean doesn't support thread CPU Time"); } Map<Long, MyThreadInfo> threadInfos = new HashMap<>(); for (long threadId : threadBean.getAllThreadIds()) { // ignore our own thread... if (Thread.currentThread().getId() == threadId) { continue; } long cpu = threadBean.getThreadCpuTime(threadId); if (cpu == -1) { continue; } ThreadInfo info = threadBean.getThreadInfo(threadId, 0); if (info == null) { continue; } threadInfos.put(threadId, new MyThreadInfo(cpu, info)); } Thread.sleep(interval.millis()); for (long threadId : threadBean.getAllThreadIds()) { // ignore our own thread... if (Thread.currentThread().getId() == threadId) { continue; } long cpu = threadBean.getThreadCpuTime(threadId); if (cpu == -1) { threadInfos.remove(threadId); continue; } ThreadInfo info = threadBean.getThreadInfo(threadId, 0); if (info == null) { threadInfos.remove(threadId); continue; } MyThreadInfo data = threadInfos.get(threadId); if (data != null) { data.setDelta(cpu, info); } else { threadInfos.remove(threadId); } } // sort by delta CPU time on thread. List<MyThreadInfo> hotties = new ArrayList<>(threadInfos.values()); final int busiestThreads = Math.min(this.busiestThreads, hotties.size()); // skip that for now CollectionUtil.introSort(hotties, new Comparator<MyThreadInfo>() { @Override public int compare(MyThreadInfo o1, MyThreadInfo o2) { if ("cpu".equals(type)) { return (int) (o2.cpuTime - o1.cpuTime); } else if ("wait".equals(type)) { return (int) (o2.waitedTime - o1.waitedTime); } else if ("block".equals(type)) { return (int) (o2.blockedTime - o1.blockedTime); } throw new IllegalArgumentException(); } }); // analyse N stack traces for M busiest threads long[] ids = new long[busiestThreads]; for (int i = 0; i < busiestThreads; i++) { MyThreadInfo info = hotties.get(i); ids[i] = info.info.getThreadId(); } ThreadInfo[][] allInfos = new ThreadInfo[threadElementsSnapshotCount][]; for (int j = 0; j < threadElementsSnapshotCount; j++) { // NOTE, javadoc of getThreadInfo says: If a thread of the given ID is not alive or does not exist, // null will be set in the corresponding element in the returned array. A thread is alive if it has // been started and has not yet died. allInfos[j] = threadBean.getThreadInfo(ids, Integer.MAX_VALUE); Thread.sleep(threadElementsSnapshotDelay.millis()); } for (int t = 0; t < busiestThreads; t++) { long time = 0; if ("cpu".equals(type)) { time = hotties.get(t).cpuTime; } else if ("wait".equals(type)) { time = hotties.get(t).waitedTime; } else if ("block".equals(type)) { time = hotties.get(t).blockedTime; } String threadName = null; for (ThreadInfo[] info : allInfos) { if (info != null && info[t] != null) { if (ignoreIdleThreads && isIdleThread(info[t])) { info[t] = null; continue; } threadName = info[t].getThreadName(); break; } } if (threadName == null) { continue; // thread is not alive yet or died before the first snapshot - ignore it! } double percent = (((double) time) / interval.nanos()) * 100; sb.append(String.format(Locale.ROOT, "%n%4.1f%% (%s out of %s) %s usage by thread '%s'%n", percent, TimeValue.timeValueNanos(time), interval, type, threadName)); // for each snapshot (2nd array index) find later snapshot for same thread with max number of // identical StackTraceElements (starting from end of each) boolean[] done = new boolean[threadElementsSnapshotCount]; for (int i = 0; i < threadElementsSnapshotCount; i++) { if (done[i]) continue; int maxSim = 1; boolean[] similars = new boolean[threadElementsSnapshotCount]; for (int j = i + 1; j < threadElementsSnapshotCount; j++) { if (done[j]) continue; int similarity = similarity(allInfos[i][t], allInfos[j][t]); if (similarity > maxSim) { maxSim = similarity; similars = new boolean[threadElementsSnapshotCount]; } if (similarity == maxSim) similars[j] = true; } // print out trace maxSim levels of i, and mark similar ones as done int count = 1; for (int j = i + 1; j < threadElementsSnapshotCount; j++) { if (similars[j]) { done[j] = true; count++; } } if (allInfos[i][t] != null) { final StackTraceElement[] show = allInfos[i][t].getStackTrace(); if (count == 1) { sb.append(String.format(Locale.ROOT, " unique snapshot%n")); for (int l = 0; l < show.length; l++) { sb.append(String.format(Locale.ROOT, " %s%n", show[l])); } } else { sb.append( String.format(Locale.ROOT, " %d/%d snapshots sharing following %d elements%n", count, threadElementsSnapshotCount, maxSim)); for (int l = show.length - maxSim; l < show.length; l++) { sb.append(String.format(Locale.ROOT, " %s%n", show[l])); } } } } } return sb.toString(); } finally { if (enabledCpu) { threadBean.setThreadCpuTimeEnabled(false); } } }
From source file:com.tcdi.zombodb.query.VisibilityQueryHelper.java
License:Apache License
static Map<Integer, FixedBitSet> determineVisibility(final Query query, final String field, final long myXid, final long xmin, final long xmax, final Set<Long> activeXids, IndexSearcher searcher, List<BytesRef> updatedCtids) throws IOException { final Map<Integer, FixedBitSet> visibilityBitSets = new HashMap<>(); if (updatedCtids.size() == 0) return visibilityBitSets; ////from w ww . j a v a 2s .c o m // build a map of {@link VisibilityInfo} objects by each _prev_ctid // // We use XConstantScoreQuery here so that we exclude deleted docs // final Map<BytesRef, List<VisibilityInfo>> map = new HashMap<>(); searcher.search( new XConstantScoreQuery( SearchContext.current().filterCache().cache(new TermsFilter(field, updatedCtids))), new ZomboDBTermsCollector(field) { private SortedDocValues prevCtids; private SortedNumericDocValues xids; private SortedNumericDocValues sequence; private int ord; private int maxdoc; @Override public void collect(int doc) throws IOException { xids.setDocument(doc); sequence.setDocument(doc); long xid = xids.valueAt(0); long seq = sequence.valueAt(0); BytesRef prevCtid = prevCtids.get(doc); List<VisibilityInfo> matchingDocs = map.get(prevCtid); if (matchingDocs == null) map.put(BytesRef.deepCopyOf(prevCtid), matchingDocs = new ArrayList<>()); matchingDocs.add(new VisibilityInfo(ord, maxdoc, doc, xid, seq)); } @Override public void setNextReader(AtomicReaderContext context) throws IOException { prevCtids = FieldCache.DEFAULT.getTermsIndex(context.reader(), field); xids = context.reader().getSortedNumericDocValues("_xid"); sequence = context.reader().getSortedNumericDocValues("_zdb_seq"); ord = context.ord; maxdoc = context.reader().maxDoc(); } }); if (map.isEmpty()) return visibilityBitSets; // // pick out the first VisibilityInfo for each document that is visible & committed // and build a FixedBitSet for each reader 'ord' that contains visible // documents. A map of these (key'd on reader ord) is what we return. // BytesRefBuilder bytesRefBuilder = new BytesRefBuilder() { /* overloaded to avoid making a copy of the byte array */ @Override public BytesRef toBytesRef() { return new BytesRef(this.bytes(), 0, this.length()); } }; Terms committedXidsTerms = MultiFields.getFields(searcher.getIndexReader()).terms("_zdb_committed_xid"); TermsEnum committedXidsEnum = committedXidsTerms == null ? null : committedXidsTerms.iterator(null); for (List<VisibilityInfo> visibility : map.values()) { CollectionUtil.introSort(visibility, new Comparator<VisibilityInfo>() { @Override public int compare(VisibilityInfo o1, VisibilityInfo o2) { int cmp = Long.compare(o2.xid, o1.xid); return cmp == 0 ? Long.compare(o2.sequence, o1.sequence) : cmp; } }); boolean foundVisible = false; for (VisibilityInfo mapping : visibility) { if (foundVisible || mapping.xid > xmax || activeXids.contains(mapping.xid) || (mapping.xid != myXid && !isCommitted(committedXidsEnum, mapping.xid, bytesRefBuilder))) { // document is not visible to us FixedBitSet visibilityBitset = visibilityBitSets.get(mapping.readerOrd); if (visibilityBitset == null) visibilityBitSets.put(mapping.readerOrd, visibilityBitset = new FixedBitSet(mapping.maxdoc)); visibilityBitset.set(mapping.docid); } else { foundVisible = true; } } } return visibilityBitSets; }
From source file:org.codelibs.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregator.java
License:Apache License
@Override public InternalAggregation buildAggregation(long owningBucketOrdinal) throws IOException { assert owningBucketOrdinal == 0; List<InternalDateHistogram.Bucket> buckets = new ArrayList<>((int) bucketOrds.size()); for (long i = 0; i < bucketOrds.size(); i++) { buckets.add(new InternalDateHistogram.Bucket(bucketOrds.get(i), bucketDocCount(i), keyed, formatter, bucketAggregations(i))); }/* w ww .ja v a 2 s.c o m*/ // the contract of the histogram aggregation is that shards must return buckets ordered by key in ascending order CollectionUtil.introSort(buckets, InternalOrder.KEY_ASC.comparator()); // value source will be null for unmapped fields InternalDateHistogram.EmptyBucketInfo emptyBucketInfo = minDocCount == 0 ? new InternalDateHistogram.EmptyBucketInfo(rounding, buildEmptySubAggregations(), extendedBounds) : null; return new InternalDateHistogram(name, buckets, order, minDocCount, offset, emptyBucketInfo, formatter, keyed, pipelineAggregators(), metaData()); }
From source file:org.codelibs.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregator.java
License:Apache License
@Override public InternalAggregation buildAggregation(long bucket) throws IOException { assert bucket == 0; List<InternalHistogram.Bucket> buckets = new ArrayList<>((int) bucketOrds.size()); for (long i = 0; i < bucketOrds.size(); i++) { double roundKey = Double.longBitsToDouble(bucketOrds.get(i)); double key = roundKey * interval + offset; buckets.add(// w ww. ja v a2 s. c om new InternalHistogram.Bucket(key, bucketDocCount(i), keyed, formatter, bucketAggregations(i))); } // the contract of the histogram aggregation is that shards must return buckets ordered by key in ascending order CollectionUtil.introSort(buckets, InternalOrder.KEY_ASC.comparator()); EmptyBucketInfo emptyBucketInfo = null; if (minDocCount == 0) { emptyBucketInfo = new EmptyBucketInfo(interval, offset, minBound, maxBound, buildEmptySubAggregations()); } return new InternalHistogram(name, buckets, order, minDocCount, emptyBucketInfo, formatter, keyed, pipelineAggregators(), metaData()); }
From source file:org.codelibs.elasticsearch.search.aggregations.bucket.histogram.InternalDateHistogram.java
License:Apache License
@Override public InternalAggregation doReduce(List<InternalAggregation> aggregations, ReduceContext reduceContext) { List<Bucket> reducedBuckets = reduceBuckets(aggregations, reduceContext); // adding empty buckets if needed if (minDocCount == 0) { addEmptyBuckets(reducedBuckets, reduceContext); }// w ww . j a v a 2s .co m if (order == InternalOrder.KEY_ASC) { // nothing to do, data are already sorted since shards return // sorted buckets and the merge-sort performed by reduceBuckets // maintains order } else if (order == InternalOrder.KEY_DESC) { // we just need to reverse here... List<Bucket> reverse = new ArrayList<>(reducedBuckets); Collections.reverse(reverse); reducedBuckets = reverse; } else { // sorted by sub-aggregation, need to fall back to a costly n*log(n) sort CollectionUtil.introSort(reducedBuckets, order.comparator()); } return new InternalDateHistogram(getName(), reducedBuckets, order, minDocCount, offset, emptyBucketInfo, format, keyed, pipelineAggregators(), getMetaData()); }
From source file:org.codelibs.elasticsearch.search.aggregations.bucket.histogram.InternalHistogram.java
License:Apache License
@Override public InternalAggregation doReduce(List<InternalAggregation> aggregations, ReduceContext reduceContext) { List<Bucket> reducedBuckets = reduceBuckets(aggregations, reduceContext); // adding empty buckets if needed if (minDocCount == 0) { addEmptyBuckets(reducedBuckets, reduceContext); }// w ww . j a va 2 s .com if (order == InternalOrder.KEY_ASC) { // nothing to do, data are already sorted since shards return // sorted buckets and the merge-sort performed by reduceBuckets // maintains order } else if (order == InternalOrder.KEY_DESC) { // we just need to reverse here... List<Bucket> reverse = new ArrayList<>(reducedBuckets); Collections.reverse(reverse); reducedBuckets = reverse; } else { // sorted by sub-aggregation, need to fall back to a costly n*log(n) sort CollectionUtil.introSort(reducedBuckets, order.comparator()); } return new InternalHistogram(getName(), reducedBuckets, order, minDocCount, emptyBucketInfo, format, keyed, pipelineAggregators(), getMetaData()); }
From source file:org.codelibs.elasticsearch.search.fetch.subphase.highlight.FragmentBuilderHelper.java
License:Apache License
/** * Fixes problems with broken analysis chains if positions and offsets are messed up that can lead to * {StringIndexOutOfBoundsException} in the {FastVectorHighlighter} */// www. j a v a 2s.com public static WeightedFragInfo fixWeightedFragInfo(FieldMapper mapper, Field[] values, WeightedFragInfo fragInfo) { assert fragInfo != null : "FragInfo must not be null"; assert mapper.fieldType().name().equals(values[0].name()) : "Expected FieldMapper for field " + values[0].name(); if (!fragInfo.getSubInfos().isEmpty() && (containsBrokenAnalysis(mapper.fieldType().indexAnalyzer()))) { /* This is a special case where broken analysis like WDF is used for term-vector creation at index-time * which can potentially mess up the offsets. To prevent a SAIIOBException we need to resort * the fragments based on their offsets rather than using soley the positions as it is done in * the FastVectorHighlighter. Yet, this is really a lucene problem and should be fixed in lucene rather * than in this hack... aka. "we are are working on in!" */ final List<SubInfo> subInfos = fragInfo.getSubInfos(); CollectionUtil.introSort(subInfos, (o1, o2) -> { int startOffset = o1.getTermsOffsets().get(0).getStartOffset(); int startOffset2 = o2.getTermsOffsets().get(0).getStartOffset(); return FragmentBuilderHelper.compare(startOffset, startOffset2); }); return new WeightedFragInfo( Math.min(fragInfo.getSubInfos().get(0).getTermsOffsets().get(0).getStartOffset(), fragInfo.getStartOffset()), fragInfo.getEndOffset(), subInfos, fragInfo.getTotalBoost()); } else { return fragInfo; } }
From source file:org.elasticsearch.common.util.IndexFolderUpgraderTests.java
License:Apache License
/** * Run upgrade on a real bwc index//from w w w. j a va 2 s .c o m */ public void testUpgradeRealIndex() throws IOException, URISyntaxException { List<Path> indexes = new ArrayList<>(); try (DirectoryStream<Path> stream = Files.newDirectoryStream(getBwcIndicesPath(), "index-*.zip")) { for (Path path : stream) { indexes.add(path); } } CollectionUtil.introSort(indexes, (o1, o2) -> o1.getFileName().compareTo(o2.getFileName())); final Path path = randomFrom(indexes); final String indexName = path.getFileName().toString().replace(".zip", "").toLowerCase(Locale.ROOT); try (NodeEnvironment nodeEnvironment = newNodeEnvironment()) { Path unzipDir = createTempDir(); Path unzipDataDir = unzipDir.resolve("data"); // decompress the index try (InputStream stream = Files.newInputStream(path)) { TestUtil.unzip(stream, unzipDir); } // check it is unique assertTrue(Files.exists(unzipDataDir)); Path[] list = FileSystemUtils.files(unzipDataDir); if (list.length != 1) { throw new IllegalStateException( "Backwards index must contain exactly one cluster but was " + list.length); } // the bwc scripts packs the indices under this path Path src = list[0].resolve("nodes/0/indices/" + indexName); assertTrue("[" + path + "] missing index dir: " + src.toString(), Files.exists(src)); final Path indicesPath = randomFrom(nodeEnvironment.nodePaths()).indicesPath; logger.info("--> injecting index [{}] into [{}]", indexName, indicesPath); OldIndexUtils.copyIndex(logger, src, indexName, indicesPath); IndexFolderUpgrader.upgradeIndicesIfNeeded(Settings.EMPTY, nodeEnvironment); // ensure old index folder is deleted Set<String> indexFolders = nodeEnvironment.availableIndexFolders(); assertEquals(indexFolders.size(), 1); // ensure index metadata is moved IndexMetaData indexMetaData = IndexMetaData.FORMAT.loadLatestState(logger, nodeEnvironment.resolveIndexFolder(indexFolders.iterator().next())); assertNotNull(indexMetaData); Index index = indexMetaData.getIndex(); assertEquals(index.getName(), indexName); Set<ShardId> shardIds = nodeEnvironment.findAllShardIds(index); // ensure all shards are moved assertEquals(shardIds.size(), indexMetaData.getNumberOfShards()); for (ShardId shardId : shardIds) { final ShardPath shardPath = ShardPath.loadShardPath(logger, nodeEnvironment, shardId, new IndexSettings(indexMetaData, Settings.EMPTY)); final Path translog = shardPath.resolveTranslog(); final Path idx = shardPath.resolveIndex(); final Path state = shardPath.getShardStatePath().resolve(MetaDataStateFormat.STATE_DIR_NAME); assertTrue(shardPath.exists()); assertTrue(Files.exists(translog)); assertTrue(Files.exists(idx)); assertTrue(Files.exists(state)); } } }
From source file:org.elasticsearch.common.util.MultiDataPathUpgraderTests.java
License:Apache License
/** * Run upgrade on a real bwc index//ww w .j ava 2s . c om */ public void testUpgradeRealIndex() throws IOException, URISyntaxException { List<Path> indexes = new ArrayList<>(); try (DirectoryStream<Path> stream = Files.newDirectoryStream(getBwcIndicesPath(), "index-*.zip")) { for (Path path : stream) { indexes.add(path); } } CollectionUtil.introSort(indexes, new Comparator<Path>() { @Override public int compare(Path o1, Path o2) { return o1.getFileName().compareTo(o2.getFileName()); } }); final ShardId shardId = new ShardId("test", 0); final Path path = randomFrom(indexes); final Path indexFile = path; final String indexName = indexFile.getFileName().toString().replace(".zip", "").toLowerCase(Locale.ROOT); try (NodeEnvironment nodeEnvironment = newNodeEnvironment()) { if (nodeEnvironment.nodeDataPaths().length == 1) { MultiDataPathUpgrader helper = new MultiDataPathUpgrader(nodeEnvironment); assertFalse(helper.needsUpgrading(shardId)); return; } Path unzipDir = createTempDir(); Path unzipDataDir = unzipDir.resolve("data"); // decompress the index try (InputStream stream = Files.newInputStream(indexFile)) { TestUtil.unzip(stream, unzipDir); } // check it is unique assertTrue(Files.exists(unzipDataDir)); Path[] list = FileSystemUtils.files(unzipDataDir); if (list.length != 1) { throw new IllegalStateException( "Backwards index must contain exactly one cluster but was " + list.length); } // the bwc scripts packs the indices under this path Path src = list[0].resolve("nodes/0/indices/" + indexName); assertTrue("[" + indexFile + "] missing index dir: " + src.toString(), Files.exists(src)); Path[] multiDataPath = new Path[nodeEnvironment.nodeDataPaths().length]; int i = 0; for (NodeEnvironment.NodePath nodePath : nodeEnvironment.nodePaths()) { multiDataPath[i++] = nodePath.indicesPath; } logger.info("--> injecting index [{}] into multiple data paths", indexName); OldIndexUtils.copyIndex(logger, src, indexName, multiDataPath); final ShardPath shardPath = new ShardPath(false, nodeEnvironment.availableShardPaths(new ShardId(indexName, 0))[0], nodeEnvironment.availableShardPaths(new ShardId(indexName, 0))[0], IndexMetaData.INDEX_UUID_NA_VALUE, new ShardId(indexName, 0)); logger.info("{}", FileSystemUtils.files(shardPath.resolveIndex())); MultiDataPathUpgrader helper = new MultiDataPathUpgrader(nodeEnvironment); helper.upgrade(new ShardId(indexName, 0), shardPath); helper.checkIndex(shardPath); assertFalse(helper.needsUpgrading(new ShardId(indexName, 0))); } }
From source file:org.elasticsearch.discovery.zen.elect.ElectMasterService.java
License:Apache License
private List<DiscoveryNode> sortedMasterNodes(Iterable<DiscoveryNode> nodes) { List<DiscoveryNode> possibleNodes = Lists.newArrayList(nodes); if (possibleNodes.isEmpty()) { return null; }/* w ww .ja v a 2 s .c o m*/ // clean non master nodes for (Iterator<DiscoveryNode> it = possibleNodes.iterator(); it.hasNext();) { DiscoveryNode node = it.next(); if (!node.masterNode()) { it.remove(); } } CollectionUtil.introSort(possibleNodes, nodeComparator); return possibleNodes; }