List of usage examples for com.google.common.collect Iterables limit
public static <T> Iterable<T> limit(final Iterable<T> iterable, final int limitSize)
From source file:org.immutables.sequence.Sequence.java
/** * Creates a fluent iterable with the first {@code size} elements of this * fluent iterable. If this fluent iterable does not contain that many elements, * the returned fluent iterable will have the same behavior as this fluent iterable. * The returned fluent iterable's iterator supports {@code remove()} if this * fluent iterable's iterator does.// ww w.j a v a 2 s .c o m * @param size the maximum number of elements in the returned fluent iterable * @return the sequence */ @CheckReturnValue public final Sequence<E> limit(int size) { return from(Iterables.limit(iterable, size)); }
From source file:org.diqube.execution.steps.OrderStep.java
@Override protected void execute() { // intermediateRun = true if NOT all final versions of all columns have been built and are available in defaultEnv // -> we only have intermediary values! boolean intermediateRun = !(columnBuiltConsumer.getNumberOfTimesWired() == 0 || allColumnsBuilt.get()); if (columnBuiltConsumer.getNumberOfTimesWired() > 0 && columnBuiltInputIsDone.get() && !allColumnsBuilt.get()) { logger.debug("Ordering needs to wait for a column to be built, but it won't be built. Skipping."); forEachOutputConsumerOfType(GenericConsumer.class, c -> c.sourceIsDone()); doneProcessing();// w ww .ja v a 2 s.c om return; } ExecutionEnvironment env; if (!intermediateRun) { env = defaultEnv; } else { env = newestTemporaryEnv; if (env == null) return; // we'll be sorting on a version of the Env that contains only intermediate columns. Check if we have at least // some values for all interesting columns. boolean allColsAvailable = sortColSet.stream().allMatch(colName -> env.getColumnShard(colName) != null); if (!allColsAvailable) return; } // new row IDs we ought to order into the result. NavigableSet<Long> activeRowIdsSet = new TreeSet<>(); Long tmpNextRowId; while ((tmpNextRowId = rowIds.poll()) != null) activeRowIdsSet.add(tmpNextRowId); SortComparator headComparator = headComparatorProvider.apply(env); if (intermediateRun) { // Make sure that we only order those rows, where we have values for all columns. // Please note the following: // We only make sure that each column contains /any/ value on the row IDs, these might be as well default values // filled in by SparseColumnShardBuilder! We therefore might order based on "wrong" values here. But this is not // as important, because as soon as we have correct values and the orderStep is executed again (either still in // 'intermediary' mode or in 'isLastRun' mode) the row will be ordered correctly. new ColumnVersionBuiltHelper().publishActiveRowIds(env, sortColSet, activeRowIdsSet, notYetProcessedRowIds); } else { if (notYetProcessedRowIds.size() > 0) { activeRowIdsSet.addAll(notYetProcessedRowIds); notYetProcessedRowIds.clear(); } } logger.trace( "Starting to order based on Env {}, having active RowIDs (limt) {}, not yet processed (limit) {}. intermediateRun: {}", env, Iterables.limit(activeRowIdsSet, 100), Iterables.limit(notYetProcessedRowIds, 100), intermediateRun); if (activeRowIdsSet.size() > 0) { Long[] activeRowIds = activeRowIdsSet.toArray(new Long[activeRowIdsSet.size()]); // be sure that we have enough space in the array sortedRowIds = resizeArrayForLength(sortedRowIds, sortedRowIdsLength + activeRowIds.length); // add new values to the array and use insertion sort to put them at the right sorted locations System.arraycopy(activeRowIds, 0, sortedRowIds, sortedRowIdsLength, activeRowIds.length); sortedRowIdsLength += activeRowIds.length; } // Use default JVM sorting, which implements a TimSort at least in OpenJDK - this executes very well even on // partly sorted arrays (which is [mostly] true in our case if the execute method is executed at least twice). // - // Execute this in each run. This is needed, as either there were new rowIds added or some rows changed their values // (otherwise execute() would not be called). Therefore we always need to sort the array. // TODO #8 support sorting only /some/ elements in case we're based on intermediary values. Arrays.sort(sortedRowIds, 0, sortedRowIdsLength, headComparator); // cutOffPoint = first index in sortedRowIds that would be cut off by a limit/softLimit clause. null otherwise. Integer cutOffPoint = null; // Find cut off point according to limit/softLimit. if (softLimit == null) { if (sortedRowIdsMaxLength != null && sortedRowIdsLength > sortedRowIdsMaxLength) { // we have a LIMIT set but our sorted array is already longer. Cut its length to save some memory. // remember point to cut off at cutOffPoint = sortedRowIdsMaxLength.intValue(); } } else { if (sortedRowIdsLength > softLimit) { // we are above the soft limit! Long lastContainedRowId = sortedRowIds[softLimit.intValue() - 1]; int softLength = softLimit.intValue(); // linearily search those rows which should be included in the result although they would have been cut-off. while (softLength < sortedRowIdsLength && headComparator.compare(lastContainedRowId, sortedRowIds[softLength]) == 0) softLength++; logger.trace("Hit soft limit {}, using length {}", softLimit.intValue(), softLength); // remember point to cut off at cutOffPoint = softLength; } } // inform RowIdConsumers about new row IDs if (activeRowIdsSet.size() > 0) { if (intermediateRun) { // if this is an intermediary run, make sure that we report all rowIds to subsequent steps, as we did not // actually execute the cut-off. Long[] rowIds = activeRowIdsSet.stream().toArray(l -> new Long[l]); forEachOutputConsumerOfType(RowIdConsumer.class, c -> c.consume(rowIds)); } else { // This is a last run. This means that sorting is now based on the defaultEnv and that none of the columns might // change its values anymore. // If there were new rowIds, we need to report only those that actually are included inside // the result value (and are not cut-off). Long[] rowIdsToBeOutput; if (cutOffPoint != null) rowIdsToBeOutput = findRowIdsToBeOutputOnCutOff(activeRowIdsSet, cutOffPoint); else rowIdsToBeOutput = activeRowIdsSet.stream().toArray(l -> new Long[l]); // report all row IDs so subsequent RowID consumers can handle them. We might report too much rowIDs here. forEachOutputConsumerOfType(RowIdConsumer.class, c -> c.consume(rowIdsToBeOutput)); } } if (!intermediateRun && cutOffPoint != null) // execute cut off if we're not in an intermediate run. While in an intermediate run we're based on arbitrary // versions of the column (see ColumnVersionBuiltConsumer). This means that the values of all interesting rows // in all columns might change arbitrarily. Therefore we cannot execute a cut-off in that case, as a row that // we'd cut off now might change its value later on so we'd actually need to include it. On the other hand, not // only that row might change its value, but all other rows might change their values and force that row to be // inside the result set - but if we cut it off before, there's no chance to recover it. So we execute no // cut-off in that case. sortedRowIdsLength = cutOffPoint; logger.trace("Ordering result (limit): {}", Iterables.limit(Arrays.asList(sortedRowIds), Math.min(20, sortedRowIdsLength))); final Integer finalIntendedCutOffPoint = cutOffPoint; // output sorted result in each run - it might be that we did not have new row IDs, but the value of specific row // IDs were changed and therefore we have a now ordering (after sorting above) - we should publicize that. forEachOutputConsumerOfType(OrderedRowIdConsumer.class, new Consumer<OrderedRowIdConsumer>() { @Override public void accept(OrderedRowIdConsumer orderedConsumer) { int startIdx; int length; if (limitStart != null) { startIdx = limitStart.intValue(); length = sortedRowIdsLength - limitStart.intValue(); } else { startIdx = 0; length = sortedRowIdsLength; } if (intermediateRun && finalIntendedCutOffPoint != null) { // we did not execute a cut-off, as we're in an intermediary run. So do at least a 'virtual cut-off' here, by // only reporting those rows inside the intendedCutOffPoint to the steps interested in the ordering. length -= sortedRowIdsLength - finalIntendedCutOffPoint; } if (length < 0 || startIdx >= sortedRowIds.length) { // ensure that we do not pass on invalid values to ArrayViewLongList. startIdx = 0; length = 0; } orderedConsumer.consumeOrderedRowIds(new ArrayViewLongList(sortedRowIds, startIdx, length)); } }); // check if we're done. if (!intermediateRun && rowIdSourceIsEmpty.get() && rowIds.isEmpty()) { forEachOutputConsumerOfType(GenericConsumer.class, c -> c.sourceIsDone()); doneProcessing(); } }
From source file:com.spotify.reaper.storage.MemoryStorage.java
@Override public Collection<RepairRunStatus> getClusterRunStatuses(String clusterName, int limit) { Optional<Cluster> cluster = getCluster(clusterName); if (!cluster.isPresent()) { return Collections.emptyList(); } else {//from w w w . ja va2 s. c o m List<RepairRunStatus> runStatuses = Lists.newArrayList(); List<RepairRun> runs = getRepairRunsForCluster(clusterName); Collections.sort(runs); for (RepairRun run : Iterables.limit(runs, limit)) { RepairUnit unit = getRepairUnit(run.getRepairUnitId()).get(); int segmentsRepaired = getSegmentAmountForRepairRunWithState(run.getId(), RepairSegment.State.DONE); int totalSegments = getSegmentAmountForRepairRun(run.getId()); runStatuses.add(new RepairRunStatus(run.getId(), clusterName, unit.getKeyspaceName(), unit.getColumnFamilies(), segmentsRepaired, totalSegments, run.getRunState(), run.getStartTime(), run.getEndTime(), run.getCause(), run.getOwner(), run.getLastEvent(), run.getCreationTime(), run.getPauseTime(), run.getIntensity(), run.getRepairParallelism())); } return runStatuses; } }
From source file:com.torodb.torod.db.backends.mysql.MySQLDbConnection.java
@Override public void insertSubdocuments(String collection, SubDocType type, Iterable<? extends SubDocument> subDocuments) { Connection connection = getJooqConf().connectionProvider().acquire(); try {/*w w w . ja v a2s.co m*/ int maxCappedSize = 10; int cappedSize = Iterables.size(Iterables.limit(subDocuments, maxCappedSize)); if (cappedSize < maxCappedSize) { //there are not enough elements on the insert => fallback LOGGER.debug("The insert window is not big enough to use copy (the " + "limit is {}, the real size is {}).", maxCappedSize, cappedSize); super.insertSubdocuments(collection, type, subDocuments); } else { super.insertSubdocuments(collection, type, subDocuments); } } finally { getJooqConf().connectionProvider().release(connection); } }
From source file:com.gnapse.common.inflector.Rule.java
/** * Returns a string representation of the given word list, shortened to 3 items. */// w ww . j a va 2s .c o m protected static String shortListStr(Iterable<String> words) { return Joiner.on(",").join(Iterables.limit(words, 3)).concat(",..."); }
From source file:io.druid.sql.calcite.schema.DruidSchema.java
/** * Attempt to refresh "segmentSignatures" for a set of segments for a particular dataSource. Returns the set of * segments actually refreshed, which may be a subset of the asked-for set. *//* w w w . j av a2 s .co m*/ private Set<DataSegment> refreshSegmentsForDataSource(final String dataSource, final Set<DataSegment> segments) throws IOException { log.debug("Refreshing metadata for dataSource[%s].", dataSource); final long startTime = System.currentTimeMillis(); // Segment identifier -> segment object. final Map<String, DataSegment> segmentMap = segments.stream() .collect(Collectors.toMap(DataSegment::getIdentifier, Function.identity())); final Set<DataSegment> retVal = new HashSet<>(); final Sequence<SegmentAnalysis> sequence = runSegmentMetadataQuery(queryLifecycleFactory, Iterables.limit(segments, MAX_SEGMENTS_PER_QUERY), escalatingAuthenticator.createEscalatedAuthenticationResult()); Yielder<SegmentAnalysis> yielder = Yielders.each(sequence); try { while (!yielder.isDone()) { final SegmentAnalysis analysis = yielder.get(); final DataSegment segment = segmentMap.get(analysis.getId()); if (segment == null) { log.warn("Got analysis for segment[%s] we didn't ask for, ignoring.", analysis.getId()); } else { final RowSignature rowSignature = analysisToRowSignature(analysis); log.debug("Segment[%s] has signature[%s].", segment.getIdentifier(), rowSignature); setSegmentSignature(segment, rowSignature); retVal.add(segment); } yielder = yielder.next(null); } } finally { yielder.close(); } log.info("Refreshed metadata for dataSource[%s] in %,d ms (%d segments queried, %d segments left).", dataSource, System.currentTimeMillis() - startTime, retVal.size(), segments.size() - retVal.size()); return retVal; }
From source file:msi.gaml.operators.Containers.java
@operator(value = { "last", "last_of" }, can_be_const = true, content_type = ITypeProvider.CONTENT_TYPE_AT_INDEX
+ 2, category = { IOperatorCategory.CONTAINER }, concept = { IConcept.CONTAINER })
@doc(value = "Returns the nth last elements of the container. If n is greater than the list size, a translation of the container to a list is returned. If it is equal or less than zero, returns an empty list")
public static IList last(final IScope scope, final Integer number, final IContainer c) {
final IList result = GamaListFactory.create(scope, c.getGamlType().getContentType(), Iterables.limit(
Lists.reverse(notNull(scope, c).listValue(scope, Types.NO_TYPE, false)), number < 0 ? 0 : number));
Collections.reverse(result);/* www. j av a2 s . c om*/
return result;
}
From source file:org.diqube.server.querymaster.query.MasterQueryExecutor.java
/** * @return A {@link RResultTable} instance of the data that is currently available in {@link #valuesByRow} etc. *///from w ww . java 2 s . c o m private RResultTable createRResultTableFromCurrentValues() { List<Long> rowIds = null; synchronized (orderedSync) { if (isOrdered && orderedRowIds != null) rowIds = new ArrayList<Long>(orderedRowIds); } RResultTable res = new RResultTable(); res.setColumnNames(selectedColumns); res.setColumnRequests(selectionRequests); if ((rowIds == null || rowIds.isEmpty()) && isOrdered) // return empty table. This could be the case if the result table is actually empty. return res; if (rowIds == null || rowIds.isEmpty()) rowIds = new ArrayList<>(valuesByRow.keySet()); // TODO lines will jump? if (rowIds.isEmpty()) // Could happen if not ordered, no row Ids. Return empty table. return res; if (isHaving) { // if we have a rowId list from the HAVING execution, remove the row IDs that are not contained in that list! Long[] activeHavingRowIds; synchronized (havingRowIdsSync) { activeHavingRowIds = havingRowIds; } logger.trace("Current HAVING result: {} rows", (activeHavingRowIds != null) ? activeHavingRowIds.length : 0); if (activeHavingRowIds == null || activeHavingRowIds.length == 0) // return empty table. This could be the case if the result table is actually empty. return res; Set<Long> havingRowIds = new HashSet<>(Arrays.asList(activeHavingRowIds)); logger.trace("Current HAVING rows (limit): {}", Iterables.limit(havingRowIds, 100)); for (Iterator<Long> rowIdIt = rowIds.iterator(); rowIdIt.hasNext();) { Long rowId = rowIdIt.next(); if (!havingRowIds.contains(rowId)) rowIdIt.remove(); } } List<List<RValue>> rows = new ArrayList<>(); for (Long rowId : rowIds) { if (!valuesByRow.containsKey(rowId)) continue; List<RValue> row = new ArrayList<>(); for (String colName : selectedColumns) row.add(RValueUtil.createRValue(valuesByRow.get(rowId).get(colName))); // fill any cells where we do not have data with an empty string. for (int i = 0; i < row.size(); i++) if (row.get(i) == null) row.set(i, RValueUtil.createRValue("")); rows.add(row); } res.setRows(rows); return res; }
From source file:org.apache.druid.sql.calcite.schema.DruidSchema.java
/** * Attempt to refresh "segmentSignatures" for a set of segments for a particular dataSource. Returns the set of * segments actually refreshed, which may be a subset of the asked-for set. *///from ww w . ja v a 2 s . com private Set<DataSegment> refreshSegmentsForDataSource(final String dataSource, final Set<DataSegment> segments) throws IOException { log.debug("Refreshing metadata for dataSource[%s].", dataSource); final long startTime = System.currentTimeMillis(); // Segment identifier -> segment object. final Map<String, DataSegment> segmentMap = segments.stream() .collect(Collectors.toMap(DataSegment::getIdentifier, Function.identity())); final Set<DataSegment> retVal = new HashSet<>(); final Sequence<SegmentAnalysis> sequence = runSegmentMetadataQuery(queryLifecycleFactory, Iterables.limit(segments, MAX_SEGMENTS_PER_QUERY), escalator.createEscalatedAuthenticationResult()); Yielder<SegmentAnalysis> yielder = Yielders.each(sequence); try { while (!yielder.isDone()) { final SegmentAnalysis analysis = yielder.get(); final DataSegment segment = segmentMap.get(analysis.getId()); if (segment == null) { log.warn("Got analysis for segment[%s] we didn't ask for, ignoring.", analysis.getId()); } else { final RowSignature rowSignature = analysisToRowSignature(analysis); log.debug("Segment[%s] has signature[%s].", segment.getIdentifier(), rowSignature); setSegmentSignature(segment, rowSignature); retVal.add(segment); } yielder = yielder.next(null); } } finally { yielder.close(); } log.info("Refreshed metadata for dataSource[%s] in %,d ms (%d segments queried, %d segments left).", dataSource, System.currentTimeMillis() - startTime, retVal.size(), segments.size() - retVal.size()); return retVal; }
From source file:com.smoketurner.notification.application.store.NotificationStore.java
/** * Returns an iterable that skips forward to a given notification ID then * only returns count more notifications. If the given notification ID is * not found/*from w ww.ja v a2 s.c om*/ * * @param notifications * Iterable of notifications * @param startId * notification ID to start at * @param inclusive * Whether to include the startId notification or not * @param limitSize * Number of notifications to return * @return Iterable containing the subset of the original notifications */ public Iterable<Notification> skip(@Nonnull final Iterable<Notification> notifications, final long startId, final boolean inclusive, final int limitSize) { Objects.requireNonNull(notifications); final int position = indexOf(notifications, startId); if (position == -1) { return Iterables.limit(notifications, limitSize); } if (inclusive) { return Iterables.limit(Iterables.skip(notifications, position), limitSize); } return Iterables.limit(Iterables.skip(notifications, position + 1), limitSize); }