List of usage examples for java.util.stream Stream collect
<R, A> R collect(Collector<? super T, A, R> collector);
From source file:com.github.aptd.simulation.elements.train.CTrain.java
/** * ctor/*from ww w . ja v a 2 s.co m*/ * * @param p_configuration agent configuration * @param p_id train identifier * @param p_timetable stream of timetable entries * @param p_doors list of door agent references * @param p_wagon stream of wagon references * @param p_time environment */ private CTrain(final IAgentConfiguration<ITrain<?>> p_configuration, final String p_id, final Stream<CTimetableEntry> p_timetable, final List<IDoor<?>> p_doors, final Stream<IWagon<?>> p_wagon, final ITime p_time) { super(p_configuration, FUNCTOR, p_id, p_time); m_wagon = p_wagon.collect(Collectors.toList()); m_doorsnotclosedlocked.addAll(p_doors); m_timetable = p_timetable.collect(Collectors.toList()); output(new CMessage(this, m_timetable.get(m_ttindex).m_platformid, EMessageType.TRAIN_TO_PLATFORM_ARRIVING, m_doorsnotclosedlocked.toArray())); // first timetable entry only has departure m_nextstatechange = determinenextstatechange(); m_nextactivation = m_time.current(); }
From source file:com.uber.hoodie.hadoop.realtime.HoodieRealtimeInputFormat.java
@Override public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException { Stream<FileSplit> fileSplits = Arrays.stream(super.getSplits(job, numSplits)).map(is -> (FileSplit) is); // obtain all unique parent folders for splits Map<Path, List<FileSplit>> partitionsToParquetSplits = fileSplits .collect(Collectors.groupingBy(split -> split.getPath().getParent())); // TODO(vc): Should we handle also non-hoodie splits here? Map<String, HoodieTableMetaClient> metaClientMap = new HashMap<>(); Map<Path, HoodieTableMetaClient> partitionsToMetaClient = partitionsToParquetSplits.keySet().stream() .collect(Collectors.toMap(Function.identity(), p -> { // find if we have a metaclient already for this partition. Optional<String> matchingBasePath = metaClientMap.keySet().stream() .filter(basePath -> p.toString().startsWith(basePath)).findFirst(); if (matchingBasePath.isPresent()) { return metaClientMap.get(matchingBasePath.get()); }//from w w w .j a v a 2 s .com try { HoodieTableMetaClient metaClient = getTableMetaClient(p.getFileSystem(conf), p); metaClientMap.put(metaClient.getBasePath(), metaClient); return metaClient; } catch (IOException e) { throw new HoodieIOException("Error creating hoodie meta client against : " + p, e); } })); // for all unique split parents, obtain all delta files based on delta commit timeline, grouped on file id List<HoodieRealtimeFileSplit> rtSplits = new ArrayList<>(); partitionsToParquetSplits.keySet().stream().forEach(partitionPath -> { // for each partition path obtain the data & log file groupings, then map back to inputsplits HoodieTableMetaClient metaClient = partitionsToMetaClient.get(partitionPath); HoodieTableFileSystemView fsView = new HoodieTableFileSystemView(metaClient, metaClient.getActiveTimeline()); String relPartitionPath = FSUtils.getRelativePartitionPath(new Path(metaClient.getBasePath()), partitionPath); try { Stream<FileSlice> latestFileSlices = fsView.getLatestFileSlices(relPartitionPath); // subgroup splits again by file id & match with log files. Map<String, List<FileSplit>> groupedInputSplits = partitionsToParquetSplits.get(partitionPath) .stream() .collect(Collectors.groupingBy(split -> FSUtils.getFileId(split.getPath().getName()))); latestFileSlices.forEach(fileSlice -> { List<FileSplit> dataFileSplits = groupedInputSplits.get(fileSlice.getFileId()); dataFileSplits.forEach(split -> { try { List<String> logFilePaths = fileSlice.getLogFiles() .map(logFile -> logFile.getPath().toString()).collect(Collectors.toList()); // Get the maxCommit from the last delta or compaction or commit - when bootstrapped from COW table String maxCommitTime = metaClient.getActiveTimeline() .getTimelineOfActions(Sets.newHashSet(HoodieTimeline.COMMIT_ACTION, HoodieTimeline.COMPACTION_ACTION, HoodieTimeline.DELTA_COMMIT_ACTION)) .filterCompletedInstants().lastInstant().get().getTimestamp(); rtSplits.add(new HoodieRealtimeFileSplit(split, logFilePaths, maxCommitTime)); } catch (IOException e) { throw new HoodieIOException("Error creating hoodie real time split ", e); } }); }); } catch (Exception e) { throw new HoodieException("Error obtaining data file/log file grouping: " + partitionPath, e); } }); LOG.info("Returning a total splits of " + rtSplits.size()); return rtSplits.toArray(new InputSplit[rtSplits.size()]); }
From source file:com.bc.fiduceo.post.PostProcessingTool.java
void runPostProcessing() throws IOException, InvalidRangeException { final Path inputDirectory = context.getMmdInputDirectory(); final Pattern pattern = Pattern.compile("mmd\\d{1,2}.*_.*_.*_\\d{4}-\\d{3}_\\d{4}-\\d{3}.nc"); try (Stream<Path> pathStream = Files.walk(inputDirectory)) { final Stream<Path> regularFiles = pathStream.filter(path -> Files.isRegularFile(path)); final Stream<Path> mmdFileStream = regularFiles .filter(path -> pattern.matcher(path.getFileName().toString()).matches()); List<Path> mmdFiles = mmdFileStream.collect(Collectors.toList()); computeFiles(mmdFiles);//from w w w. j av a2 s. c om } }
From source file:io.mandrel.data.extract.ExtractorService.java
public Pair<Set<Link>, Set<Link>> extractAndFilterOutlinks(Spider spider, Uri uri, Map<String, Instance<?>> cachedSelectors, Blob blob, OutlinkExtractor ol) { // Find outlinks in page Set<Link> outlinks = extractOutlinks(cachedSelectors, blob, ol); log.trace("Finding outlinks for url {}: {}", uri, outlinks); // Filter outlinks Set<Link> filteredOutlinks = null; if (outlinks != null) { Stream<Link> stream = outlinks.stream().filter(l -> l != null && l.getUri() != null); if (spider.getFilters() != null && CollectionUtils.isNotEmpty(spider.getFilters().getLinks())) { stream = stream//from www .j a v a 2 s . c o m .filter(link -> spider.getFilters().getLinks().stream().allMatch(f -> f.isValid(link))); } filteredOutlinks = stream.collect(Collectors.toSet()); } Set<Link> allFilteredOutlinks = null; if (filteredOutlinks != null) { Set<Uri> res = MetadataStores.get(spider.getId()) .deduplicate(filteredOutlinks.stream().map(l -> l.getUri()).collect(Collectors.toList())); allFilteredOutlinks = filteredOutlinks.stream().filter(f -> res.contains(f.getUri())) .collect(Collectors.toSet()); } log.trace("And filtering {}", allFilteredOutlinks); return Pair.of(outlinks, allFilteredOutlinks); }
From source file:io.logspace.hq.core.solr.event.SolrEventService.java
private List<String> createFilterQueries(EventFilter filter) { if (filter == null) { return Collections.emptyList(); }// www . jav a2 s. co m Stream<EventFilterElement> filterStream = StreamSupport.stream(filter.spliterator(), false); Collection<List<EventFilterElement>> groupedFilters = filterStream .collect(groupingBy(EventFilterElement::getProperty)).values(); List<String> result = new ArrayList<>(groupedFilters.size()); for (List<EventFilterElement> eachPropertyFilters : groupedFilters) { result.add(eachPropertyFilters.stream().map(this::createFilterQuery).collect(joining(" OR "))); } return result; }
From source file:org.kitodo.production.model.Subfolder.java
/** * Search for files with the file management interface. * * @param query// w w w . j a v a 2 s.c o m * search request consisting of an indication of the folder to be * searched and a pattern to which the file names must correspond * @return a map from the canonical file name part to the URI */ private Map<String, URI> listDirectory(Pair<URI, Pattern> query) { FilenameFilter filter = (dir, name) -> query.getRight().matcher(name).matches(); Stream<URI> relativeURIs = fileService.getSubUris(filter, query.getLeft()).parallelStream(); Stream<URI> absoluteURIs = relativeURIs.map( uri -> new File(FilenameUtils.concat(ConfigCore.getKitodoDataDirectory(), uri.getPath())).toURI()); Function<URI, String> keyMapper = createKeyMapperForPattern(query.getRight()); return absoluteURIs.collect(Collectors.toMap(keyMapper, Function.identity(), (previous, latest) -> latest, () -> new TreeMap<>(new MetadataImageComparator()))); }
From source file:com.github.aptd.simulation.elements.passenger.CPassenger.java
/** * ctor/*from ww w. j av a 2s . c o m*/ * * @param p_configuration agent configuration * @param p_id passenger identifier * @param p_time time reference */ private CPassenger(final IAgentConfiguration<IPassenger<?>> p_configuration, final String p_id, final ITime p_time, final Stream<CItineraryEntry> p_itinerary, final double p_platformchangeduration) { super(p_configuration, FUNCTOR, p_id, p_time); m_itinerary = new ArrayList<>(p_itinerary.collect(Collectors.toList())); m_speedatstation = m_distancetonextplatform / p_platformchangeduration; m_nextstatechange = determinenextstatechange(); m_nextactivation = m_nextstatechange; Logger.debug("passenger " + m_id + " with speed " + m_speedatstation); }
From source file:org.kitodo.production.services.image.ImageGenerator.java
/** * Gets the file list from the content folder, converts it into the required * form, and stores it in the sources field. *///from ww w .ja va 2 s . com public void determineSources() { Map<String, URI> contents = sourceFolder.listContents(); Stream<Entry<String, URI>> contentsStream = contents.entrySet().stream(); Stream<Pair<String, URI>> sourcesStream = contentsStream .map(lambda -> Pair.of(lambda.getKey(), lambda.getValue())); this.sources = sourcesStream.collect(Collectors.toList()); }
From source file:com.devicehive.dao.riak.NetworkDaoRiakImpl.java
@Override public List<NetworkWithUsersAndDevicesVO> getNetworksByIdsAndUsers(Long idForFiltering, Set<Long> networkdIds, Set<Long> permittedNetworks) { Set<Long> intersection = networkdIds; if (permittedNetworks != null) { intersection = networkdIds.stream().filter(permittedNetworks::contains).collect(Collectors.toSet()); }//w w w . j a v a2 s. c om Stream<NetworkWithUsersAndDevicesVO> networkStream = intersection.stream() .map(this::findWithUsersAndDevices).filter(Optional::isPresent).map(Optional::get); if (idForFiltering != null) { networkStream = networkStream .filter(n -> n.getUsers().stream().anyMatch(u -> u.getId().equals(idForFiltering))); } return networkStream.collect(Collectors.toList()); }
From source file:io.mandrel.metrics.impl.MongoMetricsRepository.java
@Override public void sync(Map<String, Long> accumulators) { LocalDateTime now = LocalDateTime.now(); LocalDateTime keytime = now.withMinute(0).withSecond(0).withNano(0); // {global.XXX=0, global.YYY=0, ...} to {global{XXX=O, YYY=0}, ...} Stream<Pair<String, Pair<String, Long>>> map = accumulators.entrySet().stream().map(e -> { Iterable<String> results = splitter.split(e.getKey()); List<String> elts = Lists.newArrayList(results); return Pair.of(elts.get(0), Pair.of(elts.get(1), e.getValue())); });//from ww w .j a va 2s. c o m Map<String, List<Pair<String, Long>>> byKey = map.collect(Collectors.groupingBy(e -> e.getLeft(), Collectors.mapping(e -> e.getRight(), Collectors.toList()))); List<? extends WriteModel<Document>> requests = byKey.entrySet().stream().map(e -> { Document updates = new Document(); e.getValue().stream().forEach(i -> { Iterable<String> results = splitter.split(i.getKey()); List<String> elts = Lists.newArrayList(results); if (elts.size() > 1) { updates.put(elts.get(0) + "." + JsonBsonCodec.toBson(elts.get(1)), i.getValue()); } else { updates.put(i.getKey(), i.getValue()); } }); return new UpdateOneModel<Document>(Filters.eq("_id", e.getKey()), new Document("$inc", updates), new UpdateOptions().upsert(true)); }).collect(Collectors.toList()); counters.bulkWrite(requests); requests = byKey.entrySet().stream().map(e -> { List<UpdateOneModel<Document>> tsUpdates = Lists.newArrayList(); e.getValue().stream().forEach(i -> { Iterable<String> results = splitter.split(i.getKey()); List<String> elts = Lists.newArrayList(results); if (elts.size() == 1 && e.getKey().equalsIgnoreCase(MetricKeys.global())) { tsUpdates.add(new UpdateOneModel<Document>( Filters.and(Filters.eq("type", e.getKey() + MetricKeys.METRIC_DELIM + i.getKey()), Filters.eq("timestamp_hour", keytime)), new Document("$inc", new Document("values." + Integer.toString(now.getMinute()), i.getValue())), new UpdateOptions().upsert(true))); } }); return tsUpdates; }).flatMap(list -> list.stream()).collect(Collectors.toList()); timeseries.bulkWrite(requests); }