List of usage examples for java.util.stream Collectors averagingLong
public static <T> Collector<T, ?, Double> averagingLong(ToLongFunction<? super T> mapper)
From source file:Main.java
public static void main(String[] args) { Stream<String> s = Stream.of("1", "2", "3"); double o = s.collect(Collectors.averagingLong(n -> Long.parseLong(n))); System.out.println(o);/*from w w w. j av a 2s . c o m*/ }
From source file:org.sakaiproject.tool.assessment.facade.ItemHashUtil.java
/** * Bit of a hack to allow reuse between {@link ItemFacadeQueries} and {@link PublishedItemFacadeQueries}. * Arguments are rather arbitrary extension points to support what we happen to <em>know</em> are the differences * between item and published item processing, as well as the common utilities/service dependencies. * * @param batchSize//www. j ava2s . c om * @param hqlQueries * @param concreteType * @param hashAndAssignCallback * @param hibernateTemplate * @return */ BackfillItemHashResult backfillItemHashes(int batchSize, Map<String, String> hqlQueries, Class<? extends ItemDataIfc> concreteType, Function<ItemDataIfc, ItemDataIfc> hashAndAssignCallback, HibernateTemplate hibernateTemplate) { final long startTime = System.currentTimeMillis(); log.debug("Hash backfill starting for items of type [" + concreteType.getSimpleName() + "]"); if (batchSize <= 0) { batchSize = 100; } final int flushSize = batchSize; final AtomicInteger totalItems = new AtomicInteger(0); final AtomicInteger totalItemsNeedingBackfill = new AtomicInteger(0); final AtomicInteger batchNumber = new AtomicInteger(0); final AtomicInteger recordsRead = new AtomicInteger(0); final AtomicInteger recordsUpdated = new AtomicInteger(0); final Map<Long, Throwable> hashingErrors = new TreeMap<>(); final Map<Integer, Throwable> otherErrors = new TreeMap<>(); final List<Long> batchElapsedTimes = new ArrayList<>(); // always needed as *printable* average per-batch timing value, so just store as string. and cache at this // scope b/c we sometimes need to print a single calculation multiple times, e.g. in last batch and // at method exit final AtomicReference<String> currentAvgBatchElapsedTime = new AtomicReference<>("0.00"); final AtomicBoolean areMoreItems = new AtomicBoolean(true); // Get the item totals up front since a) we know any questions created while the job is running will be // assigned hashes and thus won't need to be handled by the job and b) makes bookkeeping within the job much // easier hibernateTemplate.execute(session -> { session.setDefaultReadOnly(true); totalItems.set(countItems(hqlQueries, session)); totalItemsNeedingBackfill.set(countItemsNeedingHashBackfill(hqlQueries, session)); log.debug("Hash backfill required for [" + totalItemsNeedingBackfill + "] of [" + totalItems + "] items of type [" + concreteType.getSimpleName() + "]"); return null; }); while (areMoreItems.get()) { long batchStartTime = System.currentTimeMillis(); batchNumber.getAndIncrement(); final AtomicInteger itemsHashedInBatch = new AtomicInteger(0); final AtomicInteger itemsReadInBatch = new AtomicInteger(0); final AtomicReference<Throwable> failure = new AtomicReference<>(null); // Idea here is a) avoid very long running transactions and b) avoid reading all items into memory // and c) avoid weirdness, e.g. duplicate results, when paginating complex hibernate objects. So // there's a per-batch transaction, and each batch re-runs the same two item lookup querys, one to // get the list of IDs for the next page of items, and one to resolve those IDs to items try { new TransactionTemplate(transactionManager, requireNewTransaction()).execute(status -> { hibernateTemplate.execute(session -> { List<ItemDataIfc> itemsInBatch = null; try { // resource cleanup block session.setFlushMode(FlushMode.MANUAL); try { // initial read block (failures here are fatal) // set up the actual result set for this batch of items. use error count to skip over failed items final List<Long> itemIds = itemIdsNeedingHashBackfill(hqlQueries, flushSize, hashingErrors.size(), session); itemsInBatch = itemsById(itemIds, hqlQueries, session); } catch (RuntimeException e) { // Panic on failure to read counts and/or the actual items in the batch. // Otherwise would potentially loop indefinitely since this design has no way way to // skip this page of results. log.error("Failed to read batch of hashable items. Giving up at record [" + recordsRead + "] of [" + totalItemsNeedingBackfill + "] Type: [" + concreteType.getSimpleName() + "]", e); areMoreItems.set(false); // force overall loop to exit throw e; // force txn to give up } for (ItemDataIfc item : itemsInBatch) { recordsRead.getAndIncrement(); itemsReadInBatch.getAndIncrement(); // Assign the item's hash/es try { log.debug("Backfilling hash for item [" + recordsRead + "] of [" + totalItemsNeedingBackfill + "] Type: [" + concreteType.getSimpleName() + "] ID: [" + item.getItemId() + "]"); hashAndAssignCallback.apply(item); itemsHashedInBatch.getAndIncrement(); } catch (Throwable t) { // Failures considered ignorable here... probably some unexpected item state // that prevented hash calculation. // // Re the log statement... yes, the caller probably logs exceptions, but likely // without stack traces, and we'd like to advertise failures as quickly as possible, // so we go ahead and emit an error log here. log.error("Item hash calculation failed for item [" + recordsRead + "] of [" + totalItemsNeedingBackfill + "] Type: [" + concreteType.getSimpleName() + "] ID: [" + (item == null ? "?" : item.getItemId()) + "]", t); hashingErrors.put(item.getItemId(), t); } } if (itemsHashedInBatch.get() > 0) { session.flush(); recordsUpdated.getAndAdd(itemsHashedInBatch.get()); } areMoreItems.set(itemsInBatch.size() >= flushSize); } finally { quietlyClear(session); // potentially very large, so clear aggressively } return null; }); // end session return null; }); // end transaction } catch (Throwable t) { // We're still in the loop over all batches, but something caused the current batch (and its // transaction) to exit abnormally. Logging of both success and failure cases is quite detailed, // and needs the same timing calcs, so is consolidated into the 'finally' block below. failure.set(t); otherErrors.put(batchNumber.get(), t); } finally { // Detailed batch-level reporting final long batchElapsed = (System.currentTimeMillis() - batchStartTime); batchElapsedTimes.add(batchElapsed); currentAvgBatchElapsedTime.set(new DecimalFormat("#.00") .format(batchElapsedTimes.stream().collect(Collectors.averagingLong(l -> l)))); if (failure.get() == null) { log.debug("Item hash backfill batch flushed to database. Type: [" + concreteType.getSimpleName() + "] Batch number: [" + batchNumber + "] Items attempted in batch: [" + itemsReadInBatch + "] Items succeeded in batch: [" + itemsHashedInBatch + "] Total items attempted: [" + recordsRead + "] Total items succeeded: [" + recordsUpdated + "] Total attemptable items: [" + totalItemsNeedingBackfill + "] Elapsed batch time: [" + batchElapsed + "ms] Avg time/batch: [" + currentAvgBatchElapsedTime + "ms]"); } else { // yes, caller probably logs exceptions later, but probably without stack traces, and we'd // like to advertise failures as quickly as possible, so we go ahead and emit an error log // here. log.error("Item hash backfill failed. Type: [" + concreteType.getSimpleName() + "] Batch number: [" + batchNumber + "] Items attempted in batch: [" + itemsReadInBatch + "] Items flushable (but failed) in batch: [" + itemsHashedInBatch + "] Total items attempted: [" + recordsRead + "] Total items succeeded: [" + recordsUpdated + "] Total attemptable items: [" + totalItemsNeedingBackfill + "] Elapsed batch time: [" + batchElapsed + "ms] Avg time/batch: [" + currentAvgBatchElapsedTime + "ms]", failure.get()); } } } // end loop over all batches final long elapsedTime = System.currentTimeMillis() - startTime; log.debug("Hash backfill completed for items of type [" + concreteType.getSimpleName() + "]. Total items attempted: [" + recordsRead + "] Total items succeeded: [" + recordsUpdated + "] Target attemptable items: [" + totalItemsNeedingBackfill + "] Total elapsed time: [" + elapsedTime + "ms] Total batches: [" + batchNumber + "] Avg time/batch: [" + currentAvgBatchElapsedTime + "ms]"); return new BackfillItemHashResult(elapsedTime, totalItems.get(), totalItemsNeedingBackfill.get(), recordsRead.get(), recordsUpdated.get(), flushSize, hashingErrors, otherErrors); }