List of usage examples for com.google.common.collect Range openClosed
public static <C extends Comparable<?>> Range<C> openClosed(C lower, C upper)
From source file:com.google.cloud.genomics.dataflow.pipelines.AnnotateVariants.java
@Override public void processElement(DoFn<StreamVariantsRequest, KV<String, VariantAnnotation>>.ProcessContext c) throws Exception { Genomics genomics = GenomicsFactory.builder().build().fromOfflineAuth(auth); StreamVariantsRequest request = StreamVariantsRequest.newBuilder(c.element()).addAllCallSetIds(callSetIds) .build();/*w w w . j a v a2s .co m*/ LOG.info("processing contig " + request); Iterator<StreamVariantsResponse> iter = VariantStreamIterator.enforceShardBoundary(auth, request, ShardBoundary.Requirement.STRICT, VARIANT_FIELDS); if (!iter.hasNext()) { LOG.info("region has no variants, skipping"); return; } IntervalTree<Annotation> transcripts = retrieveTranscripts(genomics, request); ListMultimap<Range<Long>, Annotation> variantAnnotations = retrieveVariantAnnotations(genomics, request); Stopwatch stopwatch = Stopwatch.createStarted(); int varCount = 0; while (iter.hasNext()) { Iterable<Variant> varIter = FluentIterable.from(iter.next().getVariantsList()) .filter(VariantUtils.IS_SNP); for (Variant variant : varIter) { List<String> alleles = ImmutableList.<String>builder().addAll(variant.getAlternateBasesList()) .add(variant.getReferenceBases()).build(); Range<Long> pos = Range.openClosed(variant.getStart(), variant.getEnd()); for (String allele : alleles) { String outKey = Joiner.on(":").join(variant.getReferenceName(), variant.getStart(), allele, variant.getId()); for (Annotation match : variantAnnotations.get(pos)) { if (allele.equals(match.getVariant().getAlternateBases())) { // Exact match to a known variant annotation; straightforward join. c.output(KV.of(outKey, match.getVariant())); } } Iterator<Node<Annotation>> transcriptIter = transcripts .overlappers(pos.lowerEndpoint().intValue(), pos.upperEndpoint().intValue() - 1); // Inclusive. while (transcriptIter.hasNext()) { // Calculate an effect of this allele on the coding region of the given transcript. Annotation transcript = transcriptIter.next().getValue(); VariantEffect effect = AnnotationUtils.determineVariantTranscriptEffect(variant.getStart(), allele, transcript, getCachedTranscriptBases(genomics, transcript)); if (effect != null && !VariantEffect.SYNONYMOUS_SNP.equals(effect)) { c.output(KV.of(outKey, new VariantAnnotation().setAlternateBases(allele).setType("SNP") .setEffect(effect.toString()).setGeneId(transcript.getTranscript().getGeneId()) .setTranscriptIds(ImmutableList.of(transcript.getId())))); } } } varCount++; if (varCount % 1e3 == 0) { LOG.info(String.format("read %d variants (%.2f / s)", varCount, (double) varCount / stopwatch.elapsed(TimeUnit.SECONDS))); } } } LOG.info("finished reading " + varCount + " variants in " + stopwatch); }
From source file:com.google.android.apps.forscience.whistlepunk.review.RunReviewExporter.java
@Override public boolean handleMessage(Message msg) { if (msg.what == MSG_WRITE) { if (mStop) { return false; }/*from w ww.j a v a2 s.co m*/ ScalarReadingList list = (ScalarReadingList) msg.obj; list.deliver(mStreamConsumer); if (mStop) { return false; } final long start = mRun.getFirstTimestamp(); final long end = mRun.getLastTimestamp(); int progress = (int) (((mLastTimeStampWritten - start) / (double) (end - start)) * 100); mListener.onExportProgress(progress); if (list.size() == 0 || list.size() < MAX_RECORDS || mLastTimeStampWritten >= end) { endExport(); } else { Range<Long> times = Range.openClosed(mLastTimeStampWritten, end); getReadings(TimeRange.oldest(times)); } return true; } return false; }
From source file:org.apache.bookkeeper.mledger.impl.ManagedCursorImpl.java
private void recoverIndividualDeletedMessages(List<MLDataFormats.MessageRange> individualDeletedMessagesList) { lock.writeLock().lock();// w w w. j a va 2s. c o m try { individualDeletedMessages.clear(); individualDeletedMessagesList.forEach(messageRange -> individualDeletedMessages .add(Range.openClosed(new PositionImpl(messageRange.getLowerEndpoint()), new PositionImpl(messageRange.getUpperEndpoint())))); } finally { lock.writeLock().unlock(); } }
From source file:org.apache.bookkeeper.mledger.impl.ManagedCursorImpl.java
private void recoveredCursor(PositionImpl position) { // if the position was at a ledger that didn't exist (since it will be deleted if it was previously empty), // we need to move to the next existing ledger if (!ledger.ledgerExists(position.getLedgerId())) { long nextExistingLedger = ledger.getNextValidLedger(position.getLedgerId()); position = PositionImpl.get(nextExistingLedger, -1); }/*from w w w. j av a 2 s . c o m*/ log.info("[{}] Cursor {} recovered to position {}", ledger.getName(), name, position); messagesConsumedCounter = -getNumberOfEntries(Range.openClosed(position, ledger.getLastPosition())); markDeletePosition = position; readPosition = ledger.getNextValidPosition(position); STATE_UPDATER.set(this, State.NoLedger); }
From source file:org.apache.bookkeeper.mledger.impl.ManagedLedgerOfflineBacklog.java
private void calculateCursorBacklogs(final ManagedLedgerFactoryImpl factory, final DestinationName dn, final NavigableMap<Long, MLDataFormats.ManagedLedgerInfo.LedgerInfo> ledgers, final PersistentOfflineTopicStats offlineTopicStats) throws Exception { if (ledgers.size() == 0) { return;/*from w w w . java 2 s .com*/ } String managedLedgerName = dn.getPersistenceNamingEncoding(); MetaStore store = factory.getMetaStore(); BookKeeper bk = factory.getBookKeeper(); final CountDownLatch allCursorsCounter = new CountDownLatch(1); final long errorInReadingCursor = (long) -1; ConcurrentOpenHashMap<String, Long> ledgerRetryMap = new ConcurrentOpenHashMap<>(); final MLDataFormats.ManagedLedgerInfo.LedgerInfo ledgerInfo = ledgers.lastEntry().getValue(); final PositionImpl lastLedgerPosition = new PositionImpl(ledgerInfo.getLedgerId(), ledgerInfo.getEntries() - 1); if (log.isDebugEnabled()) { log.debug("[{}] Last ledger position {}", managedLedgerName, lastLedgerPosition); } store.getCursors(managedLedgerName, new MetaStore.MetaStoreCallback<List<String>>() { @Override public void operationComplete(List<String> cursors, MetaStore.Stat v) { // Load existing cursors if (log.isDebugEnabled()) { log.debug("[{}] Found {} cursors", managedLedgerName, cursors.size()); } if (cursors.isEmpty()) { allCursorsCounter.countDown(); return; } final CountDownLatch cursorCounter = new CountDownLatch(cursors.size()); for (final String cursorName : cursors) { // determine subscription position from cursor ledger if (log.isDebugEnabled()) { log.debug("[{}] Loading cursor {}", managedLedgerName, cursorName); } AsyncCallback.OpenCallback cursorLedgerOpenCb = (rc, lh, ctx1) -> { long ledgerId = lh.getId(); if (log.isDebugEnabled()) { log.debug("[{}] Opened cursor ledger {} for cursor {}. rc={}", managedLedgerName, ledgerId, cursorName, rc); } if (rc != BKException.Code.OK) { log.warn("[{}] Error opening metadata ledger {} for cursor {}: {}", managedLedgerName, ledgerId, cursorName, BKException.getMessage(rc)); cursorCounter.countDown(); return; } long lac = lh.getLastAddConfirmed(); if (log.isDebugEnabled()) { log.debug("[{}] Cursor {} LAC {} read from ledger {}", managedLedgerName, cursorName, lac, ledgerId); } if (lac == LedgerHandle.INVALID_ENTRY_ID) { // save the ledger id and cursor to retry outside of this call back // since we are trying to read the same cursor ledger, we will block until // this current callback completes, since an attempt to read the entry // will block behind this current operation to complete ledgerRetryMap.put(cursorName, ledgerId); log.info("[{}] Cursor {} LAC {} read from ledger {}", managedLedgerName, cursorName, lac, ledgerId); cursorCounter.countDown(); return; } final long entryId = lac; // read last acked message position for subscription lh.asyncReadEntries(entryId, entryId, new AsyncCallback.ReadCallback() { @Override public void readComplete(int rc, LedgerHandle lh, Enumeration<LedgerEntry> seq, Object ctx) { try { if (log.isDebugEnabled()) { log.debug("readComplete rc={} entryId={}", rc, entryId); } if (rc != BKException.Code.OK) { log.warn("[{}] Error reading from metadata ledger {} for cursor {}: {}", managedLedgerName, ledgerId, cursorName, BKException.getMessage(rc)); // indicate that this cursor should be excluded offlineTopicStats.addCursorDetails(cursorName, errorInReadingCursor, lh.getId()); } else { LedgerEntry entry = seq.nextElement(); MLDataFormats.PositionInfo positionInfo; try { positionInfo = MLDataFormats.PositionInfo.parseFrom(entry.getEntry()); } catch (InvalidProtocolBufferException e) { log.warn( "[{}] Error reading position from metadata ledger {} for cursor {}: {}", managedLedgerName, ledgerId, cursorName, e); offlineTopicStats.addCursorDetails(cursorName, errorInReadingCursor, lh.getId()); return; } final PositionImpl lastAckedMessagePosition = new PositionImpl( positionInfo); if (log.isDebugEnabled()) { log.debug("[{}] Cursor {} MD {} read last ledger position {}", managedLedgerName, cursorName, lastAckedMessagePosition, lastLedgerPosition); } // calculate cursor backlog Range<PositionImpl> range = Range.openClosed(lastAckedMessagePosition, lastLedgerPosition); if (log.isDebugEnabled()) { log.debug("[{}] Calculating backlog for cursor {} using range {}", managedLedgerName, cursorName, range); } long cursorBacklog = getNumberOfEntries(range, ledgers); offlineTopicStats.messageBacklog += cursorBacklog; offlineTopicStats.addCursorDetails(cursorName, cursorBacklog, lh.getId()); } } finally { cursorCounter.countDown(); } } }, null); }; // end of cursor meta read callback store.asyncGetCursorInfo(managedLedgerName, cursorName, new MetaStore.MetaStoreCallback<MLDataFormats.ManagedCursorInfo>() { @Override public void operationComplete(MLDataFormats.ManagedCursorInfo info, MetaStore.Stat version) { long cursorLedgerId = info.getCursorsLedgerId(); if (log.isDebugEnabled()) { log.debug("[{}] Cursor {} meta-data read ledger id {}", managedLedgerName, cursorName, cursorLedgerId); } if (cursorLedgerId != -1) { bk.asyncOpenLedgerNoRecovery(cursorLedgerId, digestType, password, cursorLedgerOpenCb, null); } else { PositionImpl lastAckedMessagePosition = new PositionImpl( info.getMarkDeleteLedgerId(), info.getMarkDeleteEntryId()); Range<PositionImpl> range = Range.openClosed(lastAckedMessagePosition, lastLedgerPosition); if (log.isDebugEnabled()) { log.debug("[{}] Calculating backlog for cursor {} using range {}", managedLedgerName, cursorName, range); } long cursorBacklog = getNumberOfEntries(range, ledgers); offlineTopicStats.messageBacklog += cursorBacklog; offlineTopicStats.addCursorDetails(cursorName, cursorBacklog, cursorLedgerId); cursorCounter.countDown(); } } @Override public void operationFailed(ManagedLedgerException.MetaStoreException e) { log.warn("[{}] Unable to obtain cursor ledger for cursor {}: {}", managedLedgerName, cursorName, e); cursorCounter.countDown(); } }); } // for every cursor find backlog try { if (accurate) { cursorCounter.await(); } else { cursorCounter.await(META_READ_TIMEOUT_SECONDS, TimeUnit.SECONDS); } } catch (Exception e) { log.warn("[{}] Error reading subscription positions{}", managedLedgerName, e); } finally { allCursorsCounter.countDown(); } } @Override public void operationFailed(ManagedLedgerException.MetaStoreException e) { log.warn("[{}] Failed to get the cursors list", managedLedgerName, e); allCursorsCounter.countDown(); } }); if (accurate) { allCursorsCounter.await(); } else { allCursorsCounter.await(META_READ_TIMEOUT_SECONDS, TimeUnit.SECONDS); } // go through ledgers where LAC was -1 if (accurate && ledgerRetryMap.size() > 0) { ledgerRetryMap.forEach((cursorName, ledgerId) -> { if (log.isDebugEnabled()) { log.debug("Cursor {} Ledger {} Trying to obtain MD from BkAdmin", cursorName, ledgerId); } PositionImpl lastAckedMessagePosition = tryGetMDPosition(bk, ledgerId, cursorName); if (lastAckedMessagePosition == null) { log.warn("[{}] Cursor {} read from ledger {}. Unable to determine cursor position", managedLedgerName, cursorName, ledgerId); } else { if (log.isDebugEnabled()) { log.debug("[{}] Cursor {} read from ledger using bk admin {}. position {}", managedLedgerName, cursorName, ledgerId, lastAckedMessagePosition); } // calculate cursor backlog Range<PositionImpl> range = Range.openClosed(lastAckedMessagePosition, lastLedgerPosition); if (log.isDebugEnabled()) { log.debug("[{}] Calculating backlog for cursor {} using range {}", managedLedgerName, cursorName, range); } long cursorBacklog = getNumberOfEntries(range, ledgers); offlineTopicStats.messageBacklog += cursorBacklog; offlineTopicStats.addCursorDetails(cursorName, cursorBacklog, ledgerId); } }); } }
From source file:com.palantir.atlasdb.keyvalue.cassandra.CassandraClientPool.java
private void refreshTokenRanges() { try {//from w ww.j a v a 2 s .c o m List<TokenRange> tokenRanges = getRandomGoodHost().runWithPooledResource(describeRing); ImmutableRangeMap.Builder<LightweightOPPToken, List<InetSocketAddress>> newTokenRing = ImmutableRangeMap .builder(); for (TokenRange tokenRange : tokenRanges) { List<InetSocketAddress> hosts = Lists.transform(tokenRange.getEndpoints(), new Function<String, InetSocketAddress>() { @Override public InetSocketAddress apply(String endpoint) { return new InetSocketAddress(endpoint, CassandraConstants.DEFAULT_THRIFT_PORT); } }); LightweightOPPToken startToken = new LightweightOPPToken( BaseEncoding.base16().decode(tokenRange.getStart_token().toUpperCase())); LightweightOPPToken endToken = new LightweightOPPToken( BaseEncoding.base16().decode(tokenRange.getEnd_token().toUpperCase())); if (startToken.compareTo(endToken) <= 0) { newTokenRing.put(Range.openClosed(startToken, endToken), hosts); } else { // Handle wrap-around newTokenRing.put(Range.greaterThan(startToken), hosts); newTokenRing.put(Range.atMost(endToken), hosts); } } tokenMap = newTokenRing.build(); } catch (Exception e) { log.error("Couldn't grab new token ranges for token aware cassandra mapping!", e); } }
From source file:org.apache.bookkeeper.mledger.impl.ManagedCursorImpl.java
public long getNumberOfEntriesSinceFirstNotAckedMessage() { return ledger.getNumberOfEntries(Range.openClosed(markDeletePosition, readPosition)); }
From source file:org.apache.bookkeeper.mledger.impl.ManagedCursorImpl.java
public long getNumberOfEntriesInStorage() { return ledger.getNumberOfEntries(Range.openClosed(markDeletePosition, ledger.getLastPosition().getNext())); }
From source file:org.apache.aurora.scheduler.thrift.SchedulerThriftInterface.java
@Override public Response addInstances(InstanceKey key, int count) { IJobKey jobKey = JobKeys.assertValid(IJobKey.build(key.getJobKey())); Response response = empty();//from ww w. j a v a2 s. c o m return storage.write(storeProvider -> { try { if (getCronJob(storeProvider, jobKey).isPresent()) { return invalidRequest("Instances may not be added to cron jobs."); } lockManager.assertNotLocked(ILockKey.build(LockKey.job(jobKey.newBuilder()))); FluentIterable<IScheduledTask> currentTasks = FluentIterable .from(storeProvider.getTaskStore().fetchTasks(Query.jobScoped(jobKey).active())); if (count <= 0) { return invalidRequest(INVALID_INSTANCE_COUNT); } Optional<IScheduledTask> templateTask = Iterables.tryFind(currentTasks, e -> e.getAssignedTask().getInstanceId() == key.getInstanceId()); if (!templateTask.isPresent()) { return invalidRequest(INVALID_INSTANCE_ID); } int lastId = currentTasks.transform(e -> e.getAssignedTask().getInstanceId()).toList().stream() .max(Comparator.naturalOrder()).get(); Set<Integer> instanceIds = ContiguousSet.create(Range.openClosed(lastId, lastId + count), DiscreteDomain.integers()); ITaskConfig task = templateTask.get().getAssignedTask().getTask(); validateTaskLimits(task, Iterables.size(currentTasks) + instanceIds.size(), quotaManager.checkInstanceAddition(task, instanceIds.size(), storeProvider)); stateManager.insertPendingTasks(storeProvider, task, instanceIds); addInstancesCounter.addAndGet(instanceIds.size()); return response.setResponseCode(OK); } catch (LockException e) { return error(LOCK_ERROR, e); } catch (TaskValidationException | IllegalArgumentException e) { return error(INVALID_REQUEST, e); } }); }
From source file:org.apache.bookkeeper.mledger.impl.ManagedCursorImpl.java
long getNumIndividualDeletedEntriesToSkip(long numEntries) { long totalEntriesToSkip = 0; long deletedMessages = 0; lock.readLock().lock();//from w ww. j a va2 s. c om try { PositionImpl startPosition = markDeletePosition; PositionImpl endPosition = null; for (Range<PositionImpl> r : individualDeletedMessages.asRanges()) { endPosition = r.lowerEndpoint(); if (startPosition.compareTo(endPosition) <= 0) { Range<PositionImpl> range = Range.openClosed(startPosition, endPosition); long entries = ledger.getNumberOfEntries(range); if (totalEntriesToSkip + entries >= numEntries) { break; } totalEntriesToSkip += entries; deletedMessages += ledger.getNumberOfEntries(r); startPosition = r.upperEndpoint(); } else { if (log.isDebugEnabled()) { log.debug("[{}] deletePosition {} moved ahead without clearing deleteMsgs {} for cursor {}", ledger.getName(), markDeletePosition, r.lowerEndpoint(), name); } } } } finally { lock.readLock().unlock(); } return deletedMessages; }