List of usage examples for org.joda.time Instant isBefore
public boolean isBefore(long instant)
From source file:com.dataartisans.flink.dataflow.translation.wrappers.streaming.FlinkAbstractParDoWrapper.java
License:Apache License
protected void checkTimestamp(WindowedValue<IN> ref, Instant timestamp) { if (timestamp.isBefore(ref.getTimestamp().minus(doFn.getAllowedTimestampSkew()))) { throw new IllegalArgumentException(String.format( "Cannot output with timestamp %s. Output timestamps must be no earlier than the " + "timestamp of the current input (%s) minus the allowed skew (%s). See the " + "DoFn#getAllowedTimestmapSkew() Javadoc for details on changing the allowed skew.", timestamp, ref.getTimestamp(), PeriodFormat.getDefault().print(doFn.getAllowedTimestampSkew().toPeriod()))); }//w ww .j a va2s . c o m }
From source file:com.google.cloud.dataflow.sdk.runners.inprocess.MockClock.java
License:Apache License
public void set(Instant newNow) { checkArgument(!newNow.isBefore(now), "Cannot move MockClock backwards in time from %s to %s", now, newNow); this.now = newNow; }
From source file:com.google.cloud.dataflow.sdk.runners.worker.WindmillStateReader.java
License:Apache License
private void consumeWatermark(Windmill.WatermarkHold watermarkHold, StateTag stateTag) { CoderAndFuture<Void, Instant> coderAndFuture = getWaiting(stateTag, false); SettableFuture<Instant> future = coderAndFuture.getNonDoneFuture(stateTag); // No coders for watermarks coderAndFuture.checkNoCoder();// w w w.j a v a2s. c o m Instant hold = null; for (long timestamp : watermarkHold.getTimestampsList()) { Instant instant = new Instant(TimeUnit.MICROSECONDS.toMillis(timestamp)); if (hold == null || instant.isBefore(hold)) { hold = instant; } } future.set(hold); }
From source file:com.google.cloud.dataflow.sdk.testing.WindowFnTestUtils.java
License:Apache License
/** * Assigns the given {@code timestamp} to windows using the specified {@code windowFn}, and * verifies that result of {@code windowFn.getOutputTimestamp} for each window is within the * proper bound.// ww w .j ava 2 s. co m */ public static <T, W extends BoundedWindow> void validateNonInterferingOutputTimes(WindowFn<T, W> windowFn, long timestamp) throws Exception { Collection<W> windows = WindowFnTestUtils.<T, W>assignedWindows(windowFn, timestamp); Instant instant = new Instant(timestamp); for (W window : windows) { Instant outputTimestamp = windowFn.getOutputTimeFn().assignOutputTime(instant, window); assertFalse("getOutputTime must be greater than or equal to input timestamp", outputTimestamp.isBefore(instant)); assertFalse("getOutputTime must be less than or equal to the max timestamp", outputTimestamp.isAfter(window.maxTimestamp())); } }
From source file:com.google.cloud.dataflow.sdk.testing.WindowFnTestUtils.java
License:Apache License
/** * Assigns the given {@code timestamp} to windows using the specified {@code windowFn}, and * verifies that result of {@link WindowFn#getOutputTime windowFn.getOutputTime} for later windows * (as defined by {@code maxTimestamp} won't prevent the watermark from passing the end of earlier * windows./*from w w w .j a v a2 s .co m*/ * * <p>This verifies that overlapping windows don't interfere at all. Depending on the * {@code windowFn} this may be stricter than desired. */ public static <T, W extends BoundedWindow> void validateGetOutputTimestamp(WindowFn<T, W> windowFn, long timestamp) throws Exception { Collection<W> windows = WindowFnTestUtils.<T, W>assignedWindows(windowFn, timestamp); List<W> sortedWindows = new ArrayList<>(windows); Collections.sort(sortedWindows, new Comparator<BoundedWindow>() { @Override public int compare(BoundedWindow o1, BoundedWindow o2) { return o1.maxTimestamp().compareTo(o2.maxTimestamp()); } }); Instant instant = new Instant(timestamp); Instant endOfPrevious = null; for (W window : sortedWindows) { Instant outputTimestamp = windowFn.getOutputTimeFn().assignOutputTime(instant, window); if (endOfPrevious == null) { // If this is the first window, the output timestamp can be anything, as long as it is in // the valid range. assertFalse("getOutputTime must be greater than or equal to input timestamp", outputTimestamp.isBefore(instant)); assertFalse("getOutputTime must be less than or equal to the max timestamp", outputTimestamp.isAfter(window.maxTimestamp())); } else { // If this is a later window, the output timestamp must be after the end of the previous // window assertTrue("getOutputTime must be greater than the end of the previous window", outputTimestamp.isAfter(endOfPrevious)); assertFalse("getOutputTime must be less than or equal to the max timestamp", outputTimestamp.isAfter(window.maxTimestamp())); } endOfPrevious = window.maxTimestamp(); } }
From source file:com.google.cloud.dataflow.sdk.transforms.windowing.AfterAll.java
License:Apache License
@Override public Instant getWatermarkThatGuaranteesFiring(W window) { // This trigger will fire after the latest of its sub-triggers. Instant deadline = BoundedWindow.TIMESTAMP_MIN_VALUE; for (Trigger<W> subTrigger : subTriggers) { Instant subDeadline = subTrigger.getWatermarkThatGuaranteesFiring(window); if (deadline.isBefore(subDeadline)) { deadline = subDeadline;/* w ww. ja v a 2s.c om*/ } } return deadline; }
From source file:com.google.cloud.dataflow.sdk.transforms.windowing.OrFinallyTrigger.java
License:Apache License
@Override public Instant getWatermarkThatGuaranteesFiring(W window) { // This trigger fires once either the trigger or the until trigger fires. Instant actualDeadline = subTriggers.get(ACTUAL).getWatermarkThatGuaranteesFiring(window); Instant untilDeadline = subTriggers.get(UNTIL).getWatermarkThatGuaranteesFiring(window); return actualDeadline.isBefore(untilDeadline) ? actualDeadline : untilDeadline; }
From source file:com.google.cloud.dataflow.sdk.transforms.windowing.SlidingWindows.java
License:Apache License
/** * Ensures that later sliding windows have an output time that is past the end of earlier windows. * * <p>If this is the earliest sliding window containing {@code inputTimestamp}, that's fine. * Otherwise, we pick the earliest time that doesn't overlap with earlier windows. */// www . j a v a2 s . c o m @Experimental(Kind.OUTPUT_TIME) @Override public OutputTimeFn<? super IntervalWindow> getOutputTimeFn() { return new OutputTimeFn.Defaults<BoundedWindow>() { @Override public Instant assignOutputTime(Instant inputTimestamp, BoundedWindow window) { Instant startOfLastSegment = window.maxTimestamp().minus(period); return startOfLastSegment.isBefore(inputTimestamp) ? inputTimestamp : startOfLastSegment.plus(1); } @Override public boolean dependsOnlyOnEarliestInputTimestamp() { return true; } }; }
From source file:com.google.cloud.dataflow.sdk.util.BatchTimerInternals.java
License:Apache License
public void advanceInputWatermark(ReduceFnRunner<?, ?, ?, ?> runner, Instant newInputWatermark) throws Exception { Preconditions.checkState(!newInputWatermark.isBefore(inputWatermarkTime), "Cannot move input watermark time backwards from %s to %s", inputWatermarkTime, newInputWatermark); inputWatermarkTime = newInputWatermark; advance(runner, newInputWatermark, TimeDomain.EVENT_TIME); }
From source file:com.google.cloud.dataflow.sdk.util.BatchTimerInternals.java
License:Apache License
public void advanceProcessingTime(ReduceFnRunner<?, ?, ?, ?> runner, Instant newProcessingTime) throws Exception { Preconditions.checkState(!newProcessingTime.isBefore(processingTime), "Cannot move processing time backwards from %s to %s", processingTime, newProcessingTime); processingTime = newProcessingTime;/*from www . jav a 2 s . co m*/ advance(runner, newProcessingTime, TimeDomain.PROCESSING_TIME); }