Example usage for org.joda.time Duration millis

Introduction

In this page you can find the example usage for org.joda.time Duration millis.

Prototype

public static Duration millis(long millis) 

Source Link

Document

Create a duration with the specified number of milliseconds.

Usage

From source file:org.apache.beam.runners.core.GroupAlsoByWindowsProperties.java

License:Apache License

/** Tests that the given GABW implementation correctly groups elements into merged sessions. */
public static void groupsElementsInMergedSessions(
        GroupAlsoByWindowsDoFnFactory<String, String, Iterable<String>> gabwFactory) throws Exception {

    WindowingStrategy<?, IntervalWindow> windowingStrategy = WindowingStrategy
            .of(Sessions.withGapDuration(Duration.millis(10)));

    List<WindowedValue<KV<String, Iterable<String>>>> result = runGABW(gabwFactory, windowingStrategy, "key",
            WindowedValue.of("v1", new Instant(0), Arrays.asList(window(0, 10)), PaneInfo.NO_FIRING),
            WindowedValue.of("v2", new Instant(5), Arrays.asList(window(5, 15)), PaneInfo.NO_FIRING),
            WindowedValue.of("v3", new Instant(15), Arrays.asList(window(15, 25)), PaneInfo.NO_FIRING));

    assertThat(result, hasSize(2));//w ww. j  a  va2s. com

    TimestampedValue<KV<String, Iterable<String>>> item0 = getOnlyElementInWindow(result, window(0, 15));
    assertThat(item0.getValue().getValue(), containsInAnyOrder("v1", "v2"));
    assertThat(item0.getTimestamp(), equalTo(window(0, 15).maxTimestamp()));

    TimestampedValue<KV<String, Iterable<String>>> item1 = getOnlyElementInWindow(result, window(15, 25));
    assertThat(item1.getValue().getValue(), contains("v3"));
    assertThat(item1.getTimestamp(), equalTo(window(15, 25).maxTimestamp()));
}

---

From source file:org.apache.beam.runners.core.GroupAlsoByWindowsProperties.java

License:Apache License

/**
 * Tests that the given {@link GroupAlsoByWindowsDoFn} implementation combines elements per
 * session window correctly according to the provided {@link CombineFn}.
 *//*from w ww  .  j a v  a2s .c  o  m*/
public static void combinesElementsPerSession(GroupAlsoByWindowsDoFnFactory<String, Long, Long> gabwFactory,
        CombineFn<Long, ?, Long> combineFn) throws Exception {

    WindowingStrategy<?, IntervalWindow> windowingStrategy = WindowingStrategy
            .of(Sessions.withGapDuration(Duration.millis(10)));

    List<WindowedValue<KV<String, Long>>> result = runGABW(gabwFactory, windowingStrategy, "k",
            WindowedValue.of(1L, new Instant(0), Arrays.asList(window(0, 10)), PaneInfo.NO_FIRING),
            WindowedValue.of(2L, new Instant(5), Arrays.asList(window(5, 15)), PaneInfo.NO_FIRING),
            WindowedValue.of(4L, new Instant(15), Arrays.asList(window(15, 25)), PaneInfo.NO_FIRING));

    assertThat(result, hasSize(2));

    TimestampedValue<KV<String, Long>> item0 = getOnlyElementInWindow(result, window(0, 15));
    assertThat(item0.getValue().getKey(), equalTo("k"));
    assertThat(item0.getValue().getValue(), equalTo(combineFn.apply(ImmutableList.of(1L, 2L))));
    assertThat(item0.getTimestamp(), equalTo(window(0, 15).maxTimestamp()));

    TimestampedValue<KV<String, Long>> item1 = getOnlyElementInWindow(result, window(15, 25));
    assertThat(item1.getValue().getKey(), equalTo("k"));
    assertThat(item1.getValue().getValue(), equalTo(combineFn.apply(ImmutableList.of(4L))));
    assertThat(item1.getTimestamp(), equalTo(window(15, 25).maxTimestamp()));
}

---

From source file:org.apache.beam.runners.core.GroupAlsoByWindowsProperties.java

License:Apache License

/**
 * Tests that for a simple sequence of elements on the same key, the given GABW implementation
 * correctly groups them according to fixed windows and also sets the output timestamp according
 * to the policy {@link OutputTimeFns#outputAtEndOfWindow()}.
 *//*from   w  ww.java  2 s .co m*/
public static void groupsElementsIntoFixedWindowsWithEndOfWindowTimestamp(
        GroupAlsoByWindowsDoFnFactory<String, String, Iterable<String>> gabwFactory) throws Exception {

    WindowingStrategy<?, IntervalWindow> windowingStrategy = WindowingStrategy
            .of(FixedWindows.of(Duration.millis(10))).withOutputTimeFn(OutputTimeFns.outputAtEndOfWindow());

    List<WindowedValue<KV<String, Iterable<String>>>> result = runGABW(gabwFactory, windowingStrategy, "key",
            WindowedValue.of("v1", new Instant(1), Arrays.asList(window(0, 10)), PaneInfo.NO_FIRING),
            WindowedValue.of("v2", new Instant(2), Arrays.asList(window(0, 10)), PaneInfo.NO_FIRING),
            WindowedValue.of("v3", new Instant(13), Arrays.asList(window(10, 20)), PaneInfo.NO_FIRING));

    assertThat(result, hasSize(2));

    TimestampedValue<KV<String, Iterable<String>>> item0 = getOnlyElementInWindow(result, window(0, 10));
    assertThat(item0.getValue().getValue(), containsInAnyOrder("v1", "v2"));
    assertThat(item0.getTimestamp(), equalTo(window(0, 10).maxTimestamp()));

    TimestampedValue<KV<String, Iterable<String>>> item1 = getOnlyElementInWindow(result, window(10, 20));
    assertThat(item1.getValue().getValue(), contains("v3"));
    assertThat(item1.getTimestamp(), equalTo(window(10, 20).maxTimestamp()));
}

---

From source file:org.apache.beam.runners.core.GroupAlsoByWindowsProperties.java

License:Apache License

/**
 * Tests that for a simple sequence of elements on the same key, the given GABW implementation
 * correctly groups them according to fixed windows and also sets the output timestamp according
 * to the policy {@link OutputTimeFns#outputAtLatestInputTimestamp()}.
 *///from  ww w .  j av a 2s  .com
public static void groupsElementsIntoFixedWindowsWithLatestTimestamp(
        GroupAlsoByWindowsDoFnFactory<String, String, Iterable<String>> gabwFactory) throws Exception {

    WindowingStrategy<?, IntervalWindow> windowingStrategy = WindowingStrategy
            .of(FixedWindows.of(Duration.millis(10)))
            .withOutputTimeFn(OutputTimeFns.outputAtLatestInputTimestamp());

    List<WindowedValue<KV<String, Iterable<String>>>> result = runGABW(gabwFactory, windowingStrategy, "k",
            WindowedValue.of("v1", new Instant(1), Arrays.asList(window(0, 10)), PaneInfo.NO_FIRING),
            WindowedValue.of("v2", new Instant(2), Arrays.asList(window(0, 10)), PaneInfo.NO_FIRING),
            WindowedValue.of("v3", new Instant(13), Arrays.asList(window(10, 20)), PaneInfo.NO_FIRING));

    assertThat(result, hasSize(2));

    TimestampedValue<KV<String, Iterable<String>>> item0 = getOnlyElementInWindow(result, window(0, 10));
    assertThat(item0.getValue().getValue(), containsInAnyOrder("v1", "v2"));
    assertThat(item0.getTimestamp(), equalTo(new Instant(2)));

    TimestampedValue<KV<String, Iterable<String>>> item1 = getOnlyElementInWindow(result, window(10, 20));
    assertThat(item1.getValue().getValue(), contains("v3"));
    assertThat(item1.getTimestamp(), equalTo(new Instant(13)));
}

---

From source file:org.apache.beam.runners.core.GroupAlsoByWindowsProperties.java

License:Apache License

/**
 * Tests that the given GABW implementation correctly groups elements into merged sessions with
 * output timestamps at the end of the merged window.
 *//* ww  w .  j  a va  2s .  c  o m*/
public static void groupsElementsInMergedSessionsWithEndOfWindowTimestamp(
        GroupAlsoByWindowsDoFnFactory<String, String, Iterable<String>> gabwFactory) throws Exception {

    WindowingStrategy<?, IntervalWindow> windowingStrategy = WindowingStrategy
            .of(Sessions.withGapDuration(Duration.millis(10)))
            .withOutputTimeFn(OutputTimeFns.outputAtEndOfWindow());

    List<WindowedValue<KV<String, Iterable<String>>>> result = runGABW(gabwFactory, windowingStrategy, "k",
            WindowedValue.of("v1", new Instant(0), Arrays.asList(window(0, 10)), PaneInfo.NO_FIRING),
            WindowedValue.of("v2", new Instant(5), Arrays.asList(window(5, 15)), PaneInfo.NO_FIRING),
            WindowedValue.of("v3", new Instant(15), Arrays.asList(window(15, 25)), PaneInfo.NO_FIRING));

    assertThat(result, hasSize(2));

    TimestampedValue<KV<String, Iterable<String>>> item0 = getOnlyElementInWindow(result, window(0, 15));
    assertThat(item0.getValue().getValue(), containsInAnyOrder("v1", "v2"));
    assertThat(item0.getTimestamp(), equalTo(window(0, 15).maxTimestamp()));

    TimestampedValue<KV<String, Iterable<String>>> item1 = getOnlyElementInWindow(result, window(15, 25));
    assertThat(item1.getValue().getValue(), contains("v3"));
    assertThat(item1.getTimestamp(), equalTo(window(15, 25).maxTimestamp()));
}

---

From source file:org.apache.beam.runners.core.GroupAlsoByWindowsProperties.java

License:Apache License

/**
 * Tests that the given GABW implementation correctly groups elements into merged sessions with
 * output timestamps at the end of the merged window.
 *//* w ww.j av a2  s .  c  om*/
public static void groupsElementsInMergedSessionsWithLatestTimestamp(
        GroupAlsoByWindowsDoFnFactory<String, String, Iterable<String>> gabwFactory) throws Exception {

    WindowingStrategy<?, IntervalWindow> windowingStrategy = WindowingStrategy
            .of(Sessions.withGapDuration(Duration.millis(10)))
            .withOutputTimeFn(OutputTimeFns.outputAtLatestInputTimestamp());

    BoundedWindow unmergedWindow = window(15, 25);
    List<WindowedValue<KV<String, Iterable<String>>>> result = runGABW(gabwFactory, windowingStrategy, "k",
            WindowedValue.of("v1", new Instant(0), Arrays.asList(window(0, 10)), PaneInfo.NO_FIRING),
            WindowedValue.of("v2", new Instant(5), Arrays.asList(window(5, 15)), PaneInfo.NO_FIRING),
            WindowedValue.of("v3", new Instant(15), Arrays.asList(unmergedWindow), PaneInfo.NO_FIRING));

    assertThat(result, hasSize(2));

    BoundedWindow mergedWindow = window(0, 15);
    TimestampedValue<KV<String, Iterable<String>>> item0 = getOnlyElementInWindow(result, mergedWindow);
    assertThat(item0.getValue().getValue(), containsInAnyOrder("v1", "v2"));
    assertThat(item0.getTimestamp(), equalTo(new Instant(5)));

    TimestampedValue<KV<String, Iterable<String>>> item1 = getOnlyElementInWindow(result, unmergedWindow);
    assertThat(item1.getValue().getValue(), contains("v3"));
    assertThat(item1.getTimestamp(), equalTo(new Instant(15)));
}

---

From source file:org.apache.beam.runners.core.GroupAlsoByWindowsProperties.java

License:Apache License

/**
 * Tests that the given {@link GroupAlsoByWindowsDoFn} implementation combines elements per
 * session window correctly according to the provided {@link CombineFn}.
 *//*from w w  w  .  j  av  a2s.c  o m*/
public static void combinesElementsPerSessionWithEndOfWindowTimestamp(
        GroupAlsoByWindowsDoFnFactory<String, Long, Long> gabwFactory, CombineFn<Long, ?, Long> combineFn)
        throws Exception {

    WindowingStrategy<?, IntervalWindow> windowingStrategy = WindowingStrategy
            .of(Sessions.withGapDuration(Duration.millis(10)))
            .withOutputTimeFn(OutputTimeFns.outputAtEndOfWindow());

    BoundedWindow secondWindow = window(15, 25);
    List<WindowedValue<KV<String, Long>>> result = runGABW(gabwFactory, windowingStrategy, "k",
            WindowedValue.of(1L, new Instant(0), Arrays.asList(window(0, 10)), PaneInfo.NO_FIRING),
            WindowedValue.of(2L, new Instant(5), Arrays.asList(window(5, 15)), PaneInfo.NO_FIRING),
            WindowedValue.of(4L, new Instant(15), Arrays.asList(secondWindow), PaneInfo.NO_FIRING));

    assertThat(result, hasSize(2));

    BoundedWindow firstResultWindow = window(0, 15);
    TimestampedValue<KV<String, Long>> item0 = getOnlyElementInWindow(result, firstResultWindow);
    assertThat(item0.getValue().getValue(), equalTo(combineFn.apply(ImmutableList.of(1L, 2L))));
    assertThat(item0.getTimestamp(), equalTo(firstResultWindow.maxTimestamp()));

    TimestampedValue<KV<String, Long>> item1 = getOnlyElementInWindow(result, secondWindow);
    assertThat(item1.getValue().getValue(), equalTo(combineFn.apply(ImmutableList.of(4L))));
    assertThat(item1.getTimestamp(), equalTo(secondWindow.maxTimestamp()));
}

---

From source file:org.apache.beam.runners.core.metrics.SimpleExecutionState.java

License:Apache License

@Override
public void reportLull(Thread trackedThread, long millis) {
    LOG.warn(getLullMessage(trackedThread, Duration.millis(millis)));
}

---

From source file:org.apache.beam.runners.core.triggers.TriggerStateMachines.java

License:Apache License

private static TriggerStateMachine stateMachineForAfterProcessingTime(
        RunnerApi.Trigger.AfterProcessingTime trigger) {
    AfterDelayFromFirstElementStateMachine stateMachine = AfterProcessingTimeStateMachine
            .pastFirstElementInPane();//from w ww .  ja va 2 s .  co m
    for (RunnerApi.TimestampTransform transform : trigger.getTimestampTransformsList()) {
        switch (transform.getTimestampTransformCase()) {
        case ALIGN_TO:
            stateMachine = stateMachine.alignedTo(Duration.millis(transform.getAlignTo().getPeriod()),
                    new Instant(transform.getAlignTo().getOffset()));
            break;
        case DELAY:
            stateMachine = stateMachine.plusDelayOf(Duration.millis(transform.getDelay().getDelayMillis()));
            break;
        case TIMESTAMPTRANSFORM_NOT_SET:
            throw new IllegalArgumentException(
                    String.format("Required field 'timestamp_transform' not set in %s", transform));
        default:
            throw new IllegalArgumentException(String.format("Unknown timestamp transform case: %s",
                    transform.getTimestampTransformCase()));
        }
    }
    return stateMachine;
}

---

From source file:org.apache.beam.runners.core.WatermarkHold.java

License:Apache License

/**
 * Attempt to add a 'garbage collection hold' if it is required. Return the {@link Instant} at
 * which the hold was added (ie the end of window time plus allowed lateness), or {@literal null}
 * if no hold was added.//from  w w  w. j  av  a2 s  .  c o m
 *
 * <p>A garbage collection hold is added in two situations:
 *
 * <ol>
 *   <li>An incoming element has a timestamp earlier than the output watermark, and was too late
 *       for placing the usual element hold or an end of window hold. Place the garbage collection
 *       hold so that we can guarantee when the pane is finally triggered its output will not be
 *       dropped due to excessive lateness by any downstream computation.
 *   <li>The {@link WindowingStrategy#getClosingBehavior()} is {@link
 *       ClosingBehavior#FIRE_ALWAYS}, and thus we guarantee a final pane will be emitted for all
 *       windows which saw at least one element. Again, the garbage collection hold guarantees
 *       that any empty final pane can be given a timestamp which will not be considered beyond
 *       allowed lateness by any downstream computation.
 * </ol>
 *
 * <p>We use {@code paneIsEmpty} to distinguish cases 1 and 2.
 */
@Nullable
private Instant addGarbageCollectionHold(ReduceFn<?, ?, ?, W>.Context context, boolean paneIsEmpty) {
    Instant outputWM = timerInternals.currentOutputWatermarkTime();
    Instant inputWM = timerInternals.currentInputWatermarkTime();
    Instant gcHold = LateDataUtils.garbageCollectionTime(context.window(), windowingStrategy);

    if (gcHold.isBefore(inputWM)) {
        WindowTracing.trace(
                "{}.addGarbageCollectionHold: gc hold would be before the input watermark "
                        + "for key:{}; window: {}; inputWatermark: {}; outputWatermark: {}",
                getClass().getSimpleName(), context.key(), context.window(), inputWM, outputWM);
        return null;
    }

    if (paneIsEmpty && context.windowingStrategy().getClosingBehavior() == ClosingBehavior.FIRE_IF_NON_EMPTY) {
        WindowTracing
                .trace("WatermarkHold.addGarbageCollectionHold: garbage collection hold at {} is unnecessary "
                        + "since empty pane and FIRE_IF_NON_EMPTY for key:{}; window:{}; inputWatermark:{}; "
                        + "outputWatermark:{}", gcHold, context.key(), context.window(), inputWM, outputWM);
        return null;
    }

    if (!gcHold.isBefore(BoundedWindow.TIMESTAMP_MAX_VALUE)) {
        // If the garbage collection hold is past the timestamp we can represent, instead truncate
        // to the maximum timestamp that is not positive infinity. This ensures all windows will
        // eventually be garbage collected.
        gcHold = BoundedWindow.TIMESTAMP_MAX_VALUE.minus(Duration.millis(1L));
    }
    checkState(!gcHold.isBefore(inputWM), "Garbage collection hold %s cannot be before input watermark %s",
            gcHold, inputWM);
    checkState(!gcHold.isAfter(BoundedWindow.TIMESTAMP_MAX_VALUE),
            "Garbage collection hold %s is beyond end-of-time", gcHold);
    // Same EXTRA_HOLD_TAG vs elementHoldTag discussion as in addEndOfWindowHold above.
    context.state().access(EXTRA_HOLD_TAG).add(gcHold);

    WindowTracing.trace(
            "WatermarkHold.addGarbageCollectionHold: garbage collection hold at {} is on time for "
                    + "key:{}; window:{}; inputWatermark:{}; outputWatermark:{}",
            gcHold, context.key(), context.window(), inputWM, outputWM);
    return gcHold;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• »