Example usage for org.joda.time Duration millis

Introduction

In this page you can find the example usage for org.joda.time Duration millis.

Prototype

public static Duration millis(long millis) 

Source Link

Document

Create a duration with the specified number of milliseconds.

Usage

From source file:org.apache.beam.sdk.transforms.Reshuffle.java

License:Apache License

@Override
public PCollection<KV<K, V>> expand(PCollection<KV<K, V>> input) {
    WindowingStrategy<?, ?> originalStrategy = input.getWindowingStrategy();
    // If the input has already had its windows merged, then the GBK that performed the merge
    // will have set originalStrategy.getWindowFn() to InvalidWindows, causing the GBK contained
    // here to fail. Instead, we install a valid WindowFn that leaves all windows unchanged.
    // The TimestampCombiner is set to ensure the GroupByKey does not shift elements forwards in
    // time.//  ww  w .jav  a  2 s  . c o m
    // Because this outputs as fast as possible, this should not hold the watermark.
    Window<KV<K, V>> rewindow = Window
            .<KV<K, V>>into(new IdentityWindowFn<>(originalStrategy.getWindowFn().windowCoder()))
            .triggering(new ReshuffleTrigger<>()).discardingFiredPanes()
            .withTimestampCombiner(TimestampCombiner.EARLIEST)
            .withAllowedLateness(Duration.millis(BoundedWindow.TIMESTAMP_MAX_VALUE.getMillis()));

    return input.apply(rewindow).apply("ReifyOriginalTimestamps", Reify.timestampsInValue())
            .apply(GroupByKey.create())
            // Set the windowing strategy directly, so that it doesn't get counted as the user having
            // set allowed lateness.
            .setWindowingStrategyInternal(originalStrategy).apply("ExpandIterable",
                    ParDo.of(new DoFn<KV<K, Iterable<TimestampedValue<V>>>, KV<K, TimestampedValue<V>>>() {
                        @ProcessElement
                        public void processElement(@Element KV<K, Iterable<TimestampedValue<V>>> element,
                                OutputReceiver<KV<K, TimestampedValue<V>>> r) {
                            K key = element.getKey();
                            for (TimestampedValue<V> value : element.getValue()) {
                                r.output(KV.of(key, value));
                            }
                        }
                    }))
            .apply("RestoreOriginalTimestamps", ReifyTimestamps.extractFromValues());
}

---

From source file:org.apache.beam.sdk.transforms.windowing.Triggers.java

License:Apache License

public static Trigger fromProto(RunnerApi.Trigger triggerProto) {
    switch (triggerProto.getTriggerCase()) {
    case AFTER_ALL:
        return AfterAll.of(protosToTriggers(triggerProto.getAfterAll().getSubtriggersList()));
    case AFTER_ANY:
        return AfterFirst.of(protosToTriggers(triggerProto.getAfterAny().getSubtriggersList()));
    case AFTER_EACH:
        return AfterEach.inOrder(protosToTriggers(triggerProto.getAfterEach().getSubtriggersList()));
    case AFTER_END_OF_WINDOW:
        RunnerApi.Trigger.AfterEndOfWindow eowProto = triggerProto.getAfterEndOfWindow();

        if (!eowProto.hasEarlyFirings() && !eowProto.hasLateFirings()) {
            return AfterWatermark.pastEndOfWindow();
        }/*from  www  .  ja v a  2  s  . co m*/

        // It either has early or late firings or both; our typing in Java makes this a smidge
        // annoying
        if (triggerProto.getAfterEndOfWindow().hasEarlyFirings()) {
            AfterWatermarkEarlyAndLate trigger = AfterWatermark.pastEndOfWindow().withEarlyFirings(
                    (OnceTrigger) fromProto(triggerProto.getAfterEndOfWindow().getEarlyFirings()));

            if (triggerProto.getAfterEndOfWindow().hasLateFirings()) {
                trigger = trigger.withLateFirings(
                        (OnceTrigger) fromProto(triggerProto.getAfterEndOfWindow().getLateFirings()));
            }
            return trigger;
        } else {
            // only late firings, so return directly
            return AfterWatermark.pastEndOfWindow()
                    .withLateFirings((OnceTrigger) fromProto(eowProto.getLateFirings()));
        }
    case AFTER_PROCESSING_TIME:
        AfterProcessingTime trigger = AfterProcessingTime.pastFirstElementInPane();
        for (RunnerApi.TimestampTransform transform : triggerProto.getAfterProcessingTime()
                .getTimestampTransformsList()) {
            switch (transform.getTimestampTransformCase()) {
            case ALIGN_TO:
                trigger = trigger.alignedTo(Duration.millis(transform.getAlignTo().getPeriod()),
                        new Instant(transform.getAlignTo().getOffset()));
                break;
            case DELAY:
                trigger = trigger.plusDelayOf(Duration.millis(transform.getDelay().getDelayMillis()));
                break;
            case TIMESTAMPTRANSFORM_NOT_SET:
                throw new IllegalArgumentException(
                        String.format("Required field 'timestamp_transform' not set in %s", transform));
            default:
                throw new IllegalArgumentException(String.format("Unknown timestamp transform case: %s",
                        transform.getTimestampTransformCase()));
            }
        }
        return trigger;
    case AFTER_SYNCHRONIZED_PROCESSING_TIME:
        return AfterSynchronizedProcessingTime.ofFirstElement();
    case ELEMENT_COUNT:
        return AfterPane.elementCountAtLeast(triggerProto.getElementCount().getElementCount());
    case NEVER:
        return Never.ever();
    case OR_FINALLY:
        return fromProto(triggerProto.getOrFinally().getMain())
                .orFinally((OnceTrigger) fromProto(triggerProto.getOrFinally().getFinally()));
    case REPEAT:
        return Repeatedly.forever(fromProto(triggerProto.getRepeat().getSubtrigger()));
    case DEFAULT:
        return DefaultTrigger.of();
    case TRIGGER_NOT_SET:
        throw new IllegalArgumentException(
                String.format("Required field 'trigger' not set in %s", triggerProto));
    default:
        throw new IllegalArgumentException(
                String.format("Unknown trigger case: %s", triggerProto.getTriggerCase()));
    }
}

---

From source file:org.apache.beam.sdk.transforms.windowing.Window.java

License:Apache License

@Override
public void populateDisplayData(DisplayData.Builder builder) {
    super.populateDisplayData(builder);

    if (getWindowFn() != null) {
        builder.add(DisplayData.item("windowFn", getWindowFn().getClass()).withLabel("Windowing Function"))
                .include("windowFn", getWindowFn());
    }//from  w  w  w  .  ja  v  a2 s. c o m

    if (getAllowedLateness() != null) {
        builder.addIfNotDefault(
                DisplayData.item("allowedLateness", getAllowedLateness()).withLabel("Allowed Lateness"),
                Duration.millis(BoundedWindow.TIMESTAMP_MAX_VALUE.getMillis()));
    }

    if (getTrigger() != null && !(getTrigger() instanceof DefaultTrigger)) {
        builder.add(DisplayData.item("trigger", getTrigger().toString()).withLabel("Trigger"));
    }

    if (getAccumulationMode() != null) {
        builder.add(DisplayData.item("accumulationMode", getAccumulationMode().toString())
                .withLabel("Accumulation Mode"));
    }

    if (getClosingBehavior() != null) {
        builder.add(DisplayData.item("closingBehavior", getClosingBehavior().toString())
                .withLabel("Window Closing Behavior"));
    }

    if (getTimestampCombiner() != null) {
        builder.add(DisplayData.item("timestampCombiner", getTimestampCombiner().toString())
                .withLabel("Timestamp Combiner"));
    }
}

---

From source file:org.apache.beam.sdk.util.GroupAlsoByWindowsProperties.java

License:Apache License

/**
 * Tests that for empty input and the given {@link WindowingStrategy}, the provided GABW
 * implementation produces no output./*  ww  w . jav a  2 s .c  om*/
 *
 * <p>The input type is deliberately left as a wildcard, since it is not relevant.
 */
public static <K, InputT, OutputT> void emptyInputEmptyOutput(
        GroupAlsoByWindowsDoFnFactory<K, InputT, OutputT> gabwFactory) throws Exception {

    WindowingStrategy<?, IntervalWindow> windowingStrategy = WindowingStrategy
            .of(FixedWindows.of(Duration.millis(10)));

    DoFnTester<KV<K, Iterable<WindowedValue<InputT>>>, KV<K, OutputT>> result = runGABW(gabwFactory,
            windowingStrategy, (K) null, // key should never be used
            Collections.<WindowedValue<InputT>>emptyList());

    assertThat(result.peekOutputElements(), hasSize(0));
}

---

From source file:org.apache.beam.sdk.util.GroupAlsoByWindowsProperties.java

License:Apache License

/**
 * Tests that for a simple sequence of elements on the same key, the given GABW implementation
 * correctly groups them according to fixed windows.
 *//*from   w w w .java  2 s . c  o  m*/
public static void groupsElementsIntoFixedWindows(
        GroupAlsoByWindowsDoFnFactory<String, String, Iterable<String>> gabwFactory) throws Exception {

    WindowingStrategy<?, IntervalWindow> windowingStrategy = WindowingStrategy
            .of(FixedWindows.of(Duration.millis(10)));

    DoFnTester<KV<String, Iterable<WindowedValue<String>>>, KV<String, Iterable<String>>> result = runGABW(
            gabwFactory, windowingStrategy, "key",
            WindowedValue.of("v1", new Instant(1), Arrays.asList(window(0, 10)), PaneInfo.NO_FIRING),
            WindowedValue.of("v2", new Instant(2), Arrays.asList(window(0, 10)), PaneInfo.NO_FIRING),
            WindowedValue.of("v3", new Instant(13), Arrays.asList(window(10, 20)), PaneInfo.NO_FIRING));

    assertThat(result.peekOutputElements(), hasSize(2));

    TimestampedValue<KV<String, Iterable<String>>> item0 = Iterables
            .getOnlyElement(result.peekOutputElementsInWindow(window(0, 10)));
    assertThat(item0.getValue().getValue(), containsInAnyOrder("v1", "v2"));
    assertThat(item0.getTimestamp(), equalTo(window(0, 10).maxTimestamp()));

    TimestampedValue<KV<String, Iterable<String>>> item1 = Iterables
            .getOnlyElement(result.peekOutputElementsInWindow(window(10, 20)));
    assertThat(item1.getValue().getValue(), contains("v3"));
    assertThat(item1.getTimestamp(), equalTo(window(10, 20).maxTimestamp()));
}

---

From source file:org.apache.beam.sdk.util.GroupAlsoByWindowsProperties.java

License:Apache License

/**
 * Tests that for a simple sequence of elements on the same key, the given GABW implementation
 * correctly groups them into sliding windows.
 *
 * <p>In the input here, each element occurs in multiple windows.
 *///from www. ja va 2 s. c o  m
public static void groupsElementsIntoSlidingWindowsWithMinTimestamp(
        GroupAlsoByWindowsDoFnFactory<String, String, Iterable<String>> gabwFactory) throws Exception {

    WindowingStrategy<?, IntervalWindow> windowingStrategy = WindowingStrategy
            .of(SlidingWindows.of(Duration.millis(20)).every(Duration.millis(10)))
            .withOutputTimeFn(OutputTimeFns.outputAtEarliestInputTimestamp());

    DoFnTester<KV<String, Iterable<WindowedValue<String>>>, KV<String, Iterable<String>>> result = runGABW(
            gabwFactory, windowingStrategy, "key",
            WindowedValue.of("v1", new Instant(5), Arrays.asList(window(-10, 10), window(0, 20)),
                    PaneInfo.NO_FIRING),
            WindowedValue.of("v2", new Instant(15), Arrays.asList(window(0, 20), window(10, 30)),
                    PaneInfo.NO_FIRING));

    assertThat(result.peekOutputElements(), hasSize(3));

    TimestampedValue<KV<String, Iterable<String>>> item0 = Iterables
            .getOnlyElement(result.peekOutputElementsInWindow(window(-10, 10)));
    assertThat(item0.getValue().getValue(), contains("v1"));
    assertThat(item0.getTimestamp(), equalTo(new Instant(5)));

    TimestampedValue<KV<String, Iterable<String>>> item1 = Iterables
            .getOnlyElement(result.peekOutputElementsInWindow(window(0, 20)));
    assertThat(item1.getValue().getValue(), containsInAnyOrder("v1", "v2"));
    assertThat(item1.getTimestamp(), equalTo(new Instant(10)));

    TimestampedValue<KV<String, Iterable<String>>> item2 = Iterables
            .getOnlyElement(result.peekOutputElementsInWindow(window(10, 30)));
    assertThat(item2.getValue().getValue(), contains("v2"));
    assertThat(item2.getTimestamp(), equalTo(new Instant(20)));
}

---

From source file:org.apache.beam.sdk.util.GroupAlsoByWindowsProperties.java

License:Apache License

/**
 * Tests that for a simple sequence of elements on the same key, the given GABW implementation
 * correctly groups and combines them according to sliding windows.
 *
 * <p>In the input here, each element occurs in multiple windows.
 *//*from w ww . j av a2s . c  o m*/
public static void combinesElementsInSlidingWindows(
        GroupAlsoByWindowsDoFnFactory<String, Long, Long> gabwFactory, CombineFn<Long, ?, Long> combineFn)
        throws Exception {

    WindowingStrategy<?, IntervalWindow> windowingStrategy = WindowingStrategy
            .of(SlidingWindows.of(Duration.millis(20)).every(Duration.millis(10)))
            .withOutputTimeFn(OutputTimeFns.outputAtEarliestInputTimestamp());

    DoFnTester<KV<String, Iterable<WindowedValue<Long>>>, KV<String, Long>> result = runGABW(gabwFactory,
            windowingStrategy, "k",
            WindowedValue.of(1L, new Instant(5), Arrays.asList(window(-10, 10), window(0, 20)),
                    PaneInfo.NO_FIRING),
            WindowedValue.of(2L, new Instant(15), Arrays.asList(window(0, 20), window(10, 30)),
                    PaneInfo.NO_FIRING),
            WindowedValue.of(4L, new Instant(18), Arrays.asList(window(0, 20), window(10, 30)),
                    PaneInfo.NO_FIRING));

    assertThat(result.peekOutputElements(), hasSize(3));

    TimestampedValue<KV<String, Long>> item0 = Iterables
            .getOnlyElement(result.peekOutputElementsInWindow(window(-10, 10)));
    assertThat(item0.getValue().getKey(), equalTo("k"));
    assertThat(item0.getValue().getValue(), equalTo(combineFn.apply(ImmutableList.of(1L))));
    assertThat(item0.getTimestamp(), equalTo(new Instant(5L)));

    TimestampedValue<KV<String, Long>> item1 = Iterables
            .getOnlyElement(result.peekOutputElementsInWindow(window(0, 20)));
    assertThat(item1.getValue().getKey(), equalTo("k"));
    assertThat(item1.getValue().getValue(), equalTo(combineFn.apply(ImmutableList.of(1L, 2L, 4L))));
    assertThat(item1.getTimestamp(), equalTo(new Instant(5L)));

    TimestampedValue<KV<String, Long>> item2 = Iterables
            .getOnlyElement(result.peekOutputElementsInWindow(window(10, 30)));
    assertThat(item2.getValue().getKey(), equalTo("k"));
    assertThat(item2.getValue().getValue(), equalTo(combineFn.apply(ImmutableList.of(2L, 4L))));
    assertThat(item2.getTimestamp(), equalTo(new Instant(15L)));
}

---

From source file:org.apache.beam.sdk.util.GroupAlsoByWindowsProperties.java

License:Apache License

/**
 * Tests that the given GABW implementation correctly groups elements that fall into overlapping
 * windows that are not merged.//from  w  w  w .j  a  v a 2 s  .  c  o m
 */
public static void groupsIntoOverlappingNonmergingWindows(
        GroupAlsoByWindowsDoFnFactory<String, String, Iterable<String>> gabwFactory) throws Exception {

    WindowingStrategy<?, IntervalWindow> windowingStrategy = WindowingStrategy
            .of(FixedWindows.of(Duration.millis(10)));

    DoFnTester<KV<String, Iterable<WindowedValue<String>>>, KV<String, Iterable<String>>> result = runGABW(
            gabwFactory, windowingStrategy, "key",
            WindowedValue.of("v1", new Instant(1), Arrays.asList(window(0, 5)), PaneInfo.NO_FIRING),
            WindowedValue.of("v2", new Instant(4), Arrays.asList(window(1, 5)), PaneInfo.NO_FIRING),
            WindowedValue.of("v3", new Instant(4), Arrays.asList(window(0, 5)), PaneInfo.NO_FIRING));

    assertThat(result.peekOutputElements(), hasSize(2));

    TimestampedValue<KV<String, Iterable<String>>> item0 = Iterables
            .getOnlyElement(result.peekOutputElementsInWindow(window(0, 5)));
    assertThat(item0.getValue().getValue(), containsInAnyOrder("v1", "v3"));
    assertThat(item0.getTimestamp(), equalTo(window(1, 5).maxTimestamp()));

    TimestampedValue<KV<String, Iterable<String>>> item1 = Iterables
            .getOnlyElement(result.peekOutputElementsInWindow(window(1, 5)));
    assertThat(item1.getValue().getValue(), contains("v2"));
    assertThat(item1.getTimestamp(), equalTo(window(0, 5).maxTimestamp()));
}

---

From source file:org.apache.beam.sdk.util.GroupAlsoByWindowsProperties.java

License:Apache License

/**
 * Tests that the given GABW implementation correctly groups elements into merged sessions.
 *//*from   w  w w .  j a  v  a  2 s . com*/
public static void groupsElementsInMergedSessions(
        GroupAlsoByWindowsDoFnFactory<String, String, Iterable<String>> gabwFactory) throws Exception {

    WindowingStrategy<?, IntervalWindow> windowingStrategy = WindowingStrategy
            .of(Sessions.withGapDuration(Duration.millis(10)));

    DoFnTester<KV<String, Iterable<WindowedValue<String>>>, KV<String, Iterable<String>>> result = runGABW(
            gabwFactory, windowingStrategy, "key",
            WindowedValue.of("v1", new Instant(0), Arrays.asList(window(0, 10)), PaneInfo.NO_FIRING),
            WindowedValue.of("v2", new Instant(5), Arrays.asList(window(5, 15)), PaneInfo.NO_FIRING),
            WindowedValue.of("v3", new Instant(15), Arrays.asList(window(15, 25)), PaneInfo.NO_FIRING));

    assertThat(result.peekOutputElements(), hasSize(2));

    TimestampedValue<KV<String, Iterable<String>>> item0 = Iterables
            .getOnlyElement(result.peekOutputElementsInWindow(window(0, 15)));
    assertThat(item0.getValue().getValue(), containsInAnyOrder("v1", "v2"));
    assertThat(item0.getTimestamp(), equalTo(window(0, 15).maxTimestamp()));

    TimestampedValue<KV<String, Iterable<String>>> item1 = Iterables
            .getOnlyElement(result.peekOutputElementsInWindow(window(15, 25)));
    assertThat(item1.getValue().getValue(), contains("v3"));
    assertThat(item1.getTimestamp(), equalTo(window(15, 25).maxTimestamp()));
}

---

From source file:org.apache.beam.sdk.util.GroupAlsoByWindowsProperties.java

License:Apache License

/**
 * Tests that the given {@link GroupAlsoByWindowsDoFn} implementation combines elements per
 * session window correctly according to the provided {@link CombineFn}.
 *//*  ww  w.ja va  2 s. c o  m*/
public static void combinesElementsPerSession(GroupAlsoByWindowsDoFnFactory<String, Long, Long> gabwFactory,
        CombineFn<Long, ?, Long> combineFn) throws Exception {

    WindowingStrategy<?, IntervalWindow> windowingStrategy = WindowingStrategy
            .of(Sessions.withGapDuration(Duration.millis(10)));

    DoFnTester<KV<String, Iterable<WindowedValue<Long>>>, KV<String, Long>> result = runGABW(gabwFactory,
            windowingStrategy, "k",
            WindowedValue.of(1L, new Instant(0), Arrays.asList(window(0, 10)), PaneInfo.NO_FIRING),
            WindowedValue.of(2L, new Instant(5), Arrays.asList(window(5, 15)), PaneInfo.NO_FIRING),
            WindowedValue.of(4L, new Instant(15), Arrays.asList(window(15, 25)), PaneInfo.NO_FIRING));

    assertThat(result.peekOutputElements(), hasSize(2));

    TimestampedValue<KV<String, Long>> item0 = Iterables
            .getOnlyElement(result.peekOutputElementsInWindow(window(0, 15)));
    assertThat(item0.getValue().getKey(), equalTo("k"));
    assertThat(item0.getValue().getValue(), equalTo(combineFn.apply(ImmutableList.of(1L, 2L))));
    assertThat(item0.getTimestamp(), equalTo(window(0, 15).maxTimestamp()));

    TimestampedValue<KV<String, Long>> item1 = Iterables
            .getOnlyElement(result.peekOutputElementsInWindow(window(15, 25)));
    assertThat(item1.getValue().getKey(), equalTo("k"));
    assertThat(item1.getValue().getValue(), equalTo(combineFn.apply(ImmutableList.of(4L))));
    assertThat(item1.getTimestamp(), equalTo(window(15, 25).maxTimestamp()));
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• »