List of usage examples for org.joda.time Duration millis
public static Duration millis(long millis)
From source file:com.google.cloud.dataflow.sdk.util.GroupAlsoByWindowsProperties.java
License:Apache License
/** * Tests that for a simple sequence of elements on the same key, the given GABW implementation * correctly groups them according to fixed windows and also sets the output timestamp * according to a custom {@link OutputTimeFn}. *///ww w .java2 s.c om public static void groupsElementsIntoFixedWindowsWithCustomTimestamp( GroupAlsoByWindowsDoFnFactory<String, String, Iterable<String>> gabwFactory) throws Exception { WindowingStrategy<?, IntervalWindow> windowingStrategy = WindowingStrategy .of(FixedWindows.of(Duration.millis(10))) .withOutputTimeFn(new OutputTimeFn.Defaults<IntervalWindow>() { @Override public Instant assignOutputTime(Instant inputTimestamp, IntervalWindow window) { return inputTimestamp.isBefore(window.maxTimestamp()) ? inputTimestamp.plus(1) : window.maxTimestamp(); } @Override public Instant combine(Instant outputTime, Instant otherOutputTime) { return outputTime.isBefore(otherOutputTime) ? outputTime : otherOutputTime; } @Override public boolean dependsOnlyOnEarliestInputTimestamp() { return true; } }); List<WindowedValue<KV<String, Iterable<String>>>> result = runGABW(gabwFactory, windowingStrategy, "key", WindowedValue.of("v1", new Instant(1), Arrays.asList(window(0, 10)), PaneInfo.NO_FIRING), WindowedValue.of("v2", new Instant(2), Arrays.asList(window(0, 10)), PaneInfo.NO_FIRING), WindowedValue.of("v3", new Instant(13), Arrays.asList(window(10, 20)), PaneInfo.NO_FIRING)); assertThat(result.size(), equalTo(2)); assertThat(result, containsInAnyOrder(gabwResult(window(0, 10), new Instant(2), "v1", "v2"), gabwResult(window(10, 20), new Instant(14), "v3"))); }
From source file:com.google.cloud.dataflow.sdk.util.GroupAlsoByWindowsProperties.java
License:Apache License
/** * Tests that for a simple sequence of elements on the same key, the given GABW implementation * correctly groups them according to fixed windows and also sets the output timestamp * according to the policy {@link OutputTimeFns#outputAtLatestInputTimestamp()}. *//*from w w w . jav a2 s . co m*/ public static void groupsElementsIntoFixedWindowsWithLatestTimestamp( GroupAlsoByWindowsDoFnFactory<String, String, Iterable<String>> gabwFactory) throws Exception { WindowingStrategy<?, IntervalWindow> windowingStrategy = WindowingStrategy .of(FixedWindows.of(Duration.millis(10))) .withOutputTimeFn(OutputTimeFns.outputAtLatestInputTimestamp()); List<WindowedValue<KV<String, Iterable<String>>>> result = runGABW(gabwFactory, windowingStrategy, "k", WindowedValue.of("v1", new Instant(1), Arrays.asList(window(0, 10)), PaneInfo.NO_FIRING), WindowedValue.of("v2", new Instant(2), Arrays.asList(window(0, 10)), PaneInfo.NO_FIRING), WindowedValue.of("v3", new Instant(13), Arrays.asList(window(10, 20)), PaneInfo.NO_FIRING)); assertThat(result.size(), equalTo(2)); assertThat(result, containsInAnyOrder(gabwResult(window(0, 10), new Instant(2), "v1", "v2"), gabwResult(window(10, 20), new Instant(13), "v3"))); }
From source file:com.google.cloud.dataflow.sdk.util.GroupAlsoByWindowsProperties.java
License:Apache License
/** * Tests that the given GABW implementation correctly groups elements into merged sessions * with output timestamps at the end of the merged window. */// w w w . ja v a 2 s . c o m public static void groupsElementsInMergedSessionsWithEndOfWindowTimestamp( GroupAlsoByWindowsDoFnFactory<String, String, Iterable<String>> gabwFactory) throws Exception { WindowingStrategy<?, IntervalWindow> windowingStrategy = WindowingStrategy .of(Sessions.withGapDuration(Duration.millis(10))) .withOutputTimeFn(OutputTimeFns.outputAtEndOfWindow()); List<WindowedValue<KV<String, Iterable<String>>>> result = runGABW(gabwFactory, windowingStrategy, "k", WindowedValue.of("v1", new Instant(0), Arrays.asList(window(0, 10)), PaneInfo.NO_FIRING), WindowedValue.of("v2", new Instant(5), Arrays.asList(window(5, 15)), PaneInfo.NO_FIRING), WindowedValue.of("v3", new Instant(15), Arrays.asList(window(15, 25)), PaneInfo.NO_FIRING)); assertThat(result.size(), equalTo(2)); assertThat(result, containsInAnyOrder(gabwResult(window(0, 15), window(0, 15).maxTimestamp(), "v1", "v2"), gabwResult(window(15, 25), window(15, 25).maxTimestamp(), "v3"))); }
From source file:com.google.cloud.dataflow.sdk.util.GroupAlsoByWindowsProperties.java
License:Apache License
/** * Tests that the given GABW implementation correctly groups elements into merged sessions * with output timestamps at the end of the merged window. *//* w w w . j a v a 2s. co m*/ public static void groupsElementsInMergedSessionsWithLatestTimestamp( GroupAlsoByWindowsDoFnFactory<String, String, Iterable<String>> gabwFactory) throws Exception { WindowingStrategy<?, IntervalWindow> windowingStrategy = WindowingStrategy .of(Sessions.withGapDuration(Duration.millis(10))) .withOutputTimeFn(OutputTimeFns.outputAtLatestInputTimestamp()); List<WindowedValue<KV<String, Iterable<String>>>> result = runGABW(gabwFactory, windowingStrategy, "k", WindowedValue.of("v1", new Instant(0), Arrays.asList(window(0, 10)), PaneInfo.NO_FIRING), WindowedValue.of("v2", new Instant(5), Arrays.asList(window(5, 15)), PaneInfo.NO_FIRING), WindowedValue.of("v3", new Instant(15), Arrays.asList(window(15, 25)), PaneInfo.NO_FIRING)); assertThat(result.size(), equalTo(2)); assertThat(result, containsInAnyOrder(gabwResult(window(0, 15), new Instant(5), "v1", "v2"), gabwResult(window(15, 25), new Instant(15), "v3"))); }
From source file:com.google.cloud.dataflow.sdk.util.GroupAlsoByWindowsProperties.java
License:Apache License
/** * Tests that the given {@link GroupAlsoByWindowsDoFn} implementation combines elements per * session window correctly according to the provided {@link CombineFn}. *//*from w ww . j a v a2 s . c o m*/ public static void combinesElementsPerSessionWithEndOfWindowTimestamp( GroupAlsoByWindowsDoFnFactory<String, Long, Long> gabwFactory, CombineFn<Long, ?, Long> combineFn) throws Exception { WindowingStrategy<?, IntervalWindow> windowingStrategy = WindowingStrategy .of(Sessions.withGapDuration(Duration.millis(10))) .withOutputTimeFn(OutputTimeFns.outputAtEndOfWindow()); List<WindowedValue<KV<String, Long>>> result = runGABW(gabwFactory, windowingStrategy, "k", WindowedValue.of(1L, new Instant(0), Arrays.asList(window(0, 10)), PaneInfo.NO_FIRING), WindowedValue.of(2L, new Instant(5), Arrays.asList(window(5, 15)), PaneInfo.NO_FIRING), WindowedValue.of(4L, new Instant(15), Arrays.asList(window(15, 25)), PaneInfo.NO_FIRING)); assertThat(result.size(), equalTo(2)); // TODO: Rewrite to use matchers rather than order-based inspection WindowedValue<KV<String, Long>> item0; WindowedValue<KV<String, Long>> item1; if (result.get(0).getWindows().iterator().next().equals(window(0, 15))) { item0 = result.get(0); item1 = result.get(1); } else { item0 = result.get(1); item1 = result.get(0); } assertThat(item0.getValue().getValue(), equalTo(combineFn.apply(ImmutableList.of(1L, 2L)))); assertThat(item0.getWindows(), contains(window(0, 15))); assertThat(item0.getTimestamp(), equalTo(Iterables.getOnlyElement(item0.getWindows()).maxTimestamp())); assertThat(item1.getValue().getValue(), equalTo(combineFn.apply(ImmutableList.of(4L)))); assertThat(item1.getWindows(), contains(window(15, 25))); assertThat(item1.getTimestamp(), equalTo(Iterables.getOnlyElement(item1.getWindows()).maxTimestamp())); }
From source file:com.google.cloud.dataflow.sdk.util.Reshuffle.java
License:Apache License
@Override public PCollection<KV<K, V>> apply(PCollection<KV<K, V>> input) { WindowingStrategy<?, ?> originalStrategy = input.getWindowingStrategy(); // If the input has already had its windows merged, then the GBK that performed the merge // will have set originalStrategy.getWindowFn() to InvalidWindows, causing the GBK contained // here to fail. Instead, we install a valid WindowFn that leaves all windows unchanged. Window.Bound<KV<K, V>> rewindow = Window .<KV<K, V>>into(new IdentityWindowFn<>(originalStrategy.getWindowFn().windowCoder(), originalStrategy.getWindowFn().assignsToSingleWindow())) .triggering(new ReshuffleTrigger<>()).discardingFiredPanes() .withAllowedLateness(Duration.millis(BoundedWindow.TIMESTAMP_MAX_VALUE.getMillis())); return input.apply(rewindow).apply(GroupByKey.<K, V>create()) // Set the windowing strategy directly, so that it doesn't get counted as the user having // set allowed lateness. .setWindowingStrategyInternal(originalStrategy) .apply(ParDo.named("ExpandIterable").of(new DoFn<KV<K, Iterable<V>>, KV<K, V>>() { @Override//from w w w. j a v a 2s . com public void processElement(ProcessContext c) { K key = c.element().getKey(); for (V value : c.element().getValue()) { c.output(KV.of(key, value)); } } })); }
From source file:com.google.cloud.dataflow.sdk.util.TimeUtil.java
License:Apache License
/** * Converts a Dataflow API duration string into a {@link Duration}. * @return the parsed duration, or null if a parse error occurs *//* w ww .j a va 2s .c o m*/ @Nullable public static Duration fromCloudDuration(String duration) { Matcher matcher = DURATION_PATTERN.matcher(duration); if (!matcher.matches()) { return null; } long millis = Long.valueOf(matcher.group(1)) * 1000; String frac = matcher.group(2); if (frac != null) { long fracs = Long.valueOf(frac); if (frac.length() == 3) { // millisecond resolution millis += fracs; } else if (frac.length() == 6) { // microsecond resolution millis += fracs / 1000; } else if (frac.length() == 9) { // nanosecond resolution millis += fracs / 1000000; } else { return null; } } return Duration.millis(millis); }
From source file:com.google.cloud.GrpcServiceOptions.java
License:Open Source License
/** * Returns a builder for API call settings. *///from w w w . j a v a 2 s. c o m protected UnaryCallSettings.Builder getApiCallSettings() { // todo(mziccard): specify timeout these settings: // retryParams().retryMaxAttempts(), retryParams().retryMinAttempts() final RetrySettings.Builder builder = RetrySettings.newBuilder() .setTotalTimeout(Duration.millis(getRetryParams().getTotalRetryPeriodMillis())) .setInitialRpcTimeout(Duration.millis(getInitialTimeout())) .setRpcTimeoutMultiplier(getTimeoutMultiplier()).setMaxRpcTimeout(Duration.millis(getMaxTimeout())) .setInitialRetryDelay(Duration.millis(getRetryParams().getInitialRetryDelayMillis())) .setRetryDelayMultiplier(getRetryParams().getRetryDelayBackoffFactor()) .setMaxRetryDelay(Duration.millis(getRetryParams().getMaxRetryDelayMillis())); return UnaryCallSettings.newBuilder().setRetrySettingsBuilder(builder); }
From source file:com.google.cloud.logging.spi.DefaultLoggingRpc.java
License:Open Source License
private static ApiCallSettings.Builder apiCallSettings(LoggingOptions options) { // todo(mziccard): specify timeout these settings: // retryParams.retryMaxAttempts(), retryParams.retryMinAttempts() RetryParams retryParams = options.retryParams(); final RetrySettings.Builder builder = RetrySettings.newBuilder() .setTotalTimeout(Duration.millis(retryParams.totalRetryPeriodMillis())) .setInitialRpcTimeout(Duration.millis(options.initialTimeout())) .setRpcTimeoutMultiplier(options.timeoutMultiplier()) .setMaxRpcTimeout(Duration.millis(options.maxTimeout())) .setInitialRetryDelay(Duration.millis(retryParams.initialRetryDelayMillis())) .setRetryDelayMultiplier(retryParams.retryDelayBackoffFactor()) .setMaxRetryDelay(Duration.millis(retryParams.maxRetryDelayMillis())); return ApiCallSettings.newBuilder().setRetrySettingsBuilder(builder); }
From source file:com.google.cloud.logging.testing.RemoteLoggingHelper.java
License:Open Source License
private static RetrySettings retrySettings() { return RetrySettings.newBuilder().setMaxRetryDelay(Duration.millis(30000L)) .setTotalTimeout(Duration.millis(120000L)).setInitialRetryDelay(Duration.millis(250L)) .setRetryDelayMultiplier(1.0).setInitialRpcTimeout(Duration.millis(120000L)) .setRpcTimeoutMultiplier(1.0).setMaxRpcTimeout(Duration.millis(120000L)).build(); }