List of usage examples for org.joda.time Duration standardMinutes
public static Duration standardMinutes(long minutes)
From source file:com.example.WindowedWordCount.java
License:Apache License
public static void main(String[] args) throws IOException { Options options = PipelineOptionsFactory.fromArgs(args).withValidation().as(Options.class); options.setBigQuerySchema(getSchema()); // DataflowExampleUtils creates the necessary input sources to simplify execution of this // Pipeline.// w w w. j av a 2 s. com DataflowExampleUtils exampleDataflowUtils = new DataflowExampleUtils(options, options.isUnbounded()); Pipeline pipeline = Pipeline.create(options); /** * Concept #1: the Dataflow SDK lets us run the same pipeline with either a bounded or * unbounded input source. */ PCollection<String> input; if (options.isUnbounded()) { LOG.info("Reading from PubSub."); /** * Concept #3: Read from the PubSub topic. A topic will be created if it wasn't * specified as an argument. The data elements' timestamps will come from the pubsub * injection. */ input = pipeline.apply(PubsubIO.Read.topic(options.getPubsubTopic())); } else { /** Else, this is a bounded pipeline. Read from the GCS file. */ input = pipeline.apply(TextIO.Read.from(options.getInputFile())) // Concept #2: Add an element timestamp, using an artificial time just to show windowing. // See AddTimestampFn for more detail on this. .apply(ParDo.of(new AddTimestampFn())); } /** * Concept #4: Window into fixed windows. The fixed window size for this example defaults to 1 * minute (you can change this with a command-line option). See the documentation for more * information on how fixed windows work, and for information on the other types of windowing * available (e.g., sliding windows). */ PCollection<String> windowedWords = input .apply(Window.<String>into(FixedWindows.of(Duration.standardMinutes(options.getWindowSize())))); /** * Concept #5: Re-use our existing CountWords transform that does not have knowledge of * windows over a PCollection containing windowed values. */ PCollection<KV<String, Long>> wordCounts = windowedWords.apply(new WordCount.CountWords()); /** * Concept #6: Format the results for a BigQuery table, then write to BigQuery. * The BigQuery output source supports both bounded and unbounded data. */ wordCounts.apply(ParDo.of(new FormatAsTableRowFn())) .apply(BigQueryIO.Write.to(getTableReference(options)).withSchema(getSchema())); PipelineResult result = pipeline.run(); /** * To mock unbounded input from PubSub, we'll now start an auxiliary 'injector' pipeline that * runs for a limited time, and publishes to the input PubSub topic. * * With an unbounded input source, you will need to explicitly shut down this pipeline when you * are done with it, so that you do not continue to be charged for the instances. You can do * this via a ctrl-C from the command line, or from the developer's console UI for Dataflow * pipelines. The PubSub topic will also be deleted at this time. */ exampleDataflowUtils.mockUnboundedSource(options.getInputFile(), result); }
From source file:com.facebook.stats.MultiWindowGauge.java
License:Apache License
private static CompositeGaugeCounter newCompositeGaugeCounter(int minutes, GaugeCounterFactory gaugeCounterFactory) { return new CompositeGaugeCounter(Duration.standardMinutes(minutes), gaugeCounterFactory); }
From source file:com.facebook.stats.MultiWindowMax.java
License:Apache License
public MultiWindowMax() { this(new CompositeMax(Duration.standardMinutes(Integer.MAX_VALUE)), new CompositeMax(Duration.standardMinutes(60)), new CompositeMax(Duration.standardMinutes(10)), new CompositeMax(Duration.standardMinutes(1))); }
From source file:com.facebook.stats.MultiWindowMin.java
License:Apache License
public MultiWindowMin() { this(new CompositeMin(Duration.standardMinutes(Integer.MAX_VALUE)), new CompositeMin(Duration.standardMinutes(60)), new CompositeMin(Duration.standardMinutes(10)), new CompositeMin(Duration.standardMinutes(1))); }
From source file:com.facebook.stats.MultiWindowRate.java
License:Apache License
MultiWindowRate(CompositeSum allTimeCounter, CompositeSum hourCounter, CompositeSum tenMinuteCounter,
CompositeSum minuteCounter, ReadableDateTime start, int timeBucketSizeMillis) {
this.allTimeCounter = allTimeCounter;
this.hourCounter = hourCounter;
this.tenMinuteCounter = tenMinuteCounter;
this.minuteCounter = minuteCounter;
this.start = start;
this.timeBucketSizeMillis = timeBucketSizeMillis;
hourRate = newEventRate(hourCounter, Duration.standardMinutes(60), start);
tenMinuteRate = newEventRate(tenMinuteCounter, Duration.standardMinutes(10), start);
minuteRate = newEventRate(minuteCounter, Duration.standardMinutes(1), start);
currentCounter = nextCurrentCounter(start.toDateTime());
}
From source file:com.facebook.stats.MultiWindowRate.java
License:Apache License
private static CompositeSum newCompositeEventCounter(int minutes) { return new CompositeSum(Duration.standardMinutes(minutes)); }
From source file:com.github.dbourdette.otto.data.DataTableUtils.java
License:Apache License
/** * Finds best Duration of rows for a {@link SimpleDataTable} based on given table interval */// w w w .j av a2s . c o m public static Duration findBest(Interval interval) { Duration duration = new Duration(interval.getStart(), interval.getEnd()); if (duration.isShorterThan(ONE_DAY) || duration.equals(ONE_DAY)) { return Duration.standardMinutes(5); } else if (duration.isShorterThan(FIVE_DAYS) || duration.equals(FIVE_DAYS)) { return Duration.standardMinutes(30); } else { return Duration.standardDays(1); } }
From source file:com.gooddata.dataload.processes.ProcessServiceAT.java
License:Open Source License
@Test(groups = "process", dependsOnMethods = "createProcess") public void createSchedule() { schedule = gd.getProcessService().createSchedule(project, new Schedule(process, "sdktest.grf", "0 0 * * *")); schedule.setReschedule(Duration.standardMinutes(15)); schedule.setName("sdkTestSchedule"); assertThat(schedule, notNullValue()); assertThat(schedule.getExecutable(), is("sdktest.grf")); assertThat(schedule.getRescheduleInMinutes(), is(15)); }
From source file:com.gooddata.dataload.processes.ProcessServiceAT.java
License:Open Source License
@Test(groups = "process", dependsOnMethods = "createSchedule") public void updateSchedule() { schedule.setState(ScheduleState.DISABLED); schedule.setReschedule(Duration.standardMinutes(26)); schedule = gd.getProcessService().updateSchedule(schedule); assertThat(schedule.isEnabled(), is(false)); assertThat(gd.getProcessService().updateSchedule(schedule).getRescheduleInMinutes(), is(26)); }
From source file:com.google.cloud.bigtable.dataflow.example.PubsubWordCount.java
License:Open Source License
/** * <p>Creates a dataflow pipeline that creates the following chain:</p> * <ol>//from www .ja v a2 s .c om * <li> Reads from a Cloud Pubsub topic * <li> Window into fixed windows of 1 minute * <li> Applies word count transform * <li> Creates Puts from each of the word counts in the array * <li> Performs a Bigtable Put on the items * </ol> * * @param args Arguments to use to configure the Dataflow Pipeline. The first three are required * when running via managed resource in Google Cloud Platform. Those options should be omitted * for LOCAL runs. The next four arguments are to configure the Bigtable connection. The last * two items are for Cloud Pubsub. * --runner=BlockingDataflowPipelineRunner * --project=[dataflow project] \\ * --stagingLocation=gs://[your google storage bucket] \\ * --bigtableProjectId=[bigtable project] \\ * --bigtableInstanceId=[bigtable instance id] \\ * --bigtableTableId=[bigtable tableName] * --inputFile=[file path on GCS] * --pubsubTopic=projects/[project name]/topics/[topic name] */ public static void main(String[] args) throws Exception { // CloudBigtableOptions is one way to retrieve the options. It's not required. BigtablePubsubOptions options = PipelineOptionsFactory.fromArgs(args).withValidation() .as(BigtablePubsubOptions.class); // CloudBigtableTableConfiguration contains the project, instance and table to connect to. CloudBigtableTableConfiguration config = new CloudBigtableTableConfiguration.Builder() .withProjectId(options.getBigtableProjectId()).withInstanceId(options.getBigtableInstanceId()) .withTableId(options.getBigtableTableId()).build(); // In order to cancel the pipelines automatically, // DataflowPipelineRunner is forced to be used. // Also enables the 2 jobs to run at the same time. options.setRunner(DataflowRunner.class); options.as(DataflowPipelineOptions.class).setStreaming(true); Pipeline p = Pipeline.create(options); FixedWindows window = FixedWindows.of(Duration.standardMinutes(options.getWindowSize())); p.apply(PubsubIO.readStrings().fromTopic(options.getPubsubTopic())).apply(Window.<String>into(window)) .apply(ParDo.of(new ExtractWordsFn())).apply(Count.<String>perElement()) .apply(ParDo.of(MUTATION_TRANSFORM)).apply(CloudBigtableIO.writeToTable(config)); p.run().waitUntilFinish(); // Start a second job to inject messages into a Cloud Pubsub topic injectMessages(options); }