List of usage examples for org.joda.time Duration millis
public static Duration millis(long millis)
From source file:my.group.id.DataflowExampleUtils.java
License:Apache License
/** * If {@literal DataflowPipelineRunner} or {@literal BlockingDataflowPipelineRunner} is used, * waits for the pipeline to finish and cancels it (and the injector) before the program exists. *//* w w w .j a v a2 s . c o m*/ public void waitToFinish(PipelineResult result) { if (result instanceof DataflowPipelineJob) { final DataflowPipelineJob job = (DataflowPipelineJob) result; jobsToCancel.add(job); if (!options.as(DataflowExampleOptions.class).getKeepJobsRunning()) { addShutdownHook(jobsToCancel); } try { job.waitUntilFinish(Duration.millis(-1L)); } catch (Exception e) { throw new RuntimeException("Failed to wait for job to finish: " + job.getJobId()); } } else { // Do nothing if the given PipelineResult doesn't support waitToFinish(), // such as EvaluationResults returned by DirectPipelineRunner. } }
From source file:oncue.worker.MaintenanceWorker.java
License:Apache License
private void processJob() throws Exception { boolean includeFailedJobs = false; if (job.getParams().containsKey(INCLUDE_FAILED_JOBS)) includeFailedJobs = new Boolean(job.getParams().get(INCLUDE_FAILED_JOBS)); if (!job.getParams().containsKey(EXPIRATION_AGE)) throw new IllegalArgumentException( "You must define the 'expiration-age' parameter in the configuration of a maintenance worker"); Duration expirationAge = Duration .millis(scala.concurrent.duration.Duration.create(job.getParams().get(EXPIRATION_AGE)).toMillis()); CleanupJobs cleanupJobs = new CleanupJobs(includeFailedJobs, expirationAge); try {//www . ja va2s .com Object object = Await.result(ask(getContext().actorFor(settings.SCHEDULER_PATH), cleanupJobs, new Timeout(settings.SCHEDULER_TIMEOUT)), settings.SCHEDULER_TIMEOUT); log.info(object.toString()); } catch (TimeoutException e) { throw new RuntimeException("Timeout waiting for scheduler to clean up jobs", e); } }
From source file:org.apache.apex.malhar.lib.join.AbstractManagedStateInnerJoinOperator.java
License:Apache License
/** * Create Managed states and stores for both the streams. *//* w ww .j a v a 2 s . c om*/ @Override public void createStores() { stream1Store = new ManagedTimeStateImpl(); stream2Store = new ManagedTimeStateImpl(); stream1Store.setNumBuckets(noOfBuckets); stream2Store.setNumBuckets(noOfBuckets); assert stream1Store.getTimeBucketAssigner() == stream2Store.getTimeBucketAssigner(); if (bucketSpanTime != null) { stream1Store.getTimeBucketAssigner().setBucketSpan(Duration.millis(bucketSpanTime)); } if (stream1Store.getTimeBucketAssigner() instanceof MovingBoundaryTimeBucketAssigner) { ((MovingBoundaryTimeBucketAssigner) stream1Store.getTimeBucketAssigner()) .setExpireBefore(Duration.millis(getExpiryTime())); } stream1Data = new ManagedTimeStateMultiValue(stream1Store, !isLeftKeyPrimary()); stream2Data = new ManagedTimeStateMultiValue(stream2Store, !isRightKeyPrimary()); }
From source file:org.apache.apex.malhar.lib.state.managed.TimeBucketAssigner.java
License:Apache License
@Override public void setup(@NotNull ManagedStateContext managedStateContext) { Context.OperatorContext context = managedStateContext.getOperatorContext(); if (!initialized && bucketSpan == null) { setBucketSpan(Duration.millis(context.getValue(Context.OperatorContext.APPLICATION_WINDOW_COUNT) * context.getValue(Context.DAGContext.STREAMING_WINDOW_SIZE_MILLIS))); }//w w w.j a v a 2s .c o m }
From source file:org.apache.apex.malhar.lib.window.impl.KeyedWindowedMergeOperatorTestApplication.java
License:Apache License
@Override public void populateDAG(DAG dag, Configuration conf) { KeyedWindowedMergeOperatorImpl<String, Integer, Integer, List<Set<Integer>>, List<List<Integer>>> op = dag .addOperator("Merge", new KeyedWindowedMergeOperatorImpl<String, Integer, Integer, List<Set<Integer>>, List<List<Integer>>>()); //op.setAccumulation(new CoGroup<Integer>()); op.setAccumulation(new InnerJoin<Integer>()); op.setDataStorage(new InMemoryWindowedKeyedStorage<String, List<Set<Integer>>>()); op.setRetractionStorage(new InMemoryWindowedKeyedStorage<String, List<List<Integer>>>()); op.setWindowStateStorage(windowStateMap); // Can select one of the following window options, or don't select any of them. op.setWindowOption(new WindowOption.GlobalWindow()); //op.setWindowOption(new WindowOption.TimeWindows(Duration.millis(4000))); op.setTriggerOption(new TriggerOption().withEarlyFiringsAtEvery(1).accumulatingFiredPanes()); op.setAllowedLateness(Duration.millis(500)); NumGen1 numGen1 = dag.addOperator("numGen1", new NumGen1()); NumGen2 numGen2 = dag.addOperator("numGen2", new NumGen2()); Collector collector = dag.addOperator("collector", new Collector()); ConsoleOutputOperator con = dag.addOperator("console", new ConsoleOutputOperator()); dag.addStream("num1", numGen1.output, op.input); dag.addStream("num2", numGen2.output, op.input2); dag.addStream("wm1", numGen1.watermarkDefaultOutputPort, op.controlInput); dag.addStream("wm2", numGen2.watermarkDefaultOutputPort, op.controlInput2); dag.addStream("MergedResult", op.output, collector.input); dag.addStream("output", collector.output, con.input); }
From source file:org.apache.apex.malhar.lib.window.impl.WindowedMergeOperatorTestApplication.java
License:Apache License
@Override public void populateDAG(DAG dag, Configuration conf) { WindowedMergeOperatorImpl<Integer, Integer, List<Set<Integer>>, List<List<Integer>>> op = dag.addOperator( "Merge", new WindowedMergeOperatorImpl<Integer, Integer, List<Set<Integer>>, List<List<Integer>>>()); op.setAccumulation(new CoGroup<Integer>()); op.setDataStorage(new InMemoryWindowedStorage<List<Set<Integer>>>()); op.setRetractionStorage(new InMemoryWindowedStorage<List<List<Integer>>>()); op.setWindowStateStorage(windowStateMap); // Can select one of the following window options, or don't select any of them. //op.setWindowOption(new WindowOption.GlobalWindow()); op.setWindowOption(new WindowOption.TimeWindows(Duration.millis(2000))); op.setTriggerOption(new TriggerOption().withEarlyFiringsAtEvery(1).accumulatingFiredPanes()); op.setAllowedLateness(Duration.millis(500)); NumGen1 numGen1 = dag.addOperator("numGen1", new NumGen1()); NumGen2 numGen2 = dag.addOperator("numGen2", new NumGen2()); Collector collector = dag.addOperator("collector", new Collector()); ConsoleOutputOperator con = dag.addOperator("console", new ConsoleOutputOperator()); dag.addStream("num1", numGen1.output, op.input); dag.addStream("num2", numGen2.output, op.input2); dag.addStream("wm1", numGen1.watermarkDefaultOutputPort, op.controlInput); dag.addStream("wm2", numGen2.watermarkDefaultOutputPort, op.controlInput2); dag.addStream("MergedResult", op.output, collector.input); dag.addStream("output", collector.output, con.input); }
From source file:org.apache.apex.malhar.lib.window.sample.pi.Application.java
License:Apache License
@Override public void populateDAG(DAG dag, Configuration configuration) { RandomNumberPairGenerator inputOperator = new RandomNumberPairGenerator(); WindowedOperatorImpl<MutablePair<Double, Double>, MutablePair<MutableLong, MutableLong>, Double> windowedOperator = new WindowedOperatorImpl<>(); Accumulation<MutablePair<Double, Double>, MutablePair<MutableLong, MutableLong>, Double> piAccumulation = new PiAccumulation(); windowedOperator.setAccumulation(piAccumulation); windowedOperator.setDataStorage(new InMemoryWindowedStorage<MutablePair<MutableLong, MutableLong>>()); windowedOperator.setWindowStateStorage(new InMemoryWindowedStorage<WindowState>()); windowedOperator.setWindowOption(new WindowOption.GlobalWindow()); windowedOperator.setTriggerOption(TriggerOption.AtWatermark().withEarlyFiringsAtEvery(Duration.millis(1000)) .accumulatingFiredPanes());/*from w w w. j av a 2 s . c o m*/ ConsoleOutputOperator outputOperator = new ConsoleOutputOperator(); dag.addOperator("inputOperator", inputOperator); dag.addOperator("windowedOperator", windowedOperator); dag.addOperator("outputOperator", outputOperator); dag.addStream("input_windowed", inputOperator.output, windowedOperator.input); dag.addStream("windowed_output", windowedOperator.output, outputOperator.input); }
From source file:org.apache.apex.malhar.lib.window.sample.wordcount.Application.java
License:Apache License
@Override public void populateDAG(DAG dag, Configuration configuration) { WordGenerator inputOperator = new WordGenerator(); KeyedWindowedOperatorImpl<String, Long, MutableLong, Long> windowedOperator = new KeyedWindowedOperatorImpl<>(); Accumulation<Long, MutableLong, Long> sum = new SumAccumulation(); windowedOperator.setAccumulation(sum); windowedOperator.setDataStorage(new InMemoryWindowedKeyedStorage<String, MutableLong>()); windowedOperator.setRetractionStorage(new InMemoryWindowedKeyedStorage<String, Long>()); windowedOperator.setWindowStateStorage(new InMemoryWindowedStorage<WindowState>()); windowedOperator.setWindowOption(new WindowOption.TimeWindows(Duration.standardMinutes(1))); windowedOperator.setTriggerOption(TriggerOption.AtWatermark().withEarlyFiringsAtEvery(Duration.millis(1000)) .accumulatingAndRetractingFiredPanes()); //windowedOperator.setAllowedLateness(Duration.millis(14000)); ConsoleOutputOperator outputOperator = new ConsoleOutputOperator(); dag.addOperator("inputOperator", inputOperator); dag.addOperator("windowedOperator", windowedOperator); dag.addOperator("outputOperator", outputOperator); dag.addStream("input_windowed", inputOperator.output, windowedOperator.input); dag.addStream("windowed_output", windowedOperator.output, outputOperator.input); }
From source file:org.apache.apex.malhar.stream.sample.ApplicationWithStreamAPI.java
License:Apache License
@Override public void populateDAG(DAG dag, Configuration configuration) { String localFolder = "./src/test/resources/data"; ApexStream<String> stream = StreamFactory.fromFolder(localFolder) .flatMap(new Function.FlatMapFunction<String, String>() { @Override/*from w w w . j ava2 s . c o m*/ public Iterable<String> f(String input) { return Arrays.asList(input.split("[\\p{Punct}\\s]+")); } }); stream.print(name("WordOutput")); stream.window(new WindowOption.GlobalWindow(), new TriggerOption().withEarlyFiringsAtEvery(Duration.millis(1000)).accumulatingFiredPanes()) .countByKey(new Function.ToKeyValue<String, String, Long>() { @Override public Tuple<KeyValPair<String, Long>> f(String input) { return new Tuple.PlainTuple(new KeyValPair<>(input, 1L)); } }).print(name("WCOutput")); stream.populateDag(dag); }
From source file:org.apache.apex.malhar.stream.sample.complete.TrafficRoutes.java
License:Apache License
@Override public void populateDAG(DAG dag, Configuration conf) { InfoGen infoGen = new InfoGen(); Collector collector = new Collector(); // Create a stream from the input operator. ApexStream<Tuple.TimestampedTuple<String>> stream = StreamFactory .fromInput(infoGen, infoGen.output, name("infoGen")) // Extract the timestamp from the input and wrap it into a TimestampedTuple. .map(new ExtractTimestamps(), name("ExtractTimestamps")); stream//from w ww . ja v a 2 s . c om // Extract the average speed of a station. .flatMap(new ExtractStationSpeedFn(), name("ExtractStationSpeedFn")) // Apply window and trigger option. .window(new WindowOption.SlidingTimeWindows(Duration.standardMinutes(WINDOW_DURATION), Duration.standardMinutes(WINDOW_SLIDE_EVERY)), new TriggerOption().withEarlyFiringsAtEvery(Duration.millis(5000)).accumulatingFiredPanes()) // Apply TrackSpeed composite transformation to compute the route information. .addCompositeStreams(new TrackSpeed()) // print the result to console. .print(name("console")).endWith(collector, collector.input, name("Collector")).populateDag(dag); }