List of usage examples for org.joda.time Duration millis
public static Duration millis(long millis)
From source file:com.google.cloud.training.dataanalyst.sandiego.AverageSpeeds.java
License:Apache License
@SuppressWarnings("serial") public static void main(String[] args) { MyOptions options = PipelineOptionsFactory.fromArgs(args).withValidation().as(MyOptions.class); options.setStreaming(true);//from w w w . j a va2 s . c o m Pipeline p = Pipeline.create(options); String topic = "projects/" + options.getProject() + "/topics/sandiego"; String avgSpeedTable = options.getProject() + ":demos.average_speeds"; // if we need to average over 60 minutes and speedup is 30x // then we need to average over 2 minutes of sped-up stream Duration averagingInterval = Duration .millis(Math.round(1000 * 60 * (options.getAveragingInterval() / options.getSpeedupFactor()))); Duration averagingFrequency = averagingInterval.dividedBy(2); // 2 times // in // window System.out.println("Averaging interval = " + averagingInterval); System.out.println("Averaging freq = " + averagingFrequency); // Build the table schema for the output table. List<TableFieldSchema> fields = new ArrayList<>(); fields.add(new TableFieldSchema().setName("timestamp").setType("TIMESTAMP")); fields.add(new TableFieldSchema().setName("latitude").setType("FLOAT")); fields.add(new TableFieldSchema().setName("longitude").setType("FLOAT")); fields.add(new TableFieldSchema().setName("highway").setType("STRING")); fields.add(new TableFieldSchema().setName("direction").setType("STRING")); fields.add(new TableFieldSchema().setName("lane").setType("INTEGER")); fields.add(new TableFieldSchema().setName("speed").setType("FLOAT")); fields.add(new TableFieldSchema().setName("sensorId").setType("STRING")); TableSchema schema = new TableSchema().setFields(fields); PCollection<LaneInfo> laneInfo = p // .apply("GetMessages", PubsubIO.<String>read().topic(topic).withCoder(StringUtf8Coder.of())) // .apply("TimeWindow", Window.into(SlidingWindows// .of(averagingInterval)// .every(averagingFrequency))) // .apply("ExtractData", ParDo.of(new DoFn<String, LaneInfo>() { @ProcessElement public void processElement(ProcessContext c) throws Exception { String line = c.element(); c.output(LaneInfo.newLaneInfo(line)); } })); PCollection<KV<String, Double>> avgSpeed = laneInfo // .apply("BySensor", ParDo.of(new DoFn<LaneInfo, KV<String, Double>>() { @ProcessElement public void processElement(ProcessContext c) throws Exception { LaneInfo info = c.element(); String key = info.getSensorKey(); Double speed = info.getSpeed(); c.output(KV.of(key, speed)); } })) // .apply("AvgBySensor", Mean.perKey()); avgSpeed.apply("ToBQRow", ParDo.of(new DoFn<KV<String, Double>, TableRow>() { @ProcessElement public void processElement(ProcessContext c) throws Exception { TableRow row = new TableRow(); String stationKey = c.element().getKey(); Double speed = c.element().getValue(); String line = Instant.now().toString() + "," + stationKey + "," + speed; // CSV LaneInfo info = LaneInfo.newLaneInfo(line); row.set("timestamp", info.getTimestamp()); row.set("latitude", info.getLatitude()); row.set("longitude", info.getLongitude()); row.set("highway", info.getHighway()); row.set("direction", info.getDirection()); row.set("lane", info.getLane()); row.set("speed", info.getSpeed()); row.set("sensorId", info.getSensorKey()); c.output(row); } })) // .apply(BigQueryIO.Write.to(avgSpeedTable)// .withSchema(schema)// .withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_APPEND) .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED)); p.run(); }
From source file:com.google.cloud.training.flights.AverageDelayPipeline.java
License:Apache License
private static WindowStats movingAverageOf(MyOptions options, Pipeline p, String event) { // if we need to average over 60 minutes and speedup is 30x // then we need to average over 2 minutes of sped-up stream Duration averagingInterval = Duration .millis(Math.round(1000 * 60 * (options.getAveragingInterval() / options.getSpeedupFactor()))); Duration averagingFrequency = averagingInterval.dividedBy(2); // 2 times in window System.out.println("Averaging interval = " + averagingInterval); System.out.println("Averaging freq = " + averagingFrequency); String topic = "projects/" + options.getProject() + "/topics/" + event; final FieldNumberLookup eventType = FieldNumberLookup.create(event); PCollection<Flight> flights = p // .apply(event + ":read", // PubsubIO.<String>read().topic(topic).withCoder(StringUtf8Coder.of())) // .apply(event + ":window", Window.into(SlidingWindows// .of(averagingInterval)// .every(averagingFrequency))) // .apply(event + ":parse", ParDo.of(new DoFn<String, Flight>() { @ProcessElement/* www . j av a 2 s . c om*/ public void processElement(ProcessContext c) throws Exception { try { String line = c.element(); Flight f = new Flight(line.split(","), eventType); c.output(f); } catch (NumberFormatException e) { // ignore errors about empty delay fields ... } } })); WindowStats stats = new WindowStats(); stats.delay = flights // .apply(event + ":airportdelay", ParDo.of(new DoFn<Flight, KV<Airport, Double>>() { @ProcessElement public void processElement(ProcessContext c) throws Exception { Flight stats = c.element(); c.output(KV.of(stats.airport, stats.delay)); // delay at airport } }))// .apply(event + ":avgdelay", Mean.perKey()); stats.timestamp = flights // .apply(event + ":timestamps", ParDo.of(new DoFn<Flight, KV<Airport, String>>() { @ProcessElement public void processElement(ProcessContext c) throws Exception { Flight stats = c.element(); c.output(KV.of(stats.airport, stats.timestamp)); } }))// .apply(event + ":lastTimeStamp", Max.perKey()); stats.num_flights = flights // .apply(event + ":numflights", ParDo.of(new DoFn<Flight, KV<Airport, Integer>>() { @ProcessElement public void processElement(ProcessContext c) throws Exception { Flight stats = c.element(); c.output(KV.of(stats.airport, 1)); } }))// .apply(event + ":total", Sum.integersPerKey()); return stats; }
From source file:com.google.cloud.translate.testing.RemoteTranslateHelper.java
License:Open Source License
private static RetrySettings retryParams() { return RetrySettings.newBuilder().setMaxAttempts(10).setMaxRetryDelay(Duration.millis(30000L)) .setTotalTimeout(Duration.millis(120000L)).setInitialRetryDelay(Duration.millis(250L)) .setRetryDelayMultiplier(1.0).setInitialRpcTimeout(Duration.millis(120000L)) .setRpcTimeoutMultiplier(1.0).setMaxRpcTimeout(Duration.millis(120000L)).build(); }
From source file:com.google.codelabs.dataflow.LatestRides.java
License:Apache License
public static void main(String[] args) { CustomPipelineOptions options = PipelineOptionsFactory.fromArgs(args).withValidation() .as(CustomPipelineOptions.class); Pipeline p = Pipeline.create(options); p.apply(PubsubIO.Read.named("read from PubSub") .topic(String.format("projects/%s/topics/%s", options.getSourceProject(), options.getSourceTopic())) .timestampLabel("ts").withCoder(TableRowJsonCoder.of())) .apply("key rides by rideid", MapElements.via((TableRow ride) -> KV.of(ride.get("ride_id").toString(), ride)) .withOutputType(new TypeDescriptor<KV<String, TableRow>>() { }))//from ww w. j a v a 2 s . c o m .apply("session windows on rides with early firings", Window .<KV<String, TableRow>>into(Sessions.withGapDuration(Duration.standardMinutes(60))) .triggering(AfterWatermark.pastEndOfWindow().withEarlyFirings( AfterProcessingTime.pastFirstElementInPane().plusDelayOf(Duration.millis(2000)))) .accumulatingFiredPanes().withAllowedLateness(Duration.ZERO)) .apply("group ride points on same ride", Combine.perKey(new LatestPointCombine())) .apply("discard key", MapElements.via((KV<String, TableRow> a) -> a.getValue()) .withOutputType(TypeDescriptor.of(TableRow.class))) .apply(PubsubIO.Write .named("WriteToPubsub").topic(String.format("projects/%s/topics/%s", options.getSinkProject(), options.getSinkTopic())) .withCoder(TableRowJsonCoder.of())); p.run(); }
From source file:com.google.codelabs.dataflow.PickupRides.java
License:Apache License
public static void main(String[] args) { CustomPipelineOptions options = PipelineOptionsFactory.fromArgs(args).withValidation() .as(CustomPipelineOptions.class); Pipeline p = Pipeline.create(options); p.apply(PubsubIO.Read.named("read from PubSub") .topic(String.format("projects/%s/topics/%s", options.getSourceProject(), options.getSourceTopic())) .timestampLabel("ts").withCoder(TableRowJsonCoder.of())) .apply("key rides by rideid", MapElements.via((TableRow ride) -> KV.of(ride.get("ride_id").toString(), ride)) .withOutputType(new TypeDescriptor<KV<String, TableRow>>() { }))/*w w w. j a v a 2 s . co m*/ .apply("session windows on rides with early firings", Window .<KV<String, TableRow>>into(Sessions.withGapDuration(Duration.standardMinutes(1))) .triggering(AfterWatermark.pastEndOfWindow().withEarlyFirings( AfterProcessingTime.pastFirstElementInPane().plusDelayOf(Duration.millis(1000)))) .accumulatingFiredPanes().withAllowedLateness(Duration.ZERO)) .apply("group ride points on same ride", Combine.perKey(new PickupPointCombine())) .apply("discard key", MapElements.via((KV<String, TableRow> a) -> a.getValue()) .withOutputType(TypeDescriptor.of(TableRow.class))) .apply("filter if no pickup", Filter.byPredicate((TableRow a) -> a.get("ride_status").equals("pickup"))) .apply(PubsubIO.Write .named("WriteToPubsub").topic(String.format("projects/%s/topics/%s", options.getSinkProject(), options.getSinkTopic())) .withCoder(TableRowJsonCoder.of())); p.run(); }
From source file:com.google.devtools.build.lib.runtime.FancyTerminalEventHandler.java
License:Open Source License
@Override public void handle(Event event) { if (terminalClosed) { return;/*from www.j a va 2 s. c o m*/ } if (!eventMask.contains(event.getKind())) { return; } if (trySpecial && !EventKind.ERRORS_AND_WARNINGS_AND_OUTPUT.contains(event.getKind()) && skipUntil.isAfterNow()) { // Short-circuit here to avoid wiping out previous terminal contents. return; } try { boolean previousLineErased = false; if (previousLineErasable) { previousLineErased = maybeOverwritePreviousMessage(); } switch (event.getKind()) { case PROGRESS: case START: { String message = event.getMessage(); Pair<String, String> progressPair = matchProgress(message); if (progressPair != null) { progress(progressPair.getFirst(), progressPair.getSecond()); if (trySpecial && ThreadLocalRandom.current().nextInt(0, 20) == 0) { message = getExtraMessage(); if (message != null) { // Should always be true, but don't crash on that! previousLineErased = maybeOverwritePreviousMessage(); progress(progressPair.getFirst(), message); // Skip unimportant messages for a bit so that this message gets some exposure. skipUntil = Instant.now() .plus(Duration.millis(ThreadLocalRandom.current().nextInt(3000, 8000))); } } } else { progress("INFO: ", message); } break; } case FINISH: { String message = event.getMessage(); Pair<String, String> progressPair = matchProgress(message); if (progressPair != null) { String percentage = progressPair.getFirst(); String rest = progressPair.getSecond(); progress(percentage, rest + " DONE"); } else { progress("INFO: ", message + " DONE"); } break; } case PASS: progress("PASS: ", event.getMessage()); break; case INFO: info(event); break; case ERROR: case FAIL: case TIMEOUT: // For errors, scroll the message, so it appears above the status // line, and highlight the word "ERROR" or "FAIL" in boldface red. errorOrFail(event); break; case WARNING: // For warnings, highlight the word "Warning" in boldface magenta, // and scroll it. warning(event); break; case SUBCOMMAND: subcmd(event); break; case STDOUT: if (previousLineErased) { terminal.flush(); } previousLineErasable = false; super.handle(event); // We don't need to flush stdout here, because // super.handle(event) will take care of that. break; case STDERR: putOutput(event); break; default: // Ignore all other event types. break; } } catch (IOException e) { // The terminal shouldn't have IO errors, unless the shell is killed, which // should also kill the blaze client. So this isn't something that should // occur here; it will show up in the client/server interface as a broken // pipe. LOG.warning("Terminal was closed during build: " + e); terminalClosed = true; } }
From source file:com.google.pubsub.clients.experimental.CPSPublisherTask.java
License:Apache License
private CPSPublisherTask(StartRequest request) { super(request, "experimental", MetricsHandler.MetricName.PUBLISH_ACK_LATENCY); this.batchSize = request.getPublishBatchSize(); this.id = (new Random()).nextInt(); try {//from www . j a va 2 s . c om this.publisher = Publisher.Builder .newBuilder("projects/" + request.getProject() + "/topics/" + request.getTopic()) .setMaxBatchDuration(Duration.millis(Durations.toMillis(request.getPublishBatchDuration()))) .setMaxBatchBytes(9500000).setMaxBatchMessages(950).setMaxOutstandingBytes(1000000000) // 1 GB .build(); } catch (Exception e) { throw new RuntimeException(e); } this.payload = ByteString.copyFromUtf8(LoadTestRunner.createMessage(request.getMessageSize())); }
From source file:com.google.pubsub.clients.gcloud.CPSPublisherTask.java
License:Apache License
private CPSPublisherTask(StartRequest request) { super(request, "gcloud", MetricsHandler.MetricName.PUBLISH_ACK_LATENCY); try {/*from ww w .j av a 2s.c o m*/ this.publisher = Publisher.newBuilder(TopicName.create(request.getProject(), request.getTopic())) .setBundlingSettings(BundlingSettings.newBuilder() .setDelayThreshold( Duration.millis(Durations.toMillis(request.getPublishBatchDuration()))) .setRequestByteThreshold(9500000L).setElementCountThreshold(950L).build()) .setFlowControlSettings( FlowControlSettings.newBuilder().setMaxOutstandingRequestBytes(1000000000).build()) // 1 GB .build(); } catch (Exception e) { throw new RuntimeException(e); } this.payload = ByteString.copyFromUtf8(LoadTestRunner.createMessage(request.getMessageSize())); this.batchSize = request.getPublishBatchSize(); this.id = (new Random()).nextInt(); }
From source file:com.google.pubsub.clients.vtk.CPSPublisherTask.java
License:Apache License
private CPSPublisherTask(StartRequest request) { super(request, "vtk", MetricsHandler.MetricName.PUBLISH_ACK_LATENCY); PublisherSettings.Builder publisherSettingsBuilder = PublisherSettings.defaultBuilder(); publisherSettingsBuilder.publishSettings().getBundlingSettingsBuilder() .setDelayThreshold(Duration.millis(Durations.toMillis(request.getPublishBatchDuration()))) .setElementCountThreshold(950L).setRequestByteThreshold(9500000L); try {/*from w w w . java2 s.c o m*/ this.publisherApi = PublisherClient.create(publisherSettingsBuilder.build()); } catch (Exception e) { throw new RuntimeException("Error creating publisher API.", e); } this.topic = TopicName.create(request.getProject(), request.getTopic()).toString(); this.payload = ByteString.copyFromUtf8(LoadTestRunner.createMessage(request.getMessageSize())); this.batchSize = request.getPublishBatchSize(); this.id = (new Random()).nextInt(); }
From source file:com.linkedin.pinot.core.data.function.FunctionInvoker.java
License:Apache License
public Object process(Object[] args) { try {//w ww . j a v a2 s .c o m return _method.invoke(_instance, _functionInfo.convertTypes(args)); } catch (IllegalAccessException | IllegalArgumentException | InvocationTargetException e) { //most likely the exception is in the udf, get the exceptio Throwable cause = e.getCause(); if (cause == null) { cause = e; } //some udf's might be configured incorrectly and we dont want to pollute the log //keep track of the last time an exception was logged and reset the counter if the last exception is more than the EXCEPTION_LIMIT_DURATION if (Duration.millis(System.currentTimeMillis() - lastExceptionTime) .getStandardMinutes() > EXCEPTION_LIMIT_DURATION) { exceptionCount = 0; } if (exceptionCount < EXCEPTION_LIMIT_RATE) { exceptionCount = exceptionCount + 1; LOGGER.error("Exception invoking method:{} with args:{}, exception message: {}", _method.getName(), Arrays.toString(args), cause.getMessage()); } return null; } }