List of usage examples for org.joda.time Duration standardSeconds
public static Duration standardSeconds(long seconds)
From source file:google.registry.config.ConfigModule.java
License:Open Source License
/** * The maximum interval (seconds) to lease tasks from the dns-pull queue. * * @see google.registry.dns.ReadDnsQueueAction * @see google.registry.dns.PublishDnsUpdatesAction */// w ww.ja v a 2 s .co m @Provides @Config("dnsWriteLockTimeout") public static Duration provideDnsWriteLockTimeout() { // Optimally, we would set this to a little less than the length of the DNS refresh cycle, since // otherwise, a new PublishDnsUpdatesAction could get kicked off before the current one has // finished, which will try and fail to acquire the lock. However, it is more important that it // be greater than the DNS write timeout, so that if that timeout occurs, it will be cleaned up // gracefully, rather than having the lock time out. So we have to live with the possible lock // failures. return Duration.standardSeconds(75); }
From source file:google.registry.config.ConfigModule.java
License:Open Source License
/** * Returns the default time to live for DNS records. * * @see google.registry.dns.writer.clouddns.CloudDnsWriter *//*from ww w . j a va 2s . c o m*/ @Provides @Config("dnsDefaultTtl") public static Duration provideDnsDefaultTtl() { return Duration.standardSeconds(180); }
From source file:google.registry.config.ConfigModule.java
License:Open Source License
/** * Maximum amount of time for sending a small XML file to ICANN via HTTP, before killing. * * @see google.registry.rde.RdeReportAction */// w w w . j a va 2 s . c om @Provides @Config("rdeReportLockTimeout") public static Duration provideRdeReportLockTimeout() { return Duration.standardSeconds(60); }
From source file:google.registry.config.ConfigModule.java
License:Open Source License
/** * Returns SSH client connection and read timeout. * * @see google.registry.rde.RdeUploadAction *///w w w . j ava 2s . c om @Provides @Config("sshTimeout") public static Duration provideSshTimeout() { return Duration.standardSeconds(30); }
From source file:google.registry.config.ConfigModule.java
License:Open Source License
/** * The reporting interval, for BigQueryMetricsEnqueuer to be sent to a {@link * google.registry.monitoring.metrics.MetricWriter}. * * @see google.registry.monitoring.metrics.MetricReporter *//*from w ww . j av a 2 s .co m*/ @Provides @Config("metricsWriteInterval") public static Duration provideMetricsWriteInterval() { return Duration.standardSeconds(60); }
From source file:google.registry.config.ConfigModule.java
License:Open Source License
/** * Returns the delay before executing async delete flow mapreduces. * * <p>This delay should be sufficiently longer than a transaction, to solve the following problem: * <ul>//from w w w.jav a 2 s . co m * <li>a domain mutation flow starts a transaction * <li>the domain flow non-transactionally reads a resource and sees that it's not in * PENDING_DELETE * <li>the domain flow creates a new reference to this resource * <li>a contact/host delete flow runs and marks the resource PENDING_DELETE and commits * <li>the domain flow commits * </ul> * * <p>Although we try not to add references to a PENDING_DELETE resource, strictly speaking that * is ok as long as the mapreduce eventually sees the new reference (and therefore asynchronously * fails the delete). Without this delay, the mapreduce might have started before the domain flow * committed, and could potentially miss the reference. * * @see google.registry.flows.async.AsyncFlowEnqueuer */ @Provides @Config("asyncDeleteFlowMapreduceDelay") public static Duration provideAsyncDeleteFlowMapreduceDelay() { return Duration.standardSeconds(90); }
From source file:google.registry.dns.writer.dnsupdate.DnsUpdateConfigModule.java
License:Open Source License
/** * Timeout on the socket for DNS update requests. *///from w w w. j a v a2 s . co m @Provides @Config("dnsUpdateTimeout") public static Duration provideDnsUpdateTimeout() { return Duration.standardSeconds(30); }
From source file:io.druid.client.DirectDruidClient.java
License:Apache License
@Override public Sequence<T> run(final QueryPlus<T> queryPlus, final Map<String, Object> context) { final Query<T> query = queryPlus.getQuery(); QueryToolChest<T, Query<T>> toolChest = warehouse.getToolChest(query); boolean isBySegment = QueryContexts.isBySegment(query); Pair<JavaType, JavaType> types = typesMap.get(query.getClass()); if (types == null) { final TypeFactory typeFactory = objectMapper.getTypeFactory(); JavaType baseType = typeFactory.constructType(toolChest.getResultTypeReference()); JavaType bySegmentType = typeFactory.constructParametricType(Result.class, typeFactory.constructParametricType(BySegmentResultValueClass.class, baseType)); types = Pair.of(baseType, bySegmentType); typesMap.put(query.getClass(), types); }// ww w. ja v a 2 s. c om final JavaType typeRef; if (isBySegment) { typeRef = types.rhs; } else { typeRef = types.lhs; } final ListenableFuture<InputStream> future; final String url = StringUtils.format("%s://%s/druid/v2/", scheme, host); final String cancelUrl = StringUtils.format("%s://%s/druid/v2/%s", scheme, host, query.getId()); try { log.debug("Querying queryId[%s] url[%s]", query.getId(), url); final long requestStartTimeNs = System.nanoTime(); long timeoutAt = ((Long) context.get(QUERY_FAIL_TIME)).longValue(); long maxScatterGatherBytes = QueryContexts.getMaxScatterGatherBytes(query); AtomicLong totalBytesGathered = (AtomicLong) context.get(QUERY_TOTAL_BYTES_GATHERED); final HttpResponseHandler<InputStream, InputStream> responseHandler = new HttpResponseHandler<InputStream, InputStream>() { private final AtomicLong byteCount = new AtomicLong(0); private final BlockingQueue<InputStream> queue = new LinkedBlockingQueue<>(); private final AtomicBoolean done = new AtomicBoolean(false); private final AtomicReference<String> fail = new AtomicReference<>(); private QueryMetrics<? super Query<T>> queryMetrics; private long responseStartTimeNs; private QueryMetrics<? super Query<T>> acquireResponseMetrics() { if (queryMetrics == null) { queryMetrics = toolChest.makeMetrics(query); queryMetrics.server(host); } return queryMetrics; } @Override public ClientResponse<InputStream> handleResponse(HttpResponse response) { checkQueryTimeout(); checkTotalBytesLimit(response.getContent().readableBytes()); log.debug("Initial response from url[%s] for queryId[%s]", url, query.getId()); responseStartTimeNs = System.nanoTime(); acquireResponseMetrics().reportNodeTimeToFirstByte(responseStartTimeNs - requestStartTimeNs) .emit(emitter); try { final String responseContext = response.headers().get("X-Druid-Response-Context"); // context may be null in case of error or query timeout if (responseContext != null) { context.putAll(objectMapper.<Map<String, Object>>readValue(responseContext, new TypeReference<Map<String, Object>>() { })); } queue.put(new ChannelBufferInputStream(response.getContent())); } catch (final IOException e) { log.error(e, "Error parsing response context from url [%s]", url); return ClientResponse.<InputStream>finished(new InputStream() { @Override public int read() throws IOException { throw e; } }); } catch (InterruptedException e) { log.error(e, "Queue appending interrupted"); Thread.currentThread().interrupt(); throw Throwables.propagate(e); } byteCount.addAndGet(response.getContent().readableBytes()); return ClientResponse .<InputStream>finished(new SequenceInputStream(new Enumeration<InputStream>() { @Override public boolean hasMoreElements() { if (fail.get() != null) { throw new RE(fail.get()); } checkQueryTimeout(); // Done is always true until the last stream has be put in the queue. // Then the stream should be spouting good InputStreams. synchronized (done) { return !done.get() || !queue.isEmpty(); } } @Override public InputStream nextElement() { if (fail.get() != null) { throw new RE(fail.get()); } try { InputStream is = queue.poll(checkQueryTimeout(), TimeUnit.MILLISECONDS); if (is != null) { return is; } else { throw new RE("Query[%s] url[%s] timed out.", query.getId(), url); } } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw Throwables.propagate(e); } } })); } @Override public ClientResponse<InputStream> handleChunk(ClientResponse<InputStream> clientResponse, HttpChunk chunk) { checkQueryTimeout(); final ChannelBuffer channelBuffer = chunk.getContent(); final int bytes = channelBuffer.readableBytes(); checkTotalBytesLimit(bytes); if (bytes > 0) { try { queue.put(new ChannelBufferInputStream(channelBuffer)); } catch (InterruptedException e) { log.error(e, "Unable to put finalizing input stream into Sequence queue for url [%s]", url); Thread.currentThread().interrupt(); throw Throwables.propagate(e); } byteCount.addAndGet(bytes); } return clientResponse; } @Override public ClientResponse<InputStream> done(ClientResponse<InputStream> clientResponse) { long stopTimeNs = System.nanoTime(); long nodeTimeNs = stopTimeNs - responseStartTimeNs; final long nodeTimeMs = TimeUnit.NANOSECONDS.toMillis(nodeTimeNs); log.debug( "Completed queryId[%s] request to url[%s] with %,d bytes returned in %,d millis [%,f b/s].", query.getId(), url, byteCount.get(), nodeTimeMs, byteCount.get() / (0.001 * nodeTimeMs) // Floating math; division by zero will yield Inf, not exception ); QueryMetrics<? super Query<T>> responseMetrics = acquireResponseMetrics(); responseMetrics.reportNodeTime(nodeTimeNs); responseMetrics.reportNodeBytes(byteCount.get()); responseMetrics.emit(emitter); synchronized (done) { try { // An empty byte array is put at the end to give the SequenceInputStream.close() as something to close out // after done is set to true, regardless of the rest of the stream's state. queue.put(ByteSource.empty().openStream()); } catch (InterruptedException e) { log.error(e, "Unable to put finalizing input stream into Sequence queue for url [%s]", url); Thread.currentThread().interrupt(); throw Throwables.propagate(e); } catch (IOException e) { // This should never happen throw Throwables.propagate(e); } finally { done.set(true); } } return ClientResponse.<InputStream>finished(clientResponse.getObj()); } @Override public void exceptionCaught(final ClientResponse<InputStream> clientResponse, final Throwable e) { String msg = StringUtils.format("Query[%s] url[%s] failed with exception msg [%s]", query.getId(), url, e.getMessage()); setupResponseReadFailure(msg, e); } private void setupResponseReadFailure(String msg, Throwable th) { fail.set(msg); queue.clear(); queue.offer(new InputStream() { @Override public int read() throws IOException { if (th != null) { throw new IOException(msg, th); } else { throw new IOException(msg); } } }); } // Returns remaining timeout or throws exception if timeout already elapsed. private long checkQueryTimeout() { long timeLeft = timeoutAt - System.currentTimeMillis(); if (timeLeft <= 0) { String msg = StringUtils.format("Query[%s] url[%s] timed out.", query.getId(), url); setupResponseReadFailure(msg, null); throw new RE(msg); } else { return timeLeft; } } private void checkTotalBytesLimit(long bytes) { if (maxScatterGatherBytes < Long.MAX_VALUE && totalBytesGathered.addAndGet(bytes) > maxScatterGatherBytes) { String msg = StringUtils.format("Query[%s] url[%s] max scatter-gather bytes limit reached.", query.getId(), url); setupResponseReadFailure(msg, null); throw new RE(msg); } } }; long timeLeft = timeoutAt - System.currentTimeMillis(); if (timeLeft <= 0) { throw new RE("Query[%s] url[%s] timed out.", query.getId(), url); } future = httpClient.go( new Request(HttpMethod.POST, new URL(url)) .setContent(objectMapper.writeValueAsBytes(QueryContexts.withTimeout(query, timeLeft))) .setHeader(HttpHeaders.Names.CONTENT_TYPE, isSmile ? SmileMediaTypes.APPLICATION_JACKSON_SMILE : MediaType.APPLICATION_JSON), responseHandler, Duration.millis(timeLeft)); queryWatcher.registerQuery(query, future); openConnections.getAndIncrement(); Futures.addCallback(future, new FutureCallback<InputStream>() { @Override public void onSuccess(InputStream result) { openConnections.getAndDecrement(); } @Override public void onFailure(Throwable t) { openConnections.getAndDecrement(); if (future.isCancelled()) { // forward the cancellation to underlying queriable node try { StatusResponseHolder res = httpClient .go(new Request(HttpMethod.DELETE, new URL(cancelUrl)) .setContent(objectMapper.writeValueAsBytes(query)) .setHeader(HttpHeaders.Names.CONTENT_TYPE, isSmile ? SmileMediaTypes.APPLICATION_JACKSON_SMILE : MediaType.APPLICATION_JSON), new StatusResponseHandler(Charsets.UTF_8), Duration.standardSeconds(1)) .get(1, TimeUnit.SECONDS); if (res.getStatus().getCode() >= 500) { throw new RE("Error cancelling query[%s]: queriable node returned status[%d] [%s].", res.getStatus().getCode(), res.getStatus().getReasonPhrase()); } } catch (IOException | ExecutionException | InterruptedException | TimeoutException e) { Throwables.propagate(e); } } } }); } catch (IOException e) { throw Throwables.propagate(e); } Sequence<T> retVal = new BaseSequence<>(new BaseSequence.IteratorMaker<T, JsonParserIterator<T>>() { @Override public JsonParserIterator<T> make() { return new JsonParserIterator<T>(typeRef, future, url, query); } @Override public void cleanup(JsonParserIterator<T> iterFromMake) { CloseQuietly.close(iterFromMake); } }); // bySegment queries are de-serialized after caching results in order to // avoid the cost of de-serializing and then re-serializing again when adding to cache if (!isBySegment) { retVal = Sequences.map(retVal, toolChest.makePreComputeManipulatorFn(query, MetricManipulatorFns.deserializing())); } return retVal; }
From source file:io.druid.indexing.jdbc.JDBCIndexTaskClient.java
License:Apache License
private FullResponseHolder submitRequest(String id, HttpMethod method, String pathSuffix, String query, byte[] content, boolean retry) { final RetryPolicy retryPolicy = retryPolicyFactory.makeRetryPolicy(); while (true) { FullResponseHolder response = null; Request request = null;/*from w ww . j av a 2 s .co m*/ TaskLocation location = TaskLocation.unknown(); String path = String.format("%s/%s/%s", BASE_PATH, id, pathSuffix); Optional<TaskStatus> status = taskInfoProvider.getTaskStatus(id); if (!status.isPresent() || !status.get().isRunnable()) { throw new TaskNotRunnableException( String.format("Aborting request because task [%s] is not runnable", id)); } try { location = taskInfoProvider.getTaskLocation(id); if (location.equals(TaskLocation.unknown())) { throw new NoTaskLocationException(String.format("No TaskLocation available for task [%s]", id)); } // Netty throws some annoying exceptions if a connection can't be opened, which happens relatively frequently // for tasks that happen to still be starting up, so test the connection first to keep the logs clean. checkConnection(location.getHost(), location.getPort()); try { URI serviceUri = new URI("http", null, location.getHost(), location.getPort(), path, query, null); request = new Request(method, serviceUri.toURL()); // used to validate that we are talking to the correct worker request.addHeader(ChatHandlerResource.TASK_ID_HEADER, id); if (content.length > 0) { request.setContent(MediaType.APPLICATION_JSON, content); } log.debug("HTTP %s: %s", method.getName(), serviceUri.toString()); response = httpClient.go(request, new FullResponseHandler(Charsets.UTF_8), httpTimeout).get(); } catch (Exception e) { Throwables.propagateIfInstanceOf(e.getCause(), IOException.class); Throwables.propagateIfInstanceOf(e.getCause(), ChannelException.class); throw Throwables.propagate(e); } int responseCode = response.getStatus().getCode(); if (responseCode / 100 == 2) { return response; } else if (responseCode == 400) { // don't bother retrying if it's a bad request throw new IAE("Received 400 Bad Request with body: %s", response.getContent()); } else { throw new IOException(String.format("Received status [%d]", responseCode)); } } catch (IOException | ChannelException e) { // Since workers are free to move tasks around to different ports, there is a chance that a task may have been // moved but our view of its location has not been updated yet from ZK. To detect this case, we send a header // identifying our expected recipient in the request; if this doesn't correspond to the worker we messaged, the // worker will return an HTTP 404 with its ID in the response header. If we get a mismatching task ID, then // we will wait for a short period then retry the request indefinitely, expecting the task's location to // eventually be updated. final Duration delay; if (response != null && response.getStatus().equals(HttpResponseStatus.NOT_FOUND)) { String headerId = response.getResponse().headers().get(ChatHandlerResource.TASK_ID_HEADER); if (headerId != null && !headerId.equals(id)) { log.warn("Expected worker to have taskId [%s] but has taskId [%s], will retry in [%d]s", id, headerId, TASK_MISMATCH_RETRY_DELAY_SECONDS); delay = Duration.standardSeconds(TASK_MISMATCH_RETRY_DELAY_SECONDS); } else { delay = retryPolicy.getAndIncrementRetryDelay(); } } else { delay = retryPolicy.getAndIncrementRetryDelay(); } String urlForLog = (request != null ? request.getUrl().toString() : String.format("http://%s:%d%s", location.getHost(), location.getPort(), path)); if (!retry) { // if retry=false, we probably aren't too concerned if the operation doesn't succeed (i.e. the request was // for informational purposes only) so don't log a scary stack trace log.info("submitRequest failed for [%s], with message [%s]", urlForLog, e.getMessage()); Throwables.propagate(e); } else if (delay == null) { log.warn(e, "Retries exhausted for [%s], last exception:", urlForLog); Throwables.propagate(e); } else { try { final long sleepTime = delay.getMillis(); log.debug("Bad response HTTP [%s] from [%s]; will try again in [%s] (body/exception: [%s])", (response != null ? response.getStatus().getCode() : "no response"), urlForLog, new Duration(sleepTime).toString(), (response != null ? response.getContent() : e.getMessage())); Thread.sleep(sleepTime); } catch (InterruptedException e2) { Throwables.propagate(e2); } } } catch (NoTaskLocationException e) { log.info( "No TaskLocation available for task [%s], this task may not have been assigned to a worker yet or " + "may have already completed", id); throw e; } catch (Exception e) { log.warn(e, "Exception while sending request"); throw e; } } }
From source file:io.druid.indexing.kafka.KafkaIndexTaskClient.java
License:Apache License
private FullResponseHolder submitRequest(String id, HttpMethod method, String pathSuffix, String query, byte[] content, boolean retry) { final RetryPolicy retryPolicy = retryPolicyFactory.makeRetryPolicy(); while (true) { FullResponseHolder response = null; Request request = null;/*from ww w .j a v a2 s.c om*/ TaskLocation location = TaskLocation.unknown(); String path = StringUtils.format("%s/%s/%s", BASE_PATH, id, pathSuffix); Optional<TaskStatus> status = taskInfoProvider.getTaskStatus(id); if (!status.isPresent() || !status.get().isRunnable()) { throw new TaskNotRunnableException( StringUtils.format("Aborting request because task [%s] is not runnable", id)); } String host = location.getHost(); String scheme = ""; int port = -1; try { location = taskInfoProvider.getTaskLocation(id); if (location.equals(TaskLocation.unknown())) { throw new NoTaskLocationException( StringUtils.format("No TaskLocation available for task [%s]", id)); } host = location.getHost(); scheme = location.getTlsPort() >= 0 ? "https" : "http"; port = location.getTlsPort() >= 0 ? location.getTlsPort() : location.getPort(); // Netty throws some annoying exceptions if a connection can't be opened, which happens relatively frequently // for tasks that happen to still be starting up, so test the connection first to keep the logs clean. checkConnection(host, port); try { URI serviceUri = new URI(scheme, null, host, port, path, query, null); request = new Request(method, serviceUri.toURL()); // used to validate that we are talking to the correct worker request.addHeader(ChatHandlerResource.TASK_ID_HEADER, id); if (content.length > 0) { request.setContent(MediaType.APPLICATION_JSON, content); } log.debug("HTTP %s: %s", method.getName(), serviceUri.toString()); response = httpClient.go(request, new FullResponseHandler(Charsets.UTF_8), httpTimeout).get(); } catch (Exception e) { Throwables.propagateIfInstanceOf(e.getCause(), IOException.class); Throwables.propagateIfInstanceOf(e.getCause(), ChannelException.class); throw Throwables.propagate(e); } int responseCode = response.getStatus().getCode(); if (responseCode / 100 == 2) { return response; } else if (responseCode == 400) { // don't bother retrying if it's a bad request throw new IAE("Received 400 Bad Request with body: %s", response.getContent()); } else { throw new IOE("Received status [%d]", responseCode); } } catch (IOException | ChannelException e) { // Since workers are free to move tasks around to different ports, there is a chance that a task may have been // moved but our view of its location has not been updated yet from ZK. To detect this case, we send a header // identifying our expected recipient in the request; if this doesn't correspond to the worker we messaged, the // worker will return an HTTP 404 with its ID in the response header. If we get a mismatching task ID, then // we will wait for a short period then retry the request indefinitely, expecting the task's location to // eventually be updated. final Duration delay; if (response != null && response.getStatus().equals(HttpResponseStatus.NOT_FOUND)) { String headerId = response.getResponse().headers().get(ChatHandlerResource.TASK_ID_HEADER); if (headerId != null && !headerId.equals(id)) { log.warn("Expected worker to have taskId [%s] but has taskId [%s], will retry in [%d]s", id, headerId, TASK_MISMATCH_RETRY_DELAY_SECONDS); delay = Duration.standardSeconds(TASK_MISMATCH_RETRY_DELAY_SECONDS); } else { delay = retryPolicy.getAndIncrementRetryDelay(); } } else { delay = retryPolicy.getAndIncrementRetryDelay(); } String urlForLog = (request != null ? request.getUrl().toString() : StringUtils.format("%s://%s:%d%s", scheme, host, port, path)); if (!retry) { // if retry=false, we probably aren't too concerned if the operation doesn't succeed (i.e. the request was // for informational purposes only) so don't log a scary stack trace log.info("submitRequest failed for [%s], with message [%s]", urlForLog, e.getMessage()); Throwables.propagate(e); } else if (delay == null) { log.warn(e, "Retries exhausted for [%s], last exception:", urlForLog); Throwables.propagate(e); } else { try { final long sleepTime = delay.getMillis(); log.debug("Bad response HTTP [%s] from [%s]; will try again in [%s] (body/exception: [%s])", (response != null ? response.getStatus().getCode() : "no response"), urlForLog, new Duration(sleepTime).toString(), (response != null ? response.getContent() : e.getMessage())); Thread.sleep(sleepTime); } catch (InterruptedException e2) { Throwables.propagate(e2); } } } catch (NoTaskLocationException e) { log.info( "No TaskLocation available for task [%s], this task may not have been assigned to a worker yet or " + "may have already completed", id); throw e; } catch (Exception e) { log.warn(e, "Exception while sending request"); throw e; } } }