List of usage examples for com.google.common.io CountingOutputStream getCount
public long getCount()
From source file:org.gradle.internal.resource.local.LocalFileStandInExternalResource.java
@Override public ExternalResourceWriteResult put(ReadableContent location) { try {// w w w . j a v a 2 s. c o m if (!localFile.canWrite()) { localFile.delete(); } Files.createParentDirs(localFile); InputStream input = location.open(); try { CountingOutputStream output = new CountingOutputStream(new FileOutputStream(localFile)); try { IOUtils.copyLarge(input, output); } finally { output.close(); } return new ExternalResourceWriteResult(output.getCount()); } finally { input.close(); } } catch (IOException e) { throw ResourceExceptions.putFailed(getURI(), e); } }
From source file:io.druid.query.aggregation.atomcube.AtomCubeQueryResource.java
@POST
@Produces({ MediaType.APPLICATION_JSON, SmileMediaTypes.APPLICATION_JACKSON_SMILE })
@Consumes({ MediaType.APPLICATION_JSON, SmileMediaTypes.APPLICATION_JACKSON_SMILE, APPLICATION_SMILE })
public Response doPost(InputStream in, @QueryParam("pretty") String pretty,
@Context final HttpServletRequest req // used only to get request content-type and remote address
) throws IOException {
final long start = System.currentTimeMillis();
final String reqContentType = req.getContentType();
final boolean isSmile = SmileMediaTypes.APPLICATION_JACKSON_SMILE.equals(reqContentType)
|| APPLICATION_SMILE.equals(reqContentType);
ObjectMapper objectMapper = isSmile ? smileMapper : jsonMapper;
final String contentType = isSmile ? SmileMediaTypes.APPLICATION_JACKSON_SMILE : MediaType.APPLICATION_JSON;
final ObjectWriter jsonWriter = pretty != null ? objectMapper.writerWithDefaultPrettyPrinter()
: objectMapper.writer();/*from ww w .j a v a 2 s. c o m*/
AtomCubeQuery _query = objectMapper.readValue(in, AtomCubeQuery.class);
final AtomCubeQuery atomQ = (AtomCubeQuery) _query.withId(UUID.randomUUID().toString());
final Map<String, Object> responseContext1 = new MapMaker().makeMap();
Sequence res = atomQ.run(this.conglomerate.findFactory(atomQ).createRunner(null), responseContext1);
final Sequence results;
if (res == null) {
results = Sequences.empty();
} else {
results = res;
}
final Yielder yielder = results.toYielder(null, new YieldingAccumulator() {
@Override
public Object accumulate(Object accumulated, Object in) {
yield();
return in;
}
});
final Map<String, Object> responseContext = new MapMaker().makeMap();
Response.ResponseBuilder builder = Response.ok(new StreamingOutput() {
@Override
public void write(OutputStream outputStream) throws IOException, WebApplicationException {
CountingOutputStream os = new CountingOutputStream(outputStream);
jsonWriter.writeValue(os, yielder);
os.flush();
os.close();
final long queryTime = System.currentTimeMillis() - start;
emitter.emit(DruidMetrics.makeQueryTimeMetric(null, jsonMapper, atomQ, req.getRemoteAddr())
.setDimension("success", "true").build("query/time", queryTime));
emitter.emit(DruidMetrics.makeQueryTimeMetric(null, jsonMapper, atomQ, req.getRemoteAddr())
.build("query/bytes", os.getCount()));
requestLogger.log(new RequestLogLine(new DateTime(), req.getRemoteAddr(), atomQ,
new QueryStats(ImmutableMap.<String, Object>of("query/time", queryTime, "query/bytes",
os.getCount(), "success", true))));
}
}, contentType).header("X-Druid-Query-Id", atomQ.getId());
String responseCtxString = jsonMapper.writeValueAsString(responseContext);
return builder.header("X-Druid-Response-Context", responseCtxString).build();
}
From source file:org.glowroot.agent.fat.storage.util.CappedDatabase.java
private long write(String type, Copier copier) throws IOException { synchronized (lock) { if (closing) { return -1; }// ww w . j ava 2 s . c om long startTick = ticker.read(); out.startBlock(); NonClosingCountingOutputStream countingStreamAfterCompression = new NonClosingCountingOutputStream(out); CountingOutputStream countingStreamBeforeCompression = new CountingOutputStream( new LZFOutputStream(countingStreamAfterCompression)); copier.copyTo(countingStreamBeforeCompression); countingStreamBeforeCompression.close(); long endTick = ticker.read(); CappedDatabaseStats stats = statsByType.get(type); if (stats == null) { stats = new CappedDatabaseStats(); statsByType.put(type, stats); } stats.record(countingStreamBeforeCompression.getCount(), countingStreamAfterCompression.getCount(), endTick - startTick); return out.endBlock(); } }
From source file:com.tinspx.util.net.MultipartBody.java
private long doCopyTo(OutputStream output) throws IOException { assert checkNotEmpty(); CountingOutputStream counter = new CountingOutputStream(output); counter.write(preamble);//from w w w.ja va2 s .com for (PartCache pc : partCache) { counter.write(pc.bytes); pc.part.copyTo(counter); } counter.write(CRLF); counter.write(DASHDASH); counter.write(boundaryBytes); counter.write(DASHDASH); counter.write(CRLF); counter.write(epilogue); return counter.getCount(); }
From source file:org.glowroot.agent.embedded.util.CappedDatabase.java
private long write(String type, Copier copier) throws IOException { long blockStartIndex; synchronized (lock) { if (closed) { return -1; }//from w w w . j a v a 2 s . c om long startTick = ticker.read(); out.startBlock(); NonClosingCountingOutputStream countingStreamAfterCompression = new NonClosingCountingOutputStream(out); CountingOutputStream countingStreamBeforeCompression = new CountingOutputStream( newLZFOutputStream(countingStreamAfterCompression)); copier.copyTo(countingStreamBeforeCompression); countingStreamBeforeCompression.close(); long endTick = ticker.read(); CappedDatabaseStats stats = statsByType.get(type); if (stats == null) { stats = new CappedDatabaseStats(); statsByType.put(type, stats); } stats.record(countingStreamBeforeCompression.getCount(), countingStreamAfterCompression.getCount(), endTick - startTick); blockStartIndex = out.endBlock(); } // fsync (if really needed here) does not need to be done under lock out.fsyncIfReallyNeeded(); return blockStartIndex; }
From source file:bear.fx.DownloadFxApp.java
public void createScene(Stage stage) { try {/* w w w . ja va2 s . c om*/ stage.setTitle("Downloading JDK " + version + "..."); instance.set(this); appStartedLatch.countDown(); final SimpleBrowser browser = SimpleBrowser.newBuilder().useFirebug(false).useJQuery(true) .createWebView(!miniMode).build(); final ProgressBar progressBar = new ProgressBar(0); final Label progressLabel = new Label("Retrieving a link..."); VBox vBox = VBoxBuilder.create().children(progressLabel, progressBar, browser).fillWidth(true).build(); Scene scene = new Scene(vBox); stage.setScene(scene); if (miniMode) { stage.setWidth(300); } else { stage.setWidth(1024); stage.setHeight(768); } stage.show(); VBox.setVgrow(browser, Priority.ALWAYS); /** * location changed to: http://www.oracle.com/technetwork/java/javase/downloads/java-archive-downloads-javase7-521261.html? location changed to: http://download.oracle.com/otn/java/jdk/7u45-b18/jdk-7u45-linux-x64.tar.gz location changed to: https://edelivery.oracle.com/akam/otn/java/jdk/7u45-b18/jdk-7u45-linux-x64.tar.gz location changed to: https://login.oracle.com/pls/orasso/orasso.wwsso_app_admin.ls_login?Site2pstoreToken=v1.2~CA55CD32~750C6EFBC9B3CB198B2ADFE87BDD4DEB60E0218858C8BFE85DCCC65865D0E810E845839B422974847E1D489D3AF25FDC9574400197F9190C389876C1EC683A6006A06F7F05D41C94455B8354559F5699F5D0EF102F26FE905E77D40487455F7829501E3A783E1354EB0B8F05B828D0FC3BA22C62D3576883850E0B99849309B0C26F286E5650F63E9C6A7C376165C9A3EED86BF2FA0FAEE3D1F7F2957F5FBD5035AF0A3522E534141FE38DFDD55C4F7F517F9E81336C993BB76512C0D30A5B5C5FD82ED1C10E9D27284B6B1633E4B7B9FA5C2E38D9C5E3845C18C009E294E881FD8B654B67050958E57F0DC20885D6FA87A59FAA7564F94F location changed to: https://login.oracle.com/mysso/signon.jsp location changed to: https://login.oracle.com/oam/server/sso/auth_cred_submit location changed to: https://edelivery.oracle.com/osso_login_success?urlc=v1.2%7E30E69346FE17F27D4F83121B0B8EC362E0B315901364AAA7D6F0B7A05CD8AA31802F5A69D70C708F34C64B65D233922B57D3C31839E82CE78E5C8DA55D729DD339893285D21A8E8B1AE8557C9240D6E33C9965956E136F4CB093779F97AF67C3DB8FF19FF2A638296BD0AA81A7801904AC5607F0568B6CEAF7ED9FCE4B7BEA80071617E4B2779F60F0C76A89F7D195965D2F003F9EDD2A1ADFD264C1C4C7F921010B08D3846CEC9524237A9337B6B0BC433BB17993A670B6C913EB4CFDC217A753F9E2943DE0CBDC41D4705AC67C2B96A4892C65F5450B146939B0EBFDF098680BBBE1F13356460C95A23D8D198D1C6762E45E62F120E32C2549E6263071DA84F8321370D2410CCA93E9A071A02ED6EB40BF40EDFC6F65AC7BA73CDB06DF4265455419D9185A6256FFE41A7FF54042374D09F5C720F3104B2EAC924778482D4BE855A45B2636CE91C7D947FF1F764674CE0E42FFCCFE411AABFE07EA0E96838AFEA263D2D5A405BD location changed to: https://edelivery.oracle.com/akam/otn/java/jdk/7u45-b18/jdk-7u45-linux-x64.tar.gz location changed to: http://download.oracle.com/otn/java/jdk/7u45-b18/jdk-7u45-linux-x64.tar.gz?AuthParam=1390405890_f9186a44471784229268632878dd89e4 */ browser.getEngine().locationProperty().addListener(new ChangeListener<String>() { @Override public void changed(ObservableValue<? extends String> observableValue, String oldLoc, final String uri) { logger.info("change: {}", uri); if (uri.contains("signon.jsp")) { browser.getEngine().executeScript( "" + "alert(document);\n" + "alert(document.getElementById('sso_username'));\n"); new Thread("signon.jsp waiter") { @Override public void run() { setStatus("waiting for the login form..."); try { Thread.sleep(1000); } catch (InterruptedException e) { throw Exceptions.runtime(e); } browser.waitFor("$('#sso_username').length > 0", 10000); System.out.println("I see it all, I see it now!"); Platform.runLater(new Runnable() { @Override public void run() { browser.getEngine().executeScript("" + "alert(document.getElementById('sso_username'));\n" + "alert($('#sso_username').val('" + oracleUser + "'));\n" + "alert($('#ssopassword').val('" + oraclePassword + "'));\n" + downloadJDKJs() + "\n" + "clickIt($('.sf-btnarea a'))"); } }); } }.start(); } if (uri.contains("download.oracle") && uri.contains("?")) { //will be here after // clicking accept license and link -> * not logged in * -> here -> download -> redirect to login // download -> fill form -> * logged in * -> here -> download Thread thread = new Thread(new Runnable() { @Override public void run() { try { DefaultHttpClient httpClient = new DefaultHttpClient(); HttpGet httppost = new HttpGet(uri); HttpResponse response = httpClient.execute(httppost); int code = response.getStatusLine().getStatusCode(); if (code != 200) { System.out.println(IOUtils.toString(response.getEntity().getContent())); throw new RuntimeException("failed to download: " + uri); } final File file = new File(tempDestDir, StringUtils.substringBefore(FilenameUtils.getName(uri), "?")); HttpEntity entity = response.getEntity(); final long length = entity.getContentLength(); final CountingOutputStream os = new CountingOutputStream( new FileOutputStream(file)); System.out.printf("Downloading %s to %s...%n", uri, file); Thread progressThread = new Thread(new Runnable() { double lastProgress; @Override public void run() { while (!Thread.currentThread().isInterrupted()) { long copied = os.getCount(); double progress = copied * 100D / length; if (progress != lastProgress) { final String s = String.format("%s: %s/%s %s%%", file.getName(), FileUtils.humanReadableByteCount(copied, false, false), FileUtils.humanReadableByteCount(length, false, true), LangUtils.toConciseString(progress, 1)); setStatus(s); System.out.print("\r" + s); } lastProgress = progress; progressBar.setProgress(copied * 1D / length); // progressProp.set(progress); try { Thread.sleep(500); } catch (InterruptedException e) { break; } } } }, "progressThread"); progressThread.start(); ByteStreams.copy(entity.getContent(), os); progressThread.interrupt(); System.out.println("Download complete."); downloadResult.set(new DownloadResult(file, "", true)); downloadLatch.countDown(); } catch (Exception e) { LoggerFactory.getLogger("log").warn("", e); downloadResult.set(new DownloadResult(null, e.getMessage(), false)); throw Exceptions.runtime(e); } } }, "fx-downloader"); thread.start(); } } public void setStatus(final String s) { Platform.runLater(new Runnable() { @Override public void run() { progressLabel.setText(s); } }); } }); // links from http://www.oracle.com/technetwork/java/archive-139210.html Map<Integer, String> archiveLinksMap = new HashMap<Integer, String>(); archiveLinksMap.put(5, "http://www.oracle.com/technetwork/java/javasebusiness/downloads/java-archive-downloads-javase5-419410.html"); archiveLinksMap.put(6, "http://www.oracle.com/technetwork/java/javase/downloads/java-archive-downloads-javase6-419409.html"); archiveLinksMap.put(7, "http://www.oracle.com/technetwork/java/javase/downloads/java-archive-downloads-javase7-521261.html"); Map<Integer, String> latestLinksMap = new HashMap<Integer, String>(); latestLinksMap.put(7, "http://www.oracle.com/technetwork/java/javase/downloads/jdk7-downloads-1880260.html"); String archiveUrl = null; String latestUrl = null; char ch = version.charAt(0); switch (ch) { case '7': case '6': case '5': latestUrl = latestLinksMap.get(ch - '0'); archiveUrl = archiveLinksMap.get(ch - '0'); break; default: archiveUrl = null; } if (latestUrl != null) { final String finalArchiveUrl = archiveUrl; tryFind(browser, latestUrl, new WhenDone() { @Override public void whenDone(boolean found) { tryArchiveLink(found, finalArchiveUrl, browser); } }); } else { tryArchiveLink(false, archiveUrl, browser); } } catch (Exception e) { e.printStackTrace(); } }
From source file:fx.DownloadFxApp.java
public void createScene(Stage stage) { try {/*from w w w . j a va2 s. c om*/ stage.setTitle("Downloading JDK " + version + "..."); instance.set(this); appStartedLatch.countDown(); final SimpleBrowser browser = SimpleBrowser.newBuilder().useFirebug(false).useJQuery(true) .createWebView(!miniMode).build(); final ProgressBar progressBar = new ProgressBar(0); final Label progressLabel = new Label("Retrieving a link..."); VBox vBox = VBoxBuilder.create().children(progressLabel, progressBar, browser).fillWidth(true).build(); Scene scene = new Scene(vBox); stage.setScene(scene); if (miniMode) { stage.setWidth(300); } else { stage.setWidth(1024); stage.setHeight(768); } stage.show(); VBox.setVgrow(browser, Priority.ALWAYS); /** The sequence of calls location changed to: http://www.oracle.com/technetwork/java/javase/downloads/java-archive-downloads-javase7-521261.html? location changed to: http://download.oracle.com/otn/java/jdk/7u45-b18/jdk-7u45-linux-x64.tar.gz location changed to: https://edelivery.oracle.com/akam/otn/java/jdk/7u45-b18/jdk-7u45-linux-x64.tar.gz location changed to: https://login.oracle.com/pls/orasso/orasso.wwsso_app_admin.ls_login?Site2pstoreToken=v1.2~CA55CD32~750C6EFBC9B3CB198B2ADFE87BDD4DEB60E0218858C8BFE85DCCC65865D0E810E845839B422974847E1D489D3AF25FDC9574400197F9190C389876C1EC683A6006A06F7F05D41C94455B8354559F5699F5D0EF102F26FE905E77D40487455F7829501E3A783E1354EB0B8F05B828D0FC3BA22C62D3576883850E0B99849309B0C26F286E5650F63E9C6A7C376165C9A3EED86BF2FA0FAEE3D1F7F2957F5FBD5035AF0A3522E534141FE38DFDD55C4F7F517F9E81336C993BB76512C0D30A5B5C5FD82ED1C10E9D27284B6B1633E4B7B9FA5C2E38D9C5E3845C18C009E294E881FD8B654B67050958E57F0DC20885D6FA87A59FAA7564F94F location changed to: https://login.oracle.com/mysso/signon.jsp location changed to: https://login.oracle.com/oam/server/sso/auth_cred_submit location changed to: https://edelivery.oracle.com/osso_login_success?urlc=v1.2%7E30E69346FE17F27D4F83121B0B8EC362E0B315901364AAA7D6F0B7A05CD8AA31802F5A69D70C708F34C64B65D233922B57D3C31839E82CE78E5C8DA55D729DD339893285D21A8E8B1AE8557C9240D6E33C9965956E136F4CB093779F97AF67C3DB8FF19FF2A638296BD0AA81A7801904AC5607F0568B6CEAF7ED9FCE4B7BEA80071617E4B2779F60F0C76A89F7D195965D2F003F9EDD2A1ADFD264C1C4C7F921010B08D3846CEC9524237A9337B6B0BC433BB17993A670B6C913EB4CFDC217A753F9E2943DE0CBDC41D4705AC67C2B96A4892C65F5450B146939B0EBFDF098680BBBE1F13356460C95A23D8D198D1C6762E45E62F120E32C2549E6263071DA84F8321370D2410CCA93E9A071A02ED6EB40BF40EDFC6F65AC7BA73CDB06DF4265455419D9185A6256FFE41A7FF54042374D09F5C720F3104B2EAC924778482D4BE855A45B2636CE91C7D947FF1F764674CE0E42FFCCFE411AABFE07EA0E96838AFEA263D2D5A405BD location changed to: https://edelivery.oracle.com/akam/otn/java/jdk/7u45-b18/jdk-7u45-linux-x64.tar.gz location changed to: http://download.oracle.com/otn/java/jdk/7u45-b18/jdk-7u45-linux-x64.tar.gz?AuthParam=1390405890_f9186a44471784229268632878dd89e4 */ browser.getEngine().locationProperty().addListener(new ChangeListener<String>() { @Override public void changed(ObservableValue<? extends String> observableValue, String oldLoc, final String uri) { logger.info("change: {}", uri); if (uri.contains("signon.jsp")) { browser.getEngine().executeScript( "" + "alert(document);\n" + "alert(document.getElementById('sso_username'));\n"); new Thread("signon.jsp waiter") { @Override public void run() { setStatus("waiting for the login form..."); try { Thread.sleep(1000); } catch (InterruptedException e) { throw Exceptions.runtime(e); } browser.waitFor("$('#sso_username').length > 0", 10000); System.out.println("I see it all, I see it now!"); Platform.runLater(new Runnable() { @Override public void run() { browser.getEngine().executeScript("" + "alert(document.getElementById('sso_username'));\n" + "alert($('#sso_username').val('" + oracleUser + "'));\n" + "alert($('#ssopassword').val('" + oraclePassword + "'));\n" + downloadJDKJs() + "\n" + "clickIt($('.sf-btnarea a'))"); } }); } }.start(); } if (uri.contains("download.oracle") && uri.contains("?")) { // will be here after // clicking accept license and link -> * not logged in * -> here -> download -> redirect to login // download -> fill form -> * logged in * -> here -> download Thread thread = new Thread(new Runnable() { @Override public void run() { try { DefaultHttpClient httpClient = new DefaultHttpClient(); HttpGet httppost = new HttpGet(uri); HttpResponse response = httpClient.execute(httppost); int code = response.getStatusLine().getStatusCode(); if (code != 200) { System.out.println(IOUtils.toString(response.getEntity().getContent())); throw new RuntimeException("failed to download: " + uri); } final File file = new File(tempDestDir, StringUtils.substringBefore(FilenameUtils.getName(uri), "?")); HttpEntity entity = response.getEntity(); final long length = entity.getContentLength(); final CountingOutputStream os = new CountingOutputStream( new FileOutputStream(file)); System.out.printf("Downloading %s to %s...%n", uri, file); Thread progressThread = new Thread(new Runnable() { double lastProgress; @Override public void run() { while (!Thread.currentThread().isInterrupted()) { long copied = os.getCount(); double progress = copied * 100D / length; if (progress != lastProgress) { final String s = String.format("%s: %s/%s %s%%", file.getName(), FileUtils.humanReadableByteCount(copied, false, false), FileUtils.humanReadableByteCount(length, false, true), LangUtils.toConciseString(progress, 1)); setStatus(s); System.out.print("\r" + s); } lastProgress = progress; progressBar.setProgress(copied * 1D / length); // progressProp.set(progress); try { Thread.sleep(500); } catch (InterruptedException e) { break; } } } }, "progressThread"); progressThread.start(); ByteStreams.copy(entity.getContent(), os); progressThread.interrupt(); System.out.println("Download complete."); downloadResult.set(new DownloadResult(file, "", true)); downloadLatch.countDown(); } catch (Exception e) { LoggerFactory.getLogger("log").warn("", e); downloadResult.set(new DownloadResult(null, e.getMessage(), false)); throw Exceptions.runtime(e); } } }, "fx-downloader"); thread.start(); } } public void setStatus(final String s) { Platform.runLater(new Runnable() { @Override public void run() { progressLabel.setText(s); } }); } }); // links from http://www.oracle.com/technetwork/java/archive-139210.html Map<Integer, String> archiveLinksMap = new HashMap<Integer, String>(); archiveLinksMap.put(5, "http://www.oracle.com/technetwork/java/javasebusiness/downloads/java-archive-downloads-javase5-419410.html"); archiveLinksMap.put(6, "http://www.oracle.com/technetwork/java/javase/downloads/java-archive-downloads-javase6-419409.html"); archiveLinksMap.put(7, "http://www.oracle.com/technetwork/java/javase/downloads/java-archive-downloads-javase7-521261.html"); Map<Integer, String> latestLinksMap = new HashMap<Integer, String>(); latestLinksMap.put(7, "http://www.oracle.com/technetwork/java/javase/downloads/jdk7-downloads-1880260.html"); String archiveUrl = null; String latestUrl = null; char ch = version.charAt(0); switch (ch) { case '7': case '6': case '5': latestUrl = latestLinksMap.get(ch - '0'); archiveUrl = archiveLinksMap.get(ch - '0'); break; default: archiveUrl = null; } if (latestUrl != null) { final String finalArchiveUrl = archiveUrl; tryFind(browser, latestUrl, new WhenDone() { @Override public void whenDone(boolean found) { tryArchiveLink(found, finalArchiveUrl, browser); } }); } else { tryArchiveLink(false, archiveUrl, browser); } } catch (Exception e) { e.printStackTrace(); } }
From source file:org.apache.druid.server.QueryResource.java
@POST
@Produces({ MediaType.APPLICATION_JSON, SmileMediaTypes.APPLICATION_JACKSON_SMILE })
@Consumes({ MediaType.APPLICATION_JSON, SmileMediaTypes.APPLICATION_JACKSON_SMILE, APPLICATION_SMILE })
public Response doPost(final InputStream in, @QueryParam("pretty") final String pretty,
@Context final HttpServletRequest req // used to get request content-type,Accept header, remote address and auth-related headers
) throws IOException {
final QueryLifecycle queryLifecycle = queryLifecycleFactory.factorize();
Query<?> query = null;// w w w . ja va 2 s. c om
String acceptHeader = req.getHeader("Accept");
if (Strings.isNullOrEmpty(acceptHeader)) {
//default to content-type
acceptHeader = req.getContentType();
}
final ResponseContext context = createContext(acceptHeader, pretty != null);
final String currThreadName = Thread.currentThread().getName();
try {
queryLifecycle.initialize(readQuery(req, in, context));
query = queryLifecycle.getQuery();
final String queryId = query.getId();
Thread.currentThread().setName(StringUtils.format("%s[%s_%s_%s]", currThreadName, query.getType(),
query.getDataSource().getNames(), queryId));
if (log.isDebugEnabled()) {
log.debug("Got query [%s]", query);
}
final Access authResult = queryLifecycle.authorize(req);
if (!authResult.isAllowed()) {
throw new ForbiddenException(authResult.toString());
}
final QueryLifecycle.QueryResponse queryResponse = queryLifecycle.execute();
final Sequence<?> results = queryResponse.getResults();
final Map<String, Object> responseContext = queryResponse.getResponseContext();
final String prevEtag = getPreviousEtag(req);
if (prevEtag != null && prevEtag.equals(responseContext.get(HEADER_ETAG))) {
return Response.notModified().build();
}
final Yielder<?> yielder = Yielders.each(results);
try {
boolean shouldFinalize = QueryContexts.isFinalize(query, true);
boolean serializeDateTimeAsLong = QueryContexts.isSerializeDateTimeAsLong(query, false)
|| (!shouldFinalize && QueryContexts.isSerializeDateTimeAsLongInner(query, false));
final ObjectWriter jsonWriter = context.newOutputWriter(serializeDateTimeAsLong);
Response.ResponseBuilder builder = Response.ok(new StreamingOutput() {
@Override
public void write(OutputStream outputStream) throws WebApplicationException {
Exception e = null;
CountingOutputStream os = new CountingOutputStream(outputStream);
try {
// json serializer will always close the yielder
jsonWriter.writeValue(os, yielder);
os.flush(); // Some types of OutputStream suppress flush errors in the .close() method.
os.close();
} catch (Exception ex) {
e = ex;
log.error(ex, "Unable to send query response.");
throw Throwables.propagate(ex);
} finally {
Thread.currentThread().setName(currThreadName);
queryLifecycle.emitLogsAndMetrics(e, req.getRemoteAddr(), os.getCount());
if (e == null) {
successfulQueryCount.incrementAndGet();
} else {
failedQueryCount.incrementAndGet();
}
}
}
}, context.getContentType()).header("X-Druid-Query-Id", queryId);
if (responseContext.get(HEADER_ETAG) != null) {
builder.header(HEADER_ETAG, responseContext.get(HEADER_ETAG));
responseContext.remove(HEADER_ETAG);
}
DirectDruidClient.removeMagicResponseContextFields(responseContext);
//Limit the response-context header, see https://github.com/apache/incubator-druid/issues/2331
//Note that Response.ResponseBuilder.header(String key,Object value).build() calls value.toString()
//and encodes the string using ASCII, so 1 char is = 1 byte
String responseCtxString = jsonMapper.writeValueAsString(responseContext);
if (responseCtxString.length() > RESPONSE_CTX_HEADER_LEN_LIMIT) {
log.warn("Response Context truncated for id [%s] . Full context is [%s].", queryId,
responseCtxString);
responseCtxString = responseCtxString.substring(0, RESPONSE_CTX_HEADER_LEN_LIMIT);
}
return builder.header("X-Druid-Response-Context", responseCtxString).build();
} catch (Exception e) {
// make sure to close yielder if anything happened before starting to serialize the response.
yielder.close();
throw Throwables.propagate(e);
} finally {
// do not close yielder here, since we do not want to close the yielder prior to
// StreamingOutput having iterated over all the results
}
} catch (QueryInterruptedException e) {
interruptedQueryCount.incrementAndGet();
queryLifecycle.emitLogsAndMetrics(e, req.getRemoteAddr(), -1);
return context.gotError(e);
} catch (ForbiddenException e) {
// don't do anything for an authorization failure, ForbiddenExceptionMapper will catch this later and
// send an error response if this is thrown.
throw e;
} catch (Exception e) {
failedQueryCount.incrementAndGet();
queryLifecycle.emitLogsAndMetrics(e, req.getRemoteAddr(), -1);
log.makeAlert(e, "Exception handling request").addData("exception", e.toString())
.addData("query", query != null ? query.toString() : "unparseable query")
.addData("peer", req.getRemoteAddr()).emit();
return context.gotError(e);
} finally {
Thread.currentThread().setName(currThreadName);
}
}
From source file:brooklyn.util.internal.ssh.sshj.SshjTool.java
/** * Executes the script in the background (`nohup ... &`), and then executes other ssh commands to poll for the * stdout, stderr and exit code of that original process (which will each have been written to separate files). * /*w ww .ja v a 2s . com*/ * The polling is a "long poll". That is, it executes a long-running ssh command to retrieve the stdout, etc. * If that long-poll command fails, then we just execute another one to pick up from where it left off. * This means we do not need to execute many ssh commands (which are expensive), but can still return promptly * when the command completes. * * Much of this was motivated by https://issues.apache.org/jira/browse/BROOKLYN-106, which is no longer * an issue. The retries (e.g. in the upload-script) are arguably overkill given that {@link #acquire(SshAction)} * will already retry. However, leaving this in place as it could prove useful when working with flakey * networks in the future. * * TODO There are (probably) issues with this method when using {@link ShellTool#PROP_RUN_AS_ROOT}. * I (Aled) saw the .pid file having an owner of root:root, and a failure message in stderr of: * -bash: line 3: /tmp/brooklyn-20150113-161203056-XMEo-move_install_dir_from_user_to_.pid: Permission denied */ protected int execScriptAsyncAndPoll(final Map<String, ?> props, final List<String> commands, final Map<String, ?> env) { return new ToolAbstractAsyncExecScript(props) { private int maxConsecutiveSshFailures = 3; private Duration maxDelayBetweenPolls = Duration.seconds(20); private Duration pollTimeout = getOptionalVal(props, PROP_EXEC_ASYNC_POLLING_TIMEOUT, Duration.FIVE_MINUTES); private int iteration = 0; private int consecutiveSshFailures = 0; private int stdoutCount = 0; private int stderrCount = 0; private Stopwatch timer; public int run() { timer = Stopwatch.createStarted(); final String scriptContents = toScript(props, commands, env); if (LOG.isTraceEnabled()) LOG.trace("Running shell command at {} as async script: {}", host, scriptContents); // Upload script; try repeatedly because have seen timeout intermittently on vcloud-director (BROOKLYN-106 related). boolean uploadSuccess = Repeater .create("async script upload on " + SshjTool.this.toString() + " (for " + getSummary() + ")") .backoffTo(maxDelayBetweenPolls).limitIterationsTo(3).rethrowException() .until(new Callable<Boolean>() { @Override public Boolean call() throws Exception { iteration++; if (LOG.isDebugEnabled()) { String msg = "Uploading (iteration=" + iteration + ") for async script on " + SshjTool.this.toString() + " (for " + getSummary() + ")"; if (iteration == 1) { LOG.trace(msg); } else { LOG.debug(msg); } } copyToServer(ImmutableMap.of("permissions", "0700"), scriptContents.getBytes(), scriptPath); return true; } }).run(); if (!uploadSuccess) { // Unexpected! Should have either returned true or have rethrown the exception; should never get false. String msg = "Unexpected state: repeated failure for async script upload on " + SshjTool.this.toString() + " (" + getSummary() + ")"; LOG.warn(msg + "; rethrowing"); throw new IllegalStateException(msg); } // Execute script asynchronously int execResult = asInt(acquire(new ShellAction(buildRunScriptCommand(), out, err, execTimeout)), -1); if (execResult != 0) return execResult; // Long polling to get the status try { final AtomicReference<Integer> result = new AtomicReference<Integer>(); boolean success = Repeater .create("async script long-poll on " + SshjTool.this.toString() + " (for " + getSummary() + ")") .backoffTo(maxDelayBetweenPolls).limitTimeTo(execTimeout) .until(new Callable<Boolean>() { @Override public Boolean call() throws Exception { iteration++; if (LOG.isDebugEnabled()) LOG.debug("Doing long-poll (iteration=" + iteration + ") for async script to complete on " + SshjTool.this.toString() + " (for " + getSummary() + ")"); Integer exitstatus = longPoll(); result.set(exitstatus); return exitstatus != null; } }).run(); if (!success) { // Timed out String msg = "Timeout for async script to complete on " + SshjTool.this.toString() + " (" + getSummary() + ")"; LOG.warn(msg + "; rethrowing"); throw new TimeoutException(msg); } return result.get(); } catch (Exception e) { LOG.debug("Problem polling for async script on " + SshjTool.this.toString() + " (for " + getSummary() + "); rethrowing after deleting temporary files", e); throw Exceptions.propagate(e); } finally { // Delete the temporary files created (and the `tail -c` commands that might have been left behind by long-polls). // Using pollTimeout so doesn't wait forever, but waits for a reasonable (configurable) length of time. // TODO also execute this if the `buildRunScriptCommand` fails, as that might have left files behind? try { int execDeleteResult = asInt( acquire(new ShellAction(deleteTemporaryFilesCommand(), out, err, pollTimeout)), -1); if (execDeleteResult != 0) { LOG.debug("Problem deleting temporary files of async script on " + SshjTool.this.toString() + " (for " + getSummary() + "): exit status " + execDeleteResult); } } catch (Exception e) { Exceptions.propagateIfFatal(e); LOG.debug("Problem deleting temporary files of async script on " + SshjTool.this.toString() + " (for " + getSummary() + "); continuing", e); } } } Integer longPoll() throws IOException { // Long-polling to get stdout, stderr + exit status of async task. // If our long-poll disconnects, we will just re-execute. // We wrap the stdout/stderr so that we can get the size count. // If we disconnect, we will pick up from that char of the stream. // TODO Additional stdout/stderr written by buildLongPollCommand() could interfere, // causing us to miss some characters. Duration nextPollTimeout = Duration.min(pollTimeout, Duration.millis(execTimeout.toMilliseconds() - timer.elapsed(TimeUnit.MILLISECONDS))); CountingOutputStream countingOut = (out == null) ? null : new CountingOutputStream(out); CountingOutputStream countingErr = (err == null) ? null : new CountingOutputStream(err); List<String> pollCommand = buildLongPollCommand(stdoutCount, stderrCount, nextPollTimeout); Duration sshJoinTimeout = nextPollTimeout.add(Duration.TEN_SECONDS); ShellAction action = new ShellAction(pollCommand, countingOut, countingErr, sshJoinTimeout); int longPollResult; try { longPollResult = asInt(acquire(action, 3, nextPollTimeout), -1); } catch (RuntimeTimeoutException e) { if (LOG.isDebugEnabled()) LOG.debug("Long-poll timed out on " + SshjTool.this.toString() + " (for " + getSummary() + "): " + e); return null; } stdoutCount += (countingOut == null) ? 0 : countingOut.getCount(); stderrCount += (countingErr == null) ? 0 : countingErr.getCount(); if (longPollResult == 0) { if (LOG.isDebugEnabled()) LOG.debug("Long-poll succeeded (exit status 0) on " + SshjTool.this.toString() + " (for " + getSummary() + ")"); return longPollResult; // success } else if (longPollResult == -1) { // probably a connection failure; try again if (LOG.isDebugEnabled()) LOG.debug("Long-poll received exit status -1; will retry on " + SshjTool.this.toString() + " (for " + getSummary() + ")"); return null; } else if (longPollResult == 125) { // 125 is the special code for timeout in long-poll (see buildLongPollCommand). // However, there is a tiny chance that the underlying command might have returned that exact exit code! // Don't treat a timeout as a "consecutiveSshFailure". if (LOG.isDebugEnabled()) LOG.debug("Long-poll received exit status " + longPollResult + "; most likely timeout; retrieving actual status on " + SshjTool.this.toString() + " (for " + getSummary() + ")"); return retrieveStatusCommand(); } else { // want to double-check whether this is the exit-code from the async process, or // some unexpected failure in our long-poll command. if (LOG.isDebugEnabled()) LOG.debug("Long-poll received exit status " + longPollResult + "; retrieving actual status on " + SshjTool.this.toString() + " (for " + getSummary() + ")"); Integer result = retrieveStatusCommand(); if (result != null) { return result; } } consecutiveSshFailures++; if (consecutiveSshFailures > maxConsecutiveSshFailures) { LOG.warn("Aborting on " + consecutiveSshFailures + " consecutive ssh connection errors (return -1) when polling for async script to complete on " + SshjTool.this.toString() + " (" + getSummary() + ")"); return -1; } else { LOG.info("Retrying after ssh connection error when polling for async script to complete on " + SshjTool.this.toString() + " (" + getSummary() + ")"); return null; } } Integer retrieveStatusCommand() throws IOException { // want to double-check whether this is the exit-code from the async process, or // some unexpected failure in our long-poll command. ByteArrayOutputStream statusOut = new ByteArrayOutputStream(); ByteArrayOutputStream statusErr = new ByteArrayOutputStream(); int statusResult = asInt( acquire(new ShellAction(buildRetrieveStatusCommand(), statusOut, statusErr, execTimeout)), -1); if (statusResult == 0) { // The status we retrieved really is valid; return it. // TODO How to ensure no additional output in stdout/stderr when parsing below? String statusOutStr = new String(statusOut.toByteArray()).trim(); if (Strings.isEmpty(statusOutStr)) { // suggests not yet completed; will retry with long-poll if (LOG.isDebugEnabled()) LOG.debug( "Long-poll retrieved status directly; command successful but no result available on " + SshjTool.this.toString() + " (for " + getSummary() + ")"); return null; } else { if (LOG.isDebugEnabled()) LOG.debug("Long-poll retrieved status directly; returning '" + statusOutStr + "' on " + SshjTool.this.toString() + " (for " + getSummary() + ")"); int result = Integer.parseInt(statusOutStr); return result; } } else if (statusResult == -1) { // probably a connection failure; try again with long-poll if (LOG.isDebugEnabled()) LOG.debug("Long-poll retrieving status directly received exit status -1; will retry on " + SshjTool.this.toString() + " (for " + getSummary() + ")"); return null; } else { if (out != null) { out.write(toUTF8ByteArray( "retrieving status failed with exit code " + statusResult + " (stdout follow)")); out.write(statusOut.toByteArray()); } if (err != null) { err.write(toUTF8ByteArray( "retrieving status failed with exit code " + statusResult + " (stderr follow)")); err.write(statusErr.toByteArray()); } if (LOG.isDebugEnabled()) LOG.debug("Long-poll retrieving status failed; returning " + statusResult + " on " + SshjTool.this.toString() + " (for " + getSummary() + ")"); return statusResult; } } }.run(); }