List of usage examples for org.apache.http.protocol HttpCoreContext HttpCoreContext
public HttpCoreContext()
From source file:org.ballerinalang.composer.service.tryit.service.HttpTryItClient.java
/** * {@inheritDoc}// ww w . j a v a2s .com */ @Override public String execute() throws TryItException { try { HttpClient client = HttpClientBuilder.create().build(); HttpCoreContext localContext = new HttpCoreContext(); // Create url for the request. String requestUrl = this.buildUrl(); String httpMethod = this.clientArgs.get(TryItConstants.HTTP_METHOD).getAsString(); switch (httpMethod.toLowerCase(Locale.getDefault())) { case "get": case "delete": case "options": case "head": case "trace": HttpRequestBase httpRequest = new TryItHttpRequestBase(httpMethod.toUpperCase(Locale.getDefault())); // Setting the url for the request. httpRequest.setURI(URI.create(requestUrl)); // Setting the request headers. JsonArray getHeaders = this.clientArgs.getAsJsonArray(TryItConstants.REQUEST_HEADERS); for (JsonElement getHeader : getHeaders) { JsonObject header = getHeader.getAsJsonObject(); if (StringUtils.isNotBlank(header.get("key").getAsString()) && StringUtils.isNotBlank(header.get("value").getAsString())) { httpRequest.setHeader(header.get("key").getAsString(), header.get("value").getAsString()); } } // Setting the content type. if (StringUtils.isBlank(this.clientArgs.get(TryItConstants.CONTENT_TYPE).getAsString())) { httpRequest.setHeader(HttpHeaders.CONTENT_TYPE, ContentType.TEXT_PLAIN.getMimeType()); } else { httpRequest.setHeader(HttpHeaders.CONTENT_TYPE, this.clientArgs.get(TryItConstants.CONTENT_TYPE).getAsString()); } // To track how long the request took. long start = System.currentTimeMillis(); // Executing the request. HttpResponse response = client.execute(httpRequest, localContext); long elapsed = System.currentTimeMillis() - start; return jsonStringifyResponse(response, localContext.getRequest().getAllHeaders(), requestUrl, elapsed); default: HttpEntityEnclosingRequestBase httpEntityRequest = new TryItHttpEntityEnclosingRequestBase( httpMethod.toUpperCase(Locale.getDefault())); // Setting the url for the request. httpEntityRequest.setURI(URI.create(requestUrl)); // Setting the request headers. JsonArray getEntityHeaders = this.clientArgs.getAsJsonArray(TryItConstants.REQUEST_HEADERS); for (JsonElement getHeader : getEntityHeaders) { JsonObject header = getHeader.getAsJsonObject(); if (StringUtils.isNotBlank(header.get("key").getAsString()) && StringUtils.isNotBlank(header.get("value").getAsString())) { httpEntityRequest.setHeader(header.get("key").getAsString(), header.get("value").getAsString()); } } // Setting the body of the request. StringEntity postEntity = new StringEntity( this.clientArgs.get(TryItConstants.REQUEST_BODY).getAsString()); httpEntityRequest.setEntity(postEntity); // Setting the content type. if (StringUtils.isBlank(this.clientArgs.get(TryItConstants.CONTENT_TYPE).getAsString())) { httpEntityRequest.setHeader(HttpHeaders.CONTENT_TYPE, ContentType.TEXT_PLAIN.getMimeType()); } else { httpEntityRequest.setHeader(HttpHeaders.CONTENT_TYPE, this.clientArgs.get(TryItConstants.CONTENT_TYPE).getAsString()); } // To track how long the request took. long entityRequestStart = System.currentTimeMillis(); // Executing the request. HttpResponse entityResponse = client.execute(httpEntityRequest, localContext); long entityRequestDuration = System.currentTimeMillis() - entityRequestStart; return jsonStringifyResponse(entityResponse, localContext.getRequest().getAllHeaders(), requestUrl, entityRequestDuration); } } catch (IOException e) { throw new TryItException(e.getMessage()); } }
From source file:org.ballerinalang.composer.service.workspace.tryit.HttpTryItClient.java
/** * {@inheritDoc}// w w w. jav a 2 s . c om */ @Override public String execute() throws TryItException { try { HttpClient client = HttpClientBuilder.create().build(); HttpCoreContext localContext = new HttpCoreContext(); // Create url for the request. String requestUrl = this.buildUrl(); String httpMethod = this.clientArgs.get(TryItConstants.HTTP_METHOD).getAsString(); switch (httpMethod.toLowerCase(Locale.getDefault())) { case "get": case "delete": case "options": case "head": case "trace": HttpRequestBase httpRequest = new TryItHttpRequestBase(httpMethod.toUpperCase(Locale.getDefault())); // Setting the url for the request. httpRequest.setURI(URI.create(requestUrl)); // Setting the request headers. JsonArray getHeaders = this.clientArgs.getAsJsonArray(TryItConstants.REQUEST_HEADERS); for (JsonElement getHeader : getHeaders) { JsonObject header = getHeader.getAsJsonObject(); if (StringUtils.isNotBlank(header.get("key").getAsString()) && StringUtils.isNotBlank(header.get("value").getAsString())) { httpRequest.setHeader(header.get("key").getAsString(), header.get("value").getAsString()); } } // Setting the content type. if (StringUtils.isBlank(this.clientArgs.get(TryItConstants.CONTENT_TYPE).getAsString())) { httpRequest.setHeader(HttpHeaders.CONTENT_TYPE, ContentType.TEXT_PLAIN.getMimeType()); } else { httpRequest.setHeader(HttpHeaders.CONTENT_TYPE, this.clientArgs.get(TryItConstants.CONTENT_TYPE).getAsString()); } // To track how long the request took. long start = System.currentTimeMillis(); // Executing the request. HttpResponse response = client.execute(httpRequest, localContext); long elapsed = System.currentTimeMillis() - start; return jsonStringifyResponse(response, localContext.getRequest().getAllHeaders(), requestUrl, elapsed); default: HttpEntityEnclosingRequestBase httpEntityRequest = new TryItHttpEntityEnclosingRequestBase( httpMethod.toUpperCase(Locale.getDefault())); // Setting the url for the request. httpEntityRequest.setURI(URI.create(requestUrl)); // Setting the request headers. JsonArray getEntityHeaders = this.clientArgs.getAsJsonArray(TryItConstants.REQUEST_HEADERS); for (JsonElement getHeader : getEntityHeaders) { JsonObject header = getHeader.getAsJsonObject(); if (StringUtils.isNotBlank(header.get("key").getAsString()) && StringUtils.isNotBlank(header.get("value").getAsString())) { httpEntityRequest.setHeader(header.get("key").getAsString(), header.get("value").getAsString()); } } // Setting the body of the request. StringEntity postEntity = new StringEntity( this.clientArgs.get(TryItConstants.REQUEST_BODY).getAsString()); httpEntityRequest.setEntity(postEntity); // Setting the content type. if (StringUtils.isBlank(this.clientArgs.get(TryItConstants.CONTENT_TYPE).getAsString())) { httpEntityRequest.setHeader(HttpHeaders.CONTENT_TYPE, ContentType.TEXT_PLAIN.getMimeType()); } else { httpEntityRequest.setHeader(HttpHeaders.CONTENT_TYPE, this.clientArgs.get(TryItConstants.CONTENT_TYPE).getAsString()); } // To track how long the request took. long entityRequestStart = System.currentTimeMillis(); // Executing the request. HttpResponse entityResponse = client.execute(httpEntityRequest, localContext); long entityRequestDuration = System.currentTimeMillis() - entityRequestStart; return jsonStringifyResponse(entityResponse, localContext.getRequest().getAllHeaders(), requestUrl, entityRequestDuration); } } catch (java.io.IOException e) { throw new TryItException(e.getMessage()); } }
From source file:org.tallison.cc.CCGetter.java
private void fetch(CCIndexRecord r, Path rootDir, BufferedWriter writer) throws IOException { Path targFile = rootDir.resolve(r.getDigest().substring(0, 2) + "/" + r.getDigest()); if (Files.isRegularFile(targFile)) { writeStatus(r, FETCH_STATUS.ALREADY_IN_REPOSITORY, writer); logger.info("already retrieved:" + targFile.toAbsolutePath()); return;//from w w w .ja v a 2 s .c om } String url = AWS_BASE + r.getFilename(); URI uri = null; try { uri = new URI(url); } catch (URISyntaxException e) { logger.warn("Bad url: " + url); writeStatus(r, FETCH_STATUS.BAD_URL, writer); return; } CloseableHttpClient httpClient = HttpClients.createDefault(); HttpHost target = new HttpHost(uri.getHost()); String urlPath = uri.getRawPath(); if (uri.getRawQuery() != null) { urlPath += "?" + uri.getRawQuery(); } HttpGet httpGet = null; try { httpGet = new HttpGet(urlPath); } catch (Exception e) { logger.warn("bad path " + uri.toString(), e); writeStatus(r, FETCH_STATUS.BAD_URL, writer); return; } if (proxyHost != null && proxyPort > -1) { HttpHost proxy = new HttpHost(proxyHost, proxyPort, "http"); RequestConfig requestConfig = RequestConfig.custom().setProxy(proxy).build(); httpGet.setConfig(requestConfig); } httpGet.addHeader("Range", r.getOffsetHeader()); HttpCoreContext coreContext = new HttpCoreContext(); CloseableHttpResponse httpResponse = null; URI lastURI = null; try { httpResponse = httpClient.execute(target, httpGet, coreContext); RedirectLocations redirectLocations = (RedirectLocations) coreContext .getAttribute(DefaultRedirectStrategy.REDIRECT_LOCATIONS); if (redirectLocations != null) { for (URI redirectURI : redirectLocations.getAll()) { lastURI = redirectURI; } } else { lastURI = httpGet.getURI(); } } catch (IOException e) { logger.warn("IOException for " + uri.toString(), e); writeStatus(r, FETCH_STATUS.FETCHED_IO_EXCEPTION, writer); return; } lastURI = uri.resolve(lastURI); if (httpResponse.getStatusLine().getStatusCode() != 200 && httpResponse.getStatusLine().getStatusCode() != 206) { logger.warn("Bad status for " + uri.toString() + " : " + httpResponse.getStatusLine().getStatusCode()); writeStatus(r, FETCH_STATUS.FETCHED_NOT_200, writer); return; } Path tmp = null; Header[] headers = null; boolean isTruncated = false; try { //this among other parts is plagiarized from centic9's CommonCrawlDocumentDownload //probably saved me hours. Thank you, Dominik! tmp = Files.createTempFile("cc-getter", ""); try (InputStream is = new GZIPInputStream(httpResponse.getEntity().getContent())) { WARCRecord warcRecord = new WARCRecord(new FastBufferedInputStream(is), "", 0); ArchiveRecordHeader archiveRecordHeader = warcRecord.getHeader(); if (archiveRecordHeader.getHeaderFields().containsKey(WARCConstants.HEADER_KEY_TRUNCATED)) { isTruncated = true; } headers = LaxHttpParser.parseHeaders(warcRecord, "UTF-8"); Files.copy(warcRecord, tmp, StandardCopyOption.REPLACE_EXISTING); } } catch (IOException e) { writeStatus(r, null, headers, 0L, isTruncated, FETCH_STATUS.FETCHED_IO_EXCEPTION_READING_ENTITY, writer); deleteTmp(tmp); return; } String digest = null; long tmpLength = 0l; try (InputStream is = Files.newInputStream(tmp)) { digest = base32.encodeAsString(DigestUtils.sha1(is)); tmpLength = Files.size(tmp); } catch (IOException e) { writeStatus(r, null, headers, tmpLength, isTruncated, FETCH_STATUS.FETCHED_IO_EXCEPTION_SHA1, writer); logger.warn("IOException during digesting: " + tmp.toAbsolutePath()); deleteTmp(tmp); return; } if (Files.exists(targFile)) { writeStatus(r, digest, headers, tmpLength, isTruncated, FETCH_STATUS.ALREADY_IN_REPOSITORY, writer); deleteTmp(tmp); return; } try { Files.createDirectories(targFile.getParent()); Files.copy(tmp, targFile); } catch (IOException e) { writeStatus(r, digest, headers, tmpLength, isTruncated, FETCH_STATUS.FETCHED_EXCEPTION_COPYING_TO_REPOSITORY, writer); deleteTmp(tmp); } writeStatus(r, digest, headers, tmpLength, isTruncated, FETCH_STATUS.ADDED_TO_REPOSITORY, writer); deleteTmp(tmp); }