List of usage examples for org.apache.http.client.methods RequestBuilder setConfig
public RequestBuilder setConfig(final RequestConfig config)
From source file:com.linkedin.pinot.common.utils.FileUploadDownloadClient.java
private static void setTimeout(RequestBuilder requestBuilder, int socketTimeoutMs) { RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(socketTimeoutMs).build(); requestBuilder.setConfig(requestConfig); }
From source file:com.aliyun.api.gateway.demo.Client.java
/** * ??// w ww .j av a 2s .c o m * * @param request * requestnull{@link NullPointerException} * @return HttpResponse HTTP? * @throws IOException * HTTP? * @throws ClientProtocolException * ???? */ public HttpResponse execute(Request request) throws ClientProtocolException, IOException { if (request == null) { return null; } RequestBuilder requestBuilder = request.getMethod().requestbuilder().setUri(request.getUrl().toString()); requestBuilder .setConfig(RequestConfig.custom().setConnectTimeout(getTimeout(request.getTimeout())).build()); requestBuilder.addHeader(SystemHeader.X_CA_TIMESTAMP, String.valueOf(System.currentTimeMillis())); requestBuilder.addHeader(SystemHeader.X_CA_NONCE, UUID.randomUUID().toString()); requestBuilder.addHeader(SystemHeader.X_CA_KEY, appKey); if (testEnv) { requestBuilder.addHeader(SystemHeader.X_CA_STAGE, "test"); } requestBuilder.build().getAllHeaders(); initialBasicHeader(requestBuilder, request.getHeaders(), request.getUrl(), request.getFormBody(), request.getSignHeaderPrefixes()); HttpEntity entity = getEntity(request); if (entity != null) { requestBuilder.setEntity(entity); } return httpClient.execute(requestBuilder.build()); }
From source file:feign.httpclient.ApacheHttpClient.java
HttpUriRequest toHttpUriRequest(Request request, Request.Options options)
throws UnsupportedEncodingException, MalformedURLException, URISyntaxException {
RequestBuilder requestBuilder = RequestBuilder.create(request.method());
//per request timeouts
RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(options.connectTimeoutMillis())
.setSocketTimeout(options.readTimeoutMillis()).build();
requestBuilder.setConfig(requestConfig);
URI uri = new URIBuilder(request.url()).build();
requestBuilder.setUri(uri.getScheme() + "://" + uri.getAuthority() + uri.getRawPath());
//request query params
List<NameValuePair> queryParams = URLEncodedUtils.parse(uri, requestBuilder.getCharset().name());
for (NameValuePair queryParam : queryParams) {
requestBuilder.addParameter(queryParam);
}/*from ww w .j ava 2 s .com*/
//request headers
boolean hasAcceptHeader = false;
for (Map.Entry<String, Collection<String>> headerEntry : request.headers().entrySet()) {
String headerName = headerEntry.getKey();
if (headerName.equalsIgnoreCase(ACCEPT_HEADER_NAME)) {
hasAcceptHeader = true;
}
if (headerName.equalsIgnoreCase(Util.CONTENT_LENGTH)) {
// The 'Content-Length' header is always set by the Apache client and it
// doesn't like us to set it as well.
continue;
}
for (String headerValue : headerEntry.getValue()) {
requestBuilder.addHeader(headerName, headerValue);
}
}
//some servers choke on the default accept string, so we'll set it to anything
if (!hasAcceptHeader) {
requestBuilder.addHeader(ACCEPT_HEADER_NAME, "*/*");
}
//request body
if (request.body() != null) {
HttpEntity entity = null;
if (request.charset() != null) {
ContentType contentType = getContentType(request);
String content = new String(request.body(), request.charset());
entity = new StringEntity(content, contentType);
} else {
entity = new ByteArrayEntity(request.body());
}
requestBuilder.setEntity(entity);
}
return requestBuilder.build();
}
From source file:org.daybreak.coccinella.webmagic.ImageDownloader.java
@Override public Page download(Request request, Task task) { Site site = null;// ww w . j a va 2s .co m if (task != null) { site = task.getSite(); } Set<Integer> acceptStatCode; String charset = null; Map<String, String> headers = null; if (site != null) { acceptStatCode = site.getAcceptStatCode(); charset = site.getCharset(); headers = site.getHeaders(); } else { acceptStatCode = Sets.newHashSet(200); } logger.info("downloading image " + request.getUrl()); RequestBuilder requestBuilder; if (request instanceof CrawlerRequest) { CrawlerRequest crawlerRequest = (CrawlerRequest) request; try { requestBuilder = RequestBuilder.post().setUri(crawlerRequest.getUrl()) .setEntity(crawlerRequest.createEntity()); } catch (UnsupportedEncodingException ex) { logger.warn("The encoding is not supported: " + crawlerRequest.getCrawler().getEncode()); return null; } } else { requestBuilder = RequestBuilder.get().setUri(request.getUrl()); } if (headers != null) { for (Map.Entry<String, String> headerEntry : headers.entrySet()) { requestBuilder.addHeader(headerEntry.getKey(), headerEntry.getValue()); } } RequestConfig.Builder requestConfigBuilder = RequestConfig.custom() .setConnectionRequestTimeout(site.getTimeOut()).setSocketTimeout(site.getTimeOut()) .setConnectTimeout(site.getTimeOut()).setCookieSpec(CookieSpecs.BEST_MATCH); if (site != null && site.getHttpProxy() != null) { requestConfigBuilder.setProxy(site.getHttpProxy()); } requestBuilder.setConfig(requestConfigBuilder.build()); CloseableHttpResponse httpResponse = null; try { httpResponse = getHttpClient(site).execute(requestBuilder.build()); int statusCode = httpResponse.getStatusLine().getStatusCode(); if (acceptStatCode.contains(statusCode)) { ImagePage imagePage = new ImagePage(ImageIO.read(httpResponse.getEntity().getContent())); imagePage.setRawText(""); imagePage.setUrl(new PlainText(request.getUrl())); imagePage.setRequest(request); imagePage.setStatusCode(httpResponse.getStatusLine().getStatusCode()); return imagePage; } else { logger.warn("code error " + statusCode + "\t" + request.getUrl()); return null; } } catch (IOException e) { logger.warn("download image " + request.getUrl() + " error", e); if (site.getCycleRetryTimes() > 0) { return addToCycleRetry(request, site); } return null; } finally { try { if (httpResponse != null) { //ensure the connection is released back to pool EntityUtils.consume(httpResponse.getEntity()); } } catch (IOException e) { logger.warn("close response fail", e); } } }
From source file:org.springframework.cloud.netflix.ribbon.apache.RibbonApacheHttpRequest.java
public HttpUriRequest toRequest(final RequestConfig requestConfig) { final RequestBuilder builder = RequestBuilder.create(this.context.getMethod()); builder.setUri(this.uri); for (final String name : this.context.getHeaders().keySet()) { final List<String> values = this.context.getHeaders().get(name); for (final String value : values) { builder.addHeader(name, value); }/*from www . java 2 s . c o m*/ } for (final String name : this.context.getParams().keySet()) { final List<String> values = this.context.getParams().get(name); for (final String value : values) { builder.addParameter(name, value); } } if (this.context.getRequestEntity() != null) { final BasicHttpEntity entity; entity = new BasicHttpEntity(); entity.setContent(this.context.getRequestEntity()); // if the entity contentLength isn't set, transfer-encoding will be set // to chunked in org.apache.http.protocol.RequestContent. See gh-1042 if (this.context.getContentLength() != null) { entity.setContentLength(this.context.getContentLength()); } else if ("GET".equals(this.context.getMethod())) { entity.setContentLength(0); } builder.setEntity(entity); } customize(this.context.getRequestCustomizers(), builder); builder.setConfig(requestConfig); return builder.build(); }
From source file:nya.miku.wishmaster.http.streamer.HttpStreamer.java
/** * HTTP ? ?. ? ? ? ?, release() !/* w w w .j a v a 2 s .c o m*/ * ? ? ?? ? ? If-Modified-Since, ? * ? ?? ? Modified ({@link #removeFromModifiedMap(String)})! * @param url ? ? * @param requestModel ? ( null, GET If-Modified) * @param httpClient HTTP , ?? ? * @param listener ? ?? ?? ( null) * @param task , ( null) * @return ? HTTP * @throws HttpRequestException ?, ? ? ? */ public HttpResponseModel getFromUrl(String url, HttpRequestModel requestModel, HttpClient httpClient, ProgressListener listener, CancellableTask task) throws HttpRequestException { if (requestModel == null) requestModel = HttpRequestModel.builder().setGET().build(); // Request HttpUriRequest request = null; try { RequestConfig requestConfigBuilder = ExtendedHttpClient .getDefaultRequestConfigBuilder(requestModel.timeoutValue) .setRedirectsEnabled(!requestModel.noRedirect).build(); RequestBuilder requestBuilder = null; switch (requestModel.method) { case HttpRequestModel.METHOD_GET: requestBuilder = RequestBuilder.get().setUri(url); break; case HttpRequestModel.METHOD_POST: requestBuilder = RequestBuilder.post().setUri(url).setEntity(requestModel.postEntity); break; default: throw new IllegalArgumentException("Incorrect type of HTTP Request"); } if (requestModel.customHeaders != null) { for (Header header : requestModel.customHeaders) { requestBuilder.addHeader(header); } } if (requestModel.checkIfModified && requestModel.method == HttpRequestModel.METHOD_GET) { synchronized (ifModifiedMap) { if (ifModifiedMap.containsKey(url)) { requestBuilder .addHeader(new BasicHeader(HttpHeaders.IF_MODIFIED_SINCE, ifModifiedMap.get(url))); } } } request = requestBuilder.setConfig(requestConfigBuilder).build(); } catch (Exception e) { Logger.e(TAG, e); HttpResponseModel.release(request, null); throw new IllegalArgumentException(e); } // ? HttpResponseModel responseModel = new HttpResponseModel(); HttpResponse response = null; try { IOException responseException = null; for (int i = 0; i < 5; ++i) { try { if (task != null && task.isCancelled()) throw new InterruptedException(); response = httpClient.execute(request); responseException = null; break; } catch (IOException e) { Logger.e(TAG, e); responseException = e; if (e.getMessage() == null) break; if (e.getMessage().indexOf("Connection reset by peer") != -1 || e.getMessage().indexOf("I/O error during system call, Broken pipe") != -1) { continue; } else { break; } } } if (responseException != null) { throw responseException; } if (task != null && task.isCancelled()) throw new InterruptedException(); // ???? HTTP StatusLine status = response.getStatusLine(); responseModel.statusCode = status.getStatusCode(); responseModel.statusReason = status.getReasonPhrase(); // (headers) String lastModifiedValue = null; if (responseModel.statusCode == 200) { Header header = response.getFirstHeader(HttpHeaders.LAST_MODIFIED); if (header != null) lastModifiedValue = header.getValue(); } Header header = response.getFirstHeader(HttpHeaders.LOCATION); if (header != null) responseModel.locationHeader = header.getValue(); responseModel.headers = response.getAllHeaders(); // HttpEntity responseEntity = response.getEntity(); if (responseEntity != null) { responseModel.contentLength = responseEntity.getContentLength(); if (listener != null) listener.setMaxValue(responseModel.contentLength); InputStream stream = responseEntity.getContent(); responseModel.stream = IOUtils.modifyInputStream(stream, listener, task); } responseModel.request = request; responseModel.response = response; if (lastModifiedValue != null) { synchronized (ifModifiedMap) { ifModifiedMap.put(url, lastModifiedValue); } } } catch (Exception e) { Logger.e(TAG, e); HttpResponseModel.release(request, response); throw new HttpRequestException(e); } return responseModel; }
From source file:cn.wanghaomiao.seimi.http.hc.HcRequestGenerator.java
public static RequestBuilder getHttpRequestBuilder(Request request, CrawlerModel crawlerModel) { RequestBuilder requestBuilder; BaseSeimiCrawler crawler = crawlerModel.getInstance(); if (request.isUseSeimiAgent()) { if (StringUtils.isBlank(crawler.seimiAgentHost())) { throw new SeimiProcessExcepiton("SeimiAgentHost is blank."); }//from www . ja v a 2s .co m String seimiAgentUrl = "http://" + crawler.seimiAgentHost() + (crawler.seimiAgentPort() != 80 ? (":" + crawler.seimiAgentPort()) : "") + "/doload"; requestBuilder = RequestBuilder.post().setUri(seimiAgentUrl); requestBuilder.addParameter("url", request.getUrl()); if (StringUtils.isNotBlank(crawler.proxy())) { requestBuilder.addParameter("proxy", crawler.proxy()); } if (request.getSeimiAgentRenderTime() > 0) { requestBuilder.addParameter("renderTime", String.valueOf(request.getSeimiAgentRenderTime())); } if (StringUtils.isNotBlank(request.getSeimiAgentScript())) { requestBuilder.addParameter("script", request.getSeimiAgentScript()); } //SeimiAgent?cookie if ((request.isSeimiAgentUseCookie() == null && crawlerModel.isUseCookie()) || (request.isSeimiAgentUseCookie() != null && request.isSeimiAgentUseCookie())) { requestBuilder.addParameter("useCookie", "1"); } if (request.getParams() != null && request.getParams().size() > 0) { requestBuilder.addParameter("postParam", JSON.toJSONString(request.getParams())); } if (request.getSeimiAgentContentType().val() > SeimiAgentContentType.HTML.val()) { requestBuilder.addParameter("contentType", request.getSeimiAgentContentType().typeVal()); } } else { if (HttpMethod.POST.equals(request.getHttpMethod())) { requestBuilder = RequestBuilder.post().setUri(request.getUrl()); } else { requestBuilder = RequestBuilder.get().setUri(request.getUrl()); } RequestConfig config = RequestConfig.custom().setProxy(crawlerModel.getProxy()) .setCircularRedirectsAllowed(true).build(); if (request.getParams() != null) { for (Map.Entry<String, String> entry : request.getParams().entrySet()) { requestBuilder.addParameter(entry.getKey(), entry.getValue()); } } requestBuilder.setConfig(config).setHeader("User-Agent", crawlerModel.isUseCookie() ? crawlerModel.getCurrentUA() : crawler.getUserAgent()); requestBuilder.setHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"); requestBuilder.setHeader("Accept-Language", "zh-CN,zh;q=0.8,en;q=0.6"); } if (!CollectionUtils.isEmpty(request.getHeader())) { for (Map.Entry<String, String> entry : request.getHeader().entrySet()) { requestBuilder.setHeader(entry.getKey(), entry.getValue()); } } return requestBuilder; }
From source file:org.bonitasoft.connectors.rest.RESTConnector.java
/** * Execute a given request/*from w w w.jav a 2s.c om*/ * * @param request The request to execute * @return The response of the executed request * @throws Exception any exception that might occur */ public void execute(final Request request) throws Exception { CloseableHttpClient httpClient = null; try { final URL url = request.getUrl(); final String urlHost = url.getHost(); final Builder requestConfigurationBuilder = RequestConfig.custom(); requestConfigurationBuilder.setConnectionRequestTimeout(CONNECTION_TIMEOUT); requestConfigurationBuilder.setRedirectsEnabled(request.isRedirect()); final HttpClientBuilder httpClientBuilder = HttpClientBuilder.create(); httpClientBuilder.setRetryHandler(new DefaultHttpRequestRetryHandler(0, false)); setSSL(request.getSsl(), httpClientBuilder); setProxy(request.getProxy(), httpClientBuilder, requestConfigurationBuilder); setCookies(requestConfigurationBuilder, httpClientBuilder, request.getCookies(), urlHost); final RequestBuilder requestBuilder = getRequestBuilderFromMethod(request.getRestMethod()); requestBuilder.setVersion( new ProtocolVersion(HTTP_PROTOCOL, HTTP_PROTOCOL_VERSION_MAJOR, HTTP_PROTOCOL_VERSION_MINOR)); int urlPort = url.getPort(); if (url.getPort() == -1) { urlPort = url.getDefaultPort(); } final String urlProtocol = url.getProtocol(); final String urlStr = url.toString(); requestBuilder.setUri(urlStr); setHeaders(requestBuilder, request.getHeaders()); if (!HTTPMethod.GET.equals(HTTPMethod.valueOf(requestBuilder.getMethod()))) { final String body = request.getBody(); if (body != null) { requestBuilder.setEntity(new StringEntity(request.getBody(), request.getContentType())); } } final HttpContext httpContext = setAuthorizations(requestConfigurationBuilder, request.getAuthorization(), request.getProxy(), urlHost, urlPort, urlProtocol, httpClientBuilder); requestBuilder.setConfig(requestConfigurationBuilder.build()); httpClientBuilder.setDefaultRequestConfig(requestConfigurationBuilder.build()); final HttpUriRequest httpRequest = requestBuilder.build(); httpClient = httpClientBuilder.build(); final CloseableHttpResponse httpResponse = httpClient.execute(httpRequest, httpContext); LOGGER.fine("Request sent."); setOutputs(httpResponse, request); } finally { try { if (httpClient != null) { httpClient.close(); } } catch (final IOException ex) { logException(ex); } } }
From source file:org.daybreak.coccinella.webmagic.CrawlerDownloader.java
@Override public Page download(Request request, Task task) { Site site = null;/*w w w .ja v a 2 s .c o m*/ if (task != null) { site = task.getSite(); } Set<Integer> acceptStatCode; String charset = null; Map<String, String> headers = null; if (site != null) { acceptStatCode = site.getAcceptStatCode(); charset = site.getCharset(); headers = site.getHeaders(); } else { acceptStatCode = Sets.newHashSet(200); } logger.info("downloading page " + request.getUrl()); RequestBuilder requestBuilder = null; if (request instanceof CrawlerRequest) { CrawlerRequest crawlerRequest = (CrawlerRequest) request; if (StringUtils.isNotBlank(crawlerRequest.getCrawler().getReferer())) { site.addHeader(HttpHeaders.REFERER, crawlerRequest.getCrawler().getReferer()); } if (crawlerRequest.getCrawler().getMethod() == HttpMethod.GET) { requestBuilder = RequestBuilder.get().setUri(request.getUrl()); } else if (crawlerRequest.getCrawler().getMethod() == HttpMethod.POST) { try { requestBuilder = RequestBuilder.post().setUri(crawlerRequest.getUrl()) .setEntity(crawlerRequest.createEntity()); } catch (UnsupportedEncodingException ex) { logger.warn("The encoding is not supported: " + crawlerRequest.getCrawler().getEncode()); return null; } } } if (requestBuilder == null) { return null; } if (headers != null) { for (Map.Entry<String, String> headerEntry : headers.entrySet()) { requestBuilder.addHeader(headerEntry.getKey(), headerEntry.getValue()); } } RequestConfig.Builder requestConfigBuilder = RequestConfig.custom() .setConnectionRequestTimeout(site.getTimeOut()).setSocketTimeout(site.getTimeOut()) .setConnectTimeout(site.getTimeOut()).setCookieSpec(CookieSpecs.BEST_MATCH); if (site != null && site.getHttpProxy() != null) { requestConfigBuilder.setProxy(site.getHttpProxy()); } requestBuilder.setConfig(requestConfigBuilder.build()); CloseableHttpResponse httpResponse = null; try { httpResponse = getHttpClient(site).execute(requestBuilder.build()); int statusCode = httpResponse.getStatusLine().getStatusCode(); if (acceptStatCode.contains(statusCode)) { // ?? Header[] resHeaders = httpResponse.getAllHeaders(); for (int i = 0; i < resHeaders.length; i++) { if (resHeaders[i].getName().equals("Set-Cookie")) { String cookie = resHeaders[i].getValue(); String cookieName = cookie.split("=")[0]; String cookieValue = cookie.split("=")[1].split(";")[0]; site.addCookie(cookieName, cookieValue); } } //charset if (charset == null) { String value = httpResponse.getEntity().getContentType().getValue(); charset = UrlUtils.getCharset(value); } return handleResponse(request, charset, httpResponse, task); } else { logger.warn("code error " + statusCode + "\t" + request.getUrl()); return null; } } catch (IOException e) { logger.warn("download page " + request.getUrl() + " error", e); if (site.getCycleRetryTimes() > 0) { return addToCycleRetry(request, site); } return null; } finally { try { if (httpResponse != null) { //ensure the connection is released back to pool EntityUtils.consume(httpResponse.getEntity()); } } catch (IOException e) { logger.warn("close response fail", e); } } }
From source file:com.wudaosoft.net.httpclient.Request.java
/** * // w w w . j a v a 2 s. com * @param workerBuilder * @param responseHandler * @return * @throws Exception */ public <T> T doRequest(WorkerBuilder workerBuilder, ResponseHandler<T> responseHandler) throws Exception { String method = workerBuilder.getMethod(); String url = workerBuilder.getUrl(); Args.notNull(workerBuilder, "WorkerBuilder"); Args.notEmpty(method, "WorkerBuilder.getMethod()"); Args.notEmpty(url, "WorkerBuilder.getUrl()"); Args.notNull(responseHandler, "responseHandler"); // if(!workerBuilder.isAnyHost()) { if (!isFullUrl(url)) { // notFullUrl(url); Args.notEmpty(hostConfig.getHostUrl(), "HostConfig.getHostUrl()"); url = hostConfig.getHostUrl() + url; } Charset charset = hostConfig.getCharset() == null ? Consts.UTF_8 : hostConfig.getCharset(); String stringBody = workerBuilder.getStringBody(); File fileBody = workerBuilder.getFileBody(); InputStream streamBody = workerBuilder.getStreamBody(); Map<String, String> params = workerBuilder.getParameters(); String contentType = null; if (responseHandler instanceof JsonResponseHandler) { contentType = MediaType.APPLICATION_JSON_VALUE; } else if (responseHandler instanceof SAXSourceResponseHandler || responseHandler instanceof XmlResponseHandler) { contentType = MediaType.APPLICATION_XML_VALUE; } else if (responseHandler instanceof FileResponseHandler || responseHandler instanceof ImageResponseHandler || responseHandler instanceof OutputStreamResponseHandler) { contentType = MediaType.ALL_VALUE; } else if (responseHandler instanceof NoResultResponseHandler) { contentType = ((NoResultResponseHandler) responseHandler).getContentType().getMimeType(); } else { contentType = MediaType.TEXT_PLAIN_VALUE; } RequestBuilder requestBuilder = RequestBuilder.create(method).setCharset(charset).setUri(url); if (stringBody != null) { StringEntity reqEntity = new StringEntity(stringBody, charset); reqEntity.setContentType(contentType + ";charset=" + charset.name()); requestBuilder.setEntity(reqEntity); } else if (fileBody != null || streamBody != null) { String filename = workerBuilder.getFilename(); MultipartEntityBuilder reqEntity = MultipartEntityBuilder.create().setLaxMode(); if (fileBody != null) { Args.check(fileBody.isFile(), "fileBody must be a file"); Args.check(fileBody.canRead(), "fileBody must be readable"); if (filename == null && streamBody == null) filename = fileBody.getName(); FileBody bin = new FileBody(fileBody, ContentType.APPLICATION_OCTET_STREAM, streamBody != null ? fileBody.getName() : filename); reqEntity.addPart(workerBuilder.getFileFieldName(), bin); } Args.notEmpty(filename, "filename"); if (streamBody != null) reqEntity.addBinaryBody(workerBuilder.getFileFieldName(), streamBody, ContentType.APPLICATION_OCTET_STREAM, filename); buildParameters(reqEntity, params, charset); requestBuilder.setEntity(reqEntity.build()); } if (fileBody == null && streamBody == null) { buildParameters(requestBuilder, params); } if (workerBuilder.getReadTimeout() > -1) { requestBuilder.setConfig(RequestConfig.copy(this.hostConfig.getRequestConfig()) .setSocketTimeout(workerBuilder.getReadTimeout()).build()); } HttpUriRequest httpRequest = ParameterRequestBuilder.build(requestBuilder); setAcceptHeader(httpRequest, contentType); if (workerBuilder.isAjax()) setAjaxHeader(httpRequest); HttpClientContext context = workerBuilder.getContext(); if (context == null) context = defaultHttpContext; T result = getHttpClient().execute(httpRequest, responseHandler, context); if (log.isDebugEnabled()) { log.debug(String.format("Send data to path:[%s]\"%s\". result: %s", method, url, result)); } return result; }