List of usage examples for org.apache.http.client.config RequestConfig custom
public static RequestConfig.Builder custom()
From source file:com.clonephpscrapper.utility.FetchPageWithProxy.java
public static String fetchPage(URI newurl) throws IOException, InterruptedException { int portNo = generateRandomPort(); CredentialsProvider credsprovider = new BasicCredentialsProvider(); credsprovider.setCredentials(new AuthScope("195.154.161.103", portNo), new UsernamePasswordCredentials("mongoose", "Fjh30fi")); HttpHost proxy = new HttpHost("195.154.161.103", portNo); //----------------------------------------------------------------------- RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(5000).setConnectTimeout(5000) .setConnectionRequestTimeout(5000).build(); CloseableHttpClient httpclient = HttpClients.custom().setDefaultCredentialsProvider(credsprovider) .setUserAgent("Mozilla/5.0 (Windows NT 6.1; WOW64; rv:35.0) Gecko/20100101 Firefox/35.0") .setDefaultRequestConfig(requestConfig).setProxy(proxy).build(); String responsebody = ""; String responsestatus = null; int count = 0; try {/*from w ww .j a va 2 s. com*/ HttpGet httpget = new HttpGet(newurl); httpget.addHeader("GET ", "http://clonephp.com/"); httpget.addHeader("Host", "clonephp.com"); httpget.addHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"); httpget.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.106 Safari/537.36"); httpget.addHeader("Accept-Encoding", "gzip, deflate, sdch"); httpget.addHeader("Accept-Language", "en-US,en;q=0.8"); httpget.addHeader("Connection", "keep-alive"); // httpget.addHeader("Cookie", "_ym_uid=14523228509491377; _gat=1; _ym_isad=0; _ga=GA1.2.438893798.1452322850; _ym_visorc_26745633=b; last-featured-visit-time=1452323357912\n" + //"If-None-Match: W/\"10d86-1fKww4Zqud7NjVeuSjZKEw\""); System.out.println("Response status " + httpget.getRequestLine()); CloseableHttpResponse resp = httpclient.execute(httpget); responsestatus = resp.getStatusLine().toString(); if (responsestatus.contains("503") || responsestatus.contains("502") || responsestatus.contains("403") || responsestatus.contains("400") || responsestatus.contains("407") || responsestatus.contains("401") || responsestatus.contains("402") || responsestatus.contains("404") || responsestatus.contains("405") || responsestatus.contains("SSLHandshakeException") || responsestatus.contains("999") || responsestatus.contains("ClientProtocolException") || responsestatus.contains("SocketTimeoutException") || "".equals(responsestatus)) { Thread.sleep(10000); do { count++; responsebody = fetchPageSourcefromClientGoogleSecond(newurl); if (responsebody == null) { Thread.sleep(10000); System.out.println("PROX FAILURE"); } if (count > 20) { Thread.sleep(1000); break; } } while (responsebody == null || "".equals(responsebody)); } else { HttpEntity entity = resp.getEntity(); System.out.println(resp.getStatusLine()); if (entity != null) { System.out.println("Response content length: " + entity.getContentLength()); BufferedReader in = new BufferedReader(new InputStreamReader(entity.getContent())); String inputLine; while ((inputLine = in.readLine()) != null) { responsebody = new StringBuilder().append(responsebody).append(inputLine).toString(); } // writeResponseFile(responsebody, pagename); } EntityUtils.consume(entity); } } catch (IOException | IllegalStateException e) { System.out.println("Exception = " + e); do { count++; responsebody = fetchPageSourcefromClientGoogleSecond(newurl); if (responsebody == null) { System.out.println("PROX FAILURE"); } if (count > 15) { Thread.sleep(50000); // responsebody = fetchPageSourcefromClientGoogleSecond(newurl); break; } } while (responsebody == null || "".equals(responsebody)); } finally { httpclient.close(); } return responsebody; }
From source file:com.adobe.acs.commons.http.impl.HttpClientFactoryImpl.java
@Activate protected void activate(Map<String, Object> config) throws Exception { boolean useSSL = PropertiesUtil.toBoolean(config.get(PROP_USE_SSL), DEFAULT_USE_SSL); String scheme = useSSL ? "https" : "http"; String hostname = PropertiesUtil.toString(config.get(PROP_HOST_DOMAIN), null); int port = PropertiesUtil.toInteger(config.get(PROP_GATEWAY_PORT), 0); if (hostname == null || port == 0) { throw new IllegalArgumentException("Configuration not valid. Both host and port must be provided."); }//w w w .j a v a 2 s .co m baseUrl = String.format("%s://%s:%s", scheme, hostname, port); int connectTimeout = PropertiesUtil.toInteger(config.get(PROP_CONNECT_TIMEOUT), DEFAULT_CONNECT_TIMEOUT); int soTimeout = PropertiesUtil.toInteger(config.get(PROP_SO_TIMEOUT), DEFAULT_SOCKET_TIMEOUT); HttpClientBuilder builder = httpClientBuilderFactory.newBuilder(); RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(connectTimeout) .setSocketTimeout(soTimeout).build(); builder.setDefaultRequestConfig(requestConfig); boolean disableCertCheck = PropertiesUtil.toBoolean(config.get(PROP_DISABLE_CERT_CHECK), DEFAULT_DISABLE_CERT_CHECK); if (useSSL && disableCertCheck) { SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, new TrustStrategy() { public boolean isTrusted(X509Certificate[] arg0, String arg1) throws CertificateException { return true; } }).build(); builder.setHostnameVerifier(new AllowAllHostnameVerifier()).setSslcontext(sslContext); } httpClient = builder.build(); executor = Executor.newInstance(httpClient); String username = PropertiesUtil.toString(config.get(PROP_USERNAME), null); String password = PropertiesUtil.toString(config.get(PROP_PASSWORD), null); if (username != null && password != null) { HttpHost httpHost = new HttpHost(hostname, port, useSSL ? "https" : "http"); executor.auth(httpHost, username, password).authPreemptive(httpHost); } }
From source file:org.archive.wayback.liveweb.StdRemoteLiveWebCache.java
/** * Gets resource object from the live web. Configure timeout to 10 seconds. * /*from w w w .j av a2 s . c om*/ * @param url to fetch from the live web. * @param maxCacheMS maximum age of resource to return - optionally honored * @param bUseOlder if true, return documents older than maxCacheMS if * a more recent copy is not available. * * @return Resource for url * * @throws LiveDocumentNotAvailableException if the resource cannot be * retrieved from the live web, but all proxying and caching * mechanisms functioned properly * @throws LiveWebCacheUnavailableException if there was a problem either * accessing the live web, in proxying to the live web, or in * maintaining the cache for the live web * @throws LiveWebTimeoutException if there is no response from the live * web cache before a timeout occurred. * @throws IOException for the usual reasons * * @see org.archive.wayback.liveweb.LiveWebCache#getCachedResource(java.net.URL, long, boolean) * @inheritDoc org.archive.wayback.liveweb.LiveWebCache#getCachedResource */ @Override public Resource getCachedResource(URL url, long maxCacheMS, boolean bUseOlder) throws LiveDocumentNotAvailableException, LiveWebCacheUnavailableException, LiveWebTimeoutException, IOException { String urlStr = url.toExternalForm(); if (requestPrefix != null) urlStr = requestPrefix + urlStr; HttpHost proxy = new HttpHost(hostConfiguration.getProxyHost(), hostConfiguration.getProxyPort()); // Set socketTimeout and connectionTimeout to 10 seconds. RequestConfig reqConf = RequestConfig.custom().setProxy(proxy).setSocketTimeout(10000) .setConnectTimeout(10000).setConnectionRequestTimeout(10000).build(); CloseableHttpClient httpclient = HttpClients.custom().setDefaultRequestConfig(reqConf).build(); HttpGet httpGet = new HttpGet(urlStr); try { // The following line gets robots.txt from live web response = httpclient.execute(httpGet); String httpHeaderStr = ""; String bodyStr = ""; /* If it fails to get robots.txt (http status code is 404), then display contents and don't throw exception (socketTimeOutException or connectTimeOutException) */ if (response.getStatusLine().getStatusCode() == 404) { httpHeaderStr = "HTTP/1.0 200 OK\n"; bodyStr = String.format("%s\n%s\n", "User-agent: *", "Allow: /"); } else if (response.getStatusLine().getStatusCode() == 200) { // The following line represents first line in http header httpHeaderStr = String.format("%s %d %s\n", response.getStatusLine().getProtocolVersion(), response.getStatusLine().getStatusCode(), response.getStatusLine().getReasonPhrase()); // Get robots.txt contents and store it into bodyStr HttpEntity entity = response.getEntity(); bodyStr = EntityUtils.toString(entity); } // Get Http Header and store complete http header in httpHeaderStr for (Header header : response.getAllHeaders()) httpHeaderStr += header.toString() + "\n"; httpHeaderStr += "\n"; int length = httpHeaderStr.length() + bodyStr.length(); /* Using httpHeaderStr and bodyStr to construct responseStr. First line in responseStr should exist. */ // TODO: the following line should be enhanced, // especially the first line in responseStr. String responseStr = String.format("%s %s %d\n%s%s", urlStr, "0.0.0.0 10000000000000 text/plain", length, httpHeaderStr, bodyStr); ByteArrayInputStream bais = new ByteArrayInputStream(responseStr.getBytes()); // TODO: Should not use ARCRecord ARCRecord r = new ARCRecord(bais, "id", 0L, false, false, true); ar = (ArcResource) ResourceFactory.ARCArchiveRecordToResource(r, null); if (ar.getStatusCode() == 502) { throw new LiveDocumentNotAvailableException(urlStr); } else if (ar.getStatusCode() == 504) { throw new LiveWebTimeoutException("Timeout:" + urlStr); } return ar; } catch (ResourceNotAvailableException e) { throw new LiveDocumentNotAvailableException(urlStr); } catch (NoHttpResponseException e) { throw new LiveWebCacheUnavailableException("No Http Response for " + urlStr); } catch (ConnectException e) { throw new LiveWebCacheUnavailableException(e.getLocalizedMessage() + " : " + urlStr); } catch (SocketException e) { throw new LiveWebCacheUnavailableException(e.getLocalizedMessage() + " : " + urlStr); } catch (SocketTimeoutException e) { throw new LiveWebTimeoutException(e.getLocalizedMessage() + " : " + urlStr); } catch (ConnectTimeoutException e) { throw new LiveWebTimeoutException(e.getLocalizedMessage() + " : " + urlStr); } finally { response.close(); } }
From source file:com.moviejukebox.tools.YamjHttpClientBuilder.java
@SuppressWarnings("resource") private static YamjHttpClient buildHttpClient() { LOG.trace("Create new YAMJ http client"); // create proxy HttpHost proxy = null;/*from w ww .j av a 2 s . c om*/ CredentialsProvider credentialsProvider = null; if (StringUtils.isNotBlank(PROXY_HOST) && PROXY_PORT > 0) { proxy = new HttpHost(PROXY_HOST, PROXY_PORT); if (StringUtils.isNotBlank(PROXY_USERNAME) && StringUtils.isNotBlank(PROXY_PASSWORD)) { credentialsProvider = new BasicCredentialsProvider(); credentialsProvider.setCredentials(new AuthScope(PROXY_HOST, PROXY_PORT), new UsernamePasswordCredentials(PROXY_USERNAME, PROXY_PASSWORD)); } } PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(); connManager.setDefaultSocketConfig(SocketConfig.custom().setSoTimeout(TIMEOUT_SOCKET).build()); connManager.setMaxTotal(20); connManager.setDefaultMaxPerRoute(2); CacheConfig cacheConfig = CacheConfig.custom().setMaxCacheEntries(1000).setMaxObjectSize(8192).build(); HttpClientBuilder builder = CachingHttpClientBuilder.create().setCacheConfig(cacheConfig) .setConnectionManager(connManager).setProxy(proxy) .setDefaultCredentialsProvider(credentialsProvider) .setDefaultRequestConfig(RequestConfig.custom().setConnectionRequestTimeout(TIMEOUT_READ) .setConnectTimeout(TIMEOUT_CONNECT).setSocketTimeout(TIMEOUT_SOCKET) .setCookieSpec(CookieSpecs.IGNORE_COOKIES).setProxy(proxy).build()); // show status showStatus(); // build the client YamjHttpClient wrapper = new YamjHttpClient(builder.build(), connManager); wrapper.setUserAgentSelector(new WebBrowserUserAgentSelector()); wrapper.addGroupLimit(".*", 1); // default limit, can be overwritten // First we have to read/create the rules String maxDownloadSlots = PropertiesUtil.getProperty("mjb.MaxDownloadSlots"); if (StringUtils.isNotBlank(maxDownloadSlots)) { LOG.debug("Using download limits: {}", maxDownloadSlots); Pattern pattern = Pattern.compile(",?\\s*([^=]+)=(\\d+)"); Matcher matcher = pattern.matcher(maxDownloadSlots); while (matcher.find()) { String group = matcher.group(1); try { final Integer maxResults = Integer.valueOf(matcher.group(2)); wrapper.addGroupLimit(group, maxResults); LOG.trace("Added download slot '{}' with max results {}", group, maxResults); } catch (NumberFormatException error) { LOG.debug("Rule '{}' is no valid regexp, ignored", group); } } } return wrapper; }
From source file:net.shibboleth.idp.cas.proxy.impl.HttpClientProxyAuthenticator.java
@Override protected int authenticateProxyCallback(@Nonnull final URI callbackUri, @Nullable final TrustEngine<? super X509Credential> x509TrustEngine) throws GeneralSecurityException { CloseableHttpClient httpClient = null; CloseableHttpResponse response = null; try {// w w w . j av a 2 s .com httpClient = createHttpClient(x509TrustEngine); log.debug("Attempting to connect to {}", callbackUri); final HttpGet request = new HttpGet(callbackUri); request.setConfig(RequestConfig.custom().setConnectTimeout(timeout).setSocketTimeout(timeout).build()); response = httpClient.execute(request); return response.getStatusLine().getStatusCode(); } catch (ClientProtocolException e) { throw new GeneralSecurityException("HTTP protocol error", e); } catch (SSLException e) { if (e.getCause() instanceof CertificateException) { throw (CertificateException) e.getCause(); } throw new GeneralSecurityException("SSL connection error", e); } catch (IOException e) { throw new GeneralSecurityException("IO error", e); } finally { close(response); close(httpClient); } }
From source file:org.springframework.cloud.netflix.ribbon.apache.RibbonApacheHttpRequestTests.java
void testEntity(String entityValue, ByteArrayInputStream requestEntity, boolean addContentLengthHeader, String method) throws IOException { String lengthString = String.valueOf(entityValue.length()); Long length = null;//from ww w . ja v a2s. com URI uri = URI.create("http://example.com"); LinkedMultiValueMap<String, String> headers = new LinkedMultiValueMap<>(); if (addContentLengthHeader) { headers.add("Content-Length", lengthString); length = (long) entityValue.length(); } RibbonRequestCustomizer requestCustomizer = new RibbonRequestCustomizer<RequestBuilder>() { @Override public boolean accepts(Class builderClass) { return builderClass == RequestBuilder.class; } @Override public void customize(RequestBuilder builder) { builder.addHeader("from-customizer", "foo"); } }; RibbonCommandContext context = new RibbonCommandContext("example", method, uri.toString(), false, headers, new LinkedMultiValueMap<String, String>(), requestEntity, Collections.singletonList(requestCustomizer)); context.setContentLength(length); RibbonApacheHttpRequest httpRequest = new RibbonApacheHttpRequest(context); HttpUriRequest request = httpRequest.toRequest(RequestConfig.custom().build()); assertThat("request is wrong type", request, is(instanceOf(HttpEntityEnclosingRequest.class))); assertThat("uri is wrong", request.getURI().toString(), startsWith(uri.toString())); if (addContentLengthHeader) { assertThat("Content-Length is missing", request.getFirstHeader("Content-Length"), is(notNullValue())); assertThat("Content-Length is wrong", request.getFirstHeader("Content-Length").getValue(), is(equalTo(lengthString))); } assertThat("from-customizer is missing", request.getFirstHeader("from-customizer"), is(notNullValue())); assertThat("from-customizer is wrong", request.getFirstHeader("from-customizer").getValue(), is(equalTo("foo"))); HttpEntityEnclosingRequest entityRequest = (HttpEntityEnclosingRequest) request; assertThat("entity is missing", entityRequest.getEntity(), is(notNullValue())); HttpEntity entity = entityRequest.getEntity(); assertThat("contentLength is wrong", entity.getContentLength(), is(equalTo((long) entityValue.length()))); assertThat("content is missing", entity.getContent(), is(notNullValue())); String string = StreamUtils.copyToString(entity.getContent(), Charset.forName("UTF-8")); assertThat("content is wrong", string, is(equalTo(entityValue))); }
From source file:com.floragunn.searchguard.HeaderAwareJestClientFactory.java
protected RequestConfig createRequestConfig() { return RequestConfig.custom().setConnectionRequestTimeout(httpClientConfig.getConnTimeout()) .setSocketTimeout(httpClientConfig.getReadTimeout()).build(); }
From source file:io.vertx.tempmail.impl.TempMailClientImpl.java
private void doGenericRequest(String url, Handler<AsyncResult<JsonObject>> handler) { vertx.executeBlocking(future -> { HttpGet httpGet = new HttpGet(url); if (options.getProxy() != null) { RequestConfig response = RequestConfig.custom().setProxy(options.getProxy()).build(); httpGet.setConfig(response); }//from w ww . j a v a2 s . co m try { CloseableHttpResponse response = httpclient.execute(httpGet); if (response.getStatusLine().getStatusCode() == 200) { HttpEntity ht = response.getEntity(); BufferedHttpEntity buf = new BufferedHttpEntity(ht); String responseContent = EntityUtils.toString(buf, "UTF-8"); JsonArray jsonArray = new JsonArray(responseContent); JsonObject jsonObject = new JsonObject(); jsonObject.put("result", jsonArray); future.complete(jsonObject); } } catch (Exception e) { log.warn(e, e); future.fail(e); } }, res -> { handler.handle(new AsyncResult<JsonObject>() { @Override public JsonObject result() { return res.succeeded() ? (JsonObject) res.result() : null; } @Override public Throwable cause() { return res.succeeded() ? null : res.cause(); } @Override public boolean succeeded() { return res.succeeded(); } @Override public boolean failed() { return res.failed(); } }); }); }
From source file:co.tuzza.swipehq.transport.ManagedHttpTransport.java
private RequestConfig getRequestConfig() { RequestConfig.Builder builder = RequestConfig.custom(); builder.setSocketTimeout(socketTimeout); builder.setConnectTimeout(connectionTimeout); return builder.build(); }
From source file:com.github.matthesrieke.realty.CrawlerServlet.java
@Override public void init() throws ServletException { super.init(); this.listTemplate = Util.parseStream(getClass().getResourceAsStream("index-template.html")); this.groupTemplate = Util.parseStream(getClass().getResourceAsStream("group-template.html")); this.properties = new Properties(); InputStream is = getClass().getResourceAsStream("/config.properties"); if (is == null) { is = getClass().getResourceAsStream("/config.properties.default"); }/* ww w . ja v a2 s. c o m*/ try { this.properties.load(is); } catch (IOException e1) { logger.warn(e1.getMessage(), e1); throw new ServletException("Could not init properties!", e1); } initializeCrawlers(); readCrawlingLinks(); String preferredDatabaseLocation = properties.getProperty("DATABASE_DIR"); storage = new H2Storage(preferredDatabaseLocation); this.timer = new Timer(); Integer crawlPeriod = Util.getIntegerProperty(this.properties, "crawlPeriodHours", 6); logger.info(String.format("Scheduling crawl every %s hours.", crawlPeriod)); this.timer.scheduleAtFixedRate(new TimerTask() { @Override public void run() { logger.info("Starting to parse ad entries..."); DateTime now = new DateTime(); int insertedCount = 0; int crawlerCount = 0; try { for (String baseLink : crawlLinks) { logger.info("Baselink = " + baseLink); Crawler crawler; try { crawler = resolveCrawler(baseLink); } catch (UnsupportedBaseLinkException e1) { logger.warn(e1.getMessage(), e1); break; } crawlerCount++; String link; int page = crawler.getFirstPageIndex(); while (true) { Thread.sleep(1000); logger.info("Parsing page " + page); link = crawler.prepareLinkForPage(baseLink, page); HttpGet get = new HttpGet(link); page++; CloseableHttpClient client = HttpClientBuilder.create().setDefaultRequestConfig( RequestConfig.custom().setConnectTimeout(20000).build()).build(); try { CloseableHttpResponse resp = client.execute(get); if (resp.getStatusLine().getStatusCode() < HttpStatus.SC_MULTIPLE_CHOICES) { InputStream content = resp.getEntity().getContent(); List<Ad> items = parse(content, crawler); if (items == null || items.size() == 0) { break; } insertedCount += compareAndStoreItems(items, now); } else { break; } } catch (IOException | CrawlerException e) { logger.warn(e.getMessage(), e); Metadata md = new Metadata("Exception during crawl: " + e.getMessage()); storage.updateMetadata(md); break; } } logger.info("finished parsing ad entries!"); } Metadata md = new Metadata(String.format("Added %s new entries. %s crawlers have been used", insertedCount, crawlerCount)); storage.updateMetadata(md); } catch (RuntimeException | InterruptedException e) { logger.warn(e.getMessage(), e); Metadata md = new Metadata("Exception during crawl: " + e.getMessage()); storage.updateMetadata(md); } } }, 0, 1000 * 60 * 60 * crawlPeriod); }