Example usage for org.apache.http.client.config RequestConfig custom

List of usage examples for org.apache.http.client.config RequestConfig custom

  1. HOME
  2. Java
  3. org.apache.http
  4. org.apache.http.client.config.*
  5. RequestConfig
  6. custom

Introduction

In this page you can find the example usage for org.apache.http.client.config RequestConfig custom.

Prototype

public static RequestConfig.Builder custom()

Source Link

Usage

From source file:justdailscrapper.vik.utility.FetchPageWithoutProxy.java

public static String fetchPageSourcefromClientGoogle(URI newurl, List<ProxyImport> proxyList)
        throws IOException, InterruptedException {

    Random r = new Random();

    ProxyImport obj = proxyList.get(r.nextInt(proxyList.size()));

    String ip = obj.proxyIP;/*from   w w  w. ja va2s.  c om*/
    int portno = Integer.parseInt(obj.proxyPort);
    String username = "";
    String password = "";
    if (obj.proxyLen > 2) {
        username = obj.proxyUserName;
        password = obj.proxyPassword;
    }

    //        int portNo = generateRandomPort();
    CredentialsProvider credsprovider = new BasicCredentialsProvider();
    credsprovider.setCredentials(new AuthScope(ip, portno),
            new UsernamePasswordCredentials(username, password));
    HttpHost proxy = new HttpHost(ip, portno);
    //-----------------------------------------------------------------------
    RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(5000).setConnectTimeout(5000)
            .setConnectionRequestTimeout(5000).build();

    CloseableHttpClient httpclient = HttpClients.custom().setDefaultCredentialsProvider(credsprovider)
            .setUserAgent("Mozilla/5.0 (Windows NT 6.3; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0")
            .setDefaultRequestConfig(requestConfig).setProxy(proxy).build();
    String responsebody = "";
    String responsestatus = null;
    int count = 0;
    try {
        HttpGet httpget = new HttpGet(newurl);
        httpget.addHeader("Accept", "*/*");
        httpget.addHeader("Accept-Encoding", "gzip, deflate, br");
        httpget.addHeader("Accept-Language", "en-US,en;q=0.5");
        httpget.addHeader("Host", "www.justdial.com");
        httpget.addHeader("Referer", "https://www.justdial.com/Bangalore/Yamaha-Two-Wheeler-Dealers");
        httpget.addHeader("Connection", "keep-alive");
        httpget.addHeader("X-Requested-With", "XMLHttpReques");

        System.out.println("Response status " + httpget.getRequestLine());
        CloseableHttpResponse resp = httpclient.execute(httpget);
        responsestatus = resp.getStatusLine().toString();
        if (responsestatus.contains("503") || responsestatus.contains("502") || responsestatus.contains("403")
                || responsestatus.contains("400") || responsestatus.contains("407")
                || responsestatus.contains("401") || responsestatus.contains("402")
                || responsestatus.contains("404") || responsestatus.contains("405")
                || responsestatus.contains("SSLHandshakeException") || responsestatus.contains("999")
                || responsestatus.contains("ClientProtocolException")
                || responsestatus.contains("SocketTimeoutException") || "".equals(responsestatus)) {
            Thread.sleep(10000);
            do {
                count++;
                responsebody = fetchPageSourcefromClientGoogleSecond(newurl, proxyList);
                if (responsebody == null) {
                    Thread.sleep(10000);
                    System.out.println("PROX FAILURE");
                }
                if (count > 20) {
                    Thread.sleep(1000);
                    break;
                }
            } while (responsebody == null || "".equals(responsebody));
        } else {
            HttpEntity entity = resp.getEntity();
            System.out.println(resp.getStatusLine());
            if (entity != null) {
                System.out.println("Response content length: " + entity.getContentLength());
                BufferedReader in = new BufferedReader(new InputStreamReader(entity.getContent()));
                String inputLine;
                while ((inputLine = in.readLine()) != null) {
                    responsebody = new StringBuilder().append(responsebody).append(inputLine).toString();
                }
                // writeResponseFile(responsebody, pagename);
            }
            EntityUtils.consume(entity);
        }
    } catch (IOException | IllegalStateException e) {
        System.out.println("Exception = " + e);
        do {
            count++;
            responsebody = fetchPageSourcefromClientGoogleSecond(newurl, proxyList);
            if (responsebody == null) {
                System.out.println("PROX FAILURE");
            }
            if (count > 15) {
                Thread.sleep(50000);
                //                    responsebody = fetchPageSourcefromClientGoogleSecond(newurl);
                break;
            }
        } while (responsebody == null || "".equals(responsebody));
    } finally {
        httpclient.close();
    }
    return responsebody;
}

From source file:guru.nidi.ramlproxy.core.CommandSender.java

protected HttpClient createClient() {
    final RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(1000)
            .setConnectionRequestTimeout(1000).setSocketTimeout(1000).build();
    return HttpClients.custom().setDefaultRequestConfig(requestConfig).build();
}

From source file:io.confluent.support.metrics.utils.WebClient.java

/**
 * Sends a POST request to a web server/*w  w  w. jav a 2  s  .c o  m*/
 * @param customerId: customer Id on behalf of which the request is sent
 * @param bytes: request payload
 * @param httpPost: A POST request structure
 * @return an HTTP Status code
 */
public static int send(String customerId, byte[] bytes, HttpPost httpPost) {
    int statusCode = DEFAULT_STATUS_CODE;
    if (bytes != null && bytes.length > 0 && httpPost != null && customerId != null) {

        // add the body to the request
        MultipartEntityBuilder builder = MultipartEntityBuilder.create();
        builder.setMode(HttpMultipartMode.BROWSER_COMPATIBLE);
        builder.addTextBody("cid", customerId);
        builder.addBinaryBody("file", bytes, ContentType.DEFAULT_BINARY, "filename");
        httpPost.setEntity(builder.build());

        // set the HTTP config
        final RequestConfig config = RequestConfig.custom().setConnectTimeout(requestTimeoutMs)
                .setConnectionRequestTimeout(requestTimeoutMs).setSocketTimeout(requestTimeoutMs).build();

        // send request
        try (CloseableHttpClient httpclient = HttpClientBuilder.create().setDefaultRequestConfig(config)
                .build(); CloseableHttpResponse response = httpclient.execute(httpPost)) {
            log.debug("POST request returned {}", response.getStatusLine().toString());
            statusCode = response.getStatusLine().getStatusCode();
        } catch (IOException e) {
            log.debug("Could not submit metrics to Confluent: {}", e.getMessage());
        }
    } else {
        statusCode = HttpStatus.SC_BAD_REQUEST;
    }
    return statusCode;
}

From source file:justdailscrapper.vik.utility.FetchPageWithProxy.java

public static String fetchPageSourcefromClientGoogle(URI newurl, List<ProxyImport> proxyList)
        throws IOException, InterruptedException {

    Random r = new Random();

    ProxyImport obj = proxyList.get(r.nextInt(proxyList.size()));

    String ip = obj.proxyIP;//from   w  ww  . j  a va2 s  . c o  m
    int portno = Integer.parseInt(obj.proxyPort);
    String username = "";
    String password = "";
    if (obj.proxyLen > 2) {
        username = obj.proxyUserName;
        password = obj.proxyPassword;
    }

    //        int portNo = generateRandomPort();
    CredentialsProvider credsprovider = new BasicCredentialsProvider();
    credsprovider.setCredentials(new AuthScope(ip, portno),
            new UsernamePasswordCredentials(username, password));
    HttpHost proxy = new HttpHost(ip, portno);
    //-----------------------------------------------------------------------
    RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(5000).setConnectTimeout(5000)
            .setConnectionRequestTimeout(5000).build();

    CloseableHttpClient httpclient = HttpClients.custom().setDefaultCredentialsProvider(credsprovider)
            .setUserAgent("Mozilla/5.0 (Windows NT 6.3; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0")
            .setDefaultRequestConfig(requestConfig).setProxy(proxy).build();
    String responsebody = "";
    String responsestatus = null;
    int count = 0;
    try {
        HttpGet httpget = new HttpGet(newurl);
        httpget.addHeader("Accept", "*/*");
        httpget.addHeader("Accept-Encoding", "gzip, deflate, br");
        httpget.addHeader("Accept-Language", "en-US,en;q=0.5");
        httpget.addHeader("Host", "www.justdial.com");
        httpget.addHeader("Referer", "https://www.justdial.com/Bangalore/Yamaha-Two-Wheeler-Dealers");
        httpget.addHeader("Connection", "keep-alive");
        httpget.addHeader("X-Requested-With", "XMLHttpReques");

        System.out.println("Response status " + httpget.getRequestLine());
        CloseableHttpResponse resp = httpclient.execute(httpget);
        responsestatus = resp.getStatusLine().toString();
        if (responsestatus.contains("503") || responsestatus.contains("502") || responsestatus.contains("403")
                || responsestatus.contains("400") || responsestatus.contains("407")
                || responsestatus.contains("401") || responsestatus.contains("402")
                || responsestatus.contains("404") || responsestatus.contains("405")
                || responsestatus.contains("SSLHandshakeException") || responsestatus.contains("999")
                || responsestatus.contains("ClientProtocolException")
                || responsestatus.contains("SocketTimeoutException") || "".equals(responsestatus)) {
            Thread.sleep(10000);
            do {
                count++;
                responsebody = fetchPageSourcefromClientGoogleSecond(newurl, proxyList);
                if (responsebody == null) {
                    Thread.sleep(10000);
                    logTextArea.append("PROX FAILURE\n");
                }
                if (count > 20) {
                    Thread.sleep(1000);
                    break;
                }
            } while (responsebody == null || "".equals(responsebody));
        } else {
            HttpEntity entity = resp.getEntity();
            System.out.println(resp.getStatusLine());
            if (entity != null) {
                logTextArea.append("Response content length: " + entity.getContentLength() + "\n");
                BufferedReader in = new BufferedReader(new InputStreamReader(entity.getContent()));
                String inputLine;
                while ((inputLine = in.readLine()) != null) {
                    responsebody = new StringBuilder().append(responsebody).append(inputLine).toString();
                }
                // writeResponseFile(responsebody, pagename);
            }
            EntityUtils.consume(entity);
        }
    } catch (IOException | IllegalStateException e) {
        System.out.println("Exception = " + e);
        do {
            count++;
            responsebody = fetchPageSourcefromClientGoogleSecond(newurl, proxyList);
            if (responsebody == null) {
                System.out.println("PROX FAILURE");
            }
            if (count > 15) {
                Thread.sleep(50000);
                //                    responsebody = fetchPageSourcefromClientGoogleSecond(newurl);
                break;
            }
        } while (responsebody == null || "".equals(responsebody));
    } finally {
        httpclient.close();
    }
    return responsebody;
}

From source file:v2.service.generic.library.utils.HttpClientUtil.java

public static HttpClient createHttpClientWithAuth(String credential, String principle) {
    CredentialsProvider provider = new BasicCredentialsProvider();
    UsernamePasswordCredentials credentials = new UsernamePasswordCredentials(credential, principle);
    provider.setCredentials(AuthScope.ANY, credentials);
    RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(10 * 1000).build();
    CloseableHttpClient httpclient = HttpClientBuilder.create().setDefaultCredentialsProvider(provider)
            .setDefaultRequestConfig(requestConfig).build();
    return httpclient;
}

From source file:com.qwazr.cluster.manager.ClusterMonitoringThread.java

ClusterMonitoringThread(int monitoring_period_seconds) {
    super("Nodes monitoring", monitoring_period_seconds);
    setDaemon(true);/* w ww .  j  a  va 2 s  .  co  m*/
    requestConfig = RequestConfig.custom().setSocketTimeout(monitoring_period)
            .setConnectTimeout(monitoring_period).build();
    httpclient = HttpAsyncClients.custom().setMaxConnPerRoute(2).setMaxConnTotal(100)
            .setDefaultRequestConfig(requestConfig).build();
    httpclient.start();
    start();
}

From source file:de.intevation.irix.PrintClient.java

/** Obtains a Report from mapfish-print service.
 *
 * @param printUrl The url to send the request to.
 * @param json The json spec for the print request.
 * @param timeout the timeout for the httpconnection.
 *
 * @return byte[] with the report.//from   ww  w  .  j  a v a 2  s .com
 *
 * @throws IOException if communication with print service failed.
 * @throws PrintException if the print job failed.
 */
public static byte[] getReport(String printUrl, String json, int timeout) throws IOException, PrintException {

    RequestConfig config = RequestConfig.custom().setConnectTimeout(timeout).build();

    CloseableHttpClient client = HttpClients.custom().setDefaultRequestConfig(config).build();

    HttpEntity entity = new StringEntity(json,
            ContentType.create("application/json", Charset.forName("UTF-8")));

    HttpPost post = new HttpPost(printUrl);
    post.setEntity(entity);
    CloseableHttpResponse resp = client.execute(post);

    StatusLine status = resp.getStatusLine();

    byte[] retval = null;
    try {
        HttpEntity respEnt = resp.getEntity();
        InputStream in = respEnt.getContent();
        if (in != null) {
            try {
                ByteArrayOutputStream out = new ByteArrayOutputStream();
                byte[] buf = new byte[BYTE_ARRAY_SIZE];
                int r;
                while ((r = in.read(buf)) >= 0) {
                    out.write(buf, 0, r);
                }
                retval = out.toByteArray();
            } finally {
                in.close();
                EntityUtils.consume(respEnt);
            }
        }
    } finally {
        resp.close();
    }

    if (status.getStatusCode() < HttpURLConnection.HTTP_OK
            || status.getStatusCode() >= HttpURLConnection.HTTP_MULT_CHOICE) {
        if (retval != null) {
            throw new PrintException(new String(retval));
        } else {
            throw new PrintException("Communication with print service '" + printUrl + "' failed."
                    + "\nNo response from print service.");
        }
    }
    return retval;
}

From source file:org.openmrs.module.webservices.rest.ITBase.java

@BeforeClass
public static void waitForServerToStart() {
    synchronized (serverStartupLock) {
        if (!serverStarted) {
            final long time = System.currentTimeMillis();
            final int timeout = 300000;
            final int retryAfter = 10000;

            final RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(retryAfter)
                    .setConnectTimeout(retryAfter).build();

            final String startupUri = TEST_URL.getScheme() + "://" + TEST_URL.getHost() + ":"
                    + TEST_URL.getPort() + TEST_URL.getPath();
            System.out.println(// w  ww. java 2s. c  om
                    "Waiting for server at " + startupUri + " for " + timeout / 1000 + " more seconds...");

            while (System.currentTimeMillis() - time < timeout) {
                try {
                    final HttpClient client = HttpClientBuilder.create().disableAutomaticRetries().build();
                    final HttpGet sessionGet = new HttpGet(startupUri);
                    sessionGet.setConfig(requestConfig);
                    final HttpClientContext context = HttpClientContext.create();
                    final HttpResponse response = client.execute(sessionGet, context);

                    int status = response.getStatusLine().getStatusCode();
                    if (status >= 400) {
                        throw new RuntimeException(status + " " + response.getStatusLine().getReasonPhrase());
                    }

                    URI finalUri = sessionGet.getURI();
                    List<URI> redirectLocations = context.getRedirectLocations();
                    if (redirectLocations != null) {
                        finalUri = redirectLocations.get(redirectLocations.size() - 1);
                    }

                    String finalUriString = finalUri.toString();
                    if (!finalUriString.contains("initialsetup")) {
                        serverStarted = true;
                        return;
                    }
                } catch (IOException e) {
                    System.out.println(e.toString());
                }

                try {
                    System.out.println("Waiting for " + (timeout - (System.currentTimeMillis() - time)) / 1000
                            + " more seconds...");
                    Thread.sleep(retryAfter);
                } catch (InterruptedException e) {
                    throw new RuntimeException(e);
                }
            }

            throw new RuntimeException("Server startup took longer than 5 minutes!");
        }
    }
}

From source file:org.n52.sos.soe.HttpUtil.java

private static InputStream executeGet(String target, CloseableHttpClient client) throws IOException {
    logger.info("HTTP GET: " + target);

    long start = System.currentTimeMillis();
    HttpGet get = new HttpGet(target);
    get.setConfig(RequestConfig.custom().setConnectTimeout(1000 * 120).build());
    HttpResponse resp = client.execute(get);
    logger.info("Request latency: " + (System.currentTimeMillis() - start));
    return resp.getEntity().getContent();
}

From source file:com.dtstack.jlogstash.distributed.http.cilent.HttpClient.java

public static String post(String url, Map<String, Object> bodyData) {
    String responseBody = null;/* w  w w  .java2s.  c o m*/
    CloseableHttpClient httpClient = null;
    try {
        httpClient = getHttpClient();
        HttpPost httPost = new HttpPost(url);
        if (SetTimeOut) {
            RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(SocketTimeout)
                    .setConnectTimeout(ConnectTimeout).build();//
            httPost.setConfig(requestConfig);
        }
        if (bodyData != null && bodyData.size() > 0) {
            httPost.setEntity(new StringEntity(objectMapper.writeValueAsString(bodyData)));
        }
        //?
        CloseableHttpResponse response = httpClient.execute(httPost);
        int status = response.getStatusLine().getStatusCode();
        if (status == HttpStatus.SC_OK) {
            HttpEntity entity = response.getEntity();
            //FIXME ?header?
            responseBody = EntityUtils.toString(entity, Charsets.UTF_8);
        } else {
            logger.error("url:" + url + "--->http return status error:" + status);
        }
    } catch (Exception e) {
        logger.error("url:" + url + "--->http request error", e);
    } finally {
        try {
            if (httpClient != null)
                httpClient.close();
        } catch (Exception e) {
            // TODO Auto-generated catch block
            logger.error(ExceptionUtil.getErrorMessage(e));
        }
    }
    return responseBody;
}