Example usage for org.apache.http.client.methods CloseableHttpResponse getStatusLine

List of usage examples for org.apache.http.client.methods CloseableHttpResponse getStatusLine

  1. HOME
  2. Java
  3. org.apache.http
  4. org.apache.http.client.methods.*
  5. CloseableHttpResponse
  6. getStatusLine

Introduction

In this page you can find the example usage for org.apache.http.client.methods CloseableHttpResponse getStatusLine.

Prototype

StatusLine getStatusLine();

Source Link

Usage

From source file:com.clonephpscrapper.utility.FetchPageWithProxy.java

public static String fetchPage(URI newurl) throws IOException, InterruptedException {

    int portNo = generateRandomPort();
    CredentialsProvider credsprovider = new BasicCredentialsProvider();
    credsprovider.setCredentials(new AuthScope("195.154.161.103", portNo),
            new UsernamePasswordCredentials("mongoose", "Fjh30fi"));
    HttpHost proxy = new HttpHost("195.154.161.103", portNo);
    //-----------------------------------------------------------------------
    RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(5000).setConnectTimeout(5000)
            .setConnectionRequestTimeout(5000).build();

    CloseableHttpClient httpclient = HttpClients.custom().setDefaultCredentialsProvider(credsprovider)
            .setUserAgent("Mozilla/5.0 (Windows NT 6.1; WOW64; rv:35.0) Gecko/20100101 Firefox/35.0")
            .setDefaultRequestConfig(requestConfig).setProxy(proxy).build();
    String responsebody = "";
    String responsestatus = null;
    int count = 0;
    try {/*  www  .  j  av  a 2  s .  co m*/
        HttpGet httpget = new HttpGet(newurl);
        httpget.addHeader("GET ", "http://clonephp.com/");
        httpget.addHeader("Host", "clonephp.com");
        httpget.addHeader("Accept",
                "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
        httpget.addHeader("User-Agent",
                "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.106 Safari/537.36");
        httpget.addHeader("Accept-Encoding", "gzip, deflate, sdch");
        httpget.addHeader("Accept-Language", "en-US,en;q=0.8");
        httpget.addHeader("Connection", "keep-alive");
        //            httpget.addHeader("Cookie", "_ym_uid=14523228509491377; _gat=1; _ym_isad=0; _ga=GA1.2.438893798.1452322850; _ym_visorc_26745633=b; last-featured-visit-time=1452323357912\n" +
        //"If-None-Match: W/\"10d86-1fKww4Zqud7NjVeuSjZKEw\"");

        System.out.println("Response status " + httpget.getRequestLine());
        CloseableHttpResponse resp = httpclient.execute(httpget);
        responsestatus = resp.getStatusLine().toString();
        if (responsestatus.contains("503") || responsestatus.contains("502") || responsestatus.contains("403")
                || responsestatus.contains("400") || responsestatus.contains("407")
                || responsestatus.contains("401") || responsestatus.contains("402")
                || responsestatus.contains("404") || responsestatus.contains("405")
                || responsestatus.contains("SSLHandshakeException") || responsestatus.contains("999")
                || responsestatus.contains("ClientProtocolException")
                || responsestatus.contains("SocketTimeoutException") || "".equals(responsestatus)) {
            Thread.sleep(10000);
            do {
                count++;
                responsebody = fetchPageSourcefromClientGoogleSecond(newurl);
                if (responsebody == null) {
                    Thread.sleep(10000);
                    System.out.println("PROX FAILURE");
                }
                if (count > 20) {
                    Thread.sleep(1000);
                    break;
                }
            } while (responsebody == null || "".equals(responsebody));
        } else {
            HttpEntity entity = resp.getEntity();
            System.out.println(resp.getStatusLine());
            if (entity != null) {
                System.out.println("Response content length: " + entity.getContentLength());
                BufferedReader in = new BufferedReader(new InputStreamReader(entity.getContent()));
                String inputLine;
                while ((inputLine = in.readLine()) != null) {
                    responsebody = new StringBuilder().append(responsebody).append(inputLine).toString();
                }
                // writeResponseFile(responsebody, pagename);
            }
            EntityUtils.consume(entity);
        }
    } catch (IOException | IllegalStateException e) {
        System.out.println("Exception = " + e);
        do {
            count++;
            responsebody = fetchPageSourcefromClientGoogleSecond(newurl);
            if (responsebody == null) {
                System.out.println("PROX FAILURE");
            }
            if (count > 15) {
                Thread.sleep(50000);
                //                    responsebody = fetchPageSourcefromClientGoogleSecond(newurl);
                break;
            }
        } while (responsebody == null || "".equals(responsebody));
    } finally {
        httpclient.close();
    }
    return responsebody;
}

From source file:com.jaspersoft.studio.community.RESTCommunityHelper.java

/**
 * Uploads the specified file to the community site. The return identifier
 * can be used later when composing other requests.
 * /* w w w  . ja  va2  s  .c  om*/
 * @param httpclient
 *            the http client
 * @param attachment
 *            the file to attach
 * @param authCookie
 *            the session cookie to use for authentication purpose
 * @return the identifier of the file uploaded, <code>null</code> otherwise
 * @throws CommunityAPIException
 */
public static String uploadFile(CloseableHttpClient httpclient, File attachment, Cookie authCookie)
        throws CommunityAPIException {
    FileInputStream fin = null;
    try {
        fin = new FileInputStream(attachment);
        byte fileContent[] = new byte[(int) attachment.length()];
        fin.read(fileContent);

        byte[] encodedFileContent = Base64.encodeBase64(fileContent);
        FileUploadRequest uploadReq = new FileUploadRequest(attachment.getName(), encodedFileContent);

        HttpPost fileuploadPOST = new HttpPost(CommunityConstants.FILE_UPLOAD_URL);
        EntityBuilder fileUploadEntity = EntityBuilder.create();
        fileUploadEntity.setText(uploadReq.getAsJSON());
        fileUploadEntity.setContentType(ContentType.create(CommunityConstants.JSON_CONTENT_TYPE));
        fileUploadEntity.setContentEncoding(CommunityConstants.REQUEST_CHARSET);
        fileuploadPOST.setEntity(fileUploadEntity.build());

        CloseableHttpResponse resp = httpclient.execute(fileuploadPOST);
        int httpRetCode = resp.getStatusLine().getStatusCode();
        String responseBodyAsString = EntityUtils.toString(resp.getEntity());

        if (HttpStatus.SC_OK == httpRetCode) {
            ObjectMapper mapper = new ObjectMapper();
            mapper.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true);
            mapper.configure(JsonParser.Feature.ALLOW_SINGLE_QUOTES, true);
            mapper.configure(JsonParser.Feature.ALLOW_COMMENTS, true);
            JsonNode jsonRoot = mapper.readTree(responseBodyAsString);
            String fid = jsonRoot.get("fid").asText(); //$NON-NLS-1$
            return fid;
        } else {
            CommunityAPIException ex = new CommunityAPIException(Messages.RESTCommunityHelper_FileUploadError);
            ex.setHttpStatusCode(httpRetCode);
            ex.setResponseBodyAsString(responseBodyAsString);
            throw ex;
        }

    } catch (FileNotFoundException e) {
        JSSCommunityActivator.getDefault().logError(Messages.RESTCommunityHelper_FileNotFoundError, e);
        throw new CommunityAPIException(Messages.RESTCommunityHelper_FileUploadError, e);
    } catch (UnsupportedEncodingException e) {
        JSSCommunityActivator.getDefault().logError(Messages.RESTCommunityHelper_EncodingNotValidError, e);
        throw new CommunityAPIException(Messages.RESTCommunityHelper_FileUploadError, e);
    } catch (IOException e) {
        JSSCommunityActivator.getDefault().logError(Messages.RESTCommunityHelper_PostMethodIOError, e);
        throw new CommunityAPIException(Messages.RESTCommunityHelper_FileUploadError, e);
    } finally {
        IOUtils.closeQuietly(fin);
    }
}

From source file:justdailscrapper.vik.utility.FetchPageWithoutProxy.java

public static String fetchPageSourcefromClientGoogleSecond(URI newurl, List<ProxyImport> proxyList)
        throws IOException {

    Random r = new Random();

    ProxyImport obj = proxyList.get(r.nextInt(proxyList.size()));

    String ip = obj.proxyIP;//from  w  w w .  java2s . c o m
    int portno = Integer.parseInt(obj.proxyPort);
    String username = "";
    String password = "";
    if (obj.proxyLen > 2) {
        username = obj.proxyUserName;
        password = obj.proxyPassword;
    }

    //        int portNo = generateRandomPort();
    CredentialsProvider credsprovider = new BasicCredentialsProvider();
    credsprovider.setCredentials(new AuthScope(ip, portno),
            new UsernamePasswordCredentials(username, password));
    HttpHost proxy = new HttpHost(ip, portno);
    //-----------------------------------------------------------------------
    RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(5000).setConnectTimeout(5000)
            .setConnectionRequestTimeout(5000).build();

    CloseableHttpClient httpclient = HttpClients.custom().setDefaultCredentialsProvider(credsprovider)
            .setUserAgent("Mozilla/5.0 (Windows NT 6.1; WOW64; rv:35.0) Gecko/20100101 Firefox/35.0")
            .setDefaultRequestConfig(requestConfig).setProxy(proxy).build();
    String responsebody = "";
    String responsestatus = null;
    try {
        HttpGet httpget = new HttpGet(newurl);
        httpget.addHeader("Accept", "*/*");
        httpget.addHeader("Accept-Encoding", "gzip, deflate, br");
        httpget.addHeader("Accept-Language", "en-US,en;q=0.5");
        httpget.addHeader("Host", "www.justdial.com");
        httpget.addHeader("Referer", "https://www.justdial.com/Bangalore/Yamaha-Two-Wheeler-Dealers");
        httpget.addHeader("Connection", "keep-alive");
        httpget.addHeader("X-Requested-With", "XMLHttpReques");

        System.out.println("Response status" + httpget.getRequestLine());
        CloseableHttpResponse resp = httpclient.execute(httpget);
        responsestatus = resp.getStatusLine().toString();
        if (responsestatus.contains("503") || responsestatus.contains("502") || responsestatus.contains("400")
                || responsestatus.contains("401") || responsestatus.contains("402")
                || responsestatus.contains("403") || responsestatus.contains("407")
                || responsestatus.contains("404") || responsestatus.contains("405")
                || responsestatus.contains("SSLHandshakeException") || responsestatus.contains("999")
                || responsestatus.contains("ClientProtocolException")
                || responsestatus.contains("SocketTimeoutException") || responsestatus == null
                || "".equals(responsestatus)) {
            return null;
        } else {
            HttpEntity entity = resp.getEntity();
            System.out.println(resp.getStatusLine());
            if (entity != null) {
                //                    System.out.println("Response content length: " + entity.getContentLength());
                BufferedReader in = new BufferedReader(new InputStreamReader(entity.getContent()));
                String inputLine;
                while ((inputLine = in.readLine()) != null) {
                    responsebody = new StringBuilder().append(responsebody).append(inputLine).toString();
                }
            }
            EntityUtils.consume(entity);
        }
    } catch (IOException | IllegalStateException e) {
        return null;
    } finally {
        httpclient.close();
    }
    return responsebody;
}

From source file:com.baidubce.util.HttpUtils.java

public static void printResponse(CloseableHttpResponse response) {
    if (!HTTP_VERBOSE) {
        return;/* w w w  . jav  a  2 s.com*/
    }
    System.out.println("\n<------------- ");
    StatusLine status = response.getStatusLine();
    System.out.println(status.getStatusCode() + " - " + status.getReasonPhrase());
    Header[] heads = response.getAllHeaders();
    for (Header h : heads) {
        System.out.println(h.getName() + " : " + h.getValue());
    }
}

From source file:org.sead.repositories.reference.util.SEADGoogleLogin.java

static void getTokensFromCode() {
    access_token = null;/*from w ww . j  a v a2s .  c  o  m*/
    expires_in = -1;
    token_start_time = -1;
    refresh_token = null;
    new File("refresh.txt").delete();

    if (gProps == null) {
        initGProps();
    }
    // Query for token now that user has gone through browser part
    // of
    // flow
    HttpPost tokenRequest = new HttpPost(gProps.token_uri);

    MultipartEntityBuilder tokenRequestParams = MultipartEntityBuilder.create();
    tokenRequestParams.addTextBody("client_id", gProps.client_id);
    tokenRequestParams.addTextBody("client_secret", gProps.client_secret);
    tokenRequestParams.addTextBody("code", device_code);
    tokenRequestParams.addTextBody("grant_type", "http://oauth.net/grant_type/device/1.0");

    HttpEntity reqEntity = tokenRequestParams.build();

    tokenRequest.setEntity(reqEntity);
    CloseableHttpClient httpclient = HttpClients.createDefault();
    CloseableHttpResponse response = null;
    try {
        response = httpclient.execute(tokenRequest);

        if (response.getStatusLine().getStatusCode() == 200) {
            HttpEntity resEntity = response.getEntity();
            if (resEntity != null) {
                String responseJSON = EntityUtils.toString(resEntity);
                ObjectNode root = (ObjectNode) new ObjectMapper().readTree(responseJSON);
                access_token = root.get("access_token").asText();
                refresh_token = root.get("refresh_token").asText();
                token_start_time = System.currentTimeMillis() / 1000;
                expires_in = root.get("expires_in").asInt();
            }
        } else {
            log.error("Error response from Google: " + response.getStatusLine().getReasonPhrase());
        }
    } catch (ParseException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    } finally {
        if (response != null) {
            try {
                response.close();
            } catch (IOException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
        }
        try {
            httpclient.close();
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }

    }

}

From source file:com.lhings.java.http.WebServiceCom.java

private static HttpResponse executeRequest(HttpRequestBase request) throws IOException {
    CloseableHttpClient hc = HttpClients.createDefault();
    CloseableHttpResponse response = hc.execute(request);
    HttpResponse httpResponse = new HttpResponse();
    httpResponse.setStatusCode(response.getStatusLine().getStatusCode());
    httpResponse.setStatusMessage(response.getStatusLine().getReasonPhrase());
    // ResponseHandler<String> handler = new BasicResponseHandler();
    // String responseBody = handler.handleResponse(response);
    httpResponse.setResponseBody(EntityUtils.toString(response.getEntity()));
    return httpResponse;
}

From source file:nl.knaw.dans.easy.sword2examples.ContinuedDeposit.java

public static URI depositPackage(File bagDir, IRI colIri, String uid, String pw, int chunkSize)
        throws Exception {
    // 0. Zip the bagDir
    File zipFile = new File(bagDir.getAbsolutePath() + ".zip");
    zipFile.delete();/*from   w w w.  j a  va  2s.c o  m*/
    Common.zipDirectory(bagDir, zipFile);

    // 1. Set up stream for calculating MD5
    FileInputStream fis = new FileInputStream(zipFile);
    MessageDigest md = MessageDigest.getInstance("MD5");
    DigestInputStream dis = new DigestInputStream(fis, md);

    // 2. Post first chunk bag to Col-IRI
    CloseableHttpClient http = Common.createHttpClient(colIri.toURI(), uid, pw);
    CloseableHttpResponse response = Common.sendChunk(dis, chunkSize, "POST", colIri.toURI(), "bag.zip.1",
            "application/octet-stream", http, chunkSize < zipFile.length());

    // 3. Check the response. If transfer corrupt (MD5 doesn't check out), report and exit.
    String bodyText = Common.readEntityAsString(response.getEntity());
    if (response.getStatusLine().getStatusCode() != 201) {
        System.err.println("FAILED. Status = " + response.getStatusLine());
        System.err.println("Response body follows:");
        System.err.println(bodyText);
        System.exit(2);
    }
    System.out.println("SUCCESS. Deposit receipt follows:");
    System.out.println(bodyText);

    Entry receipt = Common.parse(bodyText);
    Link seIriLink = receipt.getLink("edit");
    URI seIri = seIriLink.getHref().toURI();

    int remaining = (int) zipFile.length() - chunkSize;
    int count = 2;
    while (remaining > 0) {
        System.out.print(String.format("POST-ing chunk of %d bytes to SE-IRI (remaining: %d) ... ", chunkSize,
                remaining));
        response = Common.sendChunk(dis, chunkSize, "POST", seIri, "bag.zip." + count++,
                "application/octet-stream", http, remaining > chunkSize);
        remaining -= chunkSize;
        bodyText = Common.readEntityAsString(response.getEntity());
        if (response.getStatusLine().getStatusCode() != 200) {
            System.err.println("FAILED. Status = " + response.getStatusLine());
            System.err.println("Response body follows:");
            System.err.println(bodyText);
            System.exit(2);
        }
        System.out.println("SUCCESS.");
    }

    // 4. Get the statement URL. This is the URL from which to retrieve the current status of the deposit.
    System.out.println("Retrieving Statement IRI (Stat-IRI) from deposit receipt ...");
    receipt = Common.parse(bodyText);
    Link statIriLink = receipt.getLink("http://purl.org/net/sword/terms/statement");
    IRI statIri = statIriLink.getHref();
    System.out.println("Stat-IRI = " + statIri);

    // 5. Check statement every ten seconds (a bit too frantic, but okay for this test). If status changes:
    // report new status. If status is an error (INVALID, REJECTED, FAILED) or ARCHIVED: exit.
    return Common.trackDeposit(http, statIri.toURI());
}

From source file:zz.pseas.ghost.login.weibo.SinaWeiboLogin.java

public static String getSinaCookie(String username, String password) throws Exception {

    StringBuilder sb = new StringBuilder();
    HtmlUnitDriver driver = new HtmlUnitDriver(true);
    // driver.setSocksProxy("127.0.0.1", 1080);
    // driver.setProxy("127.0.0.1", 1080);
    // HtmlOption htmlOption=new Ht
    // WebDriver driver = new FirefoxDriver();
    driver.setJavascriptEnabled(true);//from www  .java2 s  .com
    // user agent switcher//
    String loginAddress = "http://login.weibo.cn/login/";
    driver.get(loginAddress);
    WebElement ele = driver.findElementByCssSelector("img");
    String src = ele.getAttribute("src");
    String cookie = concatCookie(driver);

    CloseableHttpClient httpclient = HttpClients.createDefault();
    RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(1000).setConnectTimeout(1000)
            .setCookieSpec(cookie).build();

    HttpGet httpget = new HttpGet(src);
    httpget.setConfig(requestConfig);
    CloseableHttpResponse response = httpclient.execute(httpget);

    HttpEntity entity;
    String result = null;
    try {

        if (response.getStatusLine().getStatusCode() != HttpURLConnection.HTTP_OK) {
            // try again//
        }
        entity = response.getEntity();

        if (entity != null) {
            InputStream instream = entity.getContent();
            try {
                // do something useful
                InputStream inputStream = entity.getContent();
                BufferedImage img = ImageIO.read(inputStream);
                // deal with the weibo captcha //
                String picName = LoginUtils.getCurrentTime("yyyyMMdd-hhmmss");
                // captcha//
                LoginUtils.getCaptchaDir();
                picName = "./captcha/captcha-" + picName + ".png";
                ImageIO.write(img, "png", new File(picName));
                String userInput = new CaptchaFrame(img).getUserInput();
                WebElement mobile = driver.findElementByCssSelector("input[name=mobile]");
                mobile.sendKeys(username);
                WebElement pass = driver.findElementByCssSelector("input[name^=password]");
                pass.sendKeys(password);
                WebElement code = driver.findElementByCssSelector("input[name=code]");
                code.sendKeys(userInput);
                WebElement rem = driver.findElementByCssSelector("input[name=remember]");
                rem.click();
                WebElement submit = driver.findElementByCssSelector("input[name=submit]");
                // ?//
                submit.click();
                result = concatCookie(driver);
                driver.close();
            } finally {
                instream.close();
            }
        }
    } finally {
        response.close();
    }

    if (result.contains("gsid_CTandWM")) {
        return result;
    } else {
        // throw new Exception("weibo login failed");
        return null;
    }
}

From source file:org.sead.repositories.reference.util.SEADGoogleLogin.java

static void getTokenFromRefreshToken() {
    access_token = null;/*  w ww.  j  av a  2s  .  co  m*/
    expires_in = -1;
    token_start_time = -1;

    if (gProps == null) {
        initGProps();
    }

    /* Try refresh token */
    // Query for token now that user has gone through browser part
    // of
    // flow

    // The method used in getTokensFromCode should work here as well - I
    // think URL encoded Form is the recommended way...
    HttpPost post = new HttpPost(gProps.token_uri);
    post.addHeader("accept", "application/json");
    post.addHeader("Content-Type", "application/x-www-form-urlencoded");

    List<NameValuePair> nameValuePairs = new ArrayList<NameValuePair>(4);
    nameValuePairs.add(new BasicNameValuePair("client_id", gProps.client_id));
    nameValuePairs.add(new BasicNameValuePair("client_secret", gProps.client_secret));
    nameValuePairs.add(new BasicNameValuePair("refresh_token", refresh_token));
    nameValuePairs.add(new BasicNameValuePair("grant_type", "refresh_token"));

    try {
        post.setEntity(new UrlEncodedFormEntity(nameValuePairs));
    } catch (UnsupportedEncodingException e1) {
        e1.printStackTrace();
    }

    CloseableHttpClient httpclient = HttpClients.createDefault();
    CloseableHttpResponse response = null;
    try {
        try {
            response = httpclient.execute(post);
            if (response.getStatusLine().getStatusCode() == 200) {
                HttpEntity resEntity = response.getEntity();
                if (resEntity != null) {
                    String responseJSON = EntityUtils.toString(resEntity);
                    ObjectNode root = (ObjectNode) new ObjectMapper().readTree(responseJSON);
                    access_token = root.get("access_token").asText();
                    // refresh_token =
                    // root.get("refresh_token").asText();
                    token_start_time = System.currentTimeMillis() / 1000;
                    expires_in = root.get("expires_in").asInt();
                }
            } else {
                log.error("Error response from Google: " + response.getStatusLine().getReasonPhrase());
                HttpEntity resEntity = response.getEntity();
            }
        } catch (ParseException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        } catch (IOException e) {
            log.error("Error obtaining access token: " + e.getMessage());
        } finally {
            if (response != null) {
                response.close();
            }
            httpclient.close();
        }
    } catch (IOException io) {
        log.error("Error closing connections: " + io.getMessage());
    }
}

From source file:justdailscrapper.vik.utility.FetchPageWithoutProxy.java

public static String fetchPage(URI newurl, List<ProxyImport> proxyList)
        throws IOException, InterruptedException {

    Random r = new Random();

    ProxyImport obj = proxyList.get(r.nextInt(proxyList.size()));

    String ip = obj.proxyIP;//from   w  ww  .  j a va2s .co m
    int portno = Integer.parseInt(obj.proxyPort);
    String username = "";
    String password = "";
    if (obj.proxyLen > 2) {
        username = obj.proxyUserName;
        password = obj.proxyPassword;
    }

    //        int portNo = generateRandomPort();
    CredentialsProvider credsprovider = new BasicCredentialsProvider();
    credsprovider.setCredentials(new AuthScope(ip, portno),
            new UsernamePasswordCredentials(username, password));
    HttpHost proxy = new HttpHost(ip, portno);
    //-----------------------------------------------------------------------
    RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(5000).setConnectTimeout(5000)
            .setConnectionRequestTimeout(5000).build();

    CloseableHttpClient httpclient = HttpClients.custom().setDefaultCredentialsProvider(credsprovider)
            .setUserAgent("Mozilla/5.0 (Windows NT 6.1; WOW64; rv:35.0) Gecko/20100101 Firefox/35.0")
            .setDefaultRequestConfig(requestConfig).setProxy(proxy).build();
    String responsebody = "";
    String responsestatus = null;
    int count = 0;
    try {
        HttpGet httpget = new HttpGet(newurl);
        httpget.addHeader("Accept", "*/*");
        httpget.addHeader("Accept-Encoding", "gzip, deflate, br");
        httpget.addHeader("Accept-Language", "en-US,en;q=0.5");
        httpget.addHeader("Host", "www.justdial.com");
        httpget.addHeader("Referer", "https://www.justdial.com/Bangalore/Yamaha-Two-Wheeler-Dealers");
        httpget.addHeader("Connection", "keep-alive");
        httpget.addHeader("X-Requested-With", "XMLHttpReques");
        System.out.println("Response status " + httpget.getRequestLine());
        CloseableHttpResponse resp = httpclient.execute(httpget);
        responsestatus = resp.getStatusLine().toString();
        if (responsestatus.contains("503") || responsestatus.contains("502") || responsestatus.contains("403")
                || responsestatus.contains("400") || responsestatus.contains("407")
                || responsestatus.contains("401") || responsestatus.contains("402")
                || responsestatus.contains("404") || responsestatus.contains("405")
                || responsestatus.contains("SSLHandshakeException") || responsestatus.contains("999")
                || responsestatus.contains("ClientProtocolException")
                || responsestatus.contains("SocketTimeoutException") || "".equals(responsestatus)) {
            Thread.sleep(10000);
            do {
                count++;
                responsebody = fetchPageSourcefromClientGoogleSecond(newurl, proxyList);
                if (responsebody == null) {
                    Thread.sleep(10000);
                    System.out.println("PROX FAILURE");
                }
                if (count > 20) {
                    Thread.sleep(1000);
                    break;
                }
            } while (responsebody == null || "".equals(responsebody));
        } else {
            HttpEntity entity = resp.getEntity();
            System.out.println(resp.getStatusLine());
            if (entity != null) {
                System.out.println("Response content length: " + entity.getContentLength());
                BufferedReader in = new BufferedReader(new InputStreamReader(entity.getContent()));
                String inputLine;
                while ((inputLine = in.readLine()) != null) {
                    responsebody = new StringBuilder().append(responsebody).append(inputLine).toString();
                }
                // writeResponseFile(responsebody, pagename);
            }
            EntityUtils.consume(entity);
        }
    } catch (IOException | IllegalStateException e) {
        System.out.println("Exception = " + e);
        do {
            count++;
            responsebody = fetchPageSourcefromClientGoogleSecond(newurl, proxyList);
            if (responsebody == null) {
                System.out.println("PROX FAILURE");
            }
            if (count > 15) {
                Thread.sleep(50000);
                //                    responsebody = fetchPageSourcefromClientGoogleSecond(newurl);
                break;
            }
        } while (responsebody == null || "".equals(responsebody));
    } finally {
        httpclient.close();
    }
    return responsebody;
}