Example usage for org.apache.http.client.protocol ClientContext COOKIE_STORE

List of usage examples for org.apache.http.client.protocol ClientContext COOKIE_STORE

Introduction

In this page you can find the example usage for org.apache.http.client.protocol ClientContext COOKIE_STORE.

Prototype

String COOKIE_STORE

To view the source code for org.apache.http.client.protocol ClientContext COOKIE_STORE.

Click Source Link

Document

Attribute name of a org.apache.http.client.CookieStore object that represents the actual cookie store.

Usage

From source file:net.paissad.minus.utils.HttpClientUtils.java

/**
 * //from   w ww . j  a v  a2s  .  c om
 * @param baseURL
 * @param parametersBody
 * @param sessionId
 * @param fileToUpload - The file to upload.
 * @param filename - The name of the file to use during the upload process.
 * @return The response received from the Minus API.
 * @throws MinusException
 */
public static MinusHttpResponse doUpload(final String baseURL, final Map<String, String> parametersBody,
        final String sessionId, final File fileToUpload, final String filename) throws MinusException {

    DefaultHttpClient client = null;
    HttpPost uploadRequest = null;
    InputStream responseContent = null;

    try {
        String url = baseURL;
        if (parametersBody != null && !parametersBody.isEmpty()) {
            url += CommonUtils.encodeParams(parametersBody);
        }
        uploadRequest = new HttpPost(url);
        client = new DefaultHttpClient();
        client.setHttpRequestRetryHandler(new MinusHttpRequestRetryHandler());

        FileEntity fileEntity = new FileEntity(fileToUpload, "application/octet-stream");
        uploadRequest.setEntity(fileEntity);

        // We add this headers as specified by the Minus.com API during file
        // upload.
        uploadRequest.addHeader("Content-Disposition", "attachment; filename=a.bin");
        uploadRequest.addHeader("Content-Type", "application/octet-stream");

        client.getParams().setParameter(ClientPNames.HANDLE_REDIRECTS, true);
        client.getParams().setParameter(ClientPNames.COOKIE_POLICY, CookiePolicy.BEST_MATCH);

        CookieStore cookieStore = new BasicCookieStore();

        Cookie sessionCookie = null;
        if (sessionId != null && !sessionId.trim().isEmpty()) {
            sessionCookie = new BasicClientCookie2(MINUS_COOKIE_NAME, sessionId);
            ((BasicClientCookie2) sessionCookie).setPath("/");
            ((BasicClientCookie2) sessionCookie).setDomain(MINUS_DOMAIN_NAME);
            ((BasicClientCookie2) sessionCookie).setVersion(0);
            cookieStore.addCookie(sessionCookie);
        }

        client.setCookieStore(cookieStore);
        HttpContext localContext = new BasicHttpContext();
        localContext.setAttribute(ClientContext.COOKIE_STORE, cookieStore);

        HttpResponse httpResponse = client.execute(uploadRequest);

        // Let's update the cookie have the name 'sessionid'
        for (Cookie aCookie : client.getCookieStore().getCookies()) {
            if (aCookie.getName().equals(MINUS_COOKIE_NAME)) {
                sessionCookie = aCookie;
                break;
            }
        }

        StringBuilder result = new StringBuilder();
        int statusCode = httpResponse.getStatusLine().getStatusCode();

        if (statusCode == HttpStatus.SC_OK) {
            HttpEntity respEntity = httpResponse.getEntity();
            if (respEntity != null) {
                result.append(EntityUtils.toString(respEntity));
                EntityUtils.consume(respEntity);
            }

        } else {
            // The response code is not OK.
            StringBuilder errMsg = new StringBuilder();
            errMsg.append(" Upload failed => ").append(httpResponse.getStatusLine());
            if (uploadRequest != null) {
                errMsg.append(" : ").append(uploadRequest.getURI());
            }
            throw new MinusException(errMsg.toString());
        }

        return new MinusHttpResponse(result.toString(), sessionCookie);

    } catch (Exception e) {
        if (uploadRequest != null) {
            uploadRequest.abort();
        }
        String errMsg = "Error while uploading file (" + fileToUpload + ") : " + e.getMessage();
        throw new MinusException(errMsg, e);

    } finally {
        if (client != null) {
            client.getConnectionManager().shutdown();
        }
        CommonUtils.closeAllStreamsQuietly(responseContent);
    }
}

From source file:MinimalServerTest.java

License:asdf

public void testEndOfSession() throws Exception {
    //Create client
    HttpClient client = new DefaultHttpClient();
    HttpPost mockRequest = new HttpPost("http://localhost:5555/login");
    CookieStore cookieStore = new BasicCookieStore();
    HttpContext httpContext = new BasicHttpContext();
    httpContext.setAttribute(ClientContext.COOKIE_STORE, cookieStore);
    mockRequest.setHeader("Content-type", "application/x-www-form-urlencoded");

    //Add parameters
    List<NameValuePair> urlParameters = new ArrayList<>();
    urlParameters.add(new BasicNameValuePair("email", "test"));
    urlParameters.add(new BasicNameValuePair("deviceUID", "BD655C43-3A73-4DFB-AA1F-074A4F0B0DCE"));
    mockRequest.setEntity(new UrlEncodedFormEntity(urlParameters, "UTF-8"));
    //Execute the request
    HttpResponse mockResponse = client.execute(mockRequest, httpContext);

    //Test if normal login is successful
    BufferedReader rd = new BufferedReader(new InputStreamReader(mockResponse.getEntity().getContent()));
    rd.close();/*from  w  ww.jav  a  2s . co m*/
    HttpPost mockRequest2 = new HttpPost("http://localhost:5555/start");
    mockRequest2.setHeader("Content-type", "application/x-www-form-urlencoded");
    //Add parameters
    HttpResponse mockResponse2 = client.execute(mockRequest2, httpContext);
    rd = new BufferedReader(new InputStreamReader(mockResponse2.getEntity().getContent()));
    rd.close();
    HttpPost mockRequest1 = new HttpPost("http://localhost:5555/nextImage");
    mockRequest2.setHeader("Content-type", "application/x-www-form-urlencoded");
    //Add parameters
    List<NameValuePair> urlParameters1 = new ArrayList<>();
    urlParameters1.add(new BasicNameValuePair("deviceUID", "BD655C43-3A73-4DFB-AA1F-074A4F0B0DCE"));
    urlParameters1.add(new BasicNameValuePair("comment", "asdf"));
    urlParameters1.add(new BasicNameValuePair("result", "3"));
    mockRequest1.setEntity(new UrlEncodedFormEntity(urlParameters1, "UTF-8"));
    HttpResponse mockResponse1 = client.execute(mockRequest1, httpContext);
    rd = new BufferedReader(new InputStreamReader(mockResponse1.getEntity().getContent()));
    System.out.println("++++ " + rd.readLine());
    //                "status": "1", //0 if the app should keep waiting, 1 for success, 2 if the votong session has fininshed
    //              "sessionType": "normal", //alternatively Yes/No or winner
    //              "rangeBottom": "0",
    //              "rangeTop": "15",
    //              "description": "image discription here",
    //              "comments": "True",  //True if comments are allowed, False if not
    //              "imgPath": "path/to/image.jpg" //the path where the image resides on the server

    assertEquals("Testing if login was correctly failed due to incorrect username", true, true);
}

From source file:org.opendatakit.briefcase.util.WebUtils.java

public static HttpContext createHttpContext() {
    // set up one context for all HTTP requests so that authentication
    // and cookies can be retained.
    HttpContext localContext = new SyncBasicHttpContext(new BasicHttpContext());

    // establish a local cookie store for this attempt at downloading...
    CookieStore cookieStore = new BasicCookieStore();
    localContext.setAttribute(ClientContext.COOKIE_STORE, cookieStore);

    // and establish a credentials provider...
    CredentialsProvider credsProvider = new BasicCredentialsProvider();
    localContext.setAttribute(ClientContext.CREDS_PROVIDER, credsProvider);

    return localContext;
}

From source file:com.eviware.soapui.impl.wsdl.support.http.HttpClientSupport.java

public static BasicHttpContext createEmptyContext() {
    BasicHttpContext httpContext = new BasicHttpContext();

    // always use local cookie store so we don't share cookies with other threads/executions/requests
    CookieStore cookieStore = new BasicCookieStore();
    httpContext.setAttribute(ClientContext.COOKIE_STORE, cookieStore);

    return httpContext;
}

From source file:anhttpclient.impl.DefaultWebBrowser.java

/**
 * Execute specified request/*  w  ww.j  av a  2s.  c  om*/
 *
 * @param httpUriRequest request to execute
 * @return code of http response
 * @throws java.io.IOException if errors occurs during request
 * @throws java.net.SocketTimeoutException if timeout occurs see params
 *          settings in {@link #setDefaultMethodParams} method
 */
private HttpResponse executeMethod(HttpUriRequest httpUriRequest) throws IOException {
    if (log.isDebugEnabled()) {
        for (Header header : httpUriRequest.getAllHeaders()) {
            log.debug(String.format("ANHTTPCLIENT. Request header: [%s: %s]", header.getName(),
                    header.getValue()));
        }
    }

    HttpContext localContext = new BasicHttpContext();
    localContext.setAttribute(ClientContext.COOKIE_STORE, cookieStore);
    return httpClient.execute(httpUriRequest, localContext);
}

From source file:org.opendatakit.dwc.server.GreetingServiceImpl.java

public static synchronized final HttpContext getHttpContext() {
    if (localContext == null) {
        // set up one context for all HTTP requests so that authentication
        // and cookies can be retained.
        localContext = new SyncBasicHttpContext(new BasicHttpContext());

        // establish a local cookie store for this attempt at downloading...
        CookieStore cookieStore = new BasicCookieStore();
        localContext.setAttribute(ClientContext.COOKIE_STORE, cookieStore);

        // and establish a credentials provider.
        CredentialsProvider credsProvider = new BasicCredentialsProvider();
        localContext.setAttribute(ClientContext.CREDS_PROVIDER, credsProvider);
    }/*from   w  w w.ja  v  a2  s  . c o  m*/
    return localContext;

}

From source file:gtu.youtube.JavaYoutubeVideoUrlHandler.java

public List<NameValuePair> getVideoInfo(String videoId, String format, String userAgent) {
    try {/*from   www  . j  a v  a  2  s.com*/
        List<NameValuePair> infoMap = new ArrayList<NameValuePair>();
        List<NameValuePair> qparams = new ArrayList<NameValuePair>();
        qparams.add(new BasicNameValuePair("video_id", videoId));
        qparams.add(new BasicNameValuePair("fmt", "" + format));
        URI uri = getUri("get_video_info", qparams);

        if (StringUtils.isBlank(userAgent)) {
            userAgent = DEFAULT_USER_AGENT;
        }

        CookieStore cookieStore = new BasicCookieStore();
        HttpContext localContext = new BasicHttpContext();
        localContext.setAttribute(ClientContext.COOKIE_STORE, cookieStore);

        HttpClient httpclient = new DefaultHttpClient();
        HttpGet httpget = new HttpGet(uri);
        if (userAgent != null && userAgent.length() > 0) {
            httpget.setHeader("User-Agent", userAgent);
        }

        HttpResponse response = httpclient.execute(httpget, localContext);
        HttpEntity entity = response.getEntity();
        if (entity != null && response.getStatusLine().getStatusCode() == 200) {
            InputStream instream = entity.getContent();
            String videoInfo = getStringFromInputStream(DEFAULT_ENCODING, instream);
            URLEncodedUtils.parse(infoMap, new Scanner(videoInfo), DEFAULT_ENCODING);
        }
        return infoMap;
    } catch (Exception ex) {
        throw new RuntimeException("play Err : " + ex.getMessage(), ex);
    }
}

From source file:com.heaptrip.util.http.bixo.fetcher.SimpleHttpFetcher.java

private FetchedResult doRequest(HttpRequestBase request, String url, List<TupleTwo<?, ?>> data,
        List<TupleTwo<?, ?>> headers) throws BaseFetchException {
    LOGGER.trace("Fetching " + url);

    HttpResponse response;/*from   w w  w  . ja v a  2 s. c o m*/
    long readStartTime;
    HttpHeaders headerMap = new HttpHeaders();
    String redirectedUrl = null;
    String newBaseUrl = null;
    int numRedirects = 0;
    boolean needAbort = true;
    String contentType = "";
    String hostAddress = null;

    // Create a local instance of cookie store, and bind to local context
    // Without this we get killed w/lots of threads, due to sync() on single
    // cookie store.
    HttpContext localContext = new BasicHttpContext();
    CookieStore cookieStore = new BasicCookieStore();
    localContext.setAttribute(ClientContext.COOKIE_STORE, cookieStore);

    try {
        URI uri = new URI(url);
        request.setURI(uri);
        request.setHeader("Host", uri.getHost());

        if (headers != null) {
            for (TupleTwo<?, ?> t : headers) {
                request.setHeader(t.getKey().toString(), t.getValue().toString());
            }
        }

        //collect post data if available
        if (request instanceof HttpPost && data != null) {
            List<NameValuePair> nameValuePairs = new ArrayList<NameValuePair>(1);
            for (TupleTwo<?, ?> e : data) {
                nameValuePairs.add(new BasicNameValuePair(URLEncoder.encode(e.getKey().toString(), "utf-8"),
                        URLEncoder.encode(e.getValue().toString(), "utf-8")));
            }
            ((HttpPost) (request)).setEntity(new UrlEncodedFormEntity(nameValuePairs));
        }

        readStartTime = System.currentTimeMillis();
        response = _httpClient.execute(request, localContext);

        Header[] responseHeaders = response.getAllHeaders();
        for (Header header : responseHeaders) {
            headerMap.add(header.getName(), header.getValue());
        }

        int httpStatus = response.getStatusLine().getStatusCode();
        if ((httpStatus < 200) || (httpStatus >= 300)) {
            // We can't just check against SC_OK, as some wackos return 201, 202,
            // etc
            throw new HttpFetchException(url,
                    "Error fetching " + url + " due to http status code " + httpStatus, httpStatus, headerMap);
        }

        redirectedUrl = extractRedirectedUrl(url, localContext);

        URI permRedirectUri = (URI) localContext.getAttribute(PERM_REDIRECT_CONTEXT_KEY);
        if (permRedirectUri != null) {
            newBaseUrl = permRedirectUri.toURL().toExternalForm();
        }

        Integer redirects = (Integer) localContext.getAttribute(REDIRECT_COUNT_CONTEXT_KEY);
        if (redirects != null) {
            numRedirects = redirects.intValue();
        }

        hostAddress = (String) (localContext.getAttribute(HOST_ADDRESS));
        if (hostAddress == null) {
            throw new UrlFetchException(url, "Host address not saved in context");
        }

        Header cth = response.getFirstHeader(HttpHeaderNames.CONTENT_TYPE);
        if (cth != null) {
            contentType = cth.getValue();
        }

        needAbort = false;
    } catch (ClientProtocolException e) {
        // Oleg guarantees that no abort is needed in the case of an IOException
        // (which is is a subclass of)
        needAbort = false;

        // If the root case was a "too many redirects" error, we want to map this
        // to a specific
        // exception that contains the final redirect.
        if (e.getCause() instanceof MyRedirectException) {
            MyRedirectException mre = (MyRedirectException) e.getCause();
            String redirectUrl = url;

            try {
                redirectUrl = mre.getUri().toURL().toExternalForm();
            } catch (MalformedURLException e2) {
                LOGGER.warn("Invalid URI saved during redirect handling: " + mre.getUri());
            }

            throw new RedirectFetchException(url, redirectUrl, mre.getReason());
        } else if (e.getCause() instanceof RedirectException) {
            throw new RedirectFetchException(url, extractRedirectedUrl(url, localContext),
                    RedirectExceptionReason.TOO_MANY_REDIRECTS);
        } else {
            throw new IOFetchException(url, e);
        }
    } catch (IOException e) {
        // Oleg guarantees that no abort is needed in the case of an IOException
        needAbort = false;

        if (e instanceof ConnectionPoolTimeoutException) {
            // Should never happen, so let's dump some info about the connection
            // pool.
            ThreadSafeClientConnManager cm = (ThreadSafeClientConnManager) _httpClient.getConnectionManager();
            int numConnections = cm.getConnectionsInPool();
            cm.closeIdleConnections(0, TimeUnit.MILLISECONDS);
            LOGGER.error(String.format(
                    "Got ConnectionPoolTimeoutException: %d connections before, %d after idle close",
                    numConnections, cm.getConnectionsInPool()));
        }

        throw new IOFetchException(url, e);
    } catch (URISyntaxException e) {
        throw new UrlFetchException(url, e.getMessage());
    } catch (IllegalStateException e) {
        throw new UrlFetchException(url, e.getMessage());
    } catch (BaseFetchException e) {
        throw e;
    } catch (Exception e) {
        // Map anything else to a generic IOFetchException
        // TODO KKr - create generic fetch exception
        throw new IOFetchException(url, new IOException(e));
    } finally {
        safeAbort(needAbort, request);
    }

    // Figure out how much data we want to try to fetch.
    int targetLength = _fetcherPolicy.getMaxContentSize();
    boolean truncated = false;
    String contentLengthStr = headerMap.getFirst(HttpHeaderNames.CONTENT_LENGTH);
    if (contentLengthStr != null) {
        try {
            int contentLength = Integer.parseInt(contentLengthStr);
            if (contentLength > targetLength) {
                truncated = true;
            } else {
                targetLength = contentLength;
            }
        } catch (NumberFormatException e) {
            // Ignore (and log) invalid content length values.
            LOGGER.warn("Invalid content length in header: " + contentLengthStr);
        }
    }

    // Now finally read in response body, up to targetLength bytes.
    // Note that entity might be null, for zero length responses.
    byte[] content = new byte[0];
    long readRate = 0;
    HttpEntity entity = response.getEntity();
    needAbort = true;

    if (entity != null) {
        InputStream in = null;

        try {
            in = entity.getContent();
            byte[] buffer = new byte[BUFFER_SIZE];
            int bytesRead = 0;
            int totalRead = 0;
            ByteArrayOutputStream out = new ByteArrayOutputStream(DEFAULT_BYTEARRAY_SIZE);

            int readRequests = 0;
            int minResponseRate = _fetcherPolicy.getMinResponseRate();
            // TODO KKr - we need to monitor the rate while reading a
            // single block. Look at HttpClient
            // metrics support for how to do this. Once we fix this, fix
            // the test to read a smaller (< 20K)
            // chuck of data.
            while ((totalRead < targetLength) && ((bytesRead = in.read(buffer, 0,
                    Math.min(buffer.length, targetLength - totalRead))) != -1)) {
                readRequests += 1;
                totalRead += bytesRead;
                out.write(buffer, 0, bytesRead);

                // Assume read time is at least one millisecond, to avoid DBZ
                // exception.
                long totalReadTime = Math.max(1, System.currentTimeMillis() - readStartTime);
                readRate = (totalRead * 1000L) / totalReadTime;

                // Don't bail on the first read cycle, as we can get a hiccup starting
                // out.
                // Also don't bail if we've read everything we need.
                if ((readRequests > 1) && (totalRead < targetLength) && (readRate < minResponseRate)) {
                    throw new AbortedFetchException(url, "Slow response rate of " + readRate + " bytes/sec",
                            AbortedFetchReason.SLOW_RESPONSE_RATE);
                }

                // Check to see if we got interrupted.
                if (Thread.interrupted()) {
                    throw new AbortedFetchException(url, AbortedFetchReason.INTERRUPTED);
                }
            }

            content = out.toByteArray();
            needAbort = truncated || (in.available() > 0);
        } catch (IOException e) {
            // We don't need to abort if there's an IOException
            throw new IOFetchException(url, e);
        } finally {
            safeAbort(needAbort, request);
            safeClose(in);
        }
    }

    return new FetchedResult(url, redirectedUrl, System.currentTimeMillis(), headerMap, content, contentType,
            (int) readRate, newBaseUrl, numRedirects, hostAddress);
}

From source file:bixo.fetcher.SimpleHttpFetcher.java

private FetchedResult doRequest(HttpRequestBase request, String url, List<Tuple2<?, ?>> data,
        List<Tuple2<?, ?>> headers) throws BaseFetchException {
    LOGGER.trace("Fetching " + url);

    HttpResponse response;/*from w  w w . j a v a 2s .  c  o  m*/
    long readStartTime;
    HttpHeaders headerMap = new HttpHeaders();
    String redirectedUrl = null;
    String newBaseUrl = null;
    int numRedirects = 0;
    boolean needAbort = true;
    String contentType = "";
    String hostAddress = null;

    // Create a local instance of cookie store, and bind to local context
    // Without this we get killed w/lots of threads, due to sync() on single
    // cookie store.
    HttpContext localContext = new BasicHttpContext();
    CookieStore cookieStore = new BasicCookieStore();
    localContext.setAttribute(ClientContext.COOKIE_STORE, cookieStore);

    try {
        URI uri = new URI(url);
        request.setURI(uri);
        request.setHeader("Host", uri.getHost());

        if (headers != null) {
            for (Tuple2<?, ?> t : headers) {
                request.setHeader(t.getKey().toString(), t.getValue().toString());
            }
        }

        //collect post data if available
        if (request instanceof HttpPost && data != null) {
            List<NameValuePair> nameValuePairs = new ArrayList<NameValuePair>(1);
            for (Tuple2<?, ?> e : data) {
                nameValuePairs.add(new BasicNameValuePair(URLEncoder.encode(e.getKey().toString(), "utf-8"),
                        URLEncoder.encode(e.getValue().toString(), "utf-8")));
            }
            ((HttpPost) (request)).setEntity(new UrlEncodedFormEntity(nameValuePairs));
        }

        readStartTime = System.currentTimeMillis();
        response = _httpClient.execute(request, localContext);

        Header[] responseHeaders = response.getAllHeaders();
        for (Header header : responseHeaders) {
            headerMap.add(header.getName(), header.getValue());
        }

        int httpStatus = response.getStatusLine().getStatusCode();
        if ((httpStatus < 200) || (httpStatus >= 300)) {
            // We can't just check against SC_OK, as some wackos return 201, 202,
            // etc
            throw new HttpFetchException(url,
                    "Error fetching " + url + " due to http status code " + httpStatus, httpStatus, headerMap);
        }

        redirectedUrl = extractRedirectedUrl(url, localContext);

        URI permRedirectUri = (URI) localContext.getAttribute(PERM_REDIRECT_CONTEXT_KEY);
        if (permRedirectUri != null) {
            newBaseUrl = permRedirectUri.toURL().toExternalForm();
        }

        Integer redirects = (Integer) localContext.getAttribute(REDIRECT_COUNT_CONTEXT_KEY);
        if (redirects != null) {
            numRedirects = redirects.intValue();
        }

        hostAddress = (String) (localContext.getAttribute(HOST_ADDRESS));
        if (hostAddress == null) {
            throw new UrlFetchException(url, "Host address not saved in context");
        }

        Header cth = response.getFirstHeader(HttpHeaderNames.CONTENT_TYPE);
        if (cth != null) {
            contentType = cth.getValue();
        }

        needAbort = false;
    } catch (ClientProtocolException e) {
        // Oleg guarantees that no abort is needed in the case of an IOException
        // (which is is a subclass of)
        needAbort = false;

        // If the root case was a "too many redirects" error, we want to map this
        // to a specific
        // exception that contains the final redirect.
        if (e.getCause() instanceof MyRedirectException) {
            MyRedirectException mre = (MyRedirectException) e.getCause();
            String redirectUrl = url;

            try {
                redirectUrl = mre.getUri().toURL().toExternalForm();
            } catch (MalformedURLException e2) {
                LOGGER.warn("Invalid URI saved during redirect handling: " + mre.getUri());
            }

            throw new RedirectFetchException(url, redirectUrl, mre.getReason());
        } else if (e.getCause() instanceof RedirectException) {
            throw new RedirectFetchException(url, extractRedirectedUrl(url, localContext),
                    RedirectExceptionReason.TOO_MANY_REDIRECTS);
        } else {
            throw new IOFetchException(url, e);
        }
    } catch (IOException e) {
        // Oleg guarantees that no abort is needed in the case of an IOException
        needAbort = false;

        if (e instanceof ConnectionPoolTimeoutException) {
            // Should never happen, so let's dump some info about the connection
            // pool.
            ThreadSafeClientConnManager cm = (ThreadSafeClientConnManager) _httpClient.getConnectionManager();
            int numConnections = cm.getConnectionsInPool();
            cm.closeIdleConnections(0, TimeUnit.MILLISECONDS);
            LOGGER.error(String.format(
                    "Got ConnectionPoolTimeoutException: %d connections before, %d after idle close",
                    numConnections, cm.getConnectionsInPool()));
        }

        throw new IOFetchException(url, e);
    } catch (URISyntaxException e) {
        throw new UrlFetchException(url, e.getMessage());
    } catch (IllegalStateException e) {
        throw new UrlFetchException(url, e.getMessage());
    } catch (BaseFetchException e) {
        throw e;
    } catch (Exception e) {
        // Map anything else to a generic IOFetchException
        // TODO KKr - create generic fetch exception
        throw new IOFetchException(url, new IOException(e));
    } finally {
        safeAbort(needAbort, request);
    }

    // Figure out how much data we want to try to fetch.
    int targetLength = _fetcherPolicy.getMaxContentSize();
    boolean truncated = false;
    String contentLengthStr = headerMap.getFirst(HttpHeaderNames.CONTENT_LENGTH);
    if (contentLengthStr != null) {
        try {
            int contentLength = Integer.parseInt(contentLengthStr);
            if (contentLength > targetLength) {
                truncated = true;
            } else {
                targetLength = contentLength;
            }
        } catch (NumberFormatException e) {
            // Ignore (and log) invalid content length values.
            LOGGER.warn("Invalid content length in header: " + contentLengthStr);
        }
    }

    // Now finally read in response body, up to targetLength bytes.
    // Note that entity might be null, for zero length responses.
    byte[] content = new byte[0];
    long readRate = 0;
    HttpEntity entity = response.getEntity();
    needAbort = true;

    if (entity != null) {
        InputStream in = null;

        try {
            in = entity.getContent();
            byte[] buffer = new byte[BUFFER_SIZE];
            int bytesRead = 0;
            int totalRead = 0;
            ByteArrayOutputStream out = new ByteArrayOutputStream(DEFAULT_BYTEARRAY_SIZE);

            int readRequests = 0;
            int minResponseRate = _fetcherPolicy.getMinResponseRate();
            // TODO KKr - we need to monitor the rate while reading a
            // single block. Look at HttpClient
            // metrics support for how to do this. Once we fix this, fix
            // the test to read a smaller (< 20K)
            // chuck of data.
            while ((totalRead < targetLength) && ((bytesRead = in.read(buffer, 0,
                    Math.min(buffer.length, targetLength - totalRead))) != -1)) {
                readRequests += 1;
                totalRead += bytesRead;
                out.write(buffer, 0, bytesRead);

                // Assume read time is at least one millisecond, to avoid DBZ
                // exception.
                long totalReadTime = Math.max(1, System.currentTimeMillis() - readStartTime);
                readRate = (totalRead * 1000L) / totalReadTime;

                // Don't bail on the first read cycle, as we can get a hiccup starting
                // out.
                // Also don't bail if we've read everything we need.
                if ((readRequests > 1) && (totalRead < targetLength) && (readRate < minResponseRate)) {
                    throw new AbortedFetchException(url, "Slow response rate of " + readRate + " bytes/sec",
                            AbortedFetchReason.SLOW_RESPONSE_RATE);
                }

                // Check to see if we got interrupted.
                if (Thread.interrupted()) {
                    throw new AbortedFetchException(url, AbortedFetchReason.INTERRUPTED);
                }
            }

            content = out.toByteArray();
            needAbort = truncated || (in.available() > 0);
        } catch (IOException e) {
            // We don't need to abort if there's an IOException
            throw new IOFetchException(url, e);
        } finally {
            safeAbort(needAbort, request);
            safeClose(in);
        }
    }

    return new FetchedResult(url, redirectedUrl, System.currentTimeMillis(), headerMap, content, contentType,
            (int) readRate, newBaseUrl, numRedirects, hostAddress);
}

From source file:de.geeksfactory.opacclient.apis.BaseApi.java

/**
 * Perform a HTTP GET request to a given URL
 *
 * @param url           URL to fetch/* w  w w .j  av  a2s .  c o m*/
 * @param encoding      Expected encoding of the response body
 * @param ignore_errors If true, status codes above 400 do not raise an exception
 * @param cookieStore   If set, the given cookieStore is used instead of the built-in one.
 * @return Answer content
 * @throws NotReachableException Thrown when server returns a HTTP status code greater or equal
 *                               than 400.
 */
public String httpGet(String url, String encoding, boolean ignore_errors, CookieStore cookieStore)
        throws IOException {

    HttpGet httpget = new HttpGet(cleanUrl(url));
    HttpResponse response;
    String html;
    httpget.setHeader("Accept", "*/*");

    try {
        if (cookieStore != null) {
            // Create local HTTP context
            HttpContext localContext = new BasicHttpContext();
            // Bind custom cookie store to the local context
            localContext.setAttribute(ClientContext.COOKIE_STORE, cookieStore);

            response = http_client.execute(httpget, localContext);
        } else {
            response = http_client.execute(httpget);
        }

        if (!ignore_errors && response.getStatusLine().getStatusCode() >= 400) {
            HttpUtils.consume(response.getEntity());
            throw new NotReachableException(response.getStatusLine().getReasonPhrase());
        }

        html = convertStreamToString(response.getEntity().getContent(), encoding);
        HttpUtils.consume(response.getEntity());
    } catch (javax.net.ssl.SSLPeerUnverifiedException e) {
        logHttpError(e);
        throw new SSLSecurityException(e.getMessage());
    } catch (javax.net.ssl.SSLException e) {
        // Can be "Not trusted server certificate" or can be a
        // aborted/interrupted handshake/connection
        if (e.getMessage().contains("timed out") || e.getMessage().contains("reset by")) {
            logHttpError(e);
            throw new NotReachableException(e.getMessage());
        } else {
            logHttpError(e);
            throw new SSLSecurityException(e.getMessage());
        }
    } catch (InterruptedIOException e) {
        logHttpError(e);
        throw new NotReachableException(e.getMessage());
    } catch (UnknownHostException e) {
        throw new NotReachableException(e.getMessage());
    } catch (IOException e) {
        if (e.getMessage() != null && e.getMessage().contains("Request aborted")) {
            logHttpError(e);
            throw new NotReachableException(e.getMessage());
        } else {
            throw e;
        }
    }
    return html;
}