Example usage for org.apache.http.client.methods CloseableHttpResponse getLastHeader

List of usage examples for org.apache.http.client.methods CloseableHttpResponse getLastHeader

  1. HOME
  2. Java
  3. org.apache.http
  4. org.apache.http.client.methods.*
  5. CloseableHttpResponse
  6. getLastHeader

Introduction

In this page you can find the example usage for org.apache.http.client.methods CloseableHttpResponse getLastHeader.

Prototype

Header getLastHeader(String str);

Source Link

Usage

From source file:com.webarch.common.net.http.HttpService.java

/**
 * /* ww  w .  j  a v a2 s .c o m*/
 * @param url ?
 * @param localPath ?
 * @param fileName ??
 * @return null/???
 */
public static String downloadFile(String url, String localPath, String fileName) {
    int bufferSize = 1024 * 10;
    String file = null;
    if (!localPath.endsWith(File.separator)) {
        localPath += File.separator;
    }
    CloseableHttpClient httpclient = HttpClients.createDefault();
    CloseableHttpResponse response = null;
    InputStream inputStream = null;
    FileOutputStream outputStream = null;
    try {
        HttpGet httpGet = new HttpGet(url);
        response = httpclient.execute(httpGet);
        String fileType = "dat";
        Header header = response.getLastHeader("Content-Type");
        if (header != null) {
            String headerValue = header.getValue();
            fileType = headerValue.substring(headerValue.indexOf("/") + 1, headerValue.length());
        }
        HttpEntity entity = response.getEntity();
        long length = entity.getContentLength();
        if (length <= 0) {
            logger.warn("???");
            return file;
        }
        inputStream = entity.getContent();
        file = fileName + "." + fileType;
        File outFile = new File(localPath);
        if (!outFile.exists()) {
            outFile.mkdirs();
        }
        localPath += file;
        outFile = new File(localPath);
        if (!outFile.exists()) {
            outFile.createNewFile();
        }
        outputStream = new FileOutputStream(outFile);
        byte[] buffer = new byte[bufferSize];
        int len;
        while ((len = inputStream.read(buffer)) != -1) {
            outputStream.write(buffer, 0, len);
        }
        inputStream.close();
        outputStream.close();
        return file;
    } catch (IOException e) {
        logger.error("?", e);
    } finally {
        if (inputStream != null) {
            try {
                inputStream.close();
            } catch (IOException e) {
                logger.error("?", e);
            }
        }
        if (outputStream != null) {
            try {
                outputStream.close();
            } catch (IOException e) {
                logger.error("??", e);
            }
        }
    }
    return file;
}

From source file:net.oebs.jalos.Client.java

public String get(long id) throws IOException {
    HttpGet httpGet = new HttpGet(this.serviceUrl + "/a/" + id);
    CloseableHttpResponse response = httpClient.execute(httpGet);
    HttpEntity entity = response.getEntity();
    String ret = response.getLastHeader("Location").toString();
    EntityUtils.consume(entity);/*from w  w w .j  a  v a2s  . com*/
    response.close();
    return ret;
}

From source file:org.esigate.cache.CacheAdapter.java

public ClientExecChain wrapCachingHttpClient(final ClientExecChain wrapped) {
    return new ClientExecChain() {

        /**//from   www.j av a  2  s . c  om
         * Removes client http cache directives like "Cache-control" and "Pragma". Users must not be able to bypass
         * the cache just by making a refresh in the browser. Generates X-cache header.
         * 
         */
        @Override
        public CloseableHttpResponse execute(HttpRoute route, HttpRequestWrapper request,
                HttpClientContext httpClientContext, HttpExecutionAware execAware)
                throws IOException, HttpException {
            OutgoingRequestContext context = OutgoingRequestContext.adapt(httpClientContext);

            // Switch route for the cache to generate the right cache key
            CloseableHttpResponse response = wrapped.execute(route, request, context, execAware);

            // Remove previously added Cache-control header
            if (request.getRequestLine().getMethod().equalsIgnoreCase("GET")
                    && (staleWhileRevalidate > 0 || staleIfError > 0)) {
                response.removeHeader(response.getLastHeader("Cache-control"));
            }
            // Add X-cache header
            if (xCacheHeader) {
                if (context != null) {
                    CacheResponseStatus cacheResponseStatus = (CacheResponseStatus) context
                            .getAttribute(HttpCacheContext.CACHE_RESPONSE_STATUS);
                    String xCacheString;
                    if (cacheResponseStatus.equals(CacheResponseStatus.CACHE_HIT)) {
                        xCacheString = "HIT";
                    } else if (cacheResponseStatus.equals(CacheResponseStatus.VALIDATED)) {
                        xCacheString = "VALIDATED";
                    } else {
                        xCacheString = "MISS";
                    }
                    xCacheString += " from " + route.getTargetHost().toHostString();
                    xCacheString += " (" + request.getRequestLine().getMethod() + " "
                            + request.getRequestLine().getUri() + ")";
                    response.addHeader("X-Cache", xCacheString);
                }
            }

            // Remove Via header
            if (!viaHeader && response.containsHeader("Via")) {
                response.removeHeaders("Via");
            }
            return response;
        }
    };
}

From source file:de.jetwick.snacktory.HtmlFetcher.java

/**
 * On some devices we have to hack: http://developers.sun.com/mobility/reference/techart/design_guidelines/http_redirection.html
 *
 * @param timeout Sets a specified timeout value, in milliseconds
 * @return the resolved url if any. Or null if it couldn't resolve the url (within the specified time) or the same url if response code is OK
 *//*from w  w w .j  av a 2s.  c o  m*/
public String getResolvedUrl(String urlAsString, int timeout) {
    int responseCode = -1;
    String newUrl = null;
    try {
        urlAsString = urlAsString.replace("https", "http");
        CloseableHttpResponse response = createUrlConnection(urlAsString, timeout, true, true);
        responseCode = response.getStatusLine().getStatusCode();
        if (responseCode == HttpStatus.SC_OK)
            return urlAsString;

        Header location = response.getLastHeader("Location");
        if (responseCode / 100 == 3 && location != null) {
            newUrl = location.getValue().replaceAll(" ", "+");
            // some services use (none-standard) utf8 in their location header
            if (urlAsString.startsWith("http://bit.ly") || urlAsString.startsWith("http://is.gd"))
                newUrl = encodeUriFromHeader(newUrl);

            // fix problems if shortened twice. as it is often the case after twitters' t.co bullshit
            if (furtherResolveNecessary.contains(SHelper.extractDomain(newUrl, true)))
                newUrl = getResolvedUrl(newUrl, timeout);

            return newUrl;
        } else
            return urlAsString;

    } catch (Exception ex) {
        logger.warn("getResolvedUrl:" + urlAsString + " Error:" + ex.getMessage(), ex);
        return "";
    } finally {
        if (logger.isDebugEnabled())
            logger.debug(responseCode + " url:" + urlAsString + " resolved:" + newUrl);
    }
}

From source file:at.bitfire.davdroid.webdav.WebDavResource.java

public String put(byte[] data, PutMode mode) throws URISyntaxException, IOException, HttpException {
    HttpPutHC4 put = new HttpPutHC4(location);
    put.setEntity(new ByteArrayEntityHC4(data));

    switch (mode) {
    case ADD_DONT_OVERWRITE:
        put.addHeader("If-None-Match", "*");
        break;/*from ww  w  .  j  a v  a 2 s .  c  o m*/
    case UPDATE_DONT_OVERWRITE:
        put.addHeader("If-Match", (properties.eTag != null) ? properties.eTag : "*");
        break;
    }

    if (properties.contentType != null)
        put.addHeader("Content-Type", properties.contentType.toString());

    @Cleanup
    CloseableHttpResponse response = httpClient.execute(put, context);
    checkResponse(response);

    Header eTag = response.getLastHeader("ETag");
    if (eTag != null)
        return eTag.getValue();

    return null;
}

From source file:com.granita.icloudcalsync.webdav.WebDavResource.java

public String put(byte[] data, PutMode mode) throws URISyntaxException, IOException, HttpException {
    HttpPutHC4 put = new HttpPutHC4(location);
    put.setEntity(new ByteArrayEntityHC4(data));

    switch (mode) {
    case ADD_DONT_OVERWRITE:
        put.addHeader("If-None-Match", "*");
        break;//from w  w w .ja va  2s  . co m
    case UPDATE_DONT_OVERWRITE:
        put.addHeader("If-Match", (getETag() != null) ? getETag() : "*");
        break;
    }

    if (getContentType() != null)
        put.addHeader("Content-Type", getContentType());

    @Cleanup
    CloseableHttpResponse response = httpClient.execute(put, context);
    checkResponse(response);

    Header eTag = response.getLastHeader("ETag");
    if (eTag != null)
        return eTag.getValue();

    return null;
}

From source file:org.xwiki.contrib.repository.bintray.internal.BintrayMavenExtensionRepository.java

@Override
public IterableResult<Extension> search(String pattern, int offset, int limit) throws SearchException {
    String url = null;/*from   w  w w  .ja va2  s.  c  om*/
    try {
        URIBuilder uriBuilder = new URIBuilder(BintrayParameters.BINTRAY_PACKAGE_SEARCH_URL);
        uriBuilder.addParameter(BintrayParameters.BINTRAY_API_SUBJECT_PARAM, subject);
        uriBuilder.addParameter(BintrayParameters.BINTRAY_API_REPO_PARAM, repo);
        uriBuilder.addParameter(BintrayParameters.BINTRAY_API_PACKAGE_SEARCH_NAME_PARAM, pattern);
        uriBuilder.addParameter(BintrayParameters.BINTRAY_API_PAGINATION_START_POS_PARAM, "" + offset);

        url = uriBuilder.build().toString();
    } catch (URISyntaxException e) {
        throw new SearchException("Failed to build REST URL", e);
    }

    HttpGet getMethod = new HttpGet(url);
    CloseableHttpClient httpClient = httpClientFactory.createClient(null, null);
    CloseableHttpResponse response;
    try {
        if (this.localContext != null) {
            response = httpClient.execute(getMethod, this.localContext);
        } else {
            response = httpClient.execute(getMethod);
        }
    } catch (Exception e) {
        throw new SearchException(String.format("Failed to request [%s]", getMethod.getURI()), e);
    }

    if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
        throw new SearchException(String.format("Invalid answer [%s] from the server when requesting [%s]",
                response.getStatusLine().getStatusCode(), getMethod.getURI()));
    }
    BintrayPackages bintrayPackages = null;
    int totalHits = 0;
    try {
        String totalHitsString = response.getLastHeader("X-RangeLimit-Total").getValue();
        totalHits = Integer.parseInt(totalHitsString);
        BintrayPackageDTO[] bintrayPackageDTOs = objectMapper.readValue(response.getEntity().getContent(),
                BintrayPackageDTO[].class);
        bintrayPackages = new BintrayPackages(bintrayPackageDTOs);
    } catch (IOException e) {
        throw new SearchException(
                String.format("Invalid response body from the bintray server when requesting: [%s]", pattern),
                e);
    } finally {
        IOUtils.closeQuietly(response);
        IOUtils.closeQuietly(httpClient);
    }

    bintrayPackages.limitContent(limit);

    return getItarableResultFrom(bintrayPackages, totalHits, offset, logger);
}

From source file:crawler.java.edu.uci.ics.crawler4j.fetcher.PageFetcher.java

public PageFetchResult fetchPage(WebURL webUrl)
        throws InterruptedException, IOException, PageBiggerThanMaxSizeException {
    // Getting URL, setting headers & content
    PageFetchResult fetchResult = new PageFetchResult();
    String toFetchURL = webUrl.getURL();
    HttpUriRequest request = null;//from   w  w  w .java 2s . co  m
    try {
        request = newHttpUriRequest(toFetchURL);
        // Applying Politeness delay
        synchronized (mutex) {
            long now = (new Date()).getTime();
            if ((now - lastFetchTime) < config.getPolitenessDelay()) {
                Thread.sleep(config.getPolitenessDelay() - (now - lastFetchTime));
            }
            lastFetchTime = (new Date()).getTime();
        }

        CloseableHttpResponse response = httpClient.execute(request);
        fetchResult.setEntity(response.getEntity());
        fetchResult.setResponseHeaders(response.getAllHeaders());

        // Setting HttpStatus
        int statusCode = response.getStatusLine().getStatusCode();

        // If Redirect ( 3xx )
        if (statusCode == HttpStatus.SC_MOVED_PERMANENTLY || statusCode == HttpStatus.SC_MOVED_TEMPORARILY
                || statusCode == HttpStatus.SC_MULTIPLE_CHOICES || statusCode == HttpStatus.SC_SEE_OTHER
                || statusCode == HttpStatus.SC_TEMPORARY_REDIRECT || statusCode == 308) { // todo follow https://issues.apache.org/jira/browse/HTTPCORE-389

            Header header = response.getFirstHeader("Location");
            if (header != null) {
                String movedToUrl = URLCanonicalizer.getCanonicalURL(header.getValue(), toFetchURL);
                fetchResult.setMovedToUrl(movedToUrl);
            }
        } else if (statusCode >= 200 && statusCode <= 299) { // is 2XX, everything looks ok
            fetchResult.setFetchedUrl(toFetchURL);
            String uri = request.getURI().toString();
            if (!uri.equals(toFetchURL)) {
                if (!URLCanonicalizer.getCanonicalURL(uri).equals(toFetchURL)) {
                    fetchResult.setFetchedUrl(uri);
                }
            }

            // Checking maximum size
            if (fetchResult.getEntity() != null) {
                long size = fetchResult.getEntity().getContentLength();
                if (size == -1) {
                    Header length = response.getLastHeader("Content-Length");
                    if (length == null) {
                        length = response.getLastHeader("Content-length");
                    }
                    if (length != null) {
                        size = Integer.parseInt(length.getValue());
                    }
                }
                if (size > config.getMaxDownloadSize()) {
                    //fix issue #52 - consume entity
                    response.close();
                    throw new PageBiggerThanMaxSizeException(size);
                }
            }
        }

        fetchResult.setStatusCode(statusCode);
        return fetchResult;

    } finally { // occurs also with thrown exceptions
        if ((fetchResult.getEntity() == null) && (request != null)) {
            request.abort();
        }
    }
}

From source file:crawler.PageFetcher.java

public PageFetchResult fetchPage(WebURL webUrl)
        throws InterruptedException, IOException, PageBiggerThanMaxSizeException {
    // Getting URL, setting headers & content
    PageFetchResult fetchResult = new PageFetchResult();
    String toFetchURL = webUrl.getURL();
    HttpUriRequest request = null;/* w  w w  .j  a  v a  2s  . com*/
    try {
        request = newHttpUriRequest(toFetchURL);
        // Applying Politeness delay
        synchronized (mutex) {
            long now = (new Date()).getTime();
            if ((now - lastFetchTime) < config.getPolitenessDelay()) {
                Thread.sleep(config.getPolitenessDelay() - (now - lastFetchTime));
            }
            lastFetchTime = (new Date()).getTime();
        }

        CloseableHttpResponse response = httpClient.execute(request);
        fetchResult.setEntity(response.getEntity());
        fetchResult.setResponseHeaders(response.getAllHeaders());

        // Setting HttpStatus
        int statusCode = response.getStatusLine().getStatusCode();

        // If Redirect ( 3xx )
        if (statusCode == HttpStatus.SC_MOVED_PERMANENTLY || statusCode == HttpStatus.SC_MOVED_TEMPORARILY
                || statusCode == HttpStatus.SC_MULTIPLE_CHOICES || statusCode == HttpStatus.SC_SEE_OTHER
                || statusCode == HttpStatus.SC_TEMPORARY_REDIRECT || statusCode == 308) { // todo follow
            // https://issues.apache.org/jira/browse/HTTPCORE-389

            Header header = response.getFirstHeader("Location");
            if (header != null) {
                String movedToUrl = URLCanonicalizer.getCanonicalURL(header.getValue(), toFetchURL);
                fetchResult.setMovedToUrl(movedToUrl);
            }
        } else if (statusCode >= 200 && statusCode <= 299) { // is 2XX, everything looks ok
            fetchResult.setFetchedUrl(toFetchURL);
            String uri = request.getURI().toString();
            if (!uri.equals(toFetchURL)) {
                if (!URLCanonicalizer.getCanonicalURL(uri).equals(toFetchURL)) {
                    fetchResult.setFetchedUrl(uri);
                }
            }

            // Checking maximum size
            if (fetchResult.getEntity() != null) {
                long size = fetchResult.getEntity().getContentLength();
                if (size == -1) {
                    Header length = response.getLastHeader("Content-Length");
                    if (length == null) {
                        length = response.getLastHeader("Content-length");
                    }
                    if (length != null) {
                        size = Integer.parseInt(length.getValue());
                    }
                }
                if (size > config.getMaxDownloadSize()) {
                    //fix issue #52 - consume entity
                    response.close();
                    throw new PageBiggerThanMaxSizeException(size);
                }
            }
        }

        fetchResult.setStatusCode(statusCode);
        return fetchResult;

    } finally { // occurs also with thrown exceptions
        if ((fetchResult.getEntity() == null) && (request != null)) {
            request.abort();
        }
    }
}

From source file:org.aliuge.crawler.fetcher.DefaultFetcher.java

public PageFetchResult fetch(WebURL webUrl, boolean proxy) {
    PageFetchResult fetchResult = new PageFetchResult();

    String toFetchURL = webUrl.getUrl();
    HttpGet get = new HttpGet(toFetchURL);
    get.addHeader("Accept-Encoding", "gzip");
    get.addHeader("User-Agent", config.getAgent());

    RequestConfig requestConfig = null;/*from ww w.ja  v  a2 s.c  o m*/
    CloseableHttpResponse response = null;

    synchronized (mutex) {
        long now = (new Date()).getTime();
        if (now - lastFetchTime < ((FetchConfig) config).getDelayBetweenRequests()) {
            try {
                Thread.sleep(((FetchConfig) config).getDelayBetweenRequests() - (now - lastFetchTime));
            } catch (InterruptedException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
        }
        lastFetchTime = (new Date()).getTime();
    }
    int statusCode = 0;
    int count = 5;

    while (statusCode != HttpStatus.SC_OK && count-- > 0) {
        HttpHost proxyHost = null;
        if (proxy) {
            proxyHost = getProxyIp();

            if (proxyHost != null)
                requestConfig = RequestConfig.copy(defaultRequestConfig).setSocketTimeout(10000)
                        .setConnectTimeout(10000).setProxy(proxyHost).build();
        }

        get.setConfig(requestConfig);

        try {
            response = httpClient.execute(get);
            statusCode = response.getStatusLine().getStatusCode();
            fetchResult.setEntity(response.getEntity());
            fetchResult.setResponseHeaders(response.getAllHeaders());
        } catch (IOException e) {
            // e.printStackTrace();
            // log.info("Fatal transport error: " + e.getMessage()+
            // " while fetching " + toFetchURL + " (link found in doc #"+
            // webUrl.getParentDocid() + ")");
            addFailedProxy(proxyHost.toHostString());
            /*
             * if (null != get) get.abort();
             * fetchResult.setStatusCode(CustomFetchStatus
             * .FatalTransportError);
             */
            // return fetchResult;
        }
    }

    fetchResult.setStatusCode(statusCode);

    fetchResult.setFetchedUrl(toFetchURL);

    if (fetchResult.getStatusCode() == HttpStatus.SC_OK) {

        long size = fetchResult.getEntity().getContentLength();
        if (size == -1) {
            Header length = response.getLastHeader("Content-Length");
            if (length == null) {
                length = response.getLastHeader("Content-length");
            }
            if (length != null) {
                size = Integer.parseInt(length.getValue());
            } else {
                size = -1;
            }
        }
        if (size > ((FetchConfig) config).getMaxDownloadSizePerPage()) {
            fetchResult.setStatusCode(CustomFetchStatus.PageTooBig);
            get.abort();
            return fetchResult;
        }
        // fetchResult.setStatusCode(HttpStatus.SC_OK);
        return fetchResult;
    }
    get.abort();

    fetchResult.setStatusCode(CustomFetchStatus.UnknownError);
    return fetchResult;
}