List of usage examples for org.apache.commons.httpclient HttpMethod getResponseHeader
public abstract Header getResponseHeader(String paramString);
From source file:com.kodokux.github.api.GithubApiUtil.java
@NotNull private static ResponsePage request(@NotNull GithubAuthData auth, @NotNull String path, @Nullable String requestBody, @NotNull Collection<Header> headers, @NotNull HttpVerb verb) throws IOException { HttpMethod method = null; try {/*from w ww . java 2s. c o m*/ String uri = GithubUrlUtil.getApiUrl(auth.getHost()) + path; method = doREST(auth, uri, requestBody, headers, verb); checkStatusCode(method); InputStream resp = method.getResponseBodyAsStream(); if (resp == null) { return new ResponsePage(); } JsonElement ret = parseResponse(resp); if (ret.isJsonNull()) { return new ResponsePage(); } Header header = method.getResponseHeader("Link"); if (header != null) { String value = header.getValue(); int end = value.indexOf(">; rel=\"next\""); int begin = value.lastIndexOf('<', end); if (begin >= 0 && end >= 0) { String newPath = GithubUrlUtil.removeProtocolPrefix(value.substring(begin + 1, end)); int index = newPath.indexOf('/'); return new ResponsePage(ret, newPath.substring(index)); } } return new ResponsePage(ret); } finally { if (method != null) { method.releaseConnection(); } } }
From source file:com.cyberway.issue.crawler.deciderules.NotExceedsDocumentLengthTresholdDecideRule.java
protected boolean evaluate(Object object) { try {//from w w w . jav a 2s .co m CrawlURI curi = (CrawlURI) object; int contentlength = HEADER_PREDICTS_MISSING; //filter used as midfetch filter if (getIsMidfetchRule(object)) { if (curi.containsKey(A_HTTP_TRANSACTION) == false) { // Missing header info, let pass if (logger.isLoggable(Level.INFO)) { logger.info("Error: Missing HttpMethod object in " + "CrawlURI. " + curi.toString()); } return false; } // Initially assume header info is missing HttpMethod method = (HttpMethod) curi.getObject(A_HTTP_TRANSACTION); // get content-length String newContentlength = null; if (method.getResponseHeader("content-length") != null) { newContentlength = method.getResponseHeader("content-length").getValue(); } if (newContentlength != null && newContentlength.length() > 0) { try { contentlength = Integer.parseInt(newContentlength); } catch (NumberFormatException nfe) { // Ignore. } } // If no document length was reported or format was wrong, // let pass if (contentlength == HEADER_PREDICTS_MISSING) { return false; } } else { contentlength = (int) curi.getContentSize(); } return makeDecision(contentlength, object); } catch (ClassCastException e) { // if not CrawlURI, always disregard return false; } }
From source file:com.kylinolap.job.tools.HadoopStatusChecker.java
private String getHttpResponse(String url) throws IOException { HttpClient client = new HttpClient(); String response = null;// ww w . jav a 2s. com while (response == null) { // follow redirects via 'refresh' if (url.startsWith("https://")) { registerEasyHttps(); } if (url.contains("anonymous=true") == false) { url += url.contains("?") ? "&" : "?"; url += "anonymous=true"; } HttpMethod get = new GetMethod(url); client.executeMethod(get); String redirect = null; Header h = get.getResponseHeader("Refresh"); if (h != null) { String s = h.getValue(); int cut = s.indexOf("url="); if (cut >= 0) { redirect = s.substring(cut + 4); } } if (redirect == null) { response = get.getResponseBodyAsString(); output.append("Job " + mrJobID + " get status check result.\n"); log.debug("Job " + mrJobID + " get status check result.\n"); } else { url = redirect; output.append("Job " + mrJobID + " check redirect url " + url + ".\n"); log.debug("Job " + mrJobID + " check redirect url " + url + ".\n"); } get.releaseConnection(); } return response; }
From source file:com.honnix.cheater.network.CheaterHttpClient.java
private boolean execute(HttpMethod method) { boolean result = true; try {/*from w w w . ja v a2 s.c o m*/ httpClient.executeMethod(method); // We only follow the redirection once. Header locationHeader = method.getResponseHeader("location"); if (locationHeader != null) { HttpMethod getMethod = new GetMethod(locationHeader.getValue()); httpClient.executeMethod(getMethod); } } catch (Exception e) { LOG.error("Method execution error.", e); result = false; } return result; }
From source file:ar.com.zauber.commons.web.proxy.HttpClientRequestProxy.java
/** @return el contentType */ public final String getContentType(final HttpMethod method) { final Header contenType = method.getResponseHeader("Content-Type"); String ret = null;//w ww .j av a2s . com if (contenType != null) { ret = contenType.getValue(); } return ret; }
From source file:com.cordys.coe.ac.httpconnector.samples.JIRACreateProjectResponseHandler.java
/** * @see com.cordys.coe.ac.httpconnector.samples.JIRAResponseHandler#buildXMLResponse(int,org.w3c.dom.Document, * com.eibus.xml.nom.Document)//from w w w. ja v a2s . c o m */ @Override protected void buildXMLResponse(int resNode, HttpMethod httpMethod, Document document, com.eibus.xml.nom.Document doc) throws Exception { if (httpMethod.getStatusCode() == 302) { // It's a redirect to the details screen for the newly created // project. We'll follow // the redirect. Header location = httpMethod.getResponseHeader("Location"); if (location != null) { String loc = location.getValue(); // Get the PID via a regex Matcher m = PATTERN_PID.matcher(loc); if (m.find()) { loc = m.group(1); // Now we need to get the project ID from the URL. int tuple = doc.createElementWithParentNS("tuple", null, resNode); int old = doc.createElementWithParentNS("old", null, tuple); int project = doc.createElementWithParentNS("project", null, old); doc.createElementWithParentNS("projectid", loc, project); } } } }
From source file:com.cyberway.issue.crawler.filter.HTTPMidFetchUnchangedFilter.java
protected boolean innerAccepts(Object o) { // Return FALSE when the document has NOT changed! // Return TRUE if the document has changed or we can't tell if (o instanceof CrawlURI == false) { // Only handles CrawlURIs if (logger.isLoggable(Level.INFO)) { logger.info("Error: Object passed for evaluation was not a " + "CrawlURI. " + o.toString()); }/* w ww. ja v a 2 s.c om*/ return true; } CrawlURI curi = (CrawlURI) o; if (curi.isHttpTransaction() == false) { // Only handles HTTP if (logger.isLoggable(Level.INFO)) { logger.info("Error: Non HTTP CrawlURI was passed for evalution. " + curi.toString()); } return true; } if (curi.containsKey(A_HTTP_TRANSACTION) == false) { // Missing header info, can't do anything. if (logger.isLoggable(Level.INFO)) { logger.info("Error: Missing HttpMethod object in CrawlURI. " + curi.toString()); } return true; } // Intially assume header info is missing int datestamp = HEADER_PREDICTS_MISSING; int etag = HEADER_PREDICTS_MISSING; HttpMethod method = (HttpMethod) curi.getObject(A_HTTP_TRANSACTION); // Compare datestamps (last-modified) String newDatestamp = null; if (method.getResponseHeader("last-modified") != null) { newDatestamp = method.getResponseHeader("last-modified").getValue(); } if (newDatestamp != null && newDatestamp.length() > 0) { datestamp = HEADER_PREDICTS_CHANGED; // Not missing, assume change if (curi.containsKey(A_LAST_DATESTAMP)) { if (newDatestamp.equals(curi.getString(A_LAST_DATESTAMP))) { // Both new and old are present and equal, datestamp // predicts no change datestamp = HEADER_PREDICTS_UNCHANGED; } } curi.putString(A_LAST_DATESTAMP, newDatestamp); } // Compare ETags String newETag = null; if (method.getResponseHeader("last-etag") != null) { newETag = method.getResponseHeader("last-etag").getValue(); } if (newETag != null && newETag.length() > 0) { etag = HEADER_PREDICTS_CHANGED; // Not missing, assume change if (curi.containsKey(A_LAST_ETAG)) { if (newETag.equals(curi.getString(A_LAST_ETAG))) { // Both new and old are present and equal, etag // predicts no change etag = HEADER_PREDICTS_UNCHANGED; } } curi.putString(A_LAST_ETAG, newETag); } // If both are present, predict no change only if both agree if (datestamp == HEADER_PREDICTS_UNCHANGED && etag == HEADER_PREDICTS_UNCHANGED) { // Have both and they agree, no change curi.putInt(A_CONTENT_STATE_KEY, CONTENT_UNCHANGED); return false; } // If one or the other is missing, trust the one that is present if (datestamp == HEADER_PREDICTS_MISSING && etag == HEADER_PREDICTS_UNCHANGED) { // Only have etag, and it predicts no change curi.putInt(A_CONTENT_STATE_KEY, CONTENT_UNCHANGED); return false; } if (datestamp == HEADER_PREDICTS_UNCHANGED && etag == HEADER_PREDICTS_MISSING) { // Only have last-modified, and it predicts no change curi.putInt(A_CONTENT_STATE_KEY, CONTENT_UNCHANGED); return false; } return true; // Default, assume change. }
From source file:ch.ksfx.web.services.spidering.http.WebEngine.java
private String recognizeMimeType(Resource resource, HttpMethod httpMethod) { String headerMimeType = null; try {/*from ww w .j a v a 2s.c o m*/ headerMimeType = httpMethod.getResponseHeader("Content-Type").getValue().split(";")[0]; } catch (Exception e) { logger.fine("No content type header found."); } //TODO Ouw my god, this mime type checking is a desaster.... if (headerMimeType == null || (!headerMimeType.startsWith("text") && !isBinaryFile(resource.getRawContent()))) { logger.warning("no content-type found try guessing with file extension"); String urlString = resource.getUrl(); if (urlString.endsWith(".htm") || urlString.endsWith(".html")) { logger.fine("resource seems to be html"); headerMimeType = "text/html"; } else if (urlString.endsWith(".xml")) { logger.fine("resource seems to be html"); headerMimeType = "text/xml"; } else if (urlString.endsWith(".csv")) { logger.fine("resource seems to be html"); headerMimeType = "text/csv"; } else if (urlString.endsWith(".jpg") || urlString.endsWith(".jpeg")) { logger.fine("resource seem to be jpeg"); headerMimeType = "image/jpeg"; } else if (urlString.endsWith(".gif")) { logger.fine("resource seem to be gif"); headerMimeType = "image/gif"; } else if (urlString.endsWith(".png")) { logger.fine("resource seem to be png"); headerMimeType = "image/png"; } else { logger.severe("unable to determine content-type; use application/octet-stream instead"); headerMimeType = "application/octet-stream"; } } // check if file is an image ImageInfo imageInfo = new ImageInfo(); imageInfo.setInput(new ByteArrayInputStream(resource.getRawContent())); if (imageInfo.check()) { // image found if (!imageInfo.getMimeType().equals(headerMimeType)) { logger.warning("Recognized image type of resource: " + resource.getUrl() + "does not match header mime type. using " + imageInfo.getMimeType() + " instead."); } headerMimeType = imageInfo.getMimeType(); } else { if (headerMimeType.startsWith("image")) { logger.severe("file extension recognized as image, but content was not."); headerMimeType = "application/octet-stream"; } } return headerMimeType; }
From source file:net.sf.ehcache.constructs.web.filter.PageFragmentCachingFilterTest.java
/** * Tests that a page which is not storeGzipped is not gzipped when the user agent does not accept gzip encoding */// w w w. j av a 2 s.c o m public void testNotGzippedWhenNotAcceptEncodingPageFragment() throws Exception { HttpClient httpClient = new HttpClient(); HttpMethod httpMethod = new GetMethod(buildUrl("/include/Footer.jsp")); httpMethod.setRequestHeader(new Header("Accept-encoding", "gzip")); httpClient.executeMethod(httpMethod); byte[] responseBody = httpMethod.getResponseBody(); assertFalse(PageInfo.isGzipped(responseBody)); assertNotSame("gzip", httpMethod.getResponseHeader("Accept-encoding")); }
From source file:net.sf.ehcache.constructs.web.filter.PageFragmentCachingFilterTest.java
/** * Tests that a page which is not storeGzipped is not gzipped when the user agent accepts gzip encoding *///w w w.ja v a2 s . co m public void testNotGzippedWhenAcceptEncodingPageFragment() throws Exception { HttpClient httpClient = new HttpClient(); HttpMethod httpMethod = new GetMethod(buildUrl("/include/Footer.jsp")); //httpMethod.setRequestHeader(new Header("Accept-encoding", "gzip")); httpClient.executeMethod(httpMethod); byte[] responseBody = httpMethod.getResponseBody(); assertFalse(PageInfo.isGzipped(responseBody)); assertNotSame("gzip", httpMethod.getResponseHeader("Accept-encoding")); }