List of usage examples for org.apache.commons.httpclient HttpVersion HTTP_1_0
HttpVersion HTTP_1_0
To view the source code for org.apache.commons.httpclient HttpVersion HTTP_1_0.
Click Source Link
From source file:com.zimbra.common.util.ngxlookup.ZimbraNginxLookUpClient.java
public NginxAuthServer getRouteforAccount(String userName, String authMethod, String authProtocol, String clientIP, String proxyIP, String virtualHost) throws ServiceException { Route nginxLookUpHandler = getNginxRouteHandler(); ZimbraLog.misc.debug("getting route for account %s with handler %s", userName, nginxLookUpHandler); if (nginxLookUpHandler != null) { for (String scheme : asList(ngxSchemes)) { GetMethod method = new GetMethod((new StringBuilder(scheme + "://") .append(nginxLookUpHandler.ngxServerAddress.getHostName()).append(":") .append(nginxLookUpHandler.ngxServerAddress.getPort()).append(urlExtension)).toString()); method.setRequestHeader("Auth-Method", authMethod); method.setRequestHeader("Auth-User", userName); method.setRequestHeader("Auth-Pass", ngxPassword); method.setRequestHeader("Auth-Protocol", authProtocol); // for web requests, login attempts is always 0 method.setRequestHeader("Auth-Login-Attempt", "0"); method.setRequestHeader("X-Proxy-IP", proxyIP); method.setRequestHeader("Client-IP", clientIP); method.setRequestHeader("X-Proxy-Host", virtualHost); HttpClient client = ZimbraHttpConnectionManager.getInternalHttpConnMgr().newHttpClient(); // currently we use default httpclient_internal_connmgr_connection_timeout instead of ngxConnectTimeout client.getParams().setParameter("http.protocol.version", HttpVersion.HTTP_1_0); try { int statusCode = HttpClientUtil.executeMethod(client, method); if (statusCode == 200 && method.getResponseHeader("Auth-Status").getValue().equals("OK")) { return new NginxAuthServer(method.getResponseHeader("Auth-Server").getValue(), method.getResponseHeader("Auth-Port").getValue(), method.getResponseHeader("Auth-User").getValue()); } else { ZimbraLog.misc.debug("unexpected return %d\r\n%s", statusCode, method.getResponseBodyAsString()); }/*from w w w . ja va 2 s.c o m*/ } catch (IOException e) { nginxLookUpHandler.failureTime = System.nanoTime(); ZimbraLog.misc.debug("IOException getting route", e); } finally { method.releaseConnection(); } } } return null; }
From source file:com.zimbra.soap.SoapServlet.java
private void sendResponse(HttpServletRequest req, HttpServletResponse resp, Element envelope) throws IOException { SoapProtocol soapProto = SoapProtocol.determineProtocol(envelope); int statusCode = soapProto.hasFault(envelope) ? HttpServletResponse.SC_INTERNAL_SERVER_ERROR : HttpServletResponse.SC_OK; boolean chunkingEnabled = LC.soap_response_chunked_transfer_encoding_enabled.booleanValue(); if (chunkingEnabled) { // disable chunking if proto < HTTP 1.1 String proto = req.getProtocol(); try {/*from w ww.j a v a 2 s . c om*/ HttpVersion httpVer = HttpVersion.parse(proto); chunkingEnabled = !httpVer.lessEquals(HttpVersion.HTTP_1_0); } catch (ProtocolException e) { ZimbraLog.soap.warn( "cannot parse http version in request: %s, http chunked transfer encoding disabled", proto, e); chunkingEnabled = false; } } // use jetty default if the LC key is not set int responseBufferSize = soapResponseBufferSize(); if (responseBufferSize != -1) resp.setBufferSize(responseBufferSize); resp.setContentType(soapProto.getContentType()); resp.setStatus(statusCode); resp.setHeader("Cache-Control", "no-store, no-cache"); if (chunkingEnabled) { // Let jetty chunk the response if applicable. ZimbraServletOutputStream out = new ZimbraServletOutputStream(resp.getOutputStream()); envelope.output(out); out.flush(); } else { // serialize the envelope to a byte array and send the response with Content-Length header. byte[] soapBytes = envelope.toUTF8(); resp.setContentLength(soapBytes.length); resp.getOutputStream().write(soapBytes); resp.getOutputStream().flush(); } envelope.destroy(); }
From source file:com.cyberway.issue.crawler.fetcher.FetchHTTP.java
/** * Configure the HttpMethod setting options and headers. * * @param curi CrawlURI from which we pull configuration. * @param method The Method to configure. * @return HostConfiguration copy customized for this CrawlURI *///from w w w .j a va2 s . c om protected HostConfiguration configureMethod(CrawlURI curi, HttpMethod method) { // Don't auto-follow redirects method.setFollowRedirects(false); // // set soTimeout // method.getParams().setSoTimeout( // ((Integer) getUncheckedAttribute(curi, ATTR_SOTIMEOUT_MS)) // .intValue()); // Set cookie policy. method.getParams() .setCookiePolicy((((Boolean) getUncheckedAttribute(curi, ATTR_IGNORE_COOKIES)).booleanValue()) ? CookiePolicy.IGNORE_COOKIES : CookiePolicy.BROWSER_COMPATIBILITY); // Use only HTTP/1.0 (to avoid receiving chunked responses) method.getParams().setVersion(HttpVersion.HTTP_1_0); CrawlOrder order = getSettingsHandler().getOrder(); String userAgent = curi.getUserAgent(); if (userAgent == null) { userAgent = order.getUserAgent(curi); } method.setRequestHeader("User-Agent", userAgent); method.setRequestHeader("From", order.getFrom(curi)); // Set retry handler. method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, new HeritrixHttpMethodRetryHandler()); final long maxLength = getMaxLength(curi); if (maxLength > 0 && ((Boolean) getUncheckedAttribute(curi, ATTR_SEND_RANGE)).booleanValue()) { method.addRequestHeader(RANGE, RANGE_PREFIX.concat(Long.toString(maxLength - 1))); } if (((Boolean) getUncheckedAttribute(curi, ATTR_SEND_CONNECTION_CLOSE)).booleanValue()) { method.addRequestHeader(HEADER_SEND_CONNECTION_CLOSE); } if (((Boolean) getUncheckedAttribute(curi, ATTR_SEND_REFERER)).booleanValue()) { // RFC2616 says no referer header if referer is https and the url // is not String via = curi.flattenVia(); if (via != null && via.length() > 0 && !(via.startsWith(HTTPS_SCHEME) && curi.getUURI().getScheme().equals(HTTP_SCHEME))) { method.setRequestHeader(REFERER, via); } } if (!curi.isPrerequisite()) { setConditionalGetHeader(curi, method, ATTR_SEND_IF_MODIFIED_SINCE, CoreAttributeConstants.A_LAST_MODIFIED_HEADER, "If-Modified-Since"); setConditionalGetHeader(curi, method, ATTR_SEND_IF_NONE_MATCH, CoreAttributeConstants.A_ETAG_HEADER, "If-None-Match"); } // TODO: What happens if below method adds a header already // added above: e.g. Connection, Range, or Referer? setAcceptHeaders(curi, method); HostConfiguration config = new HostConfiguration(http.getHostConfiguration()); configureProxy(curi, config); configureBindAddress(curi, config); return config; }
From source file:com.cyberway.issue.crawler.fetcher.FetchHTTP.java
protected void configureHttp() throws RuntimeException { // Get timeout. Use it for socket and for connection timeout. int timeout = (getSoTimeout(null) > 0) ? getSoTimeout(null) : 0; // HttpConnectionManager cm = new ThreadLocalHttpConnectionManager(); HttpConnectionManager cm = new SingleHttpConnectionManager(); // TODO: The following settings should be made in the corresponding // HttpConnectionManager, not here. HttpConnectionManagerParams hcmp = cm.getParams(); hcmp.setConnectionTimeout(timeout);// w w w .j a v a 2s . c o m hcmp.setStaleCheckingEnabled(true); // Minimizes bandwidth usage. Setting to true disables Nagle's // algorithm. IBM JVMs < 142 give an NPE setting this boolean // on ssl sockets. hcmp.setTcpNoDelay(false); this.http = new HttpClient(cm); HttpClientParams hcp = this.http.getParams(); // Set default socket timeout. hcp.setSoTimeout(timeout); // Set client to be version 1.0. hcp.setVersion(HttpVersion.HTTP_1_0); configureHttpCookies(); // Configure how we want the method to act. this.http.getParams().setParameter(HttpMethodParams.SINGLE_COOKIE_HEADER, new Boolean(true)); this.http.getParams().setParameter(HttpMethodParams.UNAMBIGUOUS_STATUS_LINE, new Boolean(false)); this.http.getParams().setParameter(HttpMethodParams.STRICT_TRANSFER_ENCODING, new Boolean(false)); this.http.getParams().setIntParameter(HttpMethodParams.STATUS_LINE_GARBAGE_LIMIT, 10); // modify the default config with any global settings HostConfiguration config = this.http.getHostConfiguration(); configureProxy(null, config); configureBindAddress(null, config); // Use our own protocol factory, one that gets IP to use from // heritrix cache (They're cached in CrawlHost instances). final ServerCache cache = getController().getServerCache(); hcmp.setParameter(SERVER_CACHE_KEY, cache); hcmp.setParameter(SSL_FACTORY_KEY, this.sslfactory); }
From source file:org.apache.axis.transport.http.CommonsHTTPSender.java
/** * invoke creates a socket connection, sends the request SOAP message and then * reads the response SOAP message back from the SOAP server * * @param msgContext the messsage context * * @throws AxisFault//from ww w.ja v a2s . co m */ public void invoke(MessageContext msgContext) throws AxisFault { HttpMethodBase method = null; if (log.isDebugEnabled()) { log.debug(Messages.getMessage("enter00", "CommonsHTTPSender::invoke")); } try { URL targetURL = new URL(msgContext.getStrProp(MessageContext.TRANS_URL)); // no need to retain these, as the cookies/credentials are // stored in the message context across multiple requests. // the underlying connection manager, however, is retained // so sockets get recycled when possible. HttpClient httpClient = new HttpClient(this.connectionManager); // the timeout value for allocation of connections from the pool httpClient.getParams().setConnectionManagerTimeout(this.clientProperties.getConnectionPoolTimeout()); HostConfiguration hostConfiguration = getHostConfiguration(httpClient, msgContext, targetURL); boolean posting = true; // If we're SOAP 1.2, allow the web method to be set from the // MessageContext. if (msgContext.getSOAPConstants() == SOAPConstants.SOAP12_CONSTANTS) { String webMethod = msgContext.getStrProp(SOAP12Constants.PROP_WEBMETHOD); if (webMethod != null) { posting = webMethod.equals(HTTPConstants.HEADER_POST); } } if (posting) { Message reqMessage = msgContext.getRequestMessage(); method = new PostMethod(targetURL.toString()); // set false as default, addContetInfo can overwrite method.getParams().setBooleanParameter(HttpMethodParams.USE_EXPECT_CONTINUE, false); addContextInfo(method, httpClient, msgContext, targetURL); MessageRequestEntity requestEntity = null; if (msgContext.isPropertyTrue(HTTPConstants.MC_GZIP_REQUEST)) { requestEntity = new GzipMessageRequestEntity(method, reqMessage, httpChunkStream); } else { requestEntity = new MessageRequestEntity(method, reqMessage, httpChunkStream); } ((PostMethod) method).setRequestEntity(requestEntity); } else { method = new GetMethod(targetURL.toString()); addContextInfo(method, httpClient, msgContext, targetURL); } String httpVersion = msgContext.getStrProp(MessageContext.HTTP_TRANSPORT_VERSION); if (httpVersion != null) { if (httpVersion.equals(HTTPConstants.HEADER_PROTOCOL_V10)) { method.getParams().setVersion(HttpVersion.HTTP_1_0); } // assume 1.1 } // don't forget the cookies! // Cookies need to be set on HttpState, since HttpMethodBase // overwrites the cookies from HttpState if (msgContext.getMaintainSession()) { HttpState state = httpClient.getState(); method.getParams().setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY); String host = hostConfiguration.getHost(); String path = targetURL.getPath(); boolean secure = hostConfiguration.getProtocol().isSecure(); fillHeaders(msgContext, state, HTTPConstants.HEADER_COOKIE, host, path, secure); fillHeaders(msgContext, state, HTTPConstants.HEADER_COOKIE2, host, path, secure); httpClient.setState(state); } int returnCode = httpClient.executeMethod(hostConfiguration, method, null); String contentType = getHeader(method, HTTPConstants.HEADER_CONTENT_TYPE); String contentLocation = getHeader(method, HTTPConstants.HEADER_CONTENT_LOCATION); String contentLength = getHeader(method, HTTPConstants.HEADER_CONTENT_LENGTH); if ((returnCode > 199) && (returnCode < 300)) { // SOAP return is OK - so fall through } else if (msgContext.getSOAPConstants() == SOAPConstants.SOAP12_CONSTANTS) { // For now, if we're SOAP 1.2, fall through, since the range of // valid result codes is much greater } else if ((contentType != null) && !contentType.equals("text/html") && ((returnCode > 499) && (returnCode < 600))) { // SOAP Fault should be in here - so fall through } else { String statusMessage = method.getStatusText(); AxisFault fault = new AxisFault("HTTP", "(" + returnCode + ")" + statusMessage, null, null); try { fault.setFaultDetailString( Messages.getMessage("return01", "" + returnCode, method.getResponseBodyAsString())); fault.addFaultDetail(Constants.QNAME_FAULTDETAIL_HTTPERRORCODE, Integer.toString(returnCode)); throw fault; } finally { method.releaseConnection(); // release connection back to pool. } } // wrap the response body stream so that close() also releases // the connection back to the pool. InputStream releaseConnectionOnCloseStream = createConnectionReleasingInputStream(method); Header contentEncoding = method.getResponseHeader(HTTPConstants.HEADER_CONTENT_ENCODING); if (contentEncoding != null) { if (contentEncoding.getValue().equalsIgnoreCase(HTTPConstants.COMPRESSION_GZIP)) { releaseConnectionOnCloseStream = new GZIPInputStream(releaseConnectionOnCloseStream); } else { AxisFault fault = new AxisFault("HTTP", "unsupported content-encoding of '" + contentEncoding.getValue() + "' found", null, null); throw fault; } } Message outMsg = new Message(releaseConnectionOnCloseStream, false, contentType, contentLocation); // Transfer HTTP headers of HTTP message to MIME headers of SOAP message Header[] responseHeaders = method.getResponseHeaders(); MimeHeaders responseMimeHeaders = outMsg.getMimeHeaders(); for (int i = 0; i < responseHeaders.length; i++) { Header responseHeader = responseHeaders[i]; responseMimeHeaders.addHeader(responseHeader.getName(), responseHeader.getValue()); } outMsg.setMessageType(Message.RESPONSE); msgContext.setResponseMessage(outMsg); if (log.isDebugEnabled()) { if (null == contentLength) { log.debug("\n" + Messages.getMessage("no00", "Content-Length")); } log.debug("\n" + Messages.getMessage("xmlRecd00")); log.debug("-----------------------------------------------"); log.debug(outMsg.getSOAPPartAsString()); } // if we are maintaining session state, // handle cookies (if any) if (msgContext.getMaintainSession()) { Header[] headers = method.getResponseHeaders(); for (int i = 0; i < headers.length; i++) { if (headers[i].getName().equalsIgnoreCase(HTTPConstants.HEADER_SET_COOKIE)) { handleCookie(HTTPConstants.HEADER_COOKIE, headers[i].getValue(), msgContext); } else if (headers[i].getName().equalsIgnoreCase(HTTPConstants.HEADER_SET_COOKIE2)) { handleCookie(HTTPConstants.HEADER_COOKIE2, headers[i].getValue(), msgContext); } } } // always release the connection back to the pool if // it was one way invocation if (msgContext.isPropertyTrue("axis.one.way")) { method.releaseConnection(); } } catch (Exception e) { log.debug(e); throw AxisFault.makeFault(e); } if (log.isDebugEnabled()) { log.debug(Messages.getMessage("exit00", "CommonsHTTPSender::invoke")); } }
From source file:org.apache.axis2.transport.http.AbstractHTTPSender.java
/** * Method used to copy all the common properties * * @param msgContext - The messageContext of the request message * @param url - The target URL * @param httpMethod - The http method used to send the request * @param httpClient - The httpclient used to send the request * @param soapActionString - The soap action atring of the request message * @return MessageFormatter - The messageFormatter for the relavent request message * @throws AxisFault - Thrown in case an exception occurs *//*from w w w . ja v a 2 s. c o m*/ protected MessageFormatter populateCommonProperties(MessageContext msgContext, URL url, HttpMethodBase httpMethod, HttpClient httpClient, String soapActionString) throws AxisFault { if (isAuthenticationEnabled(msgContext)) { httpMethod.setDoAuthentication(true); } MessageFormatter messageFormatter = MessageProcessorSelector.getMessageFormatter(msgContext); url = messageFormatter.getTargetAddress(msgContext, format, url); httpMethod.setPath(url.getPath()); httpMethod.setQueryString(url.getQuery()); httpMethod.setRequestHeader(HTTPConstants.HEADER_CONTENT_TYPE, messageFormatter.getContentType(msgContext, format, soapActionString)); httpMethod.setRequestHeader(HTTPConstants.HEADER_HOST, url.getHost()); if (msgContext.getOptions() != null && msgContext.getOptions().isManageSession()) { // setting the cookie in the out path Object cookieString = msgContext.getProperty(HTTPConstants.COOKIE_STRING); if (cookieString != null) { StringBuffer buffer = new StringBuffer(); buffer.append(cookieString); httpMethod.setRequestHeader(HTTPConstants.HEADER_COOKIE, buffer.toString()); } } if (httpVersion.equals(HTTPConstants.HEADER_PROTOCOL_10)) { httpClient.getParams().setVersion(HttpVersion.HTTP_1_0); } return messageFormatter; }
From source file:org.apache.axis2.transport.http.impl.httpclient3.HTTPSenderImpl.java
/** * Method used to copy all the common properties * //from w w w . ja v a 2 s . c om * @param msgContext * - The messageContext of the request message * @param url * - The target URL * @param httpMethod * - The http method used to send the request * @param httpClient * - The httpclient used to send the request * @param soapActionString * - The soap action atring of the request message * @return MessageFormatter - The messageFormatter for the relavent request * message * @throws AxisFault * - Thrown in case an exception occurs */ protected MessageFormatter populateCommonProperties(MessageContext msgContext, URL url, HttpMethodBase httpMethod, HttpClient httpClient, String soapActionString) throws AxisFault { if (isAuthenticationEnabled(msgContext)) { httpMethod.setDoAuthentication(true); } MessageFormatter messageFormatter = TransportUtils.getMessageFormatter(msgContext); url = messageFormatter.getTargetAddress(msgContext, format, url); httpMethod.setPath(url.getPath()); httpMethod.setQueryString(url.getQuery()); httpMethod.setRequestHeader(HTTPConstants.HEADER_CONTENT_TYPE, messageFormatter.getContentType(msgContext, format, soapActionString)); httpMethod.setRequestHeader(HTTPConstants.HEADER_HOST, url.getHost()); if (msgContext.getOptions() != null && msgContext.getOptions().isManageSession()) { // setting the cookie in the out path Object cookieString = msgContext.getProperty(HTTPConstants.COOKIE_STRING); if (cookieString != null) { StringBuffer buffer = new StringBuffer(); buffer.append(cookieString); httpMethod.setRequestHeader(HTTPConstants.HEADER_COOKIE, buffer.toString()); } } if (httpVersion.equals(HTTPConstants.HEADER_PROTOCOL_10)) { httpClient.getParams().setVersion(HttpVersion.HTTP_1_0); } return messageFormatter; }
From source file:org.apache.nutch.protocol.httpclient.HttpResponse.java
/** * Fetches the given <code>url</code> and prepares HTTP response. * /*from ww w . j av a 2s .c o m*/ * @param http * An instance of the implementation class of this plugin * @param url * URL to be fetched * @param datum * Crawl data * @param followRedirects * Whether to follow redirects; follows redirect if and only if this * is true * @return HTTP response * @throws IOException * When an error occurs */ HttpResponse(Http http, URL url, CrawlDatum datum, boolean followRedirects) throws IOException { // Prepare GET method for HTTP request this.url = url; GetMethod get = new GetMethod(url.toString()); get.setFollowRedirects(followRedirects); get.setDoAuthentication(true); if (http.isIfModifiedSinceEnabled() && datum.getModifiedTime() > 0) { get.setRequestHeader("If-Modified-Since", HttpDateFormat.toString(datum.getModifiedTime())); } // Set HTTP parameters HttpMethodParams params = get.getParams(); if (http.getUseHttp11()) { params.setVersion(HttpVersion.HTTP_1_1); } else { params.setVersion(HttpVersion.HTTP_1_0); } params.makeLenient(); params.setContentCharset("UTF-8"); if (http.isCookieEnabled()) { params.setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY); params.setBooleanParameter(HttpMethodParams.SINGLE_COOKIE_HEADER, true); } else { params.setCookiePolicy(CookiePolicy.IGNORE_COOKIES); } // XXX (ab) not sure about this... the default is to retry 3 times; if // XXX the request body was sent the method is not retried, so there is // XXX little danger in retrying... // params.setParameter(HttpMethodParams.RETRY_HANDLER, null); if (http.isCookieEnabled() && datum.getMetaData().containsKey(http.COOKIE)) { String cookie = ((Text) datum.getMetaData().get(http.COOKIE)).toString(); get.addRequestHeader("Cookie", cookie); } try { HttpClient client = Http.getClient(); client.getParams().setParameter("http.useragent", http.getUserAgent()); // NUTCH-1941 code = client.executeMethod(get); Header[] heads = get.getResponseHeaders(); for (int i = 0; i < heads.length; i++) { headers.set(heads[i].getName(), heads[i].getValue()); } // Limit download size int contentLength = Integer.MAX_VALUE; String contentLengthString = headers.get(Response.CONTENT_LENGTH); if (contentLengthString != null) { try { contentLength = Integer.parseInt(contentLengthString.trim()); } catch (NumberFormatException ex) { throw new HttpException("bad content length: " + contentLengthString); } } if (http.getMaxContent() >= 0 && contentLength > http.getMaxContent()) { contentLength = http.getMaxContent(); } // always read content. Sometimes content is useful to find a cause // for error. InputStream in = get.getResponseBodyAsStream(); try { byte[] buffer = new byte[HttpBase.BUFFER_SIZE]; int bufferFilled = 0; int totalRead = 0; ByteArrayOutputStream out = new ByteArrayOutputStream(); while ((bufferFilled = in.read(buffer, 0, buffer.length)) != -1 && totalRead + bufferFilled <= contentLength) { totalRead += bufferFilled; out.write(buffer, 0, bufferFilled); } content = out.toByteArray(); } catch (Exception e) { if (code == 200) throw new IOException(e.toString()); // for codes other than 200 OK, we are fine with empty content } finally { if (in != null) { in.close(); } get.abort(); } StringBuilder fetchTrace = null; if (Http.LOG.isTraceEnabled()) { // Trace message fetchTrace = new StringBuilder( "url: " + url + "; status code: " + code + "; bytes received: " + content.length); if (getHeader(Response.CONTENT_LENGTH) != null) fetchTrace.append("; Content-Length: " + getHeader(Response.CONTENT_LENGTH)); if (getHeader(Response.LOCATION) != null) fetchTrace.append("; Location: " + getHeader(Response.LOCATION)); } // Extract gzip, x-gzip and deflate content if (content != null) { // check if we have to uncompress it String contentEncoding = headers.get(Response.CONTENT_ENCODING); if (contentEncoding != null && Http.LOG.isTraceEnabled()) fetchTrace.append("; Content-Encoding: " + contentEncoding); if ("gzip".equals(contentEncoding) || "x-gzip".equals(contentEncoding)) { content = http.processGzipEncoded(content, url); if (Http.LOG.isTraceEnabled()) fetchTrace.append("; extracted to " + content.length + " bytes"); } else if ("deflate".equals(contentEncoding)) { content = http.processDeflateEncoded(content, url); if (Http.LOG.isTraceEnabled()) fetchTrace.append("; extracted to " + content.length + " bytes"); } } // Logger trace message if (Http.LOG.isTraceEnabled()) { Http.LOG.trace(fetchTrace.toString()); } } finally { get.releaseConnection(); } }
From source file:org.apache.nutch.protocol.httpclient.HttpResponseBak.java
/** * Fetches the given <code>url</code> and prepares HTTP response. * * @param http An instance of the implementation class * of this plugin * @param url URL to be fetched * @param datum Crawl data * @param followRedirects Whether to follow redirects; follows * redirect if and only if this is true * @return HTTP response * @throws IOException When an error occurs *//*from w w w.j av a 2s . c om*/ HttpResponseBak(HttpBak http, URL url, CrawlDatum datum, boolean followRedirects) throws IOException { // Prepare GET method for HTTP request this.url = url; GetMethod get = new GetMethod(url.toString()); get.setFollowRedirects(followRedirects); get.setDoAuthentication(true); if (datum.getModifiedTime() > 0) { get.setRequestHeader("If-Modified-Since", HttpDateFormat.toString(datum.getModifiedTime())); } // Set HTTP parameters HttpMethodParams params = get.getParams(); if (http.getUseHttp11()) { params.setVersion(HttpVersion.HTTP_1_1); } else { params.setVersion(HttpVersion.HTTP_1_0); } params.makeLenient(); params.setContentCharset("UTF-8"); params.setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY); params.setBooleanParameter(HttpMethodParams.SINGLE_COOKIE_HEADER, true); // XXX (ab) not sure about this... the default is to retry 3 times; if // XXX the request body was sent the method is not retried, so there is // XXX little danger in retrying... // params.setParameter(HttpMethodParams.RETRY_HANDLER, null); try { code = Http.getClient().executeMethod(get); Header[] heads = get.getResponseHeaders(); for (int i = 0; i < heads.length; i++) { headers.set(heads[i].getName(), heads[i].getValue()); } // Limit download size int contentLength = Integer.MAX_VALUE; String contentLengthString = headers.get(Response.CONTENT_LENGTH); if (contentLengthString != null) { try { contentLength = Integer.parseInt(contentLengthString.trim()); } catch (NumberFormatException ex) { throw new HttpException("bad content length: " + contentLengthString); } } if (http.getMaxContent() >= 0 && contentLength > http.getMaxContent()) { contentLength = http.getMaxContent(); } // always read content. Sometimes content is useful to find a cause // for error. InputStream in = get.getResponseBodyAsStream(); try { byte[] buffer = new byte[HttpBase.BUFFER_SIZE]; int bufferFilled = 0; int totalRead = 0; ByteArrayOutputStream out = new ByteArrayOutputStream(); while ((bufferFilled = in.read(buffer, 0, buffer.length)) != -1 && totalRead + bufferFilled <= contentLength) { totalRead += bufferFilled; out.write(buffer, 0, bufferFilled); } content = out.toByteArray(); } catch (Exception e) { if (code == 200) throw new IOException(e.toString()); // for codes other than 200 OK, we are fine with empty content } finally { if (in != null) { in.close(); } get.abort(); } StringBuilder fetchTrace = null; if (Http.LOG.isTraceEnabled()) { // Trace message fetchTrace = new StringBuilder( "url: " + url + "; status code: " + code + "; bytes received: " + content.length); if (getHeader(Response.CONTENT_LENGTH) != null) fetchTrace.append("; Content-Length: " + getHeader(Response.CONTENT_LENGTH)); if (getHeader(Response.LOCATION) != null) fetchTrace.append("; Location: " + getHeader(Response.LOCATION)); } // Extract gzip, x-gzip and deflate content if (content != null) { // check if we have to uncompress it String contentEncoding = headers.get(Response.CONTENT_ENCODING); if (contentEncoding != null && Http.LOG.isTraceEnabled()) fetchTrace.append("; Content-Encoding: " + contentEncoding); if ("gzip".equals(contentEncoding) || "x-gzip".equals(contentEncoding)) { content = http.processGzipEncoded(content, url); if (Http.LOG.isTraceEnabled()) fetchTrace.append("; extracted to " + content.length + " bytes"); } else if ("deflate".equals(contentEncoding)) { content = http.processDeflateEncoded(content, url); if (Http.LOG.isTraceEnabled()) fetchTrace.append("; extracted to " + content.length + " bytes"); } } // Logger trace message if (Http.LOG.isTraceEnabled()) { Http.LOG.trace(fetchTrace.toString()); } } finally { get.releaseConnection(); } }
From source file:org.apache.nutch.protocol.httpclient.proxy.HttpResponse.java
/** * Fetches the given <code>url</code> and prepares HTTP response. * //from w ww . ja va2s .c o m * @param http * An instance of the implementation class of this plugin * @param url * URL to be fetched * @param datum * Crawl data * @param followRedirects * Whether to follow redirects; follows redirect if and only if this * is true * @return HTTP response * @throws IOException * When an error occurs */ HttpResponse(Http http, URL url, CrawlDatum datum, boolean followRedirects) throws IOException { // Prepare GET method for HTTP request this.url = url; GetMethod get = new GetMethod(url.toString()); get.setFollowRedirects(followRedirects); get.setDoAuthentication(true); if (http.isIfModifiedSinceEnabled() && datum.getModifiedTime() > 0) { get.setRequestHeader("If-Modified-Since", HttpDateFormat.toString(datum.getModifiedTime())); } // Set HTTP parameters HttpMethodParams params = get.getParams(); if (http.getUseHttp11()) { params.setVersion(HttpVersion.HTTP_1_1); } else { params.setVersion(HttpVersion.HTTP_1_0); } params.makeLenient(); params.setContentCharset("UTF-8"); params.setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY); params.setBooleanParameter(HttpMethodParams.SINGLE_COOKIE_HEADER, true); // XXX (ab) not sure about this... the default is to retry 3 times; if // XXX the request body was sent the method is not retried, so there is // XXX little danger in retrying... // params.setParameter(HttpMethodParams.RETRY_HANDLER, null); try { HttpClient client = Http.getClient(); client.getParams().setParameter("http.useragent", http.getUserAgent()); // NUTCH-1941 code = client.executeMethod(get); Header[] heads = get.getResponseHeaders(); for (int i = 0; i < heads.length; i++) { headers.set(heads[i].getName(), heads[i].getValue()); } // Limit download size int contentLength = Integer.MAX_VALUE; String contentLengthString = headers.get(Response.CONTENT_LENGTH); if (contentLengthString != null) { try { contentLength = Integer.parseInt(contentLengthString.trim()); } catch (NumberFormatException ex) { throw new HttpException("bad content length: " + contentLengthString); } } if (http.getMaxContent() >= 0 && contentLength > http.getMaxContent()) { contentLength = http.getMaxContent(); } // always read content. Sometimes content is useful to find a cause // for error. InputStream in = get.getResponseBodyAsStream(); try { byte[] buffer = new byte[HttpBase.BUFFER_SIZE]; int bufferFilled = 0; int totalRead = 0; ByteArrayOutputStream out = new ByteArrayOutputStream(); while ((bufferFilled = in.read(buffer, 0, buffer.length)) != -1 && totalRead + bufferFilled <= contentLength) { totalRead += bufferFilled; out.write(buffer, 0, bufferFilled); } content = out.toByteArray(); } catch (Exception e) { if (code == 200) throw new IOException(e.toString()); // for codes other than 200 OK, we are fine with empty content } finally { if (in != null) { in.close(); } get.abort(); } StringBuilder fetchTrace = null; if (Http.LOG.isTraceEnabled()) { // Trace message fetchTrace = new StringBuilder( "url: " + url + "; status code: " + code + "; bytes received: " + content.length); if (getHeader(Response.CONTENT_LENGTH) != null) fetchTrace.append("; Content-Length: " + getHeader(Response.CONTENT_LENGTH)); if (getHeader(Response.LOCATION) != null) fetchTrace.append("; Location: " + getHeader(Response.LOCATION)); } // Extract gzip, x-gzip and deflate content if (content != null) { // check if we have to uncompress it String contentEncoding = headers.get(Response.CONTENT_ENCODING); if (contentEncoding != null && Http.LOG.isTraceEnabled()) fetchTrace.append("; Content-Encoding: " + contentEncoding); if ("gzip".equals(contentEncoding) || "x-gzip".equals(contentEncoding)) { content = http.processGzipEncoded(content, url); if (Http.LOG.isTraceEnabled()) fetchTrace.append("; extracted to " + content.length + " bytes"); } else if ("deflate".equals(contentEncoding)) { content = http.processDeflateEncoded(content, url); if (Http.LOG.isTraceEnabled()) fetchTrace.append("; extracted to " + content.length + " bytes"); } } // Logger trace message if (Http.LOG.isTraceEnabled()) { Http.LOG.trace(fetchTrace.toString()); } } finally { get.releaseConnection(); } }