List of usage examples for org.apache.commons.httpclient.params HttpMethodParams setCookiePolicy
public void setCookiePolicy(String paramString)
From source file:com.iflytek.spider.protocol.httpclient.HttpResponseSmiply.java
/** * Fetches the given <code>url</code> and prepares HTTP response. * /*www.j a va2 s. c o m*/ * @param http * An instance of the implementation class of this plugin * @param url * URL to be fetched * @param datum * Crawl data * @param followRedirects * Whether to follow redirects; follows redirect if and only if * this is true * @return HTTP response * @throws IOException * When an error occurs */ HttpResponseSmiply(HttpSimply http, URL url, CrawlDatum datum, boolean followRedirects) throws IOException { // Prepare GET method for HTTP request this.url = url; GetMethod get = new GetMethod(url.toString()); get.setFollowRedirects(followRedirects); get.setDoAuthentication(true); if (datum.getModifiedTime() > 0) { get.setRequestHeader("If-Modified-Since", HttpDateFormat.toString(datum.getModifiedTime())); } // Set HTTP parameters HttpMethodParams params = get.getParams(); if (http.getUseHttp11()) { params.setVersion(HttpVersion.HTTP_1_1); } else { params.setVersion(HttpVersion.HTTP_1_0); } params.makeLenient(); params.setContentCharset("UTF-8"); params.setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY); params.setBooleanParameter(HttpMethodParams.SINGLE_COOKIE_HEADER, true); // XXX (ab) not sure about this... the default is to retry 3 times; if // XXX the request body was sent the method is not retried, so there is // XXX little danger in retrying... // params.setParameter(HttpMethodParams.RETRY_HANDLER, null); try { code = HttpSimply.getClient().executeMethod(get); Header[] heads = get.getResponseHeaders(); for (int i = 0; i < heads.length; i++) { headers.set(heads[i].getName(), heads[i].getValue()); } // Limit download size int contentLength = Integer.MAX_VALUE; String contentLengthString = headers.get(Response.CONTENT_LENGTH); if (contentLengthString != null) { try { contentLength = Integer.parseInt(contentLengthString.trim()); } catch (NumberFormatException ex) { throw new HttpException("bad content length: " + contentLengthString); } } if (http.getMaxContent() >= 0 && contentLength > http.getMaxContent()) { contentLength = http.getMaxContent(); } // always read content. Sometimes content is useful to find a cause // for error. InputStream in = get.getResponseBodyAsStream(); try { byte[] buffer = new byte[HttpBaseSimply.BUFFER_SIZE]; int bufferFilled = 0; int totalRead = 0; ByteArrayOutputStream out = new ByteArrayOutputStream(); while ((bufferFilled = in.read(buffer, 0, buffer.length)) != -1 && totalRead < contentLength) { totalRead += bufferFilled; out.write(buffer, 0, bufferFilled); } content = out.toByteArray(); } catch (Exception e) { if (code == 200) throw new IOException(e.toString()); // for codes other than 200 OK, we are fine with empty content } finally { in.close(); get.abort(); } StringBuilder fetchTrace = null; if (Http.LOG.isTraceEnabled()) { // Trace message fetchTrace = new StringBuilder( "url: " + url + "; status code: " + code + "; bytes received: " + content.length); if (getHeader(Response.CONTENT_LENGTH) != null) fetchTrace.append("; Content-Length: " + getHeader(Response.CONTENT_LENGTH)); if (getHeader(Response.LOCATION) != null) fetchTrace.append("; Location: " + getHeader(Response.LOCATION)); } // Extract gzip, x-gzip and deflate content if (content != null) { // check if we have to uncompress it String contentEncoding = headers.get(Response.CONTENT_ENCODING); if (contentEncoding != null && Http.LOG.isTraceEnabled()) fetchTrace.append("; Content-Encoding: " + contentEncoding); if ("gzip".equals(contentEncoding) || "x-gzip".equals(contentEncoding)) { content = http.processGzipEncoded(content, url); if (Http.LOG.isTraceEnabled()) fetchTrace.append("; extracted to " + content.length + " bytes"); } else if ("deflate".equals(contentEncoding)) { content = http.processDeflateEncoded(content, url); if (Http.LOG.isTraceEnabled()) fetchTrace.append("; extracted to " + content.length + " bytes"); } } // Log trace message if (Http.LOG.isTraceEnabled()) { Http.LOG.trace(fetchTrace); } } finally { get.releaseConnection(); } }
From source file:com.iflytek.spider.protocol.httpclient.HttpResponse.java
/** * Fetches the given <code>url</code> and prepares HTTP response. * //from ww w .j ava 2 s. c om * @param http * An instance of the implementation class of this plugin * @param url * URL to be fetched * @param datum * Crawl data * @param followRedirects * Whether to follow redirects; follows redirect if and only if * this is true * @return HTTP response * @throws IOException * When an error occurs */ HttpResponse(Http http, URL url, CrawlDatum datum, boolean followRedirects) throws IOException { // Prepare GET method for HTTP request this.url = url; GetMethod get = new GetMethod(url.toString()); get.setFollowRedirects(followRedirects); get.setDoAuthentication(true); if (datum.getModifiedTime() > 0) { get.setRequestHeader("If-Modified-Since", HttpDateFormat.toString(datum.getModifiedTime())); } // Set HTTP parameters HttpMethodParams params = get.getParams(); if (http.getUseHttp11()) { params.setVersion(HttpVersion.HTTP_1_1); } else { params.setVersion(HttpVersion.HTTP_1_0); } params.makeLenient(); params.setContentCharset("UTF-8"); params.setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY); params.setBooleanParameter(HttpMethodParams.SINGLE_COOKIE_HEADER, true); // XXX (ab) not sure about this... the default is to retry 3 times; if // XXX the request body was sent the method is not retried, so there is // XXX little danger in retrying... // params.setParameter(HttpMethodParams.RETRY_HANDLER, null); try { code = Http.getClient().executeMethod(get); Header[] heads = get.getResponseHeaders(); for (int i = 0; i < heads.length; i++) { headers.set(heads[i].getName(), heads[i].getValue()); } // Limit download size int contentLength = Integer.MAX_VALUE; String contentLengthString = headers.get(Response.CONTENT_LENGTH); if (contentLengthString != null) { try { contentLength = Integer.parseInt(contentLengthString.trim()); } catch (NumberFormatException ex) { throw new HttpException("bad content length: " + contentLengthString); } } if (http.getMaxContent() >= 0 && contentLength > http.getMaxContent()) { contentLength = http.getMaxContent(); } // always read content. Sometimes content is useful to find a cause // for error. InputStream in = get.getResponseBodyAsStream(); try { byte[] buffer = new byte[HttpBase.BUFFER_SIZE]; int bufferFilled = 0; int totalRead = 0; ByteArrayOutputStream out = new ByteArrayOutputStream(); while ((bufferFilled = in.read(buffer, 0, buffer.length)) != -1 && totalRead < contentLength) { totalRead += bufferFilled; out.write(buffer, 0, bufferFilled); } content = out.toByteArray(); } catch (Exception e) { if (code == 200) throw new IOException(e.toString()); // for codes other than 200 OK, we are fine with empty content } finally { in.close(); get.abort(); } StringBuilder fetchTrace = null; if (Http.LOG.isTraceEnabled()) { // Trace message fetchTrace = new StringBuilder( "url: " + url + "; status code: " + code + "; bytes received: " + content.length); if (getHeader(Response.CONTENT_LENGTH) != null) fetchTrace.append("; Content-Length: " + getHeader(Response.CONTENT_LENGTH)); if (getHeader(Response.LOCATION) != null) fetchTrace.append("; Location: " + getHeader(Response.LOCATION)); } // Extract gzip, x-gzip and deflate content if (content != null) { // check if we have to uncompress it String contentEncoding = headers.get(Response.CONTENT_ENCODING); if (contentEncoding != null && Http.LOG.isTraceEnabled()) fetchTrace.append("; Content-Encoding: " + contentEncoding); if ("gzip".equals(contentEncoding) || "x-gzip".equals(contentEncoding)) { content = http.processGzipEncoded(content, url); if (Http.LOG.isTraceEnabled()) fetchTrace.append("; extracted to " + content.length + " bytes"); } else if ("deflate".equals(contentEncoding)) { content = http.processDeflateEncoded(content, url); if (Http.LOG.isTraceEnabled()) fetchTrace.append("; extracted to " + content.length + " bytes"); } } // Log trace message if (Http.LOG.isTraceEnabled()) { Http.LOG.trace(fetchTrace); } } finally { get.releaseConnection(); } }
From source file:com.sun.jersey.client.apache.DefaultApacheHttpMethodExecutor.java
@Override public void executeMethod(final HttpMethod method, final ClientRequest cr) { final Map<String, Object> props = cr.getProperties(); method.setDoAuthentication(true);/* w ww .j a v a 2 s . c o m*/ final HttpMethodParams methodParams = method.getParams(); // Set the handle cookies property if (!cr.getPropertyAsFeature(ApacheHttpClientConfig.PROPERTY_HANDLE_COOKIES)) { methodParams.setCookiePolicy(CookiePolicy.IGNORE_COOKIES); } // Set the interactive and credential provider properties if (cr.getPropertyAsFeature(ApacheHttpClientConfig.PROPERTY_INTERACTIVE)) { CredentialsProvider provider = (CredentialsProvider) props .get(ApacheHttpClientConfig.PROPERTY_CREDENTIALS_PROVIDER); if (provider == null) { provider = DEFAULT_CREDENTIALS_PROVIDER; } methodParams.setParameter(CredentialsProvider.PROVIDER, provider); } else { methodParams.setParameter(CredentialsProvider.PROVIDER, null); } // Set the read timeout final Integer readTimeout = (Integer) props.get(ApacheHttpClientConfig.PROPERTY_READ_TIMEOUT); if (readTimeout != null) { methodParams.setSoTimeout(readTimeout); } if (method instanceof EntityEnclosingMethod) { final EntityEnclosingMethod entMethod = (EntityEnclosingMethod) method; if (cr.getEntity() != null) { final RequestEntityWriter re = getRequestEntityWriter(cr); final Integer chunkedEncodingSize = (Integer) props .get(ApacheHttpClientConfig.PROPERTY_CHUNKED_ENCODING_SIZE); if (chunkedEncodingSize != null) { // There doesn't seems to be a way to set the chunk size. entMethod.setContentChunked(true); // It is not possible for a MessageBodyWriter to modify // the set of headers before writing out any bytes to // the OutputStream // This makes it impossible to use the multipart // writer that modifies the content type to add a boundary // parameter writeOutBoundHeaders(cr.getHeaders(), method); // Do not buffer the request entity when chunked encoding is // set entMethod.setRequestEntity(new RequestEntity() { @Override public boolean isRepeatable() { return false; } @Override public void writeRequest(OutputStream out) throws IOException { re.writeRequestEntity(out); } @Override public long getContentLength() { return re.getSize(); } @Override public String getContentType() { return re.getMediaType().toString(); } }); } else { entMethod.setContentChunked(false); ByteArrayOutputStream baos = new ByteArrayOutputStream(); try { re.writeRequestEntity(new CommittingOutputStream(baos) { @Override protected void commit() throws IOException { writeOutBoundHeaders(cr.getMetadata(), method); } }); } catch (IOException ex) { throw new ClientHandlerException(ex); } final byte[] content = baos.toByteArray(); entMethod.setRequestEntity(new RequestEntity() { @Override public boolean isRepeatable() { return true; } @Override public void writeRequest(OutputStream out) throws IOException { out.write(content); } @Override public long getContentLength() { return content.length; } @Override public String getContentType() { return re.getMediaType().toString(); } }); } } else { writeOutBoundHeaders(cr.getHeaders(), method); } } else { writeOutBoundHeaders(cr.getHeaders(), method); // Follow redirects method.setFollowRedirects(cr.getPropertyAsFeature(ApacheHttpClientConfig.PROPERTY_FOLLOW_REDIRECTS)); } try { httpClient.executeMethod(getHostConfiguration(httpClient, props), method, getHttpState(props)); } catch (Exception e) { method.releaseConnection(); throw new ClientHandlerException(e); } }
From source file:com.twinsoft.convertigo.engine.proxy.translated.HttpClient.java
private byte[] getData(HttpConnector connector, String resourceUrl, ParameterShuttle infoShuttle) throws IOException, EngineException { byte[] result = null; try {/*from w w w. ja v a 2 s. co m*/ Context context = infoShuttle.context; String proxyServer = Engine.theApp.proxyManager.getProxyServer(); String proxyUser = Engine.theApp.proxyManager.getProxyUser(); String proxyPassword = Engine.theApp.proxyManager.getProxyPassword(); int proxyPort = Engine.theApp.proxyManager.getProxyPort(); HostConfiguration hostConfiguration = connector.hostConfiguration; boolean trustAllServerCertificates = connector.isTrustAllServerCertificates(); // Retrieving httpState getHttpState(connector, infoShuttle); Engine.logEngine.trace("(HttpClient) Retrieving data as a bytes array..."); Engine.logEngine.debug("(HttpClient) Connecting to: " + resourceUrl); // Proxy configuration if (!proxyServer.equals("")) { hostConfiguration.setProxy(proxyServer, proxyPort); Engine.logEngine.debug("(HttpClient) Using proxy: " + proxyServer + ":" + proxyPort); } else { // Remove old proxy configuration hostConfiguration.setProxyHost(null); } Engine.logEngine.debug("(HttpClient) Https: " + connector.isHttps()); CertificateManager certificateManager = connector.certificateManager; URL url = null; String host = ""; int port = -1; if (resourceUrl.toLowerCase().startsWith("https:")) { Engine.logEngine.debug("(HttpClient) Setting up SSL properties"); certificateManager.collectStoreInformation(context); url = new URL(resourceUrl); host = url.getHost(); port = url.getPort(); if (port == -1) port = 443; Engine.logEngine.debug("(HttpClient) Host: " + host + ":" + port); Engine.logEngine .debug("(HttpClient) CertificateManager has changed: " + certificateManager.hasChanged); if (certificateManager.hasChanged || (!host.equalsIgnoreCase(hostConfiguration.getHost())) || (hostConfiguration.getPort() != port)) { Engine.logEngine.debug("(HttpClient) Using MySSLSocketFactory for creating the SSL socket"); Protocol myhttps = new Protocol("https", MySSLSocketFactory.getSSLSocketFactory(certificateManager.keyStore, certificateManager.keyStorePassword, certificateManager.trustStore, certificateManager.trustStorePassword, trustAllServerCertificates), port); hostConfiguration.setHost(host, port, myhttps); } resourceUrl = url.getFile(); Engine.logEngine.debug("(HttpClient) Updated URL for SSL purposes: " + resourceUrl); } else { url = new URL(resourceUrl); host = url.getHost(); port = url.getPort(); Engine.logEngine.debug("(HttpClient) Host: " + host + ":" + port); hostConfiguration.setHost(host, port); } Engine.logEngine.debug("(HttpClient) Building method on: " + resourceUrl); Engine.logEngine.debug("(HttpClient) postFromUser=" + infoShuttle.postFromUser); Engine.logEngine.debug("(HttpClient) postToSite=" + infoShuttle.postToSite); if (infoShuttle.postFromUser && infoShuttle.postToSite) { method = new PostMethod(resourceUrl); ((PostMethod) method).setRequestEntity( new StringRequestEntity(infoShuttle.userPostData, infoShuttle.userContentType, infoShuttle.context.httpServletRequest.getCharacterEncoding())); } else { method = new GetMethod(resourceUrl); } HttpMethodParams httpMethodParams = method.getParams(); // Cookie configuration if (handleCookie) { Engine.logEngine.debug("(HttpClient) Setting cookie policy."); httpMethodParams.setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY); } String basicUser = connector.getAuthUser(); String basicPassword = connector.getAuthPassword(); String givenBasicUser = connector.getGivenAuthUser(); String givenBasicPassword = connector.getGivenAuthPassword(); // Basic authentication configuration String realm = null; if (!basicUser.equals("") || (basicUser.equals("") && (givenBasicUser != null))) { String userName = ((givenBasicUser == null) ? basicUser : givenBasicUser); String userPassword = ((givenBasicPassword == null) ? basicPassword : givenBasicPassword); httpState.setCredentials(new AuthScope(host, port, realm), new UsernamePasswordCredentials(userName, userPassword)); Engine.logEngine.debug("(HttpClient) Credentials: " + userName + ":******"); } // Setting basic authentication for proxy if (!proxyServer.equals("") && !proxyUser.equals("")) { httpState.setProxyCredentials(new AuthScope(proxyServer, proxyPort), new UsernamePasswordCredentials(proxyUser, proxyPassword)); Engine.logEngine.debug("(HttpClient) Proxy credentials: " + proxyUser + ":******"); } // Setting HTTP headers Engine.logEngine.debug("(HttpClient) Incoming HTTP headers:"); String headerName, headerValue; for (int k = 0, kSize = infoShuttle.userHeaderNames.size(); k < kSize; k++) { headerName = (String) infoShuttle.userHeaderNames.get(k); // Cookies are handled by HttpClient, so we do not have to proxy Cookie header // See #986 (Multiples cookies don't work with some proxies) if (headerName.toLowerCase().equals("cookie")) { Engine.logEngine.debug("Cookie header ignored"); } else { headerValue = (String) infoShuttle.userHeaderValues.get(k); method.setRequestHeader(headerName, headerValue); Engine.logEngine.debug(headerName + "=" + headerValue); } } // Getting the result executeMethod(method, connector, resourceUrl, infoShuttle); result = method.getResponseBody(); } finally { if (method != null) method.releaseConnection(); } return result; }
From source file:com.zimbra.common.soap.SoapHttpTransport.java
public Element invoke(Element document, boolean raw, boolean noSession, String requestedAccountId, String changeToken, String tokenType, ResponseHandler respHandler) throws IOException, HttpException, ServiceException { PostMethod method = null;/*www. j a v a2 s . c om*/ try { // Assemble post method. Append document name, so that the request // type is written to the access log. String uri, query; int i = mUri.indexOf('?'); if (i >= 0) { uri = mUri.substring(0, i); query = mUri.substring(i); } else { uri = mUri; query = ""; } if (!uri.endsWith("/")) uri += '/'; uri += getDocumentName(document); method = new PostMethod(uri + query); // Set user agent if it's specified. String agentName = getUserAgentName(); if (agentName != null) { String agentVersion = getUserAgentVersion(); if (agentVersion != null) agentName += " " + agentVersion; method.setRequestHeader(new Header("User-Agent", agentName)); } // the content-type charset will determine encoding used // when we set the request body method.setRequestHeader("Content-Type", getRequestProtocol().getContentType()); if (getClientIp() != null) { method.setRequestHeader(RemoteIP.X_ORIGINATING_IP_HEADER, getClientIp()); if (ZimbraLog.misc.isDebugEnabled()) { ZimbraLog.misc.debug("set remote IP header [%s] to [%s]", RemoteIP.X_ORIGINATING_IP_HEADER, getClientIp()); } } Element soapReq = generateSoapMessage(document, raw, noSession, requestedAccountId, changeToken, tokenType); String soapMessage = SoapProtocol.toString(soapReq, getPrettyPrint()); HttpMethodParams params = method.getParams(); method.setRequestEntity(new StringRequestEntity(soapMessage, null, "UTF-8")); if (getRequestProtocol().hasSOAPActionHeader()) method.setRequestHeader("SOAPAction", mUri); if (mCustomHeaders != null) { for (Map.Entry<String, String> entry : mCustomHeaders.entrySet()) method.setRequestHeader(entry.getKey(), entry.getValue()); } String host = method.getURI().getHost(); HttpState state = HttpClientUtil.newHttpState(getAuthToken(), host, this.isAdmin()); String trustedToken = getTrustedToken(); if (trustedToken != null) { state.addCookie( new Cookie(host, ZimbraCookie.COOKIE_ZM_TRUST_TOKEN, trustedToken, "/", null, false)); } params.setCookiePolicy(state.getCookies().length == 0 ? CookiePolicy.IGNORE_COOKIES : CookiePolicy.BROWSER_COMPATIBILITY); params.setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler(mRetryCount - 1, true)); params.setSoTimeout(mTimeout); params.setVersion(HttpVersion.HTTP_1_1); method.setRequestHeader("Connection", mKeepAlive ? "Keep-alive" : "Close"); if (mHostConfig != null && mHostConfig.getUsername() != null && mHostConfig.getPassword() != null) { state.setProxyCredentials(new AuthScope(null, -1), new UsernamePasswordCredentials(mHostConfig.getUsername(), mHostConfig.getPassword())); } if (mHttpDebugListener != null) { mHttpDebugListener.sendSoapMessage(method, soapReq, state); } int responseCode = mClient.executeMethod(mHostConfig, method, state); // SOAP allows for "200" on success and "500" on failure; // real server issues will probably be "503" or "404" if (responseCode != HttpServletResponse.SC_OK && responseCode != HttpServletResponse.SC_INTERNAL_SERVER_ERROR) throw ServiceException.PROXY_ERROR(method.getStatusLine().toString(), uri); // Read the response body. Use the stream API instead of the byte[] // version to avoid HTTPClient whining about a large response. InputStreamReader reader = new InputStreamReader(method.getResponseBodyAsStream(), SoapProtocol.getCharset()); String responseStr = ""; try { if (respHandler != null) { respHandler.process(reader); return null; } else { responseStr = ByteUtil.getContent(reader, (int) method.getResponseContentLength(), false); Element soapResp = parseSoapResponse(responseStr, raw); if (mHttpDebugListener != null) { mHttpDebugListener.receiveSoapMessage(method, soapResp); } return soapResp; } } catch (SoapFaultException x) { // attach request/response to the exception and rethrow x.setFaultRequest(soapMessage); x.setFaultResponse(responseStr.substring(0, Math.min(10240, responseStr.length()))); throw x; } } finally { // Release the connection to the connection manager if (method != null) method.releaseConnection(); // really not necessary if running in the server because the reaper thread // of our connection manager will take care it. // if called from CLI, all connections will be closed when the CLI // exits. Leave it here anyway. if (!mKeepAlive) mClient.getHttpConnectionManager().closeIdleConnections(0); } }
From source file:com.liferay.portal.util.HttpImpl.java
protected byte[] URLtoByteArray(String location, Http.Method method, Map<String, String> headers, Cookie[] cookies, Http.Auth auth, Http.Body body, List<Http.FilePart> fileParts, Map<String, String> parts, Http.Response response, boolean followRedirects) throws IOException { byte[] bytes = null; HttpMethod httpMethod = null;//from w w w. ja va 2 s.c o m HttpState httpState = null; try { _cookies.set(null); if (location == null) { return null; } else if (!location.startsWith(Http.HTTP_WITH_SLASH) && !location.startsWith(Http.HTTPS_WITH_SLASH)) { location = Http.HTTP_WITH_SLASH + location; } HostConfiguration hostConfiguration = getHostConfiguration(location); HttpClient httpClient = getClient(hostConfiguration); if (method.equals(Http.Method.POST) || method.equals(Http.Method.PUT)) { if (method.equals(Http.Method.POST)) { httpMethod = new PostMethod(location); } else { httpMethod = new PutMethod(location); } if (body != null) { RequestEntity requestEntity = new StringRequestEntity(body.getContent(), body.getContentType(), body.getCharset()); EntityEnclosingMethod entityEnclosingMethod = (EntityEnclosingMethod) httpMethod; entityEnclosingMethod.setRequestEntity(requestEntity); } else if (method.equals(Http.Method.POST)) { PostMethod postMethod = (PostMethod) httpMethod; processPostMethod(postMethod, fileParts, parts); } } else if (method.equals(Http.Method.DELETE)) { httpMethod = new DeleteMethod(location); } else if (method.equals(Http.Method.HEAD)) { httpMethod = new HeadMethod(location); } else { httpMethod = new GetMethod(location); } if (headers != null) { for (Map.Entry<String, String> header : headers.entrySet()) { httpMethod.addRequestHeader(header.getKey(), header.getValue()); } } if ((method.equals(Http.Method.POST) || method.equals(Http.Method.PUT)) && ((body != null) || ((fileParts != null) && !fileParts.isEmpty()) | ((parts != null) && !parts.isEmpty()))) { } else if (!hasRequestHeader(httpMethod, HttpHeaders.CONTENT_TYPE)) { httpMethod.addRequestHeader(HttpHeaders.CONTENT_TYPE, ContentTypes.APPLICATION_X_WWW_FORM_URLENCODED); } if (!hasRequestHeader(httpMethod, HttpHeaders.USER_AGENT)) { httpMethod.addRequestHeader(HttpHeaders.USER_AGENT, _DEFAULT_USER_AGENT); } httpState = new HttpState(); if ((cookies != null) && (cookies.length > 0)) { org.apache.commons.httpclient.Cookie[] commonsCookies = toCommonsCookies(cookies); httpState.addCookies(commonsCookies); HttpMethodParams httpMethodParams = httpMethod.getParams(); httpMethodParams.setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY); } if (auth != null) { httpMethod.setDoAuthentication(true); httpState.setCredentials(new AuthScope(auth.getHost(), auth.getPort(), auth.getRealm()), new UsernamePasswordCredentials(auth.getUsername(), auth.getPassword())); } proxifyState(httpState, hostConfiguration); httpClient.executeMethod(hostConfiguration, httpMethod, httpState); Header locationHeader = httpMethod.getResponseHeader("location"); if ((locationHeader != null) && !locationHeader.equals(location)) { String redirect = locationHeader.getValue(); if (followRedirects) { return URLtoByteArray(redirect, Http.Method.GET, headers, cookies, auth, body, fileParts, parts, response, followRedirects); } else { response.setRedirect(redirect); } } InputStream inputStream = httpMethod.getResponseBodyAsStream(); if (inputStream != null) { Header contentLength = httpMethod.getResponseHeader(HttpHeaders.CONTENT_LENGTH); if (contentLength != null) { response.setContentLength(GetterUtil.getInteger(contentLength.getValue())); } Header contentType = httpMethod.getResponseHeader(HttpHeaders.CONTENT_TYPE); if (contentType != null) { response.setContentType(contentType.getValue()); } bytes = FileUtil.getBytes(inputStream); } for (Header header : httpMethod.getResponseHeaders()) { response.addHeader(header.getName(), header.getValue()); } return bytes; } finally { try { if (httpState != null) { _cookies.set(toServletCookies(httpState.getCookies())); } } catch (Exception e) { _log.error(e, e); } try { if (httpMethod != null) { httpMethod.releaseConnection(); } } catch (Exception e) { _log.error(e, e); } } }
From source file:com.twelve.capital.external.feed.util.HttpImpl.java
protected byte[] URLtoByteArray(String location, Http.Method method, Map<String, String> headers, Cookie[] cookies, Http.Auth auth, Http.Body body, List<Http.FilePart> fileParts, Map<String, String> parts, Http.Response response, boolean followRedirects, String progressId, PortletRequest portletRequest) throws IOException { byte[] bytes = null; HttpMethod httpMethod = null;/* www . j ava2 s . c om*/ HttpState httpState = null; try { _cookies.set(null); if (location == null) { return null; } else if (!location.startsWith(Http.HTTP_WITH_SLASH) && !location.startsWith(Http.HTTPS_WITH_SLASH)) { location = Http.HTTP_WITH_SLASH + location; } HostConfiguration hostConfiguration = getHostConfiguration(location); HttpClient httpClient = getClient(hostConfiguration); if (method.equals(Http.Method.POST) || method.equals(Http.Method.PUT)) { if (method.equals(Http.Method.POST)) { httpMethod = new PostMethod(location); } else { httpMethod = new PutMethod(location); } if (body != null) { RequestEntity requestEntity = new StringRequestEntity(body.getContent(), body.getContentType(), body.getCharset()); EntityEnclosingMethod entityEnclosingMethod = (EntityEnclosingMethod) httpMethod; entityEnclosingMethod.setRequestEntity(requestEntity); } else if (method.equals(Http.Method.POST)) { PostMethod postMethod = (PostMethod) httpMethod; if (!hasRequestHeader(postMethod, HttpHeaders.CONTENT_TYPE)) { HttpClientParams httpClientParams = httpClient.getParams(); httpClientParams.setParameter(HttpMethodParams.HTTP_CONTENT_CHARSET, StringPool.UTF8); } processPostMethod(postMethod, fileParts, parts); } } else if (method.equals(Http.Method.DELETE)) { httpMethod = new DeleteMethod(location); } else if (method.equals(Http.Method.HEAD)) { httpMethod = new HeadMethod(location); } else { httpMethod = new GetMethod(location); } if (headers != null) { for (Map.Entry<String, String> header : headers.entrySet()) { httpMethod.addRequestHeader(header.getKey(), header.getValue()); } } if ((method.equals(Http.Method.POST) || method.equals(Http.Method.PUT)) && ((body != null) || ((fileParts != null) && !fileParts.isEmpty()) || ((parts != null) && !parts.isEmpty()))) { } else if (!hasRequestHeader(httpMethod, HttpHeaders.CONTENT_TYPE)) { httpMethod.addRequestHeader(HttpHeaders.CONTENT_TYPE, ContentTypes.APPLICATION_X_WWW_FORM_URLENCODED_UTF8); } if (!hasRequestHeader(httpMethod, HttpHeaders.USER_AGENT)) { httpMethod.addRequestHeader(HttpHeaders.USER_AGENT, _DEFAULT_USER_AGENT); } httpState = new HttpState(); if (ArrayUtil.isNotEmpty(cookies)) { org.apache.commons.httpclient.Cookie[] commonsCookies = toCommonsCookies(cookies); httpState.addCookies(commonsCookies); HttpMethodParams httpMethodParams = httpMethod.getParams(); httpMethodParams.setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY); } if (auth != null) { httpMethod.setDoAuthentication(true); httpState.setCredentials(new AuthScope(auth.getHost(), auth.getPort(), auth.getRealm()), new UsernamePasswordCredentials(auth.getUsername(), auth.getPassword())); } proxifyState(httpState, hostConfiguration); int responseCode = httpClient.executeMethod(hostConfiguration, httpMethod, httpState); response.setResponseCode(responseCode); Header locationHeader = httpMethod.getResponseHeader("location"); if ((locationHeader != null) && !locationHeader.equals(location)) { String redirect = locationHeader.getValue(); if (followRedirects) { return URLtoByteArray(redirect, Http.Method.GET, headers, cookies, auth, body, fileParts, parts, response, followRedirects, progressId, portletRequest); } else { response.setRedirect(redirect); } } InputStream inputStream = httpMethod.getResponseBodyAsStream(); if (inputStream != null) { int contentLength = 0; Header contentLengthHeader = httpMethod.getResponseHeader(HttpHeaders.CONTENT_LENGTH); if (contentLengthHeader != null) { contentLength = GetterUtil.getInteger(contentLengthHeader.getValue()); response.setContentLength(contentLength); } Header contentType = httpMethod.getResponseHeader(HttpHeaders.CONTENT_TYPE); if (contentType != null) { response.setContentType(contentType.getValue()); } if (Validator.isNotNull(progressId) && (portletRequest != null)) { ProgressInputStream progressInputStream = new ProgressInputStream(portletRequest, inputStream, contentLength, progressId); UnsyncByteArrayOutputStream unsyncByteArrayOutputStream = new UnsyncByteArrayOutputStream( contentLength); try { progressInputStream.readAll(unsyncByteArrayOutputStream); } finally { progressInputStream.clearProgress(); } bytes = unsyncByteArrayOutputStream.unsafeGetByteArray(); unsyncByteArrayOutputStream.close(); } else { bytes = FileUtil.getBytes(inputStream); } } for (Header header : httpMethod.getResponseHeaders()) { response.addHeader(header.getName(), header.getValue()); } return bytes; } finally { try { if (httpState != null) { _cookies.set(toServletCookies(httpState.getCookies())); } } catch (Exception e) { _log.error(e, e); } try { if (httpMethod != null) { httpMethod.releaseConnection(); } } catch (Exception e) { _log.error(e, e); } } }
From source file:org.apache.nutch.protocol.httpclient.HttpResponse.java
/** * Fetches the given <code>url</code> and prepares HTTP response. * /* w w w . j av a2 s. co m*/ * @param http * An instance of the implementation class of this plugin * @param url * URL to be fetched * @param datum * Crawl data * @param followRedirects * Whether to follow redirects; follows redirect if and only if this * is true * @return HTTP response * @throws IOException * When an error occurs */ HttpResponse(Http http, URL url, CrawlDatum datum, boolean followRedirects) throws IOException { // Prepare GET method for HTTP request this.url = url; GetMethod get = new GetMethod(url.toString()); get.setFollowRedirects(followRedirects); get.setDoAuthentication(true); if (http.isIfModifiedSinceEnabled() && datum.getModifiedTime() > 0) { get.setRequestHeader("If-Modified-Since", HttpDateFormat.toString(datum.getModifiedTime())); } // Set HTTP parameters HttpMethodParams params = get.getParams(); if (http.getUseHttp11()) { params.setVersion(HttpVersion.HTTP_1_1); } else { params.setVersion(HttpVersion.HTTP_1_0); } params.makeLenient(); params.setContentCharset("UTF-8"); if (http.isCookieEnabled()) { params.setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY); params.setBooleanParameter(HttpMethodParams.SINGLE_COOKIE_HEADER, true); } else { params.setCookiePolicy(CookiePolicy.IGNORE_COOKIES); } // XXX (ab) not sure about this... the default is to retry 3 times; if // XXX the request body was sent the method is not retried, so there is // XXX little danger in retrying... // params.setParameter(HttpMethodParams.RETRY_HANDLER, null); if (http.isCookieEnabled() && datum.getMetaData().containsKey(http.COOKIE)) { String cookie = ((Text) datum.getMetaData().get(http.COOKIE)).toString(); get.addRequestHeader("Cookie", cookie); } try { HttpClient client = Http.getClient(); client.getParams().setParameter("http.useragent", http.getUserAgent()); // NUTCH-1941 code = client.executeMethod(get); Header[] heads = get.getResponseHeaders(); for (int i = 0; i < heads.length; i++) { headers.set(heads[i].getName(), heads[i].getValue()); } // Limit download size int contentLength = Integer.MAX_VALUE; String contentLengthString = headers.get(Response.CONTENT_LENGTH); if (contentLengthString != null) { try { contentLength = Integer.parseInt(contentLengthString.trim()); } catch (NumberFormatException ex) { throw new HttpException("bad content length: " + contentLengthString); } } if (http.getMaxContent() >= 0 && contentLength > http.getMaxContent()) { contentLength = http.getMaxContent(); } // always read content. Sometimes content is useful to find a cause // for error. InputStream in = get.getResponseBodyAsStream(); try { byte[] buffer = new byte[HttpBase.BUFFER_SIZE]; int bufferFilled = 0; int totalRead = 0; ByteArrayOutputStream out = new ByteArrayOutputStream(); while ((bufferFilled = in.read(buffer, 0, buffer.length)) != -1 && totalRead + bufferFilled <= contentLength) { totalRead += bufferFilled; out.write(buffer, 0, bufferFilled); } content = out.toByteArray(); } catch (Exception e) { if (code == 200) throw new IOException(e.toString()); // for codes other than 200 OK, we are fine with empty content } finally { if (in != null) { in.close(); } get.abort(); } StringBuilder fetchTrace = null; if (Http.LOG.isTraceEnabled()) { // Trace message fetchTrace = new StringBuilder( "url: " + url + "; status code: " + code + "; bytes received: " + content.length); if (getHeader(Response.CONTENT_LENGTH) != null) fetchTrace.append("; Content-Length: " + getHeader(Response.CONTENT_LENGTH)); if (getHeader(Response.LOCATION) != null) fetchTrace.append("; Location: " + getHeader(Response.LOCATION)); } // Extract gzip, x-gzip and deflate content if (content != null) { // check if we have to uncompress it String contentEncoding = headers.get(Response.CONTENT_ENCODING); if (contentEncoding != null && Http.LOG.isTraceEnabled()) fetchTrace.append("; Content-Encoding: " + contentEncoding); if ("gzip".equals(contentEncoding) || "x-gzip".equals(contentEncoding)) { content = http.processGzipEncoded(content, url); if (Http.LOG.isTraceEnabled()) fetchTrace.append("; extracted to " + content.length + " bytes"); } else if ("deflate".equals(contentEncoding)) { content = http.processDeflateEncoded(content, url); if (Http.LOG.isTraceEnabled()) fetchTrace.append("; extracted to " + content.length + " bytes"); } } // Logger trace message if (Http.LOG.isTraceEnabled()) { Http.LOG.trace(fetchTrace.toString()); } } finally { get.releaseConnection(); } }
From source file:org.apache.nutch.protocol.httpclient.HttpResponseBak.java
/** * Fetches the given <code>url</code> and prepares HTTP response. * * @param http An instance of the implementation class * of this plugin * @param url URL to be fetched * @param datum Crawl data * @param followRedirects Whether to follow redirects; follows * redirect if and only if this is true * @return HTTP response * @throws IOException When an error occurs *///from w w w .ja v a2 s . c o m HttpResponseBak(HttpBak http, URL url, CrawlDatum datum, boolean followRedirects) throws IOException { // Prepare GET method for HTTP request this.url = url; GetMethod get = new GetMethod(url.toString()); get.setFollowRedirects(followRedirects); get.setDoAuthentication(true); if (datum.getModifiedTime() > 0) { get.setRequestHeader("If-Modified-Since", HttpDateFormat.toString(datum.getModifiedTime())); } // Set HTTP parameters HttpMethodParams params = get.getParams(); if (http.getUseHttp11()) { params.setVersion(HttpVersion.HTTP_1_1); } else { params.setVersion(HttpVersion.HTTP_1_0); } params.makeLenient(); params.setContentCharset("UTF-8"); params.setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY); params.setBooleanParameter(HttpMethodParams.SINGLE_COOKIE_HEADER, true); // XXX (ab) not sure about this... the default is to retry 3 times; if // XXX the request body was sent the method is not retried, so there is // XXX little danger in retrying... // params.setParameter(HttpMethodParams.RETRY_HANDLER, null); try { code = Http.getClient().executeMethod(get); Header[] heads = get.getResponseHeaders(); for (int i = 0; i < heads.length; i++) { headers.set(heads[i].getName(), heads[i].getValue()); } // Limit download size int contentLength = Integer.MAX_VALUE; String contentLengthString = headers.get(Response.CONTENT_LENGTH); if (contentLengthString != null) { try { contentLength = Integer.parseInt(contentLengthString.trim()); } catch (NumberFormatException ex) { throw new HttpException("bad content length: " + contentLengthString); } } if (http.getMaxContent() >= 0 && contentLength > http.getMaxContent()) { contentLength = http.getMaxContent(); } // always read content. Sometimes content is useful to find a cause // for error. InputStream in = get.getResponseBodyAsStream(); try { byte[] buffer = new byte[HttpBase.BUFFER_SIZE]; int bufferFilled = 0; int totalRead = 0; ByteArrayOutputStream out = new ByteArrayOutputStream(); while ((bufferFilled = in.read(buffer, 0, buffer.length)) != -1 && totalRead + bufferFilled <= contentLength) { totalRead += bufferFilled; out.write(buffer, 0, bufferFilled); } content = out.toByteArray(); } catch (Exception e) { if (code == 200) throw new IOException(e.toString()); // for codes other than 200 OK, we are fine with empty content } finally { if (in != null) { in.close(); } get.abort(); } StringBuilder fetchTrace = null; if (Http.LOG.isTraceEnabled()) { // Trace message fetchTrace = new StringBuilder( "url: " + url + "; status code: " + code + "; bytes received: " + content.length); if (getHeader(Response.CONTENT_LENGTH) != null) fetchTrace.append("; Content-Length: " + getHeader(Response.CONTENT_LENGTH)); if (getHeader(Response.LOCATION) != null) fetchTrace.append("; Location: " + getHeader(Response.LOCATION)); } // Extract gzip, x-gzip and deflate content if (content != null) { // check if we have to uncompress it String contentEncoding = headers.get(Response.CONTENT_ENCODING); if (contentEncoding != null && Http.LOG.isTraceEnabled()) fetchTrace.append("; Content-Encoding: " + contentEncoding); if ("gzip".equals(contentEncoding) || "x-gzip".equals(contentEncoding)) { content = http.processGzipEncoded(content, url); if (Http.LOG.isTraceEnabled()) fetchTrace.append("; extracted to " + content.length + " bytes"); } else if ("deflate".equals(contentEncoding)) { content = http.processDeflateEncoded(content, url); if (Http.LOG.isTraceEnabled()) fetchTrace.append("; extracted to " + content.length + " bytes"); } } // Logger trace message if (Http.LOG.isTraceEnabled()) { Http.LOG.trace(fetchTrace.toString()); } } finally { get.releaseConnection(); } }
From source file:org.apache.nutch.protocol.httpclient.proxy.HttpResponse.java
/** * Fetches the given <code>url</code> and prepares HTTP response. * /*from w w w. ja v a 2s . c om*/ * @param http * An instance of the implementation class of this plugin * @param url * URL to be fetched * @param datum * Crawl data * @param followRedirects * Whether to follow redirects; follows redirect if and only if this * is true * @return HTTP response * @throws IOException * When an error occurs */ HttpResponse(Http http, URL url, CrawlDatum datum, boolean followRedirects) throws IOException { // Prepare GET method for HTTP request this.url = url; GetMethod get = new GetMethod(url.toString()); get.setFollowRedirects(followRedirects); get.setDoAuthentication(true); if (http.isIfModifiedSinceEnabled() && datum.getModifiedTime() > 0) { get.setRequestHeader("If-Modified-Since", HttpDateFormat.toString(datum.getModifiedTime())); } // Set HTTP parameters HttpMethodParams params = get.getParams(); if (http.getUseHttp11()) { params.setVersion(HttpVersion.HTTP_1_1); } else { params.setVersion(HttpVersion.HTTP_1_0); } params.makeLenient(); params.setContentCharset("UTF-8"); params.setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY); params.setBooleanParameter(HttpMethodParams.SINGLE_COOKIE_HEADER, true); // XXX (ab) not sure about this... the default is to retry 3 times; if // XXX the request body was sent the method is not retried, so there is // XXX little danger in retrying... // params.setParameter(HttpMethodParams.RETRY_HANDLER, null); try { HttpClient client = Http.getClient(); client.getParams().setParameter("http.useragent", http.getUserAgent()); // NUTCH-1941 code = client.executeMethod(get); Header[] heads = get.getResponseHeaders(); for (int i = 0; i < heads.length; i++) { headers.set(heads[i].getName(), heads[i].getValue()); } // Limit download size int contentLength = Integer.MAX_VALUE; String contentLengthString = headers.get(Response.CONTENT_LENGTH); if (contentLengthString != null) { try { contentLength = Integer.parseInt(contentLengthString.trim()); } catch (NumberFormatException ex) { throw new HttpException("bad content length: " + contentLengthString); } } if (http.getMaxContent() >= 0 && contentLength > http.getMaxContent()) { contentLength = http.getMaxContent(); } // always read content. Sometimes content is useful to find a cause // for error. InputStream in = get.getResponseBodyAsStream(); try { byte[] buffer = new byte[HttpBase.BUFFER_SIZE]; int bufferFilled = 0; int totalRead = 0; ByteArrayOutputStream out = new ByteArrayOutputStream(); while ((bufferFilled = in.read(buffer, 0, buffer.length)) != -1 && totalRead + bufferFilled <= contentLength) { totalRead += bufferFilled; out.write(buffer, 0, bufferFilled); } content = out.toByteArray(); } catch (Exception e) { if (code == 200) throw new IOException(e.toString()); // for codes other than 200 OK, we are fine with empty content } finally { if (in != null) { in.close(); } get.abort(); } StringBuilder fetchTrace = null; if (Http.LOG.isTraceEnabled()) { // Trace message fetchTrace = new StringBuilder( "url: " + url + "; status code: " + code + "; bytes received: " + content.length); if (getHeader(Response.CONTENT_LENGTH) != null) fetchTrace.append("; Content-Length: " + getHeader(Response.CONTENT_LENGTH)); if (getHeader(Response.LOCATION) != null) fetchTrace.append("; Location: " + getHeader(Response.LOCATION)); } // Extract gzip, x-gzip and deflate content if (content != null) { // check if we have to uncompress it String contentEncoding = headers.get(Response.CONTENT_ENCODING); if (contentEncoding != null && Http.LOG.isTraceEnabled()) fetchTrace.append("; Content-Encoding: " + contentEncoding); if ("gzip".equals(contentEncoding) || "x-gzip".equals(contentEncoding)) { content = http.processGzipEncoded(content, url); if (Http.LOG.isTraceEnabled()) fetchTrace.append("; extracted to " + content.length + " bytes"); } else if ("deflate".equals(contentEncoding)) { content = http.processDeflateEncoded(content, url); if (Http.LOG.isTraceEnabled()) fetchTrace.append("; extracted to " + content.length + " bytes"); } } // Logger trace message if (Http.LOG.isTraceEnabled()) { Http.LOG.trace(fetchTrace.toString()); } } finally { get.releaseConnection(); } }