List of usage examples for org.apache.commons.httpclient HttpMethod getURI
public abstract URI getURI() throws URIException;
From source file:org.parosproxy.paros.network.HttpSender.java
public int executeMethod(HttpMethod method, HttpState state) throws IOException { int responseCode = -1; String hostName;// w ww. ja v a2s .c o m hostName = method.getURI().getHost(); method.setDoAuthentication(true); HostConfiguration hc = null; HttpClient requestClient; if (param.isUseProxy(hostName)) { requestClient = clientViaProxy; } else { // ZAP: use custom client on upgrade connection and on event-source data type Header connectionHeader = method.getRequestHeader("connection"); boolean isUpgrade = connectionHeader != null && connectionHeader.getValue().toLowerCase().contains("upgrade"); // ZAP: try to apply original handling of ParosProxy requestClient = client; if (isUpgrade) { // Unless upgrade, when using another client that allows us to expose the socket // connection. requestClient = new HttpClient(new ZapHttpConnectionManager()); } } if (this.initiator == CHECK_FOR_UPDATES_INITIATOR) { // Use the 'strict' SSLConnector, ie one that performs all the usual cert checks // The 'standard' one 'trusts' everything // This is to ensure that all 'check-for update' calls are made to the expected https urls // without this is would be possible to intercept and change the response which could result // in the user downloading and installing a malicious add-on hc = new HostConfiguration() { @Override public synchronized void setHost(URI uri) { try { setHost(new HttpHost(uri.getHost(), uri.getPort(), getProtocol())); } catch (URIException e) { throw new IllegalArgumentException(e.toString()); } }; }; hc.setHost(hostName, method.getURI().getPort(), new Protocol("https", (ProtocolSocketFactory) new SSLConnector(false), 443)); if (param.isUseProxy(hostName)) { hc.setProxyHost(new ProxyHost(param.getProxyChainName(), param.getProxyChainPort())); if (param.isUseProxyChainAuth()) { requestClient.getState().setProxyCredentials(getAuthScope(param), getNTCredentials(param)); } } } // ZAP: Check if a custom state is being used if (state != null) { // Make sure cookies are enabled method.getParams().setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY); } responseCode = requestClient.executeMethod(hc, method, state); return responseCode; }
From source file:org.paxle.crawler.http.impl.HttpCrawler.java
/** * Initializes the {@link HttpMethod} with common attributes for all requests this crawler * initiates./*from w w w .java2 s .co m*/ * <p> * Currently the following attributes (represented as HTTP header values in the final request) * are set: * <ul> * <li>the cookie policy to use ({@link #PROP_COOKIE_POLICY})</li> * <li> * if enabled, content-transformation using <code>compress</code>, <code>gzip</code> and * <code>deflate</code> is supported</li> * </li> * </ul> * * @param method the method to set the standard attributes on */ private void initRequestMethod(final HttpMethod method) throws URIException { method.getParams().setCookiePolicy( this.cookiePolicy == null ? CookiePolicy.BROWSER_COMPATIBILITY : this.cookiePolicy); if (acceptEncoding && !isHostSettingSet(method.getURI().getHost(), PREF_NO_ENCODING)) method.setRequestHeader(HTTPHEADER_ACCEPT_ENCODING, "compress, gzip, identity, deflate"); // see RFC 2616, section 14.3 // set some additional http headers if (this.userAgent != null) { method.setRequestHeader("User-Agent", this.userAgent); } }
From source file:org.paxle.crawler.http.impl.HttpCrawler.java
public ICrawlerDocument request(URI requestUri) { if (requestUri == null) throw new NullPointerException("URL was null"); this.logger.debug(String.format("Crawling URL '%s' ...", requestUri)); ICrawlerDocument doc = null;//from w ww. jav a 2 s . c o m HttpMethod method = null; try { final ICrawlerContext ctx = this.contextLocal.getCurrentContext(); // creating an empty crawler-document doc = ctx.createDocument(); doc.setLocation(requestUri); final String uriAsciiString = requestUri.toASCIIString(); /* ============================================================================== * HTTP HEAD request * * first use the HEAD method to determine whether the MIME-type is supported * and to compare the content-length with the maximum allowed download size * (both only if the server provides this information, if not, the file is * fetched) * ============================================================================== */ method = new HeadMethod(uriAsciiString); // automatically follows redirects this.initRequestMethod(method); int statusCode = this.getHttpClient().executeMethod(method); final boolean headUnsupported = (statusCode == HttpStatus.SC_METHOD_FAILURE || statusCode == HttpStatus.SC_METHOD_NOT_ALLOWED); if (!headUnsupported) { if (statusCode != HttpStatus.SC_OK) { // RFC 2616 states that the GET and HEAD methods _must_ be supported by any // general purpose servers (which are in fact the ones we are connecting to here) if (statusCode == HttpStatus.SC_NOT_FOUND) { doc.setStatus(ICrawlerDocument.Status.NOT_FOUND); } else { doc.setStatus(ICrawlerDocument.Status.UNKNOWN_FAILURE, String.format("Server returned: %s", method.getStatusLine())); } this.logger.warn(String.format("Crawling of URL '%s' failed. Server returned: %s", requestUri, method.getStatusLine())); return doc; } // getting the mimetype and charset Header contentTypeHeader = method.getResponseHeader(HTTPHEADER_CONTENT_TYPE); if (!handleContentTypeHeader(contentTypeHeader, doc)) return doc; // reject the document if content-length is above our limit Header contentLengthHeader = method.getResponseHeader(HTTPHEADER_CONTENT_LENGTH); if (!handleContentLengthHeader(contentLengthHeader, doc)) return doc; // FIXME: we've been redirected, re-enqueue the new URL and abort processing //if (!requestUri.equals(method.getURI())) ; } /* ============================================================================== * HTTP GET request * * secondly - if everything is alright up to now - proceed with getting the * actual document * ============================================================================== */ HttpMethod getMethod = new GetMethod(uriAsciiString); // automatically follows redirects method.releaseConnection(); method = getMethod; this.initRequestMethod(method); // send the request to the server statusCode = this.getHttpClient().executeMethod(method); // check the response status code if (statusCode != HttpStatus.SC_OK) { if (statusCode == HttpStatus.SC_NOT_FOUND) { doc.setStatus(ICrawlerDocument.Status.NOT_FOUND); } else { doc.setStatus(ICrawlerDocument.Status.UNKNOWN_FAILURE, String.format("Server returned: %s", method.getStatusLine())); } this.logger.warn(String.format("Crawling of URL '%s' failed. Server returned: %s", requestUri, method.getStatusLine())); return doc; } // FIXME: we've been redirected, re-enqueue the new URL and abort processing // if (!requestUri.equals(method.getURI())) ; /* * HTTP Content-Type * - getting the mimetype and charset */ Header contentTypeHeader = method.getResponseHeader(HTTPHEADER_CONTENT_TYPE); if (!handleContentTypeHeader(contentTypeHeader, doc)) return doc; /* * HTTP Content-Length * - Reject the document if content-length is above our limit * * We do this a second time here because some servers may have set the content-length * of the head response to <code>0</code> */ Header contentLengthHeader = method.getResponseHeader(HTTPHEADER_CONTENT_LENGTH); if (!handleContentLengthHeader(contentLengthHeader, doc)) return doc; extractHttpHeaders(method, doc); // externalised into this method to cleanup here a bit // getting the response body InputStream respBody = method.getResponseBodyAsStream(); // handle the content-encoding, i.e. decompress the server's response Header contentEncodingHeader = method.getResponseHeader(HTTPHEADER_CONTENT_ENCODING); try { respBody = handleContentEncoding(contentEncodingHeader, respBody); /* Limit the max allowed length of the content to copy. -1 is used for no limit. * * We need to set a limit if: * a) the user has configured a max-download-size AND * b) the server returned no content-length header */ int copyLimit = (this.maxDownloadSize <= 0 || contentLengthHeader != null) ? -1 : this.maxDownloadSize; // copy the content to file final ICrawlerTools crawlerTools = ctx.getCrawlerTools(); crawlerTools.saveInto(doc, respBody, lrc, copyLimit); doc.setStatus(ICrawlerDocument.Status.OK); this.logger.debug(String.format("Crawling of URL '%s' finished.", requestUri)); } catch (IOException e) { String msg = e.getMessage(); if (msg == null || !msg.equals("Corrupt GZIP trailer")) throw e; setHostSetting(method.getURI().getHost(), PREF_NO_ENCODING); msg = String.format("server sent a corrupt gzip trailer at URL '%s'", requestUri); logger.warn(msg); // FIXME re-enqueue command doc.setStatus(ICrawlerDocument.Status.UNKNOWN_FAILURE, msg); } finally { respBody.close(); } } catch (NoRouteToHostException e) { this.logger.warn(String.format("Error crawling %s: %s", requestUri, e.getMessage())); doc.setStatus(ICrawlerDocument.Status.NOT_FOUND, e.getMessage()); } catch (UnknownHostException e) { this.logger.warn(String.format("Error crawling %s: Unknown host.", requestUri)); doc.setStatus(ICrawlerDocument.Status.NOT_FOUND, e.getMessage()); } catch (ConnectException e) { this.logger.warn(String.format("Error crawling %s: Unable to connect to host.", requestUri)); doc.setStatus(ICrawlerDocument.Status.NOT_FOUND, e.getMessage()); } catch (ConnectTimeoutException e) { this.logger.warn(String.format("Error crawling %s: %s.", requestUri, e.getMessage())); doc.setStatus(ICrawlerDocument.Status.NOT_FOUND, e.getMessage()); } catch (SocketTimeoutException e) { this.logger.warn(String.format("Error crawling %s: Connection timeout.", requestUri)); doc.setStatus(ICrawlerDocument.Status.NOT_FOUND, e.getMessage()); } catch (CircularRedirectException e) { this.logger.warn(String.format("Error crawling %s: %s", requestUri, e.getMessage())); doc.setStatus(ICrawlerDocument.Status.NOT_FOUND, e.getMessage()); } catch (NoHttpResponseException e) { this.logger.warn(String.format("Error crawling %s: %s", requestUri, e.getMessage())); doc.setStatus(ICrawlerDocument.Status.NOT_FOUND, e.getMessage()); } catch (ContentLengthLimitExceededException e) { this.logger.warn(String.format("Error crawling %s: %s", requestUri, e.getMessage())); doc.setStatus(ICrawlerDocument.Status.UNKNOWN_FAILURE, e.getMessage()); } catch (Throwable e) { String errorMsg; if (e instanceof HttpException) { errorMsg = "Unrecovered protocol exception: [%s] %s"; } else if (e instanceof IOException) { errorMsg = "Transport exceptions: [%s] %s"; } else { errorMsg = "Unexpected exception: [%s] %s"; } errorMsg = String.format(errorMsg, e.getClass().getName(), e.getMessage()); this.logger.error(String.format("Error crawling %s: %s", requestUri, errorMsg)); doc.setStatus(ICrawlerDocument.Status.UNKNOWN_FAILURE, errorMsg); e.printStackTrace(); } finally { if (method != null) method.releaseConnection(); } return doc; }
From source file:org.pentaho.di.baserver.utils.web.HttpConnectionHelperTest.java
@Test public void testGetHttpMethod() throws Exception { Map<String, String> queryParameters = new HashMap<String, String>(); queryParameters.put("param1", "value1"); queryParameters.put("param2", "value2"); queryParameters.put("param3", "value3"); String url = "http://localhost:8080/pentaho"; HttpMethod method = httpConnectionHelperSpy.getHttpMethod(url, queryParameters, "GET"); assertEquals(method.getClass(), GetMethod.class); assertTrue(method.getURI().toString().startsWith(url)); assertNotNull(method.getQueryString()); method = httpConnectionHelperSpy.getHttpMethod(url, queryParameters, "PUT"); assertEquals(method.getClass(), PutMethod.class); assertTrue(method.getURI().toString().startsWith(url)); RequestEntity requestEntity = ((PutMethod) method).getRequestEntity(); assertNotNull(requestEntity);/*from ww w . j a v a 2 s .com*/ assertEquals(requestEntity.getContentType(), "application/x-www-form-urlencoded; charset=UTF-8"); assertNull(method.getQueryString()); assertEquals(requestEntity.getClass(), StringRequestEntity.class); assertNotNull(((StringRequestEntity) requestEntity).getContent()); method = httpConnectionHelperSpy.getHttpMethod(url, queryParameters, "POST"); assertEquals(method.getClass(), PostMethod.class); assertTrue(method.getURI().toString().startsWith(url)); requestEntity = ((PostMethod) method).getRequestEntity(); assertNotNull(requestEntity); assertEquals(requestEntity.getContentType(), "application/x-www-form-urlencoded; charset=UTF-8"); assertNull(method.getQueryString()); assertEquals(requestEntity.getClass(), StringRequestEntity.class); assertNotNull(((StringRequestEntity) requestEntity).getContent()); // POST without parameters method = httpConnectionHelperSpy.getHttpMethod(url, null, "POST"); assertEquals(method.getClass(), PostMethod.class); assertTrue(method.getURI().toString().startsWith(url)); requestEntity = ((PostMethod) method).getRequestEntity(); assertNotNull(requestEntity); assertEquals(requestEntity.getContentType(), "application/x-www-form-urlencoded; charset=UTF-8"); assertNull(method.getQueryString()); assertEquals(requestEntity.getClass(), StringRequestEntity.class); assertNotNull(((StringRequestEntity) requestEntity).getContent()); method = httpConnectionHelperSpy.getHttpMethod(url, queryParameters, "DELETE"); assertEquals(method.getClass(), DeleteMethod.class); assertTrue(method.getURI().toString().startsWith(url)); assertNotNull(method.getQueryString()); method = httpConnectionHelperSpy.getHttpMethod(url, queryParameters, "HEAD"); assertEquals(method.getClass(), HeadMethod.class); assertTrue(method.getURI().toString().startsWith(url)); assertNotNull(method.getQueryString()); method = httpConnectionHelperSpy.getHttpMethod(url, queryParameters, "OPTIONS"); assertEquals(method.getClass(), OptionsMethod.class); assertTrue(method.getURI().toString().startsWith(url)); assertNotNull(method.getQueryString()); }
From source file:org.pentaho.pac.server.common.ThreadSafeHttpClient.java
/** * Execute the <param>method</param>, and return the server's response as a string * @param method the HttpMethod specifying the server URL and parameters to be * passed to the server./*w w w. j a va2 s.com*/ * @return a string containing the server's response * * @throws ProxyException if the attempt to communicate with the server fails, * if the attempt to read the response from the server fails, if the response * stream is unable to be converted into a String. */ private String executeMethod(HttpMethod method) throws ProxyException { InputStream responseStrm = null; try { int httpStatus = CLIENT.executeMethod(method); if (httpStatus != HttpStatus.SC_OK) { // If the response comes as unauthorized access we will throw a proxy exception explaining the reason and // what needs to be done to correct it if (httpStatus == HttpStatus.SC_UNAUTHORIZED) { throw new ProxyException( Messages.getErrorString("ThreadSafeHttpClient.ERROR_0003_AUTHORIZATION_FAILED")); } String status = method.getStatusLine().toString(); String uri = method.getURI().toString(); String errorMsg = Messages.getErrorString("ThreadSafeHttpClient.ERROR_0001_CLIENT_REQUEST_FAILED", //$NON-NLS-1$ uri, status); logger.error(errorMsg); throw new ProxyException(status); // TODO } responseStrm = method.getResponseBodyAsStream(); // trim() is necessary because some jsp's put \n\r at the beginning of // the returned text, and the xml processor chokes on \n\r at the beginning. String response = IOUtils.toString(responseStrm).trim(); return response; } catch (Exception e) { throw new ProxyException(e); } finally { method.releaseConnection(); } }
From source file:org.sipfoundry.sipxconfig.admin.CertificateManagerTest.java
private CertificateManagerImpl createCertificateManager() { return new CertificateManagerImpl() { @Override/*from www . j a va 2 s . com*/ protected HttpClient getNewHttpClient() { return new HttpClient() { @Override public int executeMethod(HttpMethod method) { if (StringUtils.equals(method.getName(), DELETE)) { try { URI reqURI = method.getURI(); FileUtils.deleteQuietly(new File(reqURI.getPath())); } catch (URIException uex) { //do nothing } return 200; } else { return 501; } } }; } }; }
From source file:org.sonatype.nexus.proxy.storage.remote.commonshttpclient.CommonsHttpClientRemoteStorage.java
/** * Execute method. In case of any exception thrown by HttpClient, it will release the connection. In other cases it * is the duty of caller to do it, or process the input stream. * /*from w w w .j a v a 2 s. c om*/ * @param method the method * @return the int */ protected int doExecuteMethod(ProxyRepository repository, ResourceStoreRequest request, HttpMethod method, URL remoteUrl) throws RemoteStorageException { URI methodURI = null; try { methodURI = method.getURI(); } catch (URIException e) { getLogger().debug("Could not format debug log message", e); } if (getLogger().isDebugEnabled()) { getLogger().debug("Invoking HTTP " + method.getName() + " method against remote location " + methodURI); } RemoteStorageContext ctx = getRemoteStorageContext(repository); HttpClient httpClient = (HttpClient) ctx.getContextObject(CTX_KEY_CLIENT); HostConfiguration httpConfiguration = (HostConfiguration) ctx.getContextObject(CTX_KEY_HTTP_CONFIGURATION); method.setRequestHeader(new Header("user-agent", formatUserAgentString(ctx, repository))); method.setRequestHeader(new Header("accept", "*/*")); method.setRequestHeader(new Header("accept-language", "en-us")); method.setRequestHeader(new Header("accept-encoding", "gzip, identity")); method.setRequestHeader(new Header("cache-control", "no-cache")); // HTTP keep alive should not be used, except when NTLM is used Boolean isNtlmUsed = (Boolean) ctx.getContextObject(HttpClientProxyUtil.NTLM_IS_IN_USE_KEY); if (isNtlmUsed == null || !isNtlmUsed) { method.setRequestHeader(new Header("Connection", "close")); method.setRequestHeader(new Header("Proxy-Connection", "close")); } method.setFollowRedirects(true); if (StringUtils.isNotBlank(ctx.getRemoteConnectionSettings().getQueryString())) { method.setQueryString(ctx.getRemoteConnectionSettings().getQueryString()); } int resultCode; try { resultCode = httpClient.executeMethod(httpConfiguration, method); final Header httpServerHeader = method.getResponseHeader("server"); checkForRemotePeerAmazonS3Storage(repository, httpServerHeader == null ? null : httpServerHeader.getValue()); Header proxyReturnedErrorHeader = method.getResponseHeader(NEXUS_MISSING_ARTIFACT_HEADER); boolean proxyReturnedError = proxyReturnedErrorHeader != null && Boolean.valueOf(proxyReturnedErrorHeader.getValue()); if (resultCode == HttpStatus.SC_FORBIDDEN) { throw new RemoteAccessDeniedException(repository, remoteUrl, HttpStatus.getStatusText(HttpStatus.SC_FORBIDDEN)); } else if (resultCode == HttpStatus.SC_UNAUTHORIZED) { throw new RemoteAuthenticationNeededException(repository, HttpStatus.getStatusText(HttpStatus.SC_UNAUTHORIZED)); } else if (resultCode == HttpStatus.SC_OK && proxyReturnedError) { throw new RemoteStorageException( "Invalid artifact found, most likely a proxy redirected to an HTML error page."); } } catch (RemoteStorageException e) { method.releaseConnection(); throw e; } catch (HttpException ex) { method.releaseConnection(); throw new RemoteStorageException("Protocol error while executing " + method.getName() + " method. [repositoryId=\"" + repository.getId() + "\", requestPath=\"" + request.getRequestPath() + "\", remoteUrl=\"" + methodURI + "\"]", ex); } catch (IOException ex) { method.releaseConnection(); throw new RemoteStorageException("Transport error while executing " + method.getName() + " method [repositoryId=\"" + repository.getId() + "\", requestPath=\"" + request.getRequestPath() + "\", remoteUrl=\"" + methodURI + "\"]", ex); } return resultCode; }
From source file:org.switchyard.component.test.mixins.http.HTTPMixIn.java
/** * Execute the supplied HTTP Method.//from w w w .j a v a 2 s . co m * <p/> * Does not release the {@link org.apache.commons.httpclient.HttpMethod#releaseConnection() HttpMethod connection}. * * @param method The HTTP Method. * @return The HTTP Response. */ public String execute(HttpMethod method) { if (_httpClient == null) { Assert.fail( "HTTPMixIn not initialized. You must call the initialize() method before using this MixIn"); } for (String key : _requestHeaders.keySet()) { method.setRequestHeader(key, _requestHeaders.get(key)); } if (_dumpMessages) { for (Header header : method.getRequestHeaders()) { _logger.info("Request header:[" + header.getName() + "=" + header.getValue() + "]"); } } String response = null; try { _httpClient.executeMethod(method); response = method.getResponseBodyAsString(); } catch (Exception e) { try { Assert.fail("Exception invoking HTTP endpoint '" + method.getURI() + "': " + e.getMessage()); } catch (URIException e1) { _logger.error("Unexpected error", e1); return null; } } if (_dumpMessages) { for (Header header : method.getResponseHeaders()) { _logger.info("Received response header:[" + header.getName() + "=" + header.getValue() + "]"); } _logger.info("Received response body:[" + response + "]"); } for (String key : _expectedHeaders.keySet()) { Header actual = method.getResponseHeader(key); Assert.assertNotNull("Checking response header:[" + key + "]", actual); Assert.assertEquals("Checking response header:[" + key + "]", _expectedHeaders.get(key), actual.getValue()); } return response; }
From source file:org.xwiki.test.rest.framework.AbstractHttpTest.java
protected String getHttpMethodInfo(HttpMethod method) throws Exception { return String.format("\nName: %s\nURI: %s\nStatus code: %d\nStatus text: %s", method.getName(), method.getURI(), method.getStatusCode(), method.getStatusText()); }
From source file:org.xwiki.xwoot.manager.internal.DefaultXWootManager.java
private String call(String service) { String xwootAppAddress = getXWootAppAddress(); HttpMethod method = new GetMethod(xwootAppAddress + service); /* This is needed because the xwootApp servlet might send redirects to perform initializations */ method.setFollowRedirects(true);// w w w .j av a2 s . c o m try { getLogger().debug("Requesting: " + method.getURI()); if (client.executeMethod(method) < 400) { String result = method.getResponseBodyAsString(); getLogger().debug("Result: " + result); return result; } getLogger().info("Failed call: " + method.getStatusLine()); } catch (CircularRedirectException e) { /* * Ignore. This could be normal. For example in the case of connecting/disconnecting the P2P network we call * the synchronize servlet that redirects to the boostrap that redirects to synchronize again, causing this * exception. */ } catch (Exception ex) { getLogger().warn("Exception occured while calling [" + service + "] on [" + xwootAppAddress + "]", ex); } finally { // Release the connection, since HTTPClient reuses connections for improved performance method.releaseConnection(); } return "failed"; }