List of usage examples for org.apache.commons.httpclient HostConfiguration setProxy
public void setProxy(String paramString, int paramInt)
From source file:org.apache.maven.wagon.shared.http.AbstractHttpClientWagon.java
public void openConnectionInternal() { repository.setUrl(getURL(repository)); client = new HttpClient(connectionManager); String username = null;//from w w w . j a v a2 s.c om String password = null; if (authenticationInfo != null) { username = authenticationInfo.getUserName(); password = authenticationInfo.getPassword(); } String host = getRepository().getHost(); if (StringUtils.isNotEmpty(username) && StringUtils.isNotEmpty(password)) { Credentials creds = new UsernamePasswordCredentials(username, password); int port = getRepository().getPort() > -1 ? getRepository().getPort() : AuthScope.ANY_PORT; AuthScope scope = new AuthScope(host, port); client.getState().setCredentials(scope, creds); } HostConfiguration hc = new HostConfiguration(); ProxyInfo proxyInfo = getProxyInfo(getRepository().getProtocol(), getRepository().getHost()); if (proxyInfo != null) { String proxyUsername = proxyInfo.getUserName(); String proxyPassword = proxyInfo.getPassword(); String proxyHost = proxyInfo.getHost(); int proxyPort = proxyInfo.getPort(); String proxyNtlmHost = proxyInfo.getNtlmHost(); String proxyNtlmDomain = proxyInfo.getNtlmDomain(); if (proxyHost != null) { hc.setProxy(proxyHost, proxyPort); if (proxyUsername != null && proxyPassword != null) { Credentials creds; if (proxyNtlmHost != null || proxyNtlmDomain != null) { creds = new NTCredentials(proxyUsername, proxyPassword, proxyNtlmHost, proxyNtlmDomain); } else { creds = new UsernamePasswordCredentials(proxyUsername, proxyPassword); } int port = proxyInfo.getPort() > -1 ? proxyInfo.getPort() : AuthScope.ANY_PORT; AuthScope scope = new AuthScope(proxyHost, port); client.getState().setProxyCredentials(scope, creds); } } } hc.setHost(host); //start a session with the webserver client.setHostConfiguration(hc); }
From source file:org.apache.nutch.protocol.httpclient.Http.java
/** * Configures the HTTP client//from w w w . j a va 2 s . c o m */ private void configureClient() { // Set up an HTTPS socket factory that accepts self-signed certs. // ProtocolSocketFactory factory = new SSLProtocolSocketFactory(); ProtocolSocketFactory factory = new DummySSLProtocolSocketFactory(); Protocol https = new Protocol("https", factory, 443); Protocol.registerProtocol("https", https); HttpConnectionManagerParams params = connectionManager.getParams(); params.setConnectionTimeout(timeout); params.setSoTimeout(timeout); params.setSendBufferSize(BUFFER_SIZE); params.setReceiveBufferSize(BUFFER_SIZE); // -------------------------------------------------------------------------------- // NUTCH-1836: Modification to increase the number of available connections // for multi-threaded crawls. // -------------------------------------------------------------------------------- params.setMaxTotalConnections(conf.getInt("mapreduce.tasktracker.map.tasks.maximum", 5) * conf.getInt("fetcher.threads.fetch", maxThreadsTotal)); // Also set max connections per host to maxThreadsTotal since all threads // might be used to fetch from the same host - otherwise timeout errors can // occur params.setDefaultMaxConnectionsPerHost(conf.getInt("fetcher.threads.fetch", maxThreadsTotal)); // executeMethod(HttpMethod) seems to ignore the connection timeout on the // connection manager. // set it explicitly on the HttpClient. client.getParams().setConnectionManagerTimeout(timeout); HostConfiguration hostConf = client.getHostConfiguration(); ArrayList<Header> headers = new ArrayList<Header>(); // Note: some header fields (e.g., "User-Agent") are set per GET request if (!acceptLanguage.isEmpty()) { headers.add(new Header("Accept-Language", acceptLanguage)); } if (!acceptCharset.isEmpty()) { headers.add(new Header("Accept-Charset", acceptCharset)); } if (!accept.isEmpty()) { headers.add(new Header("Accept", accept)); } // accept gzipped content headers.add(new Header("Accept-Encoding", "x-gzip, gzip, deflate")); hostConf.getParams().setParameter("http.default-headers", headers); // HTTP proxy server details if (useProxy) { hostConf.setProxy(proxyHost, proxyPort); if (proxyUsername.length() > 0) { AuthScope proxyAuthScope = getAuthScope(this.proxyHost, this.proxyPort, this.proxyRealm); NTCredentials proxyCredentials = new NTCredentials(this.proxyUsername, this.proxyPassword, Http.agentHost, this.proxyRealm); client.getState().setProxyCredentials(proxyAuthScope, proxyCredentials); } } }
From source file:org.apache.nutch.protocol.httpclient.HttpBak.java
/** * Configures the HTTP client/* w ww.j a v a2 s. c om*/ */ private void configureClient() { // Set up an HTTPS socket factory that accepts self-signed certs. Protocol https = new Protocol("https", new DummySSLProtocolSocketFactory(), 443); Protocol.registerProtocol("https", https); HttpConnectionManagerParams params = connectionManager.getParams(); params.setConnectionTimeout(timeout); params.setSoTimeout(timeout); params.setSendBufferSize(BUFFER_SIZE); params.setReceiveBufferSize(BUFFER_SIZE); params.setMaxTotalConnections(maxThreadsTotal); // executeMethod(HttpMethod) seems to ignore the connection timeout on the connection manager. // set it explicitly on the HttpClient. client.getParams().setConnectionManagerTimeout(timeout); HostConfiguration hostConf = client.getHostConfiguration(); ArrayList headers = new ArrayList(); // Set the User Agent in the header headers.add(new Header("User-Agent", userAgent)); // prefer English headers.add(new Header("Accept-Language", acceptLanguage)); // prefer UTF-8 headers.add(new Header("Accept-Charset", "utf-8,ISO-8859-1;q=0.7,*;q=0.7")); // prefer understandable formats headers.add(new Header("Accept", "text/html,application/xml;q=0.9,application/xhtml+xml,text/xml;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5")); // accept gzipped content headers.add(new Header("Accept-Encoding", "x-gzip, gzip, deflate")); hostConf.getParams().setParameter("http.default-headers", headers); // HTTP proxy server details if (useProxy) { hostConf.setProxy(proxyHost, proxyPort); if (proxyUsername.length() > 0) { AuthScope proxyAuthScope = getAuthScope(this.proxyHost, this.proxyPort, this.proxyRealm); NTCredentials proxyCredentials = new NTCredentials(this.proxyUsername, this.proxyPassword, this.agentHost, this.proxyRealm); client.getState().setProxyCredentials(proxyAuthScope, proxyCredentials); } } }
From source file:org.apache.nutch.protocol.httpclient.proxy.Http.java
/** * Configures the HTTP client/*from w ww . ja v a 2 s .co m*/ */ private void configureClient() { // Set up an HTTPS socket factory that accepts self-signed certs. //ProtocolSocketFactory factory = new SSLProtocolSocketFactory(); ProtocolSocketFactory factory = new DummySSLProtocolSocketFactory(); Protocol https = new Protocol("https", factory, 443); Protocol.registerProtocol("https", https); HttpConnectionManagerParams params = connectionManager.getParams(); params.setConnectionTimeout(timeout); params.setSoTimeout(timeout); params.setSendBufferSize(BUFFER_SIZE); params.setReceiveBufferSize(BUFFER_SIZE); params.setMaxTotalConnections(maxThreadsTotal); // Also set max connections per host to maxThreadsTotal since all threads // might be used to fetch from the same host - otherwise timeout errors can // occur params.setDefaultMaxConnectionsPerHost(maxThreadsTotal); // executeMethod(HttpMethod) seems to ignore the connection timeout on the // connection manager. // set it explicitly on the HttpClient. client.getParams().setConnectionManagerTimeout(timeout); HostConfiguration hostConf = client.getHostConfiguration(); ArrayList<Header> headers = new ArrayList<Header>(); // Set the User Agent in the header //headers.add(new Header("User-Agent", userAgent)); //NUTCH-1941 // prefer English headers.add(new Header("Accept-Language", acceptLanguage)); // prefer UTF-8 headers.add(new Header("Accept-Charset", "utf-8,ISO-8859-1;q=0.7,*;q=0.7")); // prefer understandable formats headers.add(new Header("Accept", "text/html,application/xml;q=0.9,application/xhtml+xml,text/xml;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5")); // accept gzipped content headers.add(new Header("Accept-Encoding", "x-gzip, gzip, deflate")); hostConf.getParams().setParameter("http.default-headers", headers); // HTTP proxy server details if (useProxy) { hostConf.setProxy(proxyHost, proxyPort); if (proxyUsername.length() > 0) { AuthScope proxyAuthScope = getAuthScope(this.proxyHost, this.proxyPort, this.proxyRealm); NTCredentials proxyCredentials = new NTCredentials(this.proxyUsername, this.proxyPassword, Http.agentHost, this.proxyRealm); client.getState().setProxyCredentials(proxyAuthScope, proxyCredentials); } } }
From source file:org.apache.nutch.protocol.webdriver.Http.java
/** * Configures the HTTP client//from w w w . j a v a2 s. c om */ private void configureClient() { // Set up an HTTPS socket factory that accepts self-signed certs. ProtocolSocketFactory factory = new SSLProtocolSocketFactory(); Protocol https = new Protocol("https", factory, 443); Protocol.registerProtocol("https", https); HttpConnectionManagerParams params = connectionManager.getParams(); params.setConnectionTimeout(timeout); params.setSoTimeout(timeout); params.setSendBufferSize(BUFFER_SIZE); params.setReceiveBufferSize(BUFFER_SIZE); params.setMaxTotalConnections(maxThreadsTotal); // Also set max connections per host to maxThreadsTotal since all threads // might be used to fetch from the same host - otherwise timeout errors can // occur params.setDefaultMaxConnectionsPerHost(maxThreadsTotal); // executeMethod(HttpMethod) seems to ignore the connection timeout on // the connection manager. // set it explicitly on the HttpClient. client.getParams().setConnectionManagerTimeout(timeout); HostConfiguration hostConf = client.getHostConfiguration(); if (useProxy) { hostConf.setProxy(proxyHost, proxyPort); } ArrayList<Header> headers = new ArrayList<Header>(); // prefer English headers.add(new Header("Accept-Language", "en-us,en-gb,en;q=0.7,*;q=0.3")); // prefer UTF-8 headers.add(new Header("Accept-Charset", "utf-8,ISO-8859-1;q=0.7,*;q=0.7")); // prefer understandable formats headers.add(new Header("Accept", "text/html,application/xml;q=0.9,application/xhtml+xml,text/xml;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5")); // accept gzipped content // headers.add(new Header("Accept-Encoding", "x-gzip, gzip, deflate")); hostConf.getParams().setParameter("http.default-headers", headers); }
From source file:org.apache.ode.axis2.httpbinding.ProxyConf.java
public static void configure(HostConfiguration hostConfig, HttpState state, HttpTransportProperties.ProxyProperties proxyProperties) { String proxyHost = proxyProperties.getProxyHostName(); int proxyPort = proxyProperties.getProxyPort(); //Setting credentials String userName = proxyProperties.getUserName(); String password = proxyProperties.getPassWord(); String domain = proxyProperties.getDomain(); Credentials proxyCred;//from w w w. j a v a2 s . co m if (userName != null && password != null && domain != null) { proxyCred = new NTCredentials(userName, password, proxyHost, domain); } else if (userName != null) { proxyCred = new UsernamePasswordCredentials(userName, password); } else { proxyCred = new UsernamePasswordCredentials("", ""); } //Using Java Networking Properties String host = System.getProperty(HTTP_PROXY_HOST); if (host != null) { proxyHost = host; proxyCred = new UsernamePasswordCredentials("", ""); } String port = System.getProperty(HTTP_PROXY_PORT); if (port != null) { proxyPort = Integer.parseInt(port); } state.setProxyCredentials(AuthScope.ANY, proxyCred); hostConfig.setProxy(proxyHost, proxyPort); }
From source file:org.apache.servicemix.http.processors.ProviderProcessor.java
private HostConfiguration getHostConfiguration(String locationURI, MessageExchange exchange, NormalizedMessage message) throws Exception { HostConfiguration host; URI uri = new URI(locationURI, false); if (uri.getScheme().equals("https")) { synchronized (this) { if (protocol == null) { ProtocolSocketFactory sf = new CommonsHttpSSLSocketFactory(endpoint.getSsl(), KeystoreManager.Proxy.create(endpoint.getKeystoreManager())); protocol = new Protocol("https", sf, 443); }/*from w w w . j a v a 2 s.co m*/ } HttpHost httphost = new HttpHost(uri.getHost(), uri.getPort(), protocol); host = new HostConfiguration(); host.setHost(httphost); } else { host = new HostConfiguration(); host.setHost(uri.getHost(), uri.getPort()); } if (endpoint.getProxy() != null) { if ((endpoint.getProxy().getProxyHost() != null) && (endpoint.getProxy().getProxyPort() != 0)) { host.setProxy(endpoint.getProxy().getProxyHost(), endpoint.getProxy().getProxyPort()); } if (endpoint.getProxy().getProxyCredentials() != null) { endpoint.getProxy().getProxyCredentials().applyProxyCredentials(getClient(), exchange, message); } } else if ((getConfiguration().getProxyHost() != null) && (getConfiguration().getProxyPort() != 0)) { host.setProxy(getConfiguration().getProxyHost(), getConfiguration().getProxyPort()); } //host.getParams().setLongParameter(HttpConnectionParams.CONNECTION_TIMEOUT, getClient().getParams().getSoTimeout()); return host; }
From source file:org.apache.webdav.lib.WebdavSession.java
/** * Get a <code>HttpClient</code> instance. * This method returns a new client instance, when reset is true. * * @param httpURL The http URL to connect. only used the authority part. * @param reset The reset flag to represent whether the saved information * is used or not./*from w w w. j av a 2s .c o m*/ * @return An instance of <code>HttpClient</code>. * @exception IOException */ public HttpClient getSessionInstance(HttpURL httpURL, boolean reset) throws IOException { if (reset || client == null) { client = new HttpClient(); // Set a state which allows lock tracking client.setState(new WebdavState()); HostConfiguration hostConfig = client.getHostConfiguration(); hostConfig.setHost(httpURL); if (proxyHost != null && proxyPort > 0) hostConfig.setProxy(proxyHost, proxyPort); if (hostCredentials == null) { String userName = httpURL.getUser(); if (userName != null && userName.length() > 0) { hostCredentials = new UsernamePasswordCredentials(userName, httpURL.getPassword()); } } if (hostCredentials != null) { HttpState clientState = client.getState(); clientState.setCredentials(null, httpURL.getHost(), hostCredentials); clientState.setAuthenticationPreemptive(true); } if (proxyCredentials != null) { client.getState().setProxyCredentials(null, proxyHost, proxyCredentials); } } return client; }
From source file:org.apache.wookie.proxy.ConnectionsPrefsManager.java
public static void setProxySettings(HttpClient client, Configuration properties) { if (!fParsedFile) init(properties); // just do this once - will have to reboot for changes to take effect HostConfiguration hConf = client.getHostConfiguration(); hConf.setProxy("wwwcache.open.ac.uk", 80); //client.s// w ww . j a v a2 s.c o m if (isProxyServerSet()) { int port; try { port = properties.getInt("widget.proxy.port"); } catch (Exception ex) { port = 8080; // default for now if not specified } String username = properties.getString("widget.proxy.username"); String password = properties.getString("widget.proxy.password"); hConf = client.getHostConfiguration(); hConf.setProxy(fHostname, port); hConf.setProxy("wwwcache.open.ac.uk", 80); //AuthScope scopeProxy = new AuthScope(fHostname,port , AuthScope.ANY_REALM); // if(fUseNTLMAuthentication){ // String domain = System.getenv("USERDOMAIN"); //$NON-NLS-1$ // String computerName = System.getenv("COMPUTERNAME");//$NON-NLS-1$ // if (domain!=null && computerName!=null){ // NTCredentials userNTLMCredentials = new NTCredentials(username, password, computerName, domain); // client.getState().setProxyCredentials(scopeProxy, userNTLMCredentials); // } // else { // fLogger.error("Cannot find domain or computername for NTLM proxy authentication"); // } // } // else{ // client.getState().setProxyCredentials(scopeProxy, new UsernamePasswordCredentials(username, password)); // } } }
From source file:org.apdplat.qa.util.Tools.java
public static String getRewindEvidenceText(String question, String answer) { //1??/*from w ww .j a v a 2s . co m*/ String rewindEvidenceText = MySQLUtils.getRewindEvidenceText(question, answer); if (rewindEvidenceText != null) { //? LOG.info("?" + question + " " + answer); return rewindEvidenceText; } //2???google StringBuilder text = new StringBuilder(); String query = question + answer; try { query = URLEncoder.encode(query, "UTF-8"); } catch (UnsupportedEncodingException e) { LOG.error("url", e); return null; } query = "http://ajax.googleapis.com/ajax/services/search/web?start=0&rsz=large&v=1.0&q=" + query; try { HostConfiguration hcf = new HostConfiguration(); hcf.setProxy("127.0.0.1", 8087); HttpClient httpClient = new HttpClient(); GetMethod getMethod = new GetMethod(query); //httpClient.executeMethod(hcf, getMethod); httpClient.executeMethod(getMethod); getMethod.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler()); int statusCode = httpClient.executeMethod(getMethod); if (statusCode != HttpStatus.SC_OK) { LOG.error("Method failed: " + getMethod.getStatusLine()); } byte[] responseBody = getMethod.getResponseBody(); String response = new String(responseBody, "UTF-8"); LOG.debug("??" + response); JSONObject json = new JSONObject(response); String totalResult = json.getJSONObject("responseData").getJSONObject("cursor") .getString("estimatedResultCount"); int totalResultCount = Integer.parseInt(totalResult); LOG.info("? " + totalResultCount); JSONArray results = json.getJSONObject("responseData").getJSONArray("results"); LOG.debug(" Results:"); for (int i = 0; i < results.length(); i++) { JSONObject result = results.getJSONObject(i); String title = result.getString("titleNoFormatting"); LOG.debug(title); //URL??? String url = result.get("url").toString(); String content = null;//Tools.getHTMLContent(url); if (content == null) { //???? content = result.get("content").toString(); content = content.replaceAll("<b>", ""); content = content.replaceAll("</b>", ""); content = content.replaceAll("\\.\\.\\.", ""); } LOG.debug(content); text.append(title).append(content); } LOG.info("" + question + " " + answer + " MySQL?"); MySQLUtils.saveRewindEvidenceText(question, answer, text.toString()); return text.toString(); } catch (Exception e) { LOG.debug("?", e); } return null; }