List of usage examples for org.apache.http.params HttpParams setIntParameter
HttpParams setIntParameter(String str, int i);
From source file:edu.hust.grid.crawl.fetcher.PageFetcher.java
public PageFetcher(CrawlConfig config) { super(config); HttpParams params = new BasicHttpParams(); HttpProtocolParamBean paramsBean = new HttpProtocolParamBean(params); paramsBean.setVersion(HttpVersion.HTTP_1_1); paramsBean.setContentCharset("UTF-8"); paramsBean.setUseExpectContinue(false); params.setParameter(CoreProtocolPNames.USER_AGENT, config.getUserAgentString()); params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT, config.getSocketTimeout()); params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, config.getConnectionTimeout()); params.setBooleanParameter("http.protocol.handle-redirects", false); params.setParameter("http.language.Accept-Language", "en-us"); params.setParameter("http.protocol.content-charset", "UTF-8"); SchemeRegistry schemeRegistry = new SchemeRegistry(); schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory())); if (config.isIncludeHttpsPages()) { schemeRegistry.register(new Scheme("https", 443, SSLSocketFactory.getSocketFactory())); }//from www . j a va 2s .c om connectionManager = new ThreadSafeClientConnManager(schemeRegistry); connectionManager.setMaxTotal(config.getMaxTotalConnections()); connectionManager.setDefaultMaxPerRoute(config.getMaxConnectionsPerHost()); httpClient = new DefaultHttpClient(connectionManager, params); if (config.getProxyHost() != null) { if (config.getProxyUsername() != null) { httpClient.getCredentialsProvider().setCredentials( new AuthScope(config.getProxyHost(), config.getProxyPort()), new UsernamePasswordCredentials(config.getProxyUsername(), config.getProxyPassword())); } HttpHost proxy = new HttpHost(config.getProxyHost(), config.getProxyPort()); httpClient.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, proxy); httpClient.getParams().setParameter(CoreProtocolPNames.USER_AGENT, "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:16.0) Gecko/20100101 Firefox/16.0"); } httpClient.addResponseInterceptor(new HttpResponseInterceptor() { @Override public void process(final HttpResponse response, final HttpContext context) throws HttpException, IOException { HttpEntity entity = response.getEntity(); Header contentEncoding = entity.getContentEncoding(); if (contentEncoding != null) { HeaderElement[] codecs = contentEncoding.getElements(); for (HeaderElement codec : codecs) { if (codec.getName().equalsIgnoreCase("gzip")) { response.setEntity(new GzipDecompressingEntity(response.getEntity())); return; } } } } }); if (connectionMonitorThread == null) { connectionMonitorThread = new IdleConnectionMonitorThread(connectionManager); } connectionMonitorThread.start(); }
From source file:se.kodapan.io.http.HttpAccessor.java
public synchronized void open() throws IOException { if (open) {//from www . ja va2 s . co m return; } if (temporaryContentFilePath == null) { temporaryContentFilePath = File.createTempFile(HttpAccessor.class.getSimpleName(), "contentFiles"); temporaryContentFilePath.delete(); } if (!temporaryContentFilePath.exists()) { if (!temporaryContentFilePath.mkdirs()) { throw new IOException("Could not create directory: " + temporaryContentFilePath.getAbsolutePath()); } } else if (!temporaryContentFilePath.isDirectory()) { throw new IOException("Not a directory: " + temporaryContentFilePath.getAbsolutePath()); } HttpParams params = new BasicHttpParams(); params.setIntParameter("http.socket.timeout", 10000); // 10 seconds params.setIntParameter("http.connection.timeout", 10000); // 10 seconds /* 'http.conn-manager.timeout': defines the timeout in milliseconds used when retrieving an instance of ManagedClientConnection from the ClientConnectionManager This parameter expects a value of type java.lang.Long. If this parameter is not set connection requests will not time out (infinite timeout). */ params.setLongParameter("http.conn-manager.timeout", 240000); // 4 minutes params.setIntParameter(ConnManagerPNames.MAX_TOTAL_CONNECTIONS, 50); params.setParameter(ConnManagerPNames.MAX_CONNECTIONS_PER_ROUTE, new ConnPerRoute() { @Override public int getMaxForRoute(HttpRoute httpRoute) { return 10; } }); SchemeRegistry registry = new SchemeRegistry(); registry.register(new Scheme("http", PlainSocketFactory.getSocketFactory(), 80)); registry.register(new Scheme("https", org.apache.http.conn.ssl.SSLSocketFactory.getSocketFactory(), 443)); ThreadSafeClientConnManager connManager = new ThreadSafeClientConnManager(params, registry); httpClient = new DefaultHttpClient(connManager, params); // // http proxy! // // String proxyHost = "localhost"; // int proxyPort = 8123; // String proxyUsername = null; // String proxyPassword = null; // // final HttpHost hcProxyHost = new HttpHost(proxyHost, proxyPort, "http"); // httpClient.getCredentialsProvider().setCredentials( // new AuthScope(proxyHost, proxyPort), // new UsernamePasswordCredentials(proxyUsername, proxyPassword)); // httpClient.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, hcProxyHost); open = true; }
From source file:com.nanocrawler.fetcher.PageFetcher.java
public PageFetcher(CrawlConfig config) { this.config = config; HttpParams params = new BasicHttpParams(); HttpProtocolParamBean paramsBean = new HttpProtocolParamBean(params); paramsBean.setVersion(HttpVersion.HTTP_1_1); paramsBean.setContentCharset("UTF-8"); paramsBean.setUseExpectContinue(false); params.setParameter(ClientPNames.COOKIE_POLICY, CookiePolicy.BROWSER_COMPATIBILITY); params.setParameter(CoreProtocolPNames.USER_AGENT, config.getUserAgentString()); params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT, config.getSocketTimeout()); params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, config.getConnectionTimeout()); params.setBooleanParameter("http.protocol.handle-redirects", false); SchemeRegistry schemeRegistry = new SchemeRegistry(); schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory())); if (config.isIncludeHttpsPages()) { schemeRegistry.register(new Scheme("https", 443, SSLSocketFactory.getSocketFactory())); }/*w w w . ja v a 2s. c om*/ connectionManager = new PoolingClientConnectionManager(schemeRegistry); connectionManager.setMaxTotal(config.getMaxTotalConnections()); connectionManager.setDefaultMaxPerRoute(config.getMaxConnectionsPerHost()); httpClient = new DefaultHttpClient(connectionManager, params); }
From source file:jetbrains.buildServer.buildTriggers.vcs.git.SNIHttpClientConnection.java
private HttpClient getClient() { if (client == null) client = new DefaultHttpClient(); HttpParams params = client.getParams(); if (proxy != null && !Proxy.NO_PROXY.equals(proxy)) { isUsingProxy = true;/* w w w. ja v a 2 s . co m*/ InetSocketAddress adr = (InetSocketAddress) proxy.address(); params.setParameter(ConnRoutePNames.DEFAULT_PROXY, new HttpHost(adr.getHostName(), adr.getPort())); } if (timeout != null) params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, timeout.intValue()); if (readTimeout != null) params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT, readTimeout.intValue()); if (followRedirects != null) params.setBooleanParameter(ClientPNames.HANDLE_REDIRECTS, followRedirects.booleanValue()); SSLSocketFactory sf = hostnameverifier != null ? new SNISSLSocketFactory(getSSLContext(), hostnameverifier) : new SNISSLSocketFactory(getSSLContext()); Scheme https = new Scheme("https", 443, sf); //$NON-NLS-1$ client.getConnectionManager().getSchemeRegistry().register(https); return client; }
From source file:edu.uci.ics.crawler4j.crawler.fetcher.PageFetcher.java
public PageFetcher(ICrawlerSettings config) { politenessDelay = config.getPolitenessDelay(); maxDownloadSize = config.getMaxDownloadSize(); show404Pages = config.getShow404Pages(); ignoreBinary = !config.getIncludeBinaryContent(); cache = config.getCacheProvider();/*from w w w.j a va 2s . c o m*/ HttpParams params = new BasicHttpParams(); HttpProtocolParamBean paramsBean = new HttpProtocolParamBean(params); paramsBean.setVersion(HttpVersion.HTTP_1_1); paramsBean.setContentCharset("UTF-8"); paramsBean.setUseExpectContinue(false); params.setParameter("http.useragent", config.getUserAgent()); params.setIntParameter("http.socket.timeout", config.getSocketTimeout()); params.setIntParameter("http.connection.timeout", config.getConnectionTimeout()); params.setBooleanParameter("http.protocol.handle-redirects", false); SchemeRegistry schemeRegistry = new SchemeRegistry(); schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory())); if (config.getAllowHttps()) { schemeRegistry.register(new Scheme("https", 443, SSLSocketFactory.getSocketFactory())); } connectionManager = new ThreadSafeClientConnManager(schemeRegistry); connectionManager.setDefaultMaxPerRoute(config.getMaxConnectionsPerHost()); connectionManager.setMaxTotal(config.getMaxTotalConnections()); logger.setLevel(Level.INFO); httpclient = new DefaultHttpClient(connectionManager, params); }
From source file:org.sbs.goodcrawler.fetcher.PageFetcher.java
public PageFetcher(FetchConfig config) { super(config); HttpParams params = new BasicHttpParams(); HttpProtocolParamBean paramsBean = new HttpProtocolParamBean(params); paramsBean.setVersion(HttpVersion.HTTP_1_1); paramsBean.setContentCharset("UTF-8"); paramsBean.setUseExpectContinue(false); params.setParameter(ClientPNames.COOKIE_POLICY, CookiePolicy.BROWSER_COMPATIBILITY); params.setParameter(CoreProtocolPNames.USER_AGENT, config.getAgent()); params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT, config.getSocketTimeoutMilliseconds()); params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, config.getConnectionTimeout()); params.setBooleanParameter("http.protocol.handle-redirects", false); SchemeRegistry schemeRegistry = new SchemeRegistry(); schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory())); if (config.isHttps()) { schemeRegistry.register(new Scheme("https", 443, SSLSocketFactory.getSocketFactory())); }/* w w w. j a v a 2 s. c om*/ connectionManager = new PoolingClientConnectionManager(schemeRegistry); connectionManager.setMaxTotal(config.getMaxTotalConnections()); connectionManager.setDefaultMaxPerRoute(config.getMaxConnectionsPerHost()); httpClient = new DefaultHttpClient(connectionManager, params); if (config.getProxyHost() != null) { if (config.getProxyUsername() != null) { httpClient.getCredentialsProvider().setCredentials( new AuthScope(config.getProxyHost(), config.getProxyPort()), new UsernamePasswordCredentials(config.getProxyUsername(), config.getProxyPassword())); } HttpHost proxy = new HttpHost(config.getProxyHost(), config.getProxyPort()); httpClient.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, proxy); } httpClient.addResponseInterceptor(new HttpResponseInterceptor() { @Override public void process(final HttpResponse response, final HttpContext context) throws HttpException, IOException { HttpEntity entity = response.getEntity(); Header contentEncoding = entity.getContentEncoding(); if (contentEncoding != null) { HeaderElement[] codecs = contentEncoding.getElements(); for (HeaderElement codec : codecs) { if (codec.getName().equalsIgnoreCase("gzip")) { response.setEntity(new GzipDecompressingEntity(response.getEntity())); return; } } } } }); if (connectionMonitorThread == null) { connectionMonitorThread = new IdleConnectionMonitorThread(connectionManager); } connectionMonitorThread.start(); }
From source file:com.autonomousturk.crawler.fetcher.PageFetcher.java
public PageFetcher(CrawlConfig config) { super(config); HttpParams params = new BasicHttpParams(); HttpProtocolParamBean paramsBean = new HttpProtocolParamBean(params); paramsBean.setVersion(HttpVersion.HTTP_1_1); paramsBean.setContentCharset("UTF-8"); paramsBean.setUseExpectContinue(false); params.setParameter(ClientPNames.COOKIE_POLICY, CookiePolicy.BROWSER_COMPATIBILITY); params.setParameter(CoreProtocolPNames.USER_AGENT, config.getUserAgentString()); params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT, config.getSocketTimeout()); params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, config.getConnectionTimeout()); params.setBooleanParameter("http.protocol.handle-redirects", false); SchemeRegistry schemeRegistry = new SchemeRegistry(); schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory())); if (config.isIncludeHttpsPages()) { schemeRegistry.register(new Scheme("https", 443, SSLSocketFactory.getSocketFactory())); }/* w w w .j av a 2 s . c om*/ connectionManager = new PoolingClientConnectionManager(schemeRegistry); connectionManager.setMaxTotal(config.getMaxTotalConnections()); connectionManager.setDefaultMaxPerRoute(config.getMaxConnectionsPerHost()); httpClient = new DefaultHttpClient(connectionManager, params); if (config.getProxyHost() != null) { if (config.getProxyUsername() != null) { httpClient.getCredentialsProvider().setCredentials( new AuthScope(config.getProxyHost(), config.getProxyPort()), new UsernamePasswordCredentials(config.getProxyUsername(), config.getProxyPassword())); } HttpHost proxy = new HttpHost(config.getProxyHost(), config.getProxyPort()); httpClient.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, proxy); } httpClient.addResponseInterceptor(new HttpResponseInterceptor() { @Override public void process(final HttpResponse response, final HttpContext context) throws HttpException, IOException { HttpEntity entity = response.getEntity(); Header contentEncoding = entity.getContentEncoding(); if (contentEncoding != null) { HeaderElement[] codecs = contentEncoding.getElements(); for (HeaderElement codec : codecs) { if (codec.getName().equalsIgnoreCase("gzip")) { response.setEntity(new GzipDecompressingEntity(response.getEntity())); return; } } } } }); if (connectionMonitorThread == null) { connectionMonitorThread = new IdleConnectionMonitorThread(connectionManager); } connectionMonitorThread.start(); }
From source file:com.googlecode.jsonrpc4j.JsonRpcHttpAsyncClient.java
private void initialize() { if (initialized.getAndSet(true)) { return;//from w w w . java 2 s.c o m } // HTTP parameters for the client final HttpParams params = new BasicHttpParams(); params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT, Integer.getInteger("com.googlecode.jsonrpc4j.async.socket.timeout", 30000)); params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, Integer.getInteger("com.googlecode.jsonrpc4j.async.connect.timeout", 30000)); params.setIntParameter(CoreConnectionPNames.SOCKET_BUFFER_SIZE, Integer.getInteger("com.googlecode.jsonrpc4j.async.socket.buffer", 8 * 1024)); params.setBooleanParameter(CoreConnectionPNames.TCP_NODELAY, Boolean.valueOf(System.getProperty("com.googlecode.jsonrpc4j.async.tcp.nodelay", "true"))); params.setParameter(CoreProtocolPNames.USER_AGENT, "jsonrpc4j/1.0"); // Create client-side I/O reactor final ConnectingIOReactor ioReactor; try { IOReactorConfig config = new IOReactorConfig(); config.setIoThreadCount(Integer.getInteger("com.googlecode.jsonrpc4j.async.reactor.threads", 1)); ioReactor = new DefaultConnectingIOReactor(config); } catch (IOReactorException e) { throw new RuntimeException("Exception initializing asynchronous Apache HTTP Client", e); } // Create a default SSLSetupHandler that accepts any certificate if (sslContext == null) { try { sslContext = SSLContext.getDefault(); } catch (Exception e) { throw new RuntimeException(e); } } // Create HTTP connection pool BasicNIOConnFactory nioConnFactory = new BasicNIOConnFactory(sslContext, null, params); pool = new BasicNIOConnPool(ioReactor, nioConnFactory, params); // Limit total number of connections to 500 by default pool.setDefaultMaxPerRoute(Integer.getInteger("com.googlecode.jsonrpc4j.async.max.inflight.route", 500)); pool.setMaxTotal(Integer.getInteger("com.googlecode.jsonrpc4j.async.max.inflight.total", 500)); // Run the I/O reactor in a separate thread Thread t = new Thread(new Runnable() { public void run() { try { // Create client-side HTTP protocol handler HttpAsyncRequestExecutor protocolHandler = new HttpAsyncRequestExecutor(); // Create client-side I/O event dispatch IOEventDispatch ioEventDispatch = new DefaultHttpClientIODispatch(protocolHandler, sslContext, params); // Ready to go! ioReactor.execute(ioEventDispatch); } catch (InterruptedIOException ex) { System.err.println("Interrupted"); } catch (IOException e) { System.err.println("I/O error: " + e.getMessage()); } } }, "jsonrpc4j HTTP IOReactor"); // Start the client thread t.setDaemon(true); t.start(); // Create HTTP protocol processing chain HttpProcessor httpproc = new ImmutableHttpProcessor(new HttpRequestInterceptor[] { // Use standard client-side protocol interceptors new RequestContent(), new RequestTargetHost(), new RequestConnControl(), new RequestUserAgent(), new RequestExpectContinue() }); // Create HTTP requester requester = new HttpAsyncRequester(httpproc, new DefaultConnectionReuseStrategy(), params); }
From source file:org.opensaml.util.http.HttpClientBuilder.java
/** * Constructs an {@link HttpClient} using the settings of this builder. * //www. jav a 2s . c o m * @return the constructed client */ public HttpClient buildClient() { final DefaultHttpClient client = new DefaultHttpClient(buildConnectionManager()); client.addRequestInterceptor(new RequestAcceptEncoding()); client.addResponseInterceptor(new ResponseContentEncoding()); final HttpParams httpParams = client.getParams(); if (socketLocalAddress != null) { httpParams.setParameter(AllClientPNames.LOCAL_ADDRESS, socketLocalAddress); } if (socketTimeout > 0) { httpParams.setIntParameter(AllClientPNames.SO_TIMEOUT, socketTimeout); } httpParams.setIntParameter(AllClientPNames.SOCKET_BUFFER_SIZE, socketBufferSize); if (connectionTimeout > 0) { httpParams.setIntParameter(AllClientPNames.CONNECTION_TIMEOUT, connectionTimeout); } httpParams.setBooleanParameter(AllClientPNames.STALE_CONNECTION_CHECK, connectionStalecheck); if (connectionProxyHost != null) { final HttpHost proxyHost = new HttpHost(connectionProxyHost, connectionProxyPort); httpParams.setParameter(AllClientPNames.DEFAULT_PROXY, proxyHost); if (connectionProxyUsername != null && connectionProxyPassword != null) { final CredentialsProvider credProvider = client.getCredentialsProvider(); credProvider.setCredentials(new AuthScope(connectionProxyHost, connectionProxyPort), new UsernamePasswordCredentials(connectionProxyUsername, connectionProxyPassword)); } } httpParams.setBooleanParameter(AllClientPNames.HANDLE_REDIRECTS, httpFollowRedirects); httpParams.setParameter(AllClientPNames.HTTP_CONTENT_CHARSET, httpContentCharSet); return client; }
From source file:org.sbs.goodcrawler.fetcher.Fetcher.java
public void init(File fetchConfFile) { FetchConfig fetchConfig = new FetchConfig(); Document document;//from ww w.jav a2s. c o m try { document = Jsoup.parse(fetchConfFile, "utf-8"); Fetcher.config = fetchConfig.loadConfig(document); } catch (IOException e) { e.printStackTrace(); } catch (ConfigurationException e) { e.printStackTrace(); } HttpParams params = new BasicHttpParams(); HttpProtocolParamBean paramsBean = new HttpProtocolParamBean(params); paramsBean.setVersion(HttpVersion.HTTP_1_1); paramsBean.setContentCharset("UTF-8"); paramsBean.setUseExpectContinue(false); params.setParameter(ClientPNames.COOKIE_POLICY, CookiePolicy.BROWSER_COMPATIBILITY); params.setParameter(CoreProtocolPNames.USER_AGENT, config.getAgent()); params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT, config.getSocketTimeoutMilliseconds()); params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, config.getConnectionTimeout()); params.setBooleanParameter("http.protocol.handle-redirects", false); SchemeRegistry schemeRegistry = new SchemeRegistry(); schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory())); if (config.isHttps()) { schemeRegistry.register(new Scheme("https", 443, SSLSocketFactory.getSocketFactory())); } connectionManager = new PoolingClientConnectionManager(schemeRegistry); connectionManager.setMaxTotal(config.getMaxTotalConnections()); connectionManager.setDefaultMaxPerRoute(config.getMaxConnectionsPerHost()); httpClient = new DefaultHttpClient(connectionManager, params); if (config.getProxyHost() != null) { if (config.getProxyUsername() != null) { httpClient.getCredentialsProvider().setCredentials( new AuthScope(config.getProxyHost(), config.getProxyPort()), new UsernamePasswordCredentials(config.getProxyUsername(), config.getProxyPassword())); } HttpHost proxy = new HttpHost(config.getProxyHost(), config.getProxyPort()); httpClient.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, proxy); } httpClient.addResponseInterceptor(new HttpResponseInterceptor() { @Override public void process(final HttpResponse response, final HttpContext context) throws HttpException, IOException { HttpEntity entity = response.getEntity(); Header contentEncoding = entity.getContentEncoding(); if (contentEncoding != null) { HeaderElement[] codecs = contentEncoding.getElements(); for (HeaderElement codec : codecs) { if (codec.getName().equalsIgnoreCase("gzip")) { response.setEntity(new GzipDecompressingEntity(response.getEntity())); return; } } } } }); if (connectionMonitorThread == null) { connectionMonitorThread = new IdleConnectionMonitorThread(connectionManager); } connectionMonitorThread.start(); }