Example usage for org.apache.http.params HttpParams setIntParameter

Introduction

In this page you can find the example usage for org.apache.http.params HttpParams setIntParameter.

Prototype

HttpParams setIntParameter(String str, int i);

Source Link

Usage

From source file:com.tiaoin.crawl.plugin.util.PageFetcherImpl.java

/**
 * client?Header?Cookie//  ww  w. ja v a 2 s  .  c o m
 * @param aconfig
 * @param cookies
 */
public void init(Site site) {
    //System.out.println(site.toString());
    if (null != site.getHeaders() && site.getHeaders().getHeader() != null) {
        for (com.tiaoin.crawl.core.xml.Header header : site.getHeaders().getHeader()) {
            this.addHeader(header.getName(), header.getValue());
        }
    }
    if (null != site.getCookies() && site.getCookies().getCookie() != null) {
        for (com.tiaoin.crawl.core.xml.Cookie cookie : site.getCookies().getCookie()) {
            this.addCookie(cookie.getName(), cookie.getValue(), cookie.getHost(), cookie.getPath());
        }
    }

    //HTTP?
    HttpParams params = new BasicHttpParams();
    params.setParameter(CoreProtocolPNames.USER_AGENT, config.getUserAgentString());
    params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT, config.getSocketTimeout());
    params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, config.getConnectionTimeout());

    HttpProtocolParamBean paramsBean = new HttpProtocolParamBean(params);
    paramsBean.setVersion(HttpVersion.HTTP_1_1);
    paramsBean.setContentCharset("UTF-8");
    paramsBean.setUseExpectContinue(false);

    SchemeRegistry schemeRegistry = new SchemeRegistry();
    schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory()));

    if (config.isIncludeHttpsPages())
        schemeRegistry.register(new Scheme("https", 443, SSLSocketFactory.getSocketFactory()));

    connectionManager = new ThreadSafeClientConnManager(schemeRegistry);
    connectionManager.setMaxTotal(config.getMaxTotalConnections());
    connectionManager.setDefaultMaxPerRoute(config.getMaxConnectionsPerHost());

    httpClient = new DefaultHttpClient(connectionManager, params);
    httpClient.getParams().setIntParameter("http.socket.timeout", 15000);
    httpClient.getParams().setParameter(ClientPNames.COOKIE_POLICY, CookiePolicy.BEST_MATCH);

    //?
    httpClient.addResponseInterceptor(new HttpResponseInterceptor() {
        public void process(final HttpResponse response, final HttpContext context)
                throws HttpException, IOException {
            HttpEntity entity = response.getEntity();
            Header contentEncoding = entity.getContentEncoding();
            if (contentEncoding != null) {
                HeaderElement[] codecs = contentEncoding.getElements();
                for (HeaderElement codec : codecs) {
                    //?GZIP
                    if (codec.getName().equalsIgnoreCase("gzip")) {
                        response.setEntity(new GzipDecompressingEntity(response.getEntity()));
                        return;
                    }
                }
            }
        }
    });
}

---

From source file:org.apache.droids.protocol.http.DroidsHttpClient.java

@Override
protected HttpParams createHttpParams() {
    HttpParams params = new BasicHttpParams();
    params.setParameter(CoreProtocolPNames.PROTOCOL_VERSION, HttpVersion.HTTP_1_1);
    params.setParameter(CoreProtocolPNames.HTTP_CONTENT_CHARSET, HTTP.DEFAULT_CONTENT_CHARSET);
    params.setParameter(CoreProtocolPNames.USE_EXPECT_CONTINUE, false);
    params.setParameter(CoreConnectionPNames.STALE_CONNECTION_CHECK, false);
    params.setIntParameter(CoreConnectionPNames.MAX_HEADER_COUNT, 256);
    params.setIntParameter(CoreConnectionPNames.MAX_LINE_LENGTH, 5 * 1024);
    params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT, 20000);
    params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, 20000);
    params.setParameter(CoreConnectionPNames.TCP_NODELAY, false);
    //params.setLongParameter(MAX_BODY_LENGTH, 512 * 1024);
    return params;
}

---

From source file:com.senseidb.dataprovider.http.HttpStreamDataProvider.java

public HttpStreamDataProvider(Comparator<String> versionComparator, String baseUrl, String pw, int fetchSize,
        String startingOffset, boolean disableHttps) {
    super(versionComparator);
    _baseUrl = baseUrl;/* www .j  a v a 2 s  . co  m*/
    _password = pw;
    _fetchSize = fetchSize;
    _offset = startingOffset;
    _disableHttps = disableHttps;
    _initialOffset = null;
    _currentDataIter = null;
    _stopped = true;

    _httpGetLatency = 0L;
    _responseParseLatency = 0L;

    Scheme http = new Scheme("http", 80, PlainSocketFactory.getSocketFactory());
    SchemeRegistry sr = new SchemeRegistry();
    sr.register(http);

    HttpParams params = new BasicHttpParams();
    params.setParameter(HttpProtocolParams.PROTOCOL_VERSION, HttpVersion.HTTP_1_1);
    params.setParameter(HttpProtocolParams.HTTP_CONTENT_CHARSET, "UTF-8");
    params.setIntParameter(HttpConnectionParams.CONNECTION_TIMEOUT, 5000); // 5s conn timeout
    params.setIntParameter(HttpConnectionParams.SO_LINGER, 0); //  no socket linger
    params.setBooleanParameter(HttpConnectionParams.TCP_NODELAY, true); // tcp no delay
    params.setIntParameter(HttpConnectionParams.SO_TIMEOUT, 5000); // 5s sock timeout
    params.setIntParameter(HttpConnectionParams.SOCKET_BUFFER_SIZE, 1024 * 1024); // 1mb socket buffer
    params.setBooleanParameter(HttpConnectionParams.SO_REUSEADDR, true); // 5s sock timeout

    _httpClientManager = new SingleClientConnManager(sr);
    _httpclient = new DefaultHttpClient(_httpClientManager, params);

    if (!_disableHttps) {
        _httpclient = HttpsClientDecorator.decorate(_httpclient);
    }

    _httpclient.addRequestInterceptor(new HttpRequestInterceptor() {
        public void process(final HttpRequest request, final HttpContext context)
                throws HttpException, IOException {
            if (!request.containsHeader("Accept-Encoding")) {
                request.addHeader("Accept-Encoding", "gzip");
            }
        }
    });

    _httpclient.addResponseInterceptor(new HttpResponseInterceptor() {
        public void process(final HttpResponse response, final HttpContext context)
                throws HttpException, IOException {
            HttpEntity entity = response.getEntity();
            Header ceheader = entity.getContentEncoding();
            if (ceheader != null) {
                HeaderElement[] codecs = ceheader.getElements();
                for (int i = 0; i < codecs.length; i++) {
                    if (codecs[i].getName().equalsIgnoreCase("gzip")) {
                        response.setEntity(new GzipDecompressingEntity(response.getEntity()));
                        return;
                    }
                }
            }
        }
    });

    _retryTime = DEFAULT_RETRYTIME_MS; // default retry after 5 seconds
}

---

From source file:net.oauth.client.httpclient4.HttpClient4.java

public HttpResponseMessage execute(HttpMessage request, Map<String, Object> parameters) throws IOException {
    final String method = request.method;
    final String url = request.url.toExternalForm();
    final InputStream body = request.getBody();
    final boolean isDelete = DELETE.equalsIgnoreCase(method);
    final boolean isPost = POST.equalsIgnoreCase(method);
    final boolean isPut = PUT.equalsIgnoreCase(method);
    byte[] excerpt = null;
    HttpRequestBase httpRequest;//from   w  ww.  j a  v a  2  s . com
    if (isPost || isPut) {
        HttpEntityEnclosingRequestBase entityEnclosingMethod = isPost ? new HttpPost(url) : new HttpPut(url);
        if (body != null) {
            ExcerptInputStream e = new ExcerptInputStream(body);
            excerpt = e.getExcerpt();
            String length = request.removeHeaders(HttpMessage.CONTENT_LENGTH);
            entityEnclosingMethod
                    .setEntity(new InputStreamEntity(e, (length == null) ? -1 : Long.parseLong(length)));
        }
        httpRequest = entityEnclosingMethod;
    } else if (isDelete) {
        httpRequest = new HttpDelete(url);
    } else {
        httpRequest = new HttpGet(url);
    }
    for (Map.Entry<String, String> header : request.headers) {
        httpRequest.addHeader(header.getKey(), header.getValue());
    }
    HttpParams params = httpRequest.getParams();
    for (Map.Entry<String, Object> p : parameters.entrySet()) {
        String name = p.getKey();
        String value = p.getValue().toString();
        if (FOLLOW_REDIRECTS.equals(name)) {
            params.setBooleanParameter(ClientPNames.HANDLE_REDIRECTS, Boolean.parseBoolean(value));
        } else if (READ_TIMEOUT.equals(name)) {
            params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT, Integer.parseInt(value));
        } else if (CONNECT_TIMEOUT.equals(name)) {
            params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, Integer.parseInt(value));
        }
    }
    HttpClient client = clientPool.getHttpClient(new URL(httpRequest.getURI().toString()));
    HttpResponse httpResponse = client.execute(httpRequest);
    return new HttpMethodResponse(httpRequest, httpResponse, excerpt, request.getContentCharset());
}

---

From source file:com.offbytwo.jenkins.client.JenkinsHttpClient.java

/**
 * Create an unauthenticated Jenkins HTTP client
 *
 * @param uri/*  ww w. ja  v  a  2s  . co  m*/
 *            Location of the jenkins server (ex. http://localhost:8080)
 */
public JenkinsHttpClient(URI uri) {
    this(uri, HttpClientBuilder.create());
    this.context = uri.getPath();

    if (!context.endsWith("/")) {
        context += "/";
    }
    this.uri = uri;
    this.mapper = getDefaultMapper();

    HttpParams httpParams = new BasicHttpParams();
    httpParams.setIntParameter(CoreConnectionPNames.SO_TIMEOUT, SO_TIMEOUT_IN_MILLISECONDS);
    httpParams.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, CONNECTION_TIMEOUT_IN_MILLISECONDS);

    this.httpResponseValidator = new HttpResponseValidator();
    LOGGER.debug("uri={}", uri.toString());
}

---

From source file:org.apache.marmotta.ldclient.services.ldclient.LDClient.java

public LDClient(ClientConfiguration config) {
    log.info("Initialising Linked Data Client Service ...");

    this.config = config;

    endpoints = new ArrayList<>();
    for (Endpoint endpoint : defaultEndpoints) {
        endpoints.add(endpoint);//  w w  w . j  a  v  a2  s  . c o m
    }
    endpoints.addAll(config.getEndpoints());

    Collections.sort(endpoints);
    if (log.isInfoEnabled()) {
        for (Endpoint endpoint : endpoints) {
            log.info("- LDClient Endpoint: {}", endpoint.getName());
        }
    }

    providers = new ArrayList<>();
    for (DataProvider provider : defaultProviders) {
        providers.add(provider);
    }
    providers.addAll(config.getProviders());
    if (log.isInfoEnabled()) {
        for (DataProvider provider : providers) {
            log.info("- LDClient Provider: {}", provider.getName());
        }
    }

    retrievalSemaphore = new Semaphore(config.getMaxParallelRequests());

    if (config.getHttpClient() != null) {
        log.debug("Using HttpClient provided in the configuration");
        this.client = config.getHttpClient();
    } else {
        log.debug("Creating default HttpClient based on the configuration");

        HttpParams httpParams = new BasicHttpParams();
        httpParams.setParameter(CoreProtocolPNames.USER_AGENT, "Apache Marmotta LDClient");

        httpParams.setIntParameter(CoreConnectionPNames.SO_TIMEOUT, config.getSocketTimeout());
        httpParams.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, config.getConnectionTimeout());

        httpParams.setBooleanParameter(ClientPNames.HANDLE_REDIRECTS, true);
        httpParams.setIntParameter(ClientPNames.MAX_REDIRECTS, 3);

        SchemeRegistry schemeRegistry = new SchemeRegistry();
        schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory()));

        try {
            SSLContext sslcontext = SSLContext.getInstance("TLS");
            sslcontext.init(null, null, null);
            SSLSocketFactory sf = new SSLSocketFactory(sslcontext, SSLSocketFactory.STRICT_HOSTNAME_VERIFIER);

            schemeRegistry.register(new Scheme("https", 443, sf));
        } catch (NoSuchAlgorithmException e) {
            e.printStackTrace();
        } catch (KeyManagementException e) {
            e.printStackTrace();
        }

        PoolingClientConnectionManager cm = new PoolingClientConnectionManager(schemeRegistry);
        cm.setMaxTotal(20);
        cm.setDefaultMaxPerRoute(10);

        DefaultHttpClient client = new DefaultHttpClient(cm, httpParams);
        client.setRedirectStrategy(new LMFRedirectStrategy());
        client.setHttpRequestRetryHandler(new LMFHttpRequestRetryHandler());
        idleConnectionMonitorThread = new IdleConnectionMonitorThread(client.getConnectionManager());
        idleConnectionMonitorThread.start();

        this.client = client;
    }
}

---

From source file:cn.jachohx.crawler.fetcher.PageFetcher.java

public PageFetcher(CrawlConfig config) {
    super(config);
    HttpParams params = new BasicHttpParams();
    HttpProtocolParamBean paramsBean = new HttpProtocolParamBean(params);
    paramsBean.setVersion(HttpVersion.HTTP_1_1);
    paramsBean.setContentCharset("UTF-8");
    paramsBean.setUseExpectContinue(false);

    params.setParameter(CoreProtocolPNames.USER_AGENT, config.getUserAgentString());
    params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT, config.getSocketTimeout());
    params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, config.getConnectionTimeout());

    params.setBooleanParameter("http.protocol.handle-redirects", false);

    SchemeRegistry schemeRegistry = new SchemeRegistry();
    schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory()));

    if (config.isIncludeHttpsPages()) {
        schemeRegistry.register(new Scheme("https", 443, SSLSocketFactory.getSocketFactory()));
    }// w  w w  . jav  a  2 s  .  co  m

    connectionManager = new ThreadSafeClientConnManager(schemeRegistry);
    connectionManager.setMaxTotal(config.getMaxTotalConnections());
    connectionManager.setDefaultMaxPerRoute(config.getMaxConnectionsPerHost());
    httpClient = new DefaultHttpClient(connectionManager, params);

    if (config.getProxyHost() != null) {

        if (config.getProxyUsername() != null) {
            httpClient.getCredentialsProvider().setCredentials(
                    new AuthScope(config.getProxyHost(), config.getProxyPort()),
                    new UsernamePasswordCredentials(config.getProxyUsername(), config.getProxyPassword()));
        }

        HttpHost proxy = new HttpHost(config.getProxyHost(), config.getProxyPort());
        httpClient.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, proxy);
    }

    httpClient.addResponseInterceptor(new HttpResponseInterceptor() {
        public void process(final HttpResponse response, final HttpContext context)
                throws HttpException, IOException {
            HttpEntity entity = response.getEntity();
            Header contentEncoding = entity.getContentEncoding();
            if (contentEncoding != null) {
                HeaderElement[] codecs = contentEncoding.getElements();
                for (HeaderElement codec : codecs) {
                    if (codec.getName().equalsIgnoreCase("gzip")) {
                        response.setEntity(new GzipDecompressingEntity(response.getEntity()));
                        return;
                    }
                }
            }
        }

    });

}

---

From source file:yin.autoflowcontrol.fetcher.PageFetcher.java

public PageFetcher(CrawlConfig config) {
    super(config);

    HttpParams params = new BasicHttpParams();
    HttpProtocolParamBean paramsBean = new HttpProtocolParamBean(params);
    paramsBean.setVersion(HttpVersion.HTTP_1_1);
    paramsBean.setContentCharset("UTF-8");
    paramsBean.setUseExpectContinue(false);

    params.setParameter(CoreProtocolPNames.USER_AGENT, config.getUserAgentString());
    params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT, config.getSocketTimeout());
    params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, config.getConnectionTimeout());

    params.setBooleanParameter("http.protocol.handle-redirects", false);

    SchemeRegistry schemeRegistry = new SchemeRegistry();
    schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory()));

    if (config.isIncludeHttpsPages()) {
        schemeRegistry.register(new Scheme("https", 443, SSLSocketFactory.getSocketFactory()));
    }/*  w  w  w  . j a  va 2s  .  com*/

    connectionManager = new ThreadSafeClientConnManager(schemeRegistry);
    connectionManager.setMaxTotal(config.getMaxTotalConnections());
    connectionManager.setDefaultMaxPerRoute(config.getMaxConnectionsPerHost());
    httpClient = new DefaultHttpClient(connectionManager, params);

    if (config.getProxyHost() != null) {

        if (config.getProxyUsername() != null) {
            httpClient.getCredentialsProvider().setCredentials(
                    new AuthScope(config.getProxyHost(), config.getProxyPort()),
                    new UsernamePasswordCredentials(config.getProxyUsername(), config.getProxyPassword()));
        }

        HttpHost proxy = new HttpHost(config.getProxyHost(), config.getProxyPort());
        httpClient.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, proxy);
    }

    httpClient.addResponseInterceptor(new HttpResponseInterceptor() {

        @Override
        public void process(final HttpResponse response, final HttpContext context)
                throws HttpException, IOException {
            HttpEntity entity = response.getEntity();
            Header contentEncoding = entity.getContentEncoding();
            if (contentEncoding != null) {
                HeaderElement[] codecs = contentEncoding.getElements();
                for (HeaderElement codec : codecs) {
                    if (codec.getName().equalsIgnoreCase("gzip")) {
                        response.setEntity(new GzipDecompressingEntity(response.getEntity()));
                        return;
                    }
                }
            }
        }

    });

    if (connectionMonitorThread == null) {
        connectionMonitorThread = new IdleConnectionMonitorThread(connectionManager);
    }
    connectionMonitorThread.start();

}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• »