Example usage for org.apache.http.conn.params ConnPerRouteBean setDefaultMaxPerRoute

Introduction

In this page you can find the example usage for org.apache.http.conn.params ConnPerRouteBean setDefaultMaxPerRoute.

Prototype

public void setDefaultMaxPerRoute(final int max) 

Source Link

Usage

From source file:org.apache.droids.examples.cli.SimpleRuntime.java

public static void main(String[] args) throws Exception {

    if (args.length < 1) {
        System.out.println("Please specify a URL to crawl");
        System.exit(-1);//w w w .jav a2  s. c om
    }
    String targetURL = args[0];

    // Create parser factory. Support basic HTML markup only
    ParserFactory parserFactory = new ParserFactory();
    TikaDocumentParser tikaParser = new TikaDocumentParser();
    parserFactory.getMap().put("text/html", tikaParser);

    // Create protocol factory. Support HTTP/S only.
    ProtocolFactory protocolFactory = new ProtocolFactory();

    // Create and configure HTTP client
    HttpParams params = new BasicHttpParams();
    HttpProtocolParamBean hppb = new HttpProtocolParamBean(params);
    HttpConnectionParamBean hcpb = new HttpConnectionParamBean(params);
    ConnManagerParamBean cmpb = new ConnManagerParamBean(params);

    // Set protocol parametes
    hppb.setVersion(HttpVersion.HTTP_1_1);
    hppb.setContentCharset(HTTP.ISO_8859_1);
    hppb.setUseExpectContinue(true);
    // Set connection parameters
    hcpb.setStaleCheckingEnabled(false);
    // Set connection manager parameters
    ConnPerRouteBean connPerRouteBean = new ConnPerRouteBean();
    connPerRouteBean.setDefaultMaxPerRoute(2);
    cmpb.setConnectionsPerRoute(connPerRouteBean);

    DroidsHttpClient httpclient = new DroidsHttpClient(params);

    HttpProtocol httpProtocol = new HttpProtocol(httpclient);
    protocolFactory.getMap().put("http", httpProtocol);
    protocolFactory.getMap().put("https", httpProtocol);

    // Create URL filter factory.
    URLFiltersFactory filtersFactory = new URLFiltersFactory();
    RegexURLFilter defaultURLFilter = new RegexURLFilter();
    defaultURLFilter.setFile("classpath:/regex-urlfilter.txt");
    filtersFactory.getMap().put("default", defaultURLFilter);

    // Create handler factory. Provide sysout handler only.
    HandlerFactory handlerFactory = new HandlerFactory();
    SysoutHandler defaultHandler = new SysoutHandler();
    handlerFactory.getMap().put("default", defaultHandler);

    // Create droid factory. Leave it empty for now.
    DroidFactory<Link> droidFactory = new DroidFactory<Link>();

    // Create default droid
    SimpleDelayTimer simpleDelayTimer = new SimpleDelayTimer();
    simpleDelayTimer.setDelayMillis(100);

    Queue<Link> simpleQueue = new LinkedList<Link>();

    SequentialTaskMaster<Link> taskMaster = new SequentialTaskMaster<Link>();
    taskMaster.setDelayTimer(simpleDelayTimer);
    taskMaster.setExceptionHandler(new DefaultTaskExceptionHandler());

    CrawlingDroid helloCrawler = new SysoutCrawlingDroid(simpleQueue, taskMaster);
    helloCrawler.setFiltersFactory(filtersFactory);
    helloCrawler.setParserFactory(parserFactory);
    helloCrawler.setProtocolFactory(protocolFactory);

    Collection<String> initialLocations = new ArrayList<String>();
    initialLocations.add(targetURL);
    helloCrawler.setInitialLocations(initialLocations);

    // Initialize and start the crawler
    helloCrawler.init();
    helloCrawler.start();

    // Await termination
    helloCrawler.getTaskMaster().awaitTermination(0, TimeUnit.MILLISECONDS);
    // Shut down the HTTP connection manager
    httpclient.getConnectionManager().shutdown();
}

---

From source file:eu.sisob.uma.api.crawler4j.crawler.PageFetcher.java

public synchronized static void startConnectionMonitorThread() {
    if (connectionMonitorThread == null) {
        HttpParams params = new BasicHttpParams();
        HttpProtocolParamBean paramsBean = new HttpProtocolParamBean(params);
        paramsBean.setVersion(HttpVersion.HTTP_1_1);
        paramsBean.setContentCharset("UTF-8");
        paramsBean.setUseExpectContinue(false);

        params.setParameter("http.useragent", Configurations.getStringProperty("fetcher.user_agent",
                "crawler4j (http://code.google.com/p/crawler4j/)"));

        params.setIntParameter("http.socket.timeout",
                Configurations.getIntProperty("fetcher.socket_timeout", 20000));

        params.setIntParameter("http.connection.timeout",
                Configurations.getIntProperty("fetcher.connection_timeout", 30000));

        params.setBooleanParameter("http.protocol.handle-redirects", false);

        ConnPerRouteBean connPerRouteBean = new ConnPerRouteBean();
        connPerRouteBean
                .setDefaultMaxPerRoute(Configurations.getIntProperty("fetcher.max_connections_per_host", 100));
        ConnManagerParams.setMaxConnectionsPerRoute(params, connPerRouteBean);
        ConnManagerParams.setMaxTotalConnections(params,
                Configurations.getIntProperty("fetcher.max_total_connections", 100));

        SchemeRegistry schemeRegistry = new SchemeRegistry();
        schemeRegistry.register(new Scheme("http", PlainSocketFactory.getSocketFactory(), 80));

        if (Configurations.getBooleanProperty("fetcher.crawl_https", false)) {
            schemeRegistry.register(new Scheme("https", SSLSocketFactory.getSocketFactory(), 443));
        }//w  w  w.  j  av  a 2 s  .co  m

        connectionManager = new ThreadSafeClientConnManager(params, schemeRegistry);

        ProjectLogger.LOGGER.setLevel(Level.INFO);
        httpclient = new DefaultHttpClient(connectionManager, params);
        connectionMonitorThread = new IdleConnectionMonitorThread(connectionManager);
    }
    connectionMonitorThread.start();
}

---

From source file:it.av.youeat.web.pubsubhubbub.Publisher.java

/**
 * Constructor//from ww  w.j a v  a 2 s . co m
 */
public Publisher() {
    HttpParams params = new BasicHttpParams();
    ConnManagerParams.setMaxTotalConnections(params, 200);
    ConnPerRouteBean connPerRoute = new ConnPerRouteBean(20);
    connPerRoute.setDefaultMaxPerRoute(50);
    ConnManagerParams.setMaxConnectionsPerRoute(params, connPerRoute);

    SchemeRegistry schemeRegistry = new SchemeRegistry();
    schemeRegistry.register(new Scheme("http", PlainSocketFactory.getSocketFactory(), 80));
    ClientConnectionManager cm = new ThreadSafeClientConnManager(params, schemeRegistry);
    httpClient = new DefaultHttpClient(cm, params);

    httpClient.setKeepAliveStrategy(new ConnectionKeepAliveStrategy() {

        public long getKeepAliveDuration(HttpResponse response, HttpContext context) {
            HeaderElementIterator it = new BasicHeaderElementIterator(
                    response.headerIterator(HTTP.CONN_KEEP_ALIVE));
            while (it.hasNext()) {
                HeaderElement he = it.nextElement();
                String param = he.getName();
                String value = he.getValue();
                if (value != null && param.equalsIgnoreCase("timeout")) {
                    try {
                        return Long.parseLong(value) * 1000;
                    } catch (NumberFormatException ignore) {
                    }
                }
            }
            // default keepalive is 60 seconds. This is higher than usual
            // since the number of hubs it should be talking to should be
            // small
            return 30 * 1000;
        }
    });
}

---