Example usage for org.apache.http.conn.params ConnPerRouteBean ConnPerRouteBean

Introduction

In this page you can find the example usage for org.apache.http.conn.params ConnPerRouteBean ConnPerRouteBean.

Prototype

public ConnPerRouteBean() 

Source Link

Usage

From source file:org.apache.droids.examples.cli.SimpleRuntime.java

public static void main(String[] args) throws Exception {

    if (args.length < 1) {
        System.out.println("Please specify a URL to crawl");
        System.exit(-1);//from   w w w .j a  v  a 2  s .c o m
    }
    String targetURL = args[0];

    // Create parser factory. Support basic HTML markup only
    ParserFactory parserFactory = new ParserFactory();
    TikaDocumentParser tikaParser = new TikaDocumentParser();
    parserFactory.getMap().put("text/html", tikaParser);

    // Create protocol factory. Support HTTP/S only.
    ProtocolFactory protocolFactory = new ProtocolFactory();

    // Create and configure HTTP client
    HttpParams params = new BasicHttpParams();
    HttpProtocolParamBean hppb = new HttpProtocolParamBean(params);
    HttpConnectionParamBean hcpb = new HttpConnectionParamBean(params);
    ConnManagerParamBean cmpb = new ConnManagerParamBean(params);

    // Set protocol parametes
    hppb.setVersion(HttpVersion.HTTP_1_1);
    hppb.setContentCharset(HTTP.ISO_8859_1);
    hppb.setUseExpectContinue(true);
    // Set connection parameters
    hcpb.setStaleCheckingEnabled(false);
    // Set connection manager parameters
    ConnPerRouteBean connPerRouteBean = new ConnPerRouteBean();
    connPerRouteBean.setDefaultMaxPerRoute(2);
    cmpb.setConnectionsPerRoute(connPerRouteBean);

    DroidsHttpClient httpclient = new DroidsHttpClient(params);

    HttpProtocol httpProtocol = new HttpProtocol(httpclient);
    protocolFactory.getMap().put("http", httpProtocol);
    protocolFactory.getMap().put("https", httpProtocol);

    // Create URL filter factory.
    URLFiltersFactory filtersFactory = new URLFiltersFactory();
    RegexURLFilter defaultURLFilter = new RegexURLFilter();
    defaultURLFilter.setFile("classpath:/regex-urlfilter.txt");
    filtersFactory.getMap().put("default", defaultURLFilter);

    // Create handler factory. Provide sysout handler only.
    HandlerFactory handlerFactory = new HandlerFactory();
    SysoutHandler defaultHandler = new SysoutHandler();
    handlerFactory.getMap().put("default", defaultHandler);

    // Create droid factory. Leave it empty for now.
    DroidFactory<Link> droidFactory = new DroidFactory<Link>();

    // Create default droid
    SimpleDelayTimer simpleDelayTimer = new SimpleDelayTimer();
    simpleDelayTimer.setDelayMillis(100);

    Queue<Link> simpleQueue = new LinkedList<Link>();

    SequentialTaskMaster<Link> taskMaster = new SequentialTaskMaster<Link>();
    taskMaster.setDelayTimer(simpleDelayTimer);
    taskMaster.setExceptionHandler(new DefaultTaskExceptionHandler());

    CrawlingDroid helloCrawler = new SysoutCrawlingDroid(simpleQueue, taskMaster);
    helloCrawler.setFiltersFactory(filtersFactory);
    helloCrawler.setParserFactory(parserFactory);
    helloCrawler.setProtocolFactory(protocolFactory);

    Collection<String> initialLocations = new ArrayList<String>();
    initialLocations.add(targetURL);
    helloCrawler.setInitialLocations(initialLocations);

    // Initialize and start the crawler
    helloCrawler.init();
    helloCrawler.start();

    // Await termination
    helloCrawler.getTaskMaster().awaitTermination(0, TimeUnit.MILLISECONDS);
    // Shut down the HTTP connection manager
    httpclient.getConnectionManager().shutdown();
}

---

From source file:eu.sisob.uma.api.crawler4j.crawler.PageFetcher.java

public synchronized static void startConnectionMonitorThread() {
    if (connectionMonitorThread == null) {
        HttpParams params = new BasicHttpParams();
        HttpProtocolParamBean paramsBean = new HttpProtocolParamBean(params);
        paramsBean.setVersion(HttpVersion.HTTP_1_1);
        paramsBean.setContentCharset("UTF-8");
        paramsBean.setUseExpectContinue(false);

        params.setParameter("http.useragent", Configurations.getStringProperty("fetcher.user_agent",
                "crawler4j (http://code.google.com/p/crawler4j/)"));

        params.setIntParameter("http.socket.timeout",
                Configurations.getIntProperty("fetcher.socket_timeout", 20000));

        params.setIntParameter("http.connection.timeout",
                Configurations.getIntProperty("fetcher.connection_timeout", 30000));

        params.setBooleanParameter("http.protocol.handle-redirects", false);

        ConnPerRouteBean connPerRouteBean = new ConnPerRouteBean();
        connPerRouteBean//www  .j  ava  2s .  c  om
                .setDefaultMaxPerRoute(Configurations.getIntProperty("fetcher.max_connections_per_host", 100));
        ConnManagerParams.setMaxConnectionsPerRoute(params, connPerRouteBean);
        ConnManagerParams.setMaxTotalConnections(params,
                Configurations.getIntProperty("fetcher.max_total_connections", 100));

        SchemeRegistry schemeRegistry = new SchemeRegistry();
        schemeRegistry.register(new Scheme("http", PlainSocketFactory.getSocketFactory(), 80));

        if (Configurations.getBooleanProperty("fetcher.crawl_https", false)) {
            schemeRegistry.register(new Scheme("https", SSLSocketFactory.getSocketFactory(), 443));
        }

        connectionManager = new ThreadSafeClientConnManager(params, schemeRegistry);

        ProjectLogger.LOGGER.setLevel(Level.INFO);
        httpclient = new DefaultHttpClient(connectionManager, params);
        connectionMonitorThread = new IdleConnectionMonitorThread(connectionManager);
    }
    connectionMonitorThread.start();
}

---

From source file:org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager.java

/**
 * Creates a new thread safe connection manager.
 *
 * @param schreg    the scheme registry.
 * @param connTTL   max connection lifetime, <=0 implies "infinity"
 * @param connTTLTimeUnit   TimeUnit of connTTL
 *
 * @since 4.1// w w w . j  a v  a  2  s.co m
 */
public ThreadSafeClientConnManager(final SchemeRegistry schreg, final long connTTL,
        final TimeUnit connTTLTimeUnit) {
    this(schreg, connTTL, connTTLTimeUnit, new ConnPerRouteBean());
}

---

From source file:org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager.java

/**
 * Creates a new thread safe connection manager.
 *
 * @param params    the parameters for this manager.
 * @param schreg    the scheme registry.
 *
 * @deprecated (4.1)  use {@link ThreadSafeClientConnManager#ThreadSafeClientConnManager(SchemeRegistry)}
 *///from   ww  w .  jav a 2s  .  co m
@Deprecated
public ThreadSafeClientConnManager(final HttpParams params, final SchemeRegistry schreg) {
    Args.notNull(schreg, "Scheme registry");
    this.log = LogFactory.getLog(getClass());
    this.schemeRegistry = schreg;
    this.connPerRoute = new ConnPerRouteBean();
    this.connOperator = createConnectionOperator(schreg);
    this.pool = (ConnPoolByRoute) createConnectionPool(params);
    this.connectionPool = this.pool;
}

---