List of usage examples for org.apache.http.impl.cookie BrowserCompatSpecFactory BrowserCompatSpecFactory
public BrowserCompatSpecFactory()
From source file:org.aliuge.crawler.fetcher.DefaultFetcher.java
public DefaultFetcher createFetcher(FetchConfig config) { // /*from w w w.j ava 2s . c o m*/ connectionManager = new PoolingHttpClientConnectionManager(); BasicCookieStore cookieStore = new BasicCookieStore(); CookieSpecProvider easySpecProvider = new CookieSpecProvider() { public CookieSpec create(HttpContext context) { return new BrowserCompatSpec() { @Override public void validate(Cookie cookie, CookieOrigin origin) throws MalformedCookieException { // Oh, I am easy } }; } }; Registry<CookieSpecProvider> r = RegistryBuilder.<CookieSpecProvider>create() .register(CookieSpecs.BEST_MATCH, new BestMatchSpecFactory()) .register(CookieSpecs.BROWSER_COMPATIBILITY, new BrowserCompatSpecFactory()) .register("easy", easySpecProvider).build(); // Create global request configuration defaultRequestConfig = RequestConfig.custom().setCookieSpec("easy").setSocketTimeout(10000) .setConnectTimeout(10000).build(); connectionManager.setMaxTotal(config.getMaxTotalConnections()); connectionManager.setDefaultMaxPerRoute(config.getMaxConnectionsPerHost()); // Create an HttpClient with the given custom dependencies and // configuration. httpClient = HttpClients.custom().setConnectionManager(connectionManager).setDefaultCookieStore(cookieStore) .setDefaultCookieSpecRegistry(r) /* .setProxy(new HttpHost("myproxy", 8080)) */ .setDefaultRequestConfig(defaultRequestConfig).build(); if (connectionMonitorThread == null) { connectionMonitorThread = new IdleConnectionMonitorThread(connectionManager); } /* * connectionMonitorThread.start(); try { * connectionMonitorThread.join(); } catch (InterruptedException e) { // * TODO Auto-generated catch block e.printStackTrace(); } */ return this; }
From source file:com.google.acre.appengine.script.AppEngineAsyncUrlfetch.java
private Scriptable callback_result(AsyncRequest req, HTTPResponse res) { long waiting_time = System.currentTimeMillis() - req.start_time; URL furl = res.getFinalUrl(); if (furl == null) { furl = req.url;/*from w ww . jav a2 s.c om*/ } BrowserCompatSpecFactory bcsf = new BrowserCompatSpecFactory(); CookieSpec cspec = bcsf.newInstance(null); String protocol = furl.getProtocol(); boolean issecure = ("https".equals(protocol)); int port = furl.getPort(); if (port == -1) port = 80; CookieOrigin origin = new CookieOrigin(furl.getHost(), port, furl.getPath(), issecure); Context ctx = Context.getCurrentContext(); Scriptable out = ctx.newObject(_scope); Scriptable headers = ctx.newObject(_scope); Scriptable cookies = ctx.newObject(_scope); out.put("status", out, res.getResponseCode()); String response_body = null; try { response_body = new String(res.getContent(), getResponseEncoding(res)); out.put("body", out, response_body); } catch (java.io.UnsupportedEncodingException e) { throw new RuntimeException(e); } out.put("headers", out, headers); out.put("cookies", out, cookies); StringBuilder response_header_log = new StringBuilder(); for (HTTPHeader h : res.getHeaders()) { if (h.getName().equalsIgnoreCase("set-cookie")) { String set_cookie = h.getValue(); Matcher m = Pattern.compile("\\s*(([^,]|(,\\s*\\d))+)").matcher(set_cookie); while (m.find()) { Header ch = new BasicHeader("Set-Cookie", set_cookie.substring(m.start(), m.end())); try { List<Cookie> pcookies = cspec.parse(ch, origin); for (Cookie c : pcookies) { cookies.put(c.getName(), cookies, new AcreCookie(c).toJsObject(_scope)); } } catch (MalformedCookieException e) { // we've occasionally choked on cookie-set, // e.g. www.google.com returning expires=; expires=Mon, 01-Jan-1990 00:00:00 GMT; // no solution but at least log exactly what's happening. String cookiestring = ch.toString(); _logger.warn("urlfetch.response.async", "exception thrown on bad cookie " + cookiestring); throw new RuntimeException(e); } } } headers.put(h.getName(), headers, h.getValue()); response_header_log.append(h.getName() + ": " + h.getValue() + ", "); } boolean system = req.system; boolean log_to_user = req.log_to_user; String log_body = new String(); if (res.getResponseCode() != 200 && response_body != null) { log_body = response_body; } _logger.syslog4j("INFO", "urlfetch.response.async", "URL", furl.toString(), "Status", Integer.toString(res.getResponseCode()), "Headers", response_header_log, "Body", log_body); if (system && log_to_user) { _response.userlog4j("INFO", "urlfetch.response.async", "URL", furl.toString(), "Status", Integer.toString(res.getResponseCode()), "Headers", response_header_log); } _costCollector.collect((system) ? "asuc" : "auuc").collect((system) ? "asuw" : "auuw", waiting_time); return out; }
From source file:com.google.acre.script.NHttpAsyncUrlfetch.java
private Scriptable callback_result(long start_time, URL url, HttpResponse res, boolean system, boolean log_to_user, String response_encoding) { BrowserCompatSpecFactory bcsf = new BrowserCompatSpecFactory(); CookieSpec cspec = bcsf.newInstance(null); String protocol = url.getProtocol(); boolean issecure = ("https".equals(protocol)); int port = url.getPort(); if (port == -1) port = 80;//w ww . ja va2 s . co m CookieOrigin origin = new CookieOrigin(url.getHost(), port, url.getPath(), issecure); Object body = ""; int status = res.getStatusLine().getStatusCode(); Context ctx = Context.getCurrentContext(); Scriptable out = ctx.newObject(_scope); Scriptable headers = ctx.newObject(_scope); Scriptable cookies = ctx.newObject(_scope); out.put("status", out, status); out.put("headers", out, headers); out.put("cookies", out, cookies); Header content_type_header = null; StringBuilder response_header_log = new StringBuilder(); for (Header h : res.getAllHeaders()) { if (h.getName().equalsIgnoreCase("set-cookie")) { String set_cookie = h.getValue(); Matcher m = Pattern.compile("\\s*(([^,]|(,\\s*\\d))+)").matcher(set_cookie); while (m.find()) { Header ch = new BasicHeader("Set-Cookie", set_cookie.substring(m.start(), m.end())); try { List<Cookie> pcookies = cspec.parse(ch, origin); for (Cookie c : pcookies) { cookies.put(c.getName(), cookies, new AcreCookie(c).toJsObject(_scope)); } } catch (MalformedCookieException e) { throw new RuntimeException(e); } } } else if (h.getName().equalsIgnoreCase("content-type")) { content_type_header = h; } response_header_log.append(h.getName() + ": " + h.getValue() + "\r\n"); headers.put(h.getName(), headers, h.getValue()); } String charset = null; if (content_type_header != null) { HeaderElement values[] = content_type_header.getElements(); if (values.length == 1) { NameValuePair param = values[0].getParameterByName("charset"); if (param != null) { charset = param.getValue(); } } } if (charset == null) charset = response_encoding; // read body HttpEntity ent = res.getEntity(); try { if (ent != null) { InputStream res_stream = ent.getContent(); Header cenc = ent.getContentEncoding(); if (cenc != null && res_stream != null) { HeaderElement[] codecs = cenc.getElements(); for (HeaderElement codec : codecs) { if (codec.getName().equalsIgnoreCase("gzip")) { res_stream = new GZIPInputStream(res_stream); } } } long first_byte_time = 0; long end_time = 0; if (content_type_header != null && (content_type_header.getValue().startsWith("image/") || content_type_header.getValue().startsWith("application/octet-stream") || content_type_header.getValue().startsWith("multipart/form-data"))) { // HttpClient's InputStream doesn't support mark/reset, so // wrap it with one that does. BufferedInputStream bufis = new BufferedInputStream(res_stream); bufis.mark(2); bufis.read(); first_byte_time = System.currentTimeMillis(); bufis.reset(); byte[] data = IOUtils.toByteArray(bufis); end_time = System.currentTimeMillis(); body = new JSBinary(); ((JSBinary) body).set_data(data); try { if (res_stream != null) res_stream.close(); } catch (IOException e) { // ignore } } else if (res_stream == null || charset == null) { first_byte_time = end_time = System.currentTimeMillis(); body = ""; } else { StringWriter writer = new StringWriter(); Reader reader = new InputStreamReader(res_stream, charset); int i = reader.read(); first_byte_time = System.currentTimeMillis(); writer.write(i); IOUtils.copy(reader, writer); end_time = System.currentTimeMillis(); body = writer.toString(); try { reader.close(); writer.close(); } catch (IOException e) { // ignore } } long reading_time = end_time - first_byte_time; long waiting_time = first_byte_time - start_time; String httprephdr = response_header_log.toString(); // XXX need to log start-time of request _logger.syslog4j("DEBUG", "urlfetch.response.async", "URL", url.toString(), "Status", Integer.toString(status), "Headers", httprephdr, "Reading time", reading_time, "Waiting time", waiting_time); if (system && log_to_user) { _response.userlog4j("DEBUG", "urlfetch.response.async", "URL", url.toString(), "Status", Integer.toString(status), "Headers", httprephdr); } // XXX seems like AcreResponse should be able to use // the statistics object to generate x-metaweb-cost // given a bit of extra information Statistics.instance().collectUrlfetchTime(start_time, first_byte_time, end_time); _costCollector.collect((system) ? "asuc" : "auuc").collect((system) ? "asuw" : "auuw", waiting_time); } } catch (IOException e) { throw new RuntimeException(e); } out.put("body", out, body); return out; }
From source file:org.vietspider.net.client.impl.AnonymousHttpClient.java
@Override protected CookieSpecRegistry createCookieSpecRegistry() { CookieSpecRegistry registry = new CookieSpecRegistry(); registry.register(CookiePolicy.BEST_MATCH, new BestMatchSpecFactory()); registry.register(CookiePolicy.BROWSER_COMPATIBILITY, new BrowserCompatSpecFactory()); registry.register(CookiePolicy.NETSCAPE, new NetscapeDraftSpecFactory()); registry.register(CookiePolicy.RFC_2109, new RFC2109SpecFactory()); registry.register(CookiePolicy.RFC_2965, new RFC2965SpecFactory()); return registry; }
From source file:org.apache.http.impl.client.AbstractHttpClient.java
protected CookieSpecRegistry createCookieSpecRegistry() { final CookieSpecRegistry registry = new CookieSpecRegistry(); registry.register(CookiePolicy.BEST_MATCH, new BestMatchSpecFactory()); registry.register(CookiePolicy.BROWSER_COMPATIBILITY, new BrowserCompatSpecFactory()); registry.register(CookiePolicy.NETSCAPE, new NetscapeDraftSpecFactory()); registry.register(CookiePolicy.RFC_2109, new RFC2109SpecFactory()); registry.register(CookiePolicy.RFC_2965, new RFC2965SpecFactory()); registry.register(CookiePolicy.IGNORE_COOKIES, new IgnoreSpecFactory()); return registry; }
From source file:org.apache.http.impl.client.AbstractStatisticsGatheringHttpClient.java
protected CookieSpecRegistry createCookieSpecRegistry() { CookieSpecRegistry registry = new CookieSpecRegistry(); registry.register(CookiePolicy.BEST_MATCH, new BestMatchSpecFactory()); registry.register(CookiePolicy.BROWSER_COMPATIBILITY, new BrowserCompatSpecFactory()); registry.register(CookiePolicy.NETSCAPE, new NetscapeDraftSpecFactory()); registry.register(CookiePolicy.RFC_2109, new RFC2109SpecFactory()); registry.register(CookiePolicy.RFC_2965, new RFC2965SpecFactory()); registry.register(CookiePolicy.IGNORE_COOKIES, new IgnoreSpecFactory()); return registry; }