Java tutorial
/** * Copyright (C) 2012-2014 Gist Labs, LLC. (http://gistlabs.com) * * This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ package com.gistlabs.mechanize.impl; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URL; import java.util.Collection; import java.util.HashMap; import java.util.Map; import org.apache.http.Header; import org.apache.http.HttpException; import org.apache.http.HttpResponse; import org.apache.http.HttpResponseInterceptor; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpRequestBase; import org.apache.http.entity.BufferedHttpEntity; import org.apache.http.impl.client.AbstractHttpClient; import org.apache.http.impl.client.DefaultHttpClient; import org.apache.http.params.HttpProtocolParams; import org.apache.http.protocol.BasicHttpContext; import org.apache.http.protocol.HttpContext; import com.gistlabs.mechanize.Mechanize; import com.gistlabs.mechanize.Resource; import com.gistlabs.mechanize.ResourceFactory; import com.gistlabs.mechanize.cache.HttpCacheFilter; import com.gistlabs.mechanize.cache.api.HttpCache; import com.gistlabs.mechanize.cache.inMemory.InMemoryHttpCache; import com.gistlabs.mechanize.cookie.Cookies; import com.gistlabs.mechanize.exceptions.MechanizeExceptionFactory; import com.gistlabs.mechanize.filters.DefaultMechanizeChainFilter; import com.gistlabs.mechanize.filters.MechanizeChainFilter; import com.gistlabs.mechanize.parameters.Parameters; import com.gistlabs.mechanize.requestor.PageRequestor; import com.gistlabs.mechanize.requestor.RequestBuilder; import com.gistlabs.mechanize.requestor.RequestBuilderFactory; import com.gistlabs.mechanize.util.apache.ContentType; /** * Mechanize agent acts as a focal point for HTTP interactions and also as a factory for Page objects from responses. * * <p>Interesting resources: http://en.wikipedia.org/wiki/List_of_HTTP_header_fields</p> * * <p>NOTE: The mechanize library is not synchronized and should be used in a single thread environment or with custom synchronization.</p> * * @author Martin Kersten<Martin.Kersten.mk@gmail.com> * @author John Heintz <john@gistlabs.com> */ public class MechanizeAgent implements PageRequestor<Resource>, RequestBuilderFactory<Resource>, Mechanize { static final Map<String, ResourceFactory> PAGE_FACTORIES = new HashMap<String, ResourceFactory>(); static ResourceFactory lookupFactory(final String mimeType) { return PAGE_FACTORIES.get(mimeType); } static void registerFactory(final ResourceFactory factory) { Collection<String> contentMatches = factory.getContentMatches(); for (String mimeType : contentMatches) PAGE_FACTORIES.put(mimeType, factory); } static String VERSION; public static void setVersion(final String version) { VERSION = version; } static { MechanizeInitializer.initialize(); } private final DefaultMechanizeChainFilter requestChain; private final AbstractHttpClient client; private final Cookies cookies; public MechanizeAgent() { this(buildDefaultHttpClient()); } public MechanizeAgent(final AbstractHttpClient client) { this(client, new InMemoryHttpCache()); } public MechanizeAgent(final HttpCache httpCache) { this(buildDefaultHttpClient(), httpCache); } public MechanizeAgent(final AbstractHttpClient client, final HttpCache httpCache) { this.client = client; setupClient(client); this.requestChain = new DefaultMechanizeChainFilter(new MechanizeHttpClientFilter(this.client)); addFilter(new HttpCacheFilter(httpCache)); this.cookies = new Cookies(this.client); } /** * This method is used to capture Location headers after HttpClient redirect handling. */ private void setupClient(final AbstractHttpClient client) { this.client.addResponseInterceptor(new HttpResponseInterceptor() { @Override public void process(final HttpResponse response, final HttpContext context) throws HttpException, IOException { Header header = response.getFirstHeader("Location"); if (header != null) context.setAttribute("Location", header.getValue()); } }); } public MechanizeAgent prefixFilter(final MechanizeChainFilter filter) { this.requestChain.prefix(filter); return this; } public MechanizeAgent addFilter(final MechanizeChainFilter filter) { this.requestChain.add(filter); return this; } /** * Configure the default HttpClient used by mechanize. */ public static AbstractHttpClient buildDefaultHttpClient() { DefaultHttpClient defaultHttpClient = new DefaultHttpClient(); return defaultHttpClient; } /** * * @param userAgent The value to set User-Agent HTTP parameter to for requests * @return */ public MechanizeAgent setUserAgent(final String userAgent) { HttpProtocolParams.setUserAgent(this.client.getParams(), userAgent); return this; } /** * * @return the User-Agent that HttpClient is currently using. */ public String getUserAgent() { return HttpProtocolParams.getUserAgent(this.client.getParams()); } /* (non-Javadoc) * @see com.gistlabs.mechanize.Mechanize#getClient() */ @Override public AbstractHttpClient getClient() { return client; } /* (non-Javadoc) * @see com.gistlabs.mechanize.Mechanize#doRequest(java.lang.String) */ @Override public RequestBuilder<Resource> doRequest(final String uri) { return new RequestBuilder<Resource>(this, uri); } /** * Returns the resource received uppon the request. The resource can be casted to any expected subclass of resource * but will fail with ClassCastException if the expected type of resource is not the actual returned resource. */ @SuppressWarnings("unchecked") @Override public <T extends Resource> T request(final HttpRequestBase request) { try { HttpResponse response = execute(client, request); Resource resource = toPage(request, response); return (T) resource; } catch (Exception e) { throw MechanizeExceptionFactory.newException(e); } } /* (non-Javadoc) * @see com.gistlabs.mechanize.Mechanize#get(java.lang.String) */ @Override public <T extends Resource> T get(final String uri) { return doRequest(uri).get(); } /* (non-Javadoc) * @see com.gistlabs.mechanize.Mechanize#post(java.lang.String, java.util.Map) */ @Override public <T extends Resource> T post(final String uri, final Map<String, String> params) throws UnsupportedEncodingException { return post(uri, new Parameters(unsafeCast(params))); } @SuppressWarnings({ "unchecked", "rawtypes" }) private Map<String, Object> unsafeCast(final Map<String, String> params) { return (Map) params; } /* (non-Javadoc) * @see com.gistlabs.mechanize.Mechanize#post(java.lang.String, com.gistlabs.mechanize.parameters.Parameters) */ @Override public <T extends Resource> T post(final String uri, final Parameters params) { return doRequest(uri).set(params).post(); } /** Idles / Waits for the given amount of milliseconds useful to prevent being blocked by mass sending * requests or to appear as a artificial user. */ public void idle(final int milliseconds) { long startTime = System.currentTimeMillis(); while ((System.currentTimeMillis() - startTime) < milliseconds) try { Thread.sleep(Math.max(1, milliseconds - (System.currentTimeMillis() - startTime))); } catch (InterruptedException e) { } } /* (non-Javadoc) * @see com.gistlabs.mechanize.Mechanize#cookies() */ @Override public Cookies cookies() { return cookies; } protected Resource toPage(final HttpRequestBase request, final HttpResponse response) throws IOException, UnsupportedEncodingException { ContentType contentType = getContentType(response); ResourceFactory factory = lookupFactory(contentType.getMimeType()); if (factory == null) factory = lookupFactory(ContentType.WILDCARD.getMimeType()); if (factory == null) throw MechanizeExceptionFactory.newMechanizeException( "No viable page type found, and no wildcard mime type factory registered."); return factory.buildPage(this, request, response); } protected ContentType getContentType(final HttpResponse response) { return ContentType.getOrDefault(response.getEntity()); } protected HttpResponse execute(final HttpClient client, final HttpRequestBase request) throws Exception { HttpContext context = new BasicHttpContext(); HttpResponse response = requestChain.execute(request, context); if (context.getAttribute("Location") != null) response.setHeader(MECHANIZE_LOCATION, (String) context.getAttribute("Location")); response.setEntity(new BufferedHttpEntity(response.getEntity())); return response; } @Override public String absoluteUrl(final String uri) { try { return new URL(uri).toExternalForm(); } catch (MalformedURLException e) { throw MechanizeExceptionFactory.newException(e); } } }