Java tutorial
/* * DSS - Digital Signature Services * * Copyright (C) 2013 European Commission, Directorate-General Internal Market and Services (DG MARKT), B-1049 Bruxelles/Brussel * * Developed by: 2013 ARHS Developments S.A. (rue Nicolas Bov 2B, L-1253 Luxembourg) http://www.arhs-developments.com * * This file is part of the "DSS - Digital Signature Services" project. * * "DSS - Digital Signature Services" is free software: you can redistribute it and/or modify it under the terms of * the GNU Lesser General Public License as published by the Free Software Foundation, either version 2.1 of the * License, or (at your option) any later version. * * DSS is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public License along with * "DSS - Digital Signature Services". If not, see <http://www.gnu.org/licenses/>. */ package eu.europa.ec.markt.dss.validation102853.https; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.net.URI; import java.net.URL; import java.security.SecureRandom; import java.util.HashMap; import java.util.Hashtable; import java.util.List; import java.util.Map; import javax.naming.Context; import javax.naming.directory.Attributes; import javax.naming.directory.DirContext; import javax.naming.directory.InitialDirContext; import javax.net.ssl.KeyManager; import javax.net.ssl.SSLContext; import javax.net.ssl.TrustManager; import org.apache.http.HttpEntity; import org.apache.http.HttpHost; import org.apache.http.HttpResponse; import org.apache.http.HttpStatus; import org.apache.http.auth.AuthScope; import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.client.AuthCache; import org.apache.http.client.CredentialsProvider; import org.apache.http.client.HttpClient; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.client.protocol.HttpClientContext; import org.apache.http.config.RegistryBuilder; import org.apache.http.conn.HttpClientConnectionManager; import org.apache.http.conn.socket.ConnectionSocketFactory; import org.apache.http.conn.socket.PlainConnectionSocketFactory; import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.entity.BufferedHttpEntity; import org.apache.http.entity.InputStreamEntity; import org.apache.http.impl.auth.BasicScheme; import org.apache.http.impl.client.BasicAuthCache; import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.client.HttpClients; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.message.AbstractHttpMessage; import org.apache.http.util.EntityUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import eu.europa.ec.markt.dss.DSSUtils; import eu.europa.ec.markt.dss.exception.DSSException; import eu.europa.ec.markt.dss.manager.ProxyPreferenceManager; import eu.europa.ec.markt.dss.validation102853.loader.DataLoader; import eu.europa.ec.markt.dss.validation102853.loader.Protocol; /** * Implementation of DataLoader for any protocol.<p/> * HTTP & HTTPS: using HttpClient which is more flexible for HTTPS without having to add the certificate to the JVM TrustStore. It takes into account a proxy management through * {@code ProxyPreferenceManager}. The authentication is also supported.<p/> * * @version $Revision$ - $Date$ */ public class CommonDataLoader implements DataLoader, DSSNotifier { private static final Logger LOG = LoggerFactory.getLogger(CommonDataLoader.class); public static final int TIMEOUT_CONNECTION = 6000; public static final int TIMEOUT_SOCKET = 6000; public static final String CONTENT_TYPE = "Content-Type"; protected String contentType; public static final String CONTENT_TRANSFER_ENCODING = "Content-Transfer-Encoding"; protected String contentTransferEncoding; // = "binary"; private ProxyPreferenceManager proxyPreferenceManager; private int timeoutConnection = TIMEOUT_CONNECTION; private int timeoutSocket = TIMEOUT_SOCKET; private final Map<HttpHost, UsernamePasswordCredentials> authenticationMap = new HashMap<HttpHost, UsernamePasswordCredentials>(); private HttpClient httpClient; /** * This variable indicates if any parameter has changed: authentication, proxy... */ private boolean updated; /** * The default constructor for CommonsDataLoader. */ public CommonDataLoader() { this(null); } /** * The constructor for CommonsDataLoader with defined content-type. * * @param contentType The requestBytes type of each request */ public CommonDataLoader(final String contentType) { this.contentType = contentType; } private HttpClientConnectionManager getConnectionManager() throws DSSException { LOG.debug("HTTPS TrustStore undefined, using default"); RegistryBuilder<ConnectionSocketFactory> socketFactoryRegistryBuilder = RegistryBuilder.create(); socketFactoryRegistryBuilder = setConnectionManagerSchemeHttp(socketFactoryRegistryBuilder); socketFactoryRegistryBuilder = setConnectionManagerSchemeHttps(socketFactoryRegistryBuilder); final HttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager( socketFactoryRegistryBuilder.build()); return connectionManager; } private RegistryBuilder<ConnectionSocketFactory> setConnectionManagerSchemeHttp( RegistryBuilder<ConnectionSocketFactory> socketFactoryRegistryBuilder) { return socketFactoryRegistryBuilder.register("http", PlainConnectionSocketFactory.getSocketFactory()); } private RegistryBuilder<ConnectionSocketFactory> setConnectionManagerSchemeHttps( RegistryBuilder<ConnectionSocketFactory> socketFactoryRegistryBuilder) throws DSSException { try { SSLContext sslContext = SSLContext.getInstance("TLS"); sslContext.init(new KeyManager[0], new TrustManager[] { new DefaultTrustManager() }, new SecureRandom()); SSLContext.setDefault(sslContext); final SSLConnectionSocketFactory sslConnectionSocketFactory = new SSLConnectionSocketFactory( sslContext); return socketFactoryRegistryBuilder.register("https", sslConnectionSocketFactory); } catch (Exception e) { throw new DSSException(e); } } protected synchronized HttpClient getHttpClient(final URI uri) throws DSSException { if (httpClient != null && !updated) { return httpClient; } if (LOG.isTraceEnabled() && updated) { LOG.trace(">>> Proxy preferences updated"); } final HttpClientBuilder httpClientBuilder = configCredentials(uri); final RequestConfig.Builder custom = RequestConfig.custom(); custom.setSocketTimeout(timeoutSocket); custom.setConnectionRequestTimeout(timeoutConnection); final RequestConfig requestConfig = custom.build(); httpClientBuilder.setDefaultRequestConfig(requestConfig); httpClientBuilder.setConnectionManager(getConnectionManager()); httpClient = httpClientBuilder.build(); return httpClient; } /** * Define the Credentials * * @param uri * @return {@code HttpClientBuilder} */ private HttpClientBuilder configCredentials(final URI uri) throws DSSException { HttpClientBuilder httpClientBuilder = HttpClients.custom(); final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); for (final Map.Entry<HttpHost, UsernamePasswordCredentials> entry : authenticationMap.entrySet()) { final HttpHost httpHost = entry.getKey(); final UsernamePasswordCredentials usernamePasswordCredentials = entry.getValue(); final AuthScope authscope = new AuthScope(httpHost.getHostName(), httpHost.getPort()); credentialsProvider.setCredentials(authscope, usernamePasswordCredentials); } httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider); httpClientBuilder = configureProxy(httpClientBuilder, credentialsProvider, uri); return httpClientBuilder; } /** * Configure the proxy with the required credential if needed * * @param httpClientBuilder * @param credentialsProvider * @param uri * @return */ private HttpClientBuilder configureProxy(final HttpClientBuilder httpClientBuilder, final CredentialsProvider credentialsProvider, final URI uri) throws DSSException { if (proxyPreferenceManager == null) { return httpClientBuilder; } final String protocol = uri.getScheme(); final boolean proxyHTTPS = Protocol.isHttps(protocol) && proxyPreferenceManager.isHttpsEnabled(); final boolean proxyHTTP = Protocol.isHttp(protocol) && proxyPreferenceManager.isHttpEnabled(); if (!proxyHTTPS && !proxyHTTP) { return httpClientBuilder; } String proxyHost = null; int proxyPort = 0; String proxyUser = null; String proxyPassword = null; if (proxyHTTPS) { LOG.debug("Use proxy https parameters"); final Long port = proxyPreferenceManager.getHttpsPort(); proxyPort = port != null ? port.intValue() : 0; proxyHost = proxyPreferenceManager.getHttpsHost(); proxyUser = proxyPreferenceManager.getHttpsUser(); proxyPassword = proxyPreferenceManager.getHttpsPassword(); } else if (proxyHTTP) { // noinspection ConstantConditions LOG.debug("Use proxy http parameters"); final Long port = proxyPreferenceManager.getHttpPort(); proxyPort = port != null ? port.intValue() : 0; proxyHost = proxyPreferenceManager.getHttpHost(); proxyUser = proxyPreferenceManager.getHttpUser(); proxyPassword = proxyPreferenceManager.getHttpPassword(); } if (DSSUtils.isNotEmpty(proxyUser) && DSSUtils.isNotEmpty(proxyPassword)) { AuthScope proxyAuth = new AuthScope(proxyHost, proxyPort); UsernamePasswordCredentials proxyCredentials = new UsernamePasswordCredentials(proxyUser, proxyPassword); credentialsProvider.setCredentials(proxyAuth, proxyCredentials); } LOG.debug("proxy host/port: " + proxyHost + ":" + proxyPort); // TODO SSL peer shut down incorrectly when protocol is https final HttpHost proxy = new HttpHost(proxyHost, proxyPort, Protocol.HTTP.getName()); httpClientBuilder.setProxy(proxy); updated = false; return httpClientBuilder; } @Override public byte[] get(final String urlString) { if (Protocol.isHttpUrl(urlString)) { return httpGet(urlString); } else if (Protocol.isLdapUrl(urlString)) { return ldapGet(urlString); } else if (Protocol.isFtpUrl(urlString)) { return ftpGet(urlString); } else if (Protocol.isFileUrl(urlString)) { return fileGet(urlString); } else { LOG.warn("DSS framework only supports HTTP, HTTPS, FTP, LDAP and FILE urlString."); } return httpGet(urlString); } @Override public DataAndUrl get(final List<String> urlStrings) { final int numberOfUrls = urlStrings.size(); int ii = 0; for (final String urlString : urlStrings) { try { ii++; final byte[] bytes = get(urlString); if (bytes == null) { continue; } return new DataAndUrl(bytes, urlString); } catch (Exception e) { if (ii == numberOfUrls) { if (e instanceof DSSException) { throw (DSSException) e; } throw new DSSException(e); } LOG.warn("Impossible to obtain data using {}", urlString, e); } } return null; } /** * This method is useful only with the cache handling implementation of the {@code DataLoader}. * * @param url to access * @param refresh if true indicates that the cached data should be refreshed * @return {@code byte} array of obtained data */ @Override public byte[] get(final String url, final boolean refresh) { return get(url); } private byte[] fileGet(final String urlString) { final URL url = DSSUtils.toUrlQuietly(urlString); return DSSUtils.toByteArrayQuietly(url); } // /** // * Obtains a CRL from a specified LDAP URL (Another method) // * // * @param ldapURL The LDAP URL String // * @return A CRL obtained from this LDAP URL if successful, otherwise NULL (if no CRL was resent) or an exception will be thrown. // * @throws DSSException // */ // public static byte[] ldapGet2(final String ldapURL) throws DSSException { // // try { // // //final String ldapUrlStr = URLDecoder.decode(ldapURL, "UTF-8"); // final LdapUrl ldapUrl = new LdapUrl(ldapURL); // final int port = ldapUrl.getPort() > 0 ? ldapUrl.getPort() : 389; // final LdapConnection con = new LdapNetworkConnection(ldapUrl.getHost(), port); // con.connect(); // final Entry entry = con.lookup(ldapUrl.getDn(), ldapUrl.getAttributes().toArray(new String[ldapUrl.getAttributes().size()])); // final Collection<Attribute> attributes = entry.getAttributes(); // byte[] bytes = null; // for (Attribute attr : attributes) { // // bytes = attr.getBytes(); // break; // } // con.close(); // return bytes; // } catch (Exception e) { // // LOG.warn(e.toString(), e); // } // return null; // } // /** * This method retrieves data using LDAP protocol. * - CRL from given LDAP url, e.g. ldap://ldap.infonotary.com/dc=identity-ca,dc=infonotary,dc=com * * @param urlString * @return */ private byte[] ldapGet(final String urlString) { final Hashtable<String, String> env = new Hashtable<String, String>(); env.put(Context.INITIAL_CONTEXT_FACTORY, "com.sun.jndi.ldap.LdapCtxFactory"); env.put(Context.PROVIDER_URL, urlString); try { final DirContext ctx = new InitialDirContext(env); final Attributes attributes = ctx.getAttributes(""); final javax.naming.directory.Attribute attribute = attributes.get("certificateRevocationList;binary"); final byte[] ldapBytes = (byte[]) attribute.get(); if (ldapBytes == null || ldapBytes.length == 0) { throw new DSSException("Cannot download CRL from: " + urlString); } return ldapBytes; } catch (Exception e) { LOG.warn(e.getMessage(), e); } return null; } /** * This method retrieves data using FTP protocol . * * @param urlString * @return */ protected byte[] ftpGet(final String urlString) { final URL url = DSSUtils.toUrlQuietly(urlString); return DSSUtils.toByteArrayQuietly(url); } /** * This method retrieves data using HTTP or HTTPS protocol and 'get' method. * * @param urlString to access * @return {@code byte} array of obtained data or {@code null} * @throws DSSException in case of any exception */ protected byte[] httpGet(final String urlString) throws DSSException { final URI uri = DSSUtils.toUri(urlString.trim()); HttpGet httpGet = null; HttpResponse httpResponse = null; try { httpGet = new HttpGet(uri); defineContentType(httpGet); defineContentTransferEncoding(httpGet); httpResponse = getHttpResponse(httpGet, uri); final byte[] returnedBytes = readHttpResponse(uri, httpResponse); return returnedBytes; } finally { if (httpGet != null) { httpGet.releaseConnection(); } if (httpResponse != null) { EntityUtils.consumeQuietly(httpResponse.getEntity()); } } } protected void defineContentTransferEncoding(final AbstractHttpMessage httpMessage) { if (contentTransferEncoding != null) { httpMessage.setHeader(CONTENT_TRANSFER_ENCODING, contentTransferEncoding); } } protected void defineContentType(final AbstractHttpMessage httpMessage) { if (contentType != null) { httpMessage.setHeader(CONTENT_TYPE, contentType); } } @Override public byte[] post(final String url, final byte[] requestBytes) throws DSSException { if (LOG.isDebugEnabled()) { LOG.debug("Fetching data via POST from url " + url); } HttpPost httpPost = null; HttpResponse httpResponse = null; try { final URI uri = DSSUtils.toUri(url.trim()); httpPost = new HttpPost(uri); // The length for the InputStreamEntity is needed, because some receivers (on the other side) need this information. // To determine the length, we cannot read the content-stream up to the end and re-use it afterwards. // This is because, it may not be possible to reset the stream (= go to position 0). // So, the solution is to cache temporarily the complete content data (as we do not expect much here) in a byte-array. final ByteArrayInputStream bis = new ByteArrayInputStream(requestBytes); final HttpEntity httpEntity = new InputStreamEntity(bis, requestBytes.length); final HttpEntity requestEntity = new BufferedHttpEntity(httpEntity); httpPost.setEntity(requestEntity); defineContentType(httpPost); defineContentTransferEncoding(httpPost); httpResponse = getHttpResponse(httpPost, uri); final byte[] returnedBytes = readHttpResponse(uri, httpResponse); return returnedBytes; } catch (IOException e) { throw new DSSException(e); } finally { if (httpPost != null) { httpPost.releaseConnection(); } if (httpResponse != null) { EntityUtils.consumeQuietly(httpResponse.getEntity()); } } } @Override public byte[] post(String url, byte[] requestBytes, boolean refresh) { return post(url, requestBytes); } protected HttpResponse getHttpResponse(final HttpUriRequest httpRequest, final URI uri) throws DSSException { final HttpClient client = getHttpClient(uri); final String host = uri.getHost(); final int port = uri.getPort(); final String scheme = uri.getScheme(); final HttpHost targetHost = new HttpHost(host, port, scheme); // Create AuthCache instance AuthCache authCache = new BasicAuthCache(); // Generate BASIC scheme object and add it to the local auth cache BasicScheme basicAuth = new BasicScheme(); authCache.put(targetHost, basicAuth); // Add AuthCache to the execution context HttpClientContext localContext = HttpClientContext.create(); localContext.setAuthCache(authCache); try { final HttpResponse response = client.execute(targetHost, httpRequest, localContext); return response; } catch (IOException e) { throw new DSSException(e); } } protected byte[] readHttpResponse(final URI uri, final HttpResponse httpResponse) throws DSSException { final int statusCode = httpResponse.getStatusLine().getStatusCode(); if (statusCode != HttpStatus.SC_OK) { LOG.warn("Uri: '" + uri + "' - Status code is " + statusCode + " - NOK"); return null; } else if (LOG.isDebugEnabled()) { LOG.debug("Uri: '" + uri + "' - Status code is " + statusCode + " - OK"); } final HttpEntity responseEntity = httpResponse.getEntity(); if (responseEntity == null) { LOG.warn("No message entity for this response via url: " + uri + " - returns null"); return null; } final byte[] content = getContent(responseEntity); return content; } protected byte[] getContent(final HttpEntity responseEntity) throws DSSException { try { final InputStream content = responseEntity.getContent(); final byte[] bytes = DSSUtils.toByteArray(content); return bytes; } catch (IOException e) { throw new DSSException(e); } } /** * Used when the {@code HttpClient} is created. * * @return the value (millis) */ public int getTimeoutConnection() { return timeoutConnection; } /** * Used when the {@code HttpClient} is created. * * @param timeoutConnection the value (millis) */ public void setTimeoutConnection(final int timeoutConnection) { httpClient = null; this.timeoutConnection = timeoutConnection; } /** * Used when the {@code HttpClient} is created. * * @return the value (millis) */ public int getTimeoutSocket() { return timeoutSocket; } /** * Used when the {@code HttpClient} is created. * * @param timeoutSocket the value (millis) */ public void setTimeoutSocket(final int timeoutSocket) { httpClient = null; this.timeoutSocket = timeoutSocket; } /** * @return the contentType */ public String getContentType() { return contentType; } /** * This allows to set the content type. Example: Content-Type "application/ocsp-request" * * @param contentType */ @Override public void setContentType(final String contentType) { this.contentType = contentType; } /** * @return Content-Transfer-Encoding */ public String getContentTransferEncoding() { return contentTransferEncoding; } /** * This allows to set the content transfer encoding. Example: Content-Transfer-Encoding "binary" * * @param contentTransferEncoding */ public void setContentTransferEncoding(final String contentTransferEncoding) { this.contentTransferEncoding = contentTransferEncoding; } /** * @return associated {@code ProxyPreferenceManager} */ public ProxyPreferenceManager getProxyPreferenceManager() { return proxyPreferenceManager; } /** * @param proxyPreferenceManager the proxyPreferenceManager to set */ public void setProxyPreferenceManager(final ProxyPreferenceManager proxyPreferenceManager) { httpClient = null; this.proxyPreferenceManager = proxyPreferenceManager; if (proxyPreferenceManager != null) { proxyPreferenceManager.addNotifier(this); if (LOG.isTraceEnabled()) { LOG.trace(">>> SET: " + proxyPreferenceManager); } } } /** * @param host the hostname (IP or DNS name) * @param port the port number. {@code -1} indicates the scheme default port. * @param scheme the name of the scheme. {@code null} indicates the {@link HttpHost#DEFAULT_SCHEME_NAME default scheme} * @param login login the user name * @param password password the password * @return this for fluent addAuthentication */ public CommonDataLoader addAuthentication(final String host, final int port, final String scheme, final String login, final String password) { final HttpHost httpHost = new HttpHost(host, port, scheme); final UsernamePasswordCredentials credentials = new UsernamePasswordCredentials(login, password); authenticationMap.put(httpHost, credentials); httpClient = null; return this; } /** * This method allows to propagate the authentication information from the current object. * * @param commonDataLoader {@code CommonsDataLoader} to be initialised with authentication information */ public void propagateAuthentication(final CommonDataLoader commonDataLoader) { for (final Map.Entry<HttpHost, UsernamePasswordCredentials> credentialsEntry : authenticationMap .entrySet()) { final HttpHost httpHost = credentialsEntry.getKey(); final UsernamePasswordCredentials credentials = credentialsEntry.getValue(); commonDataLoader.addAuthentication(httpHost.getHostName(), httpHost.getPort(), httpHost.getSchemeName(), credentials.getUserName(), credentials.getPassword()); } } /** * Call this method to indicate to the {@code DataLoader} if the {@code HttpClient} must be regenerated to take into account new parameters: authentication, proxy... */ @Override public void update() { updated = true; } }