org.apache.maven.wagon.providers.http.AbstractHttpClientWagonFixed.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.maven.wagon.providers.http.AbstractHttpClientWagonFixed.java

Source

//Adding to this package for Package Private Classes in original implementation
package org.apache.maven.wagon.providers.http;

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.HttpException;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.auth.AUTH;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.Credentials;
import org.apache.http.auth.MalformedChallengeException;
import org.apache.http.auth.NTCredentials;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.AuthCache;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.config.CookieSpecs;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpHead;
import org.apache.http.client.methods.HttpPut;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.client.utils.DateUtils;
import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.conn.HttpClientConnectionManager;
import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.conn.ssl.SSLContextBuilder;
import org.apache.http.conn.ssl.SSLInitializationException;
import org.apache.http.entity.AbstractHttpEntity;
import org.apache.http.impl.auth.BasicScheme;
import org.apache.http.impl.client.BasicAuthCache;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.message.BasicHeader;
import org.apache.http.protocol.HTTP;
import org.apache.http.util.EntityUtils;
import org.apache.maven.wagon.InputData;
import org.apache.maven.wagon.OutputData;
import org.apache.maven.wagon.PathUtils;
import org.apache.maven.wagon.ResourceDoesNotExistException;
import org.apache.maven.wagon.StreamWagon;
import org.apache.maven.wagon.TransferFailedException;
import org.apache.maven.wagon.Wagon;
import org.apache.maven.wagon.authorization.AuthorizationException;
import org.apache.maven.wagon.events.TransferEvent;
import org.apache.maven.wagon.proxy.ProxyInfo;
import org.apache.maven.wagon.repository.Repository;
import org.apache.maven.wagon.resource.Resource;
import org.apache.maven.wagon.shared.http.EncodingUtil;
import org.codehaus.plexus.util.IOUtil;
import org.codehaus.plexus.util.StringUtils;

import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
import java.io.ByteArrayInputStream;
import java.io.Closeable;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Locale;
import java.util.Map;
import java.util.Properties;
import java.util.TimeZone;
import java.util.concurrent.TimeUnit;
import org.adeptnet.maven.wagon.providers.http.SPNegoSchemeFactory;
import org.apache.http.auth.AuthSchemeProvider;
import org.apache.http.client.config.AuthSchemes;
import org.apache.http.impl.auth.BasicSchemeFactory;
import org.apache.http.impl.auth.DigestSchemeFactory;
import org.apache.http.impl.auth.KerberosSchemeFactory;
import org.apache.http.impl.auth.NTLMSchemeFactory;

/**
 * @author <a href="mailto:michal.maczka@dimatics.com">Michal Maczka</a>
 * @author <a href="mailto:james@atlassian.com">James William Dumay</a>
 * @author <a href="mailto:francois.s@adept.co.za">Francois Steyn</a>
 */
public abstract class AbstractHttpClientWagonFixed extends StreamWagon {

    private final class RequestEntityImplementation extends AbstractHttpEntity {

        private static final int BUFFER_SIZE = 2048;

        private final Resource resource;

        private final Wagon wagon;

        private ByteBuffer byteBuffer;

        private File source;

        private long length = -1;

        private RequestEntityImplementation(final InputStream stream, final Resource resource, final Wagon wagon,
                final File source) throws TransferFailedException {
            if (source != null) {
                this.source = source;
            } else {
                try {
                    byte[] bytes = IOUtil.toByteArray(stream);
                    byteBuffer = ByteBuffer.allocate(bytes.length);
                    byteBuffer.put(bytes);
                } catch (IOException e) {
                    throw new TransferFailedException(e.getMessage(), e);
                }
            }
            this.resource = resource;
            this.length = resource == null ? -1 : resource.getContentLength();

            this.wagon = wagon;
        }

        public long getContentLength() {
            return length;
        }

        public InputStream getContent() throws IOException, IllegalStateException {
            if (this.source != null) {
                return new FileInputStream(this.source);
            }
            return new ByteArrayInputStream(this.byteBuffer.array());
        }

        public boolean isRepeatable() {
            return true;
        }

        public void writeTo(final OutputStream outstream) throws IOException {
            if (outstream == null) {
                throw new IllegalArgumentException("Output stream may not be null");
            }
            TransferEvent transferEvent = new TransferEvent(wagon, resource, TransferEvent.TRANSFER_PROGRESS,
                    TransferEvent.REQUEST_PUT);
            transferEvent.setTimestamp(System.currentTimeMillis());
            InputStream instream = (this.source != null) ? new FileInputStream(this.source)
                    : new ByteArrayInputStream(this.byteBuffer.array());
            try {
                byte[] buffer = new byte[BUFFER_SIZE];
                int l;
                if (this.length < 0) {
                    // until EOF
                    while ((l = instream.read(buffer)) != -1) {
                        fireTransferProgress(transferEvent, buffer, -1);
                        outstream.write(buffer, 0, l);
                    }
                } else {
                    // no need to consume more than length
                    long remaining = this.length;
                    while (remaining > 0) {
                        l = instream.read(buffer, 0, (int) Math.min(BUFFER_SIZE, remaining));
                        if (l == -1) {
                            break;
                        }
                        fireTransferProgress(transferEvent, buffer, (int) Math.min(BUFFER_SIZE, remaining));
                        outstream.write(buffer, 0, l);
                        remaining -= l;
                    }
                }
            } finally {
                instream.close();
            }
        }

        public boolean isStreaming() {
            return true;
        }
    }

    private static final TimeZone GMT_TIME_ZONE = TimeZone.getTimeZone("GMT");

    /**
     * use http(s) connection pool mechanism.
     * <b>enabled by default</b>
     */
    private static boolean persistentPool = Boolean.valueOf(System.getProperty("maven.wagon.http.pool", "true"));

    /**
     * skip failure on certificate validity checks.
     * <b>disabled by default</b>
     */
    private static final boolean SSL_INSECURE = Boolean
            .valueOf(System.getProperty("maven.wagon.http.ssl.insecure", "false"));

    /**
     * if using sslInsecure, certificate date issues will be ignored
     * <b>disabled by default</b>
     */
    private static final boolean IGNORE_SSL_VALIDITY_DATES = Boolean
            .valueOf(System.getProperty("maven.wagon.http.ssl.ignore.validity.dates", "false"));

    /**
     * If enabled, ssl hostname verifier does not check hostname. Disable this
     * will use a browser compat hostname verifier <b>disabled by default</b>
     */
    private static final boolean SSL_ALLOW_ALL = Boolean
            .valueOf(System.getProperty("maven.wagon.http.ssl.allowall", "false"));

    /**
     * Maximum concurrent connections per distinct route.
     * <b>20 by default</b>
     */
    private static final int MAX_CONN_PER_ROUTE = Integer
            .parseInt(System.getProperty("maven.wagon.httpconnectionManager.maxPerRoute", "20"));

    /**
     * Maximum concurrent connections in total.
     * <b>40 by default</b>
     */
    private static final int MAX_CONN_TOTAL = Integer
            .parseInt(System.getProperty("maven.wagon.httpconnectionManager.maxTotal", "40"));

    /**
     * Internal connection manager
     */
    private static HttpClientConnectionManager httpClientConnectionManager = createConnManager();

    /**
     * See RFC6585
     */
    protected static final int SC_TOO_MANY_REQUESTS = 429;

    /**
     * For exponential backoff.
     */
    /**
     * Initial seconds to back off when a HTTP 429 received. Subsequent 429
     * responses result in exponental backoff.
     * <b>5 by default</b>
     *
     * @since 2.7
     */
    private int initialBackoffSeconds = Integer
            .parseInt(System.getProperty("maven.wagon.httpconnectionManager.backoffSeconds", "5"));

    /**
     * The maximum amount of time we want to back off in the case of repeated
     * HTTP 429 response codes.
     *
     * @since 2.7
     */
    private static final int MAX_BACKOFF_WAIT_SECONDS = Integer
            .parseInt(System.getProperty("maven.wagon.httpconnectionManager.maxBackoffSeconds", "180"));

    protected int backoff(int wait, String url) throws InterruptedException, TransferFailedException {
        TimeUnit.SECONDS.sleep(wait);
        int nextWait = wait * 2;
        if (nextWait >= getMaxBackoffWaitSeconds()) {
            throw new TransferFailedException(
                    "Waited too long to access: " + url + ". Return code is: " + SC_TOO_MANY_REQUESTS);
        }
        return nextWait;
    }

    @SuppressWarnings("checkstyle:linelength")
    private static PoolingHttpClientConnectionManager createConnManager() {

        String sslProtocolsStr = System.getProperty("https.protocols");
        String cipherSuitesStr = System.getProperty("https.cipherSuites");
        String[] sslProtocols = sslProtocolsStr != null ? sslProtocolsStr.split(" *, *") : null;
        String[] cipherSuites = cipherSuitesStr != null ? cipherSuitesStr.split(" *, *") : null;

        SSLConnectionSocketFactory sslConnectionSocketFactory;
        if (SSL_INSECURE) {
            try {
                SSLContext sslContext = new SSLContextBuilder().useSSL()
                        .loadTrustMaterial(null, new RelaxedTrustStrategy(IGNORE_SSL_VALIDITY_DATES)).build();
                sslConnectionSocketFactory = new SSLConnectionSocketFactory(sslContext, sslProtocols, cipherSuites,
                        SSL_ALLOW_ALL ? SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER
                                : SSLConnectionSocketFactory.BROWSER_COMPATIBLE_HOSTNAME_VERIFIER);
            } catch (Exception ex) {
                throw new SSLInitializationException(ex.getMessage(), ex);
            }
        } else {
            sslConnectionSocketFactory = new SSLConnectionSocketFactory(
                    HttpsURLConnection.getDefaultSSLSocketFactory(), sslProtocols, cipherSuites,
                    SSLConnectionSocketFactory.BROWSER_COMPATIBLE_HOSTNAME_VERIFIER);
        }

        Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create()
                .register("http", PlainConnectionSocketFactory.INSTANCE)
                .register("https", sslConnectionSocketFactory).build();

        PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(registry);
        if (persistentPool) {
            connManager.setDefaultMaxPerRoute(MAX_CONN_PER_ROUTE);
            connManager.setMaxTotal(MAX_CONN_TOTAL);
        } else {
            connManager.setMaxTotal(1);
        }
        return connManager;
    }

    private static final CloseableHttpClient CLIENT = createClient();

    private static CloseableHttpClient createClient() {
        return HttpClientBuilder.create() //
                .useSystemProperties() //
                .disableConnectionState() //
                .setConnectionManager(httpClientConnectionManager) //
                //Using Custom Default Schema Registry
                .setDefaultAuthSchemeRegistry(RegistryBuilder.<AuthSchemeProvider>create()
                        .register(AuthSchemes.BASIC, new BasicSchemeFactory())
                        .register(AuthSchemes.DIGEST, new DigestSchemeFactory())
                        .register(AuthSchemes.NTLM, new NTLMSchemeFactory())
                        //Using Custom SPNEGO Factory & FORCE strip port
                        //Kerberos Error: Server not found in Kerberos database (7) - LOOKING_UP_SERVER
                        .register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory(true))
                        .register(AuthSchemes.KERBEROS, new KerberosSchemeFactory(true)).build())
                .build();
    }

    private CredentialsProvider credentialsProvider;

    private AuthCache authCache;

    private HttpClientContext localContext;

    private Closeable closeable;

    /**
     * @plexus.configuration
     * @deprecated Use httpConfiguration instead.
     */
    private Properties httpHeaders;

    /**
     * @since 1.0-beta-6
     */
    private HttpConfiguration httpConfiguration;

    /**
     * Basic auth scope overrides
     *
     * @since 2.8
     */
    private BasicAuthScope basicAuth;

    /**
     * Proxy basic auth scope overrides
     *
     * @since 2.8
     */
    private BasicAuthScope proxyAuth;

    public void openConnectionInternal() {
        repository.setUrl(getURL(repository));

        localContext = HttpClientContext.create();
        credentialsProvider = new BasicCredentialsProvider();
        authCache = new BasicAuthCache();
        localContext.setCredentialsProvider(credentialsProvider);
        localContext.setAuthCache(authCache);

        if (authenticationInfo != null) {

            String username = authenticationInfo.getUserName();
            String password = authenticationInfo.getPassword();

            if (StringUtils.isNotEmpty(username) && StringUtils.isNotEmpty(password)) {
                Credentials creds = new UsernamePasswordCredentials(username, password);

                String host = getRepository().getHost();
                int port = getRepository().getPort();

                credentialsProvider.setCredentials(getBasicAuthScope().getScope(host, port), creds);
            }
        }

        ProxyInfo proxyInfo = getProxyInfo(getRepository().getProtocol(), getRepository().getHost());
        if (proxyInfo != null) {
            String proxyUsername = proxyInfo.getUserName();
            String proxyPassword = proxyInfo.getPassword();
            String proxyHost = proxyInfo.getHost();
            String proxyNtlmHost = proxyInfo.getNtlmHost();
            String proxyNtlmDomain = proxyInfo.getNtlmDomain();
            if (proxyHost != null) {
                if (proxyUsername != null && proxyPassword != null) {
                    Credentials creds;
                    if (proxyNtlmHost != null || proxyNtlmDomain != null) {
                        creds = new NTCredentials(proxyUsername, proxyPassword, proxyNtlmHost, proxyNtlmDomain);
                    } else {
                        creds = new UsernamePasswordCredentials(proxyUsername, proxyPassword);
                    }

                    int port = proxyInfo.getPort();

                    AuthScope authScope = getProxyBasicAuthScope().getScope(proxyHost, port);
                    credentialsProvider.setCredentials(authScope, creds);
                }
            }
        }
    }

    public void closeConnection() {
        if (!persistentPool) {
            httpClientConnectionManager.closeIdleConnections(0, TimeUnit.MILLISECONDS);
        }
    }

    public static void setPersistentPool(boolean persistentPool) {
        persistentPool = persistentPool;
    }

    public static void setPoolingHttpClientConnectionManager(
            PoolingHttpClientConnectionManager poolingHttpClientConnectionManager) {
        httpClientConnectionManager = poolingHttpClientConnectionManager;
    }

    public void put(File source, String resourceName)
            throws TransferFailedException, ResourceDoesNotExistException, AuthorizationException {
        Resource resource = new Resource(resourceName);

        firePutInitiated(resource, source);

        resource.setContentLength(source.length());

        resource.setLastModified(source.lastModified());

        put(null, resource, source);
    }

    public void putFromStream(final InputStream stream, String destination, long contentLength, long lastModified)
            throws TransferFailedException, ResourceDoesNotExistException, AuthorizationException {
        Resource resource = new Resource(destination);

        firePutInitiated(resource, null);

        resource.setContentLength(contentLength);

        resource.setLastModified(lastModified);

        put(stream, resource, null);
    }

    private void put(final InputStream stream, Resource resource, File source)
            throws TransferFailedException, AuthorizationException, ResourceDoesNotExistException {
        put(resource, source, new RequestEntityImplementation(stream, resource, this, source));
    }

    private void put(Resource resource, File source, HttpEntity httpEntity)
            throws TransferFailedException, AuthorizationException, ResourceDoesNotExistException {
        put(resource, source, httpEntity, buildUrl(resource));
    }

    /**
     * Builds a complete URL string from the repository URL and the relative
     * path of the resource passed.
     *
     * @param resource the resource to extract the relative path from.
     * @return the complete URL
     */
    private String buildUrl(Resource resource) {
        return EncodingUtil.encodeURLToString(getRepository().getUrl(), resource.getName());
    }

    private void put(Resource resource, File source, HttpEntity httpEntity, String url)
            throws TransferFailedException, AuthorizationException, ResourceDoesNotExistException {
        put(getInitialBackoffSeconds(), resource, source, httpEntity, url);
    }

    private void put(int wait, Resource resource, File source, HttpEntity httpEntity, String url)
            throws TransferFailedException, AuthorizationException, ResourceDoesNotExistException {

        //Parent directories need to be created before posting
        try {
            mkdirs(PathUtils.dirname(resource.getName()));
        } catch (HttpException he) {
            fireTransferError(resource, he, TransferEvent.REQUEST_GET);
        } catch (IOException e) {
            fireTransferError(resource, e, TransferEvent.REQUEST_GET);
        }

        // preemptive for put
        // TODO: is it a good idea, though? 'Expect-continue' handshake would serve much better
        Repository repo = getRepository();
        HttpHost targetHost = new HttpHost(repo.getHost(), repo.getPort(), repo.getProtocol());
        AuthScope targetScope = getBasicAuthScope().getScope(targetHost);

        if (credentialsProvider.getCredentials(targetScope) != null) {
            BasicScheme targetAuth = new BasicScheme();
            try {
                targetAuth.processChallenge(new BasicHeader(AUTH.WWW_AUTH, "BASIC preemptive"));
                authCache.put(targetHost, targetAuth);
            } catch (MalformedChallengeException ignore) {
                // ignore
            }
        }

        HttpPut putMethod = new HttpPut(url);

        firePutStarted(resource, source);

        try {
            putMethod.setEntity(httpEntity);

            CloseableHttpResponse response = execute(putMethod);
            try {
                int statusCode = response.getStatusLine().getStatusCode();
                String reasonPhrase = ", ReasonPhrase: " + response.getStatusLine().getReasonPhrase() + ".";
                fireTransferDebug(url + " - Status code: " + statusCode + reasonPhrase);

                // Check that we didn't run out of retries.
                switch (statusCode) {
                // Success Codes
                case HttpStatus.SC_OK: // 200
                case HttpStatus.SC_CREATED: // 201
                case HttpStatus.SC_ACCEPTED: // 202
                case HttpStatus.SC_NO_CONTENT: // 204
                    break;
                // handle all redirect even if http specs says " the user agent MUST NOT automatically redirect
                // the request unless it can be confirmed by the user"
                case HttpStatus.SC_MOVED_PERMANENTLY: // 301
                case HttpStatus.SC_MOVED_TEMPORARILY: // 302
                case HttpStatus.SC_SEE_OTHER: // 303
                    put(resource, source, httpEntity, calculateRelocatedUrl(response));
                    return;
                case HttpStatus.SC_FORBIDDEN:
                    fireSessionConnectionRefused();
                    throw new AuthorizationException("Access denied to: " + url + reasonPhrase);

                case HttpStatus.SC_NOT_FOUND:
                    throw new ResourceDoesNotExistException("File: " + url + " does not exist" + reasonPhrase);

                case SC_TOO_MANY_REQUESTS:
                    put(backoff(wait, url), resource, source, httpEntity, url);
                    break;
                //add more entries here
                default:
                    TransferFailedException e = new TransferFailedException(
                            "Failed to transfer file: " + url + ". Return code is: " + statusCode + reasonPhrase);
                    fireTransferError(resource, e, TransferEvent.REQUEST_PUT);
                    throw e;
                }

                firePutCompleted(resource, source);

                EntityUtils.consume(response.getEntity());
            } finally {
                response.close();
            }
        } catch (IOException e) {
            fireTransferError(resource, e, TransferEvent.REQUEST_PUT);

            throw new TransferFailedException(e.getMessage(), e);
        } catch (HttpException e) {
            fireTransferError(resource, e, TransferEvent.REQUEST_PUT);

            throw new TransferFailedException(e.getMessage(), e);
        } catch (InterruptedException e) {
            fireTransferError(resource, e, TransferEvent.REQUEST_PUT);

            throw new TransferFailedException(e.getMessage(), e);
        }

    }

    protected String calculateRelocatedUrl(HttpResponse response) {
        Header locationHeader = response.getFirstHeader("Location");
        String locationField = locationHeader.getValue();
        // is it a relative Location or a full ?
        return locationField.startsWith("http") ? locationField : getURL(getRepository()) + '/' + locationField;
    }

    protected void mkdirs(String dirname) throws HttpException, IOException {
        // nothing to do
    }

    public boolean resourceExists(String resourceName) throws TransferFailedException, AuthorizationException {
        return resourceExists(getInitialBackoffSeconds(), resourceName);
    }

    private boolean resourceExists(int wait, String resourceName)
            throws TransferFailedException, AuthorizationException {
        String repositoryUrl = getRepository().getUrl();
        String url = repositoryUrl + (repositoryUrl.endsWith("/") ? "" : "/") + resourceName;
        HttpHead headMethod = new HttpHead(url);
        try {
            CloseableHttpResponse response = execute(headMethod);
            try {
                int statusCode = response.getStatusLine().getStatusCode();
                String reasonPhrase = ", ReasonPhrase: " + response.getStatusLine().getReasonPhrase() + ".";
                boolean result;
                switch (statusCode) {
                case HttpStatus.SC_OK:
                    result = true;
                    break;
                case HttpStatus.SC_NOT_MODIFIED:
                    result = true;
                    break;
                case HttpStatus.SC_FORBIDDEN:
                    throw new AuthorizationException("Access denied to: " + url + reasonPhrase);

                case HttpStatus.SC_UNAUTHORIZED:
                    throw new AuthorizationException("Not authorized " + reasonPhrase);

                case HttpStatus.SC_PROXY_AUTHENTICATION_REQUIRED:
                    throw new AuthorizationException("Not authorized by proxy " + reasonPhrase);

                case HttpStatus.SC_NOT_FOUND:
                    result = false;
                    break;

                case SC_TOO_MANY_REQUESTS:
                    return resourceExists(backoff(wait, resourceName), resourceName);

                //add more entries here
                default:
                    throw new TransferFailedException(
                            "Failed to transfer file: " + url + ". Return code is: " + statusCode + reasonPhrase);
                }

                EntityUtils.consume(response.getEntity());
                return result;
            } finally {
                response.close();
            }
        } catch (IOException e) {
            throw new TransferFailedException(e.getMessage(), e);
        } catch (HttpException e) {
            throw new TransferFailedException(e.getMessage(), e);
        } catch (InterruptedException e) {
            throw new TransferFailedException(e.getMessage(), e);
        }

    }

    protected CloseableHttpResponse execute(HttpUriRequest httpMethod) throws HttpException, IOException {
        setHeaders(httpMethod);
        String userAgent = getUserAgent(httpMethod);
        if (userAgent != null) {
            httpMethod.setHeader(HTTP.USER_AGENT, userAgent);
        }

        RequestConfig.Builder requestConfigBuilder = RequestConfig.custom();
        // WAGON-273: default the cookie-policy to browser compatible
        requestConfigBuilder.setCookieSpec(CookieSpecs.BROWSER_COMPATIBILITY);

        Repository repo = getRepository();
        ProxyInfo proxyInfo = getProxyInfo(repo.getProtocol(), repo.getHost());
        if (proxyInfo != null) {
            HttpHost proxy = new HttpHost(proxyInfo.getHost(), proxyInfo.getPort());
            requestConfigBuilder.setProxy(proxy);
        }

        HttpMethodConfiguration config = httpConfiguration == null ? null
                : httpConfiguration.getMethodConfiguration(httpMethod);

        if (config != null) {
            ConfigurationUtils.copyConfig(config, requestConfigBuilder);
        } else {
            requestConfigBuilder.setSocketTimeout(getReadTimeout());
        }

        localContext.setRequestConfig(requestConfigBuilder.build());

        if (config != null && config.isUsePreemptive()) {
            HttpHost targetHost = new HttpHost(repo.getHost(), repo.getPort(), repo.getProtocol());
            AuthScope targetScope = getBasicAuthScope().getScope(targetHost);

            if (credentialsProvider.getCredentials(targetScope) != null) {
                BasicScheme targetAuth = new BasicScheme();
                targetAuth.processChallenge(new BasicHeader(AUTH.WWW_AUTH, "BASIC preemptive"));
                authCache.put(targetHost, targetAuth);
            }
        }

        if (proxyInfo != null) {
            if (proxyInfo.getHost() != null) {
                HttpHost proxyHost = new HttpHost(proxyInfo.getHost(), proxyInfo.getPort());
                AuthScope proxyScope = getProxyBasicAuthScope().getScope(proxyHost);

                String proxyUsername = proxyInfo.getUserName();
                String proxyPassword = proxyInfo.getPassword();
                String proxyNtlmHost = proxyInfo.getNtlmHost();
                String proxyNtlmDomain = proxyInfo.getNtlmDomain();

                if (proxyUsername != null && proxyPassword != null) {
                    Credentials creds;
                    if (proxyNtlmHost != null || proxyNtlmDomain != null) {
                        creds = new NTCredentials(proxyUsername, proxyPassword, proxyNtlmHost, proxyNtlmDomain);
                    } else {
                        creds = new UsernamePasswordCredentials(proxyUsername, proxyPassword);
                    }

                    credentialsProvider.setCredentials(proxyScope, creds);
                    BasicScheme proxyAuth = new BasicScheme();
                    proxyAuth.processChallenge(new BasicHeader(AUTH.PROXY_AUTH, "BASIC preemptive"));
                    authCache.put(proxyHost, proxyAuth);
                }
            }
        }

        return CLIENT.execute(httpMethod, localContext);
    }

    protected void setHeaders(HttpUriRequest method) {
        HttpMethodConfiguration config = httpConfiguration == null ? null
                : httpConfiguration.getMethodConfiguration(method);
        if (config == null || config.isUseDefaultHeaders()) {
            // TODO: merge with the other headers and have some better defaults, unify with lightweight headers
            method.addHeader("Cache-control", "no-cache");
            method.addHeader("Cache-store", "no-store");
            method.addHeader("Pragma", "no-cache");
            method.addHeader("Expires", "0");
            method.addHeader("Accept-Encoding", "gzip");
        }

        if (httpHeaders != null) {
            for (Map.Entry<Object, Object> entry : httpHeaders.entrySet()) {
                method.setHeader((String) entry.getKey(), (String) entry.getValue());
            }
        }

        Header[] headers = config == null ? null : config.asRequestHeaders();
        if (headers != null) {
            for (Header header : headers) {
                method.setHeader(header);
            }
        }
    }

    protected String getUserAgent(HttpUriRequest method) {
        if (httpHeaders != null) {
            String value = (String) httpHeaders.get("User-Agent");
            if (value != null) {
                return value;
            }
        }
        HttpMethodConfiguration config = httpConfiguration == null ? null
                : httpConfiguration.getMethodConfiguration(method);

        if (config != null) {
            return (String) config.getHeaders().get("User-Agent");
        }
        return null;
    }

    /**
     * getUrl Implementors can override this to remove unwanted parts of the url
     * such as role-hints
     *
     * @param repository
     * @return
     */
    protected String getURL(Repository repository) {
        return repository.getUrl();
    }

    public HttpConfiguration getHttpConfiguration() {
        return httpConfiguration;
    }

    public void setHttpConfiguration(HttpConfiguration httpConfiguration) {
        this.httpConfiguration = httpConfiguration;
    }

    /**
     * Get the override values for standard HttpClient AuthScope
     *
     * @return the basicAuth
     */
    public BasicAuthScope getBasicAuthScope() {
        if (basicAuth == null) {
            basicAuth = new BasicAuthScope();
        }
        return basicAuth;
    }

    /**
     * Set the override values for standard HttpClient AuthScope
     *
     * @param basicAuth the AuthScope to set
     */
    public void setBasicAuthScope(BasicAuthScope basicAuth) {
        this.basicAuth = basicAuth;
    }

    /**
     * Get the override values for proxy HttpClient AuthScope
     *
     * @return the proxyAuth
     */
    public BasicAuthScope getProxyBasicAuthScope() {
        if (proxyAuth == null) {
            proxyAuth = new BasicAuthScope();
        }
        return proxyAuth;
    }

    /**
     * Set the override values for proxy HttpClient AuthScope
     *
     * @param proxyAuth the AuthScope to set
     */
    public void setProxyBasicAuthScope(BasicAuthScope proxyAuth) {
        this.proxyAuth = proxyAuth;
    }

    public void fillInputData(InputData inputData)
            throws TransferFailedException, ResourceDoesNotExistException, AuthorizationException {
        fillInputData(getInitialBackoffSeconds(), inputData);
    }

    private void fillInputData(int wait, InputData inputData)
            throws TransferFailedException, ResourceDoesNotExistException, AuthorizationException {
        Resource resource = inputData.getResource();

        String repositoryUrl = getRepository().getUrl();
        String url = repositoryUrl + (repositoryUrl.endsWith("/") ? "" : "/") + resource.getName();
        HttpGet getMethod = new HttpGet(url);
        long timestamp = resource.getLastModified();
        if (timestamp > 0) {
            SimpleDateFormat fmt = new SimpleDateFormat("EEE, dd-MMM-yy HH:mm:ss zzz", Locale.US);
            fmt.setTimeZone(GMT_TIME_ZONE);
            Header hdr = new BasicHeader("If-Modified-Since", fmt.format(new Date(timestamp)));
            fireTransferDebug("sending ==> " + hdr + "(" + timestamp + ")");
            getMethod.addHeader(hdr);
        }

        try {
            CloseableHttpResponse response = execute(getMethod);
            closeable = response;
            int statusCode = response.getStatusLine().getStatusCode();

            String reasonPhrase = ", ReasonPhrase:" + response.getStatusLine().getReasonPhrase() + ".";

            fireTransferDebug(url + " - Status code: " + statusCode + reasonPhrase);

            switch (statusCode) {
            case HttpStatus.SC_OK:
                break;

            case HttpStatus.SC_NOT_MODIFIED:
                // return, leaving last modified set to original value so getIfNewer should return unmodified
                return;
            case HttpStatus.SC_FORBIDDEN:
                fireSessionConnectionRefused();
                throw new AuthorizationException("Access denied to: " + url + " " + reasonPhrase);

            case HttpStatus.SC_UNAUTHORIZED:
                fireSessionConnectionRefused();
                throw new AuthorizationException("Not authorized " + reasonPhrase);

            case HttpStatus.SC_PROXY_AUTHENTICATION_REQUIRED:
                fireSessionConnectionRefused();
                throw new AuthorizationException("Not authorized by proxy " + reasonPhrase);

            case HttpStatus.SC_NOT_FOUND:
                throw new ResourceDoesNotExistException("File: " + url + " " + reasonPhrase);

            case SC_TOO_MANY_REQUESTS:
                fillInputData(backoff(wait, url), inputData);
                break;

            // add more entries here
            default:
                cleanupGetTransfer(resource);
                TransferFailedException e = new TransferFailedException(
                        "Failed to transfer file: " + url + ". Return code is: " + statusCode + " " + reasonPhrase);
                fireTransferError(resource, e, TransferEvent.REQUEST_GET);
                throw e;
            }

            Header contentLengthHeader = response.getFirstHeader("Content-Length");

            if (contentLengthHeader != null) {
                try {
                    long contentLength = Long.parseLong(contentLengthHeader.getValue());

                    resource.setContentLength(contentLength);
                } catch (NumberFormatException e) {
                    fireTransferDebug(
                            "error parsing content length header '" + contentLengthHeader.getValue() + "' " + e);
                }
            }

            Header lastModifiedHeader = response.getFirstHeader("Last-Modified");
            if (lastModifiedHeader != null) {
                Date lastModified = DateUtils.parseDate(lastModifiedHeader.getValue());
                if (lastModified != null) {
                    resource.setLastModified(lastModified.getTime());
                    fireTransferDebug("last-modified = " + lastModifiedHeader.getValue() + " ("
                            + lastModified.getTime() + ")");
                }
            }

            HttpEntity entity = response.getEntity();
            if (entity != null) {
                inputData.setInputStream(entity.getContent());
            }
        } catch (IOException e) {
            fireTransferError(resource, e, TransferEvent.REQUEST_GET);

            throw new TransferFailedException(e.getMessage(), e);
        } catch (HttpException e) {
            fireTransferError(resource, e, TransferEvent.REQUEST_GET);

            throw new TransferFailedException(e.getMessage(), e);
        } catch (InterruptedException e) {
            fireTransferError(resource, e, TransferEvent.REQUEST_GET);

            throw new TransferFailedException(e.getMessage(), e);
        }

    }

    protected void cleanupGetTransfer(Resource resource) {
        if (closeable != null) {
            try {
                closeable.close();
            } catch (IOException ignore) {
                // ignore
            }

        }
    }

    @Override
    public void putFromStream(InputStream stream, String destination)
            throws TransferFailedException, ResourceDoesNotExistException, AuthorizationException {
        putFromStream(stream, destination, -1, -1);
    }

    @Override
    protected void putFromStream(InputStream stream, Resource resource)
            throws TransferFailedException, AuthorizationException, ResourceDoesNotExistException {
        putFromStream(stream, resource.getName(), -1, -1);
    }

    public Properties getHttpHeaders() {
        return httpHeaders;
    }

    public void setHttpHeaders(Properties httpHeaders) {
        this.httpHeaders = httpHeaders;
    }

    @Override
    public void fillOutputData(OutputData outputData) throws TransferFailedException {
        // no needed in this implementation but throw an Exception if used
        throw new IllegalStateException("this wagon http client must not use fillOutputData");
    }

    public int getInitialBackoffSeconds() {
        return initialBackoffSeconds;
    }

    public void setInitialBackoffSeconds(int initialBackoffSeconds) {
        this.initialBackoffSeconds = initialBackoffSeconds;
    }

    public static int getMaxBackoffWaitSeconds() {
        return MAX_BACKOFF_WAIT_SECONDS;
    }
}