Example usage for org.apache.hadoop.io.retry RetryUtils getDefaultRetryPolicy

Introduction

In this page you can find the example usage for org.apache.hadoop.io.retry RetryUtils getDefaultRetryPolicy.

Prototype

public static RetryPolicy getDefaultRetryPolicy(Configuration conf, String retryPolicyEnabledKey,
        boolean defaultRetryPolicyEnabled, String retryPolicySpecKey, String defaultRetryPolicySpec,
        final String remoteExceptionToRetry) 

Source Link

Document

Return the default retry policy set in conf.

Usage

From source file:com.bigstep.datalake.DLFileSystem.java

License:Apache License

@Override
public synchronized void initialize(URI uri, Configuration conf) throws IOException {
    super.initialize(uri, conf);

    uri = selectDatalakeEndpointURI(uri, conf);

    /* set user pattern based on configuration file */
    UserParam.setUserPattern(conf.get(DFSConfigKeys.DFS_WEBHDFS_USER_PATTERN_KEY,
            DFSConfigKeys.DFS_WEBHDFS_USER_PATTERN_DEFAULT));

    kerberosIdentity = initialiseKerberosIdentity(conf);

    this.shouldUseEncryption = conf.getBoolean(FS_DL_IMPL_SHOULD_USE_ENCRYPTION_CONFIG_NAME, false);
    if (this.shouldUseEncryption) {
        initialiseAesEncryption(conf);//from   www .  j a va  2  s .  c o  m
    }

    this.homeDirectory = conf.get(FS_DL_IMPL_HOME_DIRECTORY);

    if (homeDirectory == null)
        throw new IOException(
                "The Datalake requires a home directory to be configured in the fs.dl.impl.homeDirectory configuration variable. This is in the form /data_lake/dlxxxx");

    this.defaultEndpoint = conf.get(FS_DL_IMPL_DEFAULT_ENDPOINT);

    if (defaultEndpoint == null)
        throw new IOException(
                "The Datalake requires a default endpoint to be configured the fs.dl.impl.defaultEndpoint configuration variable. This is in the form /data_lake/dlxxxx");

    URI defaultEndpointURI = URI.create(defaultEndpoint);

    String authority = uri.getAuthority() == null ? defaultEndpointURI.getAuthority() : uri.getAuthority();

    this.baseUri = URI.create(uri.getScheme() + "://" + authority + this.homeDirectory);
    this.nnAddrs = resolveNNAddr();

    LOG.debug("Created kerberosIdentity " + kerberosIdentity + " for " + this.baseUri);

    boolean isHA = HAUtil.isClientFailoverConfigured(conf, this.baseUri);
    boolean isLogicalUri = isHA && HAUtil.isLogicalUri(conf, this.baseUri);
    // In non-HA or non-logical URI case, the code needs to call
    // getCanonicalUri() in order to handle the case where no port is
    // specified in the URI
    this.tokenServiceName = isLogicalUri ? HAUtil.buildTokenServiceForLogicalUri(this.baseUri, getScheme())
            : SecurityUtil.buildTokenService(getCanonicalUri());

    if (!isHA) {
        this.retryPolicy = RetryUtils.getDefaultRetryPolicy(conf,
                DFSConfigKeys.DFS_HTTP_CLIENT_RETRY_POLICY_ENABLED_KEY,
                DFSConfigKeys.DFS_HTTP_CLIENT_RETRY_POLICY_ENABLED_DEFAULT,
                DFSConfigKeys.DFS_HTTP_CLIENT_RETRY_POLICY_SPEC_KEY,
                DFSConfigKeys.DFS_HTTP_CLIENT_RETRY_POLICY_SPEC_DEFAULT, SafeModeException.class);
    } else {

        int maxFailoverAttempts = conf.getInt(DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY,
                DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_MAX_ATTEMPTS_DEFAULT);
        int maxRetryAttempts = conf.getInt(DFSConfigKeys.DFS_HTTP_CLIENT_RETRY_MAX_ATTEMPTS_KEY,
                DFSConfigKeys.DFS_HTTP_CLIENT_RETRY_MAX_ATTEMPTS_DEFAULT);
        int failoverSleepBaseMillis = conf.getInt(DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_SLEEPTIME_BASE_KEY,
                DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_SLEEPTIME_BASE_DEFAULT);
        int failoverSleepMaxMillis = conf.getInt(DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_SLEEPTIME_MAX_KEY,
                DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_SLEEPTIME_MAX_DEFAULT);

        this.retryPolicy = RetryPolicies.failoverOnNetworkException(RetryPolicies.TRY_ONCE_THEN_FAIL,
                maxFailoverAttempts, maxRetryAttempts, failoverSleepBaseMillis, failoverSleepMaxMillis);
    }

    this.workingDir = getHomeDirectory();
    //Delegation tokens don't work with httpfs
    this.canRefreshDelegationToken = false;
    this.disallowFallbackToInsecureCluster = !conf.getBoolean(
            CommonConfigurationKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_KEY,
            CommonConfigurationKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_DEFAULT);
    this.delegationToken = null;

    this.defaultFilePermissions = Short
            .decode(conf.get(FS_DL_IMPL_DEFAULT_FILE_PERMISSIONS, this.DEFAULT_FILE_PERMISSIONS));
    this.defaultUMask = Short.decode(conf.get(FS_DL_IMPL_DEFAULT_UMASK, this.DEFAULT_UMASK));

    this.transportScheme = conf.get(FS_DL_IMPL_TRANSPORT_SCHEME_CONFIG_NAME,
            FS_DL_IMPL_DEFAULT_TRANSPORT_SCHEME);

    if (!checkJCE())
        throw new IOException(JCE_ERROR);

}

---