Example usage for org.apache.hadoop.io.retry RetryPolicies TRY_ONCE_THEN

Introduction

In this page you can find the example usage for org.apache.hadoop.io.retry RetryPolicies TRY_ONCE_THEN_FAIL.

Prototype

RetryPolicy TRY_ONCE_THEN_FAIL

To view the source code for org.apache.hadoop.io.retry RetryPolicies TRY_ONCE_THEN_FAIL.

Click Source Link

Document

Try once, and fail by re-throwing the exception.

Usage

From source file:com.aliyun.fs.oss.blk.OssFileSystem.java

License:Apache License

private static FileSystemStore createDefaultStore(Configuration conf) {
    FileSystemStore store = new JetOssFileSystemStore();

    RetryPolicy basePolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
            conf.getInt("fs.oss.maxRetries", 4), conf.getLong("fs.oss.sleepTimeSeconds", 10), TimeUnit.SECONDS);
    Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap = new HashMap<Class<? extends Exception>, RetryPolicy>();
    exceptionToPolicyMap.put(IOException.class, basePolicy);
    exceptionToPolicyMap.put(OssException.class, basePolicy);

    RetryPolicy methodPolicy = RetryPolicies.retryByException(RetryPolicies.TRY_ONCE_THEN_FAIL,
            exceptionToPolicyMap);/*from ww  w. j a va 2 s . co m*/
    Map<String, RetryPolicy> methodNameToPolicyMap = new HashMap<String, RetryPolicy>();
    methodNameToPolicyMap.put("storeBlock", methodPolicy);
    methodNameToPolicyMap.put("retrieveBlock", methodPolicy);

    return (FileSystemStore) RetryProxy.create(FileSystemStore.class, store, methodNameToPolicyMap);
}

From source file:com.aliyun.fs.oss.nat.NativeOssFileSystem.java

License:Apache License

private static NativeFileSystemStore createDefaultStore(Configuration conf) {
    NativeFileSystemStore store = new JetOssNativeFileSystemStore();

    RetryPolicy basePolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
            conf.getInt("fs.oss.maxRetries", 4), conf.getLong("fs.oss.sleepTimeSeconds", 10), TimeUnit.SECONDS);
    Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap = new HashMap<Class<? extends Exception>, RetryPolicy>();
    // for reflection invoke.
    exceptionToPolicyMap.put(InvocationTargetException.class, basePolicy);
    exceptionToPolicyMap.put(IOException.class, basePolicy);
    exceptionToPolicyMap.put(OssException.class, basePolicy);

    RetryPolicy methodPolicy = RetryPolicies.retryByException(RetryPolicies.TRY_ONCE_THEN_FAIL,
            exceptionToPolicyMap);/*from  w  w  w  .  j  a  v a 2 s. c  om*/
    Map<String, RetryPolicy> methodNameToPolicyMap = new HashMap<String, RetryPolicy>();
    methodNameToPolicyMap.put("storeFile", methodPolicy);
    methodNameToPolicyMap.put("storeFiles", methodPolicy);
    methodNameToPolicyMap.put("storeEmptyFile", methodPolicy);
    methodNameToPolicyMap.put("retrieveMetadata", methodPolicy);
    methodNameToPolicyMap.put("retrieve", methodPolicy);
    methodNameToPolicyMap.put("purge", methodPolicy);
    methodNameToPolicyMap.put("dump", methodPolicy);
    methodNameToPolicyMap.put("doesObjectExist", methodPolicy);
    methodNameToPolicyMap.put("copy", methodPolicy);
    methodNameToPolicyMap.put("list", methodPolicy);
    methodNameToPolicyMap.put("delete", methodPolicy);

    return (NativeFileSystemStore) RetryProxy.create(NativeFileSystemStore.class, store, methodNameToPolicyMap);
}

From source file:com.bigstep.datalake.DLFileSystem.java

License:Apache License

@Override
public synchronized void initialize(URI uri, Configuration conf) throws IOException {
    super.initialize(uri, conf);

    uri = selectDatalakeEndpointURI(uri, conf);

    /* set user pattern based on configuration file */
    UserParam.setUserPattern(conf.get(DFSConfigKeys.DFS_WEBHDFS_USER_PATTERN_KEY,
            DFSConfigKeys.DFS_WEBHDFS_USER_PATTERN_DEFAULT));

    kerberosIdentity = initialiseKerberosIdentity(conf);

    this.shouldUseEncryption = conf.getBoolean(FS_DL_IMPL_SHOULD_USE_ENCRYPTION_CONFIG_NAME, false);
    if (this.shouldUseEncryption) {
        initialiseAesEncryption(conf);/*from  w w w.j  av a  2 s  .c  om*/
    }

    this.homeDirectory = conf.get(FS_DL_IMPL_HOME_DIRECTORY);

    if (homeDirectory == null)
        throw new IOException(
                "The Datalake requires a home directory to be configured in the fs.dl.impl.homeDirectory configuration variable. This is in the form /data_lake/dlxxxx");

    this.defaultEndpoint = conf.get(FS_DL_IMPL_DEFAULT_ENDPOINT);

    if (defaultEndpoint == null)
        throw new IOException(
                "The Datalake requires a default endpoint to be configured the fs.dl.impl.defaultEndpoint configuration variable. This is in the form /data_lake/dlxxxx");

    URI defaultEndpointURI = URI.create(defaultEndpoint);

    String authority = uri.getAuthority() == null ? defaultEndpointURI.getAuthority() : uri.getAuthority();

    this.baseUri = URI.create(uri.getScheme() + "://" + authority + this.homeDirectory);
    this.nnAddrs = resolveNNAddr();

    LOG.debug("Created kerberosIdentity " + kerberosIdentity + " for " + this.baseUri);

    boolean isHA = HAUtil.isClientFailoverConfigured(conf, this.baseUri);
    boolean isLogicalUri = isHA && HAUtil.isLogicalUri(conf, this.baseUri);
    // In non-HA or non-logical URI case, the code needs to call
    // getCanonicalUri() in order to handle the case where no port is
    // specified in the URI
    this.tokenServiceName = isLogicalUri ? HAUtil.buildTokenServiceForLogicalUri(this.baseUri, getScheme())
            : SecurityUtil.buildTokenService(getCanonicalUri());

    if (!isHA) {
        this.retryPolicy = RetryUtils.getDefaultRetryPolicy(conf,
                DFSConfigKeys.DFS_HTTP_CLIENT_RETRY_POLICY_ENABLED_KEY,
                DFSConfigKeys.DFS_HTTP_CLIENT_RETRY_POLICY_ENABLED_DEFAULT,
                DFSConfigKeys.DFS_HTTP_CLIENT_RETRY_POLICY_SPEC_KEY,
                DFSConfigKeys.DFS_HTTP_CLIENT_RETRY_POLICY_SPEC_DEFAULT, SafeModeException.class);
    } else {

        int maxFailoverAttempts = conf.getInt(DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY,
                DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_MAX_ATTEMPTS_DEFAULT);
        int maxRetryAttempts = conf.getInt(DFSConfigKeys.DFS_HTTP_CLIENT_RETRY_MAX_ATTEMPTS_KEY,
                DFSConfigKeys.DFS_HTTP_CLIENT_RETRY_MAX_ATTEMPTS_DEFAULT);
        int failoverSleepBaseMillis = conf.getInt(DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_SLEEPTIME_BASE_KEY,
                DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_SLEEPTIME_BASE_DEFAULT);
        int failoverSleepMaxMillis = conf.getInt(DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_SLEEPTIME_MAX_KEY,
                DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_SLEEPTIME_MAX_DEFAULT);

        this.retryPolicy = RetryPolicies.failoverOnNetworkException(RetryPolicies.TRY_ONCE_THEN_FAIL,
                maxFailoverAttempts, maxRetryAttempts, failoverSleepBaseMillis, failoverSleepMaxMillis);
    }

    this.workingDir = getHomeDirectory();
    //Delegation tokens don't work with httpfs
    this.canRefreshDelegationToken = false;
    this.disallowFallbackToInsecureCluster = !conf.getBoolean(
            CommonConfigurationKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_KEY,
            CommonConfigurationKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_DEFAULT);
    this.delegationToken = null;

    this.defaultFilePermissions = Short
            .decode(conf.get(FS_DL_IMPL_DEFAULT_FILE_PERMISSIONS, this.DEFAULT_FILE_PERMISSIONS));
    this.defaultUMask = Short.decode(conf.get(FS_DL_IMPL_DEFAULT_UMASK, this.DEFAULT_UMASK));

    this.transportScheme = conf.get(FS_DL_IMPL_TRANSPORT_SCHEME_CONFIG_NAME,
            FS_DL_IMPL_DEFAULT_TRANSPORT_SCHEME);

    if (!checkJCE())
        throw new IOException(JCE_ERROR);

}

From source file:com.proofpoint.zookeeper.ZookeeperClient.java

License:Apache License

private static RetryPolicy newRetryPolicy(ZookeeperClientConfig config) {
    RetryPolicy policy = RetryPolicies.exponentialBackoffRetry(config.getMaxConnectionLossRetries(),
            config.getConnectionLossSleepInMs(), TimeUnit.MILLISECONDS);
    Map<Class<? extends Exception>, RetryPolicy> map = new HashMap<Class<? extends Exception>, RetryPolicy>();
    map.put(KeeperException.ConnectionLossException.class, policy);
    return RetryPolicies.retryByException(RetryPolicies.TRY_ONCE_THEN_FAIL, map);
}

From source file:com.quixey.hadoop.fs.oss.OSSFileSystem.java

License:Apache License

private static FileSystemStore createDefaultStore(Configuration conf) {
    FileSystemStore store = new CloudOSSFileSystemStore();

    RetryPolicy basePolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
            conf.getInt(OSS_MAX_RETRIES_PROPERTY, 4), conf.getLong(OSS_SLEEP_TIME_SECONDS_PROPERTY, 10),
            TimeUnit.SECONDS);/*from w w  w.jav a2 s  .c o m*/
    Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap = new HashMap<>();
    exceptionToPolicyMap.put(IOException.class, basePolicy);

    RetryPolicy methodPolicy = RetryPolicies.retryByException(RetryPolicies.TRY_ONCE_THEN_FAIL,
            exceptionToPolicyMap);
    Map<String, RetryPolicy> methodNameToPolicyMap = new HashMap<>();
    methodNameToPolicyMap.put("storeFile", methodPolicy);
    methodNameToPolicyMap.put("rename", methodPolicy);

    return (FileSystemStore) RetryProxy.create(FileSystemStore.class, store, methodNameToPolicyMap);
}

From source file:org.apache.druid.indexer.JobHelperTest.java

License:Apache License

@Test
public void testEvilZip() throws IOException {
    final File tmpDir = temporaryFolder.newFolder("testEvilZip");

    final File evilResult = new File("/tmp/evil.txt");
    Files.deleteIfExists(evilResult.toPath());

    File evilZip = new File(tmpDir, "evil.zip");
    Files.deleteIfExists(evilZip.toPath());
    CompressionUtilsTest.makeEvilZip(evilZip);

    try {//w  ww .j  av  a 2 s.com
        JobHelper.unzipNoGuava(new Path(evilZip.getCanonicalPath()), new Configuration(), tmpDir,
                new Progressable() {
                    @Override
                    public void progress() {

                    }
                }, RetryPolicies.TRY_ONCE_THEN_FAIL);
    } catch (ISE ise) {
        Assert.assertTrue(ise.getMessage().contains("does not start with outDir"));
        Assert.assertFalse("Zip exploit triggered, /tmp/evil.txt was written.", evilResult.exists());
        return;
    }
    Assert.fail("Exception was not thrown for malicious zip file");
}

Example usage for org.apache.hadoop.io.retry RetryPolicies TRY_ONCE_THEN_FAIL

Introduction

Prototype

Document

Usage