Example usage for org.apache.hadoop.fs.s3a S3AFileSystem S3AFileSystem

Introduction

In this page you can find the example usage for org.apache.hadoop.fs.s3a S3AFileSystem S3AFileSystem.

Prototype

S3AFileSystem

Source Link

Usage

From source file:org.apache.flink.fs.s3hadoop.S3FileSystemFactory.java

License:Apache License

@Override
protected org.apache.hadoop.fs.FileSystem createHadoopFileSystem() {
    return new S3AFileSystem();
}

---

From source file:org.apache.sentry.tests.e2e.hive.fs.TestFSBase.java

License:Apache License

@BeforeClass
public static void setupTestStaticConfiguration() throws Exception {
    useSentryService = true;/*from   w ww. java2s.com*/
    AbstractTestWithStaticConfiguration.setupTestStaticConfiguration();
    AbstractTestWithStaticConfiguration.setupAdmin();
    authenticationType = System.getProperty(HiveConf.ConfVars.HIVE_SERVER2_AUTHENTICATION.varname);
    LOGGER.info("authenticationType = " + authenticationType);
    Assume.assumeThat(hiveServer2Type, equalTo(HiveServerFactory.HiveServer2Type.UnmanagedHiveServer2));

    if (DFSType.ClusterDFS.equals(DFS_TYPE) || DFSType.S3DFS.equals(DFS_TYPE)) {
        LOGGER.info("DFSType = " + DFS_TYPE);
    } else {
        LOGGER.warn("Incorrect DFSType " + DFS_TYPE);
        Assume.assumeTrue(false);
    }

    String storageUriStr = System.getProperty(TestFSContants.SENTRY_E2E_TEST_STORAGE_URI);
    if (!Strings.isNullOrEmpty(storageUriStr)) {
        storageUriStr = storageUriStr.toLowerCase();
        if (storageUriStr.startsWith("hdfs") || storageUriStr.startsWith("file")) {
            storageDFSType = DFSType.ClusterDFS;
        } else if (storageUriStr.startsWith("s3a")) {
            storageDFSType = DFSType.S3DFS;
        }
    }

    storageFileSystem = fileSystem;
    if (storageDFSType.equals(DFSType.ClusterDFS)) {
        // hdfs cluster
        defaultStorageUri = FileSystem.getDefaultUri(fileSystem.getConf());
    } else {
        // non-hdfs file sytem must specify defaultStorageUri
        if (!Strings.isNullOrEmpty(storageUriStr)) {
            defaultStorageUri = URI.create(storageUriStr);
        } else {
            LOGGER.warn(
                    "Skipping test: Unknown sentry.e2etest.storage.uri, " + "for example, s3a://bucketname");
            Assume.assumeTrue(false);
        }
        LOGGER.info("defaultStorageUri = " + defaultStorageUri.toString());

        if (storageDFSType.equals(DFSType.S3DFS)) {
            // currently defaultFS = s3a doesn't work for NN
            // needs to explicitly specify s3a's defaultUri
            String accessKey = System.getProperty(TestFSContants.S3A_ACCESS_KEY,
                    hiveServer.getProperty(TestFSContants.S3A_ACCESS_KEY));
            String secretKey = System.getProperty(TestFSContants.S3A_SECRET_KEY,
                    hiveServer.getProperty(TestFSContants.S3A_SECRET_KEY));
            LOGGER.info("accessKey = " + accessKey);
            LOGGER.info("secretKey = " + secretKey);
            Assume.assumeTrue(Strings.isNullOrEmpty(accessKey) == false);
            Assume.assumeTrue(Strings.isNullOrEmpty(secretKey) == false);

            Configuration conf = new Configuration();
            conf.set(TestFSContants.S3A_ACCESS_KEY, accessKey);
            conf.set(TestFSContants.S3A_SECRET_KEY, secretKey);
            storageFileSystem = new S3AFileSystem();
            Assume.assumeNotNull(storageFileSystem);
            LOGGER.info("Configuring S3DFS defaultStorageUri = " + defaultStorageUri.toString());
            storageFileSystem.initialize(defaultStorageUri, conf);
        }
        /*
        else if (DFS_TYPE.equals(DFSType.MSAZUREDFS)) {
        }
        */
    }
    // Get warehouse dir from hite-site.xml conf file
    StrWarehouseDirFromConfFile = hiveServer.getOrgWarehouseDir();
}

---