Example usage for org.apache.hadoop.fs.swift.snative SwiftNativeFileSystem SwiftNativeFileSystem

Introduction

In this page you can find the example usage for org.apache.hadoop.fs.swift.snative SwiftNativeFileSystem SwiftNativeFileSystem.

Prototype

public SwiftNativeFileSystem() 

Source Link

Document

Default constructor for Hadoop

Usage

From source file:org.apache.flink.fs.openstackhadoop.SwiftFileSystemFactory.java

License:Apache License

@Override
public FileSystem create(URI fsUri) throws IOException {
    LOG.debug("Creating swift file system (backed by a Hadoop native swift file system)");

    try {/* ww w  .jav a  2 s  .co m*/
        // -- (1) get the loaded Hadoop config (or fall back to one loaded from the classpath)

        org.apache.hadoop.conf.Configuration hadoopConfig = this.hadoopConfig;
        if (hadoopConfig == null) {
            if (flinkConfig != null) {
                LOG.debug("Loading Hadoop configuration for swift native file system");
                hadoopConfig = HadoopUtils.getHadoopConfiguration(flinkConfig);

                // hadoop.tmp.dir needs to be defined because it is used as buffer directory
                if (hadoopConfig.get("hadoop.tmp.dir") == null) {
                    String[] tmpDirPaths = ConfigurationUtils.parseTempDirectories(flinkConfig);
                    File tmpDir = new File(tmpDirPaths[0], "hadoop-" + System.getProperty("user.name"));
                    hadoopConfig.set("hadoop.tmp.dir", tmpDir.getPath());
                }

                // add additional config entries from the Flink config to the Hadoop config
                for (String key : flinkConfig.keySet()) {
                    if (key.startsWith(CONFIG_PREFIX)) {
                        String value = flinkConfig.getString(key, null);
                        String newKey = "fs.swift." + key.substring(CONFIG_PREFIX.length());
                        hadoopConfig.set(newKey, value);

                        LOG.debug("Adding Flink config entry for {} as {}={} to Hadoop config for "
                                + "Swift native File System", key, newKey, value);
                    }
                }

                this.hadoopConfig = hadoopConfig;
            } else {
                LOG.warn("The factory has not been configured prior to loading the Swift native file system."
                        + " Using Hadoop configuration from the classpath.");

                hadoopConfig = new org.apache.hadoop.conf.Configuration();
                this.hadoopConfig = hadoopConfig;
            }
        }

        // -- (2) Instantiate the Hadoop file system class for that scheme

        final String scheme = fsUri.getScheme();
        final String authority = fsUri.getAuthority();

        if (scheme == null && authority == null) {
            fsUri = org.apache.hadoop.fs.FileSystem.getDefaultUri(hadoopConfig);
        } else if (scheme != null && authority == null) {
            URI defaultUri = org.apache.hadoop.fs.FileSystem.getDefaultUri(hadoopConfig);
            if (scheme.equals(defaultUri.getScheme()) && defaultUri.getAuthority() != null) {
                fsUri = defaultUri;
            }
        }

        LOG.debug("Using scheme {} for swift file system backing the Swift Native File System", fsUri);

        final SwiftNativeFileSystem fs = new SwiftNativeFileSystem();
        fs.initialize(fsUri, hadoopConfig);

        return new HadoopFileSystem(fs);
    } catch (IOException e) {
        throw e;
    } catch (Exception e) {
        throw new IOException(e.getMessage(), e);
    }
}

---