Example usage for org.apache.hadoop.fs.swift.snative SwiftNativeFileSystem initialize

Introduction

In this page you can find the example usage for org.apache.hadoop.fs.swift.snative SwiftNativeFileSystem initialize.

Prototype

@Override
public void initialize(URI fsuri, Configuration conf) throws IOException 

Source Link

Document

default class initialization

Usage

From source file:org.apache.flink.fs.openstackhadoop.SwiftFileSystemFactory.java

License:Apache License

@Override
public FileSystem create(URI fsUri) throws IOException {
    LOG.debug("Creating swift file system (backed by a Hadoop native swift file system)");

    try {//from w ww.  j  ava 2s  .  c o  m
        // -- (1) get the loaded Hadoop config (or fall back to one loaded from the classpath)

        org.apache.hadoop.conf.Configuration hadoopConfig = this.hadoopConfig;
        if (hadoopConfig == null) {
            if (flinkConfig != null) {
                LOG.debug("Loading Hadoop configuration for swift native file system");
                hadoopConfig = HadoopUtils.getHadoopConfiguration(flinkConfig);

                // hadoop.tmp.dir needs to be defined because it is used as buffer directory
                if (hadoopConfig.get("hadoop.tmp.dir") == null) {
                    String[] tmpDirPaths = ConfigurationUtils.parseTempDirectories(flinkConfig);
                    File tmpDir = new File(tmpDirPaths[0], "hadoop-" + System.getProperty("user.name"));
                    hadoopConfig.set("hadoop.tmp.dir", tmpDir.getPath());
                }

                // add additional config entries from the Flink config to the Hadoop config
                for (String key : flinkConfig.keySet()) {
                    if (key.startsWith(CONFIG_PREFIX)) {
                        String value = flinkConfig.getString(key, null);
                        String newKey = "fs.swift." + key.substring(CONFIG_PREFIX.length());
                        hadoopConfig.set(newKey, value);

                        LOG.debug("Adding Flink config entry for {} as {}={} to Hadoop config for "
                                + "Swift native File System", key, newKey, value);
                    }
                }

                this.hadoopConfig = hadoopConfig;
            } else {
                LOG.warn("The factory has not been configured prior to loading the Swift native file system."
                        + " Using Hadoop configuration from the classpath.");

                hadoopConfig = new org.apache.hadoop.conf.Configuration();
                this.hadoopConfig = hadoopConfig;
            }
        }

        // -- (2) Instantiate the Hadoop file system class for that scheme

        final String scheme = fsUri.getScheme();
        final String authority = fsUri.getAuthority();

        if (scheme == null && authority == null) {
            fsUri = org.apache.hadoop.fs.FileSystem.getDefaultUri(hadoopConfig);
        } else if (scheme != null && authority == null) {
            URI defaultUri = org.apache.hadoop.fs.FileSystem.getDefaultUri(hadoopConfig);
            if (scheme.equals(defaultUri.getScheme()) && defaultUri.getAuthority() != null) {
                fsUri = defaultUri;
            }
        }

        LOG.debug("Using scheme {} for swift file system backing the Swift Native File System", fsUri);

        final SwiftNativeFileSystem fs = new SwiftNativeFileSystem();
        fs.initialize(fsUri, hadoopConfig);

        return new HadoopFileSystem(fs);
    } catch (IOException e) {
        throw e;
    } catch (Exception e) {
        throw new IOException(e.getMessage(), e);
    }
}

---