Example usage for org.apache.hadoop.fs FsUrlStreamHandlerFactory FsUrlStreamHandlerFactory

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FsUrlStreamHandlerFactory FsUrlStreamHandlerFactory.

Prototype

public FsUrlStreamHandlerFactory(Configuration conf) 

Source Link

Usage

From source file:com.thinkbiganalytics.kylo.protocol.hadoop.Handler.java

License:Apache License

/**
 * Sets the Hadoop configuration to use with this handler.
 *///from  w w w  .  j a  v  a  2 s .c o m
public static void setConfiguration(@Nonnull final Configuration conf) {
    FACTORY.set(new FsUrlStreamHandlerFactory(conf));
}

---

From source file:org.springframework.data.hadoop.configuration.ConfigurationFactoryBean.java

License:Apache License

public void afterPropertiesSet() throws Exception {
    internalConfig = createConfiguration(configuration);

    internalConfig.setClassLoader(beanClassLoader);
    if (resources != null) {
        for (Resource resource : resources) {
            internalConfig.addResource(resource.getURL());
        }//from   www  .j a va 2 s .  c o  m
    }

    ConfigurationUtils.addProperties(internalConfig, properties);

    // set hdfs / fs URI last to override all other properties
    if (StringUtils.hasText(fsUri)) {
        internalConfig.set("fs.default.name", fsUri.trim());
    }

    if (StringUtils.hasText(jtUri)) {
        internalConfig.set("mapred.job.tracker", jtUri.trim());
    }

    if (initialize) {
        internalConfig.size();
    }

    postProcessConfiguration(internalConfig);

    if (registerJvmUrl) {
        try {
            // force UGI init to prevent infinite loop - see SHDP-92
            UserGroupInformation.setConfiguration(internalConfig);
            URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory(getObject()));
            log.info("Registered HDFS URL stream handler");
        } catch (Error err) {
            log.warn("Cannot register Hadoop URL stream handler - one is already registered");
        }
    }
}

---

From source file:org.springframework.data.hadoop.fs.HdfsResourceLoaderLegacyTest.java

License:Apache License

@Test
public void testResolve() throws Exception {
    Resource resource = loader.getResource("/test");
    try {/*from w ww.j a v  a2  s.  com*/
        URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory(fs.getConf()));
    } catch (Error err) {
        //somebody already registered an URL handler...
        // get out
    }

    System.out.println(((HdfsResource) resource).getPath().makeQualified(fs));
    System.out.println(resource.getURI());

    resource = loader.getResource("test");
    System.out.println(resource.getURI());
    System.out.println(resource.getURL());
}

---

From source file:org.springframework.yarn.configuration.ConfigurationFactoryBean.java

License:Apache License

public void afterPropertiesSet() throws Exception {
    internalConfig = createConfiguration(configuration);

    internalConfig.setClassLoader(beanClassLoader);
    if (resources != null) {
        for (Resource resource : resources) {
            internalConfig.addResource(resource.getURL());
        }/*from  w  ww  .  java  2s.c  o  m*/
    }

    ConfigurationUtils.addProperties(internalConfig, properties);

    // set hdfs / fs URI last to override all other properties
    if (StringUtils.hasText(fsUri)) {
        internalConfig.set("fs.default.name", fsUri.trim());
    }

    if (StringUtils.hasText(rmAddress)) {
        internalConfig.set("yarn.resourcemanager.address", rmAddress.trim());
    }

    if (initialize) {
        internalConfig.size();
    }

    postProcessConfiguration(internalConfig);

    if (registerJvmUrl) {
        try {
            // force UGI init to prevent infinite loop - see SHDP-92
            UserGroupInformation.setConfiguration(internalConfig);
            URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory(getObject()));
            log.info("Registered HDFS URL stream handler");
        } catch (Error err) {
            log.warn("Cannot register Hadoop URL stream handler - one is already registered");
        }
    }
}

---