Example usage for org.apache.hadoop.fs AbstractFileSystem get

Introduction

In this page you can find the example usage for org.apache.hadoop.fs AbstractFileSystem get.

Prototype

public static AbstractFileSystem get(final URI uri, final Configuration conf)
        throws UnsupportedFileSystemException 

Source Link

Document

The main factory method for creating a file system.

Usage

From source file:org.apache.ignite.igfs.hadoop.v2.IgfsHadoopFileSystem.java

License:Apache License

/**
 * @param name URI passed to constructor.
 * @param cfg Configuration passed to constructor.
 * @throws IOException If initialization failed.
 *//*from   w w  w  . java 2 s  .  c o  m*/
private void initialize(URI name, Configuration cfg) throws IOException {
    enterBusy();

    try {
        if (rmtClient != null)
            throw new IOException("File system is already initialized: " + rmtClient);

        A.notNull(name, "name");
        A.notNull(cfg, "cfg");

        if (!IGFS_SCHEME.equals(name.getScheme()))
            throw new IOException("Illegal file system URI [expected=" + IGFS_SCHEME
                    + "://[name]/[optional_path], actual=" + name + ']');

        uriAuthority = name.getAuthority();

        // Override sequential reads before prefetch if needed.
        seqReadsBeforePrefetch = parameter(cfg, PARAM_IGFS_SEQ_READS_BEFORE_PREFETCH, uriAuthority, 0);

        if (seqReadsBeforePrefetch > 0)
            seqReadsBeforePrefetchOverride = true;

        // In GG replication factor is controlled by data cache affinity.
        // We use replication factor to force the whole file to be stored on local node.
        dfltReplication = (short) cfg.getInt("dfs.replication", 3);

        // Get file colocation control flag.
        colocateFileWrites = parameter(cfg, PARAM_IGFS_COLOCATED_WRITES, uriAuthority, false);
        preferLocFileWrites = cfg.getBoolean(PARAM_IGFS_PREFER_LOCAL_WRITES, false);

        // Get log directory.
        String logDirCfg = parameter(cfg, PARAM_IGFS_LOG_DIR, uriAuthority, DFLT_IGFS_LOG_DIR);

        File logDirFile = U.resolveIgnitePath(logDirCfg);

        String logDir = logDirFile != null ? logDirFile.getAbsolutePath() : null;

        rmtClient = new IgfsHadoopWrapper(uriAuthority, logDir, cfg, LOG);

        // Handshake.
        IgfsHandshakeResponse handshake = rmtClient.handshake(logDir);

        grpBlockSize = handshake.blockSize();

        IgfsPaths paths = handshake.secondaryPaths();

        Boolean logEnabled = parameter(cfg, PARAM_IGFS_LOG_ENABLED, uriAuthority, false);

        if (handshake.sampling() != null ? handshake.sampling() : logEnabled) {
            // Initiate client logger.
            if (logDir == null)
                throw new IOException("Failed to resolve log directory: " + logDirCfg);

            Integer batchSize = parameter(cfg, PARAM_IGFS_LOG_BATCH_SIZE, uriAuthority,
                    DFLT_IGFS_LOG_BATCH_SIZE);

            clientLog = IgfsLogger.logger(uriAuthority, handshake.igfsName(), logDir, batchSize);
        } else
            clientLog = IgfsLogger.disabledLogger();

        modeRslvr = new IgfsModeResolver(paths.defaultMode(), paths.pathModes());

        boolean initSecondary = paths.defaultMode() == PROXY;

        if (paths.pathModes() != null) {
            for (T2<IgfsPath, IgfsMode> pathMode : paths.pathModes()) {
                IgfsMode mode = pathMode.getValue();

                initSecondary |= mode == PROXY;
            }
        }

        if (initSecondary) {
            Map<String, String> props = paths.properties();

            String secUri = props.get(IgfsHadoopFileSystemWrapper.SECONDARY_FS_URI);
            String secConfPath = props.get(IgfsHadoopFileSystemWrapper.SECONDARY_FS_CONFIG_PATH);

            if (secConfPath == null)
                throw new IOException("Failed to connect to the secondary file system because configuration "
                        + "path is not provided.");

            if (secUri == null)
                throw new IOException(
                        "Failed to connect to the secondary file system because URI is not " + "provided.");

            if (secConfPath == null)
                throw new IOException("Failed to connect to the secondary file system because configuration "
                        + "path is not provided.");

            if (secUri == null)
                throw new IOException(
                        "Failed to connect to the secondary file system because URI is not " + "provided.");

            try {
                secondaryUri = new URI(secUri);

                URL secondaryCfgUrl = U.resolveIgniteUrl(secConfPath);

                if (secondaryCfgUrl == null)
                    throw new IOException("Failed to resolve secondary file system config URL: " + secConfPath);

                Configuration conf = new Configuration();

                conf.addResource(secondaryCfgUrl);

                String prop = String.format("fs.%s.impl.disable.cache", secondaryUri.getScheme());

                conf.setBoolean(prop, true);

                secondaryFs = AbstractFileSystem.get(secondaryUri, conf);
            } catch (URISyntaxException ignore) {
                throw new IOException("Failed to resolve secondary file system URI: " + secUri);
            } catch (IOException e) {
                throw new IOException("Failed to connect to the secondary file system: " + secUri, e);
            }
        }
    } finally {
        leaveBusy();
    }
}

---

From source file:org.apache.ignite.igfs.HadoopIgfs20FileSystemAbstractSelfTest.java

License:Apache License

/** {@inheritDoc} */
@Override//from   w  w w. j  av a2  s .  com
protected void beforeTest() throws Exception {
    primaryFsUri = new URI(primaryFileSystemUriPath());

    primaryFsCfg = new Configuration();

    primaryFsCfg.addResource(U.resolveIgniteUrl(primaryFileSystemConfigPath()));

    fs = AbstractFileSystem.get(primaryFsUri, primaryFsCfg);

    barrier = new CyclicBarrier(THREAD_CNT);
}

---

From source file:org.apache.ignite.internal.processors.hadoop.GridHadoopSortingTest.java

License:Apache License

/**
 * @throws Exception If failed./*from  ww w .java 2s .  c  om*/
 */
public void testSortSimple() throws Exception {
    // Generate test data.
    Job job = Job.getInstance();

    job.setInputFormatClass(InFormat.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(NullWritable.class);

    job.setMapperClass(Mapper.class);
    job.setNumReduceTasks(0);

    setupFileSystems(job.getConfiguration());

    FileOutputFormat.setOutputPath(job, new Path(igfsScheme() + PATH_INPUT));

    X.printerrln("Data generation started.");

    grid(0).hadoop().submit(new GridHadoopJobId(UUID.randomUUID(), 1), createJobInfo(job.getConfiguration()))
            .get(180000);

    X.printerrln("Data generation complete.");

    // Run main map-reduce job.
    job = Job.getInstance();

    setupFileSystems(job.getConfiguration());

    job.getConfiguration().set(CommonConfigurationKeys.IO_SERIALIZATIONS_KEY,
            JavaSerialization.class.getName() + "," + WritableSerialization.class.getName());

    FileInputFormat.setInputPaths(job, new Path(igfsScheme() + PATH_INPUT));
    FileOutputFormat.setOutputPath(job, new Path(igfsScheme() + PATH_OUTPUT));

    job.setSortComparatorClass(JavaSerializationComparator.class);

    job.setMapperClass(MyMapper.class);
    job.setReducerClass(MyReducer.class);

    job.setNumReduceTasks(2);

    job.setMapOutputKeyClass(UUID.class);
    job.setMapOutputValueClass(NullWritable.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(NullWritable.class);

    X.printerrln("Job started.");

    grid(0).hadoop().submit(new GridHadoopJobId(UUID.randomUUID(), 2), createJobInfo(job.getConfiguration()))
            .get(180000);

    X.printerrln("Job complete.");

    // Check result.
    Path outDir = new Path(igfsScheme() + PATH_OUTPUT);

    AbstractFileSystem fs = AbstractFileSystem.get(new URI(igfsScheme()), job.getConfiguration());

    for (FileStatus file : fs.listStatus(outDir)) {
        X.printerrln("__ file: " + file);

        if (file.getLen() == 0)
            continue;

        FSDataInputStream in = fs.open(file.getPath());

        Scanner sc = new Scanner(in);

        UUID prev = null;

        while (sc.hasNextLine()) {
            UUID next = UUID.fromString(sc.nextLine());

            //                X.printerrln("___ check: " + next);

            if (prev != null)
                assertTrue(prev.compareTo(next) < 0);

            prev = next;
        }
    }
}

---

From source file:org.apache.ignite.internal.processors.hadoop.HadoopSortingTest.java

License:Apache License

/**
 * @throws Exception If failed./*from   www .j  a v a 2 s .c o m*/
 */
public void testSortSimple() throws Exception {
    // Generate test data.
    Job job = Job.getInstance();

    job.setInputFormatClass(InFormat.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(NullWritable.class);

    job.setMapperClass(Mapper.class);
    job.setNumReduceTasks(0);

    setupFileSystems(job.getConfiguration());

    FileOutputFormat.setOutputPath(job, new Path(igfsScheme() + PATH_INPUT));

    X.printerrln("Data generation started.");

    grid(0).hadoop().submit(new HadoopJobId(UUID.randomUUID(), 1), createJobInfo(job.getConfiguration()))
            .get(180000);

    X.printerrln("Data generation complete.");

    // Run main map-reduce job.
    job = Job.getInstance();

    setupFileSystems(job.getConfiguration());

    job.getConfiguration().set(CommonConfigurationKeys.IO_SERIALIZATIONS_KEY,
            JavaSerialization.class.getName() + "," + WritableSerialization.class.getName());

    FileInputFormat.setInputPaths(job, new Path(igfsScheme() + PATH_INPUT));
    FileOutputFormat.setOutputPath(job, new Path(igfsScheme() + PATH_OUTPUT));

    job.setSortComparatorClass(JavaSerializationComparator.class);

    job.setMapperClass(MyMapper.class);
    job.setReducerClass(MyReducer.class);

    job.setNumReduceTasks(2);

    job.setMapOutputKeyClass(UUID.class);
    job.setMapOutputValueClass(NullWritable.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(NullWritable.class);

    X.printerrln("Job started.");

    grid(0).hadoop().submit(new HadoopJobId(UUID.randomUUID(), 2), createJobInfo(job.getConfiguration()))
            .get(180000);

    X.printerrln("Job complete.");

    // Check result.
    Path outDir = new Path(igfsScheme() + PATH_OUTPUT);

    AbstractFileSystem fs = AbstractFileSystem.get(new URI(igfsScheme()), job.getConfiguration());

    for (FileStatus file : fs.listStatus(outDir)) {
        X.printerrln("__ file: " + file);

        if (file.getLen() == 0)
            continue;

        FSDataInputStream in = fs.open(file.getPath());

        Scanner sc = new Scanner(in);

        UUID prev = null;

        while (sc.hasNextLine()) {
            UUID next = UUID.fromString(sc.nextLine());

            //                X.printerrln("___ check: " + next);

            if (prev != null)
                assertTrue(prev.compareTo(next) < 0);

            prev = next;
        }
    }
}

---

From source file:org.apache.ignite.internal.processors.hadoop.impl.igfs.HadoopIgfs20FileSystemAbstractSelfTest.java

License:Apache License

/** {@inheritDoc} */
@Override/*from  www  . ja v a 2 s . c  o m*/
protected void beforeTest() throws Exception {
    primaryFsUri = new URI(primaryFileSystemUriPath());

    primaryFsCfg = new Configuration();

    primaryFsCfg.addResource(U.resolveIgniteUrl(primaryFileSystemConfigPath()));

    UserGroupInformation ugi = UserGroupInformation.getBestUGI(null, getClientFsUser());

    // Create Fs on behalf of the client user:
    ugi.doAs(new PrivilegedExceptionAction<Object>() {
        @Override
        public Object run() throws Exception {
            fs = AbstractFileSystem.get(primaryFsUri, primaryFsCfg);

            return null;
        }
    });

    barrier = new CyclicBarrier(THREAD_CNT);
}

---

From source file:org.apache.ignite.internal.processors.hadoop.SecondaryFileSystemProvider.java

License:Apache License

/**
 * @return {@link org.apache.hadoop.fs.AbstractFileSystem} instance for this secondary Fs.
 * @throws IOException/*from   ww w .j a  v a2 s  .co  m*/
 */
public AbstractFileSystem createAbstractFileSystem() throws IOException {
    return AbstractFileSystem.get(uri, cfg);
}

---

From source file:org.elasticsearch.repositories.hdfs.HdfsBlobStoreContainerTests.java

License:Apache License

@SuppressForbidden(reason = "lesser of two evils (the other being a bunch of JNI/classloader nightmares)")
private FileContext createContext(URI uri) {
    // mirrors HdfsRepository.java behaviour
    Configuration cfg = new Configuration(true);
    cfg.setClassLoader(HdfsRepository.class.getClassLoader());
    cfg.reloadConfiguration();//from  ww w. ja  va  2 s  .c o m

    Constructor<?> ctor;
    Subject subject;

    try {
        Class<?> clazz = Class.forName("org.apache.hadoop.security.User");
        ctor = clazz.getConstructor(String.class);
        ctor.setAccessible(true);
    } catch (ClassNotFoundException | NoSuchMethodException e) {
        throw new RuntimeException(e);
    }

    try {
        Principal principal = (Principal) ctor.newInstance(System.getProperty("user.name"));
        subject = new Subject(false, Collections.singleton(principal), Collections.emptySet(),
                Collections.emptySet());
    } catch (InstantiationException | IllegalAccessException | InvocationTargetException e) {
        throw new RuntimeException(e);
    }

    // disable file system cache
    cfg.setBoolean("fs.hdfs.impl.disable.cache", true);

    // set file system to TestingFs to avoid a bunch of security
    // checks, similar to what is done in HdfsTests.java
    cfg.set("fs.AbstractFileSystem." + uri.getScheme() + ".impl", TestingFs.class.getName());

    // create the FileContext with our user
    return Subject.doAs(subject, new PrivilegedAction<FileContext>() {
        @Override
        public FileContext run() {
            try {
                TestingFs fs = (TestingFs) AbstractFileSystem.get(uri, cfg);
                return FileContext.getFileContext(fs, cfg);
            } catch (UnsupportedFileSystemException e) {
                throw new RuntimeException(e);
            }
        }
    });
}

---

From source file:org.gridgain.grid.ggfs.GridGgfsHadoop20FileSystemAbstractSelfTest.java

License:Open Source License

/** {@inheritDoc} */
@Override//from www  .j  a v a 2  s . c om
protected void beforeTest() throws Exception {
    primaryFsUri = new URI(primaryFileSystemUriPath());

    primaryFsCfg = new Configuration();

    primaryFsCfg.addResource(U.resolveGridGainUrl(primaryFileSystemConfigPath()));

    fs = AbstractFileSystem.get(primaryFsUri, primaryFsCfg);

    barrier = new CyclicBarrier(THREAD_CNT);
}

---

From source file:org.gridgain.grid.ggfs.hadoop.v2.GridGgfsHadoopFileSystem.java

License:Open Source License

/**
 * @param name URI passed to constructor.
 * @param cfg Configuration passed to constructor.
 * @throws IOException If initialization failed.
 *///from ww w . jav  a 2s  .  c om
private void initialize(URI name, Configuration cfg) throws IOException {
    enterBusy();

    try {
        if (rmtClient != null)
            throw new IOException("File system is already initialized: " + rmtClient);

        A.notNull(name, "name");
        A.notNull(cfg, "cfg");

        if (!GGFS_SCHEME.equals(name.getScheme()))
            throw new IOException("Illegal file system URI [expected=" + GGFS_SCHEME
                    + "://[name]/[optional_path], actual=" + name + ']');

        uriAuthority = name.getAuthority();

        // Override sequential reads before prefetch if needed.
        seqReadsBeforePrefetch = parameter(cfg, PARAM_GGFS_SEQ_READS_BEFORE_PREFETCH, uriAuthority, 0);

        if (seqReadsBeforePrefetch > 0)
            seqReadsBeforePrefetchOverride = true;

        // In GG replication factor is controlled by data cache affinity.
        // We use replication factor to force the whole file to be stored on local node.
        dfltReplication = (short) cfg.getInt("dfs.replication", 3);

        // Get file colocation control flag.
        colocateFileWrites = parameter(cfg, PARAM_GGFS_COLOCATED_WRITES, uriAuthority, false);
        preferLocFileWrites = cfg.getBoolean(PARAM_GGFS_PREFER_LOCAL_WRITES, false);

        // Get log directory.
        String logDirCfg = parameter(cfg, PARAM_GGFS_LOG_DIR, uriAuthority, DFLT_GGFS_LOG_DIR);

        File logDirFile = U.resolveGridGainPath(logDirCfg);

        String logDir = logDirFile != null ? logDirFile.getAbsolutePath() : null;

        rmtClient = new GridGgfsHadoopWrapper(uriAuthority, logDir, cfg, LOG);

        // Handshake.
        GridGgfsHandshakeResponse handshake = rmtClient.handshake(logDir);

        grpBlockSize = handshake.blockSize();

        GridGgfsPaths paths = handshake.secondaryPaths();

        Boolean logEnabled = parameter(cfg, PARAM_GGFS_LOG_ENABLED, uriAuthority, false);

        if (handshake.sampling() != null ? handshake.sampling() : logEnabled) {
            // Initiate client logger.
            if (logDir == null)
                throw new IOException("Failed to resolve log directory: " + logDirCfg);

            Integer batchSize = parameter(cfg, PARAM_GGFS_LOG_BATCH_SIZE, uriAuthority,
                    DFLT_GGFS_LOG_BATCH_SIZE);

            clientLog = GridGgfsLogger.logger(uriAuthority, handshake.ggfsName(), logDir, batchSize);
        } else
            clientLog = GridGgfsLogger.disabledLogger();

        modeRslvr = new GridGgfsModeResolver(paths.defaultMode(), paths.pathModes());

        boolean initSecondary = paths.defaultMode() == PROXY;

        if (paths.pathModes() != null) {
            for (T2<GridGgfsPath, GridGgfsMode> pathMode : paths.pathModes()) {
                GridGgfsMode mode = pathMode.getValue();

                initSecondary |= mode == PROXY;
            }
        }

        if (initSecondary) {
            Map<String, String> props = paths.properties();

            String secUri = props.get(GridGgfsHadoopFileSystemWrapper.SECONDARY_FS_URI);
            String secConfPath = props.get(GridGgfsHadoopFileSystemWrapper.SECONDARY_FS_CONFIG_PATH);

            if (secConfPath == null)
                throw new IOException("Failed to connect to the secondary file system because configuration "
                        + "path is not provided.");

            if (secUri == null)
                throw new IOException(
                        "Failed to connect to the secondary file system because URI is not " + "provided.");

            if (secConfPath == null)
                throw new IOException("Failed to connect to the secondary file system because configuration "
                        + "path is not provided.");

            if (secUri == null)
                throw new IOException(
                        "Failed to connect to the secondary file system because URI is not " + "provided.");

            try {
                secondaryUri = new URI(secUri);

                URL secondaryCfgUrl = U.resolveGridGainUrl(secConfPath);

                if (secondaryCfgUrl == null)
                    throw new IOException("Failed to resolve secondary file system config URL: " + secConfPath);

                Configuration conf = new Configuration();

                conf.addResource(secondaryCfgUrl);

                String prop = String.format("fs.%s.impl.disable.cache", secondaryUri.getScheme());

                conf.setBoolean(prop, true);

                secondaryFs = AbstractFileSystem.get(secondaryUri, conf);
            } catch (URISyntaxException ignore) {
                throw new IOException("Failed to resolve secondary file system URI: " + secUri);
            } catch (IOException e) {
                throw new IOException("Failed to connect to the secondary file system: " + secUri, e);
            }
        }
    } finally {
        leaveBusy();
    }
}

---

From source file:org.gridgain.grid.kernal.processors.hadoop.GridHadoopSortingTest.java

License:Open Source License

/**
 * @throws Exception If failed./*  ww w .  j av  a 2  s  . co  m*/
 */
public void testSortSimple() throws Exception {
    // Generate test data.
    Job job = Job.getInstance();

    job.setInputFormatClass(InFormat.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(NullWritable.class);

    job.setMapperClass(Mapper.class);
    job.setNumReduceTasks(0);

    setupFileSystems(job.getConfiguration());

    FileOutputFormat.setOutputPath(job, new Path(ggfsScheme() + PATH_INPUT));

    X.printerrln("Data generation started.");

    grid(0).hadoop().submit(new GridHadoopJobId(UUID.randomUUID(), 1), createJobInfo(job.getConfiguration()))
            .get(180000);

    X.printerrln("Data generation complete.");

    // Run main map-reduce job.
    job = Job.getInstance();

    setupFileSystems(job.getConfiguration());

    job.getConfiguration().set(CommonConfigurationKeys.IO_SERIALIZATIONS_KEY,
            JavaSerialization.class.getName() + "," + WritableSerialization.class.getName());

    FileInputFormat.setInputPaths(job, new Path(ggfsScheme() + PATH_INPUT));
    FileOutputFormat.setOutputPath(job, new Path(ggfsScheme() + PATH_OUTPUT));

    job.setSortComparatorClass(JavaSerializationComparator.class);

    job.setMapperClass(MyMapper.class);
    job.setReducerClass(MyReducer.class);

    job.setNumReduceTasks(2);

    job.setMapOutputKeyClass(UUID.class);
    job.setMapOutputValueClass(NullWritable.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(NullWritable.class);

    X.printerrln("Job started.");

    grid(0).hadoop().submit(new GridHadoopJobId(UUID.randomUUID(), 2), createJobInfo(job.getConfiguration()))
            .get(180000);

    X.printerrln("Job complete.");

    // Check result.
    Path outDir = new Path(ggfsScheme() + PATH_OUTPUT);

    AbstractFileSystem fs = AbstractFileSystem.get(new URI(ggfsScheme()), job.getConfiguration());

    for (FileStatus file : fs.listStatus(outDir)) {
        X.printerrln("__ file: " + file);

        if (file.getLen() == 0)
            continue;

        FSDataInputStream in = fs.open(file.getPath());

        Scanner sc = new Scanner(in);

        UUID prev = null;

        while (sc.hasNextLine()) {
            UUID next = UUID.fromString(sc.nextLine());

            //                X.printerrln("___ check: " + next);

            if (prev != null)
                assertTrue(prev.compareTo(next) < 0);

            prev = next;
        }
    }
}

---