Example usage for org.apache.hadoop.fs FileSystem getLocal

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem getLocal.

Prototype

public static LocalFileSystem getLocal(Configuration conf) throws IOException

Source Link

Document

Get the local FileSystem.

Usage

From source file:gobblin.util.io.StreamUtilsTest.java

License:Apache License

@Test
public void testTarDir() throws IOException {
    FileSystem localFs = FileSystem.getLocal(new Configuration());

    // Set of expected Paths to be in the resulting tar file
    Set<Path> expectedPaths = Sets.newHashSet();

    // Create input directory
    Path testInDir = new Path("testDir");
    expectedPaths.add(testInDir);// w w  w . j  a v  a2 s. c o m

    // Create output file path
    Path testOutFile = new Path("testTarOut" + UUID.randomUUID() + ".tar.gz");

    try {
        localFs.mkdirs(testInDir);

        // Create a test file path
        Path testFile1 = new Path(testInDir, "testFile1");
        expectedPaths.add(testFile1);
        FSDataOutputStream testFileOut1 = localFs.create(testFile1);
        testFileOut1.close();

        // Create a test file path
        Path testFile2 = new Path(testInDir, "testFile2");
        expectedPaths.add(testFile2);
        FSDataOutputStream testFileOut2 = localFs.create(testFile2);
        testFileOut2.close();

        // tar the input directory to the specific output file
        StreamUtils.tar(localFs, testInDir, testOutFile);

        // Confirm the contents of the tar file are valid
        try (TarArchiveInputStream tarArchiveInputStream = new TarArchiveInputStream(
                new GzipCompressorInputStream(localFs.open(testOutFile)))) {
            TarArchiveEntry tarArchiveEntry;

            while (null != (tarArchiveEntry = tarArchiveInputStream.getNextTarEntry())) {
                assertThat(new Path(tarArchiveEntry.getName()), isIn(expectedPaths));
            }
        }
    } finally {
        if (localFs.exists(testInDir)) {
            localFs.delete(testInDir, true);
        }
        if (localFs.exists(testOutFile)) {
            localFs.delete(testOutFile, true);
        }
    }
}

From source file:gobblin.util.io.StreamUtilsTest.java

License:Apache License

@Test
public void testTarFile() throws IOException {
    FileSystem localFs = FileSystem.getLocal(new Configuration());

    // Set of expected Paths to be in the resulting tar file
    Set<Path> expectedPaths = Sets.newHashSet();

    // Create input file path
    Path testFile = new Path("testFile");
    expectedPaths.add(testFile);/*from   www.  j  a v a2s  . co m*/

    // Create output file path
    Path testOutFile = new Path("testTarOut" + UUID.randomUUID() + ".tar.gz");

    try {
        // Create the input file
        FSDataOutputStream testFileOut1 = localFs.create(testFile);
        testFileOut1.close();

        // tar the input file to the specific output file
        StreamUtils.tar(localFs, testFile, testOutFile);

        // Confirm the contents of the tar file are valid
        try (TarArchiveInputStream tarArchiveInputStream = new TarArchiveInputStream(
                new GzipCompressorInputStream(localFs.open(testOutFile)))) {
            TarArchiveEntry tarArchiveEntry;

            while (null != (tarArchiveEntry = tarArchiveInputStream.getNextTarEntry())) {
                MatcherAssert.assertThat(new Path(tarArchiveEntry.getName()), Matchers.isIn(expectedPaths));
            }
        }
    } finally {
        if (localFs.exists(testFile)) {
            localFs.delete(testFile, true);
        }
        if (localFs.exists(testOutFile)) {
            localFs.delete(testOutFile, true);
        }
    }
}

From source file:gobblin.util.JobConfigurationUtils.java

License:Apache License

/**
 * Load the properties from the specified file into a {@link Properties} object.
 *
 * @param fileName the name of the file to load properties from
 * @param conf configuration object to determine the file system to be used
 * @return a new {@link Properties} instance
 *///from w ww  .  j a va  2s  .c  o m
public static Properties fileToProperties(String fileName, Configuration conf)
        throws IOException, ConfigurationException {

    PropertiesConfiguration propsConfig = new PropertiesConfiguration();
    Path filePath = new Path(fileName);
    URI fileURI = filePath.toUri();

    if (fileURI.getScheme() == null && fileURI.getAuthority() == null) {
        propsConfig.load(FileSystem.getLocal(conf).open(filePath));
    } else {
        propsConfig.load(filePath.getFileSystem(conf).open(filePath));
    }
    return ConfigurationConverter.getProperties(propsConfig);
}

From source file:gobblin.util.JobLauncherUtilsTest.java

License:Apache License

@Test
public void testDeleteStagingData() throws IOException {
    FileSystem fs = FileSystem.getLocal(new Configuration());

    Path rootDir = new Path("gobblin-test/job-launcher-utils-test");
    Path writerStagingDir0 = new Path(rootDir, "staging/fork_0");
    Path writerStagingDir1 = new Path(rootDir, "staging/fork_1");
    Path writerOutputDir0 = new Path(rootDir, "output/fork_0");
    Path writerOutputDir1 = new Path(rootDir, "output/fork_1");

    String writerPath0 = "test0";
    String writerPath1 = "test1";

    try {/*from   w w w . j  a va  2  s.  c o m*/
        WorkUnitState state = new WorkUnitState();
        state.setProp(ConfigurationKeys.FORK_BRANCHES_KEY, "2");
        state.setProp(
                ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.WRITER_FILE_SYSTEM_URI, 2, 0),
                ConfigurationKeys.LOCAL_FS_URI);
        state.setProp(
                ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.WRITER_FILE_SYSTEM_URI, 2, 1),
                ConfigurationKeys.LOCAL_FS_URI);
        state.setProp(ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.WRITER_STAGING_DIR, 2, 0),
                writerStagingDir0.toString());
        state.setProp(ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.WRITER_STAGING_DIR, 2, 1),
                writerStagingDir1.toString());
        state.setProp(ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.WRITER_OUTPUT_DIR, 2, 0),
                writerOutputDir0.toString());
        state.setProp(ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.WRITER_OUTPUT_DIR, 2, 1),
                writerOutputDir1.toString());
        state.setProp(ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.WRITER_FILE_PATH, 2, 0),
                writerPath0);
        state.setProp(ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.WRITER_FILE_PATH, 2, 1),
                writerPath1);

        Path writerStagingPath0 = new Path(writerStagingDir0, writerPath0);
        fs.mkdirs(writerStagingPath0);
        Path writerStagingPath1 = new Path(writerStagingDir1, writerPath1);
        fs.mkdirs(writerStagingPath1);
        Path writerOutputPath0 = new Path(writerOutputDir0, writerPath0);
        fs.mkdirs(writerOutputPath0);
        Path writerOutputPath1 = new Path(writerOutputDir1, writerPath1);
        fs.mkdirs(writerOutputPath1);

        JobLauncherUtils.cleanTaskStagingData(state, LoggerFactory.getLogger(JobLauncherUtilsTest.class));

        Assert.assertFalse(fs.exists(writerStagingPath0));
        Assert.assertFalse(fs.exists(writerStagingPath1));
        Assert.assertFalse(fs.exists(writerOutputPath0));
        Assert.assertFalse(fs.exists(writerOutputPath1));
    } finally {
        fs.delete(rootDir, true);
    }
}

From source file:gobblin.util.JobLauncherUtilsTest.java

License:Apache License

@Test
public void testDeleteStagingDataWithOutWriterFilePath() throws IOException {
    FileSystem fs = FileSystem.getLocal(new Configuration());

    String branchName0 = "fork_0";
    String branchName1 = "fork_1";

    String namespace = "gobblin.test";
    String tableName = "test-table";

    Path rootDir = new Path("gobblin-test/job-launcher-utils-test");

    Path writerStagingDir0 = new Path(rootDir, "staging" + Path.SEPARATOR + branchName0);
    Path writerStagingDir1 = new Path(rootDir, "staging" + Path.SEPARATOR + branchName1);
    Path writerOutputDir0 = new Path(rootDir, "output" + Path.SEPARATOR + branchName0);
    Path writerOutputDir1 = new Path(rootDir, "output" + Path.SEPARATOR + branchName1);

    try {//from   ww w  .  j a v  a2 s .  co  m
        SourceState sourceState = new SourceState();
        WorkUnitState state = new WorkUnitState(
                WorkUnit.create(new Extract(sourceState, TableType.APPEND_ONLY, namespace, tableName)));

        state.setProp(ConfigurationKeys.FORK_BRANCHES_KEY, "2");
        state.setProp(ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.FORK_BRANCH_NAME_KEY, 2, 0),
                branchName0);
        state.setProp(ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.FORK_BRANCH_NAME_KEY, 2, 1),
                branchName1);

        state.setProp(
                ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.WRITER_FILE_SYSTEM_URI, 2, 0),
                ConfigurationKeys.LOCAL_FS_URI);
        state.setProp(
                ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.WRITER_FILE_SYSTEM_URI, 2, 1),
                ConfigurationKeys.LOCAL_FS_URI);
        state.setProp(ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.WRITER_STAGING_DIR, 2, 0),
                writerStagingDir0.toString());
        state.setProp(ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.WRITER_STAGING_DIR, 2, 1),
                writerStagingDir1.toString());
        state.setProp(ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.WRITER_OUTPUT_DIR, 2, 0),
                writerOutputDir0.toString());
        state.setProp(ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.WRITER_OUTPUT_DIR, 2, 1),
                writerOutputDir1.toString());

        Path writerStagingPath0 = new Path(writerStagingDir0,
                ForkOperatorUtils.getPathForBranch(state, state.getExtract().getOutputFilePath(), 2, 0));
        fs.mkdirs(writerStagingPath0);

        Path writerStagingPath1 = new Path(writerStagingDir1,
                ForkOperatorUtils.getPathForBranch(state, state.getExtract().getOutputFilePath(), 2, 1));
        fs.mkdirs(writerStagingPath1);

        Path writerOutputPath0 = new Path(writerOutputDir0,
                ForkOperatorUtils.getPathForBranch(state, state.getExtract().getOutputFilePath(), 2, 0));
        fs.mkdirs(writerOutputPath0);

        Path writerOutputPath1 = new Path(writerOutputDir1,
                ForkOperatorUtils.getPathForBranch(state, state.getExtract().getOutputFilePath(), 2, 1));
        fs.mkdirs(writerOutputPath1);

        JobLauncherUtils.cleanTaskStagingData(state, LoggerFactory.getLogger(JobLauncherUtilsTest.class));

        Assert.assertFalse(fs.exists(writerStagingPath0));
        Assert.assertFalse(fs.exists(writerStagingPath1));
        Assert.assertFalse(fs.exists(writerOutputPath0));
        Assert.assertFalse(fs.exists(writerOutputPath1));
    } finally {
        fs.delete(rootDir, true);
    }
}

From source file:gobblin.util.ParallelRunnerTest.java

License:Apache License

@BeforeClass
public void setUp() throws IOException {
    this.fs = FileSystem.getLocal(new Configuration());
    this.outputPath = new Path(ParallelRunnerTest.class.getSimpleName());
}

From source file:gobblin.util.ProxiedFileSystemUtils.java

License:Apache License

/**
 * Retrives a {@link Token} from a given sequence file for a specified user. The sequence file should contain a list
 * of key, value pairs where each key corresponds to a user and each value corresponds to a {@link Token} for that
 * user./*from w  ww. j av a  2  s.c o m*/
 *
 * @param userNameKey The name of the user to retrieve a {@link Token} for
 * @param tokenFilePath The path to the sequence file containing the {@link Token}s
 *
 * @return A {@link Token} for the given user name
 */
public static Optional<Token<?>> getTokenFromSeqFile(String userNameKey, Path tokenFilePath)
        throws IOException {
    log.info("Reading tokens from sequence file " + tokenFilePath);

    try (Closer closer = Closer.create()) {
        FileSystem localFs = FileSystem.getLocal(new Configuration());
        @SuppressWarnings("deprecation")
        SequenceFile.Reader tokenReader = closer
                .register(new SequenceFile.Reader(localFs, tokenFilePath, localFs.getConf()));
        Text key = new Text();
        Token<?> value = new Token<>();
        while (tokenReader.next(key, value)) {
            log.debug("Found token for user: " + key);
            if (key.toString().equals(userNameKey)) {
                return Optional.<Token<?>>of(value);
            }
        }
    }
    log.warn("Did not find any tokens for user " + userNameKey);
    return Optional.absent();
}

From source file:gobblin.util.ProxiedFileSystemWrapper.java

License:Apache License

/**
 * Get token from the token sequence file.
 * @param authPath//  w  w  w.  j ava2  s .c  om
 * @param proxyUserName
 * @return Token for proxyUserName if it exists.
 * @throws IOException
 */
private static Optional<Token<?>> getTokenFromSeqFile(String authPath, String proxyUserName)
        throws IOException {
    try (Closer closer = Closer.create()) {
        FileSystem localFs = FileSystem.getLocal(new Configuration());
        SequenceFile.Reader tokenReader = closer
                .register(new SequenceFile.Reader(localFs, new Path(authPath), localFs.getConf()));
        Text key = new Text();
        Token<?> value = new Token<>();
        while (tokenReader.next(key, value)) {
            LOG.info("Found token for " + key);
            if (key.toString().equals(proxyUserName)) {
                return Optional.<Token<?>>of(value);
            }
        }
    }
    return Optional.absent();
}

From source file:gobblin.util.PullFileLoaderTest.java

License:Apache License

public PullFileLoaderTest() throws Exception {
    this.basePath = new Path(this.getClass().getClassLoader().getResource("pullFileLoaderTest").getFile());
    this.loader = new PullFileLoader(this.basePath, FileSystem.getLocal(new Configuration()),
            PullFileLoader.DEFAULT_JAVA_PROPS_PULL_FILE_EXTENSIONS,
            PullFileLoader.DEFAULT_HOCON_PULL_FILE_EXTENSIONS);
}

From source file:gobblin.util.RatedControlledFileSystemTest.java

License:Apache License

@BeforeClass
public void setUp() throws IOException, ExecutionException {
    Limiter limiter = new RateBasedLimiter(20);
    this.rateControlledFs = new TestRateControlledFileSystem(FileSystem.getLocal(new Configuration()), 20,
            limiter);//from   www .  j a v a 2s .  c  o m
    this.rateControlledFs.startRateControl();
}