Example usage for org.apache.hadoop.fs FileSystem getLocal

List of usage examples for org.apache.hadoop.fs FileSystem getLocal

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileSystem
  6. getLocal

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem getLocal.

Prototype

public static LocalFileSystem getLocal(Configuration conf) throws IOException

Source Link

Document

Get the local FileSystem.

Usage

From source file:gobblin.runtime.TestRecordStream.java

License:Apache License

@Test
public void testAcks() throws Exception {

    StreamEntity[] entities = new StreamEntity[] { new RecordEnvelope<>("a"), new BasicTestControlMessage("1"),
            new RecordEnvelope<>("b"), new BasicTestControlMessage("2") };

    BasicAckableForTesting ackable = new BasicAckableForTesting();
    for (int i = 0; i < entities.length; i++) {
        entities[i].addCallBack(ackable);
    }//from www  . j  a v  a2  s  .  c o  m

    MyExtractor extractor = new MyExtractor(entities);
    MyConverter converter = new MyConverter();
    MyDataWriter writer = new MyDataWriter();

    // Create a TaskState
    TaskState taskState = getEmptyTestTaskState("testRetryTaskId");
    taskState.setProp(ConfigurationKeys.TASK_SYNCHRONOUS_EXECUTION_MODEL_KEY, false);
    // Create a mock TaskContext
    TaskContext mockTaskContext = mock(TaskContext.class);
    when(mockTaskContext.getExtractor()).thenReturn(extractor);
    when(mockTaskContext.getForkOperator()).thenReturn(new IdentityForkOperator());
    when(mockTaskContext.getTaskState()).thenReturn(taskState);
    when(mockTaskContext.getConverters()).thenReturn(Lists.newArrayList(converter));
    when(mockTaskContext.getTaskLevelPolicyChecker(any(TaskState.class), anyInt()))
            .thenReturn(mock(TaskLevelPolicyChecker.class));
    when(mockTaskContext.getRowLevelPolicyChecker()).thenReturn(
            new RowLevelPolicyChecker(Lists.newArrayList(), "ss", FileSystem.getLocal(new Configuration())));
    when(mockTaskContext.getRowLevelPolicyChecker(anyInt())).thenReturn(
            new RowLevelPolicyChecker(Lists.newArrayList(), "ss", FileSystem.getLocal(new Configuration())));
    when(mockTaskContext.getDataWriterBuilder(anyInt(), anyInt())).thenReturn(writer);

    // Create a mock TaskPublisher
    TaskPublisher mockTaskPublisher = mock(TaskPublisher.class);
    when(mockTaskPublisher.canPublish()).thenReturn(TaskPublisher.PublisherState.SUCCESS);
    when(mockTaskContext.getTaskPublisher(any(TaskState.class), any(TaskLevelPolicyCheckResults.class)))
            .thenReturn(mockTaskPublisher);

    // Create a mock TaskStateTracker
    TaskStateTracker mockTaskStateTracker = mock(TaskStateTracker.class);

    // Create a TaskExecutor - a real TaskExecutor must be created so a Fork is run in a separate thread
    TaskExecutor taskExecutor = new TaskExecutor(new Properties());

    // Create the Task
    Task realTask = new Task(mockTaskContext, mockTaskStateTracker, taskExecutor,
            Optional.<CountDownLatch>absent());
    Task task = spy(realTask);
    doNothing().when(task).submitTaskCommittedEvent();

    task.run();
    task.commit();
    Assert.assertEquals(task.getTaskState().getWorkingState(), WorkUnitState.WorkingState.SUCCESSFUL);

    Assert.assertEquals(ackable.acked, 4);
}

From source file:gobblin.runtime.TestRecordStream.java

License:Apache License

private Task setupTask(Extractor extractor, DataWriterBuilder writer, Converter converter) throws Exception {
    // Create a TaskState
    TaskState taskState = getEmptyTestTaskState("testRetryTaskId");
    taskState.setProp(ConfigurationKeys.TASK_SYNCHRONOUS_EXECUTION_MODEL_KEY, false);
    // Create a mock TaskContext
    TaskContext mockTaskContext = mock(TaskContext.class);
    when(mockTaskContext.getExtractor()).thenReturn(extractor);
    when(mockTaskContext.getForkOperator()).thenReturn(new IdentityForkOperator());
    when(mockTaskContext.getTaskState()).thenReturn(taskState);
    when(mockTaskContext.getConverters()).thenReturn(Lists.newArrayList(converter));
    when(mockTaskContext.getTaskLevelPolicyChecker(any(TaskState.class), anyInt()))
            .thenReturn(mock(TaskLevelPolicyChecker.class));
    when(mockTaskContext.getRowLevelPolicyChecker()).thenReturn(
            new RowLevelPolicyChecker(Lists.newArrayList(), "ss", FileSystem.getLocal(new Configuration())));
    when(mockTaskContext.getRowLevelPolicyChecker(anyInt())).thenReturn(
            new RowLevelPolicyChecker(Lists.newArrayList(), "ss", FileSystem.getLocal(new Configuration())));
    when(mockTaskContext.getDataWriterBuilder(anyInt(), anyInt())).thenReturn(writer);

    // Create a mock TaskPublisher
    TaskPublisher mockTaskPublisher = mock(TaskPublisher.class);
    when(mockTaskPublisher.canPublish()).thenReturn(TaskPublisher.PublisherState.SUCCESS);
    when(mockTaskContext.getTaskPublisher(any(TaskState.class), any(TaskLevelPolicyCheckResults.class)))
            .thenReturn(mockTaskPublisher);

    // Create a mock TaskStateTracker
    TaskStateTracker mockTaskStateTracker = mock(TaskStateTracker.class);

    // Create a TaskExecutor - a real TaskExecutor must be created so a Fork is run in a separate thread
    TaskExecutor taskExecutor = new TaskExecutor(new Properties());

    // Create the Task
    Task realTask = new Task(mockTaskContext, mockTaskStateTracker, taskExecutor,
            Optional.<CountDownLatch>absent());
    Task task = spy(realTask);// w  w w .  j av a2 s .  c o m
    doNothing().when(task).submitTaskCommittedEvent();

    return task;
}

From source file:gobblin.test.integration.data.management.CopyIntegrationTest.java

License:Apache License

@Test
public void testTarGzCopy() throws Exception {

    Closer closer = Closer.create();//  w w  w. ja va  2 s.  c  o  m
    try {
        JobLauncher jobLauncher = closer.register(JobLauncherFactory.newJobLauncher(gobblinProps, jobProps));
        jobLauncher.launchJob(null);

        String file1Path = gobblinProps.getProperty(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR)
                + "/LogData/sub1/sub2/text1.txt";
        String file2Path = gobblinProps.getProperty(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR)
                + "/LogData/sub1/sub2/text2.txt";

        FileSystem fs = FileSystem.getLocal(new Configuration());

        Assert.assertEquals(IOUtils.toString(closer.register(fs.open(new Path(file1Path)))), "text1");
        Assert.assertEquals(IOUtils.toString(closer.register(fs.open(new Path(file2Path)))), "text2");

    } finally {
        closer.close();
    }
}

From source file:gobblin.test.integration.data.management.CopyIntegrationTest.java

License:Apache License

@AfterClass
@BeforeClass/*from www  .j a  va  2s . c om*/
public void cleanup() throws Exception {
    FileSystem fs = FileSystem.getLocal(new Configuration());
    fs.delete(new Path("gobblin-test-harness/testOutput"), true);
}

From source file:gobblin.util.FileListUtilsTest.java

License:Apache License

@Test
public void testListFilesRecursively() throws IOException {
    FileSystem localFs = FileSystem.getLocal(new Configuration());
    Path baseDir = new Path(FILE_UTILS_TEST_DIR, "fileListTestDir1");
    try {/*ww  w . ja  v a  2 s . com*/
        if (localFs.exists(baseDir)) {
            localFs.delete(baseDir, true);
        }
        localFs.mkdirs(baseDir);
        localFs.create(new Path(baseDir, TEST_FILE_NAME1));
        Path subDir = new Path(baseDir, "subDir");
        localFs.mkdirs(subDir);
        localFs.create(new Path(subDir, TEST_FILE_NAME2));
        List<FileStatus> testFiles = FileListUtils.listFilesRecursively(localFs, baseDir);

        Assert.assertEquals(2, testFiles.size());

        Set<String> fileNames = Sets.newHashSet();
        for (FileStatus testFileStatus : testFiles) {
            fileNames.add(testFileStatus.getPath().getName());
        }
        Assert.assertTrue(fileNames.contains(TEST_FILE_NAME1) && fileNames.contains(TEST_FILE_NAME2));
    } finally {
        localFs.delete(baseDir, true);
    }
}

From source file:gobblin.util.FileListUtilsTest.java

License:Apache License

@Test
public void testListPathsRecursively() throws IOException {
    FileSystem localFs = FileSystem.getLocal(new Configuration());
    Path baseDir = new Path(FILE_UTILS_TEST_DIR, "fileListTestDir2");
    try {/*from  w w  w  .  ja v a 2 s . c  o  m*/
        if (localFs.exists(baseDir)) {
            localFs.delete(baseDir, true);
        }
        localFs.mkdirs(baseDir);
        localFs.create(new Path(baseDir, TEST_FILE_NAME1));
        Path subDir = new Path(baseDir, "subDir");
        localFs.mkdirs(subDir);
        localFs.create(new Path(subDir, TEST_FILE_NAME2));
        List<FileStatus> testFiles = FileListUtils.listPathsRecursively(localFs, baseDir, new PathFilter() {
            @Override
            public boolean accept(Path path) {
                return true;
            }
        });
        Assert.assertEquals(4, testFiles.size());

        Set<String> fileNames = Sets.newHashSet();
        for (FileStatus testFileStatus : testFiles) {
            fileNames.add(testFileStatus.getPath().getName());
        }

        Set<String> expectedFileNames = Sets.newHashSet();
        expectedFileNames.add(baseDir.getName());
        expectedFileNames.add(subDir.getName());
        expectedFileNames.add(TEST_FILE_NAME1);
        expectedFileNames.add(TEST_FILE_NAME2);

        Assert.assertEquals(fileNames, expectedFileNames);
    } finally {
        localFs.delete(baseDir, true);
    }
}

From source file:gobblin.util.FileListUtilsTest.java

License:Apache License

@Test
public void testListMostNestedPathRecursively() throws IOException {
    FileSystem localFs = FileSystem.getLocal(new Configuration());
    Path baseDir = new Path(FILE_UTILS_TEST_DIR, "fileListTestDir3");
    String emptyDir1 = "emptyDir1";
    String emptyDir2 = "emptyDir2";
    try {/*from  w w w  . ja va  2  s . c om*/
        if (localFs.exists(baseDir)) {
            localFs.delete(baseDir, true);
        }
        localFs.mkdirs(baseDir);
        localFs.create(new Path(baseDir, TEST_FILE_NAME1));
        localFs.mkdirs(new Path(baseDir, emptyDir1));
        Path subDir = new Path(baseDir, "subDir");
        localFs.mkdirs(subDir);
        localFs.create(new Path(subDir, TEST_FILE_NAME2));
        localFs.mkdirs(new Path(subDir, emptyDir2));

        List<FileStatus> testFiles = FileListUtils.listMostNestedPathRecursively(localFs, baseDir);

        Assert.assertEquals(4, testFiles.size());

        Set<String> fileNames = Sets.newHashSet();
        for (FileStatus testFileStatus : testFiles) {
            fileNames.add(testFileStatus.getPath().getName());
        }

        Set<String> expectedFileNames = Sets.newHashSet();
        expectedFileNames.add(emptyDir1);
        expectedFileNames.add(emptyDir2);
        expectedFileNames.add(TEST_FILE_NAME1);
        expectedFileNames.add(TEST_FILE_NAME2);

        Assert.assertEquals(fileNames, expectedFileNames);
    } finally {
        localFs.delete(baseDir, true);
    }
}

From source file:gobblin.util.HadoopUtilsTest.java

License:Apache License

@Test
public void testRenameRecursively() throws Exception {

    final Path hadoopUtilsTestDir = new Path(Files.createTempDir().getAbsolutePath(), "HadoopUtilsTestDir");
    FileSystem fs = FileSystem.getLocal(new Configuration());
    try {/* ww w . jav  a  2  s . c  om*/
        fs.mkdirs(hadoopUtilsTestDir);

        fs.mkdirs(new Path(hadoopUtilsTestDir, "testRename/a/b/c"));

        fs.mkdirs(new Path(hadoopUtilsTestDir, "testRenameStaging/a/b/c"));
        fs.mkdirs(new Path(hadoopUtilsTestDir, "testRenameStaging/a/b/c/e"));
        fs.create(new Path(hadoopUtilsTestDir, "testRenameStaging/a/b/c/t1.txt"));
        fs.create(new Path(hadoopUtilsTestDir, "testRenameStaging/a/b/c/e/t2.txt"));

        HadoopUtils.renameRecursively(fs, new Path(hadoopUtilsTestDir, "testRenameStaging"),
                new Path(hadoopUtilsTestDir, "testRename"));

        Assert.assertTrue(fs.exists(new Path(hadoopUtilsTestDir, "testRename/a/b/c/t1.txt")));
        Assert.assertTrue(fs.exists(new Path(hadoopUtilsTestDir, "testRename/a/b/c/e/t2.txt")));
    } finally {
        fs.delete(hadoopUtilsTestDir, true);
    }

}

From source file:gobblin.util.HadoopUtilsTest.java

License:Apache License

@Test
public void testSafeRenameRecursively() throws Exception {
    final Logger log = LoggerFactory.getLogger("HadoopUtilsTest.testSafeRenameRecursively");

    final Path hadoopUtilsTestDir = new Path(Files.createTempDir().getAbsolutePath(), "HadoopUtilsTestDir");
    final FileSystem fs = FileSystem.getLocal(new Configuration());
    try {/*from   w w  w . jav a 2  s  .  c o m*/
        // do many iterations to catch rename race conditions
        for (int i = 0; i < 100; i++) {
            fs.mkdirs(hadoopUtilsTestDir);
            fs.mkdirs(new Path(hadoopUtilsTestDir, "testSafeRename/a/b/c"));

            fs.mkdirs(new Path(hadoopUtilsTestDir, "testRenameStaging1/a/b/c"));
            fs.mkdirs(new Path(hadoopUtilsTestDir, "testRenameStaging1/a/b/c/e"));
            fs.create(new Path(hadoopUtilsTestDir, "testRenameStaging1/a/b/c/t1.txt"));
            fs.create(new Path(hadoopUtilsTestDir, "testRenameStaging1/a/b/c/e/t2.txt"));

            fs.mkdirs(new Path(hadoopUtilsTestDir, "testRenameStaging2/a/b/c"));
            fs.mkdirs(new Path(hadoopUtilsTestDir, "testRenameStaging2/a/b/c/e"));
            fs.create(new Path(hadoopUtilsTestDir, "testRenameStaging2/a/b/c/t3.txt"));
            fs.create(new Path(hadoopUtilsTestDir, "testRenameStaging2/a/b/c/e/t4.txt"));

            ExecutorService executorService = Executors.newFixedThreadPool(2);

            final Throwable[] runnableErrors = { null, null };

            Future<?> renameFuture = executorService.submit(new Runnable() {

                @Override
                public void run() {
                    try {
                        HadoopUtils.renameRecursively(fs, new Path(hadoopUtilsTestDir, "testRenameStaging1"),
                                new Path(hadoopUtilsTestDir, "testSafeRename"));
                    } catch (Throwable e) {
                        log.error("Rename error: " + e, e);
                        runnableErrors[0] = e;
                    }
                }
            });

            Future<?> safeRenameFuture = executorService.submit(new Runnable() {

                @Override
                public void run() {
                    try {
                        HadoopUtils.safeRenameRecursively(fs,
                                new Path(hadoopUtilsTestDir, "testRenameStaging2"),
                                new Path(hadoopUtilsTestDir, "testSafeRename"));
                    } catch (Throwable e) {
                        log.error("Safe rename error: " + e, e);
                        runnableErrors[1] = e;
                    }
                }
            });

            // Wait for the executions to complete
            renameFuture.get(10, TimeUnit.SECONDS);
            safeRenameFuture.get(10, TimeUnit.SECONDS);

            executorService.shutdownNow();

            Assert.assertNull(runnableErrors[0], "Runnable 0 error: " + runnableErrors[0]);
            Assert.assertNull(runnableErrors[1], "Runnable 1 error: " + runnableErrors[1]);

            Assert.assertTrue(fs.exists(new Path(hadoopUtilsTestDir, "testSafeRename/a/b/c/t1.txt")));
            Assert.assertTrue(fs.exists(new Path(hadoopUtilsTestDir, "testSafeRename/a/b/c/t3.txt")));
            Assert.assertTrue(!fs.exists(new Path(hadoopUtilsTestDir, "testSafeRename/a/b/c/e/e/t2.txt")));
            Assert.assertTrue(fs.exists(new Path(hadoopUtilsTestDir, "testSafeRename/a/b/c/e/t2.txt")));
            Assert.assertTrue(fs.exists(new Path(hadoopUtilsTestDir, "testSafeRename/a/b/c/e/t4.txt")));
            fs.delete(hadoopUtilsTestDir, true);
        }
    } finally {
        fs.delete(hadoopUtilsTestDir, true);
    }

}

From source file:gobblin.util.HeapDumpForTaskUtilsTest.java

License:Apache License

@BeforeClass
public void setUp() throws IOException {
    this.fs = FileSystem.getLocal(new Configuration());
    this.fs.mkdirs(new Path(TEST_DIR));
}