Example usage for org.apache.hadoop.fs FileSystem getLocal

List of usage examples for org.apache.hadoop.fs FileSystem getLocal

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileSystem
  6. getLocal

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem getLocal.

Prototype

public static LocalFileSystem getLocal(Configuration conf) throws IOException

Source Link

Document

Get the local FileSystem.

Usage

From source file:gobblin.runtime.JobLauncherTestBase.java

License:Open Source License

@SuppressWarnings("unchecked")
protected void runTestWithFork(Properties jobProps) throws Exception {
    JobLauncher jobLauncher = JobLauncherFactory.newJobLauncher(this.properties);
    jobLauncher.launchJob(jobProps, null);
    String jobName = jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY);
    String jobId = jobProps.getProperty(ConfigurationKeys.JOB_ID_KEY);
    List<JobState> jobStateList = (List<JobState>) this.jobStateStore.getAll(jobName, jobId + ".jst");
    JobState jobState = jobStateList.get(0);

    Assert.assertEquals(jobState.getState(), JobState.RunningState.COMMITTED);
    Assert.assertEquals(jobState.getCompletedTasks(), 4);
    Assert.assertEquals(jobState.getPropAsInt(ConfigurationKeys.JOB_FAILURES_KEY), 0);

    FileSystem lfs = FileSystem.getLocal(new Configuration());
    for (TaskState taskState : jobState.getTaskStates()) {
        Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED);
        Path path = new Path(taskState.getProp(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR),
                new Path(taskState.getExtract().getOutputFilePath(), "fork_0"));
        Assert.assertTrue(lfs.exists(path));
        Assert.assertEquals(lfs.listStatus(path).length, 2);
        path = new Path(taskState.getProp(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR),
                new Path(taskState.getExtract().getOutputFilePath(), "fork_1"));
        Assert.assertTrue(lfs.exists(path));
        Assert.assertEquals(lfs.listStatus(path).length, 2);
    }// w  w w.j  av a2  s  . co m
}

From source file:gobblin.runtime.JobLauncherTestHelper.java

License:Apache License

public void runTestWithFork(Properties jobProps) throws Exception {
    String jobName = jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY);
    String jobId = JobLauncherUtils.newJobId(jobName).toString();
    jobProps.setProperty(ConfigurationKeys.JOB_ID_KEY, jobId);

    try (JobLauncher jobLauncher = JobLauncherFactory.newJobLauncher(this.launcherProps, jobProps)) {
        jobLauncher.launchJob(null);/*  w  ww.j a v  a2s. c o m*/
    }

    List<JobState.DatasetState> datasetStateList = this.datasetStateStore.getAll(jobName, jobId + ".jst");
    DatasetState datasetState = datasetStateList.get(0);

    Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED);
    Assert.assertEquals(datasetState.getCompletedTasks(), 4);
    Assert.assertEquals(datasetState.getJobFailures(), 0);

    FileSystem lfs = FileSystem.getLocal(new Configuration());
    for (TaskState taskState : datasetState.getTaskStates()) {
        Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED);
        Path path = new Path(this.launcherProps.getProperty(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR),
                new Path(taskState.getExtract().getOutputFilePath(), "fork_0"));
        Assert.assertTrue(lfs.exists(path));
        Assert.assertEquals(lfs.listStatus(path).length, 2);
        Assert.assertEquals(taskState.getPropAsLong(ConfigurationKeys.WRITER_RECORDS_WRITTEN + ".0"),
                TestExtractor.TOTAL_RECORDS);

        path = new Path(this.launcherProps.getProperty(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR),
                new Path(taskState.getExtract().getOutputFilePath(), "fork_1"));
        Assert.assertTrue(lfs.exists(path));
        Assert.assertEquals(lfs.listStatus(path).length, 2);
        Assert.assertEquals(taskState.getPropAsLong(ConfigurationKeys.WRITER_RECORDS_WRITTEN + ".1"),
                TestExtractor.TOTAL_RECORDS);
    }
}

From source file:gobblin.runtime.locks.FileBasedJobLockFactory.java

License:Apache License

public static FileSystem getDefaultFileSystem(Configuration hadoopConf) throws IOException {
    return FileSystem.getLocal(hadoopConf);
}

From source file:gobblin.runtime.locks.FileBasedJobLockTest.java

License:Apache License

@BeforeClass
public void setUp() throws IOException {
    BasicConfigurator.configure();//from   www  . j  ava  2 s.  c o  m
    this.fs = FileSystem.getLocal(new Configuration());
    this.path = new Path("MRJobLockTest");
    if (!this.fs.exists(this.path)) {
        this.fs.mkdirs(this.path);
    }
}

From source file:gobblin.runtime.mapreduce.GobblinOutputCommitterTest.java

License:Apache License

@BeforeClass
public void setupWorkUnitFiles() throws IOException {
    this.conf = new Configuration();
    this.fs = FileSystem.getLocal(this.conf);
    this.stagingDirs = Lists.newArrayList();

    // Create a list of WorkUnits to serialize
    WorkUnit wu1 = createAndSetWorkUnit("wu1");
    WorkUnit wu2 = createAndSetWorkUnit("wu2");
    WorkUnit wu3 = createAndSetWorkUnit("wu3");
    WorkUnit wu4 = createAndSetWorkUnit("wu4");

    // Create a MultiWorkUnit to serialize
    MultiWorkUnit mwu1 = MultiWorkUnit.createEmpty();
    mwu1.setProp(ConfigurationKeys.TASK_ID_KEY, System.nanoTime());
    mwu1.addWorkUnits(Arrays.asList(wu3, wu4));

    Path inputDir = new Path(new Path(OUTPUT_PATH, JOB_NAME), "input");

    // Writer each WorkUnit to a separate file under inputDir
    Closer closer = Closer.create();/* w ww . j  a  v a2 s. c om*/
    try {
        wu1.write(closer.register(this.fs
                .create(new Path(inputDir, wu1.getProp(ConfigurationKeys.TASK_ID_KEY) + Path.SEPARATOR + "_")
                        .suffix("wu"))));
        wu2.write(closer.register(this.fs
                .create(new Path(inputDir, wu2.getProp(ConfigurationKeys.TASK_ID_KEY) + Path.SEPARATOR + "_")
                        .suffix("wu"))));
        mwu1.write(closer.register(this.fs
                .create(new Path(inputDir, mwu1.getProp(ConfigurationKeys.TASK_ID_KEY) + Path.SEPARATOR + "_")
                        .suffix("mwu"))));
    } finally {
        closer.close();
    }
}

From source file:gobblin.runtime.mapreduce.MRJobLauncher.java

License:Apache License

/**
 * Add framework or job-specific jars to the classpath through DistributedCache
 * so the mappers can use them./*  w ww . j  a va  2 s .  co  m*/
 */
@SuppressWarnings("deprecation")
private void addJars(Path jarFileDir, String jarFileList, Configuration conf) throws IOException {
    LocalFileSystem lfs = FileSystem.getLocal(conf);
    for (String jarFile : SPLITTER.split(jarFileList)) {
        Path srcJarFile = new Path(jarFile);
        FileStatus[] fileStatusList = lfs.globStatus(srcJarFile);
        for (FileStatus status : fileStatusList) {
            // DistributedCache requires absolute path, so we need to use makeQualified.
            Path destJarFile = new Path(this.fs.makeQualified(jarFileDir), status.getPath().getName());
            if (!this.fs.exists(destJarFile)) {
                // Copy the jar file from local file system to HDFS
                this.fs.copyFromLocalFile(status.getPath(), destJarFile);
            }
            // Then add the jar file on HDFS to the classpath
            LOG.info(String.format("Adding %s to classpath", destJarFile));
            DistributedCache.addFileToClassPath(destJarFile, conf, this.fs);
        }
    }
}

From source file:gobblin.runtime.TaskContinuousTest.java

License:Apache License

private TaskContext getMockTaskContext(ArrayList<Object> recordCollector, Extractor mockExtractor)
        throws Exception {

    TaskState taskState = getStreamingTaskState();
    // Create a mock RowLevelPolicyChecker
    RowLevelPolicyChecker mockRowLevelPolicyChecker = new RowLevelPolicyChecker(Lists.newArrayList(), "stateId",
            FileSystem.getLocal(new Configuration()));

    WatermarkStorage mockWatermarkStorage = new MockWatermarkStorage();

    // Create a mock TaskPublisher
    TaskPublisher mockTaskPublisher = mock(TaskPublisher.class);
    when(mockTaskPublisher.canPublish()).thenReturn(TaskPublisher.PublisherState.SUCCESS);

    // Create a mock TaskContext
    TaskContext mockTaskContext = mock(TaskContext.class);
    when(mockTaskContext.getExtractor()).thenReturn(mockExtractor);
    when(mockTaskContext.getRawSourceExtractor()).thenReturn(mockExtractor);
    when(mockTaskContext.getWatermarkStorage()).thenReturn(mockWatermarkStorage);
    when(mockTaskContext.getForkOperator()).thenReturn(new IdentityForkOperator());
    when(mockTaskContext.getTaskState()).thenReturn(taskState);
    when(mockTaskContext.getTaskPublisher(any(TaskState.class), any(TaskLevelPolicyCheckResults.class)))
            .thenReturn(mockTaskPublisher);
    when(mockTaskContext.getRowLevelPolicyChecker()).thenReturn(mockRowLevelPolicyChecker);
    when(mockTaskContext.getRowLevelPolicyChecker(anyInt())).thenReturn(mockRowLevelPolicyChecker);
    when(mockTaskContext.getTaskLevelPolicyChecker(any(TaskState.class), anyInt()))
            .thenReturn(mock(TaskLevelPolicyChecker.class));
    when(mockTaskContext.getDataWriterBuilder(anyInt(), anyInt()))
            .thenReturn(new TestStreamingDataWriterBuilder(recordCollector));
    return mockTaskContext;
}

From source file:gobblin.runtime.TaskStateCollectorServiceTest.java

License:Apache License

@BeforeClass
public void setUp() throws Exception {
    this.localFs = FileSystem.getLocal(new Configuration());
    this.localFs.mkdirs(this.outputTaskStateDir);

    this.taskStateStore = new FsStateStore<>(this.localFs, this.outputTaskStateDir.toUri().getPath(),
            TaskState.class);

    this.taskStateCollectorService = new TaskStateCollectorService(new Properties(), this.jobState,
            this.eventBus, this.taskStateStore, new Path(this.outputTaskStateDir, JOB_ID));

    this.eventBus.register(this);
}

From source file:gobblin.runtime.TaskTest.java

License:Apache License

/**
 * Check if a {@link WorkUnitState.WorkingState} of a {@link Task} is set properly after a {@link Task} fails once,
 * but then is successful the next time.
 *//*w w w  .  java2 s .co m*/
@Test(dataProvider = "stateOverrides")
public void testRetryTask(State overrides) throws Exception {
    // Create a TaskState
    TaskState taskState = getEmptyTestTaskState("testRetryTaskId");
    taskState.addAll(overrides);
    // Create a mock TaskContext
    TaskContext mockTaskContext = mock(TaskContext.class);
    when(mockTaskContext.getExtractor()).thenReturn(new FailOnceExtractor());
    when(mockTaskContext.getForkOperator()).thenReturn(new IdentityForkOperator());
    when(mockTaskContext.getTaskState()).thenReturn(taskState);
    when(mockTaskContext.getTaskLevelPolicyChecker(any(TaskState.class), anyInt()))
            .thenReturn(mock(TaskLevelPolicyChecker.class));
    when(mockTaskContext.getRowLevelPolicyChecker()).thenReturn(
            new RowLevelPolicyChecker(Lists.newArrayList(), "ss", FileSystem.getLocal(new Configuration())));
    when(mockTaskContext.getRowLevelPolicyChecker(anyInt())).thenReturn(
            new RowLevelPolicyChecker(Lists.newArrayList(), "ss", FileSystem.getLocal(new Configuration())));

    // Create a mock TaskPublisher
    TaskPublisher mockTaskPublisher = mock(TaskPublisher.class);
    when(mockTaskPublisher.canPublish()).thenReturn(TaskPublisher.PublisherState.SUCCESS);
    when(mockTaskContext.getTaskPublisher(any(TaskState.class), any(TaskLevelPolicyCheckResults.class)))
            .thenReturn(mockTaskPublisher);

    // Create a mock TaskStateTracker
    TaskStateTracker mockTaskStateTracker = mock(TaskStateTracker.class);

    // Create a TaskExecutor - a real TaskExecutor must be created so a Fork is run in a separate thread
    TaskExecutor taskExecutor = new TaskExecutor(new Properties());

    // Create the Task
    Task realTask = new Task(mockTaskContext, mockTaskStateTracker, taskExecutor,
            Optional.<CountDownLatch>absent());
    Task task = spy(realTask);
    doNothing().when(task).submitTaskCommittedEvent();

    // The first run of the Task should fail
    task.run();
    task.commit();
    Assert.assertEquals(task.getTaskState().getWorkingState(), WorkUnitState.WorkingState.FAILED);

    // The second run of the Task should succeed
    task.run();
    task.commit();
    Assert.assertEquals(task.getTaskState().getWorkingState(), WorkUnitState.WorkingState.SUCCESSFUL);
}

From source file:gobblin.runtime.TaskTest.java

License:Apache License

private TaskContext getMockTaskContext(TaskState taskState, Extractor mockExtractor,
        ArrayList<ArrayList<Object>> writerCollectors, ForkOperator mockForkOperator) throws Exception {

    int numForks = writerCollectors.size();

    // Create a mock RowLevelPolicyChecker
    RowLevelPolicyChecker mockRowLevelPolicyChecker = spy(
            new RowLevelPolicyChecker(Lists.newArrayList(), "ss", FileSystem.getLocal(new Configuration())));
    when(mockRowLevelPolicyChecker.executePolicies(any(Object.class), any(RowLevelPolicyCheckResults.class)))
            .thenReturn(true);/*from   www .ja v  a  2 s.  c o  m*/
    when(mockRowLevelPolicyChecker.getFinalState()).thenReturn(new State());

    // Create a mock TaskPublisher
    TaskPublisher mockTaskPublisher = mock(TaskPublisher.class);
    when(mockTaskPublisher.canPublish()).thenReturn(TaskPublisher.PublisherState.SUCCESS);

    // Create a mock TaskContext
    TaskContext mockTaskContext = mock(TaskContext.class);
    when(mockTaskContext.getExtractor()).thenReturn(mockExtractor);
    when(mockTaskContext.getRawSourceExtractor()).thenReturn(mockExtractor);
    when(mockTaskContext.getForkOperator()).thenReturn(mockForkOperator);
    when(mockTaskContext.getTaskState()).thenReturn(taskState);
    when(mockTaskContext.getTaskPublisher(any(TaskState.class), any(TaskLevelPolicyCheckResults.class)))
            .thenReturn(mockTaskPublisher);
    when(mockTaskContext.getRowLevelPolicyChecker()).thenReturn(mockRowLevelPolicyChecker);
    when(mockTaskContext.getRowLevelPolicyChecker(anyInt())).thenReturn(mockRowLevelPolicyChecker);
    when(mockTaskContext.getTaskLevelPolicyChecker(any(TaskState.class), anyInt()))
            .thenReturn(mock(TaskLevelPolicyChecker.class));
    for (int i = 0; i < numForks; ++i) {
        when(mockTaskContext.getDataWriterBuilder(numForks, i))
                .thenReturn(new RecordCollectingWriterBuilder(writerCollectors.get(i)));
    }
    return mockTaskContext;
}