Example usage for org.apache.hadoop.fs FileSystem getLocal

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem getLocal.

Prototype

public static LocalFileSystem getLocal(Configuration conf) throws IOException

Source Link

Document

Get the local FileSystem.

Usage

From source file:gobblin.data.management.retention.integration.HiveRetentionTest.java

License:Apache License

@BeforeClass
public void setup() throws Exception {
    this.hiveMetastoreTestUtils = LocalHiveMetastoreTestUtils.getInstance();
    fs = FileSystem.getLocal(new Configuration());
    testTempPath = new Path(Files.createTempDir().getAbsolutePath(), "HiveRetentionTest");
    fs.mkdirs(testTempPath);//from w  w w . j  a v a  2  s  .  c o  m
}

From source file:gobblin.data.management.version.finder.DatePartitionedHiveVersionFinderTest.java

License:Apache License

@BeforeClass
public void setup() throws Exception {
    this.fs = FileSystem.getLocal(new Configuration());
    this.hiveMetastoreTestUtils = LocalHiveMetastoreTestUtils.getInstance();
    this.hiveMetastoreTestUtils.dropDatabaseIfExists(this.dbName);
    this.hiveMetastoreTestUtils.createTestDb(this.dbName);
}

From source file:gobblin.metastore.FsStateStoreTest.java

License:Apache License

@AfterClass
public void tearDown() throws IOException {
    FileSystem fs = FileSystem.getLocal(new Configuration(false));
    Path rootDir = new Path("metastore-test");
    if (fs.exists(rootDir)) {
        fs.delete(rootDir, true);//from  ww w. java  2s.  c  om
    }
}

From source file:gobblin.metastore.nameParser.GuidDatasetUrnStateStoreNameParserTest.java

License:Apache License

@BeforeTest
public void setUp() throws IOException {
    this.jobStateRootDir = new Path("testStateStoreParser");
    this.testFs = FileSystem.getLocal(new Configuration());
}

From source file:gobblin.runtime.commit.CommitSequenceTest.java

License:Apache License

@BeforeClass
public void setUp() throws IOException {
    this.fs = FileSystem.getLocal(new Configuration());

    this.fs.delete(new Path(ROOT_DIR), true);

    Path storeRootDir = new Path(ROOT_DIR, "store");

    Path dir1 = new Path(ROOT_DIR, "dir1");
    Path dir2 = new Path(ROOT_DIR, "dir2");

    this.fs.mkdirs(dir1);
    this.fs.mkdirs(dir2);

    Path src1 = new Path(dir1, "file1");
    Path src2 = new Path(dir2, "file2");
    Path dst1 = new Path(dir2, "file1");
    Path dst2 = new Path(dir1, "file2");
    this.fs.createNewFile(src1);
    this.fs.createNewFile(src2);

    DatasetState ds = new DatasetState("job-name", "job-id");
    ds.setDatasetUrn("urn");
    ds.setNoJobFailure();/*from   w ww . java  2s .  com*/

    State state = new State();
    state.setProp(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY, storeRootDir.toString());

    this.sequence = new CommitSequence.Builder().withJobName("testjob").withDatasetUrn("testurn")
            .beginStep(FsRenameCommitStep.Builder.class).from(src1).to(dst1).withProps(state).endStep()
            .beginStep(FsRenameCommitStep.Builder.class).from(src2).to(dst2).withProps(state).endStep()
            .beginStep(DatasetStateCommitStep.Builder.class).withDatasetUrn("urn").withDatasetState(ds)
            .withProps(state).endStep().build();
}

From source file:gobblin.runtime.commit.FsCommitSequenceStoreTest.java

License:Apache License

@BeforeClass
public void setUp() throws IOException {
    FileSystem fs = FileSystem.getLocal(new Configuration());
    this.store = new FsCommitSequenceStore(fs, new Path("commit-sequence-store-test"));

    State props = new State();
    props.setId("propsId");
    props.setProp("prop1", "valueOfProp1");
    props.setProp("prop2", "valueOfProp2");
    DatasetState datasetState = new DatasetState();

    datasetState.setDatasetUrn(this.datasetUrn);
    datasetState.incrementJobFailures();
    this.sequence = new CommitSequence.Builder().withJobName("testjob").withDatasetUrn("testurn")
            .beginStep(FsRenameCommitStep.Builder.class).from(new Path("/ab/cd")).to(new Path("/ef/gh"))
            .withProps(props).endStep().beginStep(DatasetStateCommitStep.Builder.class)
            .withDatasetUrn(this.datasetUrn).withDatasetState(datasetState).withProps(props).endStep().build();
}

From source file:gobblin.runtime.DummyJobContext.java

License:Apache License

@Override
protected FsDatasetStateStore createStateStore(Config config) throws IOException {
    return new NoopDatasetStateStore(FileSystem.getLocal(new Configuration()), "");
}

From source file:gobblin.runtime.FileBasedJobLockTest.java

License:Open Source License

@BeforeClass
public void setUp() throws IOException {
    this.fs = FileSystem.getLocal(new Configuration());
    this.path = new Path("MRJobLockTest");
    if (!this.fs.exists(this.path)) {
        this.fs.mkdirs(this.path);
    }//w w w .  j a v a2s  . c om
}

From source file:gobblin.runtime.FsDatasetStateStoreTest.java

License:Apache License

@AfterClass
public void tearDown() throws IOException {
    FileSystem fs = FileSystem.getLocal(new Configuration(false));
    Path rootDir = new Path(FsDatasetStateStoreTest.class.getSimpleName());
    if (fs.exists(rootDir)) {
        fs.delete(rootDir, true);/*  w ww.  j  av  a  2s .  c  o m*/
    }
}

From source file:gobblin.runtime.JobLauncherTestBase.java

License:Open Source License

@SuppressWarnings("unchecked")
protected void runTestWithCancellation(final Properties jobProps) throws Exception {
    final JobLauncher jobLauncher = JobLauncherFactory.newJobLauncher(this.properties);

    final AtomicBoolean isCancelled = new AtomicBoolean(false);
    // This thread will cancel the job after some time
    Thread thread = new Thread(new Runnable() {
        @Override// w  ww . ja v  a 2s .co m
        public void run() {
            try {
                Thread.sleep(500);
                jobLauncher.cancelJob(jobProps);
                isCancelled.set(true);
            } catch (Exception je) {
                // Ignored
            }
        }
    });
    thread.start();

    jobLauncher.launchJob(jobProps, null);

    Assert.assertTrue(isCancelled.get());

    String jobName = jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY);
    String jobId = jobProps.getProperty(ConfigurationKeys.JOB_ID_KEY);
    List<JobState> jobStateList = (List<JobState>) this.jobStateStore.getAll(jobName, jobId + ".jst");
    Assert.assertTrue(jobStateList.isEmpty());

    FileSystem lfs = FileSystem.getLocal(new Configuration());
    Path jobLockFile = new Path(jobProps.getProperty(ConfigurationKeys.JOB_LOCK_DIR_KEY),
            jobName + FileBasedJobLock.LOCK_FILE_EXTENSION);
    Assert.assertFalse(lfs.exists(jobLockFile));
}