Example usage for org.apache.hadoop.fs FileSystem getLocal

List of usage examples for org.apache.hadoop.fs FileSystem getLocal

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileSystem
  6. getLocal

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem getLocal.

Prototype

public static LocalFileSystem getLocal(Configuration conf) throws IOException

Source Link

Document

Get the local FileSystem.

Usage

From source file:gaffer.accumulostore.inputformat.InputFormatTest.java

License:Apache License

private void shouldReturnCorrectDataToMapReduceJob(final KeyPackage kp, final View view,
        final String instanceName, final Set<String> expectedResults) throws Exception {
    final AccumuloStore store = new MockAccumuloStore();
    final Schema schema = Schema.fromJson(StreamUtil.schemas(getClass()));
    final AccumuloProperties properties = AccumuloProperties
            .loadStoreProperties(StreamUtil.storeProps(getClass()));
    switch (kp) {
    case BYTE_ENTITY_KEY_PACKAGE:
        properties.setKeyPackageClass(ByteEntityKeyPackage.class.getName());
        properties.setInstanceName(instanceName + "_BYTE_ENTITY");
        break;/*from   w  w  w.  j  a va2 s.co  m*/
    case CLASSIC_KEY_PACKAGE:
        properties.setKeyPackageClass(ClassicKeyPackage.class.getName());
        properties.setInstanceName(instanceName + "_CLASSIC");
    }
    try {
        store.initialise(schema, properties);
    } catch (StoreException e) {
        fail("StoreException thrown: " + e);
    }
    setupGraph(store);

    // Set up local conf
    final JobConf conf = new JobConf();
    conf.set("fs.default.name", "file:///");
    conf.set("mapred.job.tracker", "local");
    final FileSystem fs = FileSystem.getLocal(conf);

    // Update configuration with instance, table name, etc.
    store.updateConfiguration(conf, view);

    // Run Driver
    final File outputFolder = testFolder.newFolder();
    FileUtils.deleteDirectory(outputFolder);
    final Driver driver = new Driver(outputFolder.getAbsolutePath());
    driver.setConf(conf);
    driver.run(new String[] {});

    // Read results and check correct
    final SequenceFile.Reader reader = new SequenceFile.Reader(fs, new Path(outputFolder + "/part-m-00000"),
            conf);
    final Text text = new Text();
    final Set<String> results = new HashSet<>();
    while (reader.next(text)) {
        results.add(text.toString());
    }
    reader.close();
    assertEquals(expectedResults, results);
    FileUtils.deleteDirectory(outputFolder);
}

From source file:gaffer.accumulostore.integration.AddElementsFromHdfsIT.java

License:Apache License

@Test
public void shouldAddElementsFromHdfsWhenOutputDirectoryAlreadyExists() throws Exception {
    final FileSystem fs = FileSystem.getLocal(createLocalConf());
    fs.mkdirs(new Path(outputDir));

    addElementsFromHdfs(ByteEntityKeyPackage.class);
    addElementsFromHdfs(ClassicKeyPackage.class);
}

From source file:gaffer.accumulostore.integration.AddElementsFromHdfsIT.java

License:Apache License

@Test
public void shouldAddElementsFromHdfsWhenFailureDirectoryAlreadyExists() throws Exception {
    final FileSystem fs = FileSystem.getLocal(createLocalConf());
    fs.mkdirs(new Path(failureDir));

    addElementsFromHdfs(ByteEntityKeyPackage.class);
    addElementsFromHdfs(ClassicKeyPackage.class);
}

From source file:gaffer.accumulostore.integration.AddElementsFromHdfsIT.java

License:Apache License

@Test
public void shouldThrowExceptionWhenAddElementsFromHdfsWhenOutputDirectoryContainsFiles() throws Exception {
    final FileSystem fs = FileSystem.getLocal(createLocalConf());
    fs.mkdirs(new Path(outputDir));
    try (final BufferedWriter writer = new BufferedWriter(
            new OutputStreamWriter(fs.create(new Path(outputDir + "/someFile.txt"), true)))) {
        writer.write("Some content");
    }//  w w w .j  a  v a2  s. co  m

    try {
        addElementsFromHdfs(ByteEntityKeyPackage.class);
        fail("Exception expected");
    } catch (final OperationException e) {
        assertEquals("Output directory exists and is not empty: " + outputDir, e.getCause().getMessage());
    }

    try {
        addElementsFromHdfs(ClassicKeyPackage.class);
        fail("Exception expected");
    } catch (final OperationException e) {
        assertEquals("Output directory exists and is not empty: " + outputDir, e.getCause().getMessage());
    }
}

From source file:gaffer.accumulostore.integration.AddElementsFromHdfsIT.java

License:Apache License

@Test
public void shouldThrowExceptionWhenAddElementsFromHdfsWhenFailureDirectoryContainsFiles() throws Exception {
    final FileSystem fs = FileSystem.getLocal(createLocalConf());
    fs.mkdirs(new Path(failureDir));
    try (final BufferedWriter writer = new BufferedWriter(
            new OutputStreamWriter(fs.create(new Path(failureDir + "/someFile.txt"), true)))) {
        writer.write("Some content");
    }/*  www  . ja va 2 s  .c  om*/

    try {
        addElementsFromHdfs(ByteEntityKeyPackage.class);
        fail("Exception expected");
    } catch (final OperationException e) {
        assertEquals("Failure directory is not empty: " + failureDir, e.getCause().getMessage());
    }

    try {
        addElementsFromHdfs(ClassicKeyPackage.class);
        fail("Exception expected");
    } catch (final OperationException e) {
        assertEquals("Failure directory is not empty: " + failureDir, e.getCause().getMessage());
    }
}

From source file:gaffer.accumulostore.integration.AddElementsFromHdfsIT.java

License:Apache License

private void createInputFile() throws IOException, StoreException {
    final Path inputPath = new Path(inputDir);
    final Path inputFilePath = new Path(inputDir + "/file.txt");
    final FileSystem fs = FileSystem.getLocal(createLocalConf());
    fs.mkdirs(inputPath);/*from  w w w.  jav a  2  s.c om*/

    try (final BufferedWriter writer = new BufferedWriter(
            new OutputStreamWriter(fs.create(inputFilePath, true)))) {
        for (int i = 0; i < NUM_ENTITIES; i++) {
            writer.write(TestGroups.ENTITY + "," + VERTEX_ID_PREFIX + i + "\n");
        }
    }
}

From source file:gobblin.cluster.GobblinHelixJobLauncherTest.java

License:Apache License

@BeforeClass
public void setUp() throws Exception {
    TestingServer testingZKServer = this.closer.register(new TestingServer(-1));
    LOG.info("Testing ZK Server listening on: " + testingZKServer.getConnectString());

    URL url = GobblinHelixJobLauncherTest.class.getClassLoader()
            .getResource(GobblinHelixJobLauncherTest.class.getSimpleName() + ".conf");
    Assert.assertNotNull(url, "Could not find resource " + url);

    Config config = ConfigFactory.parseURL(url).withValue("gobblin.cluster.zk.connection.string",
            ConfigValueFactory.fromAnyRef(testingZKServer.getConnectString())).resolve();

    String zkConnectingString = config.getString(GobblinClusterConfigurationKeys.ZK_CONNECTION_STRING_KEY);
    String helixClusterName = config.getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY);

    HelixUtils.createGobblinHelixCluster(zkConnectingString, helixClusterName);

    this.helixManager = HelixManagerFactory.getZKHelixManager(helixClusterName,
            TestHelper.TEST_HELIX_INSTANCE_NAME, InstanceType.CONTROLLER, zkConnectingString);
    this.closer.register(new Closeable() {
        @Override/* ww w.  j  ava 2 s .c  o  m*/
        public void close() throws IOException {
            helixManager.disconnect();
        }
    });
    this.helixManager.connect();

    Properties properties = ConfigUtils.configToProperties(config);

    this.localFs = FileSystem.getLocal(new Configuration());

    this.appWorkDir = new Path(GobblinHelixJobLauncherTest.class.getSimpleName());
    this.closer.register(new Closeable() {
        @Override
        public void close() throws IOException {
            if (localFs.exists(appWorkDir)) {
                localFs.delete(appWorkDir, true);
            }
        }
    });

    this.jobName = config.getString(ConfigurationKeys.JOB_NAME_KEY);

    this.jobOutputFile = new File(config.getString(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR),
            config.getString(ConfigurationKeys.WRITER_FILE_PATH) + File.separator
                    + config.getString(ConfigurationKeys.WRITER_FILE_NAME));

    // Prepare the source Json file
    File sourceJsonFile = new File(this.appWorkDir.toString(), TestHelper.TEST_JOB_NAME + ".json");
    TestHelper.createSourceJsonFile(sourceJsonFile);
    properties.setProperty(ConfigurationKeys.SOURCE_FILEBASED_FILES_TO_PULL, sourceJsonFile.getAbsolutePath());

    this.gobblinHelixJobLauncher = this.closer.register(new GobblinHelixJobLauncher(properties,
            this.helixManager, this.appWorkDir, ImmutableList.<Tag<?>>of()));

    this.gobblinTaskRunner = new GobblinTaskRunner(TestHelper.TEST_APPLICATION_NAME,
            TestHelper.TEST_HELIX_INSTANCE_NAME, TestHelper.TEST_APPLICATION_ID, TestHelper.TEST_TASK_RUNNER_ID,
            config, Optional.of(appWorkDir));

    String stateStoreType = properties.getProperty(ConfigurationKeys.STATE_STORE_TYPE_KEY,
            ConfigurationKeys.DEFAULT_STATE_STORE_TYPE);

    ClassAliasResolver<DatasetStateStore.Factory> resolver = new ClassAliasResolver<>(
            DatasetStateStore.Factory.class);

    DatasetStateStore.Factory stateStoreFactory = resolver.resolveClass(stateStoreType).newInstance();

    this.datasetStateStore = stateStoreFactory.createStateStore(config);

    this.thread = new Thread(new Runnable() {
        @Override
        public void run() {
            gobblinTaskRunner.start();
        }
    });
    this.thread.start();
}

From source file:gobblin.cluster.GobblinHelixTaskTest.java

License:Apache License

@BeforeClass
public void setUp() throws IOException {
    Configuration configuration = new Configuration();
    configuration.setInt(ConfigurationKeys.TASK_EXECUTOR_THREADPOOL_SIZE_KEY, 1);
    this.taskExecutor = new TaskExecutor(configuration);

    this.helixManager = Mockito.mock(HelixManager.class);
    Mockito.when(this.helixManager.getInstanceName()).thenReturn(GobblinHelixTaskTest.class.getSimpleName());
    this.taskStateTracker = new GobblinHelixTaskStateTracker(new Properties(), this.helixManager);

    this.localFs = FileSystem.getLocal(configuration);
    this.appWorkDir = new Path(GobblinHelixTaskTest.class.getSimpleName());
    this.taskOutputDir = new Path(this.appWorkDir, "output");
}

From source file:gobblin.cluster.HelixUtilsTest.java

License:Apache License

@BeforeClass
public void setUp() throws IOException {
    this.configuration = new Configuration();
    this.fileSystem = FileSystem.getLocal(this.configuration);
    this.tokenFilePath = new Path(HelixUtilsTest.class.getSimpleName(), "token");
    this.token = new Token<>();
    this.token.setKind(new Text("test"));
    this.token.setService(new Text("test"));
}

From source file:gobblin.commit.FsRenameCommitStepTest.java

License:Apache License

@BeforeClass
public void setUp() throws IOException {
    this.fs = FileSystem.getLocal(new Configuration());
    this.fs.delete(new Path(ROOT_DIR), true);

    Path dir1 = new Path(ROOT_DIR, "dir1");
    Path dir2 = new Path(ROOT_DIR, "dir2");

    this.fs.mkdirs(dir1);
    this.fs.mkdirs(dir2);

    Path src = new Path(dir1, "file");
    Path dst = new Path(dir2, "file");
    this.fs.createNewFile(src);

    this.step = (FsRenameCommitStep) new FsRenameCommitStep.Builder<>().from(src).to(dst).withProps(new State())
            .build();//from  w  w w  .j  a  v  a2 s . c  o  m
}