Example usage for org.apache.hadoop.fs FileSystem getLocal

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem getLocal.

Prototype

public static LocalFileSystem getLocal(Configuration conf) throws IOException 

Source Link

Document

Get the local FileSystem.

Usage

From source file:gobblin.data.management.copy.publisher.CopyDataPublisherTest.java

License:Apache License

@BeforeClass
public void setup() throws Exception {
    fs = FileSystem.getLocal(new Configuration());
    testClassTempPath = new Path(Files.createTempDir().getAbsolutePath(), "CopyDataPublisherTest");
    fs.delete(testClassTempPath, true);/*w  w w.j  a v  a 2s . c o  m*/
    log.info("Created a temp directory for CopyDataPublisherTest at " + testClassTempPath);
    fs.mkdirs(testClassTempPath);
}

---

From source file:gobblin.data.management.copy.publisher.DeletingCopyDataPublisherTest.java

License:Apache License

@BeforeClass
public void setup() throws Exception {
    fs = FileSystem.getLocal(new Configuration());
    testClassTempPath = new Path(Files.createTempDir().getAbsolutePath(), "DeletingCopyDataPublisherTest");
    fs.delete(testClassTempPath, true);/*w ww .j  a  va2  s. com*/
    log.info("Created a temp directory for CopyDataPublisherTest at " + testClassTempPath);
    fs.mkdirs(testClassTempPath);
}

---

From source file:gobblin.data.management.copy.recovery.RecoveryHelperTest.java

License:Apache License

@Test
public void testPersistFile() throws Exception {

    String content = "contents";

    File stagingDir = Files.createTempDir();
    stagingDir.deleteOnExit();/*from   w  ww .  j a  v a 2  s .  co  m*/

    File file = new File(stagingDir, "file");
    OutputStream os = new FileOutputStream(file);
    IOUtils.write(content, os);
    os.close();

    Assert.assertEquals(stagingDir.listFiles().length, 1);

    State state = new State();
    state.setProp(RecoveryHelper.PERSIST_DIR_KEY, this.tmpDir.getAbsolutePath());
    state.setProp(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR, "/publisher");

    File recoveryDir = new File(RecoveryHelper.getPersistDir(state).get().toUri().getPath());

    FileSystem fs = FileSystem.getLocal(new Configuration());

    CopyableFile copyableFile = CopyableFile.builder(fs, new FileStatus(0, false, 0, 0, 0, new Path("/file")),
            new Path("/dataset"), CopyConfiguration.builder(fs, state.getProperties())
                    .preserve(PreserveAttributes.fromMnemonicString("")).build())
            .build();

    CopySource.setWorkUnitGuid(state, Guid.fromHasGuid(copyableFile));

    RecoveryHelper recoveryHelper = new RecoveryHelper(FileSystem.getLocal(new Configuration()), state);

    recoveryHelper.persistFile(state, copyableFile, new Path(file.getAbsolutePath()));

    Assert.assertEquals(stagingDir.listFiles().length, 0);
    Assert.assertEquals(recoveryDir.listFiles().length, 1);

    File fileInRecovery = recoveryDir.listFiles()[0];
    Assert.assertEquals(IOUtils.readLines(new FileInputStream(fileInRecovery)).get(0), content);

    Optional<FileStatus> fileToRecover = recoveryHelper.findPersistedFile(state, copyableFile,
            Predicates.<FileStatus>alwaysTrue());
    Assert.assertTrue(fileToRecover.isPresent());
    Assert.assertEquals(fileToRecover.get().getPath().toUri().getPath(), fileInRecovery.getAbsolutePath());

    fileToRecover = recoveryHelper.findPersistedFile(state, copyableFile, Predicates.<FileStatus>alwaysFalse());
    Assert.assertFalse(fileToRecover.isPresent());

}

---

From source file:gobblin.data.management.copy.recovery.RecoveryHelperTest.java

License:Apache License

@Test
public void testPurge() throws Exception {
    String content = "contents";

    File persistDirBase = Files.createTempDir();
    persistDirBase.deleteOnExit();//from   ww  w  .  ja  v a  2  s.c  o m

    State state = new State();
    state.setProp(RecoveryHelper.PERSIST_DIR_KEY, persistDirBase.getAbsolutePath());
    state.setProp(RecoveryHelper.PERSIST_RETENTION_KEY, "1");

    RecoveryHelper recoveryHelper = new RecoveryHelper(FileSystem.getLocal(new Configuration()), state);
    File persistDir = new File(RecoveryHelper.getPersistDir(state).get().toString());
    persistDir.mkdir();

    File file = new File(persistDir, "file1");
    OutputStream os = new FileOutputStream(file);
    IOUtils.write(content, os);
    os.close();
    file.setLastModified(System.currentTimeMillis() - TimeUnit.HOURS.toMillis(2));

    File file2 = new File(persistDir, "file2");
    OutputStream os2 = new FileOutputStream(file2);
    IOUtils.write(content, os2);
    os2.close();

    Assert.assertEquals(persistDir.listFiles().length, 2);

    recoveryHelper.purgeOldPersistedFile();

    Assert.assertEquals(persistDir.listFiles().length, 1);
}

---

From source file:gobblin.data.management.copy.RecursiveCopyableDatasetTest.java

License:Apache License

@Test
public void testSimpleCopy() throws Exception {
    Path source = new Path("/source");
    Path target = new Path("/target");

    List<FileStatus> sourceFiles = Lists.newArrayList(createFileStatus(source, "file1"),
            createFileStatus(source, "file2"));
    List<FileStatus> targetFiles = Lists.newArrayList(createFileStatus(target, "file3"));

    Properties properties = new Properties();
    properties.setProperty(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR, target.toString());
    RecursiveCopyableDataset dataset = new TestRecursiveCopyableDataset(source, target, sourceFiles,
            targetFiles, properties);//  w  w w  .  j  av  a  2s .c  o m

    Collection<? extends CopyEntity> copyableFiles = dataset.getCopyableFiles(
            FileSystem.getLocal(new Configuration()),
            CopyConfiguration.builder(FileSystem.getLocal(new Configuration()), properties).build());

    Assert.assertEquals(copyableFiles.size(), 2);
    ClassifiedFiles classifiedFiles = classifyFiles(copyableFiles);
    Assert.assertTrue(classifiedFiles.getPathsToCopy().containsKey(new Path(source, "file1")));
    Assert.assertEquals(classifiedFiles.getPathsToCopy().get(new Path(source, "file1")),
            new Path(target, "file1"));
    Assert.assertTrue(classifiedFiles.getPathsToCopy().containsKey(new Path(source, "file2")));
    Assert.assertEquals(classifiedFiles.getPathsToCopy().get(new Path(source, "file2")),
            new Path(target, "file2"));
    Assert.assertEquals(classifiedFiles.getPathsToDelete().size(), 0);
}

---

From source file:gobblin.data.management.copy.RecursiveCopyableDatasetTest.java

License:Apache License

@Test
public void testCopyWithNonConflictingCollision() throws Exception {
    Path source = new Path("/source");
    Path target = new Path("/target");

    List<FileStatus> sourceFiles = Lists.newArrayList(createFileStatus(source, "file1", 1),
            createFileStatus(source, "file2"));
    List<FileStatus> targetFiles = Lists.newArrayList(createFileStatus(target, "file1", 1));

    Properties properties = new Properties();
    properties.setProperty(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR, target.toString());
    RecursiveCopyableDataset dataset = new TestRecursiveCopyableDataset(source, target, sourceFiles,
            targetFiles, properties);/* w  w w . ja v a2  s . c o m*/

    Collection<? extends CopyEntity> copyableFiles = dataset.getCopyableFiles(
            FileSystem.getLocal(new Configuration()),
            CopyConfiguration.builder(FileSystem.getLocal(new Configuration()), properties).build());

    Assert.assertEquals(copyableFiles.size(), 1);
    ClassifiedFiles classifiedFiles = classifyFiles(copyableFiles);
    Assert.assertTrue(classifiedFiles.getPathsToCopy().containsKey(new Path(source, "file2")));
    Assert.assertEquals(classifiedFiles.getPathsToCopy().get(new Path(source, "file2")),
            new Path(target, "file2"));
    Assert.assertEquals(classifiedFiles.getPathsToDelete().size(), 0);
}

---

From source file:gobblin.data.management.copy.RecursiveCopyableDatasetTest.java

License:Apache License

@Test
public void testCopyWithConflictingCollisionDueToSize() throws Exception {
    Path source = new Path("/source");
    Path target = new Path("/target");

    List<FileStatus> sourceFiles = Lists.newArrayList(createFileStatus(source, "file1", 1),
            createFileStatus(source, "file2"));
    List<FileStatus> targetFiles = Lists.newArrayList(createFileStatus(target, "file1", 2));

    Properties properties = new Properties();
    properties.setProperty(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR, target.toString());
    RecursiveCopyableDataset dataset = new TestRecursiveCopyableDataset(source, target, sourceFiles,
            targetFiles, properties);// www . ja v a 2 s  .  c o  m

    try {
        Collection<? extends CopyEntity> copyableFiles = dataset.getCopyableFiles(
                FileSystem.getLocal(new Configuration()),
                CopyConfiguration.builder(FileSystem.getLocal(new Configuration()), properties).build());
        Assert.fail();
    } catch (IOException ioe) {
        // should throw exception due to collision
    }
}

---

From source file:gobblin.data.management.copy.RecursiveCopyableDatasetTest.java

License:Apache License

@Test
public void testCopyWithConflictingCollisionDueToModtime() throws Exception {
    Path source = new Path("/source");
    Path target = new Path("/target");

    List<FileStatus> sourceFiles = Lists.newArrayList(createFileStatus(source, "file1", 1, 10),
            createFileStatus(source, "file2"));
    List<FileStatus> targetFiles = Lists.newArrayList(createFileStatus(target, "file1", 1, 9));

    Properties properties = new Properties();
    properties.setProperty(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR, target.toString());
    RecursiveCopyableDataset dataset = new TestRecursiveCopyableDataset(source, target, sourceFiles,
            targetFiles, properties);// w  w w.j a  v  a 2 s. co m

    try {
        Collection<? extends CopyEntity> copyableFiles = dataset.getCopyableFiles(
                FileSystem.getLocal(new Configuration()),
                CopyConfiguration.builder(FileSystem.getLocal(new Configuration()), properties).build());
        Assert.fail();
    } catch (IOException ioe) {
        // should throw exception due to collision
    }
}

---

From source file:gobblin.data.management.copy.RecursiveCopyableDatasetTest.java

License:Apache License

@Test
public void testCopyWithUpdate() throws Exception {
    Path source = new Path("/source");
    Path target = new Path("/target");

    FileStatus targetFile1 = createFileStatus(target, "file1", 2);

    List<FileStatus> sourceFiles = Lists.newArrayList(createFileStatus(source, "file1", 1),
            createFileStatus(source, "file2"));
    List<FileStatus> targetFiles = Lists.newArrayList(targetFile1);

    Properties properties = new Properties();
    properties.setProperty(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR, target.toString());
    properties.setProperty(RecursiveCopyableDataset.UPDATE_KEY, "true");
    RecursiveCopyableDataset dataset = new TestRecursiveCopyableDataset(source, target, sourceFiles,
            targetFiles, properties);/*from  w  w  w  . j av a 2  s. com*/

    FileSystem targetFsUnderlying = FileSystem.getLocal(new Configuration());
    FileSystem targetFs = Mockito.spy(targetFsUnderlying);
    Mockito.doReturn(targetFile1).when(targetFs).getFileStatus(new Path(target, "file1"));

    Collection<? extends CopyEntity> copyableFiles = dataset.getCopyableFiles(targetFs,
            CopyConfiguration.builder(FileSystem.getLocal(new Configuration()), properties).build());

    Assert.assertEquals(copyableFiles.size(), 3);
    ClassifiedFiles classifiedFiles = classifyFiles(copyableFiles);
    Assert.assertTrue(classifiedFiles.getPathsToCopy().containsKey(new Path(source, "file1")));
    Assert.assertEquals(classifiedFiles.getPathsToCopy().get(new Path(source, "file1")),
            new Path(target, "file1"));
    Assert.assertTrue(classifiedFiles.getPathsToCopy().containsKey(new Path(source, "file2")));
    Assert.assertEquals(classifiedFiles.getPathsToCopy().get(new Path(source, "file2")),
            new Path(target, "file2"));
    Assert.assertEquals(classifiedFiles.getPathsToDelete().size(), 1);
    Assert.assertTrue(classifiedFiles.getPathsToDelete().contains(new Path(target, "file1")));
}

---

From source file:gobblin.data.management.copy.RecursiveCopyableDatasetTest.java

License:Apache License

@Test
public void testCopyWithDeleteTarget() throws Exception {
    Path source = new Path("/source");
    Path target = new Path("/target");

    List<FileStatus> sourceFiles = Lists.newArrayList(createFileStatus(source, "file1"));
    List<FileStatus> targetFiles = Lists.newArrayList(createFileStatus(target, "file3"));

    Properties properties = new Properties();
    properties.setProperty(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR, target.toString());
    properties.setProperty(RecursiveCopyableDataset.DELETE_KEY, "true");
    RecursiveCopyableDataset dataset = new TestRecursiveCopyableDataset(source, target, sourceFiles,
            targetFiles, properties);// w w w  .jav  a 2 s  .  c o  m

    Collection<? extends CopyEntity> copyableFiles = dataset.getCopyableFiles(
            FileSystem.getLocal(new Configuration()),
            CopyConfiguration.builder(FileSystem.getLocal(new Configuration()), properties).build());

    Assert.assertEquals(copyableFiles.size(), 2);
    ClassifiedFiles classifiedFiles = classifyFiles(copyableFiles);
    Assert.assertTrue(classifiedFiles.getPathsToCopy().containsKey(new Path(source, "file1")));
    Assert.assertEquals(classifiedFiles.getPathsToCopy().get(new Path(source, "file1")),
            new Path(target, "file1"));
    Assert.assertEquals(classifiedFiles.getPathsToDelete().size(), 1);
    Assert.assertTrue(classifiedFiles.getPathsToDelete().contains(new Path(target, "file3")));

    CommitStepCopyEntity entity = (CommitStepCopyEntity) Iterables
            .filter(copyableFiles, new Predicate<CopyEntity>() {
                @Override
                public boolean apply(@Nullable CopyEntity copyEntity) {
                    return copyEntity instanceof CommitStepCopyEntity;
                }
            }).iterator().next();
    DeleteFileCommitStep step = (DeleteFileCommitStep) entity.getStep();
    Assert.assertFalse(step.getParentDeletionLimit().isPresent());
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »