Example usage for org.apache.hadoop.hdfs DistributedFileSystem DistributedFileSystem

Introduction

In this page you can find the example usage for org.apache.hadoop.hdfs DistributedFileSystem DistributedFileSystem.

Prototype

public DistributedFileSystem() 

Source Link

Usage

From source file:org.apache.carbondata.core.carbon.datastorage.filesystem.HDFSCarbonFileTest.java

License:Apache License

@Test
public void testListFilesWithOutDirectoryPermission() {
    hdfsCarbonFile = new HDFSCarbonFile(fileStatusWithOutDirectoryPermission);
    new MockUp<FileStatus>() {
        @Mock/* w  w  w .  j  a v  a 2s  .c om*/
        public boolean isDirectory() {
            return false;
        }

    };

    new MockUp<Path>() {
        @Mock
        public FileSystem getFileSystem(Configuration conf) throws IOException {
            return new DistributedFileSystem();
        }

    };

    new MockUp<FileStatus>() {
        @Mock
        public Path getPath() {
            return new Path(fileName);
        }

    };
    assertEquals(hdfsCarbonFile.listFiles(), null);
}

---

From source file:org.apache.carbondata.core.carbon.datastorage.filesystem.HDFSCarbonFileTest.java

License:Apache License

@Test
public void testListFilesForNullListStatus() {
    new MockUp<Path>() {
        @Mock/*from   w  w  w  .ja  v a 2  s .c om*/
        public FileSystem getFileSystem(Configuration conf) throws IOException {
            return new DistributedFileSystem();
        }

    };
    new MockUp<DistributedFileSystem>() {
        @Mock
        public FileStatus[] listStatus(Path var1) throws IOException {

            return null;
        }

    };
    hdfsCarbonFile = new HDFSCarbonFile(fileStatus);
    assertEquals(hdfsCarbonFile.listFiles().length, 0);
}

---

From source file:org.apache.carbondata.core.carbon.datastorage.filesystem.HDFSCarbonFileTest.java

License:Apache License

@Test
public void testListDirectory() {
    hdfsCarbonFile = new HDFSCarbonFile(fileStatus);
    new MockUp<Path>() {
        @Mock//from   w  w w  .  j a  va2 s .  c  om
        public FileSystem getFileSystem(Configuration conf) throws IOException {
            return new DistributedFileSystem();
        }

    };
    new MockUp<DistributedFileSystem>() {
        @Mock
        public FileStatus[] listStatus(Path var1) throws IOException {

            FileStatus[] fileStatus = new FileStatus[] {
                    new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName)) };
            return fileStatus;
        }

    };

    assertEquals(hdfsCarbonFile.listFiles().length, 1);
}

---

From source file:org.apache.carbondata.core.carbon.datastorage.filesystem.HDFSCarbonFileTest.java

License:Apache License

@Test
public void testListFilesWithCarbonFilter() {
    CarbonFileFilter carbonFileFilter = new CarbonFileFilter() {

        @Override/*from w ww .  j  a v a 2 s.c o m*/
        public boolean accept(CarbonFile file) {
            return true;
        }
    };
    new MockUp<FileStatus>() {
        @Mock
        public boolean isDirectory() {
            return true;
        }

    };

    new MockUp<Path>() {
        @Mock
        public FileSystem getFileSystem(Configuration conf) throws IOException {
            return new DistributedFileSystem();
        }

    };

    new MockUp<FileStatus>() {
        @Mock
        public Path getPath() {
            return new Path(fileName);
        }

    };
    new MockUp<DistributedFileSystem>() {
        @Mock
        public FileStatus[] listStatus(Path var1) throws IOException {

            FileStatus fileStatus[] = new FileStatus[] {
                    new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName)) };
            return fileStatus;
        }

    };
    hdfsCarbonFile = new HDFSCarbonFile(fileStatus);
    assertEquals(hdfsCarbonFile.listFiles(carbonFileFilter).length, 1);
}

---

From source file:org.apache.carbondata.core.carbon.datastorage.filesystem.HDFSCarbonFileTest.java

License:Apache License

@Test
public void testlistFilesWithoutFilter() {
    CarbonFileFilter carbonFileFilter = new CarbonFileFilter() {

        @Override/* w ww  .  ja  v a2 s.c o  m*/
        public boolean accept(CarbonFile file) {
            return false;
        }
    };
    new MockUp<Path>() {
        @Mock
        public FileSystem getFileSystem(Configuration conf) throws IOException {
            return new DistributedFileSystem();
        }

    };
    new MockUp<DistributedFileSystem>() {
        @Mock
        public FileStatus[] listStatus(Path var1) throws IOException {

            FileStatus[] fileStatus = new FileStatus[] {
                    new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName)) };
            return fileStatus;
        }

    };
    hdfsCarbonFile = new HDFSCarbonFile(fileStatus);
    assertEquals(hdfsCarbonFile.listFiles(carbonFileFilter).length, 0);
}

---

From source file:org.apache.carbondata.core.carbon.datastorage.filesystem.HDFSCarbonFileTest.java

License:Apache License

@Test
public void testGetParentFileForNull() {

    new MockUp<Path>() {
        @Mock/* ww  w.  j  ava2s.c  o m*/
        public Path getParent() {
            return null;
        }

    };
    new MockUp<FileStatus>() {
        @Mock
        public Path getPath() {
            return new Path(fileName);
        }

    };
    new MockUp<Path>() {
        @Mock
        public FileSystem getFileSystem(Configuration conf) throws IOException {
            return new DistributedFileSystem();
        }

    };

    new MockUp<FileStatus>() {
        @Mock
        public Path getPath() {
            return new Path(fileName);
        }

    };
    hdfsCarbonFile = new HDFSCarbonFile(fileStatus);
    assertEquals(hdfsCarbonFile.getParentFile(), null);
}

---

From source file:org.apache.carbondata.core.carbon.datastorage.filesystem.HDFSCarbonFileTest.java

License:Apache License

@Test
public void testGetParentFile() {
    new MockUp<Path>() {
        @Mock/*  w  ww  .  j ava 2s  . c  om*/
        public FileSystem getFileSystem(Configuration conf) throws IOException {
            return new DistributedFileSystem();
        }

    };
    new MockUp<Path>() {
        @Mock
        public Path getParent() {
            return new Path(fileName);
        }

    };
    new MockUp<FileStatus>() {
        @Mock
        public Path getPath() {
            return new Path(fileName);
        }

    };
    new MockUp<DistributedFileSystem>() {
        @Mock
        public FileStatus getFileStatus(Path file) throws IOException {

            return new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName));
        }

    };

    hdfsCarbonFile = new HDFSCarbonFile(fileStatus);
    assertTrue(hdfsCarbonFile.getParentFile() instanceof CarbonFile);
}

---

From source file:org.apache.carbondata.core.carbon.datastorage.filesystem.HDFSCarbonFileTest.java

License:Apache License

@Test
public void testrenameForceForDisributedSystem() {
    new MockUp<Path>() {
        @Mock/* www  .  j  a v  a2 s .com*/
        public FileSystem getFileSystem(Configuration conf) throws IOException {
            return new DistributedFileSystem();
        }

    };
    new MockUp<DistributedFileSystem>() {
        @Mock
        public void rename(Path src, Path dst, final Options.Rename... options) throws IOException {

        }

    };
    hdfsCarbonFile = new HDFSCarbonFile(fileStatus);
    assertEquals(hdfsCarbonFile.renameForce(fileName), true);

}

---

From source file:org.apache.carbondata.core.carbon.datastorage.filesystem.ViewFsCarbonFileTest.java

License:Apache License

@Test
public void testGetParentFIle() {
    new MockUp<Path>() {
        @Mock/*w  ww.j ava2  s  .  c  o  m*/
        public FileSystem getFileSystem(Configuration conf) throws IOException {
            return new DistributedFileSystem();
        }

    };
    new MockUp<Path>() {
        @Mock
        public Path getParent() {
            return new Path(file.getAbsolutePath());
        }

    };
    new MockUp<FileStatus>() {
        @Mock
        public Path getPath() {
            return new Path(file.getAbsolutePath());
        }

    };
    new MockUp<DistributedFileSystem>() {
        @Mock
        public FileStatus getFileStatus(Path f) throws IOException {

            return new FileStatus(12L, true, 60, 120l, 180L, new Path(file.getAbsolutePath()));
        }

    };

    viewFSCarbonFile = new ViewFSCarbonFile(fileStatus);
    assertFalse(viewFSCarbonFile.getParentFile().equals(null));
}

---

From source file:org.apache.druid.storage.hdfs.HdfsDataSegmentPusherTest.java

License:Apache License

@Test
public void shouldMakeHDFSCompliantSegmentOutputPath() {
    HadoopIngestionSpec schema;//w  w  w . j  a  va2s . c o m

    try {
        schema = objectMapper.readValue("{\n" + "    \"dataSchema\": {\n"
                + "        \"dataSource\": \"source\",\n" + "        \"metricsSpec\": [],\n"
                + "        \"granularitySpec\": {\n" + "            \"type\": \"uniform\",\n"
                + "            \"segmentGranularity\": \"hour\",\n"
                + "            \"intervals\": [\"2012-07-10/P1D\"]\n" + "        }\n" + "    },\n"
                + "    \"ioConfig\": {\n" + "        \"type\": \"hadoop\",\n"
                + "        \"segmentOutputPath\": \"hdfs://server:9100/tmp/druid/datatest\"\n" + "    }\n"
                + "}", HadoopIngestionSpec.class);
    } catch (Exception e) {
        throw new RuntimeException(e);
    }

    //DataSchema dataSchema = new DataSchema("dataSource", null, null, Gra)
    //schema = new HadoopIngestionSpec(dataSchema, ioConfig, HadoopTuningConfig.makeDefaultTuningConfig());
    HadoopDruidIndexerConfig cfg = new HadoopDruidIndexerConfig(
            schema.withTuningConfig(schema.getTuningConfig().withVersion("some:brand:new:version")));

    Bucket bucket = new Bucket(4711, new DateTime(2012, 07, 10, 5, 30, ISOChronology.getInstanceUTC()), 4712);
    Path path = JobHelper
            .makeFileNamePath(new Path(cfg.getSchema().getIOConfig().getSegmentOutputPath()),
                    new DistributedFileSystem(),
                    new DataSegment(cfg.getSchema().getDataSchema().getDataSource(),
                            cfg.getSchema().getDataSchema().getGranularitySpec().bucketInterval(bucket.time)
                                    .get(),
                            cfg.getSchema().getTuningConfig().getVersion(), null, null, null,
                            new NumberedShardSpec(bucket.partitionNum, 5000), -1, -1),
                    JobHelper.INDEX_ZIP, hdfsDataSegmentPusher);
    Assert.assertEquals(
            "hdfs://server:9100/tmp/druid/datatest/source/20120710T050000.000Z_20120710T060000.000Z/some_brand_new_version"
                    + "/4712_index.zip",
            path.toString());

    path = JobHelper
            .makeTmpPath(new Path(cfg.getSchema().getIOConfig().getSegmentOutputPath()),
                    new DistributedFileSystem(),
                    new DataSegment(cfg.getSchema().getDataSchema().getDataSource(),
                            cfg.getSchema().getDataSchema().getGranularitySpec().bucketInterval(bucket.time)
                                    .get(),
                            cfg.getSchema().getTuningConfig().getVersion(), null, null, null,
                            new NumberedShardSpec(bucket.partitionNum, 5000), -1, -1),
                    new TaskAttemptID("abc", 123, TaskType.REDUCE, 1, 0), hdfsDataSegmentPusher);
    Assert.assertEquals(
            "hdfs://server:9100/tmp/druid/datatest/source/20120710T050000.000Z_20120710T060000.000Z/some_brand_new_version"
                    + "/4712_index.zip.0",
            path.toString());

}

---

• «
• 1
• 2
• 3
• »