Example usage for org.apache.hadoop.fs Path Path

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path Path.

Prototype

public Path(URI aUri)

Source Link

Document

Construct a path from a URI

Usage

From source file:BwaAlignmentBase.java

License:Open Source License

public ArrayList<String> copyResults(String outputSamFileName) {
    ArrayList<String> returnedValues = new ArrayList<String>();
    String outputDir = this.bwaInterpreter.getOutputHdfsDir();

    Configuration conf = new Configuration();
    try {//from  ww  w  .  j  a  va2s .c  om
        FileSystem fs = FileSystem.get(conf);

        fs.copyFromLocalFile(new Path(this.bwaInterpreter.getOutputFile()),
                new Path(outputDir + "/" + outputSamFileName));

        // Delete the old results file
        File tmpSamFullFile = new File(this.bwaInterpreter.getOutputFile());
        tmpSamFullFile.delete();
    } catch (IOException e) {
        e.printStackTrace();
        this.LOG.error(e.toString());
    }

    returnedValues.add(outputDir + "/" + outputSamFileName);

    return returnedValues;
}

From source file:TestRawParascaleFileSystemBase.java

License:Apache License

public void testStartLocalOutput() throws IOException {
    final Path localTmpPath = new Path("/tmp/test_" + System.currentTimeMillis());
    final Path startLocalOutput = fs.startLocalOutput(new Path("input/test"), localTmpPath);
    assertEquals(localTmpPath, startLocalOutput);
}

From source file:TestRawParascaleFileSystemBase.java

License:Apache License

public void testCompleteLocalOutput() throws IOException {
    final Path localTmpPath = new Path("/tmp/test_" + System.currentTimeMillis());
    final Path psdfsPath = new Path("test");
    final Path startLocalOutput = fs.startLocalOutput(new Path("input/test"), localTmpPath);
    assertEquals(localTmpPath, startLocalOutput);
    final File tmpFile = new File(startLocalOutput.toString());
    assertTrue(tmpFile.createNewFile());
    final String testTxt = "a Test Text";
    writeToFile(testTxt, tmpFile);/*from  w  w  w  . j  a v  a 2s.  co m*/

    fs.completeLocalOutput(psdfsPath, startLocalOutput);
    final File targetFile = fs.pathToFile(psdfsPath);

    assertFalse(tmpFile.exists());
    assertEquals(testTxt, readFile(targetFile));

}

From source file:TestRawParascaleFileSystemBase.java

License:Apache License

public void testGetFileStatus() throws IOException {

    final FileStatus fileStatus = fs.getFileStatus(fs.getHomeDirectory());
    assertEquals(64 * 1024 * 1024, fileStatus.getBlockSize());

    fs.getConf().setLong(String.format(RawParascaleFileSystem.PS_VFS_BLOCKSIZE_FORMAT, "myvirtualFs"), 64);
    try {/*from  w w w.  jav  a2  s  .c om*/

        fs.getFileStatus(new Path("psdfs://myvirtualFs@10.200.2.10"));
        fail("wrong fs");
    } catch (final IllegalArgumentException e) {
        // expected
    }

}

From source file:TestRawParascaleFileSystemBase.java

License:Apache License

public void testPathToFile() {
    // test absolute
    assertEquals(//from  w w  w. j a v a 2 s.c  o  m
            new File(getTempDir(), String.format("%s/%s/%s/user/hadoop", mountPoint, controlNode, virtualFs)),
            fs.pathToFile(fs.getHomeDirectory()));
    // test relative

    assertEquals(
            new File(getTempDir(),
                    String.format("%s/%s/%s/user/hadoop/test", mountPoint, controlNode, virtualFs)),
            fs.pathToFile(new Path("test")));
}

From source file:TestRawParascaleFileSystemBase.java

License:Apache License

public void testGetVirtualFSFromPath() {
    assertEquals("myVirtualFs",
            fs.getVirtualFSFromPath(new Path("psdfs://myVirtualFs@192.168.2.2/user/hadoop"), true));
    assertEquals("filesystem", fs.getVirtualFSFromPath(new Path("user/hadoop"), false));
    assertEquals("filesystem", fs.getVirtualFSFromPath(new Path("/user/hadoop"), false));
    assertEquals("filesystem", fs.getVirtualFSFromPath(new Path("/"), false));
}

From source file:TestRawParascaleFileSystemBase.java

License:Apache License

public void testCopyFromLocal() throws Exception {
    // simple read write test -- the actual code is reused from hadoop
    // RawLocalFilesystem
    final File file = new File(tempDir, "copySrc");
    createRandomFile(RANDOM, file, 1024 * 1024);
    final String checksum = getMD5Checksum(file);
    final Path path = new Path("output");
    assertTrue(fs.mkdirs(path));//from ww  w . j a v  a 2  s .co  m
    final Path target = new Path(path, "target");
    fs.copyFromLocalFile(new Path(file.getAbsolutePath()), target);
    assertTrue("outputfile does not exist", new File(getLocalPathToWorkingDir(), "output/target").exists());
    assertTrue(fs.exists(target));
    assertEquals(checksum, getMD5Checksum(target, fs));
}

From source file:TestIndexServer.java

License:Open Source License

public void testDataPartMoniter() throws IOException, InterruptedException {
    status = initializeindexconf();/*from   w  w  w.j  av  a  2 s.co  m*/

    fs.delete(new Path(status.gettablelocation()), true);
    FSDataOutputStream fos = fs.create(new Path(status.gettablelocation() + "testfile"));
    fos.write("test".getBytes());
    fos.close();

    IndexServer server = new IndexServer(null);
    server.testmode = true;
    server.metaDataChecker.start();
    Thread.sleep(1000);
    server.dataPartMoniter.start();
    Thread.sleep(1000);
    Set<String> keys = server.tablefiles.keySet();
    for (String key : keys) {
        for (HashSet<String> hashstrs : server.tablefiles.get(key).values()) {
            for (String str : hashstrs) {
                assertEquals(new Path(str).getName(), "testfile");
            }
        }
    }
}

From source file:TestIndexServer.java

License:Open Source License

public void testIndexPartMoniter() throws IOException, InterruptedException {
    status = initializeindexconf();//w  w  w.j  a  v  a  2 s.  c  om
    fs.delete(new Path(status.gettablelocation()), true);
    fs.delete(new Path(status.gettablelocation()), true);
    FSDataOutputStream fos = fs.create(new Path(status.gettablelocation() + "testfile"));
    fos.write("test".getBytes());
    fos.close();

    IndexServer server = new IndexServer(null);
    server.testmode = true;
    server.metaDataChecker.start();
    Thread.sleep(1000);
    server.dataPartMoniter.start();
    Thread.sleep(1000);
    server.indexPartMoniter.start();
    Thread.sleep(1000);

    Set<String> keys = server.indexfiles.keySet();
    for (String key : keys) {
        for (ConcurrentHashMap<String, HashSet<String>> hashstrs : server.indexfiles.get(key).values()) {
            for (HashSet<String> str : hashstrs.values()) {
                for (String s : str) {
                    System.out.println(s);
                }
            }
        }
    }
}

From source file:TestIndexServer.java

License:Open Source License

public void testTaskQueueManager() throws IOException, InterruptedException {
    status = initializeindexconf();/*from   w  w w  .ja  v a  2  s  .  co  m*/
    fs.delete(new Path(status.gettablelocation()), true);
    fs.delete(new Path(status.gettablelocation()), true);
    FSDataOutputStream fos = fs.create(new Path(status.gettablelocation() + "testfile"));
    fos.write("test".getBytes());
    fos.close();

    IndexServer server = new IndexServer(null);
    server.testmode = true;
    server.metaDataChecker.start();
    Thread.sleep(1000);
    server.dataPartMoniter.start();
    Thread.sleep(1000);
    server.indexPartMoniter.start();
    Thread.sleep(1000);
    server.taskQueueManager.start();
    Thread.sleep(1000);

    assertTrue(server.taskQueue != null && server.taskQueue.size() > 0);
}