Example usage for org.apache.hadoop.fs Path Path

List of usage examples for org.apache.hadoop.fs Path Path

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. Path
  6. Path

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path Path.

Prototype

public Path(URI aUri)

Source Link

Document

Construct a path from a URI

Usage

From source file:alluxio.client.hadoop.FileSystemRenameIntegrationTest.java

License:Apache License

@Test
public void errorRenameTest1() throws Exception {
    // Rename /dirA to /dirA/dirB should fail
    Path dirA = new Path("/dirA");
    Path finalDst = new Path("/dirA/dirB");

    sTFS.mkdirs(dirA);/* www.j  a v  a  2 s  .  co  m*/

    Assert.assertFalse(sTFS.rename(dirA, finalDst));

    Assert.assertFalse(sTFS.exists(finalDst));
    Assert.assertTrue(sTFS.exists(dirA));
    Assert.assertFalse(sUfs.isDirectory(PathUtils.concatPath(sUfsRoot, "dirA", "dirB")));
    Assert.assertFalse(sUfs.isDirectory(PathUtils.concatPath(sUfsRoot, "dirB")));

    cleanup(sTFS);

    Assert.assertFalse(sTFS.exists(dirA));
    Assert.assertFalse(sUfs.isDirectory(PathUtils.concatPath(sUfsRoot, "dirB")));
}

From source file:alluxio.client.hadoop.FileSystemRenameIntegrationTest.java

License:Apache License

@Test
public void errorRenameTest2() throws Exception {
    // Rename /fileA to /fileB should fail if /fileB exists
    Path fileA = new Path("/fileA");
    Path fileB = new Path("/fileB");

    create(sTFS, fileA);/*from w ww.j  av  a2  s. c om*/
    create(sTFS, fileB);

    Assert.assertFalse(sTFS.rename(fileA, fileB));

    Assert.assertTrue(sTFS.exists(fileA));
    Assert.assertTrue(sTFS.exists(fileB));
    Assert.assertTrue(sUfs.isFile(PathUtils.concatPath(sUfsRoot, "fileA")));
    Assert.assertTrue(sUfs.isFile(PathUtils.concatPath(sUfsRoot, "fileB")));

    cleanup(sTFS);

    Assert.assertFalse(sTFS.exists(fileA));
    Assert.assertFalse(sTFS.exists(fileB));
    Assert.assertFalse(sUfs.isFile(PathUtils.concatPath(sUfsRoot, "fileA")));
    Assert.assertFalse(sUfs.isFile(PathUtils.concatPath(sUfsRoot, "fileB")));
}

From source file:alluxio.client.hadoop.FileSystemRenameIntegrationTest.java

License:Apache License

@Test
public void errorRenameTest3() throws Exception {
    // Rename /fileA to /dirA/fileA should fail if /dirA/fileA exists
    Path fileA = new Path("/fileA");
    Path dirA = new Path("/dirA");
    Path finalDst = new Path("/dirA/fileA");

    create(sTFS, fileA);//from w w  w .  j  av  a2  s  . c o  m
    create(sTFS, finalDst);

    Assert.assertFalse(sTFS.rename(fileA, dirA));

    Assert.assertTrue(sTFS.exists(fileA));
    Assert.assertTrue(sTFS.exists(dirA));
    Assert.assertTrue(sTFS.exists(finalDst));
    Assert.assertTrue(sUfs.isFile(PathUtils.concatPath(sUfsRoot, "fileA")));
    Assert.assertTrue(sUfs.isDirectory(PathUtils.concatPath(sUfsRoot, "dirA")));
    Assert.assertTrue(sUfs.isFile(PathUtils.concatPath(sUfsRoot, "dirA", "fileA")));

    cleanup(sTFS);

    Assert.assertFalse(sTFS.exists(fileA));
    Assert.assertFalse(sTFS.exists(dirA));
    Assert.assertFalse(sTFS.exists(finalDst));
    Assert.assertFalse(sUfs.isFile(PathUtils.concatPath(sUfsRoot, "fileA")));
    Assert.assertFalse(sUfs.isDirectory(PathUtils.concatPath(sUfsRoot, "dirA")));
    Assert.assertFalse(sUfs.isFile(PathUtils.concatPath(sUfsRoot, "dirA", "fileA")));
}

From source file:alluxio.client.hadoop.FileSystemRenameIntegrationTest.java

License:Apache License

@Test
public void errorRenameTest4() throws Exception {
    // Rename /fileA to an nonexistent path should fail
    Path fileA = new Path("/fileA");
    Path nonexistentPath = new Path("/doesNotExist/fileA");

    create(sTFS, fileA);//ww  w .  j  a va 2  s  . co m

    Assert.assertFalse(sTFS.rename(fileA, nonexistentPath));

    Assert.assertTrue(sTFS.exists(fileA));
    Assert.assertTrue(sUfs.isFile(PathUtils.concatPath(sUfsRoot, "fileA")));

    cleanup(sTFS);

    Assert.assertFalse(sTFS.exists(fileA));
    Assert.assertFalse(sUfs.isFile(PathUtils.concatPath(sUfsRoot, "fileA")));
}

From source file:alluxio.client.hadoop.FileSystemStatisticsTest.java

License:Apache License

/**
 * Test the number of bytes read./* w  w  w . j a va 2 s.c o  m*/
 */
@Test
public void bytesReadStatistics() throws Exception {
    long originStat = sStatistics.getBytesRead();
    InputStream is = sTFS.open(new Path("/testFile-read"));
    while (is.read() != -1) {
    }
    is.close();
    Assert.assertEquals(originStat + FILE_LEN, sStatistics.getBytesRead());
}

From source file:alluxio.client.hadoop.FileSystemStatisticsTest.java

License:Apache License

/**
 * Test the number of bytes written./*from ww  w.  j  a va  2  s  . c  o  m*/
 */
@Test
public void bytesWrittenStatistics() throws Exception {
    long originStat = sStatistics.getBytesWritten();
    OutputStream os = sTFS.create(new Path("/testFile-write"));
    for (int i = 0; i < FILE_LEN; i++) {
        os.write(1);
    }
    os.close();
    Assert.assertEquals(originStat + FILE_LEN, sStatistics.getBytesWritten());
}

From source file:alluxio.client.hadoop.FileSystemStatisticsTest.java

License:Apache License

/**
 * Test the number of read/write operations.
 *///from   w  ww.  j  av  a  2s .c om
@Test
public void readWriteOperationsStatistics() throws Exception {
    int exceptedReadOps = sStatistics.getReadOps();
    int exceptedWriteOps = sStatistics.getWriteOps();

    // Call all the overridden methods and check the statistics.
    sTFS.create(new Path("/testFile-create")).close();
    exceptedWriteOps++;
    Assert.assertEquals(exceptedReadOps, sStatistics.getReadOps());
    Assert.assertEquals(exceptedWriteOps, sStatistics.getWriteOps());

    sTFS.delete(new Path("/testFile-create"), true);
    exceptedWriteOps++;
    Assert.assertEquals(exceptedReadOps, sStatistics.getReadOps());
    Assert.assertEquals(exceptedWriteOps, sStatistics.getWriteOps());

    // Due to Hadoop 1 support we stick with the deprecated version. If we drop support for it
    // FileSystem.getDefaultBlockSize(new Path("/testFile-create")) will be the new one.
    sTFS.getDefaultBlockSize();
    Assert.assertEquals(exceptedReadOps, sStatistics.getReadOps());
    Assert.assertEquals(exceptedWriteOps, sStatistics.getWriteOps());

    // Due to Hadoop 1 support we stick with the deprecated version. If we drop support for it
    // FileSystem.geDefaultReplication(new Path("/testFile-create")) will be the new one.
    sTFS.getDefaultReplication();
    Assert.assertEquals(exceptedReadOps, sStatistics.getReadOps());
    Assert.assertEquals(exceptedWriteOps, sStatistics.getWriteOps());

    FileStatus fStatus = sTFS.getFileStatus(new Path("/testFile-read"));
    exceptedReadOps++;
    Assert.assertEquals(exceptedReadOps, sStatistics.getReadOps());
    Assert.assertEquals(exceptedWriteOps, sStatistics.getWriteOps());

    sTFS.getFileBlockLocations(fStatus, 0, FILE_LEN);
    exceptedReadOps++;
    Assert.assertEquals(exceptedReadOps, sStatistics.getReadOps());
    Assert.assertEquals(exceptedWriteOps, sStatistics.getWriteOps());

    sTFS.getUri();
    Assert.assertEquals(exceptedReadOps, sStatistics.getReadOps());
    Assert.assertEquals(exceptedWriteOps, sStatistics.getWriteOps());

    sTFS.getWorkingDirectory();
    Assert.assertEquals(exceptedReadOps, sStatistics.getReadOps());
    Assert.assertEquals(exceptedWriteOps, sStatistics.getWriteOps());

    sTFS.listStatus(new Path("/"));
    exceptedReadOps++;
    Assert.assertEquals(exceptedReadOps, sStatistics.getReadOps());
    Assert.assertEquals(exceptedWriteOps, sStatistics.getWriteOps());

    sTFS.mkdirs(new Path("/testDir"));
    exceptedWriteOps++;
    Assert.assertEquals(exceptedReadOps, sStatistics.getReadOps());
    Assert.assertEquals(exceptedWriteOps, sStatistics.getWriteOps());

    sTFS.open(new Path("/testFile-read")).close();
    exceptedReadOps++;
    Assert.assertEquals(exceptedReadOps, sStatistics.getReadOps());
    Assert.assertEquals(exceptedWriteOps, sStatistics.getWriteOps());

    sTFS.rename(new Path("/testDir"), new Path("/testDir-rename"));
    exceptedWriteOps++;
    Assert.assertEquals(exceptedReadOps, sStatistics.getReadOps());
    Assert.assertEquals(exceptedWriteOps, sStatistics.getWriteOps());

    sTFS.setWorkingDirectory(new Path("/testDir-rename"));
    Assert.assertEquals(exceptedReadOps, sStatistics.getReadOps());
    Assert.assertEquals(exceptedWriteOps, sStatistics.getWriteOps());
}

From source file:alluxio.client.hadoop.FileSystemUriIntegrationTest.java

License:Apache License

/**
 * Tests connections to Alluxio cluster using URIs with connect details in authorities.
 *
 * @param authority the authority to test
 *//*  w ww.j  a  va  2s  .  c o  m*/
private void testConnection(String authority) throws Exception {
    Configuration conf = new Configuration();
    conf.set("fs.alluxio.impl", FileSystem.class.getName());
    URI uri = URI.create("alluxio://" + authority + "/tmp/path.txt");
    org.apache.hadoop.fs.FileSystem fs = org.apache.hadoop.fs.FileSystem.get(uri, conf);

    mCluster.waitForAllNodesRegistered(WAIT_TIMEOUT_MS);

    Path file = new Path("/testFile");
    FsPermission permission = FsPermission.createImmutable((short) 0666);
    FSDataOutputStream o = fs.create(file, permission, false /* ignored */, 10 /* ignored */,
            (short) 1 /* ignored */, 512 /* ignored */, null /* ignored */);
    o.writeBytes("Test Bytes");
    o.close();
    // with mark of delete-on-exit, the close method will try to delete it
    fs.deleteOnExit(file);
    fs.close();
    mCluster.notifySuccess();
}

From source file:alluxio.examples.keyvalue.hadoop.CloneStoreMapReduce.java

License:Apache License

/**
 * @param args two parameters, the first is the input key-value store path, the second is the
 *    output key-value store path//from ww  w.j  a  v a  2s . c o  m
 * @throws Exception if any exception happens
 */
public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();

    // NOTE(binfan): we are using the deprecated constructor of Job instance to compile with
    // hadoop-1.0. If this is not a concern, a better way is
    //     Job job = Job.getInstance(conf);
    Job job = new Job(conf);

    job.setJobName("CloneStoreMapReduce");
    job.setJarByClass(CloneStoreMapReduce.class);

    job.setOutputKeyClass(BytesWritable.class);
    job.setOutputValueClass(BytesWritable.class);

    job.setMapperClass(CloneStoreMapper.class);
    job.setReducerClass(CloneStoreReducer.class);

    job.setInputFormatClass(KeyValueInputFormat.class);
    job.setOutputFormatClass(KeyValueOutputFormat.class);

    FileInputFormat.setInputPaths(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));

    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

From source file:alluxio.hadoop.AbstractFileSystem.java

License:Apache License

/**
 * {@inheritDoc}/*from   w  ww . jav a  2s.c  o m*/
 *
 * If the file does not exist in Alluxio, query it from HDFS.
 */
@Override
public FileStatus getFileStatus(Path path) throws IOException {
    LOG.info("getFileStatus({})", path);

    if (mStatistics != null) {
        mStatistics.incrementReadOps(1);
    }
    AlluxioURI uri = new AlluxioURI(HadoopUtils.getPathWithoutScheme(path));
    URIStatus fileStatus;
    try {
        fileStatus = sFileSystem.getStatus(uri);
    } catch (FileDoesNotExistException e) {
        throw new FileNotFoundException(e.getMessage());
    } catch (AlluxioException e) {
        throw new IOException(e);
    }

    return new FileStatus(fileStatus.getLength(), fileStatus.isFolder(), BLOCK_REPLICATION_CONSTANT,
            fileStatus.getBlockSizeBytes(), fileStatus.getCreationTimeMs(), fileStatus.getCreationTimeMs(),
            new FsPermission((short) fileStatus.getMode()), fileStatus.getOwner(), fileStatus.getGroup(),
            new Path(mAlluxioHeader + uri));
}