Example usage for org.apache.hadoop.fs Path Path

List of usage examples for org.apache.hadoop.fs Path Path

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. Path
  6. Path

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path Path.

Prototype

public Path(URI aUri)

Source Link

Document

Construct a path from a URI

Usage

From source file:GapDeduceRunner.java

License:Apache License

public static void main(String[] args) throws IOException {
    JobConf conf = new JobConf(GapDeduceRunner.class);
    conf.setJobName("gapdeduce");

    conf.setMapOutputKeyClass(Text.class);
    conf.setMapOutputValueClass(Text.class);

    conf.setOutputKeyClass(Text.class);
    conf.setOutputValueClass(Text.class);

    conf.setMapperClass(Gapper.class);
    conf.setReducerClass(Deducer.class);

    // KeyValueTextInputFormat treats each line as an input record, 
    // and splits the line by the tab character to separate it into key and value 
    conf.setInputFormat(KeyValueTextInputFormat.class);
    conf.setOutputFormat(TextOutputFormat.class);

    FileInputFormat.setInputPaths(conf, new Path(args[0]));
    FileOutputFormat.setOutputPath(conf, new Path(args[1]));

    JobClient.runJob(conf);/*from w w w.j  a v a 2s.c  om*/
}

From source file:Hw2Part1.java

License:Apache License

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length < 2) {
        System.err.println("Usage: <input file> <output directory>");
        System.exit(2);/*from   w w w  .  j a  va  2  s . c o  m*/
    }

    //    FileSystem hdfs = FileSystem.get(conf);
    String target = "hdfs://localhost:9000/";
    FileSystem fs = FileSystem.get(URI.create(target), conf);//is diffrent
    Path outputpath = new Path(otherArgs[otherArgs.length - 1]);
    if (fs.exists(outputpath)) {
        fs.delete(outputpath, true);
    }

    Job job = Job.getInstance(conf, "Hw2Part1");

    job.setJarByClass(Hw2Part1.class);

    job.setMapperClass(TokenizerMapper.class);
    job.setCombinerClass(IntSumCombiner.class);
    job.setReducerClass(IntSumReducer.class);

    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(InfoWritable.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(InfoWritable.class);

    // add the input paths as given by command line
    for (int i = 0; i < otherArgs.length - 1; ++i) {
        FileInputFormat.addInputPath(job, new Path(otherArgs[i]));
    }

    // add the output path as given by the command line
    FileOutputFormat.setOutputPath(job, new Path(otherArgs[otherArgs.length - 1]));

    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

From source file:Tmptest.java

License:Open Source License

public static void main(String[] args) throws IOException {
    Configuration conf = new Configuration();
    Path path = new Path("/se/tmp/628892613/part-00000");

    String filename = path.toString();
    IFormatDataFile ifd = new IFormatDataFile(conf);
    ifd.open(filename);/* ww w .  j av  a  2s . co m*/

    ISegmentIndex segmentIndex = ifd.segIndex();

    for (String str : ifd.fileInfo().head().getUdi().infos().values()) {
        System.out.println(str);
    }
    System.out.println(segmentIndex.getSegnum());
    IRecord record = new IRecord();
    ifd.next(record);
    record.show();
    ifd.next().show();
    ifd.next().show();
    ifd.close();

}

From source file:TestFuseDFS.java

License:Apache License

static public void startStuff() {
    try {//from   w w  w .ja v  a 2s  . c o m
        Configuration conf = new Configuration();
        conf.setBoolean("dfs.permissions", false);
        cluster = new MiniDFSCluster(conf, 1, true, null);
        fileSys = (DistributedFileSystem) cluster.getFileSystem();
        assertTrue(fileSys.getFileStatus(new Path("/")).isDir());
        mount(mpoint, fileSys.getUri());
    } catch (Exception e) {
        e.printStackTrace();
    }
}

From source file:TestFuseDFS.java

License:Apache License

/**
 * use shell to create a dir and then use filesys to see it exists.
 *//*from   ww  w.j  a v  a 2s. c o m*/
public void testMkdir() throws IOException, InterruptedException, Exception {
    try {
        // First create a new directory with mkdirs
        Path path = new Path("/foo");
        Runtime r = Runtime.getRuntime();
        String cmd = "mkdir -p " + mpoint + path.toString();
        Process p = r.exec(cmd);
        assertTrue(p.waitFor() == 0);

        // check it is there
        assertTrue(fileSys.getFileStatus(path).isDir());

        // check again through the shell
        String lsCmd = "ls " + mpoint + path.toString();
        p = r.exec(lsCmd);
        assertTrue(p.waitFor() == 0);
    } catch (Exception e) {
        e.printStackTrace();
        throw e;
    }
}

From source file:TestFuseDFS.java

License:Apache License

/**
 * use shell to create a dir and then use filesys to see it exists.
 *//*  w  w w  .jav a2 s.  c  o m*/
public void testWrites() throws IOException, InterruptedException {
    try {

        // write a hello file
        File file = new File(mpoint, "hello.txt");
        FileOutputStream f = new FileOutputStream(file);
        String s = "hello ";
        f.write(s.getBytes());
        s = "world";
        f.write(s.getBytes());
        f.flush();
        f.close();

        try {
            Thread.sleep(1000);
        } catch (Exception e) {
        }

        // check the file exists.
        Path myPath = new Path("/hello.txt");
        assertTrue(fileSys.exists(myPath));

        // check the data is ok
        FileInputStream fi = new FileInputStream(new File(mpoint, "hello.txt"));
        byte b[] = new byte[12];
        int length = fi.read(b, 0, 12);
        assertTrue(length > 0);
        String s2 = new String(b, 0, length);
        assertEquals("hello world", s2);
    } catch (Exception e) {
        e.printStackTrace();
    } finally {
    }
}

From source file:TestFuseDFS.java

License:Apache License

/**
 * Remove a dir using the shell and use filesys to see it no longer exists.
 *//*  w ww .ja va2 s.  c om*/
public void testRmdir() throws IOException, InterruptedException {
    try {
        // First create a new directory with mkdirs

        Runtime r = Runtime.getRuntime();
        Process p = r.exec("mkdir -p " + mpoint + "/test/rmdir");
        assertTrue(p.waitFor() == 0);

        Path myPath = new Path("/test/rmdir");
        assertTrue(fileSys.exists(myPath));

        // remove it
        p = r.exec("rmdir " + mpoint + "/test/rmdir");
        assertTrue(p.waitFor() == 0);

        // check it is not there
        assertFalse(fileSys.exists(myPath));

        Path trashPath = new Path("/user/root/.Trash/Current/test/rmdir");
        assertTrue(fileSys.exists(trashPath));

        // make it again to test trashing same thing twice
        p = r.exec("mkdir -p " + mpoint + "/test/rmdir");
        assertTrue(p.waitFor() == 0);

        assertTrue(fileSys.exists(myPath));

        // remove it
        p = r.exec("rmdir " + mpoint + "/test/rmdir");
        assertTrue(p.waitFor() == 0);

        // check it is not there
        assertFalse(fileSys.exists(myPath));

        trashPath = new Path("/user/root/.Trash/Current/test/rmdir.1");
        assertTrue(fileSys.exists(trashPath));

    } catch (Exception e) {
        e.printStackTrace();
    }
}

From source file:TestFuseDFS.java

License:Apache License

/**
 * use shell to create a dir and then use filesys to see it exists.
 *//*from w  w  w  .  ja v  a  2  s .  c  om*/
public void testDF() throws IOException, InterruptedException, Exception {
    try {
        // First create a new directory with mkdirs
        Path path = new Path("/foo");
        Runtime r = Runtime.getRuntime();
        String cmd = "mkdir -p " + mpoint + path.toString();
        Process p = r.exec(cmd);
        assertTrue(p.waitFor() == 0);
        File f = new File(mpoint + "/foo");

        DistributedFileSystem.DiskStatus d = fileSys.getDiskStatus();

        long fileUsedBlocks = (f.getTotalSpace() - f.getFreeSpace()) / (64 * 1024 * 1024);
        long dfsUsedBlocks = (long) Math.ceil((double) d.getDfsUsed() / (64 * 1024 * 1024));

        assertTrue(fileUsedBlocks == dfsUsedBlocks);
        assertTrue(d.getCapacity() == f.getTotalSpace());

    } catch (Exception e) {
        e.printStackTrace();
        throw e;
    }
}

From source file:TestFuseDFS.java

License:Apache License

/**
 * use shell to create a dir and then use filesys to see it exists.
 *//* ww w . j ava 2s. c o  m*/
public void testChown() throws IOException, InterruptedException, Exception {
    try {
        // First create a new directory with mkdirs
        Path path = new Path("/foo");
        Runtime r = Runtime.getRuntime();
        String cmd = "mkdir -p " + mpoint + path.toString();
        Process p = r.exec(cmd);
        assertTrue(p.waitFor() == 0);

        // check it is there
        assertTrue(fileSys.getFileStatus(path).isDir());

        FileStatus foo = fileSys.getFileStatus(path);
        System.err.println("DEBUG:owner=" + foo.getOwner());

        cmd = "chown nobody " + mpoint + path.toString();
        p = r.exec(cmd);
        assertTrue(p.waitFor() == 0);

        //      cmd = "chgrp nobody " + mpoint + path.toString();
        //      p = r.exec(cmd);
        //      assertTrue(p.waitFor() == 0);

        foo = fileSys.getFileStatus(path);

        System.err.println("DEBUG:owner=" + foo.getOwner());

        assertTrue(foo.getOwner().equals("nobody"));
        assertTrue(foo.getGroup().equals("nobody"));

    } catch (Exception e) {
        e.printStackTrace();
        throw e;
    }
}

From source file:TestFuseDFS.java

License:Apache License

/**
 * use shell to create a dir and then use filesys to see it exists.
 *///  w  ww. j av  a 2s . com
public void testChmod() throws IOException, InterruptedException, Exception {
    try {
        // First create a new directory with mkdirs
        Path path = new Path("/foo");
        Runtime r = Runtime.getRuntime();
        String cmd = "mkdir -p " + mpoint + path.toString();
        Process p = r.exec(cmd);
        assertTrue(p.waitFor() == 0);

        // check it is there
        assertTrue(fileSys.getFileStatus(path).isDir());

        cmd = "chmod 777 " + mpoint + path.toString();
        p = r.exec(cmd);
        assertTrue(p.waitFor() == 0);

        FileStatus foo = fileSys.getFileStatus(path);
        FsPermission perm = foo.getPermission();
        assertTrue(perm.toShort() == 0777);

    } catch (Exception e) {
        e.printStackTrace();
        throw e;
    }
}