Example usage for org.apache.hadoop.fs Path Path

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path Path.

Prototype

public Path(URI aUri) 

Source Link

Document

Construct a path from a URI

Usage

From source file:TestFuseDFS.java

License:Apache License

/**
 * use shell to create a dir and then use filesys to see it exists.
 *//*ww  w . j a  v a  2s  . c  om*/
public void testUtimes() throws IOException, InterruptedException, Exception {
    try {
        // First create a new directory with mkdirs
        Path path = new Path("/utimetest");
        Runtime r = Runtime.getRuntime();
        String cmd = "touch " + mpoint + path.toString();
        Process p = r.exec(cmd);
        assertTrue(p.waitFor() == 0);

        // check it is there
        assertTrue(fileSys.exists(path));

        FileStatus foo = fileSys.getFileStatus(path);
        long oldTime = foo.getModificationTime();
        try {
            Thread.sleep(1000);
        } catch (Exception e) {
        }

        cmd = "touch " + mpoint + path.toString();
        p = r.exec(cmd);
        assertTrue(p.waitFor() == 0);

        try {
            Thread.sleep(1000);
        } catch (Exception e) {
        }
        foo = fileSys.getFileStatus(path);
        long newTime = foo.getModificationTime();

        assertTrue(newTime > oldTime);

    } catch (Exception e) {
        e.printStackTrace();
        throw e;
    } finally {
    }
}

---

From source file:TestFuseDFS.java

License:Apache License

/**
 *
 * Test dfs_read on a file size that will trigger multiple internal reads. 
 * First, just check raw size reading is ok and then check with smaller reads
 * including checking the validity of the data read.
 *
 *//*from w  w  w .j  a  v a 2s  . co  m*/
public void testReads() throws IOException, InterruptedException {
    try {
        // First create a new directory with mkdirs
        Runtime r = Runtime.getRuntime();
        Process p;

        // create the file
        Path myPath = new Path("/test/hello.reads");
        FSDataOutputStream s = fileSys.create(myPath);
        String hello = "hello world!";
        int written = 0;
        int mycount = 0;
        while (written < 1024 * 9) {
            s.writeUTF(hello);
            s.writeInt(mycount++);
            written += hello.length() + 4;
        }
        s.close();

        // check it exists
        assertTrue(fileSys.exists(myPath));
        FileStatus foo = fileSys.getFileStatus(myPath);
        assertTrue(foo.getLen() >= 9 * 1024);

        {
            // cat the file
            DataInputStream is = new DataInputStream(new FileInputStream(mpoint + "/test/hello.reads"));
            byte buf[] = new byte[4096];
            // test reading 0 length
            assertTrue(is.read(buf, 0, 0) == 0);

            // test real reads
            assertTrue(is.read(buf, 0, 1024) == 1024);
            assertTrue(is.read(buf, 0, 4096) == 4096);
            assertTrue(is.read(buf, 0, 4096) == 4096);
            is.close();
        }

        {
            DataInputStream is = new DataInputStream(new FileInputStream(mpoint + "/test/hello.reads"));
            int read = 0;
            int counter = 0;
            try {
                while (true) {
                    String s2 = DataInputStream.readUTF(is);
                    int s3 = is.readInt();
                    assertTrue(s2.equals(hello));
                    assertTrue(s3 == counter++);
                    read += hello.length() + 4;
                }
            } catch (EOFException e) {
                assertTrue(read >= 9 * 1024);
            }
        }

        // check reading an empty file for EOF
        {
            // create the file
            myPath = new Path("/test/hello.reads2");
            s = fileSys.create(myPath);
            s.close();

            FSDataInputStream fs = fileSys.open(myPath);
            assertEquals(-1, fs.read());

            FileInputStream f = new FileInputStream(mpoint + "/test/hello.reads2");
            assertEquals(-1, f.read());
        }

    } catch (Exception e) {
        e.printStackTrace();
    } finally {
    }
}

---

From source file:TestFuseDFS.java

License:Apache License

/**
 * Use filesys to create the hello world! file and then cat it and see its contents are correct.
 *///from   ww  w .jav a 2  s. com
public void testCat() throws IOException, InterruptedException {
    try {
        // First create a new directory with mkdirs
        Runtime r = Runtime.getRuntime();
        Process p = r.exec("rm -rf " + mpoint + "/test/hello");
        assertTrue(p.waitFor() == 0);

        // create the file
        Path myPath = new Path("/test/hello");
        FSDataOutputStream s = fileSys.create(myPath);
        String hello = "hello world!";
        s.writeUTF(hello);
        s.writeInt(1033);
        s.close();

        // check it exists
        assertTrue(fileSys.exists(myPath));

        // cat the file
        DataInputStream is = new DataInputStream(new FileInputStream(mpoint + "/test/hello"));
        String s2 = DataInputStream.readUTF(is);
        int s3 = is.readInt();
        assertTrue(s2.equals(hello));
        assertTrue(s3 == 1033);

    } catch (Exception e) {
        e.printStackTrace();
    } finally {
    }
}

---

From source file:TestFuseDFS.java

License:Apache License

/**
 * Use filesys to create the hello world! file and then cat it and see its contents are correct.
 *///from  w  w  w. j a  v  a  2s  .  c om
public void testAppends() throws IOException, InterruptedException {
    try {
        // First create a new directory with mkdirs
        Runtime r = Runtime.getRuntime();

        {
            FileOutputStream os = new FileOutputStream(mpoint + "/appends");
            String hello = "hello";
            os.write(hello.getBytes());
            os.flush();
            os.close();
        }

        // check it exists
        Path myPath = new Path("/appends");
        assertTrue(fileSys.exists(myPath));

        try {
            Thread.sleep(1000);
        } catch (Exception e) {
        }

        FileStatus foo = fileSys.getFileStatus(myPath);

        File f = new File(mpoint + "/appends");
        assertTrue(f.length() > 0);

        {
            FileOutputStream os = new FileOutputStream(mpoint + "/appends", true);
            String hello = " world!";
            os.write(hello.getBytes());
            os.flush();
            os.close();
        }

        // cat the file
        FileInputStream is = new FileInputStream(mpoint + "/appends");
        byte b[] = new byte[1024];
        int len = is.read(b);
        assertTrue(len > 0);
        String s2 = new String(b, 0, len);
        assertTrue(s2.equals("hello world!"));

    } catch (Exception e) {
        e.printStackTrace();
    } finally {
    }
}

---

From source file:LicenseStatewiseCount.java

public static void main(String[] args) throws Exception {

    JobConf conf = new JobConf(ParkingTicketStatewiseCount.class);
    conf.setJobName("Statecounts");

    conf.setOutputKeyClass(Text.class);
    conf.setOutputValueClass(IntWritable.class);

    conf.setMapperClass(Map.class);
    conf.setReducerClass(Reduce.class);

    conf.setInputFormat(TextInputFormat.class);
    conf.setOutputFormat(TextOutputFormat.class);

    FileInputFormat.setInputPaths(conf, new Path(args[0]));
    FileOutputFormat.setOutputPath(conf, new Path(args[1]));

    JobClient.runJob(conf);/*from  ww  w .  java  2 s.  c om*/

}

---

From source file:WordCountD.java

License:Apache License

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length < 2) {
        System.err.println("Usage: wordcount <in> [<in>...] <out>");
        System.exit(2);/*from w w w . j av  a  2  s.c o m*/
    }
    Job job = Job.getInstance(conf, "word count");
    job.setJarByClass(WordCountC.class);
    job.setMapperClass(TokenizerMapper.class);
    // Disable the combiner
    // job.setCombinerClass(IntSumReducer.class);
    // Setup the Partitioner
    job.setPartitionerClass(Letterpartitioner.class);
    job.setReducerClass(IntSumReducer.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);
    for (int i = 0; i < otherArgs.length - 1; ++i) {
        FileInputFormat.addInputPath(job, new Path(otherArgs[i]));
    }
    FileOutputFormat.setOutputPath(job, new Path(otherArgs[otherArgs.length - 1]));
    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

---

From source file:BuildPersonalizedPageRankRecords.java

License:Apache License

/**
 * Runs this tool./*from  w w w.j  a  v a  2 s. com*/
 */
@SuppressWarnings({ "static-access" })
public int run(String[] args) throws Exception {
    Options options = new Options();

    options.addOption(OptionBuilder.withArgName("path").hasArg().withDescription("input path").create(INPUT));
    options.addOption(OptionBuilder.withArgName("path").hasArg().withDescription("output path").create(OUTPUT));
    options.addOption(
            OptionBuilder.withArgName("num").hasArg().withDescription("number of nodes").create(NUM_NODES));
    //parsing more than 1 integer later;
    options.addOption(
            OptionBuilder.withArgName("src").hasArg().withDescription("source of pagerank").create(SOURCES));

    CommandLine cmdline;
    CommandLineParser parser = new GnuParser();

    try {
        cmdline = parser.parse(options, args);
    } catch (ParseException exp) {
        System.err.println("Error parsing command line: " + exp.getMessage());
        return -1;
    }

    if (!cmdline.hasOption(INPUT) || !cmdline.hasOption(OUTPUT) || !cmdline.hasOption(NUM_NODES)
            || !cmdline.hasOption(SOURCES)) {
        System.out.println("args: " + Arrays.toString(args));
        HelpFormatter formatter = new HelpFormatter();
        formatter.setWidth(120);
        formatter.printHelp(this.getClass().getName(), options);
        ToolRunner.printGenericCommandUsage(System.out);
        return -1;
    }

    String inputPath = cmdline.getOptionValue(INPUT);
    String outputPath = cmdline.getOptionValue(OUTPUT);
    int n = Integer.parseInt(cmdline.getOptionValue(NUM_NODES));
    //Change to array later
    String src = cmdline.getOptionValue(SOURCES);

    LOG.info("Tool name: " + BuildPersonalizedPageRankRecords.class.getSimpleName());
    LOG.info(" - inputDir: " + inputPath);
    LOG.info(" - outputDir: " + outputPath);
    LOG.info(" - numNodes: " + n);

    Configuration conf = getConf();
    conf.setInt(NODE_CNT_FIELD, n);
    //more to be set later;
    conf.set(NODE_SRC, src);
    conf.setInt("mapred.min.split.size", 1024 * 1024 * 1024);

    Job job = Job.getInstance(conf);
    job.setJobName(BuildPersonalizedPageRankRecords.class.getSimpleName() + ":" + inputPath);
    job.setJarByClass(BuildPersonalizedPageRankRecords.class);

    job.setNumReduceTasks(0);

    FileInputFormat.addInputPath(job, new Path(inputPath));
    FileOutputFormat.setOutputPath(job, new Path(outputPath));

    job.setInputFormatClass(TextInputFormat.class);
    job.setOutputFormatClass(SequenceFileOutputFormat.class);

    job.setMapOutputKeyClass(IntWritable.class);
    job.setMapOutputValueClass(PageRankNodeMultiSrc.class);

    job.setOutputKeyClass(IntWritable.class);
    job.setOutputValueClass(PageRankNodeMultiSrc.class);

    job.setMapperClass(MyMapper.class);

    // Delete the output directory if it exists already.
    FileSystem.get(conf).delete(new Path(outputPath), true);

    job.waitForCompletion(true);

    return 0;
}

---

From source file:LookupPostingsCompressed.java

License:Apache License

/**
 * Runs this tool.//  w w  w .jav a  2s  .  c  om
 */
@SuppressWarnings({ "static-access" })
public int run(String[] args) throws Exception {
    Options options = new Options();

    options.addOption(OptionBuilder.withArgName("path").hasArg().withDescription("input path").create(INDEX));
    options.addOption(
            OptionBuilder.withArgName("path").hasArg().withDescription("output path").create(COLLECTION));

    CommandLine cmdline = null;
    CommandLineParser parser = new GnuParser();

    try {
        cmdline = parser.parse(options, args);
    } catch (ParseException exp) {
        System.err.println("Error parsing command line: " + exp.getMessage());
        System.exit(-1);
    }

    if (!cmdline.hasOption(INDEX) || !cmdline.hasOption(COLLECTION)) {
        System.out.println("args: " + Arrays.toString(args));
        HelpFormatter formatter = new HelpFormatter();
        formatter.setWidth(120);
        formatter.printHelp(LookupPostingsCompressed.class.getName(), options);
        ToolRunner.printGenericCommandUsage(System.out);
        System.exit(-1);
    }

    String indexPath = cmdline.getOptionValue(INDEX);
    String collectionPath = cmdline.getOptionValue(COLLECTION);

    if (collectionPath.endsWith(".gz")) {
        System.out.println("gzipped collection is not seekable: use compressed version!");
        System.exit(-1);
    }

    Configuration config = new Configuration();
    FileSystem fs = FileSystem.get(config);
    MapFile.Reader reader = new MapFile.Reader(new Path(indexPath + "/part-r-00000"), config);

    FSDataInputStream collection = fs.open(new Path(collectionPath));
    BufferedReader d = new BufferedReader(new InputStreamReader(collection));

    Text key = new Text();
    PairOfWritables<VIntWritable, BytesWritable> value = new PairOfWritables<VIntWritable, BytesWritable>();

    System.out.println("Looking up postings for the term \"starcross'd\"");
    key.set("starcross'd");

    reader.get(key, value);

    BytesWritable postings = value.getRightElement();
    ByteArrayInputStream buffer = new ByteArrayInputStream(postings.copyBytes());
    DataInputStream in = new DataInputStream(buffer);
    int OFFSET = 0;
    int count;
    while (in.available() != 0) {
        OFFSET = OFFSET + WritableUtils.readVInt(in);
        count = WritableUtils.readVInt(in);
        System.out.print("(" + OFFSET + ", " + count + ")");
        collection.seek(OFFSET);
        System.out.println(d.readLine());
    }

    OFFSET = 0;
    key.set("gold");
    reader.get(key, value);
    postings = value.getRightElement();
    buffer = new ByteArrayInputStream(postings.copyBytes());
    in = new DataInputStream(buffer);
    System.out.println("Complete postings list for 'gold': (" + value.getLeftElement() + ", [");
    while (in.available() != 0) {
        OFFSET = OFFSET + WritableUtils.readVInt(in);
        count = WritableUtils.readVInt(in);
        System.out.print("(" + OFFSET + ", " + count + ")");
        //collection.seek(OFFSET);
        //System.out.println(d.readLine());
        System.out.print(", ");
    }
    System.out.print("])\n");

    Int2IntFrequencyDistribution goldHist = new Int2IntFrequencyDistributionEntry();
    buffer.reset();

    OFFSET = 0;
    while (in.available() != 0) {
        OFFSET = OFFSET + WritableUtils.readVInt(in);
        count = WritableUtils.readVInt(in);
        goldHist.increment(count);
    }

    System.out.println("histogram of tf values for gold");
    for (PairOfInts pair : goldHist) {
        System.out.println(pair.getLeftElement() + "\t" + pair.getRightElement());
    }

    buffer.close();
    //Silver

    key.set("silver");
    reader.get(key, value);
    postings = value.getRightElement();
    buffer = new ByteArrayInputStream(postings.copyBytes());
    in = new DataInputStream(buffer);
    System.out.println("Complete postings list for 'silver': (" + value.getLeftElement() + ", [");
    while (in.available() != 0) {
        OFFSET = OFFSET + WritableUtils.readVInt(in);
        count = WritableUtils.readVInt(in);
        System.out.print("(" + OFFSET + ", " + count + ")");
        //collection.seek(OFFSET);
        //System.out.println(d.readLine());
        System.out.print(", ");
    }
    System.out.print("])\n");

    Int2IntFrequencyDistribution silverHist = new Int2IntFrequencyDistributionEntry();
    buffer.reset();

    OFFSET = 0;
    while (in.available() != 0) {
        OFFSET = OFFSET + WritableUtils.readVInt(in);
        count = WritableUtils.readVInt(in);
        silverHist.increment(count);
    }

    System.out.println("histogram of tf values for silver");
    for (PairOfInts pair : goldHist) {
        System.out.println(pair.getLeftElement() + "\t" + pair.getRightElement());
    }

    buffer.close();

    key.set("bronze");
    Writable w = reader.get(key, value);

    if (w == null) {
        System.out.println("the term bronze does not appear in the collection");
    }

    collection.close();
    reader.close();

    return 0;
}

---

From source file:FormatStoragePerformanceTest.java

License:Open Source License

static void doInitFile(int count, boolean var) {
    try {// ww  w . j  ava2  s  . c  om
        String textFile = "MR_input_text/testPerformanceReadText";
        if (var) {
            textFile += "_var";
        }
        Path path = new Path(textFile);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataOutputStream out = fs.create(path);

        OutputStream stream = new BufferedOutputStream(out);
        BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(stream));

        String value = null;
        if (var) {
            value = "111,22222,33333333,444444444444,5555555.5555,6666666666.666666,hello konten\n";
        } else {
            value = "111,22222,33333333,444444444444,5555555.5555,6666666666.666666\n";
        }

        long begin = System.currentTimeMillis();

        for (int i = 0; i < count; i++) {
            writer.write(value);

            if (i % 10000000 == 0) {
                String string = "write " + i + " record, delay: "
                        + ((System.currentTimeMillis() - begin) / 1000) + " s \n";
                output.write(string.getBytes());
            }
        }
        writer.close();

        long end = System.currentTimeMillis();

        String string = "write " + count + " record over, delay: " + ((end - begin) / 1000) + " s \n";
        output.write(string.getBytes());
    } catch (Exception e) {
        e.printStackTrace();
        System.out.println(e.getMessage());
    }
}

---

From source file:FormatStoragePerformanceTest.java

License:Open Source License

static void doTextReadRand(int count) {
    try {//from w w w . ja va 2s.  c o m
        String textFile = "MR_input_text/testPerformanceReadText";
        Path path = new Path(textFile);
        FileSystem fs = FileSystem.get(new Configuration());
        FSDataInputStream in = fs.open(path);

        InputStream stream = new BufferedInputStream(in);
        BufferedReader reader = new BufferedReader(new InputStreamReader(stream));

        long begin = System.currentTimeMillis();
        count = 1 * 1000;
        for (int i = 0; i < count; i++) {
            int line = (int) (Math.random() * count);
            for (int j = 0; j < line; j++) {
                String value = reader.readLine();
                value = null;
            }
        }
        reader.close();

        long end = System.currentTimeMillis();
        String string = "text read seq, count:" + count + ", delay:" + (long) ((end - begin) / 1000) + " s";
        output.write(string.getBytes());
        System.out.println(string);

    } catch (Exception e) {
        e.printStackTrace();
        System.out.println(e.getMessage());
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• »