Example usage for org.apache.hadoop.fs Path toString

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path toString.

Prototype

@Override
    public String toString() 

Source Link

Usage

From source file:WikipediaForwardIndexBuilder.java

License:Apache License

@SuppressWarnings("static-access")
@Override/*from w  w  w .j  a v  a  2 s.  c  o  m*/
public int run(String[] args) throws Exception {
    Options options = new Options();
    options.addOption(OptionBuilder.withArgName("path").hasArg().withDescription("input").create(INPUT_OPTION));
    options.addOption(
            OptionBuilder.withArgName("path").hasArg().withDescription("index file").create(INDEX_FILE_OPTION));
    options.addOption(OptionBuilder.withArgName("en|sv|de|cs|es|zh|ar|tr").hasArg()
            .withDescription("two-letter language code").create(LANGUAGE_OPTION));

    CommandLine cmdline;
    CommandLineParser parser = new GnuParser();
    try {
        cmdline = parser.parse(options, args);
    } catch (ParseException exp) {
        System.err.println("Error parsing command line: " + exp.getMessage());
        return -1;
    }

    if (!cmdline.hasOption(INPUT_OPTION) || !cmdline.hasOption(INDEX_FILE_OPTION)) {
        HelpFormatter formatter = new HelpFormatter();
        formatter.printHelp(this.getClass().getName(), options);
        ToolRunner.printGenericCommandUsage(System.out);
        return -1;
    }

    Path inputPath = new Path(cmdline.getOptionValue(INPUT_OPTION));
    String indexFile = cmdline.getOptionValue(INDEX_FILE_OPTION);

    String tmpPath = "tmp-" + WikipediaForwardIndexBuilder.class.getSimpleName() + "-" + RANDOM.nextInt(10000);

    if (!inputPath.isAbsolute()) {
        System.err.println("Error: " + INPUT_OPTION + " must be an absolute path!");
        return -1;
    }

    String language = null;
    if (cmdline.hasOption(LANGUAGE_OPTION)) {
        language = cmdline.getOptionValue(LANGUAGE_OPTION);
        if (language.length() != 2) {
            System.err.println("Error: \"" + language + "\" unknown language!");
            return -1;
        }
    }

    JobConf conf = new JobConf(getConf(), WikipediaForwardIndexBuilder.class);
    FileSystem fs = FileSystem.get(conf);

    LOG.info("Tool name: " + this.getClass().getName());
    LOG.info(" - input path: " + inputPath);
    LOG.info(" - index file: " + indexFile);
    LOG.info(" - language: " + language);
    LOG.info("Note: This tool only works on block-compressed SequenceFiles!");

    conf.setJobName(String.format("BuildWikipediaForwardIndex[%s: %s, %s: %s, %s: %s]", INPUT_OPTION, inputPath,
            INDEX_FILE_OPTION, indexFile, LANGUAGE_OPTION, language));

    conf.setNumReduceTasks(1);

    FileInputFormat.setInputPaths(conf, inputPath);
    FileOutputFormat.setOutputPath(conf, new Path(tmpPath));
    FileOutputFormat.setCompressOutput(conf, false);

    if (language != null) {
        conf.set("wiki.language", language);
    }

    conf.setInputFormat(NoSplitSequenceFileInputFormat.class);
    conf.setOutputKeyClass(IntWritable.class);
    conf.setOutputValueClass(Text.class);

    conf.setMapRunnerClass(MyMapRunner.class);
    conf.setReducerClass(IdentityReducer.class);

    // Delete the output directory if it exists already.
    fs.delete(new Path(tmpPath), true);

    RunningJob job = JobClient.runJob(conf);

    Counters counters = job.getCounters();
    int blocks = (int) counters.getCounter(Blocks.Total);

    LOG.info("number of blocks: " + blocks);

    LOG.info("Writing index file...");
    LineReader reader = new LineReader(fs.open(new Path(tmpPath + "/part-00000")));
    FSDataOutputStream out = fs.create(new Path(indexFile), true);

    out.writeUTF(edu.umd.cloud9.collection.wikipedia.WikipediaForwardIndex.class.getCanonicalName());
    out.writeUTF(inputPath.toString());
    out.writeInt(blocks);

    int cnt = 0;
    Text line = new Text();
    while (reader.readLine(line) > 0) {
        String[] arr = line.toString().split("\\s+");

        int docno = Integer.parseInt(arr[0]);
        int offset = Integer.parseInt(arr[1]);
        short fileno = Short.parseShort(arr[2]);

        out.writeInt(docno);
        out.writeInt(offset);
        out.writeShort(fileno);

        cnt++;

        if (cnt % 100000 == 0) {
            LOG.info(cnt + " blocks written");
        }
    }

    reader.close();
    out.close();

    if (cnt != blocks) {
        throw new RuntimeException("Error: mismatch in block count!");
    }

    // Clean up.
    fs.delete(new Path(tmpPath), true);

    return 0;
}

---

From source file:Script.java

License:Open Source License

/** Deserialize the Javascript function from the distributed cache.
 *  @param conf    The Hadoop configuration
 *  @return        The Javascript function
 *///w w  w. ja  va 2s  .co  m
public Object deserialize(Configuration conf, String file) throws IOException {
    Path[] cacheFiles = DistributedCache.getLocalCacheFiles(conf);
    if (null != cacheFiles && cacheFiles.length > 0) {
        for (Path path : cacheFiles) { // loop through cache files
            if (path.getName().equals(file)) { // find this file
                return evalCache(conf, path.toString(), file);
            }
        }
    }
    return null;
}

---

From source file:ClustersFilter.java

License:Apache License

public boolean accept(Path path) {
    String pathString = path.toString();
    return pathString.contains("/clusters-");
}

---

From source file:Tmptest.java

License:Open Source License

public static void main(String[] args) throws IOException {
    Configuration conf = new Configuration();
    Path path = new Path("/se/tmp/628892613/part-00000");

    String filename = path.toString();
    IFormatDataFile ifd = new IFormatDataFile(conf);
    ifd.open(filename);/*from   w ww .j  ava  2s.c  om*/

    ISegmentIndex segmentIndex = ifd.segIndex();

    for (String str : ifd.fileInfo().head().getUdi().infos().values()) {
        System.out.println(str);
    }
    System.out.println(segmentIndex.getSegnum());
    IRecord record = new IRecord();
    ifd.next(record);
    record.show();
    ifd.next().show();
    ifd.next().show();
    ifd.close();

}

---

From source file:TestFuseDFS.java

License:Apache License

/**
 * use shell to create a dir and then use filesys to see it exists.
 *//*from   ww w .  j a  v  a  2  s.c  o m*/
public void testMkdir() throws IOException, InterruptedException, Exception {
    try {
        // First create a new directory with mkdirs
        Path path = new Path("/foo");
        Runtime r = Runtime.getRuntime();
        String cmd = "mkdir -p " + mpoint + path.toString();
        Process p = r.exec(cmd);
        assertTrue(p.waitFor() == 0);

        // check it is there
        assertTrue(fileSys.getFileStatus(path).isDir());

        // check again through the shell
        String lsCmd = "ls " + mpoint + path.toString();
        p = r.exec(lsCmd);
        assertTrue(p.waitFor() == 0);
    } catch (Exception e) {
        e.printStackTrace();
        throw e;
    }
}

---

From source file:TestFuseDFS.java

License:Apache License

/**
 * use shell to create a dir and then use filesys to see it exists.
 *//*  w  w  w  .j a  v  a  2 s.  c o m*/
public void testDF() throws IOException, InterruptedException, Exception {
    try {
        // First create a new directory with mkdirs
        Path path = new Path("/foo");
        Runtime r = Runtime.getRuntime();
        String cmd = "mkdir -p " + mpoint + path.toString();
        Process p = r.exec(cmd);
        assertTrue(p.waitFor() == 0);
        File f = new File(mpoint + "/foo");

        DistributedFileSystem.DiskStatus d = fileSys.getDiskStatus();

        long fileUsedBlocks = (f.getTotalSpace() - f.getFreeSpace()) / (64 * 1024 * 1024);
        long dfsUsedBlocks = (long) Math.ceil((double) d.getDfsUsed() / (64 * 1024 * 1024));

        assertTrue(fileUsedBlocks == dfsUsedBlocks);
        assertTrue(d.getCapacity() == f.getTotalSpace());

    } catch (Exception e) {
        e.printStackTrace();
        throw e;
    }
}

---

From source file:TestFuseDFS.java

License:Apache License

/**
 * use shell to create a dir and then use filesys to see it exists.
 *///from   w  w w  . jav  a  2  s. c om
public void testChown() throws IOException, InterruptedException, Exception {
    try {
        // First create a new directory with mkdirs
        Path path = new Path("/foo");
        Runtime r = Runtime.getRuntime();
        String cmd = "mkdir -p " + mpoint + path.toString();
        Process p = r.exec(cmd);
        assertTrue(p.waitFor() == 0);

        // check it is there
        assertTrue(fileSys.getFileStatus(path).isDir());

        FileStatus foo = fileSys.getFileStatus(path);
        System.err.println("DEBUG:owner=" + foo.getOwner());

        cmd = "chown nobody " + mpoint + path.toString();
        p = r.exec(cmd);
        assertTrue(p.waitFor() == 0);

        //      cmd = "chgrp nobody " + mpoint + path.toString();
        //      p = r.exec(cmd);
        //      assertTrue(p.waitFor() == 0);

        foo = fileSys.getFileStatus(path);

        System.err.println("DEBUG:owner=" + foo.getOwner());

        assertTrue(foo.getOwner().equals("nobody"));
        assertTrue(foo.getGroup().equals("nobody"));

    } catch (Exception e) {
        e.printStackTrace();
        throw e;
    }
}

---

From source file:TestFuseDFS.java

License:Apache License

/**
 * use shell to create a dir and then use filesys to see it exists.
 *//*from w w  w .  j a va  2  s .c  om*/
public void testChmod() throws IOException, InterruptedException, Exception {
    try {
        // First create a new directory with mkdirs
        Path path = new Path("/foo");
        Runtime r = Runtime.getRuntime();
        String cmd = "mkdir -p " + mpoint + path.toString();
        Process p = r.exec(cmd);
        assertTrue(p.waitFor() == 0);

        // check it is there
        assertTrue(fileSys.getFileStatus(path).isDir());

        cmd = "chmod 777 " + mpoint + path.toString();
        p = r.exec(cmd);
        assertTrue(p.waitFor() == 0);

        FileStatus foo = fileSys.getFileStatus(path);
        FsPermission perm = foo.getPermission();
        assertTrue(perm.toShort() == 0777);

    } catch (Exception e) {
        e.printStackTrace();
        throw e;
    }
}

---

From source file:TestFuseDFS.java

License:Apache License

/**
 * use shell to create a dir and then use filesys to see it exists.
 *///from   w  w w.j  a  v a2  s  . c o m
public void testUtimes() throws IOException, InterruptedException, Exception {
    try {
        // First create a new directory with mkdirs
        Path path = new Path("/utimetest");
        Runtime r = Runtime.getRuntime();
        String cmd = "touch " + mpoint + path.toString();
        Process p = r.exec(cmd);
        assertTrue(p.waitFor() == 0);

        // check it is there
        assertTrue(fileSys.exists(path));

        FileStatus foo = fileSys.getFileStatus(path);
        long oldTime = foo.getModificationTime();
        try {
            Thread.sleep(1000);
        } catch (Exception e) {
        }

        cmd = "touch " + mpoint + path.toString();
        p = r.exec(cmd);
        assertTrue(p.waitFor() == 0);

        try {
            Thread.sleep(1000);
        } catch (Exception e) {
        }
        foo = fileSys.getFileStatus(path);
        long newTime = foo.getModificationTime();

        assertTrue(newTime > oldTime);

    } catch (Exception e) {
        e.printStackTrace();
        throw e;
    } finally {
    }
}

---

From source file:TestRawParascaleFileSystemBase.java

License:Apache License

public void testCompleteLocalOutput() throws IOException {
    final Path localTmpPath = new Path("/tmp/test_" + System.currentTimeMillis());
    final Path psdfsPath = new Path("test");
    final Path startLocalOutput = fs.startLocalOutput(new Path("input/test"), localTmpPath);
    assertEquals(localTmpPath, startLocalOutput);
    final File tmpFile = new File(startLocalOutput.toString());
    assertTrue(tmpFile.createNewFile());
    final String testTxt = "a Test Text";
    writeToFile(testTxt, tmpFile);//www. ja v a  2s . c  o  m

    fs.completeLocalOutput(psdfsPath, startLocalOutput);
    final File targetFile = fs.pathToFile(psdfsPath);

    assertFalse(tmpFile.exists());
    assertEquals(testTxt, readFile(targetFile));

}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »