Example usage for org.apache.hadoop.fs Path makeQualified

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path makeQualified.

Prototype

@Deprecated
public Path makeQualified(FileSystem fs) 

Source Link

Document

Returns a qualified path object for the FileSystem 's working directory.

Usage

From source file:org.apache.hama.bsp.BSPPeerImpl.java

License:Apache License

@SuppressWarnings("unchecked")
public final void initializeIO() throws Exception {

    if (conf.get(Constants.JOB_INPUT_DIR) != null) {
        initInput();/*from w  w w.ja  va2 s.  c om*/
    }

    String outdir = null;
    if (conf.get(Constants.JOB_OUTPUT_DIR) != null) {
        Path outputDir = new Path(conf.get(Constants.JOB_OUTPUT_DIR, "tmp-" + System.currentTimeMillis()),
                Task.getOutputName(partition));
        outdir = outputDir.makeQualified(fs).toString();
    }
    outWriter = bspJob.getOutputFormat().getRecordWriter(fs, bspJob, outdir);
    final RecordWriter<K2, V2> finalOut = outWriter;

    collector = new OutputCollector<K2, V2>() {
        @Override
        public void collect(K2 key, V2 value) throws IOException {
            finalOut.write(key, value);
        }
    };

    /* Move Files to HDFS */
    try {
        DistributedCacheUtil.moveLocalFiles(this.conf);
    } catch (Exception e) {
        LOG.error(e);
    }

    /* Add additional jars to Classpath */
    // LOG.info("conf.get(tmpjars): " + this.conf.get("tmpjars"));
    URL[] libjars = DistributedCacheUtil.addJarsToJobClasspath(this.conf);

    // ATTENTION bspJob.getConf() != this.conf
    if (libjars != null)
        bspJob.conf.setClassLoader(new URLClassLoader(libjars, bspJob.conf.getClassLoader()));
}

---

From source file:org.apache.hama.pipes.util.DistributedCacheUtil.java

License:Apache License

/**
 * Add the Files to HDFS//from ww w.  j  av a  2 s  . c  om
 * 
 * @param conf The job's configuration
 * @param files Paths that should be transfered to HDFS
 */
public static String addFilesToHDFS(Configuration conf, String files) {
    if (files == null)
        return null;
    String[] fileArr = files.split(",");
    String[] finalArr = new String[fileArr.length];

    for (int i = 0; i < fileArr.length; i++) {
        String tmp = fileArr[i];
        String finalPath;

        URI pathURI;
        try {
            pathURI = new URI(tmp);
        } catch (URISyntaxException e) {
            throw new IllegalArgumentException(e);
        }

        try {
            LocalFileSystem local = LocalFileSystem.getLocal(conf);
            Path pathSrc = new Path(pathURI);
            // LOG.info("pathSrc: " + pathSrc);

            if (local.exists(pathSrc)) {
                FileSystem hdfs = FileSystem.get(conf);
                Path pathDst = new Path(hdfs.getWorkingDirectory() + "/temp", pathSrc.getName());

                // LOG.info("WorkingDirectory: " + hdfs.getWorkingDirectory());
                LOG.debug("copyToHDFSFile: " + pathDst);
                hdfs.copyFromLocalFile(pathSrc, pathDst);
                hdfs.deleteOnExit(pathDst);

                finalPath = pathDst.makeQualified(hdfs).toString();
                finalArr[i] = finalPath;
            }
        } catch (IOException e) {
            LOG.error(e);
        }

    }
    return StringUtils.arrayToString(finalArr);
}

---

From source file:org.apache.hama.util.GenericOptionsParser.java

License:Apache License

/**
 * takes input as a comma separated list of files and verifies if they exist.
 * It defaults for file:/// if the files specified do not have a scheme. it
 * returns the paths uri converted defaulting to file:///. So an input of
 * /home/user/file1,/home/user/file2 would return
 * file:///home/user/file1,file:///home/user/file2
 * //from w  w  w.  ja  v a  2s  .c  o m
 * @param files
 * @return
 */
private String validateFiles(String files, HamaConfiguration conf) throws IOException {
    if (files == null)
        return null;
    String[] fileArr = files.split(",");
    String[] finalArr = new String[fileArr.length];
    for (int i = 0; i < fileArr.length; i++) {
        String tmp = fileArr[i];
        String finalPath;
        URI pathURI;
        try {
            pathURI = new URI(tmp);
        } catch (URISyntaxException e) {
            throw new IllegalArgumentException(e);
        }
        Path path = new Path(pathURI);
        FileSystem localFs = FileSystem.getLocal(conf);
        if (pathURI.getScheme() == null) {
            // default to the local file system
            // check if the file exists or not first
            if (!localFs.exists(path)) {
                throw new FileNotFoundException("File " + tmp + " does not exist.");
            }
            finalPath = path.makeQualified(localFs).toString();
        } else {
            // check if the file exists in this file system
            // we need to recreate this filesystem object to copy
            // these files to the file system jobtracker is running
            // on.
            FileSystem fs = path.getFileSystem(conf);
            if (!fs.exists(path)) {
                throw new FileNotFoundException("File " + tmp + " does not exist.");
            }
            finalPath = path.makeQualified(fs).toString();
        }
        finalArr[i] = finalPath;
    }
    return StringUtils.arrayToString(finalArr);
}

---

From source file:org.apache.hcatalog.security.TestHdfsAuthorizationProvider.java

License:Apache License

@Test
public void testAlterTableRelocate() throws Exception {
    exec("CREATE TABLE foo1 (foo INT) STORED AS RCFILE");
    Path tablePath = new Path(whPath, new Random().nextInt() + "/mytable");
    exec("ALTER TABLE foo1 SET LOCATION '%s'", tablePath.makeQualified(whFs));

    tablePath = new Path(whPath, new Random().nextInt() + "/mytable2");
    exec("CREATE EXTERNAL TABLE foo3 (foo INT) STORED AS RCFILE LOCATION '%s'", tablePath.makeQualified(whFs));
    tablePath = new Path(whPath, new Random().nextInt() + "/mytable2");
    exec("ALTER TABLE foo3 SET LOCATION '%s'", tablePath.makeQualified(whFs));
}

---

From source file:org.apache.hcatalog.security.TestHdfsAuthorizationProvider.java

License:Apache License

@Test
public void testAlterTableRelocateFail() throws Exception {
    exec("CREATE TABLE foo1 (foo INT) STORED AS RCFILE");
    Path tablePath = new Path(whPath, new Random().nextInt() + "/mytable");
    whFs.mkdirs(tablePath, perm500); //revoke write
    execFail("ALTER TABLE foo1 SET LOCATION '%s'", tablePath.makeQualified(whFs));

    //dont have access to new table loc
    tablePath = new Path(whPath, new Random().nextInt() + "/mytable2");
    exec("CREATE EXTERNAL TABLE foo3 (foo INT) STORED AS RCFILE LOCATION '%s'", tablePath.makeQualified(whFs));
    tablePath = new Path(whPath, new Random().nextInt() + "/mytable2");
    whFs.mkdirs(tablePath, perm500); //revoke write
    execFail("ALTER TABLE foo3 SET LOCATION '%s'", tablePath.makeQualified(whFs));

    //have access to new table loc, but not old table loc
    tablePath = new Path(whPath, new Random().nextInt() + "/mytable3");
    exec("CREATE EXTERNAL TABLE foo4 (foo INT) STORED AS RCFILE LOCATION '%s'", tablePath.makeQualified(whFs));
    whFs.mkdirs(tablePath, perm500); //revoke write
    tablePath = new Path(whPath, new Random().nextInt() + "/mytable3");
    execFail("ALTER TABLE foo4 SET LOCATION '%s'", tablePath.makeQualified(whFs));
}

---

From source file:org.apache.hcatalog.security.TestHdfsAuthorizationProvider.java

License:Apache License

@Test
public void testAlterTablePartRelocate() throws Exception {
    exec("CREATE TABLE foo1 (foo INT) PARTITIONED BY (b STRING) STORED AS RCFILE");
    exec("ALTER TABLE foo1 ADD PARTITION (b='2010-10-16')");
    Path partPath = new Path(whPath, new Random().nextInt() + "/mypart");
    exec("ALTER TABLE foo1 PARTITION (b='2010-10-16') SET LOCATION '%s'", partPath.makeQualified(whFs));
}

---

From source file:org.apache.hcatalog.security.TestHdfsAuthorizationProvider.java

License:Apache License

@Test
public void testAlterTablePartRelocateFail() throws Exception {
    exec("CREATE TABLE foo1 (foo INT) PARTITIONED BY (b STRING) STORED AS RCFILE");

    Path oldLoc = new Path(whPath, new Random().nextInt() + "/mypart");
    Path newLoc = new Path(whPath, new Random().nextInt() + "/mypart2");

    exec("ALTER TABLE foo1 ADD PARTITION (b='2010-10-16') LOCATION '%s'", oldLoc);
    whFs.mkdirs(oldLoc, perm500);//from w  w w  .j a  v a 2s  .co  m
    execFail("ALTER TABLE foo1 PARTITION (b='2010-10-16') SET LOCATION '%s'", newLoc.makeQualified(whFs));
    whFs.mkdirs(oldLoc, perm700);
    whFs.mkdirs(newLoc, perm500);
    execFail("ALTER TABLE foo1 PARTITION (b='2010-10-16') SET LOCATION '%s'", newLoc.makeQualified(whFs));
}

---

From source file:org.apache.hive.common.util.MockFileSystem.java

License:Apache License

private List<LocatedFileStatus> listLocatedFileStatuses(Path path) throws IOException {
    statistics.incrementReadOps(1);/*from  ww w . jav  a 2s  .  co m*/
    checkAccess();
    path = path.makeQualified(this);
    List<LocatedFileStatus> result = new ArrayList<>();
    String pathname = path.toString();
    String pathnameAsDir = pathname + "/";
    Set<String> dirs = new TreeSet<String>();
    MockFile file = findFile(path);
    if (file != null) {
        result.add(createLocatedStatus(file));
        return result;
    }
    findMatchingLocatedFiles(files, pathnameAsDir, dirs, result);
    findMatchingLocatedFiles(globalFiles, pathnameAsDir, dirs, result);
    // for each directory add it once
    for (String dir : dirs) {
        result.add(createLocatedDirectory(new MockPath(this, pathnameAsDir + dir)));
    }
    return result;
}

---

From source file:org.apache.hive.common.util.MockFileSystem.java

License:Apache License

@Override
public FileStatus[] listStatus(Path path) throws IOException {
    statistics.incrementReadOps(1);//from w  w w  .  java2s.c  om
    checkAccess();
    path = path.makeQualified(this);
    List<FileStatus> result = new ArrayList<FileStatus>();
    String pathname = path.toString();
    String pathnameAsDir = pathname + "/";
    Set<String> dirs = new TreeSet<String>();
    MockFile file = findFile(path);
    if (file != null) {
        return new FileStatus[] { createStatus(file) };
    }
    findMatchingFiles(files, pathnameAsDir, dirs, result);
    findMatchingFiles(globalFiles, pathnameAsDir, dirs, result);
    // for each directory add it once
    for (String dir : dirs) {
        result.add(createDirectory(new MockPath(this, pathnameAsDir + dir)));
    }
    return result.toArray(new FileStatus[result.size()]);
}

---

From source file:org.apache.hive.common.util.MockFileSystem.java

License:Apache License

@Override
public FileStatus getFileStatus(Path path) throws IOException {
    statistics.incrementReadOps(1);// w  ww.j  a v a  2  s. c om
    checkAccess();
    path = path.makeQualified(this);
    String pathnameAsDir = path.toString() + "/";
    MockFile file = findFile(path);
    if (file != null)
        return createStatus(file);
    for (MockFile dir : files) {
        if (dir.path.toString().startsWith(pathnameAsDir)) {
            return createDirectory(path);
        }
    }
    for (MockFile dir : globalFiles) {
        if (dir.path.toString().startsWith(pathnameAsDir)) {
            return createDirectory(path);
        }
    }
    throw new FileNotFoundException("File " + path + " does not exist");
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• »