Example usage for org.apache.hadoop.fs Path toUri

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path toUri.

Prototype

public URI toUri() 

Source Link

Document

Convert this Path to a URI.

Usage

From source file:com.cloudera.hoop.client.fs.TestHoopFileSystem.java

License:Open Source License

private void testRename() throws Exception {
    FileSystem fs = FileSystem.get(getHadoopConf());
    Path path = new Path(getHadoopTestDir(), "foo");
    fs.mkdirs(path);/*from www .j  av a2 s. co m*/
    fs.close();
    Configuration conf = new Configuration();
    conf.set("fs.http.impl", HoopFileSystem.class.getName());
    fs = FileSystem.get(getJettyURL().toURI(), conf);
    Path oldPath = new Path(path.toUri().getPath());
    Path newPath = new Path(path.getParent(), "bar");
    fs.rename(oldPath, newPath);
    fs.close();
    fs = FileSystem.get(getHadoopConf());
    Assert.assertFalse(fs.exists(oldPath));
    Assert.assertTrue(fs.exists(newPath));
    fs.close();
}

---

From source file:com.cloudera.hoop.client.fs.TestHoopFileSystem.java

License:Open Source License

private void testDelete() throws Exception {
    Path foo = new Path(getHadoopTestDir(), "foo");
    Path bar = new Path(getHadoopTestDir(), "bar");
    Path foe = new Path(getHadoopTestDir(), "foe");
    FileSystem fs = FileSystem.get(getHadoopConf());
    fs.mkdirs(foo);//w  ww . j av  a  2  s .  c o m
    fs.mkdirs(new Path(bar, "a"));
    fs.mkdirs(foe);

    Configuration conf = new Configuration();
    conf.set("fs.http.impl", HoopFileSystem.class.getName());
    FileSystem hoopFs = FileSystem.get(getJettyURL().toURI(), conf);
    Assert.assertTrue(hoopFs.delete(new Path(foo.toUri().getPath()), false));
    Assert.assertFalse(fs.exists(foo));
    try {
        hoopFs.delete(new Path(bar.toUri().getPath()), false);
        Assert.fail();
    } catch (IOException ex) {
    } catch (Exception ex) {
        Assert.fail();
    }
    Assert.assertTrue(fs.exists(bar));
    Assert.assertTrue(hoopFs.delete(new Path(bar.toUri().getPath()), true));
    Assert.assertFalse(fs.exists(bar));

    Assert.assertTrue(fs.exists(foe));
    Assert.assertTrue(hoopFs.delete(foe));
    Assert.assertFalse(fs.exists(foe));

    hoopFs.close();
    fs.close();
}

---

From source file:com.cloudera.hoop.client.fs.TestHoopFileSystem.java

License:Open Source License

private void testListStatus() throws Exception {
    FileSystem fs = FileSystem.get(getHadoopConf());
    Path path = new Path(getHadoopTestDir(), "foo.txt");
    OutputStream os = fs.create(path);
    os.write(1);/*from  ww w. ja  v  a  2  s.  c om*/
    os.close();
    FileStatus status1 = fs.getFileStatus(path);
    fs.close();

    Configuration conf = new Configuration();
    conf.set("fs.http.impl", HoopFileSystem.class.getName());
    fs = FileSystem.get(getJettyURL().toURI(), conf);
    FileStatus status2 = fs.getFileStatus(new Path(path.toUri().getPath()));
    fs.close();

    Assert.assertEquals(status2.getPermission(), status1.getPermission());
    Assert.assertEquals(status2.getPath().toUri().getPath(), status1.getPath().toUri().getPath());
    Assert.assertEquals(status2.getReplication(), status1.getReplication());
    Assert.assertEquals(status2.getBlockSize(), status1.getBlockSize());
    Assert.assertEquals(status2.getAccessTime(), status1.getAccessTime());
    Assert.assertEquals(status2.getModificationTime(), status1.getModificationTime());
    Assert.assertEquals(status2.getOwner(), status1.getOwner());
    Assert.assertEquals(status2.getGroup(), status1.getGroup());
    Assert.assertEquals(status2.getLen(), status1.getLen());

    FileStatus[] stati = fs.listStatus(path.getParent());
    Assert.assertEquals(stati.length, 1);
    Assert.assertEquals(stati[0].getPath().getName(), path.getName());
}

---

From source file:com.cloudera.hoop.client.fs.TestHoopFileSystem.java

License:Open Source License

private void testWorkingdirectory() throws Exception {
    FileSystem fs = FileSystem.get(getHadoopConf());
    Path workingDir = fs.getWorkingDirectory();
    fs.close();//from ww  w. ja va  2  s  . c  o m

    Configuration conf = new Configuration();
    conf.set("fs.http.impl", HoopFileSystem.class.getName());
    fs = FileSystem.get(getJettyURL().toURI(), conf);
    Path hoopWorkingDir = fs.getWorkingDirectory();
    fs.close();
    Assert.assertEquals(hoopWorkingDir.toUri().getPath(), workingDir.toUri().getPath());

    conf = new Configuration();
    conf.set("fs.http.impl", HoopFileSystem.class.getName());
    fs = FileSystem.get(getJettyURL().toURI(), conf);
    fs.setWorkingDirectory(new Path("/tmp"));
    workingDir = fs.getWorkingDirectory();
    fs.close();
    Assert.assertEquals(workingDir.toUri().getPath(), new Path("/tmp").toUri().getPath());
}

---

From source file:com.cloudera.hoop.fs.FSUtils.java

License:Open Source License

/**
 * Replaces the <code>SCHEME://HOST:PORT</code> of a Hadoop
 * <code>Path</code> with the specified base URL.
 *
 * @param path Hadoop path to replace the
 * <code>SCHEME://HOST:PORT</code>.
 * @param hoopBaseUrl base URL to replace it with.
 * @return the path using the given base URL.
 *///from  w w  w  . j  a v  a 2s .c  o  m
public static Path convertPathToHoop(Path path, String hoopBaseUrl) {
    URI uri = path.toUri();
    String filePath = uri.getRawPath();
    String hoopPath = hoopBaseUrl + filePath;
    return new Path(hoopPath);
}

---

From source file:com.cloudera.impala.catalog.TestLoadHdfsMetadataPerf.java

License:Apache License

/**
 * List file status by calling abstractFileSystem.listStatusIterator.
 */// w  w w.  ja va 2  s  .c o m
private static void listStatusIterator(String dirPath) {
    Path path = new Path(dirPath);
    boolean exceptionThrown = false;
    try {
        AbstractFileSystem fs = AbstractFileSystem.createFileSystem(path.toUri(), LoadMetadataUtil.getConf());
        RemoteIterator<FileStatus> iter = fs.listStatusIterator(path);
        while (iter.hasNext()) {
            FileStatus fileStatus = iter.next();
            BlockLocation[] locations = fs.getFileBlockLocations(fileStatus.getPath(), 0, fileStatus.getLen());
            for (BlockLocation loc : locations) {
                loc.getNames();
                loc.getHosts();
            }
        }
    } catch (IOException e) {
        exceptionThrown = true;
        LOG.error("Failed to list Status Iterator", e);
    }
    assertFalse(exceptionThrown);
}

---

From source file:com.cloudera.impala.planner.S3PlannerTest.java

License:Apache License

/**
 * Remove any non-constant components of the given file path.  For S3, the
 * actual bucket name, which will be unique to the tester's setup, needs to
 * be replaced with a fixed bucket name.
 *//*from w  ww  .  ja  v a 2  s  .c o m*/
@Override
protected Path cleanseFilePath(Path path) {
    path = super.cleanseFilePath(path);
    URI fsURI = fsName.toUri();
    URI pathURI = path.toUri();
    Assert.assertTrue("error: " + path + " is not on filesystem " + fsName,
            fsURI.getScheme().equals(pathURI.getScheme())
                    && fsURI.getAuthority().equals(pathURI.getAuthority()));
    return Path.mergePaths(S3A_CANONICAL_BUCKET, path);
}

---

From source file:com.cloudera.impala.util.LoadMetadataUtil.java

License:Apache License

/**
 * Identical to loadFileDescriptors, except using the ListStatusIterator HDFS API to
 * load file status.//ww  w  .j  a  v  a  2 s .  co m
 */
public static List<FileDescriptor> loadViaListStatusIterator(FileSystem fs, Path partDirPath,
        Map<String, List<FileDescriptor>> oldFileDescMap, HdfsFileFormat fileFormat,
        Map<FsKey, FileBlocksInfo> perFsFileBlocks, boolean isMarkedCached, String tblName,
        ListMap<TNetworkAddress> hostIndex, Map<String, List<FileDescriptor>> fileDescMap)
        throws FileNotFoundException, IOException {
    List<FileDescriptor> fileDescriptors = Lists.newArrayList();

    AbstractFileSystem abstractFs = AbstractFileSystem.createFileSystem(partDirPath.toUri(), CONF);
    RemoteIterator<FileStatus> fileStatusItor = abstractFs.listStatusIterator(partDirPath);

    while (fileStatusItor.hasNext()) {
        FileStatus fileStatus = fileStatusItor.next();
        FileDescriptor fd = getFileDescriptor(fs, fileStatus, fileFormat, oldFileDescMap, isMarkedCached,
                perFsFileBlocks, tblName, hostIndex);

        if (fd == null)
            continue;

        // Add partition dir to fileDescMap if it does not exist.
        String partitionDir = fileStatus.getPath().getParent().toString();
        if (!fileDescMap.containsKey(partitionDir)) {
            fileDescMap.put(partitionDir, new ArrayList<FileDescriptor>());
        }
        fileDescMap.get(partitionDir).add(fd);

        // Add to the list of FileDescriptors for this partition.
        fileDescriptors.add(fd);
    }

    return fileDescriptors;
}

---

From source file:com.cloudera.kitten.util.LocalDataHelper.java

License:Open Source License

private void copyToHdfs(String key, String localDataName) throws IOException {
    if (!localToHdfs.containsKey(localDataName)) {
        FileSystem fs = FileSystem.get(conf);
        Path src = new Path(localDataName);
        Path dst = getPath(fs, src.getName());
        InputStream data = getFileOrResource(localDataName);
        FSDataOutputStream os = fs.create(dst, true);
        ByteStreams.copy(data, os);//w w  w .j a va2  s  .  co m
        os.close();
        URI uri = dst.toUri();
        localToHdfs.put(key, uri);
    }
}

---

From source file:com.cloudera.livy.test.jobs.SQLGetTweets.java

License:Apache License

@Override
public List<String> call(JobContext jc) throws Exception {
    InputStream source = getClass().getResourceAsStream("/testweet.json");

    // Save the resource as a file in HDFS (or the local tmp dir when using a local filesystem).
    URI input;/*from w ww. ja  v  a  2 s.  co m*/
    File local = File.createTempFile("tweets", ".json", jc.getLocalTmpDir());
    Files.copy(source, local.toPath(), StandardCopyOption.REPLACE_EXISTING);
    FileSystem fs = FileSystem.get(jc.sc().sc().hadoopConfiguration());
    if ("file".equals(fs.getUri().getScheme())) {
        input = local.toURI();
    } else {
        String uuid = UUID.randomUUID().toString();
        Path target = new Path("/tmp/" + uuid + "-tweets.json");
        fs.copyFromLocalFile(new Path(local.toURI()), target);
        input = target.toUri();
    }

    SQLContext sqlctx = useHiveContext ? jc.hivectx() : jc.sqlctx();
    DataFrame df = sqlctx.jsonFile(input.toString());
    df.registerTempTable("tweets");

    DataFrame topTweets = sqlctx.sql("SELECT text, retweetCount FROM tweets ORDER BY retweetCount LIMIT 10");
    List<String> tweetList = new ArrayList<>();
    for (Row r : topTweets.collect()) {
        tweetList.add(r.toString());
    }
    return tweetList;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »