Example usage for org.apache.hadoop.fs FileSystem getUri

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem getUri.

Prototype

public abstract URI getUri();

Source Link

Document

Returns a URI which identifies this FileSystem.

Usage

From source file:gobblin.data.management.copy.replication.ConfigBasedMultiDatasets.java

License:Apache License

public ConfigBasedMultiDatasets(Config c, Properties props) {
    this.props = props;

    try {//  w w  w. j  a va  2 s.co m
        FileSystem executionCluster = FileSystem.get(new Configuration());
        URI executionClusterURI = executionCluster.getUri();

        ReplicationConfiguration rc = ReplicationConfiguration.buildFromConfig(c);

        // push mode
        if (this.props.containsKey(REPLICATION_PUSH_MODE)
                && Boolean.parseBoolean(this.props.getProperty(REPLICATION_PUSH_MODE))) {
            generateDatasetInPushMode(rc, executionClusterURI);
        }
        // default pull mode
        else {
            generateDatasetInPullMode(rc, executionClusterURI);
        }
    } catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) {
        log.error("Can not create Replication Configuration from raw config "
                + c.root().render(ConfigRenderOptions.defaults().setComments(false).setOriginComments(false)),
                e);
    } catch (IOException ioe) {
        log.error("Can not decide current execution cluster ", ioe);

    }
}

---

From source file:gobblin.filesystem.MetricsFileSystemInstrumentation.java

License:Apache License

public MetricsFileSystemInstrumentation(FileSystem underlying) {
    super(underlying);
    this.closer = Closer.create();
    this.metricContext = new MetricContext.Builder(underlying.getUri() + "_metrics").build();
    this.metricContext = this.closer.register(metricContext);

    this.listStatusTimer = this.metricContext.timer("listStatus");
    this.listFilesTimer = this.metricContext.timer("listFiles");
    this.globStatusTimer = this.metricContext.timer("globStatus");
    this.mkdirTimer = this.metricContext.timer("mkdirs");
    this.renameTimer = this.metricContext.timer("rename");
    this.deleteTimer = this.metricContext.timer("delete");
    this.createTimer = this.metricContext.timer("create");
    this.openTimer = this.metricContext.timer("open");
    this.setOwnerTimer = this.metricContext.timer("setOwner");
    this.getFileStatusTimer = this.metricContext.timer("getFileStatus");
    this.setPermissionTimer = this.metricContext.timer("setPermission");
    this.setTimesTimer = this.metricContext.timer("setTimes");
    this.appendTimer = this.metricContext.timer("append");
    this.concatTimer = this.metricContext.timer("concat");
}

---

From source file:gobblin.hadoop.token.TokenUtils.java

License:Open Source License

private static void getHdfsToken(Configuration conf, Credentials cred) throws IOException {
    FileSystem fs = FileSystem.get(conf);
    LOG.info("Getting DFS token from " + fs.getUri());
    Token<?> fsToken = fs.getDelegationToken(getMRTokenRenewerInternal(new JobConf()).toString());
    if (fsToken == null) {
        LOG.error("Failed to fetch DFS token for ");
        throw new IOException("Failed to fetch DFS token.");
    }/*from  w  w  w  .j a  v a2 s .c  o m*/
    LOG.info("Created DFS token: " + fsToken.toString());
    LOG.info("Token kind: " + fsToken.getKind());
    LOG.info("Token id: " + fsToken.getIdentifier());
    LOG.info("Token service: " + fsToken.getService());

    cred.addToken(fsToken.getService(), fsToken);
}

---

From source file:gobblin.publisher.BaseDataPublisher.java

License:Apache License

private ParallelRunner getParallelRunner(FileSystem fs) {
    String uri = fs.getUri().toString();
    if (!this.parallelRunners.containsKey(uri)) {
        this.parallelRunners.put(uri,
                this.parallelRunnerCloser.register(new ParallelRunner(this.parallelRunnerThreads, fs)));
    }//from   w w  w  .ja  v  a  2s.  c  om
    return this.parallelRunners.get(uri);
}

---

From source file:gobblin.util.commit.DeleteFileCommitStep.java

License:Apache License

/**
 * @param fs {@link FileSystem} where files need to be deleted.
 * @param paths Collection of {@link FileStatus}es to deleted.
 * @param properties {@link Properties} object including {@link Trash} configuration.
 * @param parentDeletionLimit if present, will delete empty parent directories up to but not including this path. If
 *                            absent, will not delete empty parent directories.
 * @throws IOException/*w w  w . java  2  s.c  o  m*/
 */
public DeleteFileCommitStep(FileSystem fs, Collection<FileStatus> paths, Properties properties,
        Optional<Path> parentDeletionLimit) throws IOException {
    this.fsUri = fs.getUri();
    this.pathsToDelete = paths;
    this.properties = properties;
    this.parentDeletionLimit = parentDeletionLimit;
}

---

From source file:gobblin.util.filesystem.InstrumentedLocalFileSystemTest.java

License:Apache License

@Test
public void testFromInstrumentedScheme() throws Exception {

    File tmpDir = Files.createTempDir();
    tmpDir.deleteOnExit();/*from  ww w. ja v a2 s.  co  m*/

    FileSystem fs = FileSystem.get(new URI(InstrumentedLocalFileSystem.SCHEME + ":///"), new Configuration());

    Assert.assertTrue(fs instanceof InstrumentedLocalFileSystem);
    Assert.assertTrue(DecoratorUtils.resolveUnderlyingObject(fs) instanceof LocalFileSystem);
    Assert.assertEquals(fs.getFileStatus(new Path("/tmp")).getPath(), new Path("instrumented-file:///tmp"));
    Assert.assertEquals(fs.getUri().getScheme(), "instrumented-file");

    Path basePath = new Path(tmpDir.getAbsolutePath());

    Assert.assertTrue(fs.exists(basePath));

    Path file = new Path(basePath, "file");

    Assert.assertFalse(fs.exists(file));
    fs.create(new Path(basePath, "file"));
    Assert.assertTrue(fs.exists(file));

    Assert.assertEquals(fs.getFileStatus(file).getLen(), 0);
    Assert.assertEquals(fs.listStatus(basePath).length, 1);

    fs.delete(file, false);
    Assert.assertFalse(fs.exists(file));
}

---

From source file:gobblin.util.filesystem.InstrumentedLocalFileSystemTest.java

License:Apache License

@Test
public void testFromConfigurationOverride() throws Exception {
    Configuration configuration = new Configuration();
    configuration.set("fs.file.impl", InstrumentedLocalFileSystem.class.getName());
    FileSystem fs = FileSystem.newInstance(new URI("file:///"), configuration);
    Assert.assertTrue(fs instanceof InstrumentedLocalFileSystem);
    Assert.assertTrue(DecoratorUtils.resolveUnderlyingObject(fs) instanceof LocalFileSystem);
    Assert.assertEquals(fs.getFileStatus(new Path("/tmp")).getPath(), new Path("file:///tmp"));
    Assert.assertEquals(fs.getUri().getScheme(), "file");
}

---

From source file:gobblin.util.hadoop.TokenUtils.java

License:Apache License

private static void getHdfsToken(Configuration conf, Credentials cred) throws IOException {
    FileSystem fs = FileSystem.get(conf);
    LOG.info("Getting DFS token from " + fs.getUri());
    Token<?> fsToken = fs.getDelegationToken(getMRTokenRenewerInternal(new JobConf()).toString());
    if (fsToken == null) {
        LOG.error("Failed to fetch DFS token for ");
        throw new IOException("Failed to fetch DFS token.");
    }/* www . j a va2  s  .com*/
    LOG.info("Created DFS token: " + fsToken.toString());
    LOG.info("Token kind: " + fsToken.getKind());
    LOG.info("Token id: " + Arrays.toString(fsToken.getIdentifier()));
    LOG.info("Token service: " + fsToken.getService());

    cred.addToken(fsToken.getService(), fsToken);
}

---

From source file:gobblin.util.HadoopUtils.java

License:Apache License

/**
 * Moves a src {@link Path} from a srcFs {@link FileSystem} to a dst {@link Path} on a dstFs {@link FileSystem}. If
 * the srcFs and the dstFs have the same scheme, and neither of them or S3 schemes, then the {@link Path} is simply
 * renamed. Otherwise, the data is from the src {@link Path} to the dst {@link Path}. So this method can handle copying
 * data between different {@link FileSystem} implementations.
 *
 * @param srcFs the source {@link FileSystem} where the src {@link Path} exists
 * @param src the source {@link Path} which will me moved
 * @param dstFs the destination {@link FileSystem} where the dst {@link Path} should be created
 * @param dst the {@link Path} to move data to
 * @param overwrite true if the destination should be overwritten; otherwise, false
 *///from  w w w  .j  a v  a  2  s.com
public static void movePath(FileSystem srcFs, Path src, FileSystem dstFs, Path dst, boolean overwrite,
        Configuration conf) throws IOException {

    if (srcFs.getUri().getScheme().equals(dstFs.getUri().getScheme())
            && !FS_SCHEMES_NON_ATOMIC.contains(srcFs.getUri().getScheme())
            && !FS_SCHEMES_NON_ATOMIC.contains(dstFs.getUri().getScheme())) {
        renamePath(srcFs, src, dst);
    } else {
        copyPath(srcFs, src, dstFs, dst, true, overwrite, conf);
    }
}

---

From source file:gobblin.util.HadoopUtils.java

License:Apache License

/**
 * Copies a src {@link Path} from a srcFs {@link FileSystem} to a dst {@link Path} on a dstFs {@link FileSystem}. If
 * either the srcFs or dstFs are S3 {@link FileSystem}s (as dictated by {@link #FS_SCHEMES_NON_ATOMIC}) then data is directly
 * copied from the src to the dst. Otherwise data is first copied to a tmp {@link Path}, which is then renamed to the
 * dst.//from  w ww  . j av a  2s  .co  m
 *
 * @param srcFs the source {@link FileSystem} where the src {@link Path} exists
 * @param src the {@link Path} to copy from the source {@link FileSystem}
 * @param dstFs the destination {@link FileSystem} where the dst {@link Path} should be created
 * @param dst the {@link Path} to copy data to
 * @param tmp the temporary {@link Path} to use when copying data
 * @param overwriteDst true if the destination and tmp path should should be overwritten, false otherwise
 */
public static void copyFile(FileSystem srcFs, Path src, FileSystem dstFs, Path dst, Path tmp,
        boolean overwriteDst, Configuration conf) throws IOException {

    Preconditions.checkArgument(srcFs.isFile(src),
            String.format("Cannot copy from %s to %s because src is not a file", src, dst));

    if (FS_SCHEMES_NON_ATOMIC.contains(srcFs.getUri().getScheme())
            || FS_SCHEMES_NON_ATOMIC.contains(dstFs.getUri().getScheme())) {
        copyFile(srcFs, src, dstFs, dst, overwriteDst, conf);
    } else {
        copyFile(srcFs, src, dstFs, tmp, overwriteDst, conf);
        try {
            boolean renamed = false;
            if (overwriteDst && dstFs.exists(dst)) {
                try {
                    deletePath(dstFs, dst, true);
                } finally {
                    renamePath(dstFs, tmp, dst);
                    renamed = true;
                }
            }
            if (!renamed) {
                renamePath(dstFs, tmp, dst);
            }
        } finally {
            deletePath(dstFs, tmp, true);
        }
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• »