Example usage for org.apache.hadoop.fs FileSystem makeQualified

List of usage examples for org.apache.hadoop.fs FileSystem makeQualified

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileSystem
  6. makeQualified

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem makeQualified.

Prototype

public Path makeQualified(Path path)

Source Link

Document

Qualify a path to one which uses this FileSystem and, if relative, made absolute.

Usage

From source file:com.asakusafw.runtime.stage.launcher.LauncherOptionsParser.java

License:Apache License

private List<Path> consumeLibraryPaths(LinkedList<String> rest) throws IOException {
    List<String> names = consumeLibraryNames(rest);
    if (names.isEmpty()) {
        return Collections.emptyList();
    }/*from w w w  .  j a v  a  2 s .co  m*/
    List<Path> results = new ArrayList<>();
    LocalFileSystem local = FileSystem.getLocal(configuration);
    for (String name : names) {
        Path path = new Path(name);
        FileSystem fs;
        if (path.toUri().getScheme() == null) {
            fs = local;
        } else {
            fs = path.getFileSystem(configuration);
        }
        path = fs.makeQualified(path);
        if (fs.exists(path) == false) {
            throw new FileNotFoundException(path.toString());
        }
        results.add(path);
    }
    return results;
}

From source file:com.asakusafw.runtime.stage.launcher.LauncherOptionsParser.java

License:Apache License

private String buildLibjars(List<Path> paths) throws IOException {
    StringBuilder buf = new StringBuilder(configuration.get(KEY_CONF_LIBRARIES, "")); //$NON-NLS-1$
    for (Path path : paths) {
        if (buf.length() != 0) {
            buf.append(',');
        }/*from  w  ww .j  a  v a  2 s  .c  om*/
        FileSystem fs = path.getFileSystem(configuration);
        buf.append(fs.makeQualified(path).toString());
    }
    String libjars = buf.toString();
    return libjars;
}

From source file:com.asakusafw.runtime.stage.resource.StageResourceDriver.java

License:Apache License

private Path findCacheForLocalMode(String resourceName, String localName) throws IOException {
    assert resourceName != null;
    assert localName != null;
    Path remotePath = null;//from   ww w  .  j a  v  a  2s .co m
    String remoteName = null;
    for (URI uri : DistributedCache.getCacheFiles(configuration)) {
        if (localName.equals(uri.getFragment())) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("fragment matched: " + uri); //$NON-NLS-1$
            }
            String rpath = uri.getPath();
            remotePath = new Path(uri);
            remoteName = rpath.substring(rpath.lastIndexOf('/') + 1);
            break;
        }
    }
    if (remoteName == null) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("fragment not matched: " + resourceName); //$NON-NLS-1$
        }
        return null;
    }
    assert remotePath != null;
    for (Path path : getLocalCacheFiles()) {
        String localFileName = path.getName();
        if (remoteName.equals(localFileName) == false) {
            continue;
        }
        if (localFileSystem.exists(path) == false) {
            continue;
        }
        if (LOG.isDebugEnabled()) {
            LOG.debug("local path matched: " + path); //$NON-NLS-1$
        }
        return localFileSystem.makeQualified(path);
    }
    FileSystem remoteFileSystem = remotePath.getFileSystem(configuration);
    remotePath = remoteFileSystem.makeQualified(remotePath);
    if (LOG.isDebugEnabled()) {
        LOG.debug("distributed cache is not localized explicitly: " + remotePath); //$NON-NLS-1$
    }
    if (isLocal(remoteFileSystem) == false) {
        LOG.warn(MessageFormat.format("Failed to resolve stage resource in local cache \"{1}\" (resource={0})",
                resourceName, localName));
    }
    return remotePath;
}

From source file:com.asakusafw.runtime.util.cache.HadoopFileCacheRepository.java

License:Apache License

@Override
public Path resolve(Path file) throws IOException, InterruptedException {
    FileSystem fs = file.getFileSystem(configuration);
    Path qualified = fs.makeQualified(file);
    return doResolve(qualified);
}

From source file:com.asakusafw.testdriver.file.FileExporterRetriever.java

License:Apache License

@Override
public void truncate(FileExporterDescription description, TestContext context) throws IOException {
    LOG.info("cleaning output files: {}", description);
    VariableTable variables = createVariables(context);
    Configuration config = configurations.newInstance();
    String resolved = variables.parse(description.getPathPrefix(), false);
    Path path = new Path(resolved);
    FileSystem fs = path.getFileSystem(config);
    Path output = path.getParent();
    Path target;//from w w w. j  a  v a 2 s  .c o m
    if (output == null) {
        LOG.warn(
                "?????????: {}",
                path);
        target = fs.makeQualified(path);
    } else {
        LOG.warn("??????: {}", output);
        target = fs.makeQualified(output);
    }
    LOG.debug("start removing file: {}", target);
    boolean succeed = fs.delete(target, true);
    LOG.debug("finish removing file (succeed={}): {}", succeed, target);
}

From source file:com.asakusafw.testdriver.file.FileImporterPreparator.java

License:Apache License

@Override
public void truncate(FileImporterDescription description, TestContext context) throws IOException {
    LOG.info("cleaning input files: {}", description);
    VariableTable variables = createVariables(context);
    Configuration config = configurations.newInstance();
    FileSystem fs = FileSystem.get(config);
    for (String path : description.getPaths()) {
        String resolved = variables.parse(path, false);
        Path target = fs.makeQualified(new Path(resolved));
        LOG.debug("start removing file: {}", target);
        boolean succeed = fs.delete(target, true);
        LOG.debug("finish removing file (succeed={}): {}", succeed, target);
    }//from  ww w.  j av  a  2 s.  c  om
}

From source file:com.asakusafw.testdriver.JobflowExecutor.java

License:Apache License

/**
 * Cleans up the working directory on the DFS.
 * @throws IOException if failed to clean up
 *//*from   ww  w  .  ja va 2s . c  o  m*/
public void cleanWorkingDirectory() throws IOException {
    Configuration conf = configurations.newInstance();
    FileSystem fs = FileSystem.get(conf);
    Path path = new Path(CompilerConstants.getRuntimeWorkingDirectory());
    Path fullPath = fs.makeQualified(path);
    LOG.debug("start initializing working directory on the testing runtime: {}", fullPath); //$NON-NLS-1$
    boolean deleted = fs.delete(fullPath, true);
    if (deleted) {
        LOG.debug("finish initializing working directory on the testing runtime: {}", fullPath); //$NON-NLS-1$
    } else {
        LOG.debug("failed to initialize working directory on the testing runtime: {}", fullPath); //$NON-NLS-1$
    }
}

From source file:com.asakusafw.testdriver.LegacyJobflowExecutor.java

License:Apache License

/**
 * Cleans up the working directory on the DFS.
 * @throws IOException if failed to clean up
 *//*from   w w w.  j  a va 2s  . com*/
public void cleanWorkingDirectory() throws IOException {
    Configuration conf = configurations.newInstance();
    FileSystem fs = FileSystem.get(conf);
    Path path = new Path(context.getClusterWorkDir());
    Path fullPath = fs.makeQualified(path);
    LOG.debug("start initializing working directory on the testing runtime: {}", fullPath); //$NON-NLS-1$
    boolean deleted = fs.delete(fullPath, true);
    if (deleted) {
        LOG.debug("finish initializing working directory on the testing runtime: {}", fullPath); //$NON-NLS-1$
    } else {
        LOG.debug("failed to initialize working directory on the testing runtime: {}", fullPath); //$NON-NLS-1$
    }
}

From source file:com.asakusafw.testdriver.mapreduce.io.TemporaryInputPreparator.java

License:Apache License

@Override
public void truncate(TemporaryInputDescription description, TestContext context) throws IOException {
    LOG.debug("Deleting input: {}", description); //$NON-NLS-1$
    VariableTable variables = createVariables(context);
    Configuration config = configurations.newInstance();
    FileSystem fs = FileSystem.get(config);
    for (String path : description.getPaths()) {
        String resolved = variables.parse(path, false);
        Path target = fs.makeQualified(new Path(resolved));
        delete(fs, target);/*  w w  w  . ja  va 2 s. c om*/
    }
    return;
}

From source file:com.asakusafw.testdriver.mapreduce.io.TemporaryOutputRetriever.java

License:Apache License

@Override
public void truncate(TemporaryOutputDescription description, TestContext context) throws IOException {
    LOG.debug("Deleting output directory: {}", description); //$NON-NLS-1$
    VariableTable variables = createVariables(context);
    Configuration config = configurations.newInstance();
    FileSystem fs = FileSystem.get(config);
    String resolved = variables.parse(description.getPathPrefix(), false);
    Path path = new Path(resolved);
    Path output = path.getParent();
    Path target;/*from ww  w  . j  a v a  2  s .c  o m*/
    if (output == null) {
        LOG.warn(MessageFormat.format(Messages.getString("TemporaryOutputRetriever.warnDeleteBaseDirectory"), //$NON-NLS-1$
                path));
        target = fs.makeQualified(path);
    } else {
        LOG.debug("output directory will be deleted: {}", output); //$NON-NLS-1$
        target = fs.makeQualified(output);
    }
    TemporaryInputPreparator.delete(fs, target);
}