Example usage for org.apache.hadoop.fs Path toString

List of usage examples for org.apache.hadoop.fs Path toString

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. Path
  6. toString

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path toString.

Prototype

@Override
    public String toString()

Source Link

Usage

From source file:com.github.gaoyangthu.demo.mapred.terasort.TeraSort.java

License:Apache License

public int run(String[] args) throws Exception {
    LOG.info("starting");
    JobConf job = (JobConf) getConf();//from   w  ww. j  a v a  2 s . co  m
    Path inputDir = new Path(args[0]);
    inputDir = inputDir.makeQualified(inputDir.getFileSystem(job));
    Path partitionFile = new Path(inputDir, TeraInputFormat.PARTITION_FILENAME);
    URI partitionUri = new URI(partitionFile.toString() + "#" + TeraInputFormat.PARTITION_FILENAME);
    TeraInputFormat.setInputPaths(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));
    job.setJobName("TeraSort");
    job.setJarByClass(TeraSort.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);
    job.setInputFormat(TeraInputFormat.class);
    job.setOutputFormat(TeraOutputFormat.class);
    job.setPartitionerClass(TotalOrderPartitioner.class);
    TeraInputFormat.writePartitionFile(job, partitionFile);
    DistributedCache.addCacheFile(partitionUri, job);
    DistributedCache.createSymlink(job);
    job.setInt("dfs.replication", 1);
    TeraOutputFormat.setFinalSync(job, true);
    JobClient.runJob(job);
    LOG.info("done");
    return 0;
}

From source file:com.github.hdl.tensorflow.yarn.app.Client.java

License:Apache License

private String copyLocalFileToDfs(FileSystem fs, String appId, String srcFilePath, String dstFileName)
        throws IOException {
    String suffix = TFYarnConstants.APP_NAME + "/" + appId + "/" + dstFileName;
    Path dst = new Path(fs.getHomeDirectory(), suffix);
    if (srcFilePath != null) {
        fs.copyFromLocalFile(new Path(srcFilePath), dst);
    }/*from  w  w  w.  j  ava 2 s  .co m*/
    LOG.info("Copy " + srcFilePath + " to " + dst.toString());
    return dst.toString();
}

From source file:com.github.hdl.tensorflow.yarn.app.TFContainer.java

License:Apache License

public void addToLocalResources(FileSystem fs, Path dst, String fileDstPath,
        Map<String, LocalResource> localResources) throws IOException {
    FileStatus scFileStatus = fs.getFileStatus(dst);
    LOG.info("Path " + dst.toString() + "->" + " " + fileDstPath);
    LocalResource scRsrc = LocalResource.newInstance(URL.fromURI(dst.toUri()), LocalResourceType.FILE,
            LocalResourceVisibility.APPLICATION, scFileStatus.getLen(), scFileStatus.getModificationTime());
    localResources.put(fileDstPath, scRsrc);
}

From source file:com.github.hdl.tensorflow.yarn.app.TFContainer.java

License:Apache License

public void addToLocalResources(FileSystem fs, String fileSrcPath, String fileDstPath, String appId,
        Map<String, LocalResource> localResources, String resources) throws IOException {

    execCmd("pwd");
    execCmd("ls -l");
    String suffix = appName + "/" + appId + "/" + fileDstPath;
    Path dst = new Path(fs.getHomeDirectory(), suffix);
    LOG.info("copy: " + fileSrcPath + " ===> " + dst.toString());
    if (fileSrcPath == null) {
        FSDataOutputStream ostream = null;
        try {/*from w ww . ja  v  a2s  . c  om*/
            ostream = FileSystem.create(fs, dst, new FsPermission((short) 0710));
            ostream.writeUTF(resources);
        } finally {
            IOUtils.closeQuietly(ostream);
        }
    } else {
        fs.copyFromLocalFile(new Path(fileSrcPath), dst);
    }

    FileStatus scFileStatus = fs.getFileStatus(dst);
    LocalResource scRsrc = LocalResource.newInstance(URL.fromURI(dst.toUri()), LocalResourceType.FILE,
            LocalResourceVisibility.APPLICATION, scFileStatus.getLen(), scFileStatus.getModificationTime());
    localResources.put(fileDstPath, scRsrc);
}

From source file:com.github.libsml.commons.util.HadoopUtils.java

License:Apache License

/**
 * Create a map-only Hadoop Job out of the passed in parameters.  Does not set the
 * Job name./* w w w.  j  a v a 2s .c o m*/
 *
 * @see #getCustomJobName(String, JobContext, Class, Class)
 */
public static Job prepareJob(Path inputPath, Path outputPath, Class<? extends InputFormat> inputFormat,
        Class<? extends Mapper> mapper, Class<? extends Writable> mapperKey,
        Class<? extends Writable> mapperValue, Class<? extends OutputFormat> outputFormat, Configuration conf)
        throws IOException {

    //    Job job = new Job(new Configuration(conf));
    Job job = Job.getInstance(conf);
    Configuration jobConf = job.getConfiguration();

    if (mapper.equals(Mapper.class)) {
        throw new IllegalStateException("Can't figure out the user class jar file from mapper/reducer");
    }
    job.setJarByClass(mapper);

    job.setInputFormatClass(inputFormat);
    jobConf.set("mapred.input.dir", inputPath.toString());

    job.setMapperClass(mapper);
    job.setMapOutputKeyClass(mapperKey);
    job.setMapOutputValueClass(mapperValue);
    job.setOutputKeyClass(mapperKey);
    job.setOutputValueClass(mapperValue);
    jobConf.setBoolean("mapred.compress.map.output", true);
    job.setNumReduceTasks(0);

    job.setOutputFormatClass(outputFormat);
    jobConf.set("mapred.output.dir", outputPath.toString());

    return job;
}

From source file:com.github.libsml.commons.util.HadoopUtils.java

License:Apache License

/**
 * Create a map and reduce Hadoop job.  Does not set the name on the job.
 *
 * @param inputPath    The input {@link Path}
 * @param outputPath   The output {@link Path}
 * @param inputFormat  The {@link InputFormat}
 * @param mapper       The {@link Mapper} class to use
 * @param mapperKey    The {@link Writable} key class.  If the Mapper is a no-op,
 *                     this value may be null
 * @param mapperValue  The {@link Writable} value class.  If the Mapper is a no-op,
 *                     this value may be null
 * @param reducer      The {@link Reducer} to use
 * @param reducerKey   The reducer key class.
 * @param reducerValue The reducer value class.
 * @param outputFormat The {@link OutputFormat}.
 * @param conf         The {@link Configuration} to use.
 * @return The {@link Job}.//from   w  w w .  java 2 s  .  com
 * @throws IOException if there is a problem with the IO.
 * @see #getCustomJobName(String, JobContext, Class, Class)
 * @see #prepareJob(Path, Path, Class, Class, Class, Class, Class,
 * Configuration)
 */
public static Job prepareJob(Path inputPath, Path outputPath, Class<? extends InputFormat> inputFormat,
        Class<? extends Mapper> mapper, Class<? extends Writable> mapperKey,
        Class<? extends Writable> mapperValue, Class<? extends Reducer> reducer,
        Class<? extends Writable> reducerKey, Class<? extends Writable> reducerValue,
        Class<? extends OutputFormat> outputFormat, Configuration conf) throws IOException {
    return prepareJob(inputPath.toString(), outputPath.toString(), inputFormat, mapper, mapperKey, mapperValue,
            reducer, reducerKey, reducerValue, outputFormat, conf);

}

From source file:com.github.libsml.commons.util.HadoopUtils.java

License:Apache License

public static void mkdir(Path path, boolean overwrite) throws IOException {

    Configuration config = new Configuration();
    FileSystem fs = path.getFileSystem(config);
    if (fs.exists(path) && !overwrite) {
        throw new IllegalStateException("Mkdir exception:path=" + path.toString() + " exists");
    }/*  ww  w.  j  a v a2 s .c om*/
    if (fs.exists(path)) {
        fs.delete(path, true);
    }
    fs.mkdirs(path);
    fs.close();
}

From source file:com.github.sakserv.minicluster.yarn.InJvmContainerExecutor.java

License:Apache License

/**
 * Overrides the parent method while still invoking it. Since
 * {@link #isContainerActive(ContainerId)} method is also overridden here and
 * always returns 'false' the super.launchContainer(..) will only go through
 * the prep routine (e.g., creating temp dirs etc.) while never launching the
 * actual container via the launch script. This will ensure that all the
 * expectations of the container to be launched (e.g., symlinks etc.) are
 * satisfied. The actual launch will be performed by invoking
 * {@link #doLaunch(Container, Path)} method.
 *//*from  ww w  .  j  a va 2 s.c  o  m*/
public int launchContainer(ContainerStartContext containerStartContext) throws IOException {
    Container container = containerStartContext.getContainer();
    Path containerWorkDir = containerStartContext.getContainerWorkDir();
    super.launchContainer(containerStartContext);
    int exitCode = 0;
    if (container.getLaunchContext().getCommands().toString().contains("bin/java")) {
        ExecJavaCliParser result = this.createExecCommandParser(containerWorkDir.toString());
        try {
            exitCode = this.doLaunch(container, containerWorkDir);
            if (logger.isInfoEnabled()) {
                logger.info(("Returned: " + exitCode));
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    } else {
        String cmd = container.getLaunchContext().getCommands().get(0);
        if (logger.isInfoEnabled()) {
            logger.info("Running Command: " + cmd);
        }
        ExecShellCliParser execShellCliParser = new ExecShellCliParser(cmd);
        try {
            exitCode = execShellCliParser.runCommand();
        } catch (Exception e) {
            e.printStackTrace();
        }
        if (logger.isInfoEnabled()) {
            logger.info(("Returned: " + exitCode));
        }
    }
    return exitCode;
}

From source file:com.github.sakserv.minicluster.yarn.InJvmContainerExecutor.java

License:Apache License

/**
 * Overrides the parent method while still invoking it. Since
 * {@link #isContainerActive(ContainerId)} method is also overridden here and
 * always returns 'false' the super.launchContainer(..) will only go through
 * the prep routine (e.g., creating temp dirs etc.) while never launching the
 * actual container via the launch script. This will ensure that all the
 * expectations of the container to be launched (e.g., symlinks etc.) are
 * satisfied. The actual launch will be performed by invoking
 * {@link #doLaunch(Container, Path)} method.
 *//*from w  ww. j av  a  2  s. c  o m*/
public int launchContainer(Container container, Path nmPrivateContainerScriptPath, Path nmPrivateTokensPath,
        String userName, String appId, Path containerWorkDir, List<String> localDirs, List<String> logDirs)
        throws IOException {
    ContainerStartContext containerStartContext = new ContainerStartContext.Builder().setContainer(container)
            .setLocalizedResources(container.getLocalizedResources())
            .setNmPrivateContainerScriptPath(nmPrivateContainerScriptPath)
            .setNmPrivateTokensPath(nmPrivateTokensPath).setUser(userName).setAppId(appId)
            .setContainerWorkDir(containerWorkDir).setLocalDirs(localDirs).setLocalDirs(logDirs).build();

    super.launchContainer(containerStartContext);
    int exitCode = 0;
    if (container.getLaunchContext().getCommands().toString().contains("bin/java")) {
        ExecJavaCliParser result = this.createExecCommandParser(containerWorkDir.toString());
        try {
            exitCode = this.doLaunch(container, containerWorkDir);
            if (logger.isInfoEnabled()) {
                logger.info(("Returned: " + exitCode));
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    } else {
        String cmd = container.getLaunchContext().getCommands().get(0);
        if (logger.isInfoEnabled()) {
            logger.info("Running Command: " + cmd);
        }
        ExecShellCliParser execShellCliParser = new ExecShellCliParser(cmd);
        try {
            exitCode = execShellCliParser.runCommand();
        } catch (Exception e) {
            e.printStackTrace();
        }
        if (logger.isInfoEnabled()) {
            logger.info(("Returned: " + exitCode));
        }
    }
    return exitCode;
}

From source file:com.github.sakserv.minicluster.yarn.InJvmContainerExecutor.java

License:Apache License

/**
 * Will launch containers within the same JVM as this Container Executor. It
 * will do so by: - extracting Container's class name and program arguments
 * from the launch script (e.g., launch_container.sh) - Creating an isolated
 * ClassLoader for each container - Calling doLaunchContainer(..) method to
 * launch Container/*from www  . j  av a2 s.  c o  m*/
 */
private int doLaunch(Container container, Path containerWorkDir) throws Exception {
    Map<String, String> environment = container.getLaunchContext().getEnvironment();
    EnvironmentUtils.putAll(environment);

    Set<URL> additionalClassPathUrls = this.filterAndBuildUserClasspath(container);

    ExecJavaCliParser javaCliParser = this.createExecCommandParser(containerWorkDir.toString());

    UserGroupInformation.setLoginUser(null);
    try {
        // create Isolated Class Loader for each container and set it as context
        // class loader
        URLClassLoader containerCl = new URLClassLoader(
                additionalClassPathUrls.toArray(additionalClassPathUrls.toArray(new URL[] {})), null);
        Thread.currentThread().setContextClassLoader(containerCl);
        String containerLauncher = javaCliParser.getMain();

        Class<?> containerClass = Class.forName(containerLauncher, true, containerCl);
        Method mainMethod = containerClass.getMethod("main", new Class[] { String[].class });
        mainMethod.setAccessible(true);
        String[] arguments = javaCliParser.getMainArguments();

        this.doLaunchContainer(containerClass, mainMethod, arguments);

    } catch (Exception e) {
        logger.error("Failed to launch container " + container, e);
        container.handle(new ContainerDiagnosticsUpdateEvent(container.getContainerId(), e.getMessage()));
        return -1;
    } finally {
        logger.info("Removing symlinks");
        this.cleanUp();
    }
    return 0;
}