Example usage for org.apache.hadoop.mapred JobConf get

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred JobConf get.

Prototype

public String get(String name)

Source Link

Document

Get the value of the name property, null if no such property exists.

Usage

From source file:cascading.tap.hadoop.Hadoop18TapUtil.java

License:Open Source License

static boolean isInflow(JobConf conf) {
    return conf.get("cascading.flow.step") != null;
}

From source file:cascading.tap.hadoop.Hadoop18TapUtil.java

License:Open Source License

private static Path getTaskOutputPath(JobConf conf) {
    String taskId = conf.get("mapred.task.id");

    Path p = new Path(FileOutputFormat.getOutputPath(conf), TEMPORARY_PATH + Path.SEPARATOR + "_" + taskId);

    try {//from ww w. jav  a2 s. c  o  m
        FileSystem fs = p.getFileSystem(conf);
        return p.makeQualified(fs);
    } catch (IOException ie) {
        return p;
    }
}

From source file:cascading.tap.hadoop.Hadoop18TapUtil.java

License:Open Source License

private static void moveTaskOutputs(JobConf conf, FileSystem fs, Path jobOutputDir, Path taskOutput)
        throws IOException {
    String taskId = conf.get("mapred.task.id");

    if (fs.isFile(taskOutput)) {
        Path finalOutputPath = getFinalPath(jobOutputDir, taskOutput, getTaskOutputPath(conf));
        if (!fs.rename(taskOutput, finalOutputPath)) {
            if (!fs.delete(finalOutputPath, true)) {
                throw new IOException("Failed to delete earlier output of task: " + taskId);
            }/*from  ww  w .  j  a  v a2s  . co m*/
            if (!fs.rename(taskOutput, finalOutputPath)) {
                throw new IOException("Failed to save output of task: " + taskId);
            }
        }
        LOG.debug("Moved " + taskOutput + " to " + finalOutputPath);
    } else if (fs.getFileStatus(taskOutput).isDir()) {
        FileStatus[] paths = fs.listStatus(taskOutput);
        Path finalOutputPath = getFinalPath(jobOutputDir, taskOutput, getTaskOutputPath(conf));
        fs.mkdirs(finalOutputPath);
        if (paths != null) {
            for (FileStatus path : paths) {
                moveTaskOutputs(conf, fs, jobOutputDir, path.getPath());
            }
        }
    }
}

From source file:cascading.tap.hadoop.io.MultiInputFormat.java

License:Open Source License

/**
 * Used to set the current JobConf with all sub jobs configurations.
 *
 * @param toJob//  w w w.  j  a va  2  s.  c o  m
 * @param fromJobs
 */
public static void addInputFormat(JobConf toJob, JobConf... fromJobs) {
    toJob.setInputFormat(MultiInputFormat.class);
    List<Map<String, String>> configs = new ArrayList<Map<String, String>>();
    List<Path> allPaths = new ArrayList<Path>();

    boolean isLocal = false;

    for (JobConf fromJob : fromJobs) {
        if (fromJob.get("mapred.input.format.class") == null)
            throw new CascadingException(
                    "mapred.input.format.class is required, should be set in source Scheme#sourceConfInit");

        configs.add(HadoopUtil.getConfig(toJob, fromJob));
        Collections.addAll(allPaths, FileInputFormat.getInputPaths(fromJob));

        if (!isLocal)
            isLocal = HadoopUtil.isLocal(fromJob);
    }

    if (!allPaths.isEmpty()) // it's possible there aren't any
        FileInputFormat.setInputPaths(toJob, (Path[]) allPaths.toArray(new Path[allPaths.size()]));

    try {
        toJob.set("cascading.multiinputformats", HadoopUtil.serializeBase64(configs, toJob, true));
    } catch (IOException exception) {
        throw new CascadingException("unable to pack input formats", exception);
    }

    if (isLocal)
        HadoopUtil.setLocal(toJob);
}

From source file:cascading.tap.hadoop.io.MultiInputFormat.java

License:Open Source License

private List<Map<String, String>> getConfigs(JobConf job) throws IOException {
    return (List<Map<String, String>>) HadoopUtil.deserializeBase64(job.get("cascading.multiinputformats"), job,
            ArrayList.class, true);
}

From source file:cascading.tap.hadoop.io.MultiInputSplit.java

License:Open Source License

/**
 * Method getCurrentTapSourcePath finds and returns the current source Tap filename path, if any.
 * <p/>/*from w w w  .ja va 2 s  .  c o m*/
 * Use this method inside an Operation to find the current file being processed.
 *
 * @param jobConf
 * @return a String
 */
public static String getCurrentTapSourcePath(JobConf jobConf) {
    return jobConf.get(CASCADING_SOURCE_PATH);
}

From source file:cascading.tap.hadoop.MultiInputFormat.java

License:Open Source License

/**
 * Used to set the current JobConf with all sub jobs configurations.
 *
 * @param toJob/*  w ww  .ja va2 s.c  o m*/
 * @param fromJobs
 */
public static void addInputFormat(JobConf toJob, JobConf... fromJobs) {
    toJob.setInputFormat(MultiInputFormat.class);
    List<Map<String, String>> configs = new ArrayList<Map<String, String>>();
    List<Path> allPaths = new ArrayList<Path>();

    boolean isLocal = false;

    for (JobConf fromJob : fromJobs) {
        configs.add(getConfig(toJob, fromJob));
        Collections.addAll(allPaths, FileInputFormat.getInputPaths(fromJob));

        if (!isLocal)
            isLocal = fromJob.get("mapred.job.tracker").equalsIgnoreCase("local");
    }

    FileInputFormat.setInputPaths(toJob, (Path[]) allPaths.toArray(new Path[allPaths.size()]));

    try {
        toJob.set("cascading.multiinputformats", Util.serializeBase64(configs));
    } catch (IOException exception) {
        throw new CascadingException("unable to pack input formats", exception);
    }

    if (isLocal)
        toJob.set("mapred.job.tracker", "local");
}

From source file:cascading.tap.hadoop.MultiInputFormat.java

License:Open Source License

private List<Map<String, String>> getConfigs(JobConf job) throws IOException {
    return (List<Map<String, String>>) Util.deserializeBase64(job.get("cascading.multiinputformats"));
}

From source file:cascading.tap.hadoop.MultiInputSplit.java

License:Open Source License

/**
 * Method getCurrentTapSourcePath finds and returns the current source Tap filename path, if any.
 * <p/>/*www.jav  a 2s . c  o  m*/
 * Use this method inside an Operation to find the current file being processed.
 *
 * @param jobConf
 * @return a String
 */
public static String getCurrentTapSourcePath(JobConf jobConf) {
    return jobConf.get("cascading.source.path");
}

From source file:cascading.tap.Hfs.java

License:Open Source License

@Override
public void sinkInit(JobConf conf) throws IOException {
    // do not delete if initialized from within a task
    if (isReplace() && conf.get("mapred.task.partition") == null)
        deletePath(conf);/*from w  w  w  . j ava 2  s . c om*/

    Path qualifiedPath = getQualifiedPath(conf);

    FileOutputFormat.setOutputPath(conf, qualifiedPath);
    super.sinkInit(conf);

    makeLocal(conf, qualifiedPath, "forcing job to local mode, via sink: ");

    TupleSerialization.setSerializations(conf); // allows Hfs to be used independent of Flow
}