Example usage for org.apache.hadoop.mapred JobConf get

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred JobConf get.

Prototype

public String get(String name)

Source Link

Document

Get the value of the name property, null if no such property exists.

Usage

From source file:com.ibm.bi.dml.runtime.transform.ApplyTfHelper.java

License:Open Source License

public ApplyTfHelper(JobConf job) throws IllegalArgumentException, IOException {
    _hasHeader = Boolean.parseBoolean(job.get(MRJobConfiguration.TF_HAS_HEADER));

    _delimString = job.get(MRJobConfiguration.TF_DELIM);
    _delim = Pattern.compile(Pattern.quote(_delimString));

    _naStrings = DataTransform.parseNAStrings(job);

    _numCols = UtilFunctions.parseToLong(job.get(MRJobConfiguration.TF_NUM_COLS)); // #of columns in input data
    _tmpPath = job.get(MRJobConfiguration.TF_TMP_LOC);

    _specFile = job.get(MRJobConfiguration.TF_SPEC_FILE);

    _numTransformedRows = 0;/*w w w .j  a  v  a2  s.com*/
    _numTransformedColumns = 0;

    _rJob = job;
}

From source file:com.ibm.bi.dml.runtime.transform.GTFMTDMapper.java

License:Open Source License

/**
 * Configure the information used in the mapper, and setup transformation agents.
 *//*from   www . j a  v  a  2s  .co  m*/
@Override
public void configure(JobConf job) {
    String[] parts = job.get("mapred.task.id").split("_");
    if (parts[0].equalsIgnoreCase("task")) {
        _mapTaskID = Integer.parseInt(parts[parts.length - 1]);
    } else if (parts[0].equalsIgnoreCase("attempt")) {
        _mapTaskID = Integer.parseInt(parts[parts.length - 2]);
    } else {
        throw new RuntimeException("Unrecognized format for taskID: " + job.get("mapred.task.id"));
    }

    try {
        _partFileName = TfUtils.getPartFileName(job);
        _partFileWithHeader = TfUtils.isPartFileWithHeader(job);
        _agents = new TfUtils(job);
    } catch (IOException e) {
        throw new RuntimeException(e);
    } catch (JSONException e) {
        throw new RuntimeException(e);
    }

}

From source file:com.ibm.bi.dml.runtime.transform.TfUtils.java

License:Open Source License

public static String getPartFileName(JobConf job) throws IOException {
    FileSystem fs = FileSystem.get(job);
    Path thisPath = new Path(job.get("map.input.file")).makeQualified(fs);
    return thisPath.toString();
}

From source file:com.ibm.bi.dml.runtime.transform.TfUtils.java

License:Open Source License

public static boolean isPartFileWithHeader(JobConf job) throws IOException {
    FileSystem fs = FileSystem.get(job);

    String thisfile = getPartFileName(job);
    Path smallestFilePath = new Path(job.get(MRJobConfiguration.TF_SMALLEST_FILE)).makeQualified(fs);

    if (thisfile.toString().equals(smallestFilePath.toString()))
        return true;
    else/*from ww  w. ja  v a 2 s  .c om*/
        return false;
}

From source file:com.ibm.bi.dml.runtime.transform.TfUtils.java

License:Open Source License

public static String[] parseNAStrings(JobConf job) {
    return parseNAStrings(job.get(MRJobConfiguration.TF_NA_STRINGS));
}

From source file:com.ibm.bi.dml.runtime.transform.TfUtils.java

License:Open Source License

public TfUtils(JobConf job, boolean minimal) throws IOException, JSONException {
    if (!InfrastructureAnalyzer.isLocalMode(job)) {
        ConfigurationManager.setCachedJobConf(job);
    }/* ww  w .j a v  a2s .  co m*/

    _NAstrings = TfUtils.parseNAStrings(job);
    _specFile = job.get(MRJobConfiguration.TF_SPEC_FILE);

    FileSystem fs = FileSystem.get(job);
    JSONObject spec = TfUtils.readSpec(fs, _specFile);

    _oa = new OmitAgent(spec);
}

From source file:com.ibm.bi.dml.runtime.transform.TfUtils.java

License:Open Source License

public TfUtils(JobConf job) throws IOException, JSONException {
    if (!InfrastructureAnalyzer.isLocalMode(job)) {
        ConfigurationManager.setCachedJobConf(job);
    }/*from  ww w.ja v  a2  s  .  co m*/

    boolean hasHeader = Boolean.parseBoolean(job.get(MRJobConfiguration.TF_HAS_HEADER));
    //Pattern delim = Pattern.compile(Pattern.quote(job.get(MRJobConfiguration.TF_DELIM)));
    String[] naStrings = TfUtils.parseNAStrings(job);

    long numCols = UtilFunctions.parseToLong(job.get(MRJobConfiguration.TF_NUM_COLS)); // #of columns in input data

    String specFile = job.get(MRJobConfiguration.TF_SPEC_FILE);
    String offsetFile = job.get(MRJobConfiguration.TF_OFFSETS_FILE);
    String tmpPath = job.get(MRJobConfiguration.TF_TMP_LOC);
    String outputPath = FileOutputFormat.getOutputPath(job).toString();
    FileSystem fs = FileSystem.get(job);
    JSONObject spec = TfUtils.readSpec(fs, specFile);

    init(job.get(MRJobConfiguration.TF_HEADER), hasHeader, job.get(MRJobConfiguration.TF_DELIM), naStrings,
            spec, numCols, offsetFile, tmpPath, outputPath);
}

From source file:com.ibm.bi.dml.runtime.transform.TfUtils.java

License:Open Source License

private Reader initOffsetsReader(JobConf job) throws IOException {
    Path path = new Path(job.get(CSVReblockMR.ROWID_FILE_NAME));
    FileSystem fs = FileSystem.get(job);
    Path[] files = MatrixReader.getSequenceFilePaths(fs, path);
    if (files.length != 1)
        throw new IOException("Expecting a single file under counters file: " + path.toString());

    Reader reader = new SequenceFile.Reader(fs, files[0], job);

    return reader;
}

From source file:com.ibm.bi.dml.runtime.util.MapReduceTool.java

License:Open Source License

public static String getUniqueKeyPerTask(JobConf job, boolean inMapper) {
    //TODO: investigate ID pattern, required for parallel jobs
    /*String nodePrefix = job.get("mapred.task.id");
    return String.valueOf(IDHandler.extractLongID(nodePrefix));*/

    String nodePrefix = job.get("mapred.task.id");
    int i;//  w ww . jav  a  2 s .com
    if (inMapper)
        i = nodePrefix.indexOf("_m_");
    else
        i = nodePrefix.indexOf("_r_");
    int j = nodePrefix.lastIndexOf("_");
    nodePrefix = nodePrefix.substring(i + 3, j);
    // remove all the leading 0s
    return String.valueOf(Long.parseLong(nodePrefix));
}

From source file:com.ibm.bi.dml.runtime.util.MapReduceTool.java

License:Open Source License

@Deprecated
public static String getUniqueKeyPerTaskWithLeadingZros(JobConf job, boolean inMapper) {
    String nodePrefix = job.get("mapred.task.id");
    int i;//from ww  w. ja  v  a  2s  .  com
    if (inMapper)
        i = nodePrefix.indexOf("_m_");
    else
        i = nodePrefix.indexOf("_r_");
    int j = nodePrefix.lastIndexOf("_");
    nodePrefix = nodePrefix.substring(i + 3, j);
    return nodePrefix;
}