Example usage for org.apache.hadoop.mapred JobConf getBoolean

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred JobConf getBoolean.

Prototype

public boolean getBoolean(String name, boolean defaultValue)

Source Link

Document

Get the value of the name property as a boolean.

Usage

From source file:com.ibm.bi.dml.runtime.matrix.data.MultipleOutputCommitter.java

License:Open Source License

/**
 * //from w w  w.  j ava2  s.  c om
 * @param context
 * @param fs
 * @param file
 * @throws IOException
 */
private void moveFileToDestination(TaskAttemptContext context, FileSystem fs, Path file) throws IOException {
    JobConf conf = context.getJobConf();
    TaskAttemptID attemptId = context.getTaskAttemptID();

    //get output index and final destination
    String taskType = (conf.getBoolean(JobContext.TASK_ISMAP, true)) ? "m" : "r";
    String name = file.getName();
    int charIx = name.indexOf("-" + taskType + "-");
    int index = Integer.parseInt(name.substring(0, charIx));
    Path finalPath = new Path(outputs[index], file.getName());

    //move file from 'file' to 'finalPath'
    if (!fs.rename(file, finalPath)) {
        if (!fs.delete(finalPath, true))
            throw new IOException("Failed to delete earlier output " + finalPath + " for rename of " + file
                    + " in task " + attemptId);
        if (!fs.rename(file, finalPath))
            throw new IOException(
                    "Failed to save output " + finalPath + " for rename of " + file + " in task: " + attemptId);
    }
}

From source file:com.ibm.bi.dml.runtime.matrix.mapred.CSVAssignRowIDMapper.java

License:Open Source License

@Override
@SuppressWarnings("deprecation")
public void configure(JobConf job) {
    byte thisIndex;
    try {//from w ww. ja v  a2s.  com
        //it doesn't make sense to have repeated file names in the input, since this is for reblock
        thisIndex = MRJobConfiguration.getInputMatrixIndexesInMapper(job).get(0);
        outKey.set(thisIndex);
        FileSystem fs = FileSystem.get(job);
        Path thisPath = new Path(job.get("map.input.file")).makeQualified(fs);
        filename = thisPath.toString();
        String[] strs = job.getStrings(CSVReblockMR.SMALLEST_FILE_NAME_PER_INPUT);
        Path headerPath = new Path(strs[thisIndex]).makeQualified(fs);
        if (headerPath.toString().equals(filename))
            headerFile = true;
    } catch (IOException e) {
        throw new RuntimeException(e);
    }

    try {
        CSVReblockInstruction[] reblockInstructions = MRJobConfiguration.getCSVReblockInstructions(job);
        for (CSVReblockInstruction ins : reblockInstructions) {
            if (ins.input == thisIndex) {
                delim = Pattern.quote(ins.delim);
                ignoreFirstLine = ins.hasHeader;
                break;
            }
        }
    } catch (DMLUnsupportedOperationException e) {
        throw new RuntimeException(e);
    } catch (DMLRuntimeException e) {
        throw new RuntimeException(e);
    }

    // load properties relevant to transform
    try {
        boolean omit = job.getBoolean(MRJobConfiguration.TF_TRANSFORM, false);
        if (omit)
            _agents = new TfUtils(job, true);
    } catch (IOException e) {
        throw new RuntimeException(e);
    } catch (JSONException e) {
        throw new RuntimeException(e);
    }
}

From source file:com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration.java

License:Open Source License

public static Class<? extends MatrixValue> getMatrixValueClass(JobConf job) {
    if (job.getBoolean(WEIGHTEDCELL_REPRESENTATION_CONFIG, false))
        return WeightedPair.class;

    if (job.getBoolean(BLOCK_REPRESENTATION_CONFIG, true))
        return MatrixBlock.class;
    else/*from w w w. ja  va2 s . co m*/
        return MatrixCell.class;
}

From source file:com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration.java

License:Open Source License

public static boolean getParforCachingConfig(JobConf job) {
    return job.getBoolean(PARFOR_CACHING_CONFIG, true);
}

From source file:com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration.java

License:Open Source License

public static boolean getPartitioningTransposedCol(JobConf job) {
    return job.getBoolean(PARTITIONING_TRANSPOSE_COL_CONFIG, false);
}

From source file:com.ibm.bi.dml.runtime.matrix.sort.IndexSortMapper.java

License:Open Source License

@Override
public void configure(JobConf job) {
    super.configure(job);
    _brlen = MRJobConfiguration.getNumRowsPerBlock(job, (byte) 0);
    boolean desc = job.getBoolean(SortMR.SORT_DECREASING, false);
    if (!desc)/*from ww w. ja v a2 s  . c  om*/
        _tmpSortKey = new IndexSortComparable();
    else
        _tmpSortKey = new IndexSortComparableDesc();
}

From source file:com.ibm.bi.dml.runtime.matrix.sort.PickFromCompactInputFormat.java

License:Open Source License

public RecordReader<MatrixIndexes, MatrixCell> getRecordReader(InputSplit split, JobConf job, Reporter reporter)
        throws IOException {
    if (job.getBoolean(INPUT_IS_VECTOR, true))
        return new PickRecordReader(job, (FileSplit) split);
    else/*w w w .j av a 2  s.c o  m*/
        return new RangePickRecordReader(job, (FileSplit) split);
}

From source file:com.ibm.bi.dml.runtime.matrix.sort.ValueSortReducer.java

License:Open Source License

public void configure(JobConf job) {
    taskID = MapReduceTool.getUniqueKeyPerTask(job, false);
    valueIsWeight = job.getBoolean(SortMR.VALUE_IS_WEIGHT, false);
}

From source file:com.intel.hadoop.graphbuilder.idnormalize.mapreduce.SortDictMapper.java

License:Open Source License

@Override
public void configure(JobConf job) {
    super.configure(job);
    this.hashRawVid = job.getBoolean("hashRawVid", true);
    this.numChunks = job.getInt("numChunks", 256);
    try {/*from w ww  . j a  va 2  s . c  o  m*/
        this.vidparser = (FieldParser) Class.forName(job.get("VidParser")).newInstance();
    } catch (InstantiationException e) {
        e.printStackTrace();
    } catch (IllegalAccessException e) {
        e.printStackTrace();
    } catch (ClassNotFoundException e) {
        e.printStackTrace();
    }
}

From source file:com.intel.hadoop.graphbuilder.idnormalize.mapreduce.SortDictReducer.java

License:Open Source License

@Override
public void configure(JobConf job) {
    super.configure(job);
    this.mos = new MultipleOutputs(job);
    this.hashRawVid = job.getBoolean("hashRawVid", true);
}