Example usage for org.apache.hadoop.mapred JobConf setLong

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred JobConf setLong.

Prototype

public void setLong(String name, long value)

Source Link

Document

Set the value of the name property to a long.

Usage

From source file:com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration.java

License:Open Source License

public static void setMMCJCacheSize(JobConf job, long size) {
    job.setLong(MMCJ_CACHE_SIZE, size);
}

From source file:com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration.java

License:Open Source License

public static void setMatrixDimension(JobConf job, byte matrixIndex, long rlen, long clen) {
    job.setLong(INPUT_MATRIX_NUM_ROW_PREFIX_CONFIG + matrixIndex, rlen);
    job.setLong(INPUT_MATRIX_NUM_COLUMN_PREFIX_CONFIG + matrixIndex, clen);
}

From source file:com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration.java

License:Open Source License

public static void setMatrixDimension(JobConf job, byte matrixIndex, long rlen, long clen, long nnz) {
    job.setLong(INPUT_MATRIX_NUM_ROW_PREFIX_CONFIG + matrixIndex, rlen);
    job.setLong(INPUT_MATRIX_NUM_COLUMN_PREFIX_CONFIG + matrixIndex, clen);
    job.setLong(INPUT_MATRIX_NUM_NNZ_PREFIX_CONFIG + matrixIndex, nnz);
}

From source file:com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration.java

License:Open Source License

/**
 * /*w w w  . j  ava2 s .  c om*/
 * @param job
 * @param inputIndexes
 * @param inputs
 * @param inputInfos
 * @param brlens
 * @param bclens
 * @param distCacheOnly
 * @param setConverter
 * @param target
 * @throws Exception
 */
public static void setUpMultipleInputs(JobConf job, byte[] inputIndexes, String[] inputs,
        InputInfo[] inputInfos, int[] brlens, int[] bclens, boolean[] distCacheOnly, boolean setConverter,
        ConvertTarget target) throws Exception {
    if (inputs.length != inputInfos.length)
        throw new Exception("number of inputs and inputInfos does not match");

    //set up names of the input matrices and their inputformat information
    job.setStrings(INPUT_MATRICIES_DIRS_CONFIG, inputs);
    MRJobConfiguration.setMapFunctionInputMatrixIndexes(job, inputIndexes);

    //set up converter infos (converter determined implicitly)
    if (setConverter) {
        for (int i = 0; i < inputs.length; i++)
            setInputInfo(job, inputIndexes[i], inputInfos[i], brlens[i], bclens[i], target);
    }

    //remove redundant inputs and pure broadcast variables
    ArrayList<Path> lpaths = new ArrayList<Path>();
    ArrayList<InputInfo> liinfos = new ArrayList<InputInfo>();
    for (int i = 0; i < inputs.length; i++) {
        Path p = new Path(inputs[i]);

        //check and skip redundant inputs
        if (lpaths.contains(p) //path already included
                || distCacheOnly[i]) //input only required in dist cache
        {
            continue;
        }

        lpaths.add(p);
        liinfos.add(inputInfos[i]);
    }

    boolean combineInputFormat = false;
    if (OptimizerUtils.ALLOW_COMBINE_FILE_INPUT_FORMAT) {
        //determine total input sizes
        double totalInputSize = 0;
        for (int i = 0; i < inputs.length; i++)
            totalInputSize += MapReduceTool.getFilesizeOnHDFS(new Path(inputs[i]));

        //set max split size (default blocksize) to 2x blocksize if (1) sort buffer large enough, 
        //(2) degree of parallelism not hurt, and only a single input (except broadcasts)
        //(the sort buffer size is relevant for pass-through of, potentially modified, inputs to the reducers)
        //(the single input constraint stems from internal runtime assumptions used to relate meta data to inputs)
        long sizeSortBuff = InfrastructureAnalyzer.getRemoteMaxMemorySortBuffer();
        long sizeHDFSBlk = InfrastructureAnalyzer.getHDFSBlockSize();
        long newSplitSize = sizeHDFSBlk * 2;
        double spillPercent = job.getDouble("mapreduce.map.sort.spill.percent", 1.0);
        int numPMap = OptimizerUtils.getNumMappers();
        if (numPMap < totalInputSize / newSplitSize && sizeSortBuff * spillPercent >= newSplitSize
                && lpaths.size() == 1) {
            job.setLong("mapreduce.input.fileinputformat.split.maxsize", newSplitSize);
            combineInputFormat = true;
        }
    }

    //add inputs to jobs input (incl input format configuration)
    for (int i = 0; i < lpaths.size(); i++) {
        //add input to job inputs (for binaryblock we use CombineSequenceFileInputFormat to reduce task latency)
        if (combineInputFormat && liinfos.get(i) == InputInfo.BinaryBlockInputInfo)
            MultipleInputs.addInputPath(job, lpaths.get(i), CombineSequenceFileInputFormat.class);
        else
            MultipleInputs.addInputPath(job, lpaths.get(i), liinfos.get(i).inputFormatClass);
    }
}

From source file:com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration.java

License:Open Source License

public static void setIntermediateMatrixCharactristics(JobConf job, byte tag, MatrixCharacteristics dim) {

    job.setLong(INTERMEDIATE_MATRIX_NUM_ROW_PREFIX_CONFIG + tag, dim.getRows());
    job.setLong(INTERMEDIATE_MATRIX_NUM_COLUMN_PREFIX_CONFIG + tag, dim.getCols());
    job.setInt(INTERMEDIATE_BLOCK_NUM_ROW_PREFIX_CONFIG + tag, dim.getRowsPerBlock());
    job.setInt(INTERMEDIATE_BLOCK_NUM_COLUMN_PREFIX_CONFIG + tag, dim.getColsPerBlock());
}

From source file:com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration.java

License:Open Source License

public static void setMatrixCharactristicsForOutput(JobConf job, byte tag, MatrixCharacteristics dim) {
    job.setLong(OUTPUT_MATRIX_NUM_ROW_PREFIX_CONFIG + tag, dim.getRows());
    job.setLong(OUTPUT_MATRIX_NUM_COLUMN_PREFIX_CONFIG + tag, dim.getCols());
    job.setInt(OUTPUT_BLOCK_NUM_ROW_PREFIX_CONFIG + tag, dim.getRowsPerBlock());
    job.setInt(OUTPUT_BLOCK_NUM_COLUMN_PREFIX_CONFIG + tag, dim.getColsPerBlock());
}

From source file:com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration.java

License:Open Source License

public static void setMatrixCharactristicsForMapperOutput(JobConf job, byte tag, MatrixCharacteristics dim) {
    job.setLong(MAPOUTPUT_MATRIX_NUM_ROW_PREFIX_CONFIG + tag, dim.getRows());
    job.setLong(MAPOUTPUT_MATRIX_NUM_COLUMN_PREFIX_CONFIG + tag, dim.getCols());
    job.setInt(MAPOUTPUT_BLOCK_NUM_ROW_PREFIX_CONFIG + tag, dim.getRowsPerBlock());
    job.setInt(MAPOUTPUT_BLOCK_NUM_COLUMN_PREFIX_CONFIG + tag, dim.getColsPerBlock());
}

From source file:com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration.java

License:Open Source License

public static void setMatrixCharactristicsForReblock(JobConf job, byte tag, MatrixCharacteristics dim) {
    job.setLong(REBLOCK_MATRIX_NUM_ROW_PREFIX_CONFIG + tag, dim.getRows());
    job.setLong(REBLOCK_MATRIX_NUM_COLUMN_PREFIX_CONFIG + tag, dim.getCols());
    job.setInt(REBLOCK_BLOCK_NUM_ROW_PREFIX_CONFIG + tag, dim.getRowsPerBlock());
    job.setInt(REBLOCK_BLOCK_NUM_COLUMN_PREFIX_CONFIG + tag, dim.getColsPerBlock());
    job.setLong(REBLOCK_MATRIX_NUM_NNZ_PREFIX_CONFIG + tag, dim.getNonZeros());
}

From source file:com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration.java

License:Open Source License

public static void setMatrixCharactristicsForBinAgg(JobConf job, byte tag, MatrixCharacteristics dim) {
    job.setLong(AGGBIN_MATRIX_NUM_ROW_PREFIX_CONFIG + tag, dim.getRows());
    job.setLong(AGGBIN_MATRIX_NUM_COLUMN_PREFIX_CONFIG + tag, dim.getCols());
    job.setInt(AGGBIN_BLOCK_NUM_ROW_PREFIX_CONFIG + tag, dim.getRowsPerBlock());
    job.setInt(AGGBIN_BLOCK_NUM_COLUMN_PREFIX_CONFIG + tag, dim.getColsPerBlock());
}

From source file:com.ibm.bi.dml.runtime.matrix.sort.PickFromCompactInputFormat.java

License:Open Source License

public static void setZeroValues(JobConf job, NumItemsByEachReducerMetaData metadata) {
    job.setInt(PARTITION_OF_ZERO, metadata.getPartitionOfZero());
    job.setLong(NUMBER_OF_ZERO, metadata.getNumberOfZero());
}