Example usage for org.apache.hadoop.mapred JobConf set

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred JobConf set.

Prototype

public void set(String name, String value)

Source Link

Document

Set the value of the name property.

Usage

From source file:com.ibm.bi.dml.runtime.matrix.data.hadoopfix.MultipleInputs.java

License:Apache License

/**
 * Add a {@link Path} with a custom {@link InputFormat} to the list of
 * inputs for the map-reduce job.//  www  .  java 2 s.  co  m
 * 
 * @param conf The configuration of the job
 * @param path {@link Path} to be added to the list of inputs for the job
 * @param inputFormatClass {@link InputFormat} class to use for this path
 */
public static void addInputPath(JobConf conf, Path path, Class<? extends InputFormat> inputFormatClass) {

    String inputFormatMapping = path.toString() + ";" + inputFormatClass.getName();
    String inputFormats = conf.get("mapred.input.dir.formats");
    conf.set("mapred.input.dir.formats",
            inputFormats == null ? inputFormatMapping : inputFormats + "," + inputFormatMapping);

    conf.setInputFormat(DelegatingInputFormat.class);
}

From source file:com.ibm.bi.dml.runtime.matrix.data.hadoopfix.MultipleInputs.java

License:Apache License

/**
 * Add a {@link Path} with a custom {@link InputFormat} and
 * {@link Mapper} to the list of inputs for the map-reduce job.
 * /* w  w w.jav a 2  s.co m*/
 * @param conf The configuration of the job
 * @param path {@link Path} to be added to the list of inputs for the job
 * @param inputFormatClass {@link InputFormat} class to use for this path
 * @param mapperClass {@link Mapper} class to use for this path
 */
public static void addInputPath(JobConf conf, Path path, Class<? extends InputFormat> inputFormatClass,
        Class<? extends Mapper> mapperClass) {

    addInputPath(conf, path, inputFormatClass);

    String mapperMapping = path.toString() + ";" + mapperClass.getName();
    String mappers = conf.get("mapred.input.dir.mappers");
    conf.set("mapred.input.dir.mappers", mappers == null ? mapperMapping : mappers + "," + mapperMapping);

    conf.setMapperClass(DelegatingMapper.class);
}

From source file:com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration.java

License:Open Source License

/**
 * Unique working dirs required for thread-safe submission of parallel jobs;
 * otherwise job.xml and other files might be overridden (in local mode).
 * //w w w  .j ava2 s .  com
 * @param job
 * @param mode
 */
public static void setUniqueWorkingDir(JobConf job) {
    if (InfrastructureAnalyzer.isLocalMode(job)) {
        StringBuilder tmp = new StringBuilder();
        tmp.append(Lop.FILE_SEPARATOR);
        tmp.append(Lop.PROCESS_PREFIX);
        tmp.append(DMLScript.getUUID());
        tmp.append(Lop.FILE_SEPARATOR);
        tmp.append(seq.getNextID());
        String uniqueSubdir = tmp.toString();

        //unique local dir
        String[] dirlist = job.get("mapred.local.dir", "/tmp").split(",");
        StringBuilder sb2 = new StringBuilder();
        for (String dir : dirlist) {
            if (sb2.length() > 0)
                sb2.append(",");
            sb2.append(dir);
            sb2.append(uniqueSubdir);
        }
        job.set("mapred.local.dir", sb2.toString());

        //unique system dir 
        job.set("mapred.system.dir", job.get("mapred.system.dir") + uniqueSubdir);

        //unique staging dir 
        job.set("mapreduce.jobtracker.staging.root.dir",
                job.get("mapreduce.jobtracker.staging.root.dir") + uniqueSubdir);
    }
}

From source file:com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration.java

License:Open Source License

/**
 * //from ww w .  ja v a 2 s  . co m
 * @param job
 */
public static void setStagingDir(JobConf job) {
    String dir = DMLConfig.LOCAL_MR_MODE_STAGING_DIR + Lop.FILE_SEPARATOR + Lop.PROCESS_PREFIX
            + DMLScript.getUUID() + Lop.FILE_SEPARATOR;
    job.set("mapreduce.jobtracker.staging.root.dir", dir);
}

From source file:com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration.java

License:Open Source License

public static void setProgramBlocks(JobConf job, String sProgramBlocks) {
    job.set(PARFOR_PROGRAMBLOCKS_CONFIG, sProgramBlocks);
}

From source file:com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration.java

License:Open Source License

public static void setPartitioningInfo(JobConf job, long rlen, long clen, int brlen, int bclen, InputInfo ii,
        OutputInfo oi, PDataPartitionFormat dpf, int n, String fnameNew) throws DMLRuntimeException {
    job.set(PARTITIONING_INPUT_MATRIX_NUM_ROW_CONFIG, String.valueOf(rlen));
    job.set(PARTITIONING_INPUT_MATRIX_NUM_COLUMN_CONFIG, String.valueOf(clen));
    job.set(PARTITIONING_INPUT_BLOCK_NUM_ROW_CONFIG, String.valueOf(brlen));
    job.set(PARTITIONING_INPUT_BLOCK_NUM_COLUMN_CONFIG, String.valueOf(bclen));
    job.set(PARTITIONING_INPUT_INFO_CONFIG, InputInfo.inputInfoToString(ii));
    job.set(PARTITIONING_OUTPUT_INFO_CONFIG, OutputInfo.outputInfoToString(oi));
    job.set(PARTITIONING_OUTPUT_FORMAT_CONFIG, dpf.toString());
    job.set(PARTITIONING_OUTPUT_N_CONFIG, String.valueOf(n));
    job.set(PARTITIONING_OUTPUT_FILENAME_CONFIG, fnameNew);
}

From source file:com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration.java

License:Open Source License

public static void setPartitioningInfo(JobConf job, long rlen, long clen, int brlen, int bclen, InputInfo ii,
        OutputInfo oi, PDataPartitionFormat dpf, int n, String fnameNew, String itervar, String matrixvar,
        boolean tSparseCol) throws DMLRuntimeException {
    //set basic partitioning information
    setPartitioningInfo(job, rlen, clen, brlen, bclen, ii, oi, dpf, n, fnameNew);

    //set iteration variable name (used for ParFor-DPE)
    job.set(PARTITIONING_ITERVAR_CONFIG, itervar);

    //set iteration variable name (used for ParFor-DPE)
    job.set(PARTITIONING_MATRIXVAR_CONFIG, matrixvar);

    //set transpose sparse column vector
    job.setBoolean(PARTITIONING_TRANSPOSE_COL_CONFIG, tSparseCol);
}

From source file:com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration.java

License:Open Source License

public static void setPartitioningBlockNumRows(JobConf job, int brlen) {
    job.set(PARTITIONING_INPUT_BLOCK_NUM_ROW_CONFIG, String.valueOf(brlen));
}

From source file:com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration.java

License:Open Source License

public static void setPartitioningBlockNumCols(JobConf job, int bclen) {
    job.set(PARTITIONING_INPUT_BLOCK_NUM_COLUMN_CONFIG, String.valueOf(bclen));
}

From source file:com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration.java

License:Open Source License

public static void setPartitioningFormat(JobConf job, PDataPartitionFormat dpf) {
    job.set(PARTITIONING_OUTPUT_FORMAT_CONFIG, dpf.toString());
}