Example usage for org.apache.hadoop.mapred JobConf set

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred JobConf set.

Prototype

public void set(String name, String value)

Source Link

Document

Set the value of the name property.

Usage

From source file:com.ibm.bi.dml.yarn.DMLAppMasterUtils.java

License:Open Source License

/**
 * /*from  w ww . jav  a2  s .c  o  m*/
 * @param job
 * @param conf
 */
public static void setupMRJobRemoteMaxMemory(JobConf job, DMLConfig conf) {
    if (DMLScript.isActiveAM() && conf.getBooleanValue(DMLConfig.YARN_APPMASTER)) {
        int memMB = -1;

        //obtain the current configuation (optimized or user-specified)
        if (isResourceOptimizerEnabled())
            memMB = (int) (InfrastructureAnalyzer.getRemoteMaxMemoryMap() / (1024 * 1024));
        else
            memMB = conf.getIntValue(DMLConfig.YARN_MAPREDUCEMEM);

        //set the memory configuration into the job conf
        if (memMB > 0) { //ignored if negative
            String memOpts = "-Xmx" + memMB + "m -Xms" + memMB + "m -Xmn" + (int) (memMB / 10) + "m";

            //set mapper heapsizes
            job.set("mapreduce.map.java.opts", memOpts);
            job.set("mapreduce.map.memory.mb", String.valueOf(DMLYarnClient.computeMemoryAllocation(memMB)));

            //set reducer heapsizes
            job.set("mapreduce.reduce.java.opts", memOpts);
            job.set("mapreduce.reduce.memory.mb", String.valueOf(DMLYarnClient.computeMemoryAllocation(memMB)));
        }
    }
}

From source file:com.ibm.jaql.fail.io.ErrorInputConfigurator.java

License:Apache License

@Override
protected void set(JobConf conf) throws Exception {
    // TODO Auto-generated method stub
    super.set(conf);
    conf.set(ErrorInputFormat.ERROR_NAME, err.toString());
    conf.setInt(ErrorInputFormat.ERROR_NEXT_MAX, errorMax);
}

From source file:com.ibm.jaql.fail.io.ErrorOutputConfigurator.java

License:Apache License

@Override
public void setParallel(JobConf conf) throws Exception {
    // TODO Auto-generated method stub
    super.setParallel(conf);
    conf.set(ErrorOutputFormat.ERROR_NAME, err.toString());
    conf.setInt(ErrorOutputFormat.ERROR_NEXT_MAX, errMax);
}

From source file:com.ibm.jaql.fail.io.ErrorOutputConfigurator.java

License:Apache License

@Override
public void setSequential(JobConf conf) throws Exception {
    // TODO Auto-generated method stub
    super.setSequential(conf);
    conf.set(ErrorOutputFormat.ERROR_NAME, err.toString());
    conf.setInt(ErrorOutputFormat.ERROR_NEXT_MAX, errMax);
}

From source file:com.ibm.jaql.io.hadoop.CompositeInputAdapter.java

License:Apache License

/**
 * @param conf/*w  w  w  . j a  va  2s. com*/
 * @throws Exception
 */
protected void set(JobConf conf) throws Exception {
    conf.setInputFormat(this.getClass());

    // write out the input adapter args array
    ConfUtil.writeConfArray(conf, ConfSetter.CONFINOPTIONS_NAME, this.args);
    conf.set(ADD_INDEX_NAME, Boolean.toString(addIndex));
}

From source file:com.ibm.jaql.io.hadoop.CompositeInputAdapter.java

License:Apache License

/**
 * @param job//w  w w.j a va 2  s. com
 * @param idx
 * @throws Exception
 */
private void writeCurrentIndex(JobConf job, int idx) throws Exception {
    job.set(CURRENT_IDX_NAME, String.valueOf(idx));
}

From source file:com.ibm.jaql.io.hadoop.CompositeOutputAdapter.java

License:Apache License

/**
 * Parse the JSON record in confText.//from   w w  w . ja v a  2s. c  o  m
 * Added each field and its value to a new JobConf.
 *  
 * @param conf
 * @param confText
 * @return
 */
private static JobConf restoreConf(JobConf parent, String confText) // TODO: move to general library
{
    try {
        JsonParser parser = new JsonParser(new StringReader(confText));
        JsonRecord jrec = (JsonRecord) parser.JsonVal();
        JobConf conf = new JobConf(parent);
        for (Map.Entry<JsonString, JsonValue> entry : jrec) {
            JsonValue val = entry.getValue();
            conf.set(entry.getKey().toString(), val == null ? null : val.toString());
        }
        return conf;
    } catch (ParseException pe) {
        throw new UndeclaredThrowableException(pe); // IOException(pe);
    }
}

From source file:com.ibm.jaql.io.hadoop.CompositeOutputAdapter.java

License:Apache License

@Override
public void setParallel(JobConf conf) throws Exception {
    set(conf);// www.  j a va  2  s .  co m

    subconfs = new JobConf[outputs.length];
    for (int i = 0; i < outputs.length; i++) {
        subconfs[i] = new JobConf(conf);
        // We need to set the committer because many class rely on this default (instead of our committer)
        subconfs[i].setOutputCommitter(FileOutputCommitter.class);
        outputs[i].setParallel(subconfs[i]);
        JsonRecord confText = saveConf(conf, subconfs[i]);
        conf.set(SUBCONF_NAME + i, confText.toString());
    }
}

From source file:com.ibm.jaql.io.hadoop.ConfUtil.java

License:Apache License

public static void writeConfOptions(JobConf conf, JsonRecord args) {
    JsonRecord extraConf = (JsonRecord) args.get(CONF_FIELD);
    if (extraConf != null) {
        for (Map.Entry<JsonString, JsonValue> f : extraConf) {
            String key = f.getKey().toString();
            JsonValue jvalue = f.getValue();
            String value = (jvalue == null) ? null : jvalue.toString();
            conf.set(key, value);
        }/*from w ww .  j a  va  2s . co  m*/
    }
}

From source file:com.ibm.jaql.io.hbase.TableInputConfigurator.java

License:Apache License

/**
 * @param conf//from  w  w w. j  ava2s  .  c om
 * @throws Exception
 */
protected void set(JobConf conf) throws Exception {
    conf.set(JaqlTableInputFormat.JOB_TABLE, location);
    // set column family
    JsonArray columnNames = null;
    if (options != null) {
        columnNames = (JsonArray) options.get(new JsonString("columns"));
    }
    if (columnNames == null) {
        conf.set(JaqlTableInputFormat.JOB_COLUMNS, HBaseStore.Util.DEFAULT_HBASE_COLUMN_FAMILY_NAME);
    } else {
        JsonIterator colIter = columnNames.iter();
        StringBuilder colList = new StringBuilder();
        boolean first = true;
        for (JsonValue current : colIter) {
            if (!first) {
                first = false;
                colList.append(",");
            }
            JsonString colName = JaqlUtil.enforceNonNull((JsonString) current);
            colList.append(HBaseStore.Util.convertColumn(colName));
        }
        conf.set(JaqlTableInputFormat.JOB_COLUMNS, colList.toString());
    }
    // get the other arguments

    if (options != null) {
        // set timestamp
        JsonLong timestampValue = (JsonLong) options.get(new JsonString("timestamp"));
        if (timestampValue != null) {
            conf.set(JaqlTableInputFormat.JOB_TS, String.valueOf(timestampValue.get()));
        }

        // set start key
        JsonString lowKeyArg = (JsonString) options.get(new JsonString("lowKey"));
        if (lowKeyArg != null) {
            conf.set(JaqlTableInputFormat.JOB_LOWKEY, lowKeyArg.toString());
        }

        // set the end key
        JsonString highKeyArg = (JsonString) options.get(new JsonString("highKey"));
        if (highKeyArg != null) {
            conf.set(JaqlTableInputFormat.JOB_HIGHKEY, highKeyArg.toString());
        }
    }
}