Example usage for org.apache.hadoop.mapred JobConf set

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred JobConf set.

Prototype

public void set(String name, String value)

Source Link

Document

Set the value of the name property.

Usage

From source file:cascading.flow.tez.Hadoop2TezFlowStep.java

License:Open Source License

protected Map<FlowElement, Configuration> initFromSinks(FlowNode flowNode,
        FlowProcess<? extends Configuration> flowProcess, Configuration conf) {
    Set<FlowElement> sinks = flowNode.getSinkElements();
    Map<FlowElement, Configuration> configs = new HashMap<>();

    for (FlowElement element : sinks) {
        JobConf current = new JobConf(conf);

        if (element instanceof Tap) {
            Tap tap = (Tap) element;/*from  w w  w  .  j  av  a  2 s.c o  m*/

            if (tap.getIdentifier() == null)
                throw new IllegalStateException("tap may not have null identifier: " + element.toString());

            tap.sinkConfInit(flowProcess, current);

            setLocalMode(conf, current, tap);
        }

        String id = FlowElements.id(element);

        current.set("cascading.node.sink", id);

        configs.put(element, current);
    }

    return configs;
}

From source file:cascading.hcatalog.HCatScheme.java

License:Apache License

@Override
public void sinkConfInit(FlowProcess<JobConf> flowProcess, Tap<JobConf, RecordReader, OutputCollector> tap,
        JobConf conf) {
    conf.setOutputFormat(outputFormat);//from w  w  w  . j a  v a2 s .  c om
    conf.set(HiveProps.HIVE_COLUMNS, (String) tableMetadata.get(HiveProps.HIVE_COLUMNS));
    conf.set(HiveProps.HIVE_COLUMN_TYPES, (String) tableMetadata.get(HiveProps.HIVE_COLUMN_TYPES));
    createSerDe(conf);
}

From source file:cascading.hive.ORCFile.java

License:Apache License

@Override
public void sourceConfInit(FlowProcess<JobConf> flowProcess, Tap<JobConf, RecordReader, OutputCollector> tap,
        JobConf conf) {
    conf.setInputFormat(OrcInputFormat.class);
    if (selectedColIds != null) {
        conf.set(HiveProps.HIVE_SELECTD_COLUMN_IDS, selectedColIds);
        conf.set(HiveProps.HIVE_READ_ALL_COLUMNS, "false");
    }/*from   ww  w  . j a  v a  2s.c  o m*/
}

From source file:cascading.hive.RCFile.java

License:Apache License

@Override
public void sourceConfInit(FlowProcess<JobConf> flowProcess, Tap<JobConf, RecordReader, OutputCollector> tap,
        JobConf conf) {
    conf.setInputFormat(RCFileInputFormat.class);
    if (selectedColIds != null) {
        conf.set(HiveProps.HIVE_SELECTD_COLUMN_IDS, selectedColIds);
    }/*  w ww .  j a  v a  2s. c  o m*/
}

From source file:cascading.hive.RCFile.java

License:Apache License

@Override
public void sinkConfInit(FlowProcess<JobConf> flowProcess, Tap<JobConf, RecordReader, OutputCollector> tap,
        JobConf conf) {
    conf.setOutputKeyClass(WritableComparable.class);
    conf.setOutputValueClass(BytesRefArrayWritable.class);
    conf.setOutputFormat(RCFileOutputFormat.class);
    conf.set(HiveProps.HIVE_COLUMN_NUMBER, String.valueOf(getSinkFields().size()));
}

From source file:cascading.jdbc.db.DBConfiguration.java

License:Apache License

/**
 * Sets the DB access related fields in the JobConf.
 *
 * @param job         the job/*  www.ja  v  a2s .c o  m*/
 * @param driverClass JDBC Driver class name
 * @param dbUrl       JDBC DB access URL.
 * @param userName    DB access username
 * @param passwd      DB access passwd
 */
public static void configureDB(JobConf job, String driverClass, String dbUrl, String userName, String passwd) {
    job.set(DRIVER_CLASS_PROPERTY, driverClass);
    job.set(URL_PROPERTY, dbUrl);

    if (userName != null)
        job.set(USERNAME_PROPERTY, userName);

    if (passwd != null)
        job.set(PASSWORD_PROPERTY, passwd);
}

From source file:cascading.mongodb.MongoDBConfiguration.java

License:Apache License

public static void configureMongoDB(JobConf jobConf, String database, String collection) {

    if (database != null && !"".equals(database))
        jobConf.set(DATABASE, database);

    if (collection != null && !"".equals(collection))
        jobConf.set(COLLECTION, collection);
}

From source file:cascading.plumber.grids.AbstractGridTest.java

License:Apache License

@Test
public void shouldCopyJobConfIntoProperties() {
    JobConf jobConf = new JobConf();
    jobConf.set(KEY, VALUE);
    new MockGrid().createFlowConnector(jobConf);
}

From source file:cascading.tap.hadoop.Hadoop18TapUtil.java

License:Open Source License

/**
 * should only be called if not in a Flow
 *
 * @param conf//ww  w . ja  v a 2s .co m
 * @throws IOException
 */
public static void setupJob(JobConf conf) throws IOException {
    Path outputPath = FileOutputFormat.getOutputPath(conf);

    if (outputPath == null)
        return;

    if (getFSSafe(conf, outputPath) == null)
        return;

    if (conf.get("mapred.task.id") == null) // need to stuff a fake id
    {
        String mapper = conf.getBoolean("mapred.task.is.map", true) ? "m" : "r";
        conf.set("mapred.task.id", String.format("attempt_%012d_0000_%s_000000_0",
                (int) Math.rint(System.currentTimeMillis()), mapper));
    }

    makeTempPath(conf);

    if (writeDirectlyToWorkingPath(conf, outputPath)) {
        LOG.info("writing directly to output path: " + outputPath);
        setWorkOutputPath(conf, outputPath);
        return;
    }

    // "mapred.work.output.dir"
    Path taskOutputPath = getTaskOutputPath(conf);
    setWorkOutputPath(conf, taskOutputPath);
}

From source file:cascading.tap.hadoop.Hadoop18TapUtil.java

License:Open Source License

static void setWorkOutputPath(JobConf conf, Path outputDir) {
    outputDir = new Path(conf.getWorkingDirectory(), outputDir);
    conf.set("mapred.work.output.dir", outputDir.toString());
}