Example usage for org.apache.hadoop.mapred JobConf set

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred JobConf set.

Prototype

public void set(String name, String value) 

Source Link

Document

Set the value of the name property.

Usage

From source file:com.cloudera.lib.service.hadoop.TestHadoopService.java

License:Open Source License

@Test(expectedExceptions = HadoopException.class, expectedExceptionsMessageRegExp = "H06.*")
@TestDir/*from  w w w .j  a  va 2 s .c  o  m*/
@TestHadoop
public void fileSystemExecutorNoNameNode() throws Exception {
    String dir = getTestDir().getAbsolutePath();
    String services = StringUtils.toString(
            Arrays.asList(InstrumentationService.class.getName(), HadoopService.class.getName()), ",");
    XConfiguration conf = new XConfiguration();
    conf.set("server.services", services);
    Server server = new Server("server", dir, dir, dir, dir, conf);
    server.init();
    Hadoop hadoop = server.get(Hadoop.class);

    JobConf jobConf = getHadoopConf();
    jobConf.set("fs.default.name", "");
    hadoop.execute("u", jobConf, new Hadoop.FileSystemExecutor<Void>() {
        @Override
        public Void execute(FileSystem fs) throws IOException {
            return null;
        }
    });
}

---

From source file:com.cloudera.lib.service.hadoop.TestHadoopService.java

License:Open Source License

@Test(expectedExceptions = HadoopException.class, expectedExceptionsMessageRegExp = "H06.*")
@TestDir//ww w  .  j  a  v  a 2s. c  om
@TestHadoop
public void jobClientExecutorNoNameNode() throws Exception {
    String dir = getTestDir().getAbsolutePath();
    String services = StringUtils.toString(
            Arrays.asList(InstrumentationService.class.getName(), HadoopService.class.getName()), ",");
    XConfiguration conf = new XConfiguration();
    conf.set("server.services", services);
    Server server = new Server("server", dir, dir, dir, dir, conf);
    server.init();
    Hadoop hadoop = server.get(Hadoop.class);

    JobConf jobConf = getHadoopConf();
    jobConf.set("fs.default.name", "");
    hadoop.execute("u", jobConf, new Hadoop.JobClientExecutor<Void>() {
        @Override
        public Void execute(JobClient jobClient, FileSystem fs) throws IOException {
            return null;
        }
    });
}

---

From source file:com.cloudera.lib.service.hadoop.TestHadoopService.java

License:Open Source License

@Test(expectedExceptions = HadoopException.class, expectedExceptionsMessageRegExp = "H06.*")
@TestDir//from  w  w  w  .java 2  s . c om
@TestHadoop
public void jobClientExecutorNoJobTracker() throws Exception {
    String dir = getTestDir().getAbsolutePath();
    String services = StringUtils.toString(
            Arrays.asList(InstrumentationService.class.getName(), HadoopService.class.getName()), ",");
    XConfiguration conf = new XConfiguration();
    conf.set("server.services", services);
    Server server = new Server("server", dir, dir, dir, dir, conf);
    server.init();
    Hadoop hadoop = server.get(Hadoop.class);

    JobConf jobConf = getHadoopConf();
    jobConf.set("mapred.job.tracker", "");
    hadoop.execute("u", jobConf, new Hadoop.JobClientExecutor<Void>() {
        @Override
        public Void execute(JobClient jobClient, FileSystem fs) throws IOException {
            return null;
        }
    });
}

---

From source file:com.cloudera.recordservice.hive.RecordServiceHiveInputFormat.java

License:Apache License

private void addSplitsForGroup(List<Path> dirs, TableScanOperator tableScan, JobConf conf,
        InputFormat inputFormat, Class<? extends InputFormat> inputFormatClass, int splits, TableDesc table,
        List<InputSplit> result) throws IOException {
    Utilities.copyTableJobPropertiesToConf(table, conf);

    // The table and database name to scan.
    // TODO: This is commented out until we have a pluggable way to configure the
    // SerDe. Until then, the create a separate table and set the job conf properties.
    // String fqTblName[] = table.getTableName().split("\\.");
    ///*w w  w.j av a 2 s . c o m*/
    // conf.set("recordservice.table.name", table.getTableName());

    if (tableScan != null) {
        pushFilters(conf, tableScan);
        // Set the projected column and table info for the RecordServiceRecordReader.
        conf.set("recordservice.col.names", Joiner.on(",").join(tableScan.getNeededColumns()));
    }
    // Unset the file config. We're going to be just reading from the table.
    conf.unset(org.apache.hadoop.mapreduce.lib.input.FileInputFormat.INPUT_DIR);
    conf.setInputFormat(inputFormat.getClass());

    // Generate the RecordService
    InputSplit[] iss = inputFormat.getSplits(conf, splits);
    for (InputSplit is : iss) {
        if (is instanceof FileSplit) {
            FileSplit fileSplit = (FileSplit) is;
            LOG.info("INPUT SPLIT: " + fileSplit.getPath().toString());
        }
    }

    // Wrap the InputSplits in HiveInputSplits. We use modified version of the
    // HiveInputSplit to work around some issues with the base one.
    // TODO: Get changes incorporated into Hive.
    for (InputSplit is : iss) {
        result.add(new HiveInputSplitShim(dirs.get(0), is, inputFormatClass.getName()));
    }
}

---

From source file:com.cloudera.recordservice.tests.MiniClusterController.java

License:Apache License

/**
 * This method returns a JobConf object that allows a map reduce job to be run
 * on the minicluster/*  ww  w .  j a v a 2 s  .c om*/
 */
public JobConf getJobConf(Class<?> mrClass) {
    if (clusterList_.size() == 0) {
        System.err.println("Cannot run MR job because the cluster has no active nodes");
        return null;
    }
    JobConf conf = new JobConf(mrClass);
    conf.set(RecordServiceConfig.ConfVars.PLANNER_HOSTPORTS_CONF.name,
            "localhost:" + getRandomNode().plannerPort_);
    return conf;
}

---

From source file:com.cloudera.recordservice.tests.MiniClusterController.java

License:Apache License

/**
 * This method takes JobConf and executes it
 *///from ww  w  .j  a v  a  2s .  c  o m
public RunningJob runJob(JobConf mrJob) throws IOException {
    if (clusterList_.size() == 0) {
        System.err.println("Cannot run MR job because the cluster has no active nodes");
        return null;
    }
    mrJob.set(RecordServiceConfig.ConfVars.PLANNER_HOSTPORTS_CONF.name,
            "localhost:" + getRandomNode().plannerPort_);
    System.out.println("Running Job");
    return JobClient.runJob(mrJob);
}

---

From source file:com.cloudera.sqoop.orm.TestParseMethods.java

License:Apache License

public void runParseTest(String fieldTerminator, String lineTerminator, String encloser, String escape,
        boolean encloseRequired) throws IOException {

    ClassLoader prevClassLoader = null;

    String[] argv = getArgv(true, fieldTerminator, lineTerminator, encloser, escape, encloseRequired);
    runImport(argv);//w  ww  . ja v  a2  s .com
    try {
        String tableClassName = getTableName();

        argv = getArgv(false, fieldTerminator, lineTerminator, encloser, escape, encloseRequired);
        SqoopOptions opts = new ImportTool().parseArguments(argv, null, null, true);

        CompilationManager compileMgr = new CompilationManager(opts);
        String jarFileName = compileMgr.getJarFilename();

        // Make sure the user's class is loaded into our address space.
        prevClassLoader = ClassLoaderStack.addJarFile(jarFileName, tableClassName);

        JobConf job = new JobConf();
        job.setJar(jarFileName);

        // Tell the job what class we're testing.
        job.set(ReparseMapper.USER_TYPE_NAME_KEY, tableClassName);

        // use local mode in the same JVM.
        ConfigurationHelper.setJobtrackerAddr(job, "local");
        if (!BaseSqoopTestCase.isOnPhysicalCluster()) {
            job.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
        }
        String warehouseDir = getWarehouseDir();
        Path warehousePath = new Path(warehouseDir);
        Path inputPath = new Path(warehousePath, getTableName());
        Path outputPath = new Path(warehousePath, getTableName() + "-out");

        job.setMapperClass(ReparseMapper.class);
        job.setNumReduceTasks(0);
        FileInputFormat.addInputPath(job, inputPath);
        FileOutputFormat.setOutputPath(job, outputPath);

        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(NullWritable.class);

        JobClient.runJob(job);
    } catch (InvalidOptionsException ioe) {
        fail(ioe.toString());
    } catch (ParseException pe) {
        fail(pe.toString());
    } finally {
        if (null != prevClassLoader) {
            ClassLoaderStack.setCurrentClassLoader(prevClassLoader);
        }
    }
}

---

From source file:com.cloudera.sqoop.orm.TestParseMethods.java

License:Apache License

public void testFieldSetter() throws IOException {
    ClassLoader prevClassLoader = null;

    String[] types = { "VARCHAR(32)", "VARCHAR(32)" };
    String[] vals = { "'meep'", "'foo'" };
    createTableWithColTypes(types, vals);

    String[] argv = getArgv(true, ",", "\\n", "\\\'", "\\", false);
    runImport(argv);//from  w w  w.j  av  a  2  s .  c o m
    try {
        String tableClassName = getTableName();

        argv = getArgv(false, ",", "\\n", "\\\'", "\\", false);
        SqoopOptions opts = new ImportTool().parseArguments(argv, null, null, true);

        CompilationManager compileMgr = new CompilationManager(opts);
        String jarFileName = compileMgr.getJarFilename();

        // Make sure the user's class is loaded into our address space.
        prevClassLoader = ClassLoaderStack.addJarFile(jarFileName, tableClassName);

        JobConf job = new JobConf();
        job.setJar(jarFileName);

        // Tell the job what class we're testing.
        job.set(ExplicitSetMapper.USER_TYPE_NAME_KEY, tableClassName);
        job.set(ExplicitSetMapper.SET_COL_KEY, BASE_COL_NAME + "0");
        job.set(ExplicitSetMapper.SET_VAL_KEY, "this-is-a-test");

        // use local mode in the same JVM.
        ConfigurationHelper.setJobtrackerAddr(job, "local");
        if (!BaseSqoopTestCase.isOnPhysicalCluster()) {
            job.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
        }
        String warehouseDir = getWarehouseDir();
        Path warehousePath = new Path(warehouseDir);
        Path inputPath = new Path(warehousePath, getTableName());
        Path outputPath = new Path(warehousePath, getTableName() + "-out");

        job.setMapperClass(ExplicitSetMapper.class);
        job.setNumReduceTasks(0);
        FileInputFormat.addInputPath(job, inputPath);
        FileOutputFormat.setOutputPath(job, outputPath);

        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(NullWritable.class);

        JobClient.runJob(job);
    } catch (InvalidOptionsException ioe) {
        fail(ioe.toString());
    } catch (ParseException pe) {
        fail(pe.toString());
    } finally {
        if (null != prevClassLoader) {
            ClassLoaderStack.setCurrentClassLoader(prevClassLoader);
        }
    }
}

---

From source file:com.dataartisans.flink.cascading.planner.FlinkFlowStep.java

License:Apache License

private DataSet<Tuple> translateSource(FlowProcess flowProcess, ExecutionEnvironment env, FlowNode node,
        int dop) {

    Tap tap = this.getSingle(node.getSourceTaps());
    JobConf tapConfig = new JobConf(this.getNodeConfig(node));
    tap.sourceConfInit(flowProcess, tapConfig);
    tapConfig.set("cascading.step.source", Tap.id(tap));

    Fields outFields = tap.getSourceFields();
    registerKryoTypes(outFields);/*from  w w  w .j  a va  2 s.c om*/

    JobConf sourceConfig = new JobConf(this.getNodeConfig(node));
    MultiInputFormat.addInputFormat(sourceConfig, tapConfig);

    DataSet<Tuple> src = env.createInput(new TapInputFormat(node), new TupleTypeInfo(outFields))
            .name(tap.getIdentifier()).setParallelism(dop)
            .withParameters(FlinkConfigConverter.toFlinkConfig(new Configuration(sourceConfig)));

    return src;

}

---

From source file:com.dataartisans.flink.cascading.runtime.util.FlinkFlowProcess.java

License:Apache License

@Override
public TupleEntryCollector openTrapForWrite(Tap trap) throws IOException {

    if (trap instanceof Hfs) {

        JobConf jobConf = new JobConf(this.getConfigCopy());

        int stepNum = jobConf.getInt("cascading.flow.step.num", 0);
        int nodeNum = jobConf.getInt("cascading.flow.node.num", 0);

        String partname = String.format("-%05d-%05d-%05d", stepNum, nodeNum, this.getCurrentSliceNum());
        jobConf.set("cascading.tapcollector.partname", "%s%spart" + partname);

        String value = String.format("attempt_%012d_0000_m_%06d_0", (int) Math.rint(System.currentTimeMillis()),
                this.getCurrentSliceNum());
        jobConf.set("mapred.task.id", value);
        jobConf.set("mapreduce.task.id", value);

        return trap.openForWrite(new FlinkFlowProcess(jobConf), null);
    } else {//from   ww  w . jav  a 2s  . c o  m
        throw new UnsupportedOperationException("Only Hfs taps are supported as traps");
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »