Example usage for org.apache.hadoop.mapred JobConf set

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred JobConf set.

Prototype

public void set(String name, String value) 

Source Link

Document

Set the value of the name property.

Usage

From source file:infinidb.hadoop.db.InfiniDBConfiguration.java

License:Apache License

/**
 * Sets the DB access related fields in the JobConf.  
 * @param job the job//  w  ww . j  a  v  a 2s.c  o m
 * @param driverClass JDBC Driver class name
 * @param dbUrl JDBC DB access URL. 
 * @param userName DB access username 
 * @param passwd DB access passwd
 */
public static void configureDB(JobConf job, String driverClass, String dbUrl, String userName, String passwd) {

    job.set(DBConfiguration.DRIVER_CLASS_PROPERTY, driverClass);
    job.set(DBConfiguration.URL_PROPERTY, dbUrl);
    if (userName != null)
        job.set(DBConfiguration.USERNAME_PROPERTY, userName);
    if (passwd != null)
        job.set(DBConfiguration.PASSWORD_PROPERTY, passwd);
}

---

From source file:infinidb.hadoop.example.InfiniDoopDriver.java

License:Apache License

public int run(String[] args) throws Exception {
    Configuration conf = new Configuration();
    JobConf jobconf = new JobConf(conf, InfiniDoopDriver.class);
    DBConfiguration.configureDB(jobconf, "com.mysql.jdbc.Driver", "jdbc:mysql://srvswint4/tpch1", "root", "");
    String[] fields = { "n_nationkey", "n_name" };
    jobconf.setInputFormat(InfiniDBInputFormat.class);

    jobconf.setOutputKeyClass(LongWritable.class);
    jobconf.setOutputValueClass(Text.class);

    InfiniDBInputFormat.setInput(jobconf, InfiniDoopRecord.class, "nation", null, "n_nationkey", fields);

    InfiniDBConfiguration idbconf = new InfiniDBConfiguration(jobconf);
    idbconf.setOutputPath("output2");
    jobconf.setMapperClass(InfiniDoopInputMapper.class);
    jobconf.setNumMapTasks(4);//  w ww .  j  a v a2  s  . c o m
    jobconf.setNumReduceTasks(1);
    jobconf.set("mapred.textoutputformat.separator", "|");
    JobClient client = new JobClient();

    client.setConf(jobconf);
    try {
        JobClient.runJob(jobconf);
    } catch (Exception e) {
        e.printStackTrace();
    }

    return 0;
}

---

From source file:io.fabric8.hadoop.mapred.JobTrackerFactory.java

License:Apache License

@Override
protected JobTracker doCreate(Dictionary properties) throws Exception {
    JobConf conf = new JobConf();
    for (Enumeration e = properties.keys(); e.hasMoreElements();) {
        Object key = e.nextElement();
        Object val = properties.get(key);
        conf.set(key.toString(), val.toString());
    }/*  ww w.  j av  a  2s.com*/
    JobTracker jobTracker = JobTracker.startTracker(conf);
    jobTracker.offerService();
    return jobTracker;
}

---

From source file:io.fabric8.hadoop.mapred.TaskTrackerFactory.java

License:Apache License

@Override
protected TaskTracker doCreate(Dictionary properties) throws Exception {
    JobConf conf = new JobConf();
    for (Enumeration e = properties.keys(); e.hasMoreElements();) {
        Object key = e.nextElement();
        Object val = properties.get(key);
        conf.set(key.toString(), val.toString());
    }//  w w w.  j a  v a2s. c om
    TaskTracker taskTracker = new TaskTracker(conf);
    new Daemon(taskTracker).start();
    return taskTracker;
}

---

From source file:io.fluo.mapreduce.FluoFileOutputFormatIT.java

License:Apache License

@Test
public void testImportFile() throws Exception {

    File inDir = new File(tempFolder.getRoot(), "in");
    inDir.mkdir();// w w  w.ja  v a  2s  .c o  m
    File outDir = new File(tempFolder.getRoot(), "out");
    File failDir = new File(tempFolder.getRoot(), "fail");
    failDir.mkdir();

    // generate some data for map reduce to read
    PrintWriter writer = new PrintWriter(new File(inDir, "file1.txt"), "UTF-8");
    writer.println("a,b,c,1");
    writer.println("d,b,c,2");
    writer.println("foo,moo,moo,90");
    writer.close();

    // run map reduce job to generate rfiles
    JobConf jconf = new JobConf();
    jconf.set("mapred.job.tracker", "true");
    jconf.set("fs.defaultFS", "file:///");
    @SuppressWarnings("deprecation")
    Job job = new Job(jconf);
    job.setInputFormatClass(TextInputFormat.class);
    FileInputFormat.setInputPaths(job, inDir.toURI().toString());
    job.setOutputFormatClass(FluoFileOutputFormat.class);
    FileOutputFormat.setOutputPath(job, new Path(outDir.toURI()));
    job.setMapperClass(TestMapper.class);
    job.setNumReduceTasks(0);
    job.waitForCompletion(false);

    // bulk import rfiles
    conn.tableOperations().importDirectory(table, outDir.toString(), failDir.toString(), false);

    // read and update data using transactions
    TestTransaction tx1 = new TestTransaction(env);
    TestTransaction tx2 = new TestTransaction(env);

    Assert.assertEquals("1", tx1.get().row("a").fam("b").qual("c").toString());
    Assert.assertEquals("2", tx1.get().row("d").fam("b").qual("c").toString());
    Assert.assertEquals("90", tx1.get().row("foo").fam("moo").qual("moo").toString());

    tx1.mutate().row("a").fam("b").qual("c").set("3");
    tx1.mutate().row("d").fam("b").qual("c").delete();

    tx1.done();

    // should not see changes from tx1
    Assert.assertEquals("1", tx2.get().row("a").fam("b").qual("c").toString());
    Assert.assertEquals("2", tx2.get().row("d").fam("b").qual("c").toString());
    Assert.assertEquals("90", tx2.get().row("foo").fam("moo").qual("moo").toString());

    TestTransaction tx3 = new TestTransaction(env);

    // should see changes from tx1
    Assert.assertEquals("3", tx3.get().row("a").fam("b").qual("c").toString());
    Assert.assertNull(tx3.get().row("d").fam("b").qual("c").toString());
    Assert.assertEquals("90", tx3.get().row("foo").fam("moo").qual("moo").toString());
}

---

From source file:io.fluo.stress.trie.NumberIngest.java

License:Apache License

private static void loadConfig(JobConf conf, Properties props) {
    for (Entry<Object, Object> entry : props.entrySet()) {
        conf.set((String) entry.getKey(), (String) entry.getValue());
    }//from  www. ja  v  a  2  s  . c  o  m
}

---

From source file:io.prestosql.orc.OrcTester.java

License:Apache License

private static void assertFileContentsOrcHive(Type type, TempFile tempFile, Iterable<?> expectedValues)
        throws Exception {
    JobConf configuration = new JobConf(new Configuration(false));
    configuration.set(READ_COLUMN_IDS_CONF_STR, "0");
    configuration.setBoolean(READ_ALL_COLUMNS, false);

    Reader reader = OrcFile.createReader(new Path(tempFile.getFile().getAbsolutePath()),
            new ReaderOptions(configuration));
    org.apache.hadoop.hive.ql.io.orc.RecordReader recordReader = reader.rows();

    StructObjectInspector rowInspector = (StructObjectInspector) reader.getObjectInspector();
    StructField field = rowInspector.getStructFieldRef("test");

    Iterator<?> iterator = expectedValues.iterator();
    Object rowData = null;/*w w  w. j a va 2  s.c o m*/
    while (recordReader.hasNext()) {
        rowData = recordReader.next(rowData);
        Object expectedValue = iterator.next();

        Object actualValue = rowInspector.getStructFieldData(rowData, field);
        actualValue = decodeRecordReaderValue(type, actualValue);
        assertColumnValueEquals(type, actualValue, expectedValue);
    }
    assertFalse(iterator.hasNext());
}

---

From source file:io.prestosql.orc.OrcTester.java

License:Apache License

private static void assertFileContentsDwrfHive(Type type, TempFile tempFile, Iterable<?> expectedValues)
        throws Exception {
    JobConf configuration = new JobConf(new Configuration(false));
    configuration.set(READ_COLUMN_IDS_CONF_STR, "0");
    configuration.setBoolean(READ_ALL_COLUMNS, false);

    Path path = new Path(tempFile.getFile().getAbsolutePath());
    com.facebook.hive.orc.Reader reader = com.facebook.hive.orc.OrcFile
            .createReader(path.getFileSystem(configuration), path, configuration);
    boolean[] include = new boolean[reader.getTypes().size() + 100000];
    Arrays.fill(include, true);/* w ww.j  a  va 2  s. c  o m*/
    com.facebook.hive.orc.RecordReader recordReader = reader.rows(include);

    StructObjectInspector rowInspector = (StructObjectInspector) reader.getObjectInspector();
    StructField field = rowInspector.getStructFieldRef("test");

    Iterator<?> iterator = expectedValues.iterator();
    Object rowData = null;
    while (recordReader.hasNext()) {
        rowData = recordReader.next(rowData);
        Object expectedValue = iterator.next();

        Object actualValue = rowInspector.getStructFieldData(rowData, field);
        actualValue = decodeRecordReaderValue(type, actualValue);
        assertColumnValueEquals(type, actualValue, expectedValue);
    }
    assertFalse(iterator.hasNext());
}

---

From source file:io.prestosql.plugin.hive.HiveUtil.java

License:Apache License

public static RecordReader<?, ?> createRecordReader(Configuration configuration, Path path, long start,
        long length, Properties schema, List<HiveColumnHandle> columns) {
    // determine which hive columns we will read
    List<HiveColumnHandle> readColumns = ImmutableList
            .copyOf(filter(columns, column -> column.getColumnType() == REGULAR));
    List<Integer> readHiveColumnIndexes = ImmutableList
            .copyOf(transform(readColumns, HiveColumnHandle::getHiveColumnIndex));

    // Tell hive the columns we would like to read, this lets hive optimize reading column oriented files
    setReadColumns(configuration, readHiveColumnIndexes);

    InputFormat<?, ?> inputFormat = getInputFormat(configuration, schema, true);
    JobConf jobConf = toJobConf(configuration);
    FileSplit fileSplit = new FileSplit(path, start, length, (String[]) null);

    // propagate serialization configuration to getRecordReader
    schema.stringPropertyNames().stream().filter(name -> name.startsWith("serialization."))
            .forEach(name -> jobConf.set(name, schema.getProperty(name)));

    // add Airlift LZO and LZOP to head of codecs list so as to not override existing entries
    List<String> codecs = newArrayList(
            Splitter.on(",").trimResults().omitEmptyStrings().split(jobConf.get("io.compression.codecs", "")));
    if (!codecs.contains(LzoCodec.class.getName())) {
        codecs.add(0, LzoCodec.class.getName());
    }//from www .  java2s.  c  o m
    if (!codecs.contains(LzopCodec.class.getName())) {
        codecs.add(0, LzopCodec.class.getName());
    }
    jobConf.set("io.compression.codecs", codecs.stream().collect(joining(",")));

    try {
        RecordReader<WritableComparable, Writable> recordReader = (RecordReader<WritableComparable, Writable>) inputFormat
                .getRecordReader(fileSplit, jobConf, Reporter.NULL);

        int headerCount = getHeaderCount(schema);
        if (headerCount > 0) {
            Utilities.skipHeader(recordReader, headerCount, recordReader.createKey(),
                    recordReader.createValue());
        }

        int footerCount = getFooterCount(schema);
        if (footerCount > 0) {
            recordReader = new FooterAwareRecordReader<>(recordReader, footerCount, jobConf);
        }

        return recordReader;
    } catch (IOException e) {
        if (e instanceof TextLineLengthLimitExceededException) {
            throw new PrestoException(HIVE_BAD_DATA, "Line too long in text file: " + path, e);
        }

        throw new PrestoException(HIVE_CANNOT_OPEN_SPLIT,
                format("Error opening Hive split %s (offset=%s, length=%s) using %s: %s", path, start, length,
                        getInputFormatName(schema), firstNonNull(e.getMessage(), e.getClass().getName())),
                e);
    }
}

---

From source file:io.prestosql.plugin.hive.TestOrcPageSourceMemoryTracking.java

License:Apache License

public static FileSplit createTestFile(String filePath, HiveOutputFormat<?, ?> outputFormat,
        Serializer serializer, String compressionCodec, List<TestColumn> testColumns, int numRows,
        int stripeRows) throws Exception {
    // filter out partition keys, which are not written to the file
    testColumns = ImmutableList.copyOf(filter(testColumns, not(TestColumn::isPartitionKey)));

    Properties tableProperties = new Properties();
    tableProperties.setProperty("columns", Joiner.on(',').join(transform(testColumns, TestColumn::getName)));
    tableProperties.setProperty("columns.types",
            Joiner.on(',').join(transform(testColumns, TestColumn::getType)));
    serializer.initialize(CONFIGURATION, tableProperties);

    JobConf jobConf = new JobConf();
    if (compressionCodec != null) {
        CompressionCodec codec = new CompressionCodecFactory(CONFIGURATION).getCodecByName(compressionCodec);
        jobConf.set(COMPRESS_CODEC, codec.getClass().getName());
        jobConf.set(COMPRESS_TYPE, SequenceFile.CompressionType.BLOCK.toString());
    }//from www  .jav a 2s  .  co m

    RecordWriter recordWriter = createRecordWriter(new Path(filePath), CONFIGURATION);

    try {
        SettableStructObjectInspector objectInspector = getStandardStructObjectInspector(
                ImmutableList.copyOf(transform(testColumns, TestColumn::getName)),
                ImmutableList.copyOf(transform(testColumns, TestColumn::getObjectInspector)));

        Object row = objectInspector.create();

        List<StructField> fields = ImmutableList.copyOf(objectInspector.getAllStructFieldRefs());

        for (int rowNumber = 0; rowNumber < numRows; rowNumber++) {
            for (int i = 0; i < testColumns.size(); i++) {
                Object writeValue = testColumns.get(i).getWriteValue();
                if (writeValue instanceof Slice) {
                    writeValue = ((Slice) writeValue).getBytes();
                }
                objectInspector.setStructFieldData(row, fields.get(i), writeValue);
            }

            Writable record = serializer.serialize(row, objectInspector);
            recordWriter.write(record);
            if (rowNumber % stripeRows == stripeRows - 1) {
                flushStripe(recordWriter);
            }
        }
    } finally {
        recordWriter.close(false);
    }

    Path path = new Path(filePath);
    path.getFileSystem(CONFIGURATION).setVerifyChecksum(true);
    File file = new File(filePath);
    return new FileSplit(path, 0, file.length(), new String[0]);
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »