Example usage for org.apache.hadoop.mapred JobConf JobConf

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred JobConf JobConf.

Prototype

public JobConf() 

Source Link

Document

Construct a map/reduce job configuration.

Usage

From source file:cascading.tap.hive.HiveTapTest.java

License:Open Source License

@Test(expected = HiveTableValidationException.class)
public void testResourceExistsStrictModeColumnCountMismatch() throws IOException {
    HiveTableDescriptor desc = new HiveTableDescriptor("myTable3", new String[] { "key" },
            new String[] { "string" });
    HiveTap tap = new HiveTap(desc, new NullScheme());
    tap.createResource(new JobConf());

    HiveTableDescriptor mismatch = new HiveTableDescriptor("myTable3", new String[] { "key", "value" },
            new String[] { "string", "string" });

    tap = new HiveTap(mismatch, new NullScheme(), SinkMode.REPLACE, true);
    tap.resourceExists(new JobConf());

}

---

From source file:cascading.tap.hive.HiveTapTest.java

License:Open Source License

@Test(expected = HiveTableValidationException.class)
public void testResourceExistsStrictModeNameMismatch() throws IOException {
    HiveTableDescriptor desc = new HiveTableDescriptor("myTable4", new String[] { "key" },
            new String[] { "string" });
    HiveTap tap = new HiveTap(desc, new NullScheme());
    tap.createResource(new JobConf());

    HiveTableDescriptor mismatch = new HiveTableDescriptor("myTable4", new String[] { "key2" },
            new String[] { "string" });

    tap = new HiveTap(mismatch, new NullScheme(), SinkMode.REPLACE, true);
    tap.resourceExists(new JobConf());
}

---

From source file:cascading.tap.hive.HiveTapTest.java

License:Open Source License

@Test(expected = HiveTableValidationException.class)
public void testResourceExistsStrictModeTypeMismatch() throws IOException {
    HiveTableDescriptor desc = new HiveTableDescriptor("myTable5", new String[] { "key" },
            new String[] { "string" });
    HiveTap tap = new HiveTap(desc, new NullScheme());
    tap.createResource(new JobConf());

    HiveTableDescriptor mismatch = new HiveTableDescriptor("myTable5", new String[] { "key" },
            new String[] { "int" });
    tap = new HiveTap(mismatch, new NullScheme(), SinkMode.REPLACE, true);
    tap.resourceExists(new JobConf());
}

---

From source file:cascading.tap.hive.HiveTapTest.java

License:Open Source License

@Test
public void testResourceExistsStrictModeCaseInsensitivity() throws IOException {
    HiveTableDescriptor desc = new HiveTableDescriptor("myTable4", new String[] { "key" },
            new String[] { "string" });
    HiveTap tap = new HiveTap(desc, new NullScheme());
    tap.createResource(new JobConf());

    HiveTableDescriptor mismatch = new HiveTableDescriptor("MYTABLE4", new String[] { "KeY" },
            new String[] { "StRinG" });

    tap = new HiveTap(mismatch, new NullScheme(), SinkMode.REPLACE, true);
    assertTrue(tap.resourceExists(new JobConf()));
}

---

From source file:cascading.tap.hive.HiveTapTest.java

License:Open Source License

@Test
public void testResourceExistsStrictModeWithPartitionedTable() throws IOException {
    HiveTableDescriptor desc = new HiveTableDescriptor("myTable8", new String[] { "name", "id" },
            new String[] { "string", "string" }, new String[] { "id" });
    HiveTap tap = new HiveTap(desc, new NullScheme());
    tap.createResource(new JobConf());

    HiveTableDescriptor mismatch = new HiveTableDescriptor("MYTABLE8", new String[] { "NAME", "ID" },
            new String[] { "StRinG", "string" }, new String[] { "ID" });

    tap = new HiveTap(mismatch, new NullScheme(), SinkMode.REPLACE, true);
    assertTrue(tap.resourceExists(new JobConf()));
}

---

From source file:cascading.tap.hive.HiveTapTest.java

License:Open Source License

@Test(expected = HiveTableValidationException.class)
public void testResourceExistsStrictModeLocationMismatch() throws IOException {
    HiveTableDescriptor desc = new HiveTableDescriptor(HiveTableDescriptor.HIVE_DEFAULT_DATABASE_NAME,
            "mytable9", new String[] { "one", "two", "three" }, new String[] { "int", "string", "boolean" },
            new String[] {}, ",", HiveTableDescriptor.HIVE_DEFAULT_SERIALIZATION_LIB_NAME,
            new Path(HIVE_WAREHOUSE_DIR + "/custompath"));
    HiveTap tap = new HiveTap(desc, new NullScheme());
    tap.createResource(new JobConf());

    HiveTableDescriptor mismatch = new HiveTableDescriptor(HiveTableDescriptor.HIVE_DEFAULT_DATABASE_NAME,
            "mytable9", new String[] { "one", "two", "three" }, new String[] { "int", "string", "boolean" },
            new String[] {}, ",");
    tap = new HiveTap(mismatch, new NullScheme(), SinkMode.REPLACE, true);
    tap.resourceExists(new JobConf());
}

---

From source file:cascading.tap.hive.HiveTapTest.java

License:Open Source License

@Test
public void testDeleteRessource() throws Exception {
    HiveTableDescriptor desc = new HiveTableDescriptor("myTable5", new String[] { "key" },
            new String[] { "string" });
    HiveTap tap = new HiveTap(desc, new NullScheme());

    JobConf conf = new JobConf();

    tap.createResource(conf);/* w  ww.java  2 s .com*/
    assertTrue(tap.resourceExists(conf));
    assertTableExists(desc);

    tap.deleteResource(conf);
    assertFalse(tap.resourceExists(conf));
}

---

From source file:cascading.tap.hive.HiveTapTest.java

License:Open Source License

@Test
public void testRegisterPartition() throws Exception {
    HiveTableDescriptor desc = new HiveTableDescriptor("myTable6", new String[] { "one", "two" },
            new String[] { "string", "string" }, new String[] { "two" });
    HiveTap tap = new HiveTap(desc, new NullScheme());
    JobConf conf = new JobConf();
    int now = (int) (System.currentTimeMillis() / 1000);
    Partition part = new Partition(Arrays.asList("2"), desc.getDatabaseName(), desc.getTableName(), now, now,
            desc.toHiveTable().getSd(), new HashMap<String, String>());

    tap.registerPartition(conf, part);/*from w ww .ja  v  a2  s  . co m*/

    assertTableExists(desc);

    IMetaStoreClient client = createMetaStoreClient();
    Partition result = client.getPartition(desc.getDatabaseName(), desc.getTableName(), Arrays.asList("2"));
    assertNotNull(result);
    client.close();

}

---

From source file:cascading.tap.TapTest.java

License:Open Source License

public void testMultiSourceIterator() throws Exception {
    if (!new File(inputFileLower).exists())
        fail("data file not found");

    copyFromLocal(inputFileLower);// w ww  .  j  a v a2 s.  c o  m
    copyFromLocal(inputFileUpper);

    Tap sourceLower = new Hfs(new TextLine(new Fields("offset", "line")), inputFileLower);
    Tap sourceUpper = new Hfs(new TextLine(new Fields("offset", "line")), inputFileUpper);

    Tap source = new MultiSourceTap(sourceLower, sourceUpper);

    validateLength(source.openForRead(new JobConf()), 10, null);

    GlobHfs source1 = new GlobHfs(new TextLine(new Fields("offset", "line")), "build/test/data/?{ppe[_r]}.txt");
    GlobHfs source2 = new GlobHfs(new TextLine(new Fields("offset", "line")), "build/test/data/?{owe?}.txt");

    source = new MultiSourceTap(source1, source2);

    validateLength(source.openForRead(new JobConf()), 10, null);

    GlobHfs sourceMulti = new GlobHfs(new TextLine(new Fields("offset", "line")),
            "build/test/data/?{ppe[_r],owe?}.txt");

    source = new MultiSourceTap(sourceMulti);

    validateLength(source.openForRead(new JobConf()), 10, null);
}

---

From source file:cascading.tuple.hadoop.HadoopSerializationTest.java

License:Open Source License

public void testInputOutputSerialization() throws IOException {
    long time = System.currentTimeMillis();

    JobConf jobConf = new JobConf();

    jobConf.set("io.serializations",
            TestSerialization.class.getName() + "," + WritableSerialization.class.getName()); // disable/replace WritableSerialization class
    jobConf.set("cascading.serialization.tokens",
            "1000=" + BooleanWritable.class.getName() + ",10001=" + Text.class.getName()); // not using Text, just testing parsing

    TupleSerialization tupleSerialization = new TupleSerialization(jobConf);

    File file = new File(outputPath);

    file.mkdirs();//from   ww  w.j a va2s.c  o m
    file = new File(file, "/test.bytes");

    TupleOutputStream output = new TupleOutputStream(new FileOutputStream(file, false),
            tupleSerialization.getElementWriter());

    for (int i = 0; i < 501; i++) // 501 is arbitrary
    {
        String aString = "string number " + i;
        double random = Math.random();

        output.writeTuple(new Tuple(i, aString, random, new TestText(aString),
                new Tuple("inner tuple", new BytesWritable("some string".getBytes())),
                new BytesWritable(Integer.toString(i).getBytes("UTF-8")), new BooleanWritable(false)));
    }

    output.close();

    assertEquals("wrong size", 89967L, file.length()); // just makes sure the file size doesnt change from expected

    TupleInputStream input = new TupleInputStream(new FileInputStream(file),
            tupleSerialization.getElementReader());

    int k = -1;
    for (int i = 0; i < 501; i++) {
        Tuple tuple = input.readTuple();
        int value = tuple.getInteger(0);
        assertTrue("wrong diff", value - k == 1);
        assertTrue("wrong type", tuple.get(3) instanceof TestText);
        assertTrue("wrong type", tuple.get(4) instanceof Tuple);
        assertTrue("wrong type", tuple.get(5) instanceof BytesWritable);

        byte[] bytes = ((BytesWritable) tuple.get(5)).getBytes();
        String string = new String(bytes, 0, bytes.length > 1 ? bytes.length - 1 : bytes.length, "UTF-8");
        assertEquals("wrong value", Integer.parseInt(string), i);
        assertTrue("wrong type", tuple.get(6) instanceof BooleanWritable);
        k = value;
    }

    input.close();

    System.out.println("time = " + (System.currentTimeMillis() - time));
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• »