Example usage for org.apache.hadoop.mapred JobConf JobConf

List of usage examples for org.apache.hadoop.mapred JobConf JobConf

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.mapred.*
  5. JobConf
  6. JobConf

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred JobConf JobConf.

Prototype

public JobConf()

Source Link

Document

Construct a map/reduce job configuration.

Usage

From source file:cascading.RegressionMiscTest.java

License:Open Source License

public void testTupleEntryNextTwice() throws IOException {
    Tap tap = new Hfs(new TextLine(), inputFileNums10);

    TupleEntryIterator iterator = tap.openForRead(new JobConf());

    int count = 0;
    while (iterator.hasNext()) {
        iterator.next();/*  w  w w  .ja  v a 2s .c o m*/
        count++;
    }

    assertFalse(iterator.hasNext());
    assertEquals(10, count);
}

From source file:cascading.SortedValuesTest.java

License:Open Source License

private void validateCase(String test, Boolean[] testCase, Tap sink) throws IOException {
    TupleEntryIterator iterator = sink.openForRead(new JobConf());
    LinkedHashMap<Long, List<String>> group = new LinkedHashMap<Long, List<String>>();

    while (iterator.hasNext()) {
        Tuple tuple = iterator.next().getTuple();

        String[] values = tuple.getString(0).split("\\s");

        long num = Long.parseLong(values[0]);

        if (!group.containsKey(num))
            group.put(num, new ArrayList<String>());

        group.get(num).add(values[2]);/*ww w  .  ja va 2s  . co  m*/
    }

    boolean groupIsReversed = testCase[0];

    if (testCase[2])
        groupIsReversed = !groupIsReversed;

    compare("grouping+" + test, groupIsReversed, group.keySet());

    if (testCase[1] == null)
        return;

    boolean valueIsReversed = testCase[1];

    if (testCase[2])
        valueIsReversed = !valueIsReversed;

    for (Long grouping : group.keySet())
        compare("values+" + test, valueIsReversed, group.get(grouping));
}

From source file:cascading.SortedValuesTest.java

License:Open Source License

private void validateFile(Tap tap, int length, int uniqueValues, boolean isReversed, int comparePosition)
        throws IOException, ParseException {
    TupleEntryIterator iterator = tap.openForRead(new JobConf());

    Set<Long> values = new HashSet<Long>();

    long lastValue = isReversed ? Long.MAX_VALUE : Long.MIN_VALUE;
    int count = 0;

    while (iterator.hasNext()) {
        Tuple tuple = iterator.next().getTuple();
        count++;//from  w ww .jav  a  2  s.c  om

        tuple = new Tuple((Object[]) tuple.getString(1).split("\t"));

        long value = tuple.getLong(comparePosition);

        values.add(value);

        if (isReversed)
            assertTrue("out of order in " + tap, lastValue >= value);
        else
            assertTrue("out of order in " + tap, lastValue <= value);

        lastValue = value;
    }

    if (length != -1)
        assertEquals("length of " + tap, length, count);

    if (uniqueValues != -1)
        assertEquals("unique values of " + tap, uniqueValues, values.size());
}

From source file:cascading.tap.GlobHfs.java

License:Open Source License

@Override
protected Tap[] getTaps() {
    if (taps != null)
        return taps;

    try {/*from   ww w. j a  va  2 s .  c o  m*/
        taps = makeTaps(new JobConf());
    } catch (IOException exception) {
        throw new TapException("unable to resolve taps for globbing path: " + pathPattern);
    }

    return taps;
}

From source file:cascading.tap.hadoop.GlobHfs.java

License:Open Source License

@Override
protected Hfs[] getTaps() {
    return initTapsInternal(new JobConf());
}

From source file:cascading.tap.hadoop.TapCollectorTest.java

License:Open Source License

private void runTest(Tap tap) throws IOException {
    JobConf conf = new JobConf();

    TapCollector collector = (TapCollector) tap.openForWrite(conf); // casting for test

    for (int i = 0; i < 100; i++)
        collector.collect(new Tuple("string", "" + i, i));

    collector.close();//  w  w w . jav a 2s.  co  m

    TupleEntryIterator iterator = tap.openForRead(conf);

    int count = 0;
    while (iterator.hasNext()) {
        iterator.next();
        count++;
    }

    iterator.close();

    assertEquals("wrong size", 100, count);
}

From source file:cascading.tap.hadoop.ZipInputFormatTest.java

License:Open Source License

public void testSplits() throws Exception {
    JobConf job = new JobConf();
    FileSystem currentFs = FileSystem.get(job);

    Path file = new Path(workDir, "test.zip");

    Reporter reporter = Reporter.NULL;//from w  w w  .j a v a  2 s.  c o m

    int seed = new Random().nextInt();
    LOG.info("seed = " + seed);
    Random random = new Random(seed);
    FileInputFormat.setInputPaths(job, file);

    for (int entries = 1; entries < MAX_ENTRIES; entries += random.nextInt(MAX_ENTRIES / 10) + 1) {
        ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
        ZipOutputStream zos = new ZipOutputStream(byteArrayOutputStream);
        long length = 0;

        LOG.debug("creating; zip file with entries = " + entries);

        // for each entry in the zip file
        for (int entryCounter = 0; entryCounter < entries; entryCounter++) {
            // construct zip entries splitting MAX_LENGTH between entries
            long entryLength = MAX_LENGTH / entries;
            ZipEntry zipEntry = new ZipEntry("/entry" + entryCounter + ".txt");
            zipEntry.setMethod(ZipEntry.DEFLATED);
            zos.putNextEntry(zipEntry);

            for (length = entryCounter * entryLength; length < (entryCounter + 1) * entryLength; length++) {
                zos.write(Long.toString(length).getBytes());
                zos.write("\n".getBytes());
            }

            zos.flush();
            zos.closeEntry();
        }

        zos.flush();
        zos.close();

        currentFs.delete(file, true);

        OutputStream outputStream = currentFs.create(file);

        byteArrayOutputStream.writeTo(outputStream);
        outputStream.close();

        ZipInputFormat format = new ZipInputFormat();
        format.configure(job);
        LongWritable key = new LongWritable();
        Text value = new Text();
        InputSplit[] splits = format.getSplits(job, 100);

        BitSet bits = new BitSet((int) length);
        for (int j = 0; j < splits.length; j++) {
            LOG.debug("split[" + j + "]= " + splits[j]);
            RecordReader<LongWritable, Text> reader = format.getRecordReader(splits[j], job, reporter);

            try {
                int count = 0;

                while (reader.next(key, value)) {
                    int v = Integer.parseInt(value.toString());
                    LOG.debug("read " + v);

                    if (bits.get(v))
                        LOG.warn("conflict with " + v + " in split " + j + " at position " + reader.getPos());

                    assertFalse("key in multiple partitions.", bits.get(v));
                    bits.set(v);
                    count++;
                }

                LOG.debug("splits[" + j + "]=" + splits[j] + " count=" + count);
            } finally {
                reader.close();
            }
        }

        assertEquals("some keys in no partition.", length, bits.cardinality());
    }
}

From source file:cascading.tap.hive.HiveTapTest.java

License:Open Source License

@Test
public void testResourceExistsWithNonExistingTable() throws IOException {
    HiveTableDescriptor desc = new HiveTableDescriptor("myTable", new String[] { "key" },
            new String[] { "string" });
    HiveTap tap = new HiveTap(desc, new NullScheme());
    assertFalse(tap.resourceExists(new JobConf()));
}

From source file:cascading.tap.hive.HiveTapTest.java

License:Open Source License

@Test
public void testCreateResource() throws IOException {
    HiveTableDescriptor desc = new HiveTableDescriptor("myTable2", new String[] { "key" },
            new String[] { "string" });
    HiveTap tap = new HiveTap(desc, new NullScheme());
    assertTrue(tap.createResource(new JobConf()));
    assertTrue(tap.resourceExists(new JobConf()));
    assertNotNull(tap.getPath());/*from www .j a  v a  2  s  .  co  m*/
}

From source file:cascading.tap.hive.HiveTapTest.java

License:Open Source License

@Test
public void testCreateResourceInNonExistingDatabase() throws IOException {
    HiveTableDescriptor desc = new HiveTableDescriptor("myDatabase", "myTable2", new String[] { "key" },
            new String[] { "string" });
    HiveTap tap = new HiveTap(desc, new NullScheme());
    assertTrue(tap.createResource(new JobConf()));
    assertTrue(tap.resourceExists(new JobConf()));
    assertNotNull(tap.getPath());//  ww w .  ja  v a  2  s. co  m
}