Example usage for org.apache.hadoop.io Text Text

Introduction

In this page you can find the example usage for org.apache.hadoop.io Text Text.

Prototype

public Text(byte[] utf8)

Source Link

Document

Construct from a byte array.

Usage

From source file:com.alectenharmsel.research.MoabLicensesReducerTest.java

@Before
public void setUp() throws IOException {
    reducer = new MoabLicensesReducer();
    driver = new ReduceDriver(reducer);
    key = new Text("cfd_solv_ser-05-11");
    vals = new ArrayList<Text>();
    vals.add(new Text("0,6"));
    vals.add(new Text("0,6"));
    vals.add(new Text("0,6"));
    vals.add(new Text("0,6"));
    vals.add(new Text("2,8"));
    vals.add(new Text("2,8"));
    vals.add(new Text("1,8"));
    vals.add(new Text("0,8"));
    driver.withInput(key, vals);//from  ww w  .j av a  2  s  . c  o  m
    res = driver.run();
}

From source file:com.alectenharmsel.research.SrcTokMapper.java

License:Apache License

public void map(LongWritable key, Text contents, Mapper.Context context)
        throws IOException, InterruptedException {
    StringBuilder line = new StringBuilder(contents.toString());
    for (int i = 0; i < line.length(); i++) {
        if (!Character.isLetter(line.charAt(i))) {
            line.replace(i, i + 1, " ");
        }/*from www.  j av a 2s.com*/
    }
    String[] tokens = line.toString().split(" ");
    for (String s : tokens) {
        if (s.length() > 0) {
            context.write(new Text(s), one);
        }
    }
}

From source file:com.alectenharmsel.research.SrcTokReducer.java

License:Apache License

public void reduce(Text key, Iterable<LongWritable> counts, Context context)
        throws IOException, InterruptedException {
    long sum = 0;
    for (LongWritable tmp : counts) {
        sum += tmp.get();/*from w w w .  j av a2s.  c  o m*/
    }

    context.write(key, new Text(String.valueOf(sum)));
}

From source file:com.alexholmes.hadooputils.combine.avro.AvroFileGenerator.java

License:Apache License

public int run(final String[] args) throws Exception {

    if (args.length != 2) {
        System.err.println(/* www.j a  v a  2 s  .  c  o  m*/
                String.format("Usage: %s: <file path> <number of records>", AvroFileGenerator.class.getName()));
        return 1;
    }

    Path file = new Path(args[0]);
    int numRecords = Integer.valueOf(args[1]);

    FileSystem fs = FileSystem.get(super.getConf());

    SequenceFile.Writer writer = SequenceFile.createWriter(fs, super.getConf(), file, Text.class, Text.class,
            SequenceFile.CompressionType.BLOCK, new DefaultCodec());
    try {
        for (int i = 0; i < numRecords; i++) {
            writer.append(new Text("k" + i), new Text("v" + i));
        }
    } finally {
        writer.close();
    }

    return 0;
}

From source file:com.alexholmes.hadooputils.combine.seqfile.SequenceFileGenerator.java

License:Apache License

public int run(final String[] args) throws Exception {

    if (args.length != 2) {
        System.err.println(String.format("Usage: %s: <file path> <number of records>",
                SequenceFileGenerator.class.getName()));
        return 1;
    }/*  www . j  a va 2 s .c o  m*/

    Path file = new Path(args[0]);
    int numRecords = Integer.valueOf(args[1]);

    FileSystem fs = FileSystem.get(super.getConf());

    SequenceFile.Writer writer = SequenceFile.createWriter(fs, super.getConf(), file, Text.class, Text.class,
            SequenceFile.CompressionType.BLOCK, new DefaultCodec());
    try {
        for (int i = 0; i < numRecords; i++) {
            writer.append(new Text("k" + i), new Text("v" + i));
        }
    } finally {
        writer.close();
    }

    return 0;
}

From source file:com.alexholmes.hadooputils.sort.SortRecordReaderTest.java

License:Apache License

@Test
public void testDefaultExtractKey() throws IOException {
    assertEquals("asd", SortRecordReader.extractKey(new Text("asd"), null, // start key
            null, // end key
            " ", // separator
            false // ignore case
    ).toString());/*from   w ww  .j a va  2  s. c om*/

    assertEquals("asd def", SortRecordReader.extractKey(new Text("asd def"), null, // start key
            null, // end key
            " ", // separator
            false // ignore case
    ).toString());
}

From source file:com.alexholmes.hadooputils.sort.SortRecordReaderTest.java

License:Apache License

@Test
public void testStartKey() throws IOException {
    assertEquals("asd", SortRecordReader.extractKey(new Text("asd"), 1, // start key
            null, // end key
            " ", // separator
            false // ignore case
    ).toString());//from   w w w . j  ava 2  s . c om

    assertEquals("asd def", SortRecordReader.extractKey(new Text("asd def"), 1, // start key
            null, // end key
            " ", // separator
            false // ignore case
    ).toString());

    assertEquals("asd def feg", SortRecordReader.extractKey(new Text("asd def feg"), 1, // start key
            null, // end key
            " ", // separator
            false // ignore case
    ).toString());

    assertEquals("def", SortRecordReader.extractKey(new Text("asd def"), 2, // start key
            null, // end key
            " ", // separator
            false // ignore case
    ).toString());

    assertEquals("def feg", SortRecordReader.extractKey(new Text("asd def feg"), 2, // start key
            null, // end key
            " ", // separator
            false // ignore case
    ).toString());
}

From source file:com.alexholmes.hadooputils.sort.SortRecordReaderTest.java

License:Apache License

@Test(expected = IOException.class)
public void testStartKeyOutOfBounds() throws IOException {
    SortRecordReader.extractKey(new Text("asd"), 2, // start key
            null, // end key
            " ", // separator
            false // ignore case
    );/*from ww w  .j  a va 2 s  .  c  o  m*/
}

From source file:com.alexholmes.hadooputils.sort.SortRecordReaderTest.java

License:Apache License

@Test
public void testStartEndKeys() throws IOException {
    assertEquals("asd", SortRecordReader.extractKey(new Text("asd"), 1, // start key
            1, // end key
            " ", // separator
            false // ignore case
    ).toString());//from   ww w.  j  a  va 2 s.  com

    assertEquals("asd def", SortRecordReader.extractKey(new Text("asd def"), 1, // start key
            2, // end key
            " ", // separator
            false // ignore case
    ).toString());

    assertEquals("asd def feg", SortRecordReader.extractKey(new Text("asd def feg"), 1, // start key
            3, // end key
            " ", // separator
            false // ignore case
    ).toString());

    assertEquals("def", SortRecordReader.extractKey(new Text("asd def"), 2, // start key
            2, // end key
            " ", // separator
            false // ignore case
    ).toString());

    assertEquals("def feg", SortRecordReader.extractKey(new Text("asd def feg"), 2, // start key
            3, // end key
            " ", // separator
            false // ignore case
    ).toString());
}

From source file:com.alexholmes.hadooputils.sort.SortRecordReaderTest.java

License:Apache License

@Test(expected = IOException.class)
public void testEndKeyOutOfBounds() throws IOException {
    SortRecordReader.extractKey(new Text("asd"), 1, // start key
            2, // end key
            " ", // separator
            false // ignore case
    );/*from   w ww  . j av  a 2 s . c  o m*/
}