Example usage for org.apache.hadoop.io Text Text

List of usage examples for org.apache.hadoop.io Text Text

Introduction

In this page you can find the example usage for org.apache.hadoop.io Text Text.

Prototype

public Text(byte[] utf8) 

Source Link

Document

Construct from a byte array.

Usage

From source file:com.alectenharmsel.research.MoabLicensesReducerTest.java

@Before
public void setUp() throws IOException {
    reducer = new MoabLicensesReducer();
    driver = new ReduceDriver(reducer);
    key = new Text("cfd_solv_ser-05-11");
    vals = new ArrayList<Text>();
    vals.add(new Text("0,6"));
    vals.add(new Text("0,6"));
    vals.add(new Text("0,6"));
    vals.add(new Text("0,6"));
    vals.add(new Text("2,8"));
    vals.add(new Text("2,8"));
    vals.add(new Text("1,8"));
    vals.add(new Text("0,8"));
    driver.withInput(key, vals);//from  ww w  .j av a  2  s  . c  o  m
    res = driver.run();
}

From source file:com.alectenharmsel.research.SrcTokMapper.java

License:Apache License

public void map(LongWritable key, Text contents, Mapper.Context context)
        throws IOException, InterruptedException {
    StringBuilder line = new StringBuilder(contents.toString());
    for (int i = 0; i < line.length(); i++) {
        if (!Character.isLetter(line.charAt(i))) {
            line.replace(i, i + 1, " ");
        }/*from www.  j av a 2s.com*/
    }
    String[] tokens = line.toString().split(" ");
    for (String s : tokens) {
        if (s.length() > 0) {
            context.write(new Text(s), one);
        }
    }
}

From source file:com.alectenharmsel.research.SrcTokReducer.java

License:Apache License

public void reduce(Text key, Iterable<LongWritable> counts, Context context)
        throws IOException, InterruptedException {
    long sum = 0;
    for (LongWritable tmp : counts) {
        sum += tmp.get();/*from w w w .  j av a2s.  c  o m*/
    }

    context.write(key, new Text(String.valueOf(sum)));
}

From source file:com.alexholmes.hadooputils.combine.avro.AvroFileGenerator.java

License:Apache License

public int run(final String[] args) throws Exception {

    if (args.length != 2) {
        System.err.println(/* www.j a  v a  2 s  .  c  o  m*/
                String.format("Usage: %s: <file path> <number of records>", AvroFileGenerator.class.getName()));
        return 1;
    }

    Path file = new Path(args[0]);
    int numRecords = Integer.valueOf(args[1]);

    FileSystem fs = FileSystem.get(super.getConf());

    SequenceFile.Writer writer = SequenceFile.createWriter(fs, super.getConf(), file, Text.class, Text.class,
            SequenceFile.CompressionType.BLOCK, new DefaultCodec());
    try {
        for (int i = 0; i < numRecords; i++) {
            writer.append(new Text("k" + i), new Text("v" + i));
        }
    } finally {
        writer.close();
    }

    return 0;
}

From source file:com.alexholmes.hadooputils.combine.seqfile.SequenceFileGenerator.java

License:Apache License

public int run(final String[] args) throws Exception {

    if (args.length != 2) {
        System.err.println(String.format("Usage: %s: <file path> <number of records>",
                SequenceFileGenerator.class.getName()));
        return 1;
    }/*  www . j  a va 2 s .c o  m*/

    Path file = new Path(args[0]);
    int numRecords = Integer.valueOf(args[1]);

    FileSystem fs = FileSystem.get(super.getConf());

    SequenceFile.Writer writer = SequenceFile.createWriter(fs, super.getConf(), file, Text.class, Text.class,
            SequenceFile.CompressionType.BLOCK, new DefaultCodec());
    try {
        for (int i = 0; i < numRecords; i++) {
            writer.append(new Text("k" + i), new Text("v" + i));
        }
    } finally {
        writer.close();
    }

    return 0;
}

From source file:com.alexholmes.hadooputils.sort.SortRecordReaderTest.java

License:Apache License

@Test
public void testDefaultExtractKey() throws IOException {
    assertEquals("asd", SortRecordReader.extractKey(new Text("asd"), null, // start key
            null, // end key
            " ", // separator
            false // ignore case
    ).toString());/*from   w ww  .j a va  2  s. c om*/

    assertEquals("asd def", SortRecordReader.extractKey(new Text("asd def"), null, // start key
            null, // end key
            " ", // separator
            false // ignore case
    ).toString());
}

From source file:com.alexholmes.hadooputils.sort.SortRecordReaderTest.java

License:Apache License

@Test
public void testStartKey() throws IOException {
    assertEquals("asd", SortRecordReader.extractKey(new Text("asd"), 1, // start key
            null, // end key
            " ", // separator
            false // ignore case
    ).toString());//from   w w w . j  ava 2  s . c om

    assertEquals("asd def", SortRecordReader.extractKey(new Text("asd def"), 1, // start key
            null, // end key
            " ", // separator
            false // ignore case
    ).toString());

    assertEquals("asd def feg", SortRecordReader.extractKey(new Text("asd def feg"), 1, // start key
            null, // end key
            " ", // separator
            false // ignore case
    ).toString());

    assertEquals("def", SortRecordReader.extractKey(new Text("asd def"), 2, // start key
            null, // end key
            " ", // separator
            false // ignore case
    ).toString());

    assertEquals("def feg", SortRecordReader.extractKey(new Text("asd def feg"), 2, // start key
            null, // end key
            " ", // separator
            false // ignore case
    ).toString());
}

From source file:com.alexholmes.hadooputils.sort.SortRecordReaderTest.java

License:Apache License

@Test(expected = IOException.class)
public void testStartKeyOutOfBounds() throws IOException {
    SortRecordReader.extractKey(new Text("asd"), 2, // start key
            null, // end key
            " ", // separator
            false // ignore case
    );/*from ww w  .j  a va 2 s  .  c  o  m*/
}

From source file:com.alexholmes.hadooputils.sort.SortRecordReaderTest.java

License:Apache License

@Test
public void testStartEndKeys() throws IOException {
    assertEquals("asd", SortRecordReader.extractKey(new Text("asd"), 1, // start key
            1, // end key
            " ", // separator
            false // ignore case
    ).toString());//from   ww w.  j  a  va 2 s.  com

    assertEquals("asd def", SortRecordReader.extractKey(new Text("asd def"), 1, // start key
            2, // end key
            " ", // separator
            false // ignore case
    ).toString());

    assertEquals("asd def feg", SortRecordReader.extractKey(new Text("asd def feg"), 1, // start key
            3, // end key
            " ", // separator
            false // ignore case
    ).toString());

    assertEquals("def", SortRecordReader.extractKey(new Text("asd def"), 2, // start key
            2, // end key
            " ", // separator
            false // ignore case
    ).toString());

    assertEquals("def feg", SortRecordReader.extractKey(new Text("asd def feg"), 2, // start key
            3, // end key
            " ", // separator
            false // ignore case
    ).toString());
}

From source file:com.alexholmes.hadooputils.sort.SortRecordReaderTest.java

License:Apache License

@Test(expected = IOException.class)
public void testEndKeyOutOfBounds() throws IOException {
    SortRecordReader.extractKey(new Text("asd"), 1, // start key
            2, // end key
            " ", // separator
            false // ignore case
    );/*from   w ww  . j av  a 2 s . c  o m*/
}