List of usage examples for org.apache.hadoop.io Text Text
public Text(byte[] utf8)
From source file:com.alectenharmsel.research.MoabLicensesReducerTest.java
@Before public void setUp() throws IOException { reducer = new MoabLicensesReducer(); driver = new ReduceDriver(reducer); key = new Text("cfd_solv_ser-05-11"); vals = new ArrayList<Text>(); vals.add(new Text("0,6")); vals.add(new Text("0,6")); vals.add(new Text("0,6")); vals.add(new Text("0,6")); vals.add(new Text("2,8")); vals.add(new Text("2,8")); vals.add(new Text("1,8")); vals.add(new Text("0,8")); driver.withInput(key, vals);//from ww w .j av a 2 s . c o m res = driver.run(); }
From source file:com.alectenharmsel.research.SrcTokMapper.java
License:Apache License
public void map(LongWritable key, Text contents, Mapper.Context context) throws IOException, InterruptedException { StringBuilder line = new StringBuilder(contents.toString()); for (int i = 0; i < line.length(); i++) { if (!Character.isLetter(line.charAt(i))) { line.replace(i, i + 1, " "); }/*from www. j av a 2s.com*/ } String[] tokens = line.toString().split(" "); for (String s : tokens) { if (s.length() > 0) { context.write(new Text(s), one); } } }
From source file:com.alectenharmsel.research.SrcTokReducer.java
License:Apache License
public void reduce(Text key, Iterable<LongWritable> counts, Context context) throws IOException, InterruptedException { long sum = 0; for (LongWritable tmp : counts) { sum += tmp.get();/*from w w w . j av a2s. c o m*/ } context.write(key, new Text(String.valueOf(sum))); }
From source file:com.alexholmes.hadooputils.combine.avro.AvroFileGenerator.java
License:Apache License
public int run(final String[] args) throws Exception { if (args.length != 2) { System.err.println(/* www.j a v a 2 s . c o m*/ String.format("Usage: %s: <file path> <number of records>", AvroFileGenerator.class.getName())); return 1; } Path file = new Path(args[0]); int numRecords = Integer.valueOf(args[1]); FileSystem fs = FileSystem.get(super.getConf()); SequenceFile.Writer writer = SequenceFile.createWriter(fs, super.getConf(), file, Text.class, Text.class, SequenceFile.CompressionType.BLOCK, new DefaultCodec()); try { for (int i = 0; i < numRecords; i++) { writer.append(new Text("k" + i), new Text("v" + i)); } } finally { writer.close(); } return 0; }
From source file:com.alexholmes.hadooputils.combine.seqfile.SequenceFileGenerator.java
License:Apache License
public int run(final String[] args) throws Exception { if (args.length != 2) { System.err.println(String.format("Usage: %s: <file path> <number of records>", SequenceFileGenerator.class.getName())); return 1; }/* www . j a va 2 s .c o m*/ Path file = new Path(args[0]); int numRecords = Integer.valueOf(args[1]); FileSystem fs = FileSystem.get(super.getConf()); SequenceFile.Writer writer = SequenceFile.createWriter(fs, super.getConf(), file, Text.class, Text.class, SequenceFile.CompressionType.BLOCK, new DefaultCodec()); try { for (int i = 0; i < numRecords; i++) { writer.append(new Text("k" + i), new Text("v" + i)); } } finally { writer.close(); } return 0; }
From source file:com.alexholmes.hadooputils.sort.SortRecordReaderTest.java
License:Apache License
@Test public void testDefaultExtractKey() throws IOException { assertEquals("asd", SortRecordReader.extractKey(new Text("asd"), null, // start key null, // end key " ", // separator false // ignore case ).toString());/*from w ww .j a va 2 s. c om*/ assertEquals("asd def", SortRecordReader.extractKey(new Text("asd def"), null, // start key null, // end key " ", // separator false // ignore case ).toString()); }
From source file:com.alexholmes.hadooputils.sort.SortRecordReaderTest.java
License:Apache License
@Test public void testStartKey() throws IOException { assertEquals("asd", SortRecordReader.extractKey(new Text("asd"), 1, // start key null, // end key " ", // separator false // ignore case ).toString());//from w w w . j ava 2 s . c om assertEquals("asd def", SortRecordReader.extractKey(new Text("asd def"), 1, // start key null, // end key " ", // separator false // ignore case ).toString()); assertEquals("asd def feg", SortRecordReader.extractKey(new Text("asd def feg"), 1, // start key null, // end key " ", // separator false // ignore case ).toString()); assertEquals("def", SortRecordReader.extractKey(new Text("asd def"), 2, // start key null, // end key " ", // separator false // ignore case ).toString()); assertEquals("def feg", SortRecordReader.extractKey(new Text("asd def feg"), 2, // start key null, // end key " ", // separator false // ignore case ).toString()); }
From source file:com.alexholmes.hadooputils.sort.SortRecordReaderTest.java
License:Apache License
@Test(expected = IOException.class) public void testStartKeyOutOfBounds() throws IOException { SortRecordReader.extractKey(new Text("asd"), 2, // start key null, // end key " ", // separator false // ignore case );/*from ww w .j a va 2 s . c o m*/ }
From source file:com.alexholmes.hadooputils.sort.SortRecordReaderTest.java
License:Apache License
@Test public void testStartEndKeys() throws IOException { assertEquals("asd", SortRecordReader.extractKey(new Text("asd"), 1, // start key 1, // end key " ", // separator false // ignore case ).toString());//from ww w. j a va 2 s. com assertEquals("asd def", SortRecordReader.extractKey(new Text("asd def"), 1, // start key 2, // end key " ", // separator false // ignore case ).toString()); assertEquals("asd def feg", SortRecordReader.extractKey(new Text("asd def feg"), 1, // start key 3, // end key " ", // separator false // ignore case ).toString()); assertEquals("def", SortRecordReader.extractKey(new Text("asd def"), 2, // start key 2, // end key " ", // separator false // ignore case ).toString()); assertEquals("def feg", SortRecordReader.extractKey(new Text("asd def feg"), 2, // start key 3, // end key " ", // separator false // ignore case ).toString()); }
From source file:com.alexholmes.hadooputils.sort.SortRecordReaderTest.java
License:Apache License
@Test(expected = IOException.class) public void testEndKeyOutOfBounds() throws IOException { SortRecordReader.extractKey(new Text("asd"), 1, // start key 2, // end key " ", // separator false // ignore case );/*from w ww . j av a 2 s . c o m*/ }