Example usage for org.apache.hadoop.io Text Text

Introduction

In this page you can find the example usage for org.apache.hadoop.io Text Text.

Prototype

public Text(byte[] utf8)

Source Link

Document

Construct from a byte array.

Usage

From source file:co.nubetech.hiho.dedup.TestDedupKeyReducer.java

License:Apache License

@Test
public void testReducerForNullValues() throws IOException, InterruptedException {
    Text key = new Text("key123");
    HihoTuple hihoTuple = new HihoTuple();
    hihoTuple.setKey(key);//from w ww.  j  a v a  2 s  .c  o  m

    String value1 = null;
    ArrayList<String> values = new ArrayList<String>();
    values.add(value1);

    Reducer.Context context = mock(Reducer.Context.class);
    Counters counters = new Counters();
    Counter counter = counters.findCounter(DedupRecordCounter.OUTPUT);
    when(context.getCounter(DedupRecordCounter.OUTPUT)).thenReturn(counter);
    DedupKeyReducer dedupReducer = new DedupKeyReducer();
    dedupReducer.reduce(hihoTuple, values, context);
    verify(context).write(key, value1);
    assertEquals(1, context.getCounter(DedupRecordCounter.OUTPUT).getValue());
}

From source file:co.nubetech.hiho.dedup.TestDedupValueMapper.java

License:Apache License

@Test
public final void testMapperValidValues() throws IOException, InterruptedException {
    Mapper.Context context = mock(Mapper.Context.class);
    Counters counters = new Counters();
    Counter counter = counters.findCounter(DedupRecordCounter.TOTAL_RECORDS_READ);
    when(context.getCounter(DedupRecordCounter.TOTAL_RECORDS_READ)).thenReturn(counter);
    DedupValueMapper mapper = new DedupValueMapper();
    Text key = new Text("abc123");
    Text val = new Text("valueOfKey");
    mapper.map(key, val, context);

    HihoTuple<Text> hihoTuple = new HihoTuple<Text>();
    hihoTuple.setKey(val);
    verify(context).write(hihoTuple, key);
    assertEquals(1, context.getCounter(DedupRecordCounter.TOTAL_RECORDS_READ).getValue());
}

From source file:co.nubetech.hiho.dedup.TestDedupValueMapper.java

License:Apache License

@Test(expected = IOException.class)
public final void testMapperForNullKeyValue() throws IOException, InterruptedException {
    Mapper.Context context = mock(Mapper.Context.class);
    Counters counters = new Counters();
    Counter counter = counters.findCounter(DedupRecordCounter.BAD_RECORD);
    when(context.getCounter(DedupRecordCounter.BAD_RECORD)).thenReturn(counter);
    DedupValueMapper mapper = new DedupValueMapper();
    Text val = new Text("valueOfKey");
    mapper.map(null, val, context);
}

From source file:co.nubetech.hiho.dedup.TestDedupValueReducer.java

License:Apache License

@Test
public void testReducerValidValues() throws IOException, InterruptedException {
    Text key = new Text("key123");
    HihoTuple hihoTuple = new HihoTuple();
    hihoTuple.setKey(key);/* w  ww .  j ava 2  s  .c  om*/

    String value1 = new String("value1");
    String value2 = new String("value2");
    String value3 = new String("value3");
    ArrayList<String> values = new ArrayList<String>();
    values.add(value1);
    values.add(value2);
    values.add(value3);

    Reducer.Context context = mock(Reducer.Context.class);
    Counters counters = new Counters();
    Counter counter = counters.findCounter(DedupRecordCounter.OUTPUT);
    when(context.getCounter(DedupRecordCounter.OUTPUT)).thenReturn(counter);
    DedupValueReducer dedupValueReducer = new DedupValueReducer();
    dedupValueReducer.reduce(hihoTuple, values, context);
    verify(context).write(value1, key);
    assertEquals(1, context.getCounter(DedupRecordCounter.OUTPUT).getValue());
}

From source file:co.nubetech.hiho.dedup.TestDedupValueReducer.java

License:Apache License

@Test
public void testReducerForNullValues() throws IOException, InterruptedException {
    Text key = new Text("key123");
    HihoTuple hihoTuple = new HihoTuple();
    hihoTuple.setKey(key);// ww w .j  ava2  s  .com

    String value1 = null;
    ArrayList<String> values = new ArrayList<String>();
    values.add(value1);

    Reducer.Context context = mock(Reducer.Context.class);
    Counters counters = new Counters();
    Counter counter = counters.findCounter(DedupRecordCounter.OUTPUT);
    when(context.getCounter(DedupRecordCounter.OUTPUT)).thenReturn(counter);
    DedupValueReducer dedupValueReducer = new DedupValueReducer();
    dedupValueReducer.reduce(hihoTuple, values, context);
    verify(context).write(value1, key);
    assertEquals(1, context.getCounter(DedupRecordCounter.OUTPUT).getValue());
}

From source file:co.nubetech.hiho.dedup.TestDedupValueReducer.java

License:Apache License

@Test
public void testReducerForLongWritableKey() throws IOException, InterruptedException {
    LongWritable key = new LongWritable(Long.parseLong("123"));
    HihoTuple hihoTuple = new HihoTuple();
    hihoTuple.setKey(key);/*from w ww .j  av a 2s  .c o  m*/

    Text value1 = new Text("value1");
    ArrayList<Text> values = new ArrayList<Text>();
    values.add(value1);

    Reducer.Context context = mock(Reducer.Context.class);
    Counters counters = new Counters();
    Counter counter = counters.findCounter(DedupRecordCounter.OUTPUT);
    when(context.getCounter(DedupRecordCounter.OUTPUT)).thenReturn(counter);
    DedupValueReducer dedupReducer = new DedupValueReducer();
    dedupReducer.reduce(hihoTuple, values, context);
    verify(context).write(value1, key);
    assertEquals(1, context.getCounter(DedupRecordCounter.OUTPUT).getValue());
}

From source file:co.nubetech.hiho.dedup.TestDelimitedLineRecordReader.java

License:Apache License

@Test
public void testGetColumnFixedSizeCols() throws IOException {
    DelimitedLineRecordReader delimitedLineRecordReader = new DelimitedLineRecordReader();
    Text val = new Text("abd,cdd,efd,ghd,ijd,kld");
    Text key = delimitedLineRecordReader.getColumn(val, 5, ",");
    System.out.println("key is: " + key);
    assertEquals(new Text("ijd"), key);
    assertEquals(new Text("abd,cdd,efd,ghd,ijd,kld"), val); // checking
    // parsed text
    // value/*from  w  ww .  ja v a  2 s. c o m*/
}

From source file:co.nubetech.hiho.dedup.TestDelimitedLineRecordReader.java

License:Apache License

@Test
public void testGetColumnVariableSize() throws IOException {
    DelimitedLineRecordReader delimitedLineRecordReader = new DelimitedLineRecordReader();
    Text val = new Text("abcde,cedd,efd,ghd,ijd,kld");
    Text key = delimitedLineRecordReader.getColumn(val, 5, ",");
    System.out.println("key is: " + key);
    assertEquals(new Text("ijd"), key);
    assertEquals(new Text("abcde,cedd,efd,ghd,ijd,kld"), val);
}

From source file:co.nubetech.hiho.dedup.TestDelimitedLineRecordReader.java

License:Apache License

@Test
public void testGetColumnForDelimiterSizeGreaterThanOne() throws IOException {
    DelimitedLineRecordReader delimitedLineRecordReader = new DelimitedLineRecordReader();
    Text val = new Text("abd::cdd::efd::ghd::ijd::kld");
    Text key = delimitedLineRecordReader.getColumn(val, 5, "::");
    System.out.println("key is: " + key);
    assertEquals(new Text("ijd"), key);
    assertEquals(new Text("abd::cdd::efd::ghd::ijd::kld"), val);
}

From source file:co.nubetech.hiho.dedup.TestDelimitedLineRecordReader.java

License:Apache License

@Test
public void testGetColumnStartingWithDelimiter() throws IOException {
    DelimitedLineRecordReader delimitedLineRecordReader = new DelimitedLineRecordReader();
    Text val = new Text(",abd,cdd,efd,ghd,ijd,kld");
    Text key = delimitedLineRecordReader.getColumn(val, 5, ",");
    System.out.println("key is: " + key);
    assertEquals(new Text("ghd"), key);
    assertEquals(new Text(",abd,cdd,efd,ghd,ijd,kld"), val);
}