Example usage for org.apache.hadoop.mapreduce Counters Counters

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce Counters Counters.

Prototype

public Counters() 

Source Link

Document

Default constructor

Usage

From source file:co.nubetech.hiho.dedup.TestDedupKeyMapper.java

License:Apache License

@Test
public final void testMapperValidValues() throws IOException, InterruptedException {
    Mapper.Context context = mock(Mapper.Context.class);
    Counters counters = new Counters();
    Counter counter = counters.findCounter(DedupRecordCounter.TOTAL_RECORDS_READ);
    when(context.getCounter(DedupRecordCounter.TOTAL_RECORDS_READ)).thenReturn(counter);
    DedupKeyMapper<Text, String> mapper = new DedupKeyMapper<Text, String>();
    Text key = new Text("abc123");
    String val = "valueOfKey";
    mapper.map(key, val, context);

    HihoTuple<Text> hihoTuple = new HihoTuple<Text>();
    hihoTuple.setKey(key);//from   ww w.  j a v  a 2  s . com
    verify(context).write(hihoTuple, val);
    assertEquals(1, context.getCounter(DedupRecordCounter.TOTAL_RECORDS_READ).getValue());
}

---

From source file:co.nubetech.hiho.dedup.TestDedupKeyMapper.java

License:Apache License

@Test(expected = IOException.class)
public final void testMapperForNullKeyValue() throws IOException, InterruptedException {
    Mapper.Context context = mock(Mapper.Context.class);
    Counters counters = new Counters();
    Counter counter = counters.findCounter(DedupRecordCounter.BAD_RECORD);
    when(context.getCounter(DedupRecordCounter.BAD_RECORD)).thenReturn(counter);
    DedupKeyMapper mapper = new DedupKeyMapper();
    String val = "valueOfKey";
    mapper.map(null, val, context);
}

---

From source file:co.nubetech.hiho.dedup.TestDedupKeyReducer.java

License:Apache License

@Test
public void testReducerValidValues() throws IOException, InterruptedException {
    Text key = new Text("key123");
    HihoTuple hihoTuple = new HihoTuple();
    hihoTuple.setKey(key);/* ww w  .  j  a v  a 2 s .  co  m*/

    String value1 = new String("value1");
    String value2 = new String("value2");
    String value3 = new String("value3");
    ArrayList<String> values = new ArrayList<String>();
    values.add(value1);
    values.add(value2);
    values.add(value3);

    Reducer.Context context = mock(Reducer.Context.class);
    Counters counters = new Counters();
    Counter counter = counters.findCounter(DedupRecordCounter.OUTPUT);
    when(context.getCounter(DedupRecordCounter.OUTPUT)).thenReturn(counter);
    DedupKeyReducer dedupReducer = new DedupKeyReducer();
    dedupReducer.reduce(hihoTuple, values, context);
    verify(context).write(key, value1);
    assertEquals(1, context.getCounter(DedupRecordCounter.OUTPUT).getValue());
}

---

From source file:co.nubetech.hiho.dedup.TestDedupKeyReducer.java

License:Apache License

@Test
public void testReducerForLongWritableKey() throws IOException, InterruptedException {
    LongWritable key = new LongWritable(Long.parseLong("123"));
    HihoTuple hihoTuple = new HihoTuple();
    hihoTuple.setKey(key);/* ww w .j a  v a 2 s .com*/

    Text value1 = new Text("value1");
    ArrayList<Text> values = new ArrayList<Text>();
    values.add(value1);

    Reducer.Context context = mock(Reducer.Context.class);
    Counters counters = new Counters();
    Counter counter = counters.findCounter(DedupRecordCounter.OUTPUT);
    when(context.getCounter(DedupRecordCounter.OUTPUT)).thenReturn(counter);
    DedupKeyReducer dedupReducer = new DedupKeyReducer();
    dedupReducer.reduce(hihoTuple, values, context);
    verify(context).write(key, value1);
    assertEquals(1, context.getCounter(DedupRecordCounter.OUTPUT).getValue());
}

---

From source file:co.nubetech.hiho.dedup.TestDedupKeyReducer.java

License:Apache License

@Test
public void testReducerForBytesWritableKeyAndValue() throws IOException, InterruptedException {
    BytesWritable key = new BytesWritable("abc123".getBytes());
    HihoTuple hihoTuple = new HihoTuple();
    hihoTuple.setKey(key);/*from   www .  j a v a2s.  c  o  m*/

    BytesWritable value1 = new BytesWritable("value1".getBytes());
    ArrayList<BytesWritable> values = new ArrayList<BytesWritable>();
    values.add(value1);

    Reducer.Context context = mock(Reducer.Context.class);
    Counters counters = new Counters();
    Counter counter = counters.findCounter(DedupRecordCounter.OUTPUT);
    when(context.getCounter(DedupRecordCounter.OUTPUT)).thenReturn(counter);
    DedupKeyReducer dedupReducer = new DedupKeyReducer();
    dedupReducer.reduce(hihoTuple, values, context);
    verify(context).write(key, value1);
    assertEquals(1, context.getCounter(DedupRecordCounter.OUTPUT).getValue());
}

---

From source file:co.nubetech.hiho.dedup.TestDedupKeyReducer.java

License:Apache License

@Test
public void testReducerForIntWritableKeyAndValue() throws IOException, InterruptedException {
    IntWritable key = new IntWritable(123);
    HihoTuple hihoTuple = new HihoTuple();
    hihoTuple.setKey(key);/* ww w  .  ja v  a  2 s  .  c  o m*/

    IntWritable value1 = new IntWritable(456);
    ArrayList<IntWritable> values = new ArrayList<IntWritable>();
    values.add(value1);

    Reducer.Context context = mock(Reducer.Context.class);
    Counters counters = new Counters();
    Counter counter = counters.findCounter(DedupRecordCounter.OUTPUT);
    when(context.getCounter(DedupRecordCounter.OUTPUT)).thenReturn(counter);
    DedupKeyReducer dedupReducer = new DedupKeyReducer();
    dedupReducer.reduce(hihoTuple, values, context);
    verify(context).write(key, value1);
    assertEquals(1, context.getCounter(DedupRecordCounter.OUTPUT).getValue());
}

---

From source file:co.nubetech.hiho.dedup.TestDedupKeyReducer.java

License:Apache License

@Test
public void testReducerForNullValues() throws IOException, InterruptedException {
    Text key = new Text("key123");
    HihoTuple hihoTuple = new HihoTuple();
    hihoTuple.setKey(key);//from w  ww  .j av a 2  s  .c  om

    String value1 = null;
    ArrayList<String> values = new ArrayList<String>();
    values.add(value1);

    Reducer.Context context = mock(Reducer.Context.class);
    Counters counters = new Counters();
    Counter counter = counters.findCounter(DedupRecordCounter.OUTPUT);
    when(context.getCounter(DedupRecordCounter.OUTPUT)).thenReturn(counter);
    DedupKeyReducer dedupReducer = new DedupKeyReducer();
    dedupReducer.reduce(hihoTuple, values, context);
    verify(context).write(key, value1);
    assertEquals(1, context.getCounter(DedupRecordCounter.OUTPUT).getValue());
}

---

From source file:co.nubetech.hiho.dedup.TestDedupValueMapper.java

License:Apache License

@Test
public final void testMapperValidValues() throws IOException, InterruptedException {
    Mapper.Context context = mock(Mapper.Context.class);
    Counters counters = new Counters();
    Counter counter = counters.findCounter(DedupRecordCounter.TOTAL_RECORDS_READ);
    when(context.getCounter(DedupRecordCounter.TOTAL_RECORDS_READ)).thenReturn(counter);
    DedupValueMapper mapper = new DedupValueMapper();
    Text key = new Text("abc123");
    Text val = new Text("valueOfKey");
    mapper.map(key, val, context);

    HihoTuple<Text> hihoTuple = new HihoTuple<Text>();
    hihoTuple.setKey(val);
    verify(context).write(hihoTuple, key);
    assertEquals(1, context.getCounter(DedupRecordCounter.TOTAL_RECORDS_READ).getValue());
}

---

From source file:co.nubetech.hiho.dedup.TestDedupValueMapper.java

License:Apache License

@Test(expected = IOException.class)
public final void testMapperForNullKeyValue() throws IOException, InterruptedException {
    Mapper.Context context = mock(Mapper.Context.class);
    Counters counters = new Counters();
    Counter counter = counters.findCounter(DedupRecordCounter.BAD_RECORD);
    when(context.getCounter(DedupRecordCounter.BAD_RECORD)).thenReturn(counter);
    DedupValueMapper mapper = new DedupValueMapper();
    Text val = new Text("valueOfKey");
    mapper.map(null, val, context);
}

---

From source file:co.nubetech.hiho.dedup.TestDedupValueReducer.java

License:Apache License

@Test
public void testReducerValidValues() throws IOException, InterruptedException {
    Text key = new Text("key123");
    HihoTuple hihoTuple = new HihoTuple();
    hihoTuple.setKey(key);/*ww w. j  ava 2  s.  c o m*/

    String value1 = new String("value1");
    String value2 = new String("value2");
    String value3 = new String("value3");
    ArrayList<String> values = new ArrayList<String>();
    values.add(value1);
    values.add(value2);
    values.add(value3);

    Reducer.Context context = mock(Reducer.Context.class);
    Counters counters = new Counters();
    Counter counter = counters.findCounter(DedupRecordCounter.OUTPUT);
    when(context.getCounter(DedupRecordCounter.OUTPUT)).thenReturn(counter);
    DedupValueReducer dedupValueReducer = new DedupValueReducer();
    dedupValueReducer.reduce(hihoTuple, values, context);
    verify(context).write(value1, key);
    assertEquals(1, context.getCounter(DedupRecordCounter.OUTPUT).getValue());
}

---

• «
• 1
• 2
• 3
• 4
• 5
• »