Example usage for org.apache.hadoop.mapreduce Counters Counters

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce Counters Counters.

Prototype

public Counters() 

Source Link

Document

Default constructor

Usage

From source file:co.nubetech.hiho.dedup.TestDedupValueReducer.java

License:Apache License

@Test
public void testReducerForNullValues() throws IOException, InterruptedException {
    Text key = new Text("key123");
    HihoTuple hihoTuple = new HihoTuple();
    hihoTuple.setKey(key);/* ww  w  . j  ava 2s  .c om*/

    String value1 = null;
    ArrayList<String> values = new ArrayList<String>();
    values.add(value1);

    Reducer.Context context = mock(Reducer.Context.class);
    Counters counters = new Counters();
    Counter counter = counters.findCounter(DedupRecordCounter.OUTPUT);
    when(context.getCounter(DedupRecordCounter.OUTPUT)).thenReturn(counter);
    DedupValueReducer dedupValueReducer = new DedupValueReducer();
    dedupValueReducer.reduce(hihoTuple, values, context);
    verify(context).write(value1, key);
    assertEquals(1, context.getCounter(DedupRecordCounter.OUTPUT).getValue());
}

---

From source file:co.nubetech.hiho.dedup.TestDedupValueReducer.java

License:Apache License

@Test
public void testReducerForLongWritableKey() throws IOException, InterruptedException {
    LongWritable key = new LongWritable(Long.parseLong("123"));
    HihoTuple hihoTuple = new HihoTuple();
    hihoTuple.setKey(key);/*  ww w.  j  av  a2  s  .  c  o  m*/

    Text value1 = new Text("value1");
    ArrayList<Text> values = new ArrayList<Text>();
    values.add(value1);

    Reducer.Context context = mock(Reducer.Context.class);
    Counters counters = new Counters();
    Counter counter = counters.findCounter(DedupRecordCounter.OUTPUT);
    when(context.getCounter(DedupRecordCounter.OUTPUT)).thenReturn(counter);
    DedupValueReducer dedupReducer = new DedupValueReducer();
    dedupReducer.reduce(hihoTuple, values, context);
    verify(context).write(value1, key);
    assertEquals(1, context.getCounter(DedupRecordCounter.OUTPUT).getValue());
}

---

From source file:co.nubetech.hiho.dedup.TestDedupValueReducer.java

License:Apache License

@Test
public void testReducerForBytesWritableKeyAndValue() throws IOException, InterruptedException {
    BytesWritable key = new BytesWritable("abc123".getBytes());
    HihoTuple hihoTuple = new HihoTuple();
    hihoTuple.setKey(key);//  ww w  .j a  v a2 s  .c  o m

    BytesWritable value1 = new BytesWritable("value1".getBytes());
    ArrayList<BytesWritable> values = new ArrayList<BytesWritable>();
    values.add(value1);

    Reducer.Context context = mock(Reducer.Context.class);
    Counters counters = new Counters();
    Counter counter = counters.findCounter(DedupRecordCounter.OUTPUT);
    when(context.getCounter(DedupRecordCounter.OUTPUT)).thenReturn(counter);
    DedupValueReducer dedupReducer = new DedupValueReducer();
    dedupReducer.reduce(hihoTuple, values, context);
    verify(context).write(value1, key);
    assertEquals(1, context.getCounter(DedupRecordCounter.OUTPUT).getValue());
}

---

From source file:co.nubetech.hiho.dedup.TestDedupValueReducer.java

License:Apache License

@Test
public void testReducerForIntWritableKeyAndValue() throws IOException, InterruptedException {
    IntWritable key = new IntWritable(123);
    HihoTuple hihoTuple = new HihoTuple();
    hihoTuple.setKey(key);/*ww w.j a  v  a  2s .  c o  m*/

    IntWritable value1 = new IntWritable(456);
    ArrayList<IntWritable> values = new ArrayList<IntWritable>();
    values.add(value1);

    Reducer.Context context = mock(Reducer.Context.class);
    Counters counters = new Counters();
    Counter counter = counters.findCounter(DedupRecordCounter.OUTPUT);
    when(context.getCounter(DedupRecordCounter.OUTPUT)).thenReturn(counter);
    DedupValueReducer dedupReducer = new DedupValueReducer();
    dedupReducer.reduce(hihoTuple, values, context);
    verify(context).write(value1, key);
    assertEquals(1, context.getCounter(DedupRecordCounter.OUTPUT).getValue());
}

---

From source file:co.nubetech.hiho.merge.TestMergeKeyMapper.java

License:Apache License

@Test(expected = IOException.class)
public final void testMapperForNullKeyValue() throws IOException, InterruptedException {
    Mapper.Context context = mock(Mapper.Context.class);
    Counters counters = new Counters();
    Counter counter = counters.findCounter(MergeRecordCounter.BAD_RECORD);
    when(context.getCounter(MergeRecordCounter.BAD_RECORD)).thenReturn(counter);
    MergeKeyMapper mapper = new MergeKeyMapper();
    Text val = new Text("valueOfKey");
    mapper.map(null, val, context);
}

---

From source file:co.nubetech.hiho.merge.TestMergeKeyMapper.java

License:Apache License

@Test
public final void testMapperValidValues() throws IOException, InterruptedException {
    Mapper.Context context = mock(Mapper.Context.class);
    Counters counters = new Counters();
    Counter counter = counters.findCounter(MergeRecordCounter.TOTAL_RECORDS_NEW);
    when(context.getCounter(MergeRecordCounter.TOTAL_RECORDS_NEW)).thenReturn(counter);

    MergeKeyMapper mapper = new MergeKeyMapper();
    Text key = new Text("abc123");
    Text val = new Text("valueOfKey");
    mapper.isOld = false;/* ww  w.j a v a2  s . co m*/
    mapper.map(key, val, context);

    HihoValue hihoValue = new HihoValue();
    hihoValue.setVal(val);
    hihoValue.setIsOld(false);
    HihoTuple hihoTuple = new HihoTuple();
    hihoTuple.setKey(key);
    verify(context).write(hihoTuple, hihoValue);
    assertEquals(1, context.getCounter(MergeRecordCounter.TOTAL_RECORDS_NEW).getValue());
}

---

From source file:co.nubetech.hiho.merge.TestMergeKeyReducer.java

License:Apache License

@Test
public void testReducerValidValues() throws IOException, InterruptedException {
    Text key = new Text("key123");
    HihoTuple hihoTuple = new HihoTuple();
    hihoTuple.setKey(key);/*from ww w . jav a 2  s  . c  o  m*/

    HihoValue hihoValue1 = new HihoValue();
    HihoValue hihoValue2 = new HihoValue();
    Text value1 = new Text("value1");
    Text value2 = new Text("value2");
    hihoValue1.setVal(value1);
    hihoValue2.setVal(value2);
    hihoValue1.setIsOld(true);
    hihoValue2.setIsOld(false);
    ArrayList<HihoValue> values = new ArrayList<HihoValue>();
    values.add(hihoValue1);
    values.add(hihoValue2);

    Reducer.Context context = mock(Reducer.Context.class);
    Counters counters = new Counters();
    Counter counter = counters.findCounter(MergeRecordCounter.OUTPUT);
    when(context.getCounter(MergeRecordCounter.OUTPUT)).thenReturn(counter);
    MergeKeyReducer mergeReducer = new MergeKeyReducer();
    mergeReducer.reduce(hihoTuple, values, context);
    verify(context).write(key, value2);
    assertEquals(1, context.getCounter(MergeRecordCounter.OUTPUT).getValue());
}

---

From source file:co.nubetech.hiho.merge.TestMergeKeyReducer.java

License:Apache License

@Test
public void testReducerNullValues() throws IOException, InterruptedException {
    Text key = new Text("key123");
    HihoTuple hihoTuple = new HihoTuple();
    hihoTuple.setKey(key);/*from ww  w.j  a  va 2s.  c  om*/

    HihoValue hihoValue1 = new HihoValue();
    HihoValue hihoValue2 = new HihoValue();
    Text value1 = new Text("value1");
    Text value2 = new Text("value2");
    hihoValue1.setVal(value1);
    hihoValue2.setVal(value2);
    hihoValue1.setIsOld(true);
    hihoValue2.setIsOld(false);
    ArrayList<HihoValue> values = new ArrayList<HihoValue>();

    Reducer.Context context = mock(Reducer.Context.class);
    Counters counters = new Counters();
    Counter counter = counters.findCounter(MergeRecordCounter.OUTPUT);
    when(context.getCounter(MergeRecordCounter.OUTPUT)).thenReturn(counter);
    MergeKeyReducer mergeReducer = new MergeKeyReducer();
    mergeReducer.reduce(hihoTuple, values, context);
    verify(context).write(key, null);
    assertEquals(1, context.getCounter(MergeRecordCounter.OUTPUT).getValue());
}

---

From source file:co.nubetech.hiho.merge.TestMergeKeyReducer.java

License:Apache License

@Test
public void testReducerForLongWritableKey() throws IOException, InterruptedException {
    LongWritable key = new LongWritable(Long.parseLong("123"));
    HihoTuple hihoTuple = new HihoTuple();
    hihoTuple.setKey(key);/*from  w w w . j  a  va2  s.co  m*/

    HihoValue hihoValue1 = new HihoValue();
    HihoValue hihoValue2 = new HihoValue();
    Text value1 = new Text("value1");
    Text value2 = new Text("value2");
    hihoValue1.setVal(value1);
    hihoValue2.setVal(value2);
    hihoValue1.setIsOld(true);
    hihoValue2.setIsOld(false);
    ArrayList<HihoValue> values = new ArrayList<HihoValue>();
    values.add(hihoValue1);
    values.add(hihoValue2);

    Reducer.Context context = mock(Reducer.Context.class);
    Counters counters = new Counters();
    Counter counter = counters.findCounter(MergeRecordCounter.OUTPUT);
    when(context.getCounter(MergeRecordCounter.OUTPUT)).thenReturn(counter);
    MergeKeyReducer mergeReducer = new MergeKeyReducer();
    mergeReducer.reduce(hihoTuple, values, context);
    verify(context).write(key, value2);
    assertEquals(1, context.getCounter(MergeRecordCounter.OUTPUT).getValue());
}

---

From source file:co.nubetech.hiho.merge.TestMergeKeyReducer.java

License:Apache License

@Test
public void testReducerForBytesWritableKeyAndValue() throws IOException, InterruptedException {
    BytesWritable key = new BytesWritable("abc123".getBytes());
    HihoTuple hihoTuple = new HihoTuple();
    hihoTuple.setKey(key);/*from   w  ww.  j  a  v a 2s.  c o m*/

    HihoValue hihoValue1 = new HihoValue();
    HihoValue hihoValue2 = new HihoValue();
    BytesWritable value1 = new BytesWritable("value1".getBytes());
    BytesWritable value2 = new BytesWritable("value2".getBytes());
    hihoValue1.setVal(value1);
    hihoValue2.setVal(value2);
    hihoValue1.setIsOld(true);
    hihoValue2.setIsOld(false);
    ArrayList<HihoValue> values = new ArrayList<HihoValue>();
    values.add(hihoValue1);
    values.add(hihoValue2);

    Reducer.Context context = mock(Reducer.Context.class);
    Counters counters = new Counters();
    Counter counter = counters.findCounter(MergeRecordCounter.OUTPUT);
    when(context.getCounter(MergeRecordCounter.OUTPUT)).thenReturn(counter);
    MergeKeyReducer mergeReducer = new MergeKeyReducer();
    mergeReducer.reduce(hihoTuple, values, context);
    verify(context).write(key, value2);
    assertEquals(1, context.getCounter(MergeRecordCounter.OUTPUT).getValue());
}

---

• «
• 1
• 2
• 3
• 4
• 5
• »