Example usage for org.apache.hadoop.mapred Counters Counters

List of usage examples for org.apache.hadoop.mapred Counters Counters

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.mapred.*
  5. Counters
  6. Counters

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred Counters Counters.

Prototype

public Counters()

Source Link

Usage

From source file:be.uantwerpen.adrem.bigfim.AprioriPhaseReducerTest.java

License:Apache License

@Test
public void numberOfPGCounter_One_Group()
        throws IOException, InterruptedException, NoSuchFieldException, IllegalAccessException {
    AprioriPhaseReducer.Context ctx = createMock(Reducer.Context.class);
    Counter pgCounter = createMockCounter(1);

    EasyMock.expect(ctx.getCounter(COUNTER_GROUPNAME, COUNTER_NRPREFIXGROUPS)).andReturn(pgCounter).anyTimes();
    EasyMock.expect(ctx.getCounter(COUNTER_GROUPNAME, COUNTER_NRLARGEPREFIXGROUPS))
            .andReturn(new Counters().findCounter(COUNTER_GROUPNAME, COUNTER_NRLARGEPREFIXGROUPS)).anyTimes();

    EasyMock.replay(ctx, pgCounter);// w  w  w.  j a  v a  2 s .c o m

    AprioriPhaseReducer reducer = createAprioriPhaseReducer(1, createMultipleOutputsOnePrefix(1));
    reducer.reduce(new Text(""), createReducerInput(), ctx);
    reducer.cleanup(ctx);

    EasyMock.verify(ctx, pgCounter);
}

From source file:be.uantwerpen.adrem.bigfim.AprioriPhaseReducerTest.java

License:Apache License

@Test
public void numberOfPGCounter_Multiple_Groups()
        throws IOException, InterruptedException, NoSuchFieldException, IllegalAccessException {
    AprioriPhaseReducer.Context ctx = createMock(Reducer.Context.class);
    Counter pgCounter = createMockCounter(3);

    EasyMock.expect(ctx.getCounter(COUNTER_GROUPNAME, COUNTER_NRPREFIXGROUPS)).andReturn(pgCounter).anyTimes();
    EasyMock.expect(ctx.getCounter(COUNTER_GROUPNAME, COUNTER_NRLARGEPREFIXGROUPS))
            .andReturn(new Counters().findCounter(COUNTER_GROUPNAME, COUNTER_NRLARGEPREFIXGROUPS)).anyTimes();

    EasyMock.replay(ctx, pgCounter);//from  www  .  j  a  v a2 s .c  o m

    AprioriPhaseReducer reducer = createAprioriPhaseReducer(1, createMultipleOutputsTwoPrefix(1));
    for (String prefix : new String[] { "1", "2", "4" }) {
        reducer.reduce(new Text(prefix), createReducerInputTwoPrefix(prefix), ctx);
    }
    reducer.cleanup(ctx);

    EasyMock.verify(ctx, pgCounter);
}

From source file:be.uantwerpen.adrem.bigfim.AprioriPhaseReducerTest.java

License:Apache License

@Test
public void numberOfLargePGCounter_One_Group_Not_Large()
        throws IOException, InterruptedException, NoSuchFieldException, IllegalAccessException {
    AprioriPhaseReducer.Context ctx = createMock(Reducer.Context.class);
    Counter pgCounter = createMockCounter(0);

    EasyMock.expect(ctx.getCounter(COUNTER_GROUPNAME, COUNTER_NRPREFIXGROUPS))
            .andReturn(new Counters().findCounter(COUNTER_GROUPNAME, COUNTER_NRPREFIXGROUPS)).anyTimes();
    EasyMock.expect(ctx.getCounter(COUNTER_GROUPNAME, COUNTER_NRLARGEPREFIXGROUPS)).andReturn(pgCounter)
            .anyTimes();/* ww  w . j av  a 2 s  . c om*/

    EasyMock.replay(ctx, pgCounter);

    int tmp = ComputeTidListReducer.MAX_NUMBER_OF_TIDS;
    ComputeTidListReducer.MAX_NUMBER_OF_TIDS = 50;

    AprioriPhaseReducer reducer = createAprioriPhaseReducer(1, createMultipleOutputsOnePrefix(1));
    reducer.reduce(new Text(""), createReducerInput(), ctx);
    reducer.cleanup(ctx);

    EasyMock.verify(ctx, pgCounter);

    ComputeTidListReducer.MAX_NUMBER_OF_TIDS = tmp;
}

From source file:be.uantwerpen.adrem.bigfim.AprioriPhaseReducerTest.java

License:Apache License

@Test
public void numberOfLargePGCounter_One_Group_Is_Large()
        throws IOException, InterruptedException, NoSuchFieldException, IllegalAccessException {
    AprioriPhaseReducer.Context ctx = createMock(Reducer.Context.class);
    Counter pgCounter = createMockCounter(1);

    EasyMock.expect(ctx.getCounter(COUNTER_GROUPNAME, COUNTER_NRPREFIXGROUPS))
            .andReturn(new Counters().findCounter(COUNTER_GROUPNAME, COUNTER_NRPREFIXGROUPS)).anyTimes();
    EasyMock.expect(ctx.getCounter(COUNTER_GROUPNAME, COUNTER_NRLARGEPREFIXGROUPS)).andReturn(pgCounter)
            .anyTimes();/*from   www.j  a va2s  .c o  m*/

    EasyMock.replay(ctx, pgCounter);

    int tmp = ComputeTidListReducer.MAX_NUMBER_OF_TIDS;
    ComputeTidListReducer.MAX_NUMBER_OF_TIDS = 10;

    AprioriPhaseReducer reducer = createAprioriPhaseReducer(1, createMultipleOutputsOnePrefix(1));
    reducer.reduce(new Text(""), createReducerInput(), ctx);
    reducer.cleanup(ctx);

    EasyMock.verify(ctx, pgCounter);

    ComputeTidListReducer.MAX_NUMBER_OF_TIDS = tmp;
}

From source file:be.uantwerpen.adrem.bigfim.AprioriPhaseReducerTest.java

License:Apache License

@Test
public void numberOfLargePGCounter_Multiple_Groups()
        throws IOException, InterruptedException, NoSuchFieldException, IllegalAccessException {
    AprioriPhaseReducer.Context ctx = createMock(Reducer.Context.class);
    Counter pgCounter = createMockCounter(2);

    EasyMock.expect(ctx.getCounter(COUNTER_GROUPNAME, COUNTER_NRPREFIXGROUPS))
            .andReturn(new Counters().findCounter(COUNTER_GROUPNAME, COUNTER_NRPREFIXGROUPS)).anyTimes();
    EasyMock.expect(ctx.getCounter(COUNTER_GROUPNAME, COUNTER_NRLARGEPREFIXGROUPS)).andReturn(pgCounter)
            .anyTimes();//from  w  w  w  .  j av  a 2s . co m

    EasyMock.replay(ctx, pgCounter);

    int tmp = ComputeTidListReducer.MAX_NUMBER_OF_TIDS;
    ComputeTidListReducer.MAX_NUMBER_OF_TIDS = 20;

    AprioriPhaseReducer reducer = createAprioriPhaseReducer(1, createMultipleOutputsTwoPrefix(1));
    for (String prefix : new String[] { "1", "2", "4" }) {
        reducer.reduce(new Text(prefix), createReducerInputTwoPrefix(prefix), ctx);
    }
    reducer.cleanup(ctx);

    EasyMock.verify(ctx, pgCounter);

    ComputeTidListReducer.MAX_NUMBER_OF_TIDS = tmp;
}

From source file:com.atlantbh.jmeter.plugins.hadooputilities.jobstatistics.RunningJobEmul.java

License:Apache License

@Override
public Counters getCounters() throws IOException {
    return new Counters();
}

From source file:com.cloudera.crunch.test.TestCounters.java

License:Open Source License

public static void clearCounters() {
    COUNTERS = new Counters();
}

From source file:com.datatorrent.demos.mroperator.MapOperator.java

License:Open Source License

@Override
public void setup(OperatorContext context) {
    if (context != null) {
        operatorId = context.getId();/* ww w  .j  a v a  2 s . c o m*/
    }
    reporter = new ReporterImpl(ReporterType.Mapper, new Counters());
    outputCollector = new OutputCollectorImpl<K2, V2>();
    Configuration conf = new Configuration();
    try {
        inputFormat = inputFormatClass.newInstance();
        SerializationFactory serializationFactory = new SerializationFactory(conf);
        Deserializer keyDesiralizer = serializationFactory.getDeserializer(inputSplitClass);
        keyDesiralizer.open(new ByteArrayInputStream(outstream.toByteArray()));
        inputSplit = (InputSplit) keyDesiralizer.deserialize(null);
        ((ReporterImpl) reporter).setInputSplit(inputSplit);
        reader = inputFormat.getRecordReader(inputSplit, new JobConf(conf), reporter);
    } catch (Exception e) {
        logger.info("failed to initialize inputformat obj {}", inputFormat);
        throw new RuntimeException(e);
    }
    InputStream stream = null;
    if (configFile != null && configFile.length() > 0) {
        stream = ClassLoader.getSystemResourceAsStream("/" + configFile);
        if (stream == null) {
            stream = ClassLoader.getSystemResourceAsStream(configFile);
        }
    }
    if (stream != null) {
        conf.addResource(stream);
    }
    jobConf = new JobConf(conf);
    if (mapClass != null) {
        try {
            mapObject = mapClass.newInstance();
        } catch (Exception e) {
            logger.info("can't instantiate object {}", e.getMessage());
        }

        mapObject.configure(jobConf);
    }
    if (combineClass != null) {
        try {
            combineObject = combineClass.newInstance();
        } catch (Exception e) {
            logger.info("can't instantiate object {}", e.getMessage());
        }
        combineObject.configure(jobConf);
    }
}

From source file:com.datatorrent.demos.mroperator.ReduceOperator.java

License:Open Source License

@Override
public void setup(OperatorContext context) {
    reporter = new ReporterImpl(ReporterType.Reducer, new Counters());
    if (context != null) {
        operatorId = context.getId();/*  w  w w  .j ava 2  s  . co  m*/
    }
    cacheObject = new HashMap<K1, List<V1>>();
    outputCollector = new OutputCollectorImpl<K2, V2>();
    if (reduceClass != null) {
        try {
            reduceObj = reduceClass.newInstance();
        } catch (Exception e) {
            logger.info("can't instantiate object {}", e.getMessage());
            throw new RuntimeException(e);
        }
        Configuration conf = new Configuration();
        InputStream stream = null;
        if (configFile != null && configFile.length() > 0) {
            logger.info("system /{}", configFile);
            stream = ClassLoader.getSystemResourceAsStream("/" + configFile);
            if (stream == null) {
                logger.info("system {}", configFile);
                stream = ClassLoader.getSystemResourceAsStream(configFile);
            }
        }
        if (stream != null) {
            logger.info("found our stream... so adding it");
            conf.addResource(stream);
        }
        reduceObj.configure(new JobConf(conf));
    }

}

From source file:com.hdfs.concat.crush.CountersMapperTest.java

License:Apache License

@Test
public void map() throws IOException {
    Counters counters = new Counters();

    counters.incrCounter(MapperCounter.DIRS_FOUND, 1);
    reporter.incrCounter(MapperCounter.class.getName(), MapperCounter.DIRS_FOUND.name(), 1);

    counters.incrCounter(MapperCounter.DIRS_ELIGIBLE, 2);
    reporter.incrCounter(MapperCounter.class.getName(), MapperCounter.DIRS_ELIGIBLE.name(), 2);

    counters.incrCounter(MapperCounter.DIRS_SKIPPED, 3);
    reporter.incrCounter(MapperCounter.class.getName(), MapperCounter.DIRS_SKIPPED.name(), 3);

    counters.incrCounter(MapperCounter.FILES_FOUND, 4);
    reporter.incrCounter(MapperCounter.class.getName(), MapperCounter.FILES_FOUND.name(), 4);

    counters.incrCounter(MapperCounter.FILES_SKIPPED, 5);
    reporter.incrCounter(MapperCounter.class.getName(), MapperCounter.FILES_SKIPPED.name(), 5);

    replayAll();//from  w ww.  j  a  v  a 2  s. c  om

    mapper.map(counters, null, null, reporter);

    verifyAll();
}