List of usage examples for org.apache.hadoop.mapreduce Counters Counters
public Counters()
From source file:com.google.appengine.tools.mapreduce.MapReduceState.java
License:Apache License
/** * Reconstitutes a Counters object from a MR state entity. * The returned counters is a copy. You must call * {@link #setCounters(Counters)} to persist updated counters to the * datastore./*from www. ja v a2 s. c o m*/ * * @return the reconstituted Counters object */ public Counters getCounters() { Blob serializedMap = (Blob) entity.getProperty(COUNTERS_MAP_PROPERTY); Counters counters = new Counters(); Writables.initializeWritableFromByteArray(serializedMap.getBytes(), counters); return counters; }
From source file:com.google.appengine.tools.mapreduce.ShardState.java
License:Apache License
/** * Creates a shard state that's active but hasn't made any progress as of yet. * //from ww w. j a v a 2 s. com * The shard state isn't persisted when returned (so {@link #getKey()} will * return {@code null} until {@link #persist()} is called. * * @param service the datastore to persist the ShardState to * @param taskAttemptId the TaskAttemptID corresponding to the returned * ShardState * @return the initialized shard state */ public static ShardState generateInitializedShardState(DatastoreService service, TaskAttemptID taskAttemptId) { ShardState shardState = new ShardState(service); shardState.entity = new Entity("ShardState", taskAttemptId.toString()); shardState.entity.setProperty(JOB_ID_PROPERTY, taskAttemptId.getJobID().toString()); Counters counters = new Counters(); shardState.setCounters(counters); shardState.setStatusString(""); shardState.entity.setProperty(STATUS_PROPERTY, Status.ACTIVE.name()); return shardState; }
From source file:com.google.appengine.tools.mapreduce.v2.impl.ShardState.java
License:Apache License
/** * Creates a shard state that's active but hasn't made any progress as of yet. * * The shard state isn't persisted when returned (so {@link #getKey()} will * return {@code null} until {@link #persist()} is called. * * @param service the datastore to persist the ShardState to * @param taskAttemptId the TaskAttemptID corresponding to the returned * ShardState//w w w . j a v a 2 s . c o m * @return the initialized shard state */ public static ShardState generateInitializedShardState(DatastoreService service, TaskAttemptID taskAttemptId) { ShardState shardState = new ShardState(service); shardState.entity = new Entity("ShardState", taskAttemptId.toString()); shardState.entity.setProperty(JOB_ID_PROPERTY, taskAttemptId.getJobID().toString()); Counters counters = new Counters(); shardState.setCounters(counters); shardState.setStatusString(""); shardState.entity.setProperty(STATUS_PROPERTY, "" + Status.ACTIVE); return shardState; }
From source file:com.inmobi.conduit.distcp.tools.mapred.TestCopyCommitter.java
License:Apache License
public void testValidationPass() { config.setLong(DistCpConstants.CONF_LABEL_TOTAL_BYTES_TO_BE_COPIED, 100); Counters counters = new Counters(); CounterGroup grp = counters.getGroup(CopyMapper.Counter.class.getName()); grp.findCounter(CopyMapper.Counter.BYTES_COPIED.name()).increment(50); grp.findCounter(CopyMapper.Counter.BYTES_FAILED.name()).increment(20); grp.findCounter(CopyMapper.Counter.BYTES_SKIPPED.name()).increment(30); counterProvider.setCounters(counters); try {//www . ja v a 2s . c o m TaskAttemptContext taskAttemptContext = getTaskAttemptContext(config); JobContext jobContext = Mockito.mock(JobContext.class); Mockito.when(jobContext.getConfiguration()).thenReturn(config); JobID jobID = new JobID(); Mockito.when(jobContext.getJobID()).thenReturn(jobID); final String[] statusString = new String[1]; try { Mockito.doAnswer(new Answer() { @Override public Object answer(InvocationOnMock invocationOnMock) throws Throwable { LOG.info("XXXX crap I am called now " + invocationOnMock.getArguments()[0]); statusString[0] = (String) invocationOnMock.getArguments()[0]; return null; //To change body of implemented methods use File | Settings | File Templates. } }).when(taskAttemptContext).setStatus(Mockito.anyString()); } catch (Throwable e) { } try { OutputCommitter committer = new CopyCommitter(null, taskAttemptContext); committer.commitJob(jobContext); Assert.assertEquals(statusString[0], "Commit Successful"); } catch (IOException e) { LOG.error("Exception encountered ", e); Assert.fail("Commit failed"); } } finally { config.setLong(DistCpConstants.CONF_LABEL_TOTAL_BYTES_TO_BE_COPIED, 0); counterProvider.setCounters(EMPTY_COUNTERS); } }
From source file:com.linkedin.cubert.utils.ScriptStats.java
License:Open Source License
public ScriptStats() { conf = new Configuration(); try {//from ww w . j a va 2 s . c o m jobClient = new JobClient(new JobConf(conf)); } catch (IOException e) { throw new RuntimeException(e); } aggregate.startTime = System.currentTimeMillis(); aggregate.counters = new Counters(); }
From source file:com.scaleunlimited.cascading.LoggingFlowProcess.java
License:Apache License
/** * @param counter whose value should be returned * @return current value of the counter, local to the task * <br/><br/><b>Note:</b> Only the JobTracker aggregates task counter values * to report the job-wide total.//from w w w. j a v a 2 s. c o m */ public long getCounter(Enum counter) { if (_isLocal) { AtomicLong count = _localCounters.get(counter); if (count != null) { return count.get(); } else { return 0; } } else { Counters counters = new Counters(); Counter hadoopCounter = counters.findCounter(counter); if (hadoopCounter != null) { return (int) hadoopCounter.getValue(); } else { return 0; } } }
From source file:com.yahoo.glimmer.indexing.generator.DocumentMapperTest.java
License:Open Source License
@SuppressWarnings("unchecked") @Before//from w w w .jav a 2 s . co m public void before() { context = new Mockery(); context.setImposteriser(ClassImposteriser.INSTANCE); mapperContext = context.mock(Context.class, "mapperContext"); mapperConf = new Configuration(); doc = context.mock(RDFDocument.class, "doc"); counters = new Counters(); }
From source file:crunch.MaxTemperature.java
License:Apache License
@Test public void parsesMalformedTemperature() throws IOException, InterruptedException {//from w ww . jav a 2 s.c om Text value = new Text("0335999999433181957042302005+37950+139117SAO +0004" + // Year ^^^^ "RJSN V02011359003150070356999999433201957010100005+353"); // Temperature ^^^^^ Counters counters = new Counters(); new MapDriver<LongWritable, Text, Text, IntWritable>() .withMapper(new MaxTemperatureMapper()) .withInputValue(value) .withCounters(counters) .runTest(); Counter c = counters.findCounter(MaxTemperatureMapper.Temperature.MALFORMED); assertThat(c.getValue(), is(1L)); }
From source file:org.apache.crunch.impl.mem.CountersWrapper.java
License:Apache License
CountersWrapper() {
this.active = new Counters();
allCounters.add(active);
}
From source file:org.apache.crunch.impl.mem.CountersWrapper.java
License:Apache License
@Override public Counter findCounter(String groupName, String counterName) { Map<String, Counter> c = lookupCache.get(groupName); if (c == null) { c = Maps.newHashMap();// ww w .j av a2 s . com lookupCache.put(groupName, c); } Counter counter = c.get(counterName); if (counter == null) { try { counter = active.findCounter(groupName, counterName); } catch (Exception e) { // Recover from this by creating a new active instance active = new Counters(); allCounters.add(active); counter = active.findCounter(groupName, counterName); } c.put(counterName, counter); } return counter; }