Java tutorial
/* * Copyright 2015 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of * the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations under * the License. */ package co.cask.cdap.app.mapreduce; import co.cask.cdap.api.metrics.MetricStore; import co.cask.cdap.api.metrics.MetricType; import co.cask.cdap.api.metrics.MetricValues; import co.cask.cdap.app.metrics.MapReduceMetrics; import co.cask.cdap.common.conf.CConfiguration; import co.cask.cdap.common.conf.Constants; import co.cask.cdap.data2.datafabric.dataset.service.DatasetService; import co.cask.cdap.data2.datafabric.dataset.service.executor.DatasetOpExecutor; import co.cask.cdap.internal.guice.AppFabricTestModule; import co.cask.cdap.proto.Id; import co.cask.cdap.proto.MRJobInfo; import co.cask.cdap.proto.MRTaskInfo; import co.cask.cdap.proto.ProgramType; import co.cask.tephra.TransactionManager; import com.google.common.collect.ImmutableMap; import com.google.inject.Guice; import com.google.inject.Injector; import org.apache.hadoop.mapreduce.TaskCounter; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; import java.util.List; import java.util.Map; public class LocalMRJobInfoFetcherTest { private static Injector injector; private static MetricStore metricStore; @BeforeClass public static void beforeClass() throws Exception { CConfiguration conf = CConfiguration.create(); conf.set(Constants.AppFabric.OUTPUT_DIR, System.getProperty("java.io.tmpdir")); conf.set(Constants.AppFabric.TEMP_DIR, System.getProperty("java.io.tmpdir")); injector = startMetricsService(conf); metricStore = injector.getInstance(MetricStore.class); } public static Injector startMetricsService(CConfiguration conf) { Injector injector = Guice.createInjector(new AppFabricTestModule(conf)); injector.getInstance(TransactionManager.class).startAndWait(); injector.getInstance(DatasetOpExecutor.class).startAndWait(); injector.getInstance(DatasetService.class).startAndWait(); return injector; } @Test public void testGetMRJobInfo() throws Exception { Id.Program programId = Id.Program.from("fooNamespace", "testApp", ProgramType.MAPREDUCE, "fooMapReduce"); Id.Run runId = new Id.Run(programId, "run10878"); Map<String, String> runContext = ImmutableMap.of(Constants.Metrics.Tag.NAMESPACE, programId.getNamespaceId(), Constants.Metrics.Tag.APP, programId.getApplicationId(), Constants.Metrics.Tag.MAPREDUCE, programId.getId(), Constants.Metrics.Tag.RUN_ID, runId.getId()); Map<String, String> mapTypeContext = addToContext(runContext, Constants.Metrics.Tag.MR_TASK_TYPE, MapReduceMetrics.TaskType.Mapper.getId()); Map<String, String> reduceTypeContext = addToContext(runContext, Constants.Metrics.Tag.MR_TASK_TYPE, MapReduceMetrics.TaskType.Reducer.getId()); String mapTask1Name = "task_m_01"; Map<String, String> mapTask1Context = addToContext(mapTypeContext, Constants.Metrics.Tag.INSTANCE_ID, mapTask1Name); String mapTask2Name = "task_m_02"; Map<String, String> mapTask2Context = addToContext(mapTypeContext, Constants.Metrics.Tag.INSTANCE_ID, mapTask2Name); String reduceTaskName = "task_r_01"; Map<String, String> reduceTaskContext = addToContext(reduceTypeContext, Constants.Metrics.Tag.INSTANCE_ID, reduceTaskName); // Imitate a MapReduce Job running (gauge mapper and reducer metrics) long measureTime = System.currentTimeMillis() / 1000; gauge(mapTypeContext, MapReduceMetrics.METRIC_COMPLETION, measureTime, 76L); gauge(reduceTypeContext, MapReduceMetrics.METRIC_COMPLETION, measureTime, 52L); gauge(mapTask1Context, MapReduceMetrics.METRIC_TASK_COMPLETION, measureTime, 100L); gauge(mapTask1Context, MapReduceMetrics.METRIC_TASK_INPUT_RECORDS, measureTime, 32L); gauge(mapTask1Context, MapReduceMetrics.METRIC_TASK_OUTPUT_RECORDS, measureTime, 320L); gauge(mapTask2Context, MapReduceMetrics.METRIC_TASK_COMPLETION, measureTime, 12L); gauge(mapTask2Context, MapReduceMetrics.METRIC_TASK_INPUT_RECORDS, measureTime, 6L); gauge(mapTask2Context, MapReduceMetrics.METRIC_TASK_OUTPUT_RECORDS, measureTime, 60L); // gauge job-level counters for mappers gauge(mapTypeContext, MapReduceMetrics.METRIC_INPUT_RECORDS, measureTime, 38L); gauge(mapTypeContext, MapReduceMetrics.METRIC_OUTPUT_RECORDS, measureTime, 380L); gauge(reduceTaskContext, MapReduceMetrics.METRIC_TASK_COMPLETION, measureTime, 76L); gauge(reduceTaskContext, MapReduceMetrics.METRIC_TASK_INPUT_RECORDS, measureTime, 320L); gauge(reduceTaskContext, MapReduceMetrics.METRIC_TASK_OUTPUT_RECORDS, measureTime, 1L); // gauge job-level counters for reducers gauge(reduceTypeContext, MapReduceMetrics.METRIC_INPUT_RECORDS, measureTime, 320L); gauge(reduceTypeContext, MapReduceMetrics.METRIC_OUTPUT_RECORDS, measureTime, 1L); LocalMRJobInfoFetcher localMRJobInfoFetcher = injector.getInstance(LocalMRJobInfoFetcher.class); MRJobInfo mrJobInfo = localMRJobInfoFetcher.getMRJobInfo(runId); // Incomplete because MapReduceMetricsInfo does not provide task-level state and start/end times. Assert.assertFalse(mrJobInfo.isComplete()); // Check job-level counters Map<String, Long> jobCounters = mrJobInfo.getCounters(); Assert.assertEquals((Long) 38L, jobCounters.get(TaskCounter.MAP_INPUT_RECORDS.name())); Assert.assertEquals((Long) 380L, jobCounters.get(TaskCounter.MAP_OUTPUT_RECORDS.name())); Assert.assertEquals((Long) 320L, jobCounters.get(TaskCounter.REDUCE_INPUT_RECORDS.name())); Assert.assertEquals((Long) 1L, jobCounters.get(TaskCounter.REDUCE_OUTPUT_RECORDS.name())); // Ensure all tasks show up List<MRTaskInfo> mapTasks = mrJobInfo.getMapTasks(); List<MRTaskInfo> reduceTasks = mrJobInfo.getReduceTasks(); Assert.assertEquals(2, mapTasks.size()); Assert.assertEquals(1, reduceTasks.size()); MRTaskInfo mapTask1 = findByTaskId(mapTasks, mapTask1Name); MRTaskInfo mapTask2 = findByTaskId(mapTasks, mapTask2Name); MRTaskInfo reduceTask = findByTaskId(reduceTasks, reduceTaskName); // Check task-level counters Map<String, Long> mapTask1Counters = mapTask1.getCounters(); Assert.assertEquals((Long) 32L, mapTask1Counters.get(TaskCounter.MAP_INPUT_RECORDS.name())); Assert.assertEquals((Long) 320L, mapTask1Counters.get(TaskCounter.MAP_OUTPUT_RECORDS.name())); Map<String, Long> mapTask2Counters = mapTask2.getCounters(); Assert.assertEquals((Long) 6L, mapTask2Counters.get(TaskCounter.MAP_INPUT_RECORDS.name())); Assert.assertEquals((Long) 60L, mapTask2Counters.get(TaskCounter.MAP_OUTPUT_RECORDS.name())); Map<String, Long> reduceTaskCounters = reduceTask.getCounters(); Assert.assertEquals((Long) 320L, reduceTaskCounters.get(TaskCounter.REDUCE_INPUT_RECORDS.name())); Assert.assertEquals((Long) 1L, reduceTaskCounters.get(TaskCounter.REDUCE_OUTPUT_RECORDS.name())); // Checking progress float permittedProgressDelta = 0.01F; Assert.assertEquals(0.76F, mrJobInfo.getMapProgress(), permittedProgressDelta); Assert.assertEquals(0.52F, mrJobInfo.getReduceProgress(), permittedProgressDelta); Assert.assertEquals(1.0F, mapTask1.getProgress(), permittedProgressDelta); Assert.assertEquals(0.12F, mapTask2.getProgress(), permittedProgressDelta); Assert.assertEquals(0.76F, reduceTask.getProgress(), permittedProgressDelta); } private void gauge(Map<String, String> context, String metric, long timestamp, Long value) throws Exception { metricStore.add(new MetricValues(context, metric, timestamp, value, MetricType.GAUGE)); } // Returned copied map, with new key-value pair. private Map<String, String> addToContext(Map<String, String> context, String key, String value) { return ImmutableMap.<String, String>builder().putAll(context).put(key, value).build(); } private MRTaskInfo findByTaskId(List<MRTaskInfo> taskInfos, String taskId) { for (MRTaskInfo taskInfo : taskInfos) { if (taskInfo.getTaskId().equals(taskId)) { return taskInfo; } } throw new IllegalArgumentException( String.format("TaskId: %s not found in list of TaskInfos: %s", taskId, taskInfos)); } }