Example usage for org.apache.hadoop.mapreduce TaskType MAP

List of usage examples for org.apache.hadoop.mapreduce TaskType MAP

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.mapreduce.*
  5. TaskType
  6. MAP

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce TaskType MAP.

Prototype

TaskType MAP

To view the source code for org.apache.hadoop.mapreduce TaskType MAP.

Click Source Link

Usage

From source file:com.moz.fiji.mapreduce.output.TestFijiHFileOutputFormat.java

License:Apache License

@Test
public void testMaxHFileSizeNewRow() throws Exception {
    final HFileKeyValue entry1 = entry("row-key1", mDefaultLGId, "a", 1L, makeBytes(0, 1024));
    final HFileKeyValue entry2 = entry("row-key2", mDefaultLGId, "b", 1L, makeBytes(0, 1024));

    mConf.setInt(FijiHFileOutputFormat.CONF_HREGION_MAX_FILESIZE, entry1.getLength() + 1);

    final TaskAttemptID taskAttemptId = FijiMRPlatformBridge.get().newTaskAttemptID("jobTracker_jtPort", 314,
            TaskType.MAP, 159, 2);
    final TaskAttemptContext context = FijiMRPlatformBridge.get().newTaskAttemptContext(mConf, taskAttemptId);
    final Path outputDir = mFormat.getDefaultWorkFile(context, FijiHFileOutputFormat.OUTPUT_EXTENSION);
    final FileSystem fs = outputDir.getFileSystem(mConf);

    final RecordWriter<HFileKeyValue, NullWritable> writer = mFormat.getRecordWriter(context);
    writer.write(entry1, NW);/*  w w w  .  j a v  a 2s .  c  o  m*/
    writer.write(entry2, NW);
    writer.close(context);

    final Path defaultDir = new Path(outputDir, mDefaultLGId.toString());
    assertTrue(fs.exists(defaultDir));

    final Path inMemoryDir = new Path(outputDir, mInMemoryLGId.toString());
    assertFalse(fs.exists(inMemoryDir));

    assertHFileContent(new Path(defaultDir, "00000"), entry1.getKeyValue());
    assertHFileContent(new Path(defaultDir, "00001"), entry2.getKeyValue());
    assertFalse(fs.exists(new Path(defaultDir, "00002")));

    mFormat.getOutputCommitter(context).commitTask(context);
}

From source file:com.moz.fiji.mapreduce.output.TestFijiHFileOutputFormat.java

License:Apache License

@Test
public void testMultipleLayouts() throws Exception {
    final TaskAttemptID taskAttemptId = FijiMRPlatformBridge.get().newTaskAttemptID("jobTracker_jtPort", 314,
            TaskType.MAP, 159, 2);
    final TaskAttemptContext context = FijiMRPlatformBridge.get().newTaskAttemptContext(mConf, taskAttemptId);
    final Path outputDir = mFormat.getDefaultWorkFile(context, FijiHFileOutputFormat.OUTPUT_EXTENSION);
    final FileSystem fs = outputDir.getFileSystem(mConf);

    final RecordWriter<HFileKeyValue, NullWritable> writer = mFormat.getRecordWriter(context);

    final HFileKeyValue defaultEntry = entry("row-key", mDefaultLGId, "a", 1L, makeBytes(0, 1024));
    writer.write(defaultEntry, NW);//from ww w  . j a v  a2  s.com
    final HFileKeyValue inMemoryEntry = entry("row-key", mInMemoryLGId, "a", 1L, makeBytes(2, 1024));
    writer.write(inMemoryEntry, NW);

    try {
        // Test with an invalid locality group ID:
        final ColumnId invalid = new ColumnId(1234);
        assertTrue(!mLayout.getLocalityGroupIdNameMap().containsKey(invalid));
        writer.write(entry("row-key", invalid, "a", 1L, HConstants.EMPTY_BYTE_ARRAY), NW);
        fail("Output format did not fail on unknown locality group IDs.");
    } catch (IllegalArgumentException iae) {
        LOG.info("Expected error: " + iae);
    }

    writer.close(context);

    final Path defaultDir = new Path(outputDir, mDefaultLGId.toString());
    assertTrue(fs.exists(defaultDir));

    final Path inMemoryDir = new Path(outputDir, mInMemoryLGId.toString());
    assertTrue(fs.exists(inMemoryDir));

    assertHFileContent(new Path(defaultDir, "00000"), defaultEntry.getKeyValue());
    assertHFileContent(new Path(inMemoryDir, "00000"), inMemoryEntry.getKeyValue());

    mFormat.getOutputCommitter(context).commitTask(context);
}

From source file:com.moz.fiji.mapreduce.output.TestFijiHFileOutputFormat.java

License:Apache License

@Test
public void testTombstonesInHFile() throws Exception {
    final HFileKeyValue put = entry("row-key1", mDefaultLGId, "a", 1L, makeBytes(0, 1024));
    final HFileKeyValue deleteCell = entry("row-key2", mDefaultLGId, "a", 1L, HFileKeyValue.Type.DeleteCell);
    final HFileKeyValue deleteColumn = entry("row-key3", mDefaultLGId, "a", 1L,
            HFileKeyValue.Type.DeleteColumn);
    final HFileKeyValue deleteFamily = entry("row-key4", mDefaultLGId, "a", 1L,
            HFileKeyValue.Type.DeleteFamily);

    final TaskAttemptID taskAttemptId = FijiMRPlatformBridge.get().newTaskAttemptID("jobTracker_jtPort", 314,
            TaskType.MAP, 159, 2);
    final TaskAttemptContext context = FijiMRPlatformBridge.get().newTaskAttemptContext(mConf, taskAttemptId);
    final Path outputDir = mFormat.getDefaultWorkFile(context, FijiHFileOutputFormat.OUTPUT_EXTENSION);
    final FileSystem fs = outputDir.getFileSystem(mConf);

    final RecordWriter<HFileKeyValue, NullWritable> writer = mFormat.getRecordWriter(context);
    writer.write(put, NW);/*from   ww  w.j a v  a2 s . c  o  m*/
    writer.write(deleteCell, NW);
    writer.write(deleteColumn, NW);
    writer.write(deleteFamily, NW);
    writer.close(context);

    final Path defaultDir = new Path(outputDir, mDefaultLGId.toString());
    assertTrue(fs.exists(defaultDir));

    assertHFileContent(new Path(defaultDir, "00000"), put.getKeyValue(), deleteCell.getKeyValue(),
            deleteColumn.getKeyValue(), deleteFamily.getKeyValue());
    assertFalse(fs.exists(new Path(defaultDir, "00001")));

    mFormat.getOutputCommitter(context).commitTask(context);
}

From source file:com.twitter.hraven.hadoopJobMonitor.AppStatusChecker.java

License:Apache License

@Override
public void run() {
    // 0. set thread name
    setThreadName();/*from w  w  w.  ja v a 2s . co  m*/
    if (isInWhitelist(appReport)) {
        LOG.warn("Skipping whitelisted app " + appId);
        return;
    }
    LOG.debug("Running " + Thread.currentThread().getName());
    try {
        try {
            init();
        } catch (Exception e) {
            LOG.error("Skipping app " + appId + " due to initialization error: " + e.getMessage());
            return;
        }
        if (!checkApp())
            return;
        loadClientService();
        checkTasks(TaskType.MAP);
        checkTasks(TaskType.REDUCE);
        clientCache.stopClient(jobId);
    } catch (YarnException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    } finally {
        appCheckerProgress.finished();
    }
}

From source file:com.twitter.hraven.hadoopJobMonitor.AppStatusCheckerTest.java

License:Apache License

public AppStatusCheckerTest() throws ConfigurationAccessException, RestException, SAXException, IOException,
        ParserConfigurationException, YarnException {
    appId = new MyApplicationId();
    appId.setId(oldJobId.getId());//w w  w.  j av a  2  s. c o  m
    appId.setClusterTimestamp(Long.parseLong(oldJobId.getJtIdentifier()));

    taskId = new TaskID(oldJobId, TaskType.MAP, 0);
    taskAttemptId = new TaskAttemptID(taskId, 0);

    vConf.setFloat(HadoopJobMonitorConfiguration.TASK_PROGRESS_THRESHOLD, 0.2f);
    vConf.getInt(HadoopJobMonitorConfiguration.MAX_CACHED_TASK_PROGRESSES, 10);
    vConf.getInt(HadoopJobMonitorConfiguration.MAX_CACHED_APP_CONFS, 10);
    AppConfCache.init(vConf);
    ProgressCache.init(vConf);
    HadoopJobMonitorMetrics.initSingleton(vConf);
    taskProgressCache = ProgressCache.getTaskProgressCache();
    attemptProgressCache = ProgressCache.getAttemptProgressCache();

    when(clientCache.getClient(any(JobID.class))).thenReturn(clientService);
    appReport = mock(ApplicationReport.class);
    when(appReport.getApplicationId()).thenReturn(appId);
    appStatusChecker = new AppStatusChecker(vConf, appReport, clientCache, rm, new AppCheckerProgress() {
        @Override
        public void finished() {
        }
    });

    mockStatic(RestClient.class);
    restClient = mock(RestClient.class);
    when(RestClient.getInstance()).thenReturn(restClient);
}

From source file:com.twitter.hraven.hadoopJobMonitor.AppStatusCheckerTest.java

License:Apache License

@Test(timeout = 30000)
public void testMapTasks() throws Exception {
    killCounter = 0;/*from  ww  w  . j  a  v  a  2 s .c o m*/
    final String pName = HadoopJobMonitorConfiguration.MAP_MAX_RUNTIME_MIN;
    final boolean passCheck = true, killed = true, dryRun = true, enforce = true;
    testTask(TaskType.MAP, pName, 5, 10, enforce, !dryRun, TIPStatus.RUNNING, passCheck, !killed);
    testTask(TaskType.MAP, pName, 15, 10, enforce, !dryRun, TIPStatus.FAILED, passCheck, !killed);
    testTask(TaskType.MAP, pName, 15, 10, enforce, !dryRun, TIPStatus.RUNNING, !passCheck, killed);
    testTask(TaskType.MAP, pName, 15, 10, !enforce, !dryRun, TIPStatus.RUNNING, !passCheck, !killed);
    testTask(TaskType.MAP, pName, 15, 10, !enforce, dryRun, TIPStatus.RUNNING, !passCheck, !killed);
    testTask(TaskType.MAP, pName, 15, 10, enforce, dryRun, TIPStatus.RUNNING, !passCheck, !killed);
}

From source file:com.twitter.hraven.hadoopJobMonitor.AppStatusCheckerTest.java

License:Apache License

@Test
public void testMapProgress() throws Exception {
    testProgress(TaskType.MAP, HadoopJobMonitorConfiguration.MAP_MAX_RUNTIME_MIN);
}

From source file:com.twitter.hraven.hadoopJobMonitor.notification.Notifier.java

License:Apache License

public static String tooLongTaskAttempt(AppConfiguraiton appConf, ApplicationReport appReport,
        TaskReport taskReport, TaskType taskType, TaskAttemptID taskAttemptId, long duration, long max) {
    metrics.badBehavedTasks.incr();// ww  w. j  a  v  a2 s.c om
    if (taskType.equals(TaskType.MAP))
        metrics.badBehavedMappers.incr();
    else if (taskType.equals(TaskType.REDUCE))
        metrics.badBehavedReducers.incr();
    String body = longTaskMsg(appReport, taskReport, taskAttemptId, duration, max);
    if (DRYRUN)
        body += "\n" + DRYRUN_NOTE;
    if (appConf.getNotifyUser())
        Mail.send(SUBJECT, body, getUserEmail(appConf, appReport));
    else
        Mail.send(SUBJECT, body);
    return body;
}

From source file:com.twitter.hraven.hadoopJobMonitor.notification.Notifier.java

License:Apache License

public static String tooLittleProgress(AppConfiguraiton appConf, ApplicationReport appReport,
        TaskReport taskReport, TaskType taskType, TaskAttemptID taskAttemptId, float progress,
        float progressThreshold, long max) {
    metrics.badBehavedTasks.incr();/*from  w w w.  j av a 2 s  . com*/
    if (taskType.equals(TaskType.MAP))
        metrics.badBehavedMappers.incr();
    else if (taskType.equals(TaskType.REDUCE))
        metrics.badBehavedReducers.incr();
    String body = littleProgressMsg(appReport, taskReport, taskAttemptId, progress, progressThreshold, max);
    if (DRYRUN)
        body += "\n" + DRYRUN_NOTE;
    if (appConf.getNotifyUser())
        Mail.send(SUBJECT, body, getUserEmail(appConf, appReport));
    else
        Mail.send(SUBJECT, body);
    return body;
}

From source file:com.twitter.hraven.hadoopJobMonitor.policy.DefaultPolicy.java

License:Apache License

/**
 * Check the status of an attempt of a task. This method is invoked only if
 * {@link checkTask} returns false./*  w w w.  j  a v a2 s . c  o m*/
 * 
 * @param taskType
 * @param taskReport
 * @param appConf
 * @param taskAttemptId
 * @param taskAttemptXml The task attempt detail in xml format
 * @return true if task attempt is well-behaved
 */
@Override
public String checkTaskAttempt(ApplicationReport appReport, TaskType taskType, TaskReport taskReport,
        AppConfiguraiton appConf, TaskAttemptID taskAttemptId, Document taskAttemptXml, long currTime) {
    long maxRunTimeMs = appConf.getMaxTaskLenMin(taskType) * MINtoMS;
    // Iterating through the nodes and extracting the data.
    NodeList nodeList = taskAttemptXml.getDocumentElement().getChildNodes();
    int checkedNodes = 0, maxRequiredNodes = 2;//elapsedtime and progress
    for (int i = 0; i < nodeList.getLength(); i++) {
        Node node = nodeList.item(i);
        if (node instanceof Element) {
            String name = node.getNodeName();
            //check duration
            if (name.equals("elapsedTime")) {
                checkedNodes++;
                String timeStr = node.getTextContent();
                long timeMs = Long.parseLong(timeStr);
                LOG.debug(name + " = " + timeMs + " ? " + maxRunTimeMs);
                boolean badTask = (timeMs > maxRunTimeMs);
                if (badTask) {
                    String msg = Notifier.tooLongTaskAttempt(appConf, appReport, taskReport, taskType,
                            taskAttemptId, timeMs, maxRunTimeMs);
                    if (taskType == TaskType.MAP
                            && appConf.isEnforced(HadoopJobMonitorConfiguration.MAP_MAX_RUNTIME_MIN))
                        return msg;
                    if (taskType == TaskType.REDUCE
                            && appConf.isEnforced(HadoopJobMonitorConfiguration.REDUCE_MAX_RUNTIME_MIN))
                        return msg;
                }
                if (checkedNodes >= maxRequiredNodes)
                    break;
                continue;
            }
            //check progress
            if (name.equals("progress")) {
                checkedNodes++;
                String progressStr = node.getTextContent();
                float progress = Float.parseFloat(progressStr);
                boolean badTask = !checkProgress(progress, appConf.getProgressThreshold(), maxRunTimeMs,
                        taskAttemptId, currTime);
                if (badTask) {
                    String msg = Notifier.tooLittleProgress(appConf, appReport, taskReport, taskType,
                            taskAttemptId, progress, appConf.getProgressThreshold(), maxRunTimeMs);
                    if (taskType == TaskType.MAP
                            && appConf.isEnforced(HadoopJobMonitorConfiguration.MAP_MAX_RUNTIME_MIN))
                        return msg;
                    if (taskType == TaskType.REDUCE
                            && appConf.isEnforced(HadoopJobMonitorConfiguration.REDUCE_MAX_RUNTIME_MIN))
                        return msg;
                }
                if (checkedNodes >= maxRequiredNodes)
                    break;
                continue;
            }
        }
    }
    return null;
}