Example usage for org.apache.hadoop.mapreduce.jobhistory JobHistoryParser JobHistoryParser

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce.jobhistory JobHistoryParser JobHistoryParser.

Prototype

public JobHistoryParser(FSDataInputStream in) 

Source Link

Document

Create the history parser based on the input stream

Usage

From source file:com.netflix.bdp.inviso.history.TraceService.java

License:Apache License

/**
 * Returns a json object representing the job history.
 *
 * @param jobId/*  w  ww.  ja  va 2  s.c o  m*/
 * @param path Use the given path as opposed to the history locator
 * @param summary Return just the top level details of the job
 * @param counters Include counters
 * @return Json string
 * @throws Exception
 */
@Path("load/{jobId}")
@GET
@Produces("application/json")
public String trace(@PathParam("jobId") final String jobId, @QueryParam("path") final String path,
        @QueryParam("summary") boolean summary, @QueryParam("counters") @DefaultValue("true") boolean counters)
        throws Exception {

    Pair<org.apache.hadoop.fs.Path, org.apache.hadoop.fs.Path> historyPath;

    if (path != null) {
        historyPath = new ImmutablePair<>(null, new org.apache.hadoop.fs.Path(path));
    } else {
        historyPath = historyLocator.locate(jobId);
    }

    if (historyPath == null) {
        throw new WebApplicationException(404);
    }

    TraceJobHistoryLoader loader = new TraceJobHistoryLoader(properties);

    FileSystem fs = FileSystem.get(historyPath.getRight().toUri(), config);
    CompressionCodec codec = new CompressionCodecFactory(config).getCodec(historyPath.getRight());

    FSDataInputStream fin = fs.open(historyPath.getRight());

    if (codec != null) {
        fin = new FSDataInputStream(new WrappedCompressionInputStream(codec.createInputStream(fin)));
    }

    JobHistoryParser parser = new JobHistoryParser(fin);
    parser.parse(loader);

    String[] ignore = { "counters" };

    ObjectMapper mapper = new ObjectMapper();
    SimpleModule module = new SimpleModule("MyModule", new Version(1, 0, 0, null));

    //Job
    JavaType jobMapType = MapLikeType.construct(Job.class, SimpleType.construct(String.class),
            SimpleType.construct(Object.class));
    module.addSerializer(Job.class, MapSerializer.construct(ignore, jobMapType, false, null, null, null, null));

    //Task
    JavaType taskMapType = MapLikeType.construct(Task.class, SimpleType.construct(String.class),
            SimpleType.construct(Object.class));
    module.addSerializer(Task.class,
            MapSerializer.construct(ignore, taskMapType, false, null, null, null, null));

    //Attempt
    JavaType attemptMapType = MapLikeType.construct(TaskAttempt.class, SimpleType.construct(String.class),
            SimpleType.construct(Object.class));
    module.addSerializer(TaskAttempt.class,
            MapSerializer.construct(ignore, attemptMapType, false, null, null, null, null));

    if (!counters) {
        mapper.registerModule(module);
    }

    if (summary) {
        loader.getJob().clearTasks();
    }

    return mapper.writeValueAsString(loader.getJob());
}

---