Example usage for org.apache.hadoop.mapreduce.jobhistory JobHistoryParser parse

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce.jobhistory JobHistoryParser parse.

Prototype

@Private
public synchronized JobInfo parse(EventReader reader) throws IOException 

Source Link

Document

Only used for unit tests.

Usage

From source file:com.netflix.bdp.inviso.history.TraceJobHistoryLoader.java

License:Apache License

public static void main(String[] args) throws Exception {
    FileSystem fs = FileSystem.newInstanceLocal(new Configuration());

    JobHistoryParser parser = new JobHistoryParser(fs, "/tmp/job_1405808155709_124465.history");

    //JobInfo jobInfo = parser.parse();
    TraceJobHistoryLoader loader = new TraceJobHistoryLoader(new PropertiesConfiguration());
    parser.parse(loader);

    ObjectMapper mapper = new ObjectMapper();
    mapper.configure(Feature.INDENT_OUTPUT, true);
    mapper.writeValue(new File("/tmp/mr2-hist.json"), loader.getJob());
}

---

From source file:com.netflix.bdp.inviso.history.TraceService.java

License:Apache License

/**
 * Returns a json object representing the job history.
 *
 * @param jobId/*from  ww  w  .  ja  v  a  2s .c  o  m*/
 * @param path Use the given path as opposed to the history locator
 * @param summary Return just the top level details of the job
 * @param counters Include counters
 * @return Json string
 * @throws Exception
 */
@Path("load/{jobId}")
@GET
@Produces("application/json")
public String trace(@PathParam("jobId") final String jobId, @QueryParam("path") final String path,
        @QueryParam("summary") boolean summary, @QueryParam("counters") @DefaultValue("true") boolean counters)
        throws Exception {

    Pair<org.apache.hadoop.fs.Path, org.apache.hadoop.fs.Path> historyPath;

    if (path != null) {
        historyPath = new ImmutablePair<>(null, new org.apache.hadoop.fs.Path(path));
    } else {
        historyPath = historyLocator.locate(jobId);
    }

    if (historyPath == null) {
        throw new WebApplicationException(404);
    }

    TraceJobHistoryLoader loader = new TraceJobHistoryLoader(properties);

    FileSystem fs = FileSystem.get(historyPath.getRight().toUri(), config);
    CompressionCodec codec = new CompressionCodecFactory(config).getCodec(historyPath.getRight());

    FSDataInputStream fin = fs.open(historyPath.getRight());

    if (codec != null) {
        fin = new FSDataInputStream(new WrappedCompressionInputStream(codec.createInputStream(fin)));
    }

    JobHistoryParser parser = new JobHistoryParser(fin);
    parser.parse(loader);

    String[] ignore = { "counters" };

    ObjectMapper mapper = new ObjectMapper();
    SimpleModule module = new SimpleModule("MyModule", new Version(1, 0, 0, null));

    //Job
    JavaType jobMapType = MapLikeType.construct(Job.class, SimpleType.construct(String.class),
            SimpleType.construct(Object.class));
    module.addSerializer(Job.class, MapSerializer.construct(ignore, jobMapType, false, null, null, null, null));

    //Task
    JavaType taskMapType = MapLikeType.construct(Task.class, SimpleType.construct(String.class),
            SimpleType.construct(Object.class));
    module.addSerializer(Task.class,
            MapSerializer.construct(ignore, taskMapType, false, null, null, null, null));

    //Attempt
    JavaType attemptMapType = MapLikeType.construct(TaskAttempt.class, SimpleType.construct(String.class),
            SimpleType.construct(Object.class));
    module.addSerializer(TaskAttempt.class,
            MapSerializer.construct(ignore, attemptMapType, false, null, null, null, null));

    if (!counters) {
        mapper.registerModule(module);
    }

    if (summary) {
        loader.getJob().clearTasks();
    }

    return mapper.writeValueAsString(loader.getJob());
}

---