Example usage for org.apache.hadoop.yarn.conf YarnConfiguration NM_REMOTE_APP_LOG_DIR

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.conf YarnConfiguration NM_REMOTE_APP_LOG_DIR.

Prototype

String NM_REMOTE_APP_LOG_DIR

To view the source code for org.apache.hadoop.yarn.conf YarnConfiguration NM_REMOTE_APP_LOG_DIR.

Click Source Link

Document

Where to aggregate logs to.

Usage

From source file:AggregatedLogsPurger.java

License:Apache License

@Override
public int run(String[] args) throws Exception {
    this.conf = getConf();
    this.deleteOlderThanDays = getConf().getInt("deleteOlderThan", 0);
    Preconditions.checkArgument(deleteOlderThanDays > 1,
            "Usage: yarn jar " + "./target/yarn-logs-purger-1.0-SNAPSHOT.jar -DdeleteOlderThan=300 "
                    + "-DdeleteFiles=true.  Please provide valid argument for deleteOlderThanDays. It has to "
                    + "be > 0");
    this.shouldDelete = getConf().getBoolean("deleteFiles", false);

    this.suffix = LogAggregationUtils.getRemoteNodeLogDirSuffix(conf);
    this.rootLogDir = new Path(
            conf.get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR));
    return (purge()) ? 0 : -1;
}

---

From source file:com.netflix.bdp.inviso.log.LogService.java

License:Apache License

@javax.ws.rs.Path("load/{owner}/{appId}/{containerId}/{nodeId}")
@GET//w w  w . j a  v a2 s  . c o  m
@Produces("text/plain")
public Response log(@PathParam("owner") String owner, @PathParam("appId") String appId,
        @PathParam("containerId") String containerId, @PathParam("nodeId") String nodeId,
        @QueryParam("fs") String fs, @QueryParam("root") String root) throws IOException {

    Configuration conf = new Configuration();

    if (fs != null) {
        conf.set("fs.default.name", fs);
    }

    Path logRoot = new Path(
            conf.get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR));

    if (root != null) {
        logRoot = new Path(root);
    }

    Path logPath = LogAggregationUtils.getRemoteNodeLogFileForApp(logRoot,
            ConverterUtils.toApplicationId(appId), owner, ConverterUtils.toNodeId(nodeId),
            LogAggregationUtils.getRemoteNodeLogDirSuffix(conf));

    AggregatedLogFormat.LogReader reader = new AggregatedLogFormat.LogReader(conf, logPath);

    LogKey key = new LogKey();

    DataInputStream in = reader.next(key);

    while (in != null && !key.toString().equals(containerId)) {
        key = new LogKey();
        in = reader.next(key);
    }

    if (in == null) {
        throw new WebApplicationException(404);
    }

    final DataInputStream fin = in;

    StreamingOutput stream = new StreamingOutput() {
        @Override
        public void write(OutputStream os) throws IOException, WebApplicationException {
            PrintStream out = new PrintStream(os);

            while (true) {
                try {
                    LogReader.readAContainerLogsForALogType(fin, out);
                    out.flush();
                } catch (EOFException e) {
                    break;
                }
            }
        }
    };

    return Response.ok(stream).build();
}

---

From source file:org.apache.oozie.tools.diag.OozieLauncherLogFetcher.java

License:Apache License

public int dumpAllContainersLogs(ApplicationId appId, String appOwner, PrintStream out) throws IOException {
    Path remoteRootLogDir = new Path(hadoopConfig.get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
            YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR));
    String logDirSuffix = getRemoteNodeLogDirSuffix(hadoopConfig);
    Path remoteAppLogDir = getRemoteAppLogDir(remoteRootLogDir, appId, appOwner, logDirSuffix);

    RemoteIterator nodeFiles;//from w ww  . j  av  a 2  s  . com
    try {
        Path qualifiedLogDir = FileContext.getFileContext(hadoopConfig).makeQualified(remoteAppLogDir);
        nodeFiles = FileContext.getFileContext(qualifiedLogDir.toUri(), hadoopConfig)
                .listStatus(remoteAppLogDir);
    } catch (FileNotFoundException fileNotFoundException) {
        logDirNotExist(remoteAppLogDir.toString());
        return -1;
    }

    boolean foundAnyLogs = false;

    while (true) {
        FileStatus thisNodeFile;
        do {
            if (!nodeFiles.hasNext()) {
                if (!foundAnyLogs) {
                    emptyLogDir(remoteAppLogDir.toString());
                    return -1;
                }

                return 0;
            }

            thisNodeFile = (FileStatus) nodeFiles.next();
        } while (thisNodeFile.getPath().getName().endsWith(TMP_FILE_SUFFIX));

        AggregatedLogFormat.LogReader reader = new AggregatedLogFormat.LogReader(hadoopConfig,
                thisNodeFile.getPath());

        try {
            AggregatedLogFormat.LogKey key = new AggregatedLogFormat.LogKey();
            DataInputStream valueStream = reader.next(key);

            while (valueStream != null) {
                String containerString = "\n\nContainer: " + key + " on " + thisNodeFile.getPath().getName();
                out.println(containerString);
                out.println(StringUtils.repeat("=", containerString.length()));

                while (true) {
                    try {
                        AggregatedLogFormat.LogReader.readAContainerLogsForALogType(valueStream, out,
                                thisNodeFile.getModificationTime());
                        foundAnyLogs = true;
                    } catch (EOFException eofException) {
                        key = new AggregatedLogFormat.LogKey();
                        valueStream = reader.next(key);
                        break;
                    }
                }
            }
        } finally {
            reader.close();
        }
    }
}

---