Example usage for org.apache.hadoop.yarn.conf YarnConfiguration NM_LOG_DIRS

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.conf YarnConfiguration NM_LOG_DIRS.

Prototype

String NM_LOG_DIRS

To view the source code for org.apache.hadoop.yarn.conf YarnConfiguration NM_LOG_DIRS.

Click Source Link

Document

Where to store container logs.

Usage

From source file:com.datatorrent.stram.util.ConfigUtils.java

License:Apache License

public static String getRawContainerLogsUrl(YarnConfiguration conf, String nodeHttpAddress, String appId,
        String containerId) {//from w ww . j av  a2  s. co m
    String logDirs = conf.get(YarnConfiguration.NM_LOG_DIRS);
    if (logDirs.startsWith("${yarn.log.dir}")) {
        return ConfigUtils.getSchemePrefix(conf) + nodeHttpAddress + "/logs"
                + logDirs.substring("${yarn.log.dir}".length()) + "/" + appId + "/" + containerId;
    } else {
        try {
            String logDirsPath = new File(logDirs).getCanonicalPath();
            String yarnLogDirPath = new File(getYarnLogDir()).getCanonicalPath();
            if (logDirsPath.startsWith(yarnLogDirPath)) {
                return ConfigUtils.getSchemePrefix(conf) + nodeHttpAddress + "/logs"
                        + logDirsPath.substring(yarnLogDirPath.length()) + "/" + appId + "/" + containerId;
            } else {
                if (!rawContainerLogWarningPrinted) {
                    LOG.warn(
                            "Cannot determine the location of container logs because of incompatible node manager log location ({}) and yarn log location ({})",
                            logDirsPath, yarnLogDirPath);
                    rawContainerLogWarningPrinted = true;
                }
            }
        } catch (Exception ex) {
            if (!rawContainerLogWarningPrinted) {
                LOG.warn("Cannot determine the location of container logs because of error: ", ex);
                rawContainerLogWarningPrinted = true;
            }
        }
    }
    return null;
}

---

From source file:io.hops.tensorflow.TestUtils.java

License:Apache License

public static int verifyContainerLog(MiniYARNCluster yarnCluster, int containerNum,
        List<String> expectedContent, boolean count, String expectedWord) {
    File logFolder = new File(yarnCluster.getNodeManager(0).getConfig().get(YarnConfiguration.NM_LOG_DIRS,
            YarnConfiguration.DEFAULT_NM_LOG_DIRS));

    File[] listOfFiles = logFolder.listFiles();
    int currentContainerLogFileIndex = -1;
    for (int i = listOfFiles.length - 1; i >= 0; i--) {
        if (listOfFiles[i].listFiles().length == containerNum + 1) {
            currentContainerLogFileIndex = i;
            break;
        }/* w w  w .j a  v a 2  s .  co m*/
    }
    Assert.assertTrue(currentContainerLogFileIndex != -1);
    File[] containerFiles = listOfFiles[currentContainerLogFileIndex].listFiles();

    int numOfWords = 0;
    for (int i = 0; i < containerFiles.length; i++) {
        for (File output : containerFiles[i].listFiles()) {
            if (output.getName().trim().contains("stdout")) {
                BufferedReader br = null;
                List<String> stdOutContent = new ArrayList<String>();
                try {

                    String sCurrentLine;
                    br = new BufferedReader(new FileReader(output));
                    int numOfline = 0;
                    while ((sCurrentLine = br.readLine()) != null) {
                        if (count) {
                            if (sCurrentLine.contains(expectedWord)) {
                                numOfWords++;
                            }
                        } else if (output.getName().trim().equals("stdout")) {
                            if (!Shell.WINDOWS) {
                                Assert.assertEquals("The current is" + sCurrentLine,
                                        expectedContent.get(numOfline), sCurrentLine.trim());
                                numOfline++;
                            } else {
                                stdOutContent.add(sCurrentLine.trim());
                            }
                        }
                    }
                    /* By executing bat script using cmd /c,
                     * it will output all contents from bat script first
                     * It is hard for us to do check line by line
                     * Simply check whether output from bat file contains
                     * all the expected messages
                     */
                    if (Shell.WINDOWS && !count && output.getName().trim().equals("stdout")) {
                        Assert.assertTrue(stdOutContent.containsAll(expectedContent));
                    }
                } catch (IOException e) {
                    e.printStackTrace();
                } finally {
                    try {
                        if (br != null) {
                            br.close();
                        }
                    } catch (IOException ex) {
                        ex.printStackTrace();
                    }
                }
            }
        }
    }
    return numOfWords;
}

---

From source file:io.hops.tensorflow.TestUtils.java

License:Apache License

public static boolean dumpAllRemoteContainersLogs(MiniYARNCluster yarnCluster, ApplicationId appId) {
    File logFolder = new File(yarnCluster.getNodeManager(0).getConfig().get(YarnConfiguration.NM_LOG_DIRS,
            YarnConfiguration.DEFAULT_NM_LOG_DIRS));
    File appFolder = new File(logFolder, appId.toString());
    File[] containerFolders = appFolder.listFiles();

    if (containerFolders == null) {
        System.out.println(appFolder + " does not have any log files.");
        return false;
    }//from   w  w  w.j av a 2 s .  co  m

    for (File containerFolder : appFolder.listFiles()) {
        System.out.println("\n\nContainer: " + containerFolder.getName());
        System.out.println("======================================================================");
        for (File logFile : containerFolder.listFiles()) {
            System.out.println("LogType:" + logFile.getName());
            BufferedReader br = null;
            try {
                String sCurrentLine;
                br = new BufferedReader(new FileReader(logFile));
                while ((sCurrentLine = br.readLine()) != null) {
                    System.out.println(sCurrentLine);
                }
            } catch (IOException e) {
                e.printStackTrace();
            } finally {
                try {
                    if (br != null) {
                        br.close();
                    }
                } catch (IOException ex) {
                    ex.printStackTrace();
                }
            }
            System.out.println("End of LogType:" + logFile.getName() + "\n");
        }
    }
    return true;
}

---

From source file:org.apache.metron.maas.service.MaasIntegrationTest.java

License:Apache License

private int verifyContainerLog(int containerNum, List<String> expectedContent, boolean count,
        String expectedWord) {//from w w w.j  a v a 2s .  com
    File logFolder = new File(yarnComponent.getYARNCluster().getNodeManager(0).getConfig()
            .get(YarnConfiguration.NM_LOG_DIRS, YarnConfiguration.DEFAULT_NM_LOG_DIRS));

    File[] listOfFiles = logFolder.listFiles();
    int currentContainerLogFileIndex = -1;
    for (int i = listOfFiles.length - 1; i >= 0; i--) {
        if (listOfFiles[i].listFiles().length == containerNum + 1) {
            currentContainerLogFileIndex = i;
            break;
        }
    }
    Assert.assertTrue(currentContainerLogFileIndex != -1);
    File[] containerFiles = listOfFiles[currentContainerLogFileIndex].listFiles();

    int numOfWords = 0;
    for (int i = 0; i < containerFiles.length; i++) {
        for (File output : containerFiles[i].listFiles()) {
            if (output.getName().trim().contains("stdout")) {
                BufferedReader br = null;
                List<String> stdOutContent = new ArrayList<String>();
                try {

                    String sCurrentLine;
                    br = new BufferedReader(new FileReader(output));
                    int numOfline = 0;
                    while ((sCurrentLine = br.readLine()) != null) {
                        if (count) {
                            if (sCurrentLine.contains(expectedWord)) {
                                numOfWords++;
                            }
                        } else if (output.getName().trim().equals("stdout")) {
                            if (!Shell.WINDOWS) {
                                Assert.assertEquals("The current is" + sCurrentLine,
                                        expectedContent.get(numOfline), sCurrentLine.trim());
                                numOfline++;
                            } else {
                                stdOutContent.add(sCurrentLine.trim());
                            }
                        }
                    }
                    /* By executing bat script using cmd /c,
                     * it will output all contents from bat script first
                     * It is hard for us to do check line by line
                     * Simply check whether output from bat file contains
                     * all the expected messages
                     */
                    if (Shell.WINDOWS && !count && output.getName().trim().equals("stdout")) {
                        Assert.assertTrue(stdOutContent.containsAll(expectedContent));
                    }
                } catch (IOException e) {
                    e.printStackTrace();
                } finally {
                    try {
                        if (br != null)
                            br.close();
                    } catch (IOException ex) {
                        ex.printStackTrace();
                    }
                }
            }
        }
    }
    return numOfWords;
}

---