Example usage for org.apache.hadoop.yarn.conf YarnConfiguration DEFAULT_NM_LOG_DIRS

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.conf YarnConfiguration DEFAULT_NM_LOG_DIRS.

Prototype

String DEFAULT_NM_LOG_DIRS

To view the source code for org.apache.hadoop.yarn.conf YarnConfiguration DEFAULT_NM_LOG_DIRS.

Click Source Link

Usage

From source file:io.hops.tensorflow.TestUtils.java

License:Apache License

public static int verifyContainerLog(MiniYARNCluster yarnCluster, int containerNum,
        List<String> expectedContent, boolean count, String expectedWord) {
    File logFolder = new File(yarnCluster.getNodeManager(0).getConfig().get(YarnConfiguration.NM_LOG_DIRS,
            YarnConfiguration.DEFAULT_NM_LOG_DIRS));

    File[] listOfFiles = logFolder.listFiles();
    int currentContainerLogFileIndex = -1;
    for (int i = listOfFiles.length - 1; i >= 0; i--) {
        if (listOfFiles[i].listFiles().length == containerNum + 1) {
            currentContainerLogFileIndex = i;
            break;
        }//w  w  w .  j ava 2  s .c om
    }
    Assert.assertTrue(currentContainerLogFileIndex != -1);
    File[] containerFiles = listOfFiles[currentContainerLogFileIndex].listFiles();

    int numOfWords = 0;
    for (int i = 0; i < containerFiles.length; i++) {
        for (File output : containerFiles[i].listFiles()) {
            if (output.getName().trim().contains("stdout")) {
                BufferedReader br = null;
                List<String> stdOutContent = new ArrayList<String>();
                try {

                    String sCurrentLine;
                    br = new BufferedReader(new FileReader(output));
                    int numOfline = 0;
                    while ((sCurrentLine = br.readLine()) != null) {
                        if (count) {
                            if (sCurrentLine.contains(expectedWord)) {
                                numOfWords++;
                            }
                        } else if (output.getName().trim().equals("stdout")) {
                            if (!Shell.WINDOWS) {
                                Assert.assertEquals("The current is" + sCurrentLine,
                                        expectedContent.get(numOfline), sCurrentLine.trim());
                                numOfline++;
                            } else {
                                stdOutContent.add(sCurrentLine.trim());
                            }
                        }
                    }
                    /* By executing bat script using cmd /c,
                     * it will output all contents from bat script first
                     * It is hard for us to do check line by line
                     * Simply check whether output from bat file contains
                     * all the expected messages
                     */
                    if (Shell.WINDOWS && !count && output.getName().trim().equals("stdout")) {
                        Assert.assertTrue(stdOutContent.containsAll(expectedContent));
                    }
                } catch (IOException e) {
                    e.printStackTrace();
                } finally {
                    try {
                        if (br != null) {
                            br.close();
                        }
                    } catch (IOException ex) {
                        ex.printStackTrace();
                    }
                }
            }
        }
    }
    return numOfWords;
}

---

From source file:io.hops.tensorflow.TestUtils.java

License:Apache License

public static boolean dumpAllRemoteContainersLogs(MiniYARNCluster yarnCluster, ApplicationId appId) {
    File logFolder = new File(yarnCluster.getNodeManager(0).getConfig().get(YarnConfiguration.NM_LOG_DIRS,
            YarnConfiguration.DEFAULT_NM_LOG_DIRS));
    File appFolder = new File(logFolder, appId.toString());
    File[] containerFolders = appFolder.listFiles();

    if (containerFolders == null) {
        System.out.println(appFolder + " does not have any log files.");
        return false;
    }/*from  w  w  w .  j  a  v  a2  s .  c o  m*/

    for (File containerFolder : appFolder.listFiles()) {
        System.out.println("\n\nContainer: " + containerFolder.getName());
        System.out.println("======================================================================");
        for (File logFile : containerFolder.listFiles()) {
            System.out.println("LogType:" + logFile.getName());
            BufferedReader br = null;
            try {
                String sCurrentLine;
                br = new BufferedReader(new FileReader(logFile));
                while ((sCurrentLine = br.readLine()) != null) {
                    System.out.println(sCurrentLine);
                }
            } catch (IOException e) {
                e.printStackTrace();
            } finally {
                try {
                    if (br != null) {
                        br.close();
                    }
                } catch (IOException ex) {
                    ex.printStackTrace();
                }
            }
            System.out.println("End of LogType:" + logFile.getName() + "\n");
        }
    }
    return true;
}

---

From source file:org.apache.metron.maas.service.MaasIntegrationTest.java

License:Apache License

private int verifyContainerLog(int containerNum, List<String> expectedContent, boolean count,
        String expectedWord) {//from   w  w  w  .  j  a  v  a 2  s  .  com
    File logFolder = new File(yarnComponent.getYARNCluster().getNodeManager(0).getConfig()
            .get(YarnConfiguration.NM_LOG_DIRS, YarnConfiguration.DEFAULT_NM_LOG_DIRS));

    File[] listOfFiles = logFolder.listFiles();
    int currentContainerLogFileIndex = -1;
    for (int i = listOfFiles.length - 1; i >= 0; i--) {
        if (listOfFiles[i].listFiles().length == containerNum + 1) {
            currentContainerLogFileIndex = i;
            break;
        }
    }
    Assert.assertTrue(currentContainerLogFileIndex != -1);
    File[] containerFiles = listOfFiles[currentContainerLogFileIndex].listFiles();

    int numOfWords = 0;
    for (int i = 0; i < containerFiles.length; i++) {
        for (File output : containerFiles[i].listFiles()) {
            if (output.getName().trim().contains("stdout")) {
                BufferedReader br = null;
                List<String> stdOutContent = new ArrayList<String>();
                try {

                    String sCurrentLine;
                    br = new BufferedReader(new FileReader(output));
                    int numOfline = 0;
                    while ((sCurrentLine = br.readLine()) != null) {
                        if (count) {
                            if (sCurrentLine.contains(expectedWord)) {
                                numOfWords++;
                            }
                        } else if (output.getName().trim().equals("stdout")) {
                            if (!Shell.WINDOWS) {
                                Assert.assertEquals("The current is" + sCurrentLine,
                                        expectedContent.get(numOfline), sCurrentLine.trim());
                                numOfline++;
                            } else {
                                stdOutContent.add(sCurrentLine.trim());
                            }
                        }
                    }
                    /* By executing bat script using cmd /c,
                     * it will output all contents from bat script first
                     * It is hard for us to do check line by line
                     * Simply check whether output from bat file contains
                     * all the expected messages
                     */
                    if (Shell.WINDOWS && !count && output.getName().trim().equals("stdout")) {
                        Assert.assertTrue(stdOutContent.containsAll(expectedContent));
                    }
                } catch (IOException e) {
                    e.printStackTrace();
                } finally {
                    try {
                        if (br != null)
                            br.close();
                    } catch (IOException ex) {
                        ex.printStackTrace();
                    }
                }
            }
        }
    }
    return numOfWords;
}

---