Java tutorial
/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.github.sakserv.minicluster.yarn; import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.File; import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; import java.lang.reflect.Field; import java.lang.reflect.Method; import java.net.URI; import java.net.URL; import java.net.URLClassLoader; import java.security.Permission; import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.locks.LockSupport; import org.apache.commons.io.FileUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor; import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent; import com.github.sakserv.minicluster.yarn.util.EnvironmentUtils; import com.github.sakserv.minicluster.yarn.util.ExecJavaCliParser; import com.github.sakserv.minicluster.yarn.util.ExecShellCliParser; import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerStartContext; /** * !!!!! FOR TESTING WITH MINI CLUSTER ONLY !!!!! * * Container executor which will launch Java containers in the same JVM. * * In order to use it you must override * 'yarn.nodemanager.container-executor.class' property in the server * configuration (e.g., mini cluster) and set it to the fully qualified name of * this class * * @author Oleg Zhurakousky * */ public class InJvmContainerExecutor extends DefaultContainerExecutor { private static final Log logger = LogFactory.getLog(InJvmContainerExecutor.class); /** * Will construct the instance of this {@link ContainerExecutor} and will * install a {@link SystemExitDisallowingSecurityManager} which will help with * managing the life-cycle of the containers that contain System.exit calls. */ public InJvmContainerExecutor() { logger.info("Adding SystemExitDisallowingSecurityManager"); System.setSecurityManager(new SystemExitDisallowingSecurityManager()); } /** * Overrides the parent method while still invoking it. Since * {@link #isContainerActive(ContainerId)} method is also overridden here and * always returns 'false' the super.launchContainer(..) will only go through * the prep routine (e.g., creating temp dirs etc.) while never launching the * actual container via the launch script. This will ensure that all the * expectations of the container to be launched (e.g., symlinks etc.) are * satisfied. The actual launch will be performed by invoking * {@link #doLaunch(Container, Path)} method. */ public int launchContainer(ContainerStartContext containerStartContext) throws IOException { Container container = containerStartContext.getContainer(); Path containerWorkDir = containerStartContext.getContainerWorkDir(); super.launchContainer(containerStartContext); int exitCode = 0; if (container.getLaunchContext().getCommands().toString().contains("bin/java")) { ExecJavaCliParser result = this.createExecCommandParser(containerWorkDir.toString()); try { exitCode = this.doLaunch(container, containerWorkDir); if (logger.isInfoEnabled()) { logger.info(("Returned: " + exitCode)); } } catch (Exception e) { e.printStackTrace(); } } else { String cmd = container.getLaunchContext().getCommands().get(0); if (logger.isInfoEnabled()) { logger.info("Running Command: " + cmd); } ExecShellCliParser execShellCliParser = new ExecShellCliParser(cmd); try { exitCode = execShellCliParser.runCommand(); } catch (Exception e) { e.printStackTrace(); } if (logger.isInfoEnabled()) { logger.info(("Returned: " + exitCode)); } } return exitCode; } /** * Overrides the parent method while still invoking it. Since * {@link #isContainerActive(ContainerId)} method is also overridden here and * always returns 'false' the super.launchContainer(..) will only go through * the prep routine (e.g., creating temp dirs etc.) while never launching the * actual container via the launch script. This will ensure that all the * expectations of the container to be launched (e.g., symlinks etc.) are * satisfied. The actual launch will be performed by invoking * {@link #doLaunch(Container, Path)} method. */ public int launchContainer(Container container, Path nmPrivateContainerScriptPath, Path nmPrivateTokensPath, String userName, String appId, Path containerWorkDir, List<String> localDirs, List<String> logDirs) throws IOException { ContainerStartContext containerStartContext = new ContainerStartContext.Builder().setContainer(container) .setLocalizedResources(container.getLocalizedResources()) .setNmPrivateContainerScriptPath(nmPrivateContainerScriptPath) .setNmPrivateTokensPath(nmPrivateTokensPath).setUser(userName).setAppId(appId) .setContainerWorkDir(containerWorkDir).setLocalDirs(localDirs).setLocalDirs(logDirs).build(); super.launchContainer(containerStartContext); int exitCode = 0; if (container.getLaunchContext().getCommands().toString().contains("bin/java")) { ExecJavaCliParser result = this.createExecCommandParser(containerWorkDir.toString()); try { exitCode = this.doLaunch(container, containerWorkDir); if (logger.isInfoEnabled()) { logger.info(("Returned: " + exitCode)); } } catch (Exception e) { e.printStackTrace(); } } else { String cmd = container.getLaunchContext().getCommands().get(0); if (logger.isInfoEnabled()) { logger.info("Running Command: " + cmd); } ExecShellCliParser execShellCliParser = new ExecShellCliParser(cmd); try { exitCode = execShellCliParser.runCommand(); } catch (Exception e) { e.printStackTrace(); } if (logger.isInfoEnabled()) { logger.info(("Returned: " + exitCode)); } } return exitCode; } /** * This is to ensure that call to super.launchContainer(..) doesn't actually * execute anything other then prep work (e.g., sets up directories etc.) */ @Override protected boolean isContainerActive(ContainerId containerId) { return false; } /** * Will launch containers within the same JVM as this Container Executor. It * will do so by: - extracting Container's class name and program arguments * from the launch script (e.g., launch_container.sh) - Creating an isolated * ClassLoader for each container - Calling doLaunchContainer(..) method to * launch Container */ private int doLaunch(Container container, Path containerWorkDir) throws Exception { Map<String, String> environment = container.getLaunchContext().getEnvironment(); EnvironmentUtils.putAll(environment); Set<URL> additionalClassPathUrls = this.filterAndBuildUserClasspath(container); ExecJavaCliParser javaCliParser = this.createExecCommandParser(containerWorkDir.toString()); UserGroupInformation.setLoginUser(null); try { // create Isolated Class Loader for each container and set it as context // class loader URLClassLoader containerCl = new URLClassLoader( additionalClassPathUrls.toArray(additionalClassPathUrls.toArray(new URL[] {})), null); Thread.currentThread().setContextClassLoader(containerCl); String containerLauncher = javaCliParser.getMain(); Class<?> containerClass = Class.forName(containerLauncher, true, containerCl); Method mainMethod = containerClass.getMethod("main", new Class[] { String[].class }); mainMethod.setAccessible(true); String[] arguments = javaCliParser.getMainArguments(); this.doLaunchContainer(containerClass, mainMethod, arguments); } catch (Exception e) { logger.error("Failed to launch container " + container, e); container.handle(new ContainerDiagnosticsUpdateEvent(container.getContainerId(), e.getMessage())); return -1; } finally { logger.info("Removing symlinks"); this.cleanUp(); } return 0; } /** * Will invoke Container's main method blocking if necessary. This method * contains a hack that I am not proud of it, but given the fact that some * containers rely on System.exit to manage its life-cycle instead of proper * exit this will ensure that together with the * SystemExitDisallowingSecurityManager (see constructor of this class) this * method will block until such container invokes System.exit * * ByteCodeUtils.hasSystemExit(..) will check if a container that was invoked * has calls to System.exit and if it does it will block this thread until * SystemExitException is thrown which will be caught allowing this method to * exit normally. * * Of course this doesn't guarantee anything since the underlying * implementation of the container may still be implemented in such way where * it exits gracefully while also has some shortcut method for some * exceptional conditions where System.exit is called and if that's the case * this process will block infinitely. * * The bottom line: DON'T USE System.exit when implementing application * containers!!! */ private void doLaunchContainer(Class<?> containerClass, Method mainMethod, String[] arguments) throws Exception { if (logger.isInfoEnabled()) { logger.info("Launching container for " + containerClass.getName() + " with arguments: " + Arrays.asList(arguments)); } try { mainMethod.invoke(null, (Object) arguments); logger.info("Keeping " + containerClass.getName() + " process alive"); LockSupport.park(); } catch (SystemExitException e) { logger.warn("Ignoring System.exit(..) call in " + containerClass.getName()); } if (logger.isInfoEnabled()) { logger.warn("Container " + containerClass.getName() + " is finished"); } } /** * YARN provides ability to pass resources (e.g., classpath) through * {@link LocalResource}s which allows user to provision all the resources * required to run the app. This method will extract those resources as a * {@link Set} of {@link URL}s so they are used when {@link ClassLoader} for a * container is created. * * This is done primarily as a convenience for applications that rely on * automatic classpath propagation (e.g., pull everything from my dev * classpath) instead of manual. * * @param container * @return */ private Set<URL> filterAndBuildUserClasspath(Container container) { if (logger.isDebugEnabled()) { logger.debug("Building additional classpath for the container: " + container); } Set<URL> additionalClassPathUrls = new HashSet<URL>(); Set<Path> userClassPath = this.extractUserProvidedClassPathEntries(container); for (Path resourcePath : userClassPath) { String resourceName = "file:///" + new File(resourcePath.getName()).getAbsolutePath(); if (logger.isDebugEnabled()) { logger.debug("\t adding " + resourceName + " to the classpath"); } try { additionalClassPathUrls.add(new URI(resourceName).toURL()); } catch (Exception e) { throw new IllegalArgumentException(e); } } return additionalClassPathUrls; } /** * Creates CLI parser which can be used to extract Container's class name and * its launch arguments. * * @param containerWorkDir * @return */ private ExecJavaCliParser createExecCommandParser(String containerWorkDir) { String execLine = this.filterAndExecuteLaunchScriptAndReturnExecLine(containerWorkDir); String[] values = execLine.split("\""); String javaCli = values[1]; String[] javaCliValues = javaCli.split(" "); StringBuffer buffer = new StringBuffer(); for (int i = 0; i < javaCliValues.length; i++) { if (i > 0) { buffer.append(javaCliValues[i]); if (javaCliValues.length - i > 1) { buffer.append(" "); } } } String extractedJavaCli = buffer.toString(); ExecJavaCliParser execJavaCliParser = new ExecJavaCliParser(extractedJavaCli); return execJavaCliParser; } /** * This method does three things 1. It creates an updated version of the * initial launch script where it simply copies its contents less the 'exec' * line 2. It extract the 'exec' line and returns it so the Container's class * name and launch arguments could be retrieved. 3. It executes the * 'exec'-less launch script to ensure that all symlinks and other prepwork * expected by the underlying container is performed. * * @param containerWorkDir * @return */ private String filterAndExecuteLaunchScriptAndReturnExecLine(String containerWorkDir) { BufferedReader reader = null; BufferedWriter writer = null; String execLine = null; File inJvmlaunchScript = null; try { File launchScript = new File(containerWorkDir, "launch_container.sh"); inJvmlaunchScript = new File(containerWorkDir.toString(), "injvm_launch_container.sh"); inJvmlaunchScript.setExecutable(true); reader = new BufferedReader(new FileReader(launchScript)); writer = new BufferedWriter(new FileWriter(inJvmlaunchScript)); String line; while ((line = reader.readLine()) != null) { if (!line.startsWith("exec")) { writer.write(line); writer.write("\n"); } else { execLine = line; } } } catch (Exception e) { throw new IllegalStateException("Failed to override default launch script", e); } finally { try { reader.close(); } catch (IOException e) { // ignore } try { writer.close(); } catch (IOException e) { // ignore } } if (inJvmlaunchScript != null) { try { inJvmlaunchScript.setExecutable(true); Process process = Runtime.getRuntime().exec(inJvmlaunchScript.getAbsolutePath()); int exitCode = process.waitFor(); if (exitCode != 0) { throw new IllegalStateException("Failed to execute launch script. Exit code: " + exitCode); } } catch (Exception e) { throw new IllegalStateException("Failed to execute " + inJvmlaunchScript.getAbsolutePath(), e); } } return execLine; } /** * Extracts {@link LocalResource}s from the {@link Container}. */ @SuppressWarnings("unchecked") private Set<Path> extractUserProvidedClassPathEntries(Container container) { Map<Path, List<String>> localizedResources; try { Field lf = container.getClass().getDeclaredField("localizedResources"); lf.setAccessible(true); localizedResources = (Map<Path, List<String>>) lf.get(container); Set<Path> paths = localizedResources.keySet(); // Needed for Tez for (Path path : paths) { if (path.toString().endsWith("tez-conf.pb") || path.toString().endsWith("tez-dag.pb")) { File sourceFile = new File(path.toUri()); File targetFile = new File(System.getenv(Environment.PWD.name()) + "/" + sourceFile.getName()); FileUtils.copyFile(sourceFile, targetFile); // System.out.println("######## Copied file: " + targetFile); // FileInputStream fis = new FileInputStream(new File(System.getenv(Environment.PWD.name()), targetFile.getName())); // System.out.println(fis.available()); // fis.close(); // break; } } return paths; } catch (Exception e) { throw new RuntimeException(e); } } /** * Will clean up symlinks that were created by a launch script */ private void cleanUp() { try { File file = new File(System.getProperty("user.dir")); String[] links = file.list(); for (String name : links) { File potentialSymLink = new File(file, name); if (FileUtils.isSymlink(potentialSymLink)) { if (logger.isDebugEnabled()) { logger.debug("DELETING: " + potentialSymLink); } potentialSymLink.delete(); } } } catch (Exception e) { logger.warn("Failed to remove symlinks", e); } } /** * An implementation of the {@link SecurityManager} which can be used to * intercept System.exit. This implementation will simply throw an exception * when such call is made essentially making such call ineffective. * * It is used by this class to intercept System.exit calls made by some * implementations of YARN containers (e.g., Tez's DAGAppMaster). Since this * container executor will use the same JVM its running in to start those * containers any System.exit call will shut down the entire cluster. Using * such {@link SecurityManager} would allow such calls to be intercepted by * catching {@link SystemExitException}. */ private static class SystemExitDisallowingSecurityManager extends SecurityManager { @Override public void checkPermission(Permission perm) { // allow anything. } @Override public void checkPermission(Permission perm, Object context) { // allow anything. } @Override public void checkExit(int status) { throw new SystemExitException(); } } /** * An implementation of the {@link SecurityManager} which can be used to * intercept System.exit. This implementation will simply throw an exception * when such call is made essentially making such call ineffective. * * It is used by this class to intercept System.exit calls made by some * implementations of YARN containers (e.g., Tez's DAGAppMaster). Since this * container executor will use the same JVM its running in to start those * containers any System.exit call will shut down the entire cluster. Using * such {@link SecurityManager} would allow such calls to be intercepted by * catching {@link SystemExitException}. */ public static class SystemExitAllowSecurityManager extends SecurityManager { @Override public void checkPermission(Permission perm) { // allow anything. } @Override public void checkPermission(Permission perm, Object context) { // allow anything. } @Override public void checkExit(int status) { super.checkExit(status); } } }