List of usage examples for java.io File pathSeparatorChar
char pathSeparatorChar
To view the source code for java.io File pathSeparatorChar.
Click Source Link
From source file:org.apache.cocoon.bean.CocoonWrapper.java
/** * This builds the important ClassPath used by this class. It * does so in a neutral way./* w w w . j a v a 2 s . c o m*/ * It iterates in alphabetical order through every file in the * lib directory and adds it to the classpath. * * Also, we add the files to the ClassLoader for the Cocoon system. * In order to protect ourselves from skitzofrantic classloaders, * we need to work with a known one. * * @param context The context path * @return a <code>String</code> value */ protected String getClassPath(final String context) { StringBuffer buildClassPath = new StringBuffer(); String classDir = context + "/WEB-INF/classes"; buildClassPath.append(classDir); File root = new File(context + "/WEB-INF/lib"); if (root.isDirectory()) { File[] libraries = root.listFiles(); Arrays.sort(libraries); for (int i = 0; i < libraries.length; i++) { if (libraries[i].getAbsolutePath().endsWith(".jar")) { buildClassPath.append(File.pathSeparatorChar).append(IOUtils.getFullFilename(libraries[i])); } } } buildClassPath.append(File.pathSeparatorChar).append(SystemUtils.JAVA_CLASS_PATH); // Extra class path is necessary for non-classloader-aware java compilers to compile XSPs // buildClassPath.append(File.pathSeparatorChar) // .append(getExtraClassPath(context)); if (log.isDebugEnabled()) { log.debug("Context classpath: " + buildClassPath); } return buildClassPath.toString(); }
From source file:com.asakusafw.lang.compiler.cli.BatchCompilerCliTest.java
private String files(File... files) { StringBuilder buf = new StringBuilder(); for (File file : files) { if (buf.length() > 0) { buf.append(File.pathSeparatorChar); }/*from w w w. j ava2s . c o m*/ buf.append(file.getPath()); } return buf.toString(); }
From source file:com.simpligility.maven.plugins.android.phase08preparepackage.DexMojo.java
private File generateMainDexClassesFile() throws MojoExecutionException { CommandExecutor executor = CommandExecutor.Factory.createDefaultCommmandExecutor(); executor.setLogger(getLog());//w w w . j a va 2s .com List<String> commands = new ArrayList<>(); commands.add("--output"); File mainDexClasses = new File(targetDirectory, "mainDexClasses.txt"); commands.add(mainDexClasses.getAbsolutePath()); Set<File> inputFiles = getDexInputFiles(); StringBuilder sb = new StringBuilder(); sb.append(StringUtils.join(inputFiles, File.pathSeparatorChar)); if (Os.isFamily(Os.FAMILY_WINDOWS)) { commands.add(StringUtils.wrap(sb.toString(), '"')); } else { commands.add(sb.toString()); } String executable = getAndroidSdk().getMainDexClasses().getAbsolutePath(); try { executor.executeCommand(executable, commands, project.getBasedir(), false); } catch (ExecutionException ex) { throw new MojoExecutionException("Failed to execute mainDexClasses", ex); } return mainDexClasses; }
From source file:kieker.tools.bridge.cli.CLIServerMain.java
/** * Compile the options for the CLI server. * * @return The composed options for the CLI server *//* w ww.j a v a2 s. co m*/ private static Options declareOptions() { options = new Options(); Option option; // Type selection option = new Option(CMD_TYPE, CMD_TYPE_LONG, true, "select the service type: tcp-client, tcp-server, tcp-single-server, jms-client, jms-embedded, http-rest"); option.setArgName("type"); option.setRequired(true); options.addOption(option); // TCP client option = new Option(CMD_HOST, CMD_HOST_LONG, true, "connect to server named <hostname>"); option.setArgName("hostname"); options.addOption(option); // TCP server option = new Option(CMD_PORT, CMD_PORT_LONG, true, "listen at port (tcp-server, jms-embedded, or http-rest) or connect to port (tcp-client)"); option.setArgName("number"); option.setType(Number.class); options.addOption(option); // JMS client option = new Option(CMD_USER, CMD_USER_LONG, true, "user name for a JMS service"); option.setArgName("username"); options.addOption(option); option = new Option(CMD_PASSWORD, CMD_PASSWORD_LONG, true, "password for a JMS service"); option.setArgName("password"); options.addOption(option); option = new Option(CMD_URL, CMD_URL_LONG, true, "URL for JMS server or HTTP servlet"); option.setArgName("jms-url"); option.setType(URL.class); options.addOption(option); // HTTP client option = new Option(CMD_CONTEXT, CMD_CONTEXT_LONG, true, "context for the HTTP servlet"); option.setArgName("context"); options.addOption(option); // kieker configuration file option = new Option(CMD_KIEKER_CONFIGURATION, CMD_KIEKER_CONFIGURATION_LONG, true, "kieker configuration file"); option.setArgName("configuration"); options.addOption(option); // mapping file for TCP and JMS option = new Option(CMD_MAP_FILE, CMD_MAP_FILE_LONG, true, "Class name to id (integer or string) mapping"); option.setArgName("map-file"); option.setType(File.class); option.setRequired(true); options.addOption(option); // libraries option = new Option(CMD_LIBRARIES, CMD_LIBRARIES_LONG, true, "List of library paths separated by " + File.pathSeparatorChar); option.setArgName("paths"); option.setType(File.class); option.setRequired(true); option.setValueSeparator(File.pathSeparatorChar); options.addOption(option); // verbose option = new Option(CMD_VERBOSE, CMD_VERBOSE_LONG, true, "output processing information"); option.setRequired(false); option.setOptionalArg(true); options.addOption(option); // statistics option = new Option(CMD_STATS, CMD_STATS_LONG, false, "output performance statistics"); option.setRequired(false); options.addOption(option); // daemon mode option = new Option(CMD_DAEMON, CMD_DAEMON_LONG, false, "detach from console; TCP server allows multiple connections"); option.setRequired(false); options.addOption(option); return options; }
From source file:org.fornax.toolsupport.sculptor.maven.plugin.GeneratorMojo.java
/** * Returns the generators classpath by adding the artifacts from the plugins * dependencies. The dependencies used by the plugin itself (the Maven * stuff) are skipped.// w ww .j a va2 s .c o m */ @SuppressWarnings("rawtypes") protected String getGeneratorClasspath() { StringBuilder classpath = new StringBuilder(); // First add the maven projects resource folder to the classpath List resources = project.getResources(); for (Iterator iterator = resources.iterator(); iterator.hasNext();) { Resource resource = (Resource) iterator.next(); if (isVerbose() || getLog().isDebugEnabled()) { getLog().info("Adding resource to classpath: " + resource.getDirectory()); } if (classpath.length() > 0) { classpath.append(File.pathSeparatorChar); } classpath.append(resource.getDirectory()); } // Now add plugin dependencies to the classpath for (Artifact artifact : pluginDependencies) { String groupId = artifact.getGroupId(); // Skip all dependencies only used by Maven if (!groupId.startsWith("org.apache.maven") && !groupId.startsWith("org.codehaus.plexus") && !groupId.startsWith("junit")) { if (isVerbose() || getLog().isDebugEnabled()) { getLog().info("Adding dependency to classpath: " + artifact); } if (classpath.length() > 0) { classpath.append(File.pathSeparatorChar); } classpath.append(artifact.getFile()); } } return classpath.toString(); }
From source file:com.srini.hadoopYarn.Client.java
/** * Main run function for the client/*w w w . j a va 2 s. c om*/ * @return true if application completed successfully * @throws IOException * @throws YarnException */ public boolean run() throws IOException, YarnException { LOG.info("Running Client"); yarnClient.start(); YarnClusterMetrics clusterMetrics = yarnClient.getYarnClusterMetrics(); LOG.info("Got Cluster metric info from ASM" + ", numNodeManagers=" + clusterMetrics.getNumNodeManagers()); List<NodeReport> clusterNodeReports = yarnClient.getNodeReports(NodeState.RUNNING); LOG.info("Got Cluster node info from ASM"); for (NodeReport node : clusterNodeReports) { LOG.info("Got node report from ASM for" + ", nodeId=" + node.getNodeId() + ", nodeAddress" + node.getHttpAddress() + ", nodeRackName" + node.getRackName() + ", nodeNumContainers" + node.getNumContainers()); } QueueInfo queueInfo = yarnClient.getQueueInfo(this.amQueue); LOG.info("Queue info" + ", queueName=" + queueInfo.getQueueName() + ", queueCurrentCapacity=" + queueInfo.getCurrentCapacity() + ", queueMaxCapacity=" + queueInfo.getMaximumCapacity() + ", queueApplicationCount=" + queueInfo.getApplications().size() + ", queueChildQueueCount=" + queueInfo.getChildQueues().size()); List<QueueUserACLInfo> listAclInfo = yarnClient.getQueueAclsInfo(); for (QueueUserACLInfo aclInfo : listAclInfo) { for (QueueACL userAcl : aclInfo.getUserAcls()) { LOG.info("User ACL Info for Queue" + ", queueName=" + aclInfo.getQueueName() + ", userAcl=" + userAcl.name()); } } // Get a new application id YarnClientApplication app = yarnClient.createApplication(); GetNewApplicationResponse appResponse = app.getNewApplicationResponse(); // TODO get min/max resource capabilities from RM and change memory ask if needed // If we do not have min/max, we may not be able to correctly request // the required resources from the RM for the app master // Memory ask has to be a multiple of min and less than max. // Dump out information about cluster capability as seen by the resource manager int maxMem = appResponse.getMaximumResourceCapability().getMemory(); LOG.info("Max mem capabililty of resources in this cluster " + maxMem); // A resource ask cannot exceed the max. if (amMemory > maxMem) { LOG.info("AM memory specified above max threshold of cluster. Using max value." + ", specified=" + amMemory + ", max=" + maxMem); amMemory = maxMem; } // set the application name ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); ApplicationId appId = appContext.getApplicationId(); appContext.setApplicationName(appName); // Set up the container launch context for the application master ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); // set local resources for the application master // local files or archives as needed // In this scenario, the jar file for the application master is part of the local resources Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); LOG.info("Copy App Master jar from local filesystem and add to local environment"); // Copy the application master jar to the filesystem // Create a local resource to point to the destination jar path FileSystem fs = FileSystem.get(conf); Path src = new Path(appMasterJar); String pathSuffix = appName + "/" + appId.getId() + "/AppMaster.jar"; Path dst = new Path(fs.getHomeDirectory(), pathSuffix); fs.copyFromLocalFile(false, true, src, dst); FileStatus destStatus = fs.getFileStatus(dst); LocalResource amJarRsrc = Records.newRecord(LocalResource.class); // Set the type of resource - file or archive // archives are untarred at destination // we don't need the jar file to be untarred for now amJarRsrc.setType(LocalResourceType.FILE); // Set visibility of the resource // Setting to most private option amJarRsrc.setVisibility(LocalResourceVisibility.APPLICATION); // Set the resource to be copied over amJarRsrc.setResource(ConverterUtils.getYarnUrlFromPath(dst)); // Set timestamp and length of file so that the framework // can do basic sanity checks for the local resource // after it has been copied over to ensure it is the same // resource the client intended to use with the application amJarRsrc.setTimestamp(destStatus.getModificationTime()); amJarRsrc.setSize(destStatus.getLen()); localResources.put("AppMaster.jar", amJarRsrc); // Set the log4j properties if needed if (!log4jPropFile.isEmpty()) { Path log4jSrc = new Path(log4jPropFile); Path log4jDst = new Path(fs.getHomeDirectory(), "log4j.props"); fs.copyFromLocalFile(false, true, log4jSrc, log4jDst); FileStatus log4jFileStatus = fs.getFileStatus(log4jDst); LocalResource log4jRsrc = Records.newRecord(LocalResource.class); log4jRsrc.setType(LocalResourceType.FILE); log4jRsrc.setVisibility(LocalResourceVisibility.APPLICATION); log4jRsrc.setResource(ConverterUtils.getYarnUrlFromURI(log4jDst.toUri())); log4jRsrc.setTimestamp(log4jFileStatus.getModificationTime()); log4jRsrc.setSize(log4jFileStatus.getLen()); localResources.put("log4j.properties", log4jRsrc); } // The shell script has to be made available on the final container(s) // where it will be executed. // To do this, we need to first copy into the filesystem that is visible // to the yarn framework. // We do not need to set this as a local resource for the application // master as the application master does not need it. String hdfsShellScriptLocation = ""; long hdfsShellScriptLen = 0; long hdfsShellScriptTimestamp = 0; if (!shellScriptPath.isEmpty()) { Path shellSrc = new Path(shellScriptPath); String shellPathSuffix = appName + "/" + appId.getId() + "/ExecShellScript.sh"; Path shellDst = new Path(fs.getHomeDirectory(), shellPathSuffix); fs.copyFromLocalFile(false, true, shellSrc, shellDst); hdfsShellScriptLocation = shellDst.toUri().toString(); FileStatus shellFileStatus = fs.getFileStatus(shellDst); hdfsShellScriptLen = shellFileStatus.getLen(); hdfsShellScriptTimestamp = shellFileStatus.getModificationTime(); } // Set local resource info into app master container launch context amContainer.setLocalResources(localResources); // Set the necessary security tokens as needed //amContainer.setContainerTokens(containerToken); // Set the env variables to be setup in the env where the application master will be run LOG.info("Set the environment for the application master"); Map<String, String> env = new HashMap<String, String>(); // put location of shell script into env // using the env info, the application master will create the correct local resource for the // eventual containers that will be launched to execute the shell scripts env.put(DSConstants.DISTRIBUTEDSHELLSCRIPTLOCATION, hdfsShellScriptLocation); env.put(DSConstants.DISTRIBUTEDSHELLSCRIPTTIMESTAMP, Long.toString(hdfsShellScriptTimestamp)); env.put(DSConstants.DISTRIBUTEDSHELLSCRIPTLEN, Long.toString(hdfsShellScriptLen)); // Add AppMaster.jar location to classpath // At some point we should not be required to add // the hadoop specific classpaths to the env. // It should be provided out of the box. // For now setting all required classpaths including // the classpath to "." for the application jar StringBuilder classPathEnv = new StringBuilder(Environment.CLASSPATH.$()).append(File.pathSeparatorChar) .append("./*"); for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH, YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) { classPathEnv.append(File.pathSeparatorChar); classPathEnv.append(c.trim()); } classPathEnv.append(File.pathSeparatorChar).append("./log4j.properties"); // add the runtime classpath needed for tests to work if (conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) { classPathEnv.append(':'); classPathEnv.append(System.getProperty("java.class.path")); } env.put("CLASSPATH", classPathEnv.toString()); amContainer.setEnvironment(env); // Set the necessary command to execute the application master Vector<CharSequence> vargs = new Vector<CharSequence>(30); // Set java executable command LOG.info("Setting up app master command"); vargs.add(Environment.JAVA_HOME.$() + "/bin/java"); // Set Xmx based on am memory size vargs.add("-Xmx" + amMemory + "m"); // Set class name vargs.add(appMasterMainClass); // Set params for Application Master vargs.add("--container_memory " + String.valueOf(containerMemory)); vargs.add("--num_containers " + String.valueOf(numContainers)); vargs.add("--priority " + String.valueOf(shellCmdPriority)); if (!shellCommand.isEmpty()) { vargs.add("--shell_command " + shellCommand + ""); } if (!shellArgs.isEmpty()) { vargs.add("--shell_args " + shellArgs + ""); } for (Map.Entry<String, String> entry : shellEnv.entrySet()) { vargs.add("--shell_env " + entry.getKey() + "=" + entry.getValue()); } if (debugFlag) { vargs.add("--debug"); } vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stdout"); vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stderr"); // Get final commmand StringBuilder command = new StringBuilder(); for (CharSequence str : vargs) { command.append(str).append(" "); } LOG.info("Completed setting up app master command " + command.toString()); List<String> commands = new ArrayList<String>(); commands.add(command.toString()); amContainer.setCommands(commands); // Set up resource type requirements // For now, only memory is supported so we set memory requirements Resource capability = Records.newRecord(Resource.class); capability.setMemory(amMemory); appContext.setResource(capability); // Service data is a binary blob that can be passed to the application // Not needed in this scenario // amContainer.setServiceData(serviceData); // Setup security tokens if (UserGroupInformation.isSecurityEnabled()) { Credentials credentials = new Credentials(); String tokenRenewer = conf.get(YarnConfiguration.RM_PRINCIPAL); if (tokenRenewer == null || tokenRenewer.length() == 0) { throw new IOException("Can't get Master Kerberos principal for the RM to use as renewer"); } // For now, only getting tokens for the default file-system. final Token<?> tokens[] = fs.addDelegationTokens(tokenRenewer, credentials); if (tokens != null) { for (Token<?> token : tokens) { LOG.info("Got dt for " + fs.getUri() + "; " + token); } } DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); ByteBuffer fsTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); amContainer.setTokens(fsTokens); } appContext.setAMContainerSpec(amContainer); // Set the priority for the application master Priority pri = Records.newRecord(Priority.class); // TODO - what is the range for priority? how to decide? pri.setPriority(amPriority); appContext.setPriority(pri); // Set the queue to which this application is to be submitted in the RM appContext.setQueue(amQueue); // Submit the application to the applications manager // SubmitApplicationResponse submitResp = applicationsManager.submitApplication(appRequest); // Ignore the response as either a valid response object is returned on success // or an exception thrown to denote some form of a failure LOG.info("Submitting application to ASM"); yarnClient.submitApplication(appContext); // TODO // Try submitting the same request again // app submission failure? // Monitor the application return monitorApplication(appId); }
From source file:org.apache.axis2.jaxws.description.builder.JAXWSRIWSDLGenerator.java
/** * Get the default classpath from various thingies in the message context * * @param msgContext//from w w w. j a v a 2 s . c om * @return default classpath */ public String getDefaultClasspath(String webBase) { HashSet classpath = new HashSet(); ClassLoader cl = Thread.currentThread().getContextClassLoader(); fillClassPath(cl, classpath); // Just to be safe (the above doesn't seem to return the webapp // classpath in all cases), manually do this: if (webBase != null) { addPath(classpath, webBase + File.separatorChar + "classes"); try { String libBase = webBase + File.separatorChar + "lib"; File libDir = new File(libBase); String[] jarFiles = libDir.list(); for (int i = 0; i < jarFiles.length; i++) { String jarFile = jarFiles[i]; if (jarFile.endsWith(".jar")) { addPath(classpath, libBase + File.separatorChar + jarFile); } } } catch (Exception e) { // Oh well. No big deal. } } URL serviceArchive = axisService.getFileName(); if (serviceArchive != null) { try { /** * If the service contains libraries in the 'lib' folder, we have to add those also * into classpath */ URL[] urls = Utils.getURLsForAllJars(serviceArchive, null); for (URL url : urls) { classpath.add(Utils.toFile(url).getCanonicalPath()); } } catch (UnsupportedEncodingException e) { log.error(e.getMessage(), e); } catch (IOException e) { log.error(e.getMessage(), e); } } // axis.ext.dirs can be used in any appserver getClassPathFromDirectoryProperty(classpath, "axis.ext.dirs"); // classpath used by Jasper getClassPathFromProperty(classpath, "org.apache.catalina.jsp_classpath"); // websphere stuff. getClassPathFromProperty(classpath, "ws.ext.dirs"); getClassPathFromProperty(classpath, "com.ibm.websphere.servlet.application.classpath"); // java class path getClassPathFromProperty(classpath, "java.class.path"); // Load jars from java external directory getClassPathFromDirectoryProperty(classpath, "java.ext.dirs"); // boot classpath isn't found in above search getClassPathFromProperty(classpath, "sun.boot.class.path"); StringBuffer path = new StringBuffer(); for (Iterator iterator = classpath.iterator(); iterator.hasNext();) { String s = (String) iterator.next(); path.append(s); path.append(File.pathSeparatorChar); } log.debug(path); return path.toString(); }
From source file:fr.gouv.finances.cp.xemelios.importers.batch.BatchRealImporter.java
private ImportContent files(final String extension, final String titreEtat) { ImportContent ic = new ImportContent(); Vector<File> ret = new Vector<File>(); ret.addAll(files);/*from ww w .j a va 2 s . c om*/ // on regarde si l'un des fichiers a importer est un zip for (int i = 0; i < ret.size(); i++) { if (ret.get(i).getName().toLowerCase().endsWith(".zip")) { if (ret.get(i).exists()) { ZipFile zf = null; try { zf = new ZipFile(ret.get(i)); for (Enumeration<? extends ZipEntry> enumer = zf.entries(); enumer.hasMoreElements();) { ZipEntry ze = enumer.nextElement(); if (!ze.isDirectory()) { String fileName = ze.getName(); String entryName = fileName.toLowerCase(); fileName = fileName.replace(File.pathSeparatorChar, '_') .replace(File.separatorChar, '_').replace(':', '|').replace('\'', '_') .replace('/', '_'); logger.debug(entryName); if (PJRef.isPJ(ze)) { PJRef pj = new PJRef(ze); File tmpFile = pj.writeTmpFile(FileUtils.getTempDir(), zf); ic.pjs.add(pj); filesToDrop.add(tmpFile); } else if ((entryName.endsWith(extension.toLowerCase()) || entryName.endsWith(".xml")) && !fileName.startsWith("_")) { // on decompresse le fichier dans le // repertoire temporaire, comme ca il sera // supprime en quittant InputStream is = zf.getInputStream(ze); BufferedInputStream bis = new BufferedInputStream(is); File output = new File(FileUtils.getTempDir(), fileName); BufferedOutputStream bos = new BufferedOutputStream( new FileOutputStream(output)); byte[] buffer = new byte[1024]; int read = bis.read(buffer); while (read > 0) { bos.write(buffer, 0, read); read = bis.read(buffer); } bos.flush(); bos.close(); bis.close(); ic.filesToImport.add(output); filesToDrop.add(output); } } } zf.close(); } catch (ZipException zEx) { System.out.println( "Le fichier " + ret.get(i).getName() + " n'est pas une archive ZIP valide."); } catch (IOException ioEx) { ioEx.printStackTrace(); } finally { if (zf != null) { try { zf.close(); } catch (Throwable t) { } } } } } else if (ret.get(i).getName().toLowerCase().endsWith(".gz")) { try { String fileName = ret.get(i).getName(); fileName = fileName.substring(0, fileName.length() - 3); File output = new File(FileUtils.getTempDir(), fileName); GZIPInputStream gis = new GZIPInputStream(new FileInputStream(ret.get(i))); BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(output)); byte[] buffer = new byte[1024]; int read = gis.read(buffer); while (read > 0) { bos.write(buffer, 0, read); read = gis.read(buffer); } bos.flush(); bos.close(); gis.close(); ic.filesToImport.add(output); filesToDrop.add(output); } catch (IOException ioEx) { // nothing to do } } else { ic.filesToImport.add(ret.get(i)); // dans ce cas l, on ne le supprime pas } } return ic; }
From source file:org.dknight.app.Client.java
/** * Main run function for the client//from www . j av a2 s .co m * @return true if application completed successfully * @throws IOException * @throws YarnException */ public boolean run() throws IOException, YarnException { LOG.info("Running Client"); yarnClient.start(); YarnClusterMetrics clusterMetrics = yarnClient.getYarnClusterMetrics(); LOG.info("Got Cluster metric info from ASM" + ", numNodeManagers=" + clusterMetrics.getNumNodeManagers()); List<NodeReport> clusterNodeReports = yarnClient.getNodeReports(NodeState.RUNNING); LOG.info("Got Cluster node info from ASM"); for (NodeReport node : clusterNodeReports) { LOG.info("Got node report from ASM for" + ", nodeId=" + node.getNodeId() + ", nodeAddress" + node.getHttpAddress() + ", nodeRackName" + node.getRackName() + ", nodeNumContainers" + node.getNumContainers()); } QueueInfo queueInfo = yarnClient.getQueueInfo(this.amQueue); LOG.info("Queue info" + ", queueName=" + queueInfo.getQueueName() + ", queueCurrentCapacity=" + queueInfo.getCurrentCapacity() + ", queueMaxCapacity=" + queueInfo.getMaximumCapacity() + ", queueApplicationCount=" + queueInfo.getApplications().size() + ", queueChildQueueCount=" + queueInfo.getChildQueues().size()); List<QueueUserACLInfo> listAclInfo = yarnClient.getQueueAclsInfo(); for (QueueUserACLInfo aclInfo : listAclInfo) { for (QueueACL userAcl : aclInfo.getUserAcls()) { LOG.info("User ACL Info for Queue" + ", queueName=" + aclInfo.getQueueName() + ", userAcl=" + userAcl.name()); } } // Get a new application id YarnClientApplication app = yarnClient.createApplication(); GetNewApplicationResponse appResponse = app.getNewApplicationResponse(); // TODO get min/max resource capabilities from RM and change memory ask if needed // If we do not have min/max, we may not be able to correctly request // the required resources from the RM for the app master // Memory ask has to be a multiple of min and less than max. // Dump out information about cluster capability as seen by the resource manager int maxMem = appResponse.getMaximumResourceCapability().getMemory(); LOG.info("Max mem capabililty of resources in this cluster " + maxMem); // A resource ask cannot exceed the max. if (amMemory > maxMem) { LOG.info("AM memory specified above max threshold of cluster. Using max value." + ", specified=" + amMemory + ", max=" + maxMem); amMemory = maxMem; } // set the application name ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); ApplicationId appId = appContext.getApplicationId(); appContext.setApplicationName(appName); // Set up the container launch context for the application master ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); // set local resources for the application master // local files or archives as needed // In this scenario, the jar file for the application master is part of the local resources Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); LOG.info("Copy App Master jar from local filesystem and add to local environment"); // Copy the application master jar to the filesystem // Create a local resource to point to the destination jar path FileSystem fs = FileSystem.get(conf); Path src = new Path(appMasterJar); String pathSuffix = appName + "/" + appId.getId() + "/AppMaster.jar"; Path dst = new Path(fs.getHomeDirectory(), pathSuffix); fs.copyFromLocalFile(false, true, src, dst); FileStatus destStatus = fs.getFileStatus(dst); LocalResource amJarRsrc = Records.newRecord(LocalResource.class); // Set the type of resource - file or archive // archives are untarred at destination // we don't need the jar file to be untarred for now amJarRsrc.setType(LocalResourceType.FILE); // Set visibility of the resource // Setting to most private option amJarRsrc.setVisibility(LocalResourceVisibility.APPLICATION); // Set the resource to be copied over amJarRsrc.setResource(ConverterUtils.getYarnUrlFromPath(dst)); // Set timestamp and length of file so that the framework // can do basic sanity checks for the local resource // after it has been copied over to ensure it is the same // resource the client intended to use with the application amJarRsrc.setTimestamp(destStatus.getModificationTime()); amJarRsrc.setSize(destStatus.getLen()); localResources.put("AppMaster.jar", amJarRsrc); String confXMLFSPath = ""; { File clusterConfXML = new File("cluster-conf.xml"); conf.writeXml(new FileOutputStream(clusterConfXML)); Path confSrc = new Path(clusterConfXML.getAbsolutePath()); String confPathSuffix = appName + "/" + appId.getId() + "/cluster-conf.xml"; Path confDst = new Path(fs.getHomeDirectory(), confPathSuffix); fs.copyFromLocalFile(false, true, confSrc, confDst); FileStatus confFileStatus = fs.getFileStatus(confDst); LocalResource confRsrc = Records.newRecord(LocalResource.class); confRsrc.setType(LocalResourceType.FILE); confRsrc.setVisibility(LocalResourceVisibility.APPLICATION); confRsrc.setResource(ConverterUtils.getYarnUrlFromURI(confDst.toUri())); confRsrc.setSize(confFileStatus.getLen()); confRsrc.setTimestamp(confFileStatus.getModificationTime()); localResources.put("cluster-conf.xml", confRsrc); confXMLFSPath = confDst.toUri().getPath(); } // Set the log4j properties if needed if (!log4jPropFile.isEmpty()) { Path log4jSrc = new Path(log4jPropFile); Path log4jDst = new Path(fs.getHomeDirectory(), "log4j.props"); fs.copyFromLocalFile(false, true, log4jSrc, log4jDst); FileStatus log4jFileStatus = fs.getFileStatus(log4jDst); LocalResource log4jRsrc = Records.newRecord(LocalResource.class); log4jRsrc.setType(LocalResourceType.FILE); log4jRsrc.setVisibility(LocalResourceVisibility.APPLICATION); log4jRsrc.setResource(ConverterUtils.getYarnUrlFromURI(log4jDst.toUri())); log4jRsrc.setTimestamp(log4jFileStatus.getModificationTime()); log4jRsrc.setSize(log4jFileStatus.getLen()); localResources.put("log4j.properties", log4jRsrc); } // The shell script has to be made available on the final container(s) // where it will be executed. // To do this, we need to first copy into the filesystem that is visible // to the yarn framework. // We do not need to set this as a local resource for the application // master as the application master does not need it. String hdfsShellScriptLocation = ""; long hdfsShellScriptLen = 0; long hdfsShellScriptTimestamp = 0; if (!shellScriptPath.isEmpty()) { Path shellSrc = new Path(shellScriptPath); String shellPathSuffix = appName + "/" + appId.getId() + "/ExecShellScript.sh"; Path shellDst = new Path(fs.getHomeDirectory(), shellPathSuffix); fs.copyFromLocalFile(false, true, shellSrc, shellDst); hdfsShellScriptLocation = shellDst.toUri().toString(); FileStatus shellFileStatus = fs.getFileStatus(shellDst); hdfsShellScriptLen = shellFileStatus.getLen(); hdfsShellScriptTimestamp = shellFileStatus.getModificationTime(); } // Set local resource info into app master container launch context amContainer.setLocalResources(localResources); // Set the necessary security tokens as needed //amContainer.setContainerTokens(containerToken); // Set the env variables to be setup in the env where the application master will be run LOG.info("Set the environment for the application master"); Map<String, String> env = new HashMap<String, String>(); // put location of shell script into env // using the env info, the application master will create the correct local resource for the // eventual containers that will be launched to execute the shell scripts env.put(DSConstants.DISTRIBUTEDSHELLSCRIPTLOCATION, hdfsShellScriptLocation); env.put(DSConstants.DISTRIBUTEDSHELLSCRIPTTIMESTAMP, Long.toString(hdfsShellScriptTimestamp)); env.put(DSConstants.DISTRIBUTEDSHELLSCRIPTLEN, Long.toString(hdfsShellScriptLen)); env.put(DSConstants.CLUSTER_CONF_XML_PATH, confXMLFSPath); // Add AppMaster.jar location to classpath // At some point we should not be required to add // the hadoop specific classpaths to the env. // It should be provided out of the box. // For now setting all required classpaths including // the classpath to "." for the application jar StringBuilder classPathEnv = new StringBuilder(Environment.CLASSPATH.$()).append(File.pathSeparatorChar) .append("./*"); for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH, YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) { classPathEnv.append(File.pathSeparatorChar); classPathEnv.append(c.trim()); } classPathEnv.append(File.pathSeparatorChar).append("./log4j.properties"); // add the runtime classpath needed for tests to work if (conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) { classPathEnv.append(':'); classPathEnv.append(System.getProperty("java.class.path")); } env.put("CLASSPATH", classPathEnv.toString()); amContainer.setEnvironment(env); // Set the necessary command to execute the application master Vector<CharSequence> vargs = new Vector<CharSequence>(30); // Set java executable command LOG.info("Setting up app master command"); vargs.add(Environment.JAVA_HOME.$() + "/bin/java"); // Set Xmx based on am memory size vargs.add("-Xmx" + amMemory + "m"); // if (debugFlag) { // vargs.add("-Xdebug -Xrunjdwp:transport=dt_socket,address=9998,server=y,suspend=y"); // } // Set class name vargs.add(appMasterMainClass); // Set params for Application Master vargs.add("--container_memory " + String.valueOf(containerMemory)); vargs.add("--num_containers " + String.valueOf(numContainers)); vargs.add("--priority " + String.valueOf(shellCmdPriority)); if (!shellCommand.isEmpty()) { vargs.add("--shell_command " + shellCommand + ""); } if (!shellArgs.isEmpty()) { vargs.add("--shell_args " + shellArgs + ""); } for (Map.Entry<String, String> entry : shellEnv.entrySet()) { vargs.add("--shell_env " + entry.getKey() + "=" + entry.getValue()); } if (debugFlag) { vargs.add("--debug"); } vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stdout"); vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stderr"); // Get final commmand StringBuilder command = new StringBuilder(); for (CharSequence str : vargs) { command.append(str).append(" "); } LOG.info("Completed setting up app master command " + command.toString()); List<String> commands = new ArrayList<String>(); commands.add(command.toString()); amContainer.setCommands(commands); // Set up resource type requirements // For now, only memory is supported so we set memory requirements Resource capability = Records.newRecord(Resource.class); capability.setMemory(amMemory); appContext.setResource(capability); // Service data is a binary blob that can be passed to the application // Not needed in this scenario // amContainer.setServiceData(serviceData); // Setup security tokens if (UserGroupInformation.isSecurityEnabled()) { Credentials credentials = new Credentials(); String tokenRenewer = conf.get(YarnConfiguration.RM_PRINCIPAL); if (tokenRenewer == null || tokenRenewer.length() == 0) { throw new IOException("Can't get Master Kerberos principal for the RM to use as renewer"); } // For now, only getting tokens for the default file-system. final Token<?> tokens[] = fs.addDelegationTokens(tokenRenewer, credentials); if (tokens != null) { for (Token<?> token : tokens) { LOG.info("Got dt for " + fs.getUri() + "; " + token); } } DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); ByteBuffer fsTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); amContainer.setTokens(fsTokens); } appContext.setAMContainerSpec(amContainer); // Set the priority for the application master Priority pri = Records.newRecord(Priority.class); // TODO - what is the range for priority? how to decide? pri.setPriority(amPriority); appContext.setPriority(pri); // Set the queue to which this application is to be submitted in the RM appContext.setQueue(amQueue); // Submit the application to the applications manager // SubmitApplicationResponse submitResp = applicationsManager.submitApplication(appRequest); // Ignore the response as either a valid response object is returned on success // or an exception thrown to denote some form of a failure LOG.info("Submitting application to ASM"); yarnClient.submitApplication(appContext); // TODO // Try submitting the same request again // app submission failure? // Monitor the application return monitorApplication(appId); }
From source file:com.sogou.dockeronyarn.client.DockerClient.java
/** * Main run function for the client/* ww w . ja va 2 s . c om*/ * @return true if application completed successfully * @throws IOException * @throws YarnException */ public ApplicationId run() throws IOException, YarnException { LOG.info("Running Client"); yarnClient.start(); YarnClusterMetrics clusterMetrics = yarnClient.getYarnClusterMetrics(); LOG.info("Got Cluster metric info from ASM" + ", numNodeManagers=" + clusterMetrics.getNumNodeManagers()); List<NodeReport> clusterNodeReports = yarnClient.getNodeReports(NodeState.RUNNING); LOG.info("Got Cluster node info from ASM"); for (NodeReport node : clusterNodeReports) { LOG.info("Got node report from ASM for" + ", nodeId=" + node.getNodeId() + ", nodeAddress" + node.getHttpAddress() + ", nodeRackName" + node.getRackName() + ", nodeNumContainers" + node.getNumContainers()); } QueueInfo queueInfo = yarnClient.getQueueInfo(this.amQueue); LOG.info("Queue info" + ", queueName=" + queueInfo.getQueueName() + ", queueCurrentCapacity=" + queueInfo.getCurrentCapacity() + ", queueMaxCapacity=" + queueInfo.getMaximumCapacity() + ", queueApplicationCount=" + queueInfo.getApplications().size() + ", queueChildQueueCount=" + queueInfo.getChildQueues().size()); List<QueueUserACLInfo> listAclInfo = yarnClient.getQueueAclsInfo(); for (QueueUserACLInfo aclInfo : listAclInfo) { for (QueueACL userAcl : aclInfo.getUserAcls()) { LOG.info("User ACL Info for Queue" + ", queueName=" + aclInfo.getQueueName() + ", userAcl=" + userAcl.name()); } } // Get a new application id YarnClientApplication app = yarnClient.createApplication(); GetNewApplicationResponse appResponse = app.getNewApplicationResponse(); // TODO get min/max resource capabilities from RM and change memory ask if needed // If we do not have min/max, we may not be able to correctly request // the required resources from the RM for the app master // Memory ask has to be a multiple of min and less than max. // Dump out information about cluster capability as seen by the resource manager int maxMem = appResponse.getMaximumResourceCapability().getMemory(); LOG.info("Max mem capabililty of resources in this cluster " + maxMem); // A resource ask cannot exceed the max. if (amMemory > maxMem) { LOG.info("AM memory specified above max threshold of cluster. Using max value." + ", specified=" + amMemory + ", max=" + maxMem); amMemory = maxMem; } int maxVCores = appResponse.getMaximumResourceCapability().getVirtualCores(); LOG.info("Max virtual cores capabililty of resources in this cluster " + maxVCores); if (amVCores > maxVCores) { LOG.info("AM virtual cores specified above max threshold of cluster. " + "Using max value." + ", specified=" + amVCores + ", max=" + maxVCores); amVCores = maxVCores; } // set the application name ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); ApplicationId appId = appContext.getApplicationId(); //appContext.setKeepContainersAcrossApplicationAttempts(keepContainers); appContext.setApplicationName(appName); // set local resources for the application master // local files or archives as needed // In this scenario, the jar file for the application master is part of the local resources Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); LOG.info("Copy App Master jar from local filesystem and add to local environment"); // Copy the application master jar to the filesystem // Create a local resource to point to the destination jar path FileSystem fs = FileSystem.get(conf); addToLocalResources(fs, appMasterJar, appMasterJarPath, appId.toString(), localResources, null); // Set the log4j properties if needed if (!log4jPropFile.isEmpty()) { addToLocalResources(fs, log4jPropFile, log4jPath, appId.toString(), localResources, null); } // The shell script has to be made available on the final container(s) // where it will be executed. // To do this, we need to first copy into the filesystem that is visible // to the yarn framework. // We do not need to set this as a local resource for the application // master as the application master does not need it. String hdfsShellScriptLocation = ""; long hdfsShellScriptLen = 0; long hdfsShellScriptTimestamp = 0; //if (!shellScriptPath.isEmpty()) { // Path shellSrc = new Path(fs.getHomeDirectory(), SCRIPT_PATH); String shellPathSuffix = SCRIPT_PATH; Path shellDst = new Path(fs.getHomeDirectory(), shellPathSuffix); //fs.copyFromLocalFile(false, true, shellSrc, shellDst); hdfsShellScriptLocation = shellDst.toUri().toString(); FileStatus shellFileStatus = fs.getFileStatus(shellDst); hdfsShellScriptLen = shellFileStatus.getLen(); hdfsShellScriptTimestamp = shellFileStatus.getModificationTime(); //} if (shellArgs.length > 0) { addToLocalResources(fs, null, shellArgsPath, appId.toString(), localResources, StringUtils.join(shellArgs, " ")); } // Set the necessary security tokens as needed //amContainer.setContainerTokens(containerToken); // Set the env variables to be setup in the env where the application master will be run LOG.info("Set the environment for the application master"); Map<String, String> env = new HashMap<String, String>(); // put location of shell script into env // using the env info, the application master will create the correct local resource for the // eventual containers that will be launched to execute the shell scripts env.put(DSConstants.DISTRIBUTEDSHELLSCRIPTLOCATION, hdfsShellScriptLocation); env.put(DSConstants.DISTRIBUTEDSHELLSCRIPTTIMESTAMP, Long.toString(hdfsShellScriptTimestamp)); env.put(DSConstants.DISTRIBUTEDSHELLSCRIPTLEN, Long.toString(hdfsShellScriptLen)); // Add AppMaster.jar location to classpath // At some point we should not be required to add // the hadoop specific classpaths to the env. // It should be provided out of the box. // For now setting all required classpaths including // the classpath to "." for the application jar StringBuilder classPathEnv = new StringBuilder(Environment.CLASSPATH.$()).append(File.pathSeparatorChar) .append("./*"); // StringBuilder classPathEnv = new StringBuilder(Environment.CLASSPATH.$$()) // .append(ApplicationConstants.CLASS_PATH_SEPARATOR).append("./*"); for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH, YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) { classPathEnv.append(File.pathSeparatorChar); classPathEnv.append(c.trim()); } classPathEnv.append(File.pathSeparatorChar).append("./log4j.properties"); // for (String c : conf.getStrings( // YarnConfiguration.YARN_APPLICATION_CLASSPATH, // YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH)) { // classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR); // classPathEnv.append(c.trim()); // } // classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR).append( // "./log4j.properties"); // add the runtime classpath needed for tests to work if (conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) { classPathEnv.append(':'); classPathEnv.append(System.getProperty("java.class.path")); } env.put("CLASSPATH", classPathEnv.toString()); // Set the necessary command to execute the application master Vector<CharSequence> vargs = new Vector<CharSequence>(30); // Set java executable command LOG.info("Setting up app master command"); vargs.add(Environment.JAVA_HOME.$() + "/bin/java"); //vargs.add(Environment.JAVA_HOME.$$() + "/bin/java"); // Set Xmx based on am memory size vargs.add("-Xmx" + amMemory + "m"); // Set class name vargs.add(appMasterMainClass); // Set params for Application Master vargs.add("--container_memory " + String.valueOf(containerMemory)); vargs.add("--container_vcores " + String.valueOf(containerVirtualCores)); vargs.add("--num_containers " + String.valueOf(numContainers)); vargs.add("--priority " + String.valueOf(shellCmdPriority)); vargs.add("--container_retry " + String.valueOf(this.container_retry)); for (Map.Entry<String, String> entry : shellEnv.entrySet()) { vargs.add("--shell_env " + entry.getKey() + "=" + entry.getValue()); } if (debugFlag) { vargs.add("--debug"); } vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stdout"); vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stderr"); // Get final commmand StringBuilder command = new StringBuilder(); for (CharSequence str : vargs) { command.append(str).append(" "); } LOG.info("Completed setting up app master command " + command.toString()); List<String> commands = new ArrayList<String>(); commands.add(command.toString()); // Set up the container launch context for the application master ContainerLaunchContext amContainer = ContainerLaunchContext.newInstance(localResources, env, commands, null, null, null); // Set up resource type requirements // For now, both memory and vcores are supported, so we set memory and // vcores requirements Resource capability = Resource.newInstance(amMemory, amVCores); appContext.setResource(capability); // Service data is a binary blob that can be passed to the application // Not needed in this scenario // amContainer.setServiceData(serviceData); // Setup security tokens if (UserGroupInformation.isSecurityEnabled()) { // Note: Credentials class is marked as LimitedPrivate for HDFS and MapReduce Credentials credentials = new Credentials(); String tokenRenewer = conf.get(YarnConfiguration.RM_PRINCIPAL); if (tokenRenewer == null || tokenRenewer.length() == 0) { throw new IOException("Can't get Master Kerberos principal for the RM to use as renewer"); } // For now, only getting tokens for the default file-system. final Token<?> tokens[] = fs.addDelegationTokens(tokenRenewer, credentials); if (tokens != null) { for (Token<?> token : tokens) { LOG.info("Got dt for " + fs.getUri() + "; " + token); } } DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); ByteBuffer fsTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); amContainer.setTokens(fsTokens); } appContext.setAMContainerSpec(amContainer); // Set the priority for the application master // TODO - what is the range for priority? how to decide? Priority pri = Priority.newInstance(amPriority); appContext.setPriority(pri); // Set the queue to which this application is to be submitted in the RM appContext.setQueue(amQueue); // Submit the application to the applications manager // SubmitApplicationResponse submitResp = applicationsManager.submitApplication(appRequest); // Ignore the response as either a valid response object is returned on success // or an exception thrown to denote some form of a failure LOG.info("Submitting application to ASM"); yarnClient.submitApplication(appContext); // TODO // Try submitting the same request again // app submission failure? // Monitor the application return appId; }