List of usage examples for org.apache.hadoop.fs Path toUri
public URI toUri()
From source file:com.ibm.stocator.fs.common.ObjectStoreGlobber.java
License:Open Source License
private String schemeFromPath(Path path) throws IOException { String scheme = path.toUri().getScheme(); if (scheme == null) { if (fs != null) { scheme = fs.getUri().getScheme(); }/*from w w w. j a v a2 s. co m*/ } return scheme; }
From source file:com.ibm.stocator.fs.common.ObjectStoreGlobber.java
License:Open Source License
private String authorityFromPath(Path path) throws IOException { String authority = path.toUri().getAuthority(); if (authority == null) { if (fs != null) { authority = fs.getUri().getAuthority(); }//from w w w. java 2 s . c om } return authority; }
From source file:com.ibm.stocator.fs.common.ObjectStoreGlobber.java
License:Open Source License
public FileStatus[] glob() throws IOException { // First we get the scheme and authority of the pattern that was passed // in./*from w ww.j ava 2s. c o m*/ LOG.debug("Welcome to glob : " + pathPattern.toString()); String scheme = schemeFromPath(pathPattern); String authority = authorityFromPath(pathPattern); // Next we strip off everything except the pathname itself, and expand all // globs. Expansion is a process which turns "grouping" clauses, // expressed as brackets, into separate path patterns. String pathPatternString = pathPattern.toUri().getPath(); List<String> flattenedPatterns = ObjectStoreGlobExpander.expand(pathPatternString); LOG.debug("expanded : " + pathPatternString); // Now loop over all flattened patterns. In every case, we'll be trying to // match them to entries in the filesystem. ArrayList<FileStatus> results = new ArrayList<FileStatus>(flattenedPatterns.size()); boolean sawWildcard = false; for (String flatPattern : flattenedPatterns) { LOG.debug("pattern from list: " + flatPattern); Path absPattern = new Path(flatPattern.isEmpty() ? Path.CUR_DIR : flatPattern); List<String> components = getPathComponents(absPattern.toUri().getPath()); ArrayList<FileStatus> candidates = new ArrayList<FileStatus>(1); FileStatus rootPlaceholder = new FileStatus(0, true, 0, 0, 0, new Path(scheme, authority, Path.SEPARATOR)); LOG.debug("Going to add candidate: " + rootPlaceholder.getPath().toString()); candidates.add(rootPlaceholder); String cmpCombined = ""; ObjectStoreGlobFilter globFilter = null; for (int componentIdx = 0; componentIdx < components.size() && !sawWildcard; componentIdx++) { globFilter = new ObjectStoreGlobFilter(components.get(componentIdx)); if (globFilter.hasPattern()) { sawWildcard = true; } else { cmpCombined = cmpCombined + "/" + components.get(componentIdx); } } String component = unescapePathComponent(cmpCombined); if (component != null && component.length() > 0) { for (FileStatus candidate : candidates) { candidate.setPath(new Path(candidate.getPath(), component)); } } else { globFilter = new ObjectStoreGlobFilter(components.get(0)); } ArrayList<FileStatus> newCandidates = new ArrayList<FileStatus>(candidates.size()); for (FileStatus candidate : candidates) { if (globFilter.hasPattern()) { FileStatus[] children = listStatus(candidate.getPath()); if (children.length == 1) { if (!getFileStatus(candidate.getPath()).isDirectory()) { continue; } } for (FileStatus child : children) { if (globFilter.accept(child.getPath())) { newCandidates.add(child); } } } else { FileStatus childStatus = null; childStatus = getFileStatus(new Path(candidate.getPath(), component)); if (childStatus != null) { newCandidates.add(childStatus); } } } candidates = newCandidates; for (FileStatus status : candidates) { if (status == rootPlaceholder) { status = getFileStatus(rootPlaceholder.getPath()); if (status == null) { continue; } } if (filter.accept(status.getPath())) { results.add(status); } } } if (!sawWildcard && results.isEmpty() && (flattenedPatterns.size() <= 1)) { return null; } return results.toArray(new FileStatus[0]); }
From source file:com.ibm.stocator.fs.cos.COSAPIClient.java
License:Apache License
/** * Turns a path (relative or otherwise) into an COS key * * @host hostName host of the object//w w w. j a v a 2 s .c o m * @param path object full path */ private String pathToKey(String hostName, Path path) { if (!path.isAbsolute()) { String pathStr = path.toUri().getPath(); if (pathStr.startsWith(mBucket) && !pathStr.equals(mBucket)) { path = new Path(pathStr.substring(mBucket.length() + 1)); } path = new Path(hostName, path); } if (path.toUri().getScheme() != null && path.toUri().getPath().isEmpty()) { return ""; } return path.toUri().getPath().substring(1); }
From source file:com.ikanow.infinit.e.data_model.custom.InfiniteFileInputReader.java
License:Apache License
private static Path createNewName(Path subFile, String replacement) throws MalformedURLException, UnsupportedEncodingException, URISyntaxException { String path = subFile.toUri().toString(); // (currently the entire string) String name = subFile.getName(); int startOfName = path.lastIndexOf(name); return new Path(replacement.replace("$name", name).replace("$path", path.substring(0, startOfName - 1))); }
From source file:com.ikanow.infinit.e.processing.custom.launcher.CustomHadoopTaskLauncher.java
License:Open Source License
@SuppressWarnings({ "unchecked", "rawtypes" })
public String runHadoopJob(CustomMapReduceJobPojo job, String tempJarLocation)
throws IOException, SAXException, ParserConfigurationException {
StringWriter xml = new StringWriter();
String outputCollection = job.outputCollectionTemp;// (non-append mode)
if ((null != job.appendResults) && job.appendResults)
outputCollection = job.outputCollection; // (append mode, write directly in....)
else if (null != job.incrementalMode)
job.incrementalMode = false; // (not allowed to be in incremental mode and not update mode)
createConfigXML(xml, job.jobtitle, job.inputCollection,
InfiniteHadoopUtils.getQueryOrProcessing(job.query, InfiniteHadoopUtils.QuerySpec.INPUTFIELDS),
job.isCustomTable, job.getOutputDatabase(), job._id.toString(), outputCollection, job.mapper,
job.reducer, job.combiner,//from ww w . j a v a 2s . c o m
InfiniteHadoopUtils.getQueryOrProcessing(job.query, InfiniteHadoopUtils.QuerySpec.QUERY),
job.communityIds, job.outputKey, job.outputValue, job.arguments, job.incrementalMode,
job.submitterID, job.selfMerge, job.outputCollection, job.appendResults);
ClassLoader savedClassLoader = Thread.currentThread().getContextClassLoader();
URLClassLoader child = new URLClassLoader(new URL[] { new File(tempJarLocation).toURI().toURL() },
savedClassLoader);
Thread.currentThread().setContextClassLoader(child);
// Check version: for now, any infinit.e.data_model with an VersionTest class is acceptable
boolean dataModelLoaded = true;
try {
URLClassLoader versionTest = new URLClassLoader(new URL[] { new File(tempJarLocation).toURI().toURL() },
null);
try {
Class.forName("com.ikanow.infinit.e.data_model.custom.InfiniteMongoInputFormat", true, versionTest);
} catch (ClassNotFoundException e2) {
//(this is fine, will use the cached version)
dataModelLoaded = false;
}
if (dataModelLoaded)
Class.forName("com.ikanow.infinit.e.data_model.custom.InfiniteMongoVersionTest", true, versionTest);
} catch (ClassNotFoundException e1) {
throw new RuntimeException(
"This JAR is compiled with too old a version of the data-model, please recompile with Jan 2014 (rc2) onwards");
}
// Now load the XML into a configuration object:
Configuration config = new Configuration();
// Add the client configuration overrides:
if (!bLocalMode) {
String hadoopConfigPath = props_custom.getHadoopConfigPath() + "/hadoop/";
config.addResource(new Path(hadoopConfigPath + "core-site.xml"));
config.addResource(new Path(hadoopConfigPath + "mapred-site.xml"));
config.addResource(new Path(hadoopConfigPath + "hadoop-site.xml"));
} //TESTED
try {
DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
Document doc = dBuilder.parse(new ByteArrayInputStream(xml.toString().getBytes()));
NodeList nList = doc.getElementsByTagName("property");
for (int temp = 0; temp < nList.getLength(); temp++) {
Node nNode = nList.item(temp);
if (nNode.getNodeType() == Node.ELEMENT_NODE) {
Element eElement = (Element) nNode;
String name = getTagValue("name", eElement);
String value = getTagValue("value", eElement);
if ((null != name) && (null != value)) {
config.set(name, value);
}
}
}
} catch (Exception e) {
throw new IOException(e.getMessage());
}
// Some other config defaults:
// (not sure if these are actually applied, or derived from the defaults - for some reason they don't appear in CDH's client config)
config.set("mapred.map.tasks.speculative.execution", "false");
config.set("mapred.reduce.tasks.speculative.execution", "false");
// (default security is ignored here, have it set via HADOOP_TASKTRACKER_CONF in cloudera)
// Now run the JAR file
try {
BasicDBObject advancedConfigurationDbo = null;
try {
advancedConfigurationDbo = (null != job.query)
? ((BasicDBObject) com.mongodb.util.JSON.parse(job.query))
: (new BasicDBObject());
} catch (Exception e) {
advancedConfigurationDbo = new BasicDBObject();
}
boolean esMode = advancedConfigurationDbo.containsField("qt") && !job.isCustomTable;
if (esMode && !job.inputCollection.equals("doc_metadata.metadata")) {
throw new RuntimeException(
"Infinit.e Queries are only supported on doc_metadata - use MongoDB queries instead.");
}
config.setBoolean("mapred.used.genericoptionsparser", true); // (just stops an annoying warning from appearing)
if (bLocalMode) { // local job tracker and FS mode
config.set("mapred.job.tracker", "local");
config.set("fs.default.name", "local");
} else {
if (bTestMode) { // run job tracker locally but FS mode remotely
config.set("mapred.job.tracker", "local");
} else { // normal job tracker
String trackerUrl = HadoopUtils.getXMLProperty(
props_custom.getHadoopConfigPath() + "/hadoop/mapred-site.xml", "mapred.job.tracker");
config.set("mapred.job.tracker", trackerUrl);
}
String fsUrl = HadoopUtils.getXMLProperty(
props_custom.getHadoopConfigPath() + "/hadoop/core-site.xml", "fs.default.name");
config.set("fs.default.name", fsUrl);
}
if (!dataModelLoaded && !(bTestMode || bLocalMode)) { // If running distributed and no data model loaded then add ourselves
Path jarToCache = InfiniteHadoopUtils.cacheLocalFile("/opt/infinite-home/lib/",
"infinit.e.data_model.jar", config);
DistributedCache.addFileToClassPath(jarToCache, config);
jarToCache = InfiniteHadoopUtils.cacheLocalFile("/opt/infinite-home/lib/",
"infinit.e.processing.custom.library.jar", config);
DistributedCache.addFileToClassPath(jarToCache, config);
} //TESTED
// Debug scripts (only if they exist), and only in non local/test mode
if (!bLocalMode && !bTestMode) {
try {
Path scriptToCache = InfiniteHadoopUtils.cacheLocalFile("/opt/infinite-home/scripts/",
"custom_map_error_handler.sh", config);
config.set("mapred.map.task.debug.script", "custom_map_error_handler.sh " + job.jobtitle);
config.set("mapreduce.map.debug.script", "custom_map_error_handler.sh " + job.jobtitle);
DistributedCache.createSymlink(config);
DistributedCache.addCacheFile(scriptToCache.toUri(), config);
} catch (Exception e) {
} // just carry on
try {
Path scriptToCache = InfiniteHadoopUtils.cacheLocalFile("/opt/infinite-home/scripts/",
"custom_reduce_error_handler.sh", config);
config.set("mapred.reduce.task.debug.script", "custom_reduce_error_handler.sh " + job.jobtitle);
config.set("mapreduce.reduce.debug.script", "custom_reduce_error_handler.sh " + job.jobtitle);
DistributedCache.createSymlink(config);
DistributedCache.addCacheFile(scriptToCache.toUri(), config);
} catch (Exception e) {
} // just carry on
} //TODO (???): TOTEST
// (need to do these 2 things here before the job is created, at which point the config class has been copied across)
//1)
Class<?> mapperClazz = Class.forName(job.mapper, true, child);
if (ICustomInfiniteInternalEngine.class.isAssignableFrom(mapperClazz)) { // Special case: internal custom engine, so gets an additional integration hook
ICustomInfiniteInternalEngine preActivities = (ICustomInfiniteInternalEngine) mapperClazz
.newInstance();
preActivities.preTaskActivities(job._id, job.communityIds, config, !(bTestMode || bLocalMode));
} //TESTED
//2)
if (job.inputCollection.equalsIgnoreCase("file.binary_shares")) {
// Need to download the GridFSZip file
try {
Path jarToCache = InfiniteHadoopUtils.cacheLocalFile("/opt/infinite-home/lib/unbundled/",
"GridFSZipFile.jar", config);
DistributedCache.addFileToClassPath(jarToCache, config);
} catch (Throwable t) {
} // (this is fine, will already be on the classpath .. otherwise lots of other stuff will be failing all over the place!)
}
if (job.inputCollection.equals("records")) {
InfiniteElasticsearchHadoopUtils.handleElasticsearchInput(job, config, advancedConfigurationDbo);
//(won't run under 0.19 so running with "records" should cause all sorts of exceptions)
} //TESTED (by hand)
if (bTestMode || bLocalMode) { // If running locally, turn "snappy" off - tomcat isn't pointing its native library path in the right place
config.set("mapred.map.output.compression.codec", "org.apache.hadoop.io.compress.DefaultCodec");
}
// Manually specified caches
List<URL> localJarCaches = InfiniteHadoopUtils.handleCacheList(advancedConfigurationDbo.get("$caches"),
job, config, props_custom);
Job hj = new Job(config); // (NOTE: from here, changes to config are ignored)
try {
if (null != localJarCaches) {
if (bLocalMode || bTestMode) {
Method method = URLClassLoader.class.getDeclaredMethod("addURL", new Class[] { URL.class });
method.setAccessible(true);
method.invoke(child, localJarCaches.toArray());
} //TOTEST (tested logically)
}
Class<?> classToLoad = Class.forName(job.mapper, true, child);
hj.setJarByClass(classToLoad);
if (job.inputCollection.equalsIgnoreCase("filesystem")) {
String inputPath = null;
try {
inputPath = MongoDbUtil.getProperty(advancedConfigurationDbo, "file.url");
if (!inputPath.endsWith("/")) {
inputPath = inputPath + "/";
}
} catch (Exception e) {
}
if (null == inputPath) {
throw new RuntimeException("Must specify 'file.url' if reading from filesystem.");
}
inputPath = InfiniteHadoopUtils.authenticateInputDirectory(job, inputPath);
InfiniteFileInputFormat.addInputPath(hj, new Path(inputPath + "*/*")); // (that extra bit makes it recursive)
InfiniteFileInputFormat.setMaxInputSplitSize(hj, 33554432); // (32MB)
InfiniteFileInputFormat.setInfiniteInputPathFilter(hj, config);
hj.setInputFormatClass((Class<? extends InputFormat>) Class.forName(
"com.ikanow.infinit.e.data_model.custom.InfiniteFileInputFormat", true, child));
} else if (job.inputCollection.equalsIgnoreCase("file.binary_shares")) {
String[] oidStrs = null;
try {
String inputPath = MongoDbUtil.getProperty(advancedConfigurationDbo, "file.url");
Pattern oidExtractor = Pattern.compile("inf://share/([^/]+)");
Matcher m = oidExtractor.matcher(inputPath);
if (m.find()) {
oidStrs = m.group(1).split("\\s*,\\s*");
} else {
throw new RuntimeException(
"file.url must be in format inf://share/<oid-list>/<string>: " + inputPath);
}
InfiniteHadoopUtils.authenticateShareList(job, oidStrs);
} catch (Exception e) {
throw new RuntimeException(
"Authentication error: " + e.getMessage() + ": " + advancedConfigurationDbo, e);
}
hj.getConfiguration().setStrings("mapred.input.dir", oidStrs);
hj.setInputFormatClass((Class<? extends InputFormat>) Class.forName(
"com.ikanow.infinit.e.data_model.custom.InfiniteShareInputFormat", true, child));
} else if (job.inputCollection.equals("records")) {
hj.setInputFormatClass((Class<? extends InputFormat>) Class
.forName("com.ikanow.infinit.e.data_model.custom.InfiniteEsInputFormat", true, child));
} else {
if (esMode) {
hj.setInputFormatClass((Class<? extends InputFormat>) Class.forName(
"com.ikanow.infinit.e.processing.custom.utils.InfiniteElasticsearchMongoInputFormat",
true, child));
} else {
hj.setInputFormatClass((Class<? extends InputFormat>) Class.forName(
"com.ikanow.infinit.e.data_model.custom.InfiniteMongoInputFormat", true, child));
}
}
if ((null != job.exportToHdfs) && job.exportToHdfs) {
//TODO (INF-2469): Also, if the output key is BSON then also run as text (but output as JSON?)
Path outPath = InfiniteHadoopUtils.ensureOutputDirectory(job, props_custom);
if ((null != job.outputKey) && (null != job.outputValue)
&& job.outputKey.equalsIgnoreCase("org.apache.hadoop.io.text")
&& job.outputValue.equalsIgnoreCase("org.apache.hadoop.io.text")) {
// (slight hack before I sort out the horrendous job class - if key/val both text and exporting to HDFS then output as Text)
hj.setOutputFormatClass((Class<? extends OutputFormat>) Class
.forName("org.apache.hadoop.mapreduce.lib.output.TextOutputFormat", true, child));
TextOutputFormat.setOutputPath(hj, outPath);
} //TESTED
else {
hj.setOutputFormatClass((Class<? extends OutputFormat>) Class.forName(
"org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat", true, child));
SequenceFileOutputFormat.setOutputPath(hj, outPath);
} //TESTED
} else { // normal case, stays in MongoDB
hj.setOutputFormatClass((Class<? extends OutputFormat>) Class.forName(
"com.ikanow.infinit.e.data_model.custom.InfiniteMongoOutputFormat", true, child));
}
hj.setMapperClass((Class<? extends Mapper>) mapperClazz);
String mapperOutputKeyOverride = advancedConfigurationDbo.getString("$mapper_key_class", null);
if (null != mapperOutputKeyOverride) {
hj.setMapOutputKeyClass(Class.forName(mapperOutputKeyOverride));
} //TESTED
String mapperOutputValueOverride = advancedConfigurationDbo.getString("$mapper_value_class", null);
if (null != mapperOutputValueOverride) {
hj.setMapOutputValueClass(Class.forName(mapperOutputValueOverride));
} //TESTED
if ((null != job.reducer) && !job.reducer.startsWith("#") && !job.reducer.equalsIgnoreCase("null")
&& !job.reducer.equalsIgnoreCase("none")) {
hj.setReducerClass((Class<? extends Reducer>) Class.forName(job.reducer, true, child));
// Variable reducers:
if (null != job.query) {
try {
hj.setNumReduceTasks(advancedConfigurationDbo.getInt("$reducers", 1));
} catch (Exception e) {
try {
// (just check it's not a string that is a valid int)
hj.setNumReduceTasks(
Integer.parseInt(advancedConfigurationDbo.getString("$reducers", "1")));
} catch (Exception e2) {
}
}
} //TESTED
} else {
hj.setNumReduceTasks(0);
}
if ((null != job.combiner) && !job.combiner.startsWith("#")
&& !job.combiner.equalsIgnoreCase("null") && !job.combiner.equalsIgnoreCase("none")) {
hj.setCombinerClass((Class<? extends Reducer>) Class.forName(job.combiner, true, child));
}
hj.setOutputKeyClass(Class.forName(job.outputKey, true, child));
hj.setOutputValueClass(Class.forName(job.outputValue, true, child));
hj.setJobName(job.jobtitle);
currJobName = job.jobtitle;
} catch (Error e) { // (messing about with class loaders = lots of chances for errors!)
throw new RuntimeException(e.getMessage(), e);
}
if (bTestMode || bLocalMode) {
hj.submit();
currThreadId = null;
Logger.getRootLogger().addAppender(this);
currLocalJobId = hj.getJobID().toString();
currLocalJobErrs.setLength(0);
while (!hj.isComplete()) {
Thread.sleep(1000);
}
Logger.getRootLogger().removeAppender(this);
if (hj.isSuccessful()) {
if (this.currLocalJobErrs.length() > 0) {
return "local_done: " + this.currLocalJobErrs.toString();
} else {
return "local_done";
}
} else {
return "Error: " + this.currLocalJobErrs.toString();
}
} else {
hj.submit();
String jobId = hj.getJobID().toString();
return jobId;
}
} catch (Exception e) {
e.printStackTrace();
Thread.currentThread().setContextClassLoader(savedClassLoader);
return "Error: " + InfiniteHadoopUtils.createExceptionMessage(e);
} finally {
Thread.currentThread().setContextClassLoader(savedClassLoader);
}
}
From source file:com.ikanow.infinit.e.processing.custom.utils.InfiniteHadoopUtils.java
License:Open Source License
@SuppressWarnings("rawtypes") public static List<URL> handleCacheList(Object cacheFileList, CustomMapReduceJobPojo job, Configuration config, PropertiesManager prop_custom) throws MalformedURLException, IOException, Exception { if (null == cacheFileList) { return null; }//w ww .ja v a2 s.com LinkedList<URL> localJarCache = null; Collection cacheFiles = null; if (cacheFileList instanceof String) { // comma separated list String[] cacheFilesArray = ((String) cacheFileList).split("\\s*,\\s*"); cacheFiles = Arrays.asList(cacheFilesArray); } else { cacheFiles = (Collection) cacheFileList; } for (Object cache : cacheFiles) { String cacheStr = (String) cache; ObjectId cacheId = null; try { cacheId = new ObjectId(cacheStr); } catch (Exception e) { } // fine FileSystem fs = null; if ((null != cacheId) || cacheStr.startsWith("http:") || cacheStr.startsWith("https:") || cacheStr.startsWith("$")) { if (null != cacheId) { // this might be a custom cache in which case just bypass all this, handled in the main list if (checkIfSourceOrCustomAndAuthenticate(null, cacheId, job)) { continue; } } //TESTED (by hand = skip and continue) // Use existing code to cache to local fs (and then onwards to HDFS!) URL localPathURL = new File( downloadJarFile(cacheStr, job.communityIds, prop_custom, job.submitterID)).toURI().toURL(); String localPath = localPathURL.getPath(); String pathMinusName = localPath.substring(0, localPath.lastIndexOf('/') + 1); String name = localPath.substring(localPath.lastIndexOf('/') + 1); Path distPath = cacheLocalFile(pathMinusName, name, config); if (name.endsWith(".jar")) { if (null == localJarCache) { localJarCache = new LinkedList<URL>(); } localJarCache.add(localPathURL); DistributedCache.addFileToClassPath(distPath, config); } //TESTED else if (name.endsWith(".zip") || name.endsWith("gz")) { DistributedCache.addCacheArchive(distPath.toUri(), config); } //TESTED else { DistributedCache.addCacheFile(distPath.toUri(), config); } //TESTED } else { // this is the location of a file (it is almost certainly an input/output path) if (checkIfSourceOrCustomAndAuthenticate(cacheStr, null, job)) { continue; } //TESTED (by hand - seen it skip if not a jobid/sourcekey - currently not possible for it to be one anyway; c/p from checkIfSourceOrCustomAndAuthenticate call in previous call anyway^2) String path = authenticateInputDirectory(job, cacheStr); if (null == fs) { fs = FileSystem.get(config); } Path distPath = new Path(fs.getFileStatus(new Path(path)).getPath().toUri().getPath()); if (path.endsWith(".jar")) { DistributedCache.addFileToClassPath(distPath, config); } //TESTED else if (path.endsWith(".zip") || path.endsWith("gz")) { DistributedCache.addCacheArchive(distPath.toUri(), config); } //TESTED else { DistributedCache.addCacheFile(distPath.toUri(), config); } //TESTED } //TESTED } return localJarCache; }
From source file:com.inforefiner.hdata.SubmitClient.java
License:Apache License
/** * Main run function for the client// w w w .j av a 2 s. c o m * * @return true if application completed successfully * @throws IOException * @throws YarnException */ public boolean run() throws IOException, YarnException { LOG.info("Running Client"); yarnClient.start(); YarnClusterMetrics clusterMetrics = yarnClient.getYarnClusterMetrics(); LOG.info("Got Cluster metric info from ASM" + ", numNodeManagers=" + clusterMetrics.getNumNodeManagers()); List<NodeReport> clusterNodeReports = yarnClient.getNodeReports(NodeState.RUNNING); LOG.info("Got Cluster node info from ASM"); for (NodeReport node : clusterNodeReports) { LOG.info("Got node report from ASM for" + ", nodeId=" + node.getNodeId() + ", nodeAddress" + node.getHttpAddress() + ", nodeRackName" + node.getRackName() + ", nodeNumContainers" + node.getNumContainers()); } QueueInfo queueInfo = yarnClient.getQueueInfo(this.amQueue); LOG.info("Queue info" + ", queueName=" + queueInfo.getQueueName() + ", queueCurrentCapacity=" + queueInfo.getCurrentCapacity() + ", queueMaxCapacity=" + queueInfo.getMaximumCapacity() + ", queueApplicationCount=" + queueInfo.getApplications().size() + ", queueChildQueueCount=" + queueInfo.getChildQueues().size()); List<QueueUserACLInfo> listAclInfo = yarnClient.getQueueAclsInfo(); for (QueueUserACLInfo aclInfo : listAclInfo) { for (QueueACL userAcl : aclInfo.getUserAcls()) { LOG.info("User ACL Info for Queue" + ", queueName=" + aclInfo.getQueueName() + ", userAcl=" + userAcl.name()); } } if (domainId != null && domainId.length() > 0 && toCreateDomain) { prepareTimelineDomain(); } // Get a new application id YarnClientApplication app = yarnClient.createApplication(); GetNewApplicationResponse appResponse = app.getNewApplicationResponse(); // TODO get min/max resource capabilities from RM and change memory ask if needed // If we do not have min/max, we may not be able to correctly request // the required resources from the RM for the app master // Memory ask has to be a multiple of min and less than max. // Dump out information about cluster capability as seen by the resource manager int maxMem = appResponse.getMaximumResourceCapability().getMemory(); LOG.info("Max mem capabililty of resources in this cluster " + maxMem); // A resource ask cannot exceed the max. if (amMemory > maxMem) { LOG.info("AM memory specified above max threshold of cluster. Using max value." + ", specified=" + amMemory + ", max=" + maxMem); amMemory = maxMem; } int maxVCores = appResponse.getMaximumResourceCapability().getVirtualCores(); LOG.info("Max virtual cores capabililty of resources in this cluster " + maxVCores); if (amVCores > maxVCores) { LOG.info("AM virtual cores specified above max threshold of cluster. " + "Using max value." + ", specified=" + amVCores + ", max=" + maxVCores); amVCores = maxVCores; } // set the application name ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); ApplicationId appId = appContext.getApplicationId(); appContext.setKeepContainersAcrossApplicationAttempts(keepContainers); appContext.setApplicationName(appName); if (attemptFailuresValidityInterval >= 0) { appContext.setAttemptFailuresValidityInterval(attemptFailuresValidityInterval); } // set local resources for the application master // local files or archives as needed // In this scenario, the jar file for the application master is part of the local resources Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); LOG.info("Copy App Master jar from local filesystem and add to local environment"); // Copy the application master jar to the filesystem // Create a local resource to point to the destination jar path FileSystem fs = FileSystem.get(conf); addToLocalResources(fs, appMasterJar, appMasterJarPath, appId.toString(), localResources, null); // Set the log4j properties if needed if (!log4jPropFile.isEmpty()) { addToLocalResources(fs, log4jPropFile, log4jPath, appId.toString(), localResources, null); } // The shell script has to be made available on the final container(s) // where it will be executed. // To do this, we need to first copy into the filesystem that is visible // to the yarn framework. // We do not need to set this as a local resource for the application // master as the application master does not need it. String hdfsShellScriptLocation = ""; long hdfsShellScriptLen = 0; long hdfsShellScriptTimestamp = 0; if (!shellScriptPath.isEmpty()) { Path shellSrc = new Path(shellScriptPath); String shellPathSuffix = appName + "/" + appId.toString() + "/" + SCRIPT_PATH; Path shellDst = new Path(fs.getHomeDirectory(), shellPathSuffix); fs.copyFromLocalFile(false, true, shellSrc, shellDst); hdfsShellScriptLocation = shellDst.toUri().toString(); FileStatus shellFileStatus = fs.getFileStatus(shellDst); hdfsShellScriptLen = shellFileStatus.getLen(); hdfsShellScriptTimestamp = shellFileStatus.getModificationTime(); } if (!shellCommand.isEmpty()) { addToLocalResources(fs, null, shellCommandPath, appId.toString(), localResources, shellCommand); } if (shellArgs.length > 0) { addToLocalResources(fs, null, shellArgsPath, appId.toString(), localResources, StringUtils.join(shellArgs, " ")); } // Set the necessary security tokens as needed //amContainer.setContainerTokens(containerToken); // Set the env variables to be setup in the env where the application master will be run LOG.info("Set the environment for the application master"); Map<String, String> env = new HashMap<String, String>(); // put location of shell script into env // using the env info, the application master will create the correct local resource for the // eventual containers that will be launched to execute the shell scripts env.put(DSConstants.DISTRIBUTEDSHELLSCRIPTLOCATION, hdfsShellScriptLocation); env.put(DSConstants.DISTRIBUTEDSHELLSCRIPTTIMESTAMP, Long.toString(hdfsShellScriptTimestamp)); env.put(DSConstants.DISTRIBUTEDSHELLSCRIPTLEN, Long.toString(hdfsShellScriptLen)); if (domainId != null && domainId.length() > 0) { env.put(DSConstants.DISTRIBUTEDSHELLTIMELINEDOMAIN, domainId); } // Add AppMaster.jar location to classpath // At some point we should not be required to add // the hadoop specific classpaths to the env. // It should be provided out of the box. // For now setting all required classpaths including // the classpath to "." for the application jar StringBuilder classPathEnv = new StringBuilder(Environment.CLASSPATH.$$()) .append(ApplicationConstants.CLASS_PATH_SEPARATOR).append("./*"); for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH, YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH)) { classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR); classPathEnv.append(c.trim()); } classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR).append("./log4j.properties"); // add the runtime classpath needed for tests to work if (conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) { classPathEnv.append(':'); classPathEnv.append(System.getProperty("java.class.path")); } env.put("CLASSPATH", classPathEnv.toString()); // Set the necessary command to execute the application master Vector<CharSequence> vargs = new Vector<CharSequence>(30); // Set java executable command LOG.info("Setting up app master command"); vargs.add(Environment.JAVA_HOME.$$() + "/bin/java"); // Set Xmx based on am memory size vargs.add("-Xmx" + amMemory + "m"); // Set class name vargs.add(appMasterMainClass); // Set params for Application Master vargs.add("--container_memory " + String.valueOf(containerMemory)); vargs.add("--container_vcores " + String.valueOf(containerVirtualCores)); vargs.add("--num_containers " + String.valueOf(numContainers)); if (null != nodeLabelExpression) { appContext.setNodeLabelExpression(nodeLabelExpression); } vargs.add("--priority " + String.valueOf(shellCmdPriority)); for (Map.Entry<String, String> entry : shellEnv.entrySet()) { vargs.add("--shell_env " + entry.getKey() + "=" + entry.getValue()); } if (debugFlag) { vargs.add("--debug"); } vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stdout"); vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stderr"); // Get final commmand StringBuilder command = new StringBuilder(); for (CharSequence str : vargs) { command.append(str).append(" "); } LOG.info("Completed setting up app master command " + command.toString()); List<String> commands = new ArrayList<String>(); commands.add(command.toString()); // Set up the container launch context for the application master ContainerLaunchContext amContainer = ContainerLaunchContext.newInstance(localResources, env, commands, null, null, null); // Set up resource type requirements // For now, both memory and vcores are supported, so we set memory and // vcores requirements Resource capability = Resource.newInstance(amMemory, amVCores); appContext.setResource(capability); // Service data is a binary blob that can be passed to the application // Not needed in this scenario // amContainer.setServiceData(serviceData); // Setup security tokens if (UserGroupInformation.isSecurityEnabled()) { // Note: Credentials class is marked as LimitedPrivate for HDFS and MapReduce Credentials credentials = new Credentials(); String tokenRenewer = conf.get(YarnConfiguration.RM_PRINCIPAL); if (tokenRenewer == null || tokenRenewer.length() == 0) { throw new IOException("Can't get Master Kerberos principal for the RM to use as renewer"); } // For now, only getting tokens for the default file-system. final Token<?> tokens[] = fs.addDelegationTokens(tokenRenewer, credentials); if (tokens != null) { for (Token<?> token : tokens) { LOG.info("Got dt for " + fs.getUri() + "; " + token); } } DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); ByteBuffer fsTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); amContainer.setTokens(fsTokens); } appContext.setAMContainerSpec(amContainer); // Set the priority for the application master // TODO - what is the range for priority? how to decide? Priority pri = Priority.newInstance(amPriority); appContext.setPriority(pri); // Set the queue to which this application is to be submitted in the RM appContext.setQueue(amQueue); // Submit the application to the applications manager // SubmitApplicationResponse submitResp = applicationsManager.submitApplication(appRequest); // Ignore the response as either a valid response object is returned on success // or an exception thrown to denote some form of a failure LOG.info("Submitting application to ASM"); yarnClient.submitApplication(appContext); // TODO // Try submitting the same request again // app submission failure? Thread t = new Thread(new LogReceiver()); t.start(); // Monitor the application return monitorApplication(appId); }
From source file:com.inmobi.conduit.Cluster.java
License:Apache License
public String getUnqaulifiedFinalDestDirRoot() { Path absolutePath = new Path(hdfsUrl); String dest = File.separator + absolutePath.toUri().getPath() + rootDir + File.separator + "streams" + File.separator;/* w w w . ja va 2 s . c o m*/ return dest; }
From source file:com.inmobi.conduit.Cluster.java
License:Apache License
public String getUnqaulifiedReadUrlFinalDestDirRoot() { Path absolutePath = new Path(readUrl); String dest = File.separator + absolutePath.toUri().getPath() + rootDir + File.separator + "streams" + File.separator;// w w w . j ava2 s . co m return dest; }