List of usage examples for org.apache.hadoop.fs FileSystem getHomeDirectory
public Path getHomeDirectory()
From source file:edu.uci.ics.asterix.aoya.AsterixYARNClient.java
License:Apache License
/** * Uploads a AsterixDB cluster configuration to HDFS for the AM to use. * @param overwrite Overwrite existing configurations by the same name. * @throws IllegalStateException// ww w . ja va2 s . c om * @throws IOException */ private void installAsterixConfig(boolean overwrite) throws IllegalStateException, IOException { FileSystem fs = FileSystem.get(conf); File srcfile = new File(MERGED_PARAMETERS_PATH); Path src = new Path(srcfile.getCanonicalPath()); String pathSuffix = CONF_DIR_REL + instanceFolder + File.separator + PARAMS_DEFAULT_NAME; Path dst = new Path(fs.getHomeDirectory(), pathSuffix); if (fs.exists(dst) && !overwrite) { throw new IllegalStateException( "Instance exists. Please delete an existing instance before trying to overwrite"); } fs.copyFromLocalFile(false, true, src, dst); }
From source file:edu.uci.ics.asterix.aoya.AsterixYARNClient.java
License:Apache License
/** * Uploads binary resources to HDFS for use by the AM * @return/*w w w.jav a2s . co m*/ * @throws IOException * @throws YarnException */ public List<DFSResourceCoordinate> distributeBinaries() throws IOException, YarnException { List<DFSResourceCoordinate> resources = new ArrayList<DFSResourceCoordinate>(2); // Copy the application master jar to the filesystem // Create a local resource to point to the destination jar path FileSystem fs = FileSystem.get(conf); Path src, dst; FileStatus destStatus; String pathSuffix; // adding info so we can add the jar to the App master container path // Add the asterix tarfile to HDFS for easy distribution // Keep it all archived for now so add it as a file... pathSuffix = CONF_DIR_REL + instanceFolder + "asterix-server.zip"; dst = new Path(fs.getHomeDirectory(), pathSuffix); if (refresh) { if (fs.exists(dst)) { fs.delete(dst, false); } } if (!fs.exists(dst)) { src = new Path(asterixZip); LOG.info("Copying Asterix distributable to DFS"); fs.copyFromLocalFile(false, true, src, dst); } destStatus = fs.getFileStatus(dst); LocalResource asterixTarLoc = Records.newRecord(LocalResource.class); asterixTarLoc.setType(LocalResourceType.ARCHIVE); asterixTarLoc.setVisibility(LocalResourceVisibility.PRIVATE); asterixTarLoc.setResource(ConverterUtils.getYarnUrlFromPath(dst)); asterixTarLoc.setTimestamp(destStatus.getModificationTime()); // adding info so we can add the tarball to the App master container path DFSResourceCoordinate tar = new DFSResourceCoordinate(); tar.envs.put(dst.toUri().toString(), AConstants.TARLOCATION); tar.envs.put(Long.toString(asterixTarLoc.getSize()), AConstants.TARLEN); tar.envs.put(Long.toString(asterixTarLoc.getTimestamp()), AConstants.TARTIMESTAMP); tar.res = asterixTarLoc; tar.name = "asterix-server.zip"; resources.add(tar); // Set the log4j properties if needed if (!log4jPropFile.isEmpty()) { Path log4jSrc = new Path(log4jPropFile); Path log4jDst = new Path(fs.getHomeDirectory(), "log4j.props"); fs.copyFromLocalFile(false, true, log4jSrc, log4jDst); FileStatus log4jFileStatus = fs.getFileStatus(log4jDst); LocalResource log4jRsrc = Records.newRecord(LocalResource.class); log4jRsrc.setType(LocalResourceType.FILE); log4jRsrc.setVisibility(LocalResourceVisibility.PRIVATE); log4jRsrc.setResource(ConverterUtils.getYarnUrlFromURI(log4jDst.toUri())); log4jRsrc.setTimestamp(log4jFileStatus.getModificationTime()); log4jRsrc.setSize(log4jFileStatus.getLen()); DFSResourceCoordinate l4j = new DFSResourceCoordinate(); tar.res = log4jRsrc; tar.name = "log4j.properties"; resources.add(l4j); } resources.addAll(installAmLibs()); return resources; }
From source file:edu.uci.ics.asterix.aoya.AsterixYARNClient.java
License:Apache License
/** * Submits the request to start the AsterixApplicationMaster to the YARN ResourceManager. * //from w w w . j av a2 s .c om * @param app * The application attempt handle. * @param resources * Resources to be distributed as part of the container launch * @param mode * The mode of the ApplicationMaster * @return The application ID of the new Asterix instance. * @throws IOException * @throws YarnException */ public ApplicationId deployAM(YarnClientApplication app, List<DFSResourceCoordinate> resources, Mode mode) throws IOException, YarnException { // Set up the container launch context for the application master ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); // Set local resource info into app master container launch context Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); for (DFSResourceCoordinate res : resources) { localResources.put(res.name, res.res); } amContainer.setLocalResources(localResources); // Set the env variables to be setup in the env where the application // master will be run LOG.info("Set the environment for the application master"); Map<String, String> env = new HashMap<String, String>(); // using the env info, the application master will create the correct // local resource for the // eventual containers that will be launched to execute the shell // scripts for (DFSResourceCoordinate res : resources) { if (res.envs == null) { //some entries may not have environment variables. continue; } for (Map.Entry<String, String> e : res.envs.entrySet()) { env.put(e.getValue(), e.getKey()); } } //this is needed for when the RM address isn't known from the environment of the AM env.put(AConstants.RMADDRESS, conf.get("yarn.resourcemanager.address")); env.put(AConstants.RMSCHEDULERADDRESS, conf.get("yarn.resourcemanager.scheduler.address")); ///add miscellaneous environment variables. env.put(AConstants.INSTANCESTORE, CONF_DIR_REL + instanceFolder); env.put(AConstants.DFS_BASE, FileSystem.get(conf).getHomeDirectory().toUri().toString()); env.put(AConstants.CC_JAVA_OPTS, ccJavaOpts); env.put(AConstants.NC_JAVA_OPTS, ncJavaOpts); // Add AppMaster.jar location to classpath // At some point we should not be required to add // the hadoop specific classpaths to the env. // It should be provided out of the box. // For now setting all required classpaths including // the classpath to "." for the application jar StringBuilder classPathEnv = new StringBuilder("").append("./*"); for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH, YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) { classPathEnv.append(File.pathSeparatorChar); classPathEnv.append(c.trim()); } classPathEnv.append(File.pathSeparatorChar).append("." + File.separator + "log4j.properties"); // add the runtime classpath needed for tests to work if (conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) { LOG.info("In YARN MiniCluster"); classPathEnv.append(System.getProperty("path.separator")); classPathEnv.append(System.getProperty("java.class.path")); env.put("HADOOP_CONF_DIR", System.getProperty("user.dir") + File.separator + "target" + File.separator); } LOG.info("AM Classpath:" + classPathEnv.toString()); env.put("CLASSPATH", classPathEnv.toString()); amContainer.setEnvironment(env); // Set the necessary command to execute the application master Vector<CharSequence> vargs = new Vector<CharSequence>(30); // Set java executable command LOG.info("Setting up app master command"); vargs.add(JAVA_HOME + File.separator + "bin" + File.separator + "java"); // Set class name vargs.add(appMasterMainClass); //Set params for Application Master if (debugFlag) { vargs.add("-debug"); } if (mode == Mode.DESTROY) { vargs.add("-obliterate"); } else if (mode == Mode.BACKUP) { vargs.add("-backup"); } else if (mode == Mode.RESTORE) { vargs.add("-restore " + snapName); } else if (mode == Mode.INSTALL) { vargs.add("-initial "); } if (refresh) { vargs.add("-refresh"); } //vargs.add("/bin/ls -alh asterix-server.zip/repo"); vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + File.separator + "AppMaster.stdout"); vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + File.separator + "AppMaster.stderr"); // Get final commmand StringBuilder command = new StringBuilder(); for (CharSequence str : vargs) { command.append(str).append(" "); } LOG.info("Completed setting up app master command " + command.toString()); List<String> commands = new ArrayList<String>(); commands.add(command.toString()); amContainer.setCommands(commands); // Set up resource type requirements // For now, only memory is supported so we set memory requirements Resource capability = Records.newRecord(Resource.class); capability.setMemory(amMemory); appContext.setResource(capability); // Service data is a binary blob that can be passed to the application // Not needed in this scenario // amContainer.setServiceData(serviceData); // The following are not required for launching an application master // amContainer.setContainerId(containerId); appContext.setAMContainerSpec(amContainer); // Set the priority for the application master Priority pri = Records.newRecord(Priority.class); // TODO - what is the range for priority? how to decide? pri.setPriority(amPriority); appContext.setPriority(pri); // Set the queue to which this application is to be submitted in the RM appContext.setQueue(amQueue); // Submit the application to the applications manager // SubmitApplicationResponse submitResp = // applicationsManager.submitApplication(appRequest); // Ignore the response as either a valid response object is returned on // success // or an exception thrown to denote some form of a failure LOG.info("Submitting application to ASM"); yarnClient.submitApplication(appContext); //now write the instance lock if (mode == Mode.INSTALL || mode == Mode.START) { FileSystem fs = FileSystem.get(conf); Path lock = new Path(fs.getHomeDirectory(), CONF_DIR_REL + instanceFolder + instanceLock); if (fs.exists(lock)) { throw new IllegalStateException("Somehow, this instance has been launched twice. "); } BufferedWriter out = new BufferedWriter(new OutputStreamWriter(fs.create(lock, true))); try { out.write(app.getApplicationSubmissionContext().getApplicationId().toString()); out.close(); } finally { out.close(); } } return app.getApplicationSubmissionContext().getApplicationId(); }
From source file:edu.uci.ics.asterix.aoya.AsterixYARNClient.java
License:Apache License
/** * Tries to stop a running AsterixDB instance gracefully. * @throws IOException/*from w w w . j ava 2s. c om*/ * @throws YarnException */ private void stopInstanceIfRunning() throws IOException, YarnException { FileSystem fs = FileSystem.get(conf); String pathSuffix = CONF_DIR_REL + instanceFolder + CONFIG_DEFAULT_NAME; Path dstConf = new Path(fs.getHomeDirectory(), pathSuffix); //if the instance is up, fix that if (isRunning()) { try { this.stopInstance(); } catch (IOException e) { throw new YarnException(e); } } else if (!fs.exists(dstConf)) { throw new YarnException("No instance configured with that name exists"); } }
From source file:edu.uci.ics.asterix.aoya.AsterixYARNClient.java
License:Apache License
private void deleteLockFile() throws IOException { if (instanceName == null || instanceName == "") { return;/*from www . j av a 2s . c o m*/ } FileSystem fs = FileSystem.get(conf); Path lockPath = new Path(fs.getHomeDirectory(), CONF_DIR_REL + instanceName + '/' + instanceLock); if (fs.exists(lockPath)) { fs.delete(lockPath, false); } }
From source file:edu.uci.ics.asterix.aoya.AsterixYARNClient.java
License:Apache License
private boolean instanceExists() throws IOException { FileSystem fs = FileSystem.get(conf); String pathSuffix = CONF_DIR_REL + instanceFolder + CONFIG_DEFAULT_NAME; Path dstConf = new Path(fs.getHomeDirectory(), pathSuffix); return fs.exists(dstConf); }
From source file:edu.uci.ics.asterix.aoya.AsterixYARNClient.java
License:Apache License
private boolean isRunning() throws IOException { FileSystem fs = FileSystem.get(conf); String pathSuffix = CONF_DIR_REL + instanceFolder + CONFIG_DEFAULT_NAME; Path dstConf = new Path(fs.getHomeDirectory(), pathSuffix); if (fs.exists(dstConf)) { Path lock = new Path(fs.getHomeDirectory(), CONF_DIR_REL + instanceFolder + instanceLock); return fs.exists(lock); } else {// ww w . j av a 2s . c om return false; } }
From source file:edu.uci.ics.asterix.aoya.AsterixYARNClient.java
License:Apache License
private ApplicationId getLockFile() throws IOException, YarnException { if (instanceFolder == "") { throw new IllegalStateException("Instance name not given."); }//from w w w .j a v a 2 s . co m FileSystem fs = FileSystem.get(conf); Path lockPath = new Path(fs.getHomeDirectory(), CONF_DIR_REL + instanceFolder + instanceLock); if (!fs.exists(lockPath)) { throw new YarnException("Instance appears to not be running. If you know it is, try using kill"); } BufferedReader br = new BufferedReader(new InputStreamReader(fs.open(lockPath))); String lockAppId = br.readLine(); br.close(); return ConverterUtils.toApplicationId(lockAppId); }
From source file:edu.uci.ics.asterix.aoya.AsterixYARNClient.java
License:Apache License
public static ApplicationId getLockFile(String instanceName, Configuration conf) throws IOException { if (instanceName == "") { throw new IllegalStateException("Instance name not given."); }/*from w w w . jav a 2s .co m*/ FileSystem fs = FileSystem.get(conf); Path lockPath = new Path(fs.getHomeDirectory(), CONF_DIR_REL + instanceName + '/' + instanceLock); if (!fs.exists(lockPath)) { return null; } BufferedReader br = new BufferedReader(new InputStreamReader(fs.open(lockPath))); String lockAppId = br.readLine(); br.close(); return ConverterUtils.toApplicationId(lockAppId); }
From source file:edu.uci.ics.asterix.aoya.test.AsterixYARNInstanceUtil.java
License:Apache License
public YarnConfiguration setUp() throws Exception { File asterixProjectDir = new File(System.getProperty("user.dir")); File installerTargetDir = new File(asterixProjectDir, "target"); String[] dirsInTarget = installerTargetDir.list(new FilenameFilter() { @Override//w ww . j a v a 2 s . c o m public boolean accept(File dir, String name) { return new File(dir, name).isDirectory() && name.startsWith("asterix-yarn") && name.endsWith("binary-assembly"); } }); if (dirsInTarget.length != 1) { throw new IllegalStateException("Could not find binary to run YARN integration test with"); } aoyaHome = installerTargetDir.getAbsolutePath() + File.separator + dirsInTarget[0]; File asterixServerInstallerDir = new File(aoyaHome, "asterix"); String[] zipsInFolder = asterixServerInstallerDir.list(new FilenameFilter() { @Override public boolean accept(File dir, String name) { return name.startsWith("asterix-server") && name.endsWith("binary-assembly.zip"); } }); if (zipsInFolder.length != 1) { throw new IllegalStateException("Could not find server binary to run YARN integration test with"); } aoyaServerPath = asterixServerInstallerDir.getAbsolutePath() + File.separator + zipsInFolder[0]; configPath = aoyaHome + File.separator + "configs" + File.separator + "local.xml"; parameterPath = aoyaHome + File.separator + "conf" + File.separator + "base-asterix-configuration.xml"; YARNCluster.getInstance().setup(); appConf = new YarnConfiguration(); File baseDir = new File("./target/hdfs/").getAbsoluteFile(); FileUtil.fullyDelete(baseDir); appConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.getAbsolutePath()); MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(appConf); MiniDFSCluster hdfsCluster = builder.build(); miniCluster = YARNCluster.getInstance().getCluster(); appConf.set("fs.defaultFS", "hdfs://localhost:" + hdfsCluster.getNameNodePort()); miniCluster.init(appConf); Cluster defaultConfig = Utils.parseYarnClusterConfig(configPath); for (Node n : defaultConfig.getNode()) { n.setClusterIp(MiniYARNCluster.getHostname()); } defaultConfig.getMasterNode().setClusterIp(MiniYARNCluster.getHostname()); configPath = "target" + File.separator + "localized-aoya-config.xml"; Utils.writeYarnClusterConfig(configPath, defaultConfig); miniCluster.start(); appConf = new YarnConfiguration(miniCluster.getConfig()); appConf.set("fs.defaultFS", "hdfs://localhost:" + hdfsCluster.getNameNodePort()); //TODO:why must I do this!? what is not being passed properly via environment variables??? appConf.writeXml(new FileOutputStream("target" + File.separator + "yarn-site.xml")); //once the cluster is created, you can get its configuration //with the binding details to the cluster added from the minicluster FileSystem fs = FileSystem.get(appConf); Path instanceState = new Path(fs.getHomeDirectory(), AsterixYARNClient.CONF_DIR_REL + INSTANCE_NAME + "/"); fs.delete(instanceState, true); Assert.assertFalse(fs.exists(instanceState)); File outdir = new File(PATH_ACTUAL); outdir.mkdirs(); return appConf; }