List of usage examples for org.apache.hadoop.fs FileSystem getHomeDirectory
public Path getHomeDirectory()
From source file:org.apache.giraph.yarn.YarnUtils.java
License:Apache License
/** * Get the base HDFS dir we will be storing our LocalResources in. * @param fs the file system.//from ww w.j a v a2 s. c o m * @param appId the ApplicationId under which our resources will be stored. * @return the path */ public static Path getFsCachePath(final FileSystem fs, final ApplicationId appId) { return new Path(fs.getHomeDirectory(), HDFS_RESOURCE_DIR + "/" + appId); }
From source file:org.apache.gobblin.cluster.GobblinClusterUtils.java
License:Apache License
public static Path getAppWorkDirPathFromConfig(Config config, FileSystem fs, String applicationName, String applicationId) {//from w ww . j ava 2s . c om if (config.hasPath(CLUSTER_WORK_DIR)) { return new Path(config.getString(CLUSTER_WORK_DIR)); } return new Path(fs.getHomeDirectory(), getAppWorkDirPath(applicationName, applicationId)); }
From source file:org.apache.gobblin.config.store.hdfs.SimpleHDFSConfigStoreFactory.java
License:Apache License
@Override protected URI getDefaultRootDir(Config factoryConfig, FileSystem defaultFileSystem, Optional<URI> configDefinedDefaultURI) { return configDefinedDefaultURI.or(defaultFileSystem.getHomeDirectory().toUri()); }
From source file:org.apache.gobblin.util.HadoopUtilsTest.java
License:Apache License
@Test public void testMoveToTrash() throws IOException { Path hadoopUtilsTestDir = new Path(Files.createTempDir().getAbsolutePath(), "HadoopUtilsTestDir"); Configuration conf = new Configuration(); // Set the time to keep it in trash to 10 minutes. // 0 means object will be deleted instantly. conf.set("fs.trash.interval", "10"); FileSystem fs = FileSystem.getLocal(conf); Trash trash = new Trash(fs, conf); TrashPolicy trashPolicy = TrashPolicy.getInstance(conf, fs, fs.getHomeDirectory()); Path trashPath = trashPolicy.getCurrentTrashDir(); fs.mkdirs(hadoopUtilsTestDir);// www . jav a 2s . co m Assert.assertTrue(fs.exists(hadoopUtilsTestDir)); trash.moveToTrash(hadoopUtilsTestDir.getParent()); Assert.assertFalse(fs.exists(hadoopUtilsTestDir)); Assert.assertTrue(fs.exists(trashPath)); }
From source file:org.apache.hama.HamaClusterTestCase.java
License:Apache License
@Override protected void setUp() throws Exception { try {//w w w .j a va2s. c o m if (this.startDfs) { // This spews a bunch of warnings about missing scheme. TODO: fix. this.dfsCluster = new MiniDFSCluster(0, this.conf, 2, true, true, true, null, null, null, null); // mangle the conf so that the fs parameter points to the minidfs we // just started up FileSystem filesystem = dfsCluster.getFileSystem(); conf.set("fs.defaultFS", filesystem.getUri().toString()); Path parentdir = filesystem.getHomeDirectory(); filesystem.mkdirs(parentdir); } // do the super setup now. if we had done it first, then we would have // gotten our conf all mangled and a local fs started up. super.setUp(); // start the instance hamaClusterSetup(); } catch (Exception e) { if (zooKeeperCluster != null) { zooKeeperCluster.shutdown(); } if (dfsCluster != null) { shutdownDfs(dfsCluster); } throw e; } }
From source file:org.apache.helix.provisioning.yarn.AppLauncher.java
License:Apache License
private Path copyToHDFS(FileSystem fs, String name, URI uri) throws Exception { // will throw exception if the file name is without extension String extension = uri.getPath().substring(uri.getPath().lastIndexOf(".") + 1); String pathSuffix = _applicationSpec.getAppName() + "/" + _appId.getId() + "/" + name + "." + extension; Path dst = new Path(fs.getHomeDirectory(), pathSuffix); Path src = new Path(uri); fs.copyFromLocalFile(false, true, src, dst); return dst;/*from www . java 2s. co m*/ }
From source file:org.apache.helix.provisioning.yarn.AppMasterLauncher.java
License:Apache License
private static InputStream readFromHDFS(FileSystem fs, String name, URI uri, ApplicationSpec appSpec, ApplicationId appId) throws Exception { // will throw exception if the file name is without extension String extension = uri.getPath().substring(uri.getPath().lastIndexOf(".") + 1); String pathSuffix = appSpec.getAppName() + "/" + appId.getId() + "/" + name + "." + extension; Path dst = new Path(fs.getHomeDirectory(), pathSuffix); return fs.open(dst).getWrappedStream(); }
From source file:org.apache.helix.provisioning.yarn.YarnProvisioner.java
License:Apache License
private ContainerLaunchContext createLaunchContext(ContainerId containerId, Container container, Participant participant) throws Exception { ContainerLaunchContext participantContainer = Records.newRecord(ContainerLaunchContext.class); // Map<String, String> envs = System.getenv(); String appName = applicationMasterConfig.getAppName(); int appId = applicationMasterConfig.getAppId(); String serviceName = _resourceConfig.getId().stringify(); String serviceClasspath = applicationMasterConfig.getClassPath(serviceName); String mainClass = applicationMasterConfig.getMainClass(serviceName); String zkAddress = applicationMasterConfig.getZKAddress(); // set the localresources needed to launch container Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); LocalResource servicePackageResource = Records.newRecord(LocalResource.class); YarnConfiguration conf = new YarnConfiguration(); FileSystem fs; fs = FileSystem.get(conf);/*from ww w . j a v a2 s. com*/ String pathSuffix = appName + "/" + appId + "/" + serviceName + ".tar"; Path dst = new Path(fs.getHomeDirectory(), pathSuffix); FileStatus destStatus = fs.getFileStatus(dst); // Set the type of resource - file or archive // archives are untarred at destination // we don't need the jar file to be untarred for now servicePackageResource.setType(LocalResourceType.ARCHIVE); // Set visibility of the resource // Setting to most private option servicePackageResource.setVisibility(LocalResourceVisibility.APPLICATION); // Set the resource to be copied over servicePackageResource.setResource(ConverterUtils.getYarnUrlFromPath(dst)); // Set timestamp and length of file so that the framework // can do basic sanity checks for the local resource // after it has been copied over to ensure it is the same // resource the client intended to use with the application servicePackageResource.setTimestamp(destStatus.getModificationTime()); servicePackageResource.setSize(destStatus.getLen()); LOG.info("Setting local resource:" + servicePackageResource + " for service" + serviceName); localResources.put(serviceName, servicePackageResource); // Set local resource info into app master container launch context participantContainer.setLocalResources(localResources); // Set the necessary security tokens as needed // amContainer.setContainerTokens(containerToken); // Set the env variables to be setup in the env where the application master will be run LOG.info("Set the environment for the application master"); Map<String, String> env = new HashMap<String, String>(); env.put(serviceName, dst.getName()); // Add AppMaster.jar location to classpath // At some point we should not be required to add // the hadoop specific classpaths to the env. // It should be provided out of the box. // For now setting all required classpaths including // the classpath to "." for the application jar StringBuilder classPathEnv = new StringBuilder(Environment.CLASSPATH.$()).append(File.pathSeparatorChar) .append("./*"); classPathEnv.append(File.pathSeparatorChar); classPathEnv.append(serviceClasspath); for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH, YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) { classPathEnv.append(File.pathSeparatorChar); classPathEnv.append(c.trim()); } classPathEnv.append(File.pathSeparatorChar).append("./log4j.properties"); LOG.info("Setting classpath for service:\n" + classPathEnv.toString()); env.put("CLASSPATH", classPathEnv.toString()); participantContainer.setEnvironment(env); if (applicationMaster.allTokens != null) { LOG.info("Setting tokens: " + applicationMaster.allTokens); participantContainer.setTokens(applicationMaster.allTokens); } // Set the necessary command to execute the application master Vector<CharSequence> vargs = new Vector<CharSequence>(30); // Set java executable command LOG.info("Setting up app master command"); vargs.add(Environment.JAVA_HOME.$() + "/bin/java"); // Set Xmx based on am memory size vargs.add("-Xmx" + 4096 + "m"); // Set class name vargs.add(ParticipantLauncher.class.getCanonicalName()); // Set params for container participant vargs.add("--zkAddress " + zkAddress); vargs.add("--cluster " + appName); vargs.add("--participantId " + participant.getId().stringify()); vargs.add("--participantClass " + mainClass); vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/ContainerParticipant.stdout"); vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/ContainerParticipant.stderr"); // Get final commmand StringBuilder command = new StringBuilder(); for (CharSequence str : vargs) { command.append(str).append(" "); } LOG.info("Completed setting up container launch command " + command.toString() + " with arguments \n" + vargs); List<String> commands = new ArrayList<String>(); commands.add(command.toString()); participantContainer.setCommands(commands); return participantContainer; }
From source file:org.apache.ignite.internal.processors.hadoop.GridHadoopPopularWordsTest.java
License:Apache License
/** * Runs the Hadoop job.//from w w w. j a v a2 s. com * * @return {@code True} if succeeded, {@code false} otherwise. * @throws Exception If failed. */ private boolean runWordCountConfigBasedHadoopJob() throws Exception { Job job = createConfigBasedHadoopJob(); // Distributed file system this job will work with. FileSystem fs = FileSystem.get(job.getConfiguration()); X.println(">>> Using distributed file system: " + fs.getHomeDirectory()); // Prepare input and output job directories. prepareDirectories(fs); long time = System.currentTimeMillis(); // Run job. boolean res = job.waitForCompletion(true); X.println(">>> Job execution time: " + (System.currentTimeMillis() - time) / 1000 + " sec."); // Move job results into local file system, so you can view calculated results. publishResults(fs); return res; }
From source file:org.apache.metron.maas.service.Client.java
License:Apache License
private Path addToLocalResources(FileSystem fs, String fileSrcPath, String fileDstPath, String appId, Map<String, LocalResource> localResources, String resources) throws IOException { String suffix = appName + "/" + appId + "/" + fileDstPath; Path dst = new Path(fs.getHomeDirectory(), suffix); if (fileSrcPath == null) { FSDataOutputStream ostream = null; try {/*from ww w . j a v a 2 s . com*/ ostream = FileSystem.create(fs, dst, new FsPermission((short) 0710)); ostream.writeUTF(resources); } finally { IOUtils.closeQuietly(ostream); } } else { fs.copyFromLocalFile(new Path(fileSrcPath), dst); } FileStatus scFileStatus = fs.getFileStatus(dst); LocalResource scRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(dst.toUri()), LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, scFileStatus.getLen(), scFileStatus.getModificationTime()); localResources.put(fileDstPath, scRsrc); return dst; }