List of usage examples for org.apache.hadoop.fs FileSystem getHomeDirectory
public Path getHomeDirectory()
From source file:org.apache.metron.maas.service.MaasIntegrationTest.java
License:Apache License
public void testDSShell(boolean haveDomain) throws Exception { MaaSConfig config = new MaaSConfig() { {//w ww . ja v a 2 s . com setServiceRoot("/maas/service"); setQueueConfig(new HashMap<String, Object>() { { put(ZKQueue.ZK_PATH, "/maas/queue"); } }); } }; String configRoot = "/maas/config"; byte[] configData = ConfigUtil.INSTANCE.toBytes(config); try { client.setData().forPath(configRoot, configData); } catch (KeeperException.NoNodeException e) { client.create().creatingParentsIfNeeded().forPath(configRoot, configData); } String[] args = { "--jar", yarnComponent.getAppMasterJar(), "--zk_quorum", zkServerComponent.getConnectionString(), "--zk_root", configRoot, "--master_memory", "512", "--master_vcores", "2", }; if (haveDomain) { String[] domainArgs = { "--domain", "TEST_DOMAIN", "--view_acls", "reader_user reader_group", "--modify_acls", "writer_user writer_group", "--create" }; List<String> argsList = new ArrayList<String>(Arrays.asList(args)); argsList.addAll(Arrays.asList(domainArgs)); args = argsList.toArray(new String[argsList.size()]); } YarnConfiguration conf = yarnComponent.getConfig(); LOG.info("Initializing DS Client"); final Client client = new Client(new Configuration(conf)); boolean initSuccess = client.init(args); Assert.assertTrue(initSuccess); LOG.info("Running DS Client"); final AtomicBoolean result = new AtomicBoolean(false); Thread t = new Thread() { @Override public void run() { try { result.set(client.run()); } catch (Exception e) { throw new RuntimeException(e); } } }; t.start(); YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(new Configuration(conf)); yarnClient.start(); String hostName = NetUtils.getHostname(); boolean verified = false; String errorMessage = ""; while (!verified) { List<ApplicationReport> apps = yarnClient.getApplications(); if (apps.size() == 0) { Thread.sleep(10); continue; } ApplicationReport appReport = apps.get(0); if (appReport.getHost().equals("N/A")) { Thread.sleep(10); continue; } errorMessage = "Expected host name to start with '" + hostName + "', was '" + appReport.getHost() + "'. Expected rpc port to be '-1', was '" + appReport.getRpcPort() + "'."; if (checkHostname(appReport.getHost()) && appReport.getRpcPort() == -1) { verified = true; } if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED) { break; } } Assert.assertTrue(errorMessage, verified); FileSystem fs = FileSystem.get(conf); try { new ModelSubmission().execute(FileSystem.get(conf), new String[] { "--name", "dummy", "--version", "1.0", "--zk_quorum", zkServerComponent.getConnectionString(), "--zk_root", configRoot, "--local_model_path", "src/test/resources/maas", "--hdfs_model_path", new Path(fs.getHomeDirectory(), "maas/dummy").toString(), "--num_instances", "1", "--memory", "100", "--mode", "ADD", "--log4j", "src/test/resources/log4j.properties" }); ServiceDiscoverer discoverer = new ServiceDiscoverer(this.client, config.getServiceRoot()); discoverer.start(); { boolean passed = false; for (int i = 0; i < 100; ++i) { try { List<ModelEndpoint> endpoints = discoverer.getEndpoints(new Model("dummy", "1.0")); if (endpoints != null && endpoints.size() == 1) { LOG.trace("Found endpoints: " + endpoints.get(0)); String output = makeRESTcall( new URL(endpoints.get(0).getEndpoint().getUrl() + "/echo/casey")); if (output.contains("casey")) { passed = true; break; } } } catch (Exception e) { } Thread.sleep(2000); } Assert.assertTrue(passed); } { List<ModelEndpoint> endpoints = discoverer.getEndpoints(new Model("dummy", "1.0")); Assert.assertNotNull(endpoints); Assert.assertEquals(1, endpoints.size()); } new ModelSubmission().execute(FileSystem.get(conf), new String[] { "--name", "dummy", "--version", "1.0", "--zk_quorum", zkServerComponent.getConnectionString(), "--zk_root", configRoot, "--num_instances", "1", "--mode", "REMOVE", }); { boolean passed = false; for (int i = 0; i < 100; ++i) { try { List<ModelEndpoint> endpoints = discoverer.getEndpoints(new Model("dummy", "1.0")); //ensure that the endpoint is dead. if (endpoints == null || endpoints.size() == 0) { passed = true; break; } } catch (Exception e) { } Thread.sleep(2000); } Assert.assertTrue(passed); } } finally { cleanup(); } }
From source file:org.apache.oozie.action.hadoop.TestFsActionExecutor.java
License:Apache License
public void testMovetoTrash() throws Exception { FsActionExecutor ae = new FsActionExecutor(); FileSystem fs = getFileSystem(); Path path = new Path(getFsTestCaseDir(), "dir1"); Context context = createContext("<fs/>"); fs.mkdirs(path);/*from www . jav a 2s . c o m*/ ae.delete(context, null, null, path, false); Path trashPath = new Path(fs.getHomeDirectory() + "/.Trash/Current/" + path.toUri().getPath()); assertTrue(!fs.exists(path)); assertTrue(fs.exists(trashPath)); }
From source file:org.apache.oozie.command.wf.WfEndXCommand.java
License:Apache License
private void deleteWFDir() throws CommandException { FileSystem fs; try {//from ww w . ja v a 2s . co m fs = getAppFileSystem(job); String wfDir = Services.get().getSystemId() + "/" + job.getId(); Path wfDirPath = new Path(fs.getHomeDirectory(), wfDir); LOG.debug("WF tmp dir :" + wfDirPath); if (fs.exists(wfDirPath)) { fs.delete(wfDirPath, true); } else { LOG.debug("Tmp dir doesn't exist :" + wfDirPath); } } catch (Exception e) { LOG.error("Unable to delete WF temp dir of wf id :" + job.getId(), e); throw new CommandException(ErrorCode.E0819, job.getId(), e); } }
From source file:org.apache.spark.tez.utils.HadoopUtils.java
License:Apache License
/** * Provisions resource represented as {@link File} to the {@link FileSystem} for a given application * //from w ww. j av a 2 s . c om * @param localResource * @param fs * @param applicationName * @return */ public static Path provisionResourceToFs(File localResource, FileSystem fs, String applicationName) { String destinationFilePath = applicationName + "/" + localResource.getName(); Path provisionedPath = new Path(fs.getHomeDirectory(), destinationFilePath); provisioinResourceToFs(fs, new Path(localResource.getAbsolutePath()), provisionedPath); return provisionedPath; }
From source file:org.apache.spark.tez.utils.HadoopUtils.java
License:Apache License
/** * Will provision current classpath to YARN and return an array of * {@link Path}s representing provisioned resources * If 'generate-jar' system property is set it will also generate the JAR for the current * working directory (mainly used when executing from IDE) * // w ww .j ava2s. c om * @return */ private static Path[] provisionClassPath(FileSystem fs, String applicationName, String[] classPathExclusions) { String genJarProperty = System.getProperty(TezConstants.GENERATE_JAR); boolean generateJar = genJarProperty != null && Boolean.parseBoolean(genJarProperty); List<Path> provisionedPaths = new ArrayList<Path>(); List<File> generatedJars = new ArrayList<File>(); boolean confFromHadoopConfDir = generateConfigJarFromHadoopConfDir(fs, applicationName, provisionedPaths, generatedJars); TezConfiguration tezConf = new TezConfiguration(fs.getConf()); boolean provisionTez = true; if (tezConf.get("tez.lib.uris") != null) { provisionTez = false; } URL[] classpath = ((URLClassLoader) ClassLoader.getSystemClassLoader()).getURLs(); for (URL classpathUrl : classpath) { File f = new File(classpathUrl.getFile()); if (f.isDirectory()) { if (generateJar) { String jarFileName = ClassPathUtils.generateJarFileName("application"); f = doGenerateJar(f, jarFileName, generatedJars, "application"); } else if (f.getName().equals("conf") && !confFromHadoopConfDir) { String jarFileName = ClassPathUtils.generateJarFileName("conf_application"); f = doGenerateJar(f, jarFileName, generatedJars, "configuration"); } else { f = null; } } if (f != null) { if (f.getName().startsWith("tez-") && !provisionTez) { logger.info("Skipping provisioning of " + f.getName() + " since Tez libraries are already provisioned"); continue; } String destinationFilePath = applicationName + "/" + f.getName(); Path provisionedPath = new Path(fs.getHomeDirectory(), destinationFilePath); if (shouldProvision(provisionedPath.getName(), classPathExclusions)) { provisioinResourceToFs(fs, new Path(f.getAbsolutePath()), provisionedPath); provisionedPaths.add(provisionedPath); } } } for (File generatedJar : generatedJars) { try { generatedJar.delete(); } catch (Exception e) { logger.warn("Failed to delete generated jars", e); } } return provisionedPaths.toArray(new Path[] {}); }
From source file:org.apache.spark.tez.utils.HadoopUtils.java
License:Apache License
/** * //from ww w. java 2s. c o m */ private static boolean generateConfigJarFromHadoopConfDir(FileSystem fs, String applicationName, List<Path> provisionedPaths, List<File> generatedJars) { boolean generated = false; String hadoopConfDir = System.getenv().get("HADOOP_CONF_DIR"); if (hadoopConfDir != null && hadoopConfDir.trim().length() > 0) { String jarFileName = ClassPathUtils.generateJarFileName("conf_"); File confDir = new File(hadoopConfDir.trim()); File jarFile = doGenerateJar(confDir, jarFileName, generatedJars, "configuration (HADOOP_CONF_DIR)"); String destinationFilePath = applicationName + "/" + jarFile.getName(); Path provisionedPath = new Path(fs.getHomeDirectory(), destinationFilePath); provisioinResourceToFs(fs, new Path(jarFile.getAbsolutePath()), provisionedPath); provisionedPaths.add(provisionedPath); generated = true; } String tezConfDir = System.getenv().get("TEZ_CONF_DIR"); if (tezConfDir != null && tezConfDir.trim().length() > 0) { String jarFileName = ClassPathUtils.generateJarFileName("conf_tez"); File confDir = new File(tezConfDir.trim()); File jarFile = doGenerateJar(confDir, jarFileName, generatedJars, "configuration (TEZ_CONF_DIR)"); try { URLClassLoader cl = (URLClassLoader) ClassLoader.getSystemClassLoader(); Method m = URLClassLoader.class.getDeclaredMethod("addURL", URL.class); m.setAccessible(true); m.invoke(cl, jarFile.toURI().toURL()); } catch (Exception e) { e.printStackTrace(); } String destinationFilePath = applicationName + "/" + jarFile.getName(); Path provisionedPath = new Path(fs.getHomeDirectory(), destinationFilePath); provisioinResourceToFs(fs, new Path(jarFile.getAbsolutePath()), provisionedPath); provisionedPaths.add(provisionedPath); generated = true; } return generated; }
From source file:org.apache.tajo.HBaseTestClusterUtil.java
License:Apache License
/** * Returns the path to the default root dir the minicluster uses. * Note: this does not cause the root dir to be created. * @return Fully qualified path for the default hbase root dir * @throws java.io.IOException//from w w w . jav a 2 s.co m */ public Path getDefaultRootDirPath() throws IOException { FileSystem fs = FileSystem.get(this.conf); return new Path(fs.makeQualified(fs.getHomeDirectory()), "hbase"); }
From source file:org.apache.tajo.master.QueryMaster.java
License:Apache License
/** * It initializes the final output and staging directory and sets * them to variables.//from www. j a v a2s.c o m */ private void initStagingDir() throws IOException { QueryConf conf = getContext().getConf(); String realUser; String currentUser; UserGroupInformation ugi; ugi = UserGroupInformation.getLoginUser(); realUser = ugi.getShortUserName(); currentUser = UserGroupInformation.getCurrentUser().getShortUserName(); String givenOutputTableName = conf.getOutputTable(); Path stagingDir; // If final output directory is not given by an user, // we use the query id as a output directory. if (givenOutputTableName.equals("")) { this.isCreateTableStmt = false; FileSystem defaultFS = FileSystem.get(conf); Path homeDirectory = defaultFS.getHomeDirectory(); if (!defaultFS.exists(homeDirectory)) { defaultFS.mkdirs(homeDirectory, new FsPermission(USER_DIR_PERMISSION)); } Path userQueryDir = new Path(homeDirectory, TajoConstants.USER_QUERYDIR_PREFIX); if (defaultFS.exists(userQueryDir)) { FileStatus fsStatus = defaultFS.getFileStatus(userQueryDir); String owner = fsStatus.getOwner(); if (!(owner.equals(currentUser) || owner.equals(realUser))) { throw new IOException("The ownership on the user's query " + "directory " + userQueryDir + " is not as expected. " + "It is owned by " + owner + ". The directory must " + "be owned by the submitter " + currentUser + " or " + "by " + realUser); } if (!fsStatus.getPermission().equals(USER_DIR_PERMISSION)) { LOG.info("Permissions on staging directory " + userQueryDir + " are " + "incorrect: " + fsStatus.getPermission() + ". Fixing permissions " + "to correct value " + USER_DIR_PERMISSION); defaultFS.setPermission(userQueryDir, new FsPermission(USER_DIR_PERMISSION)); } } else { defaultFS.mkdirs(userQueryDir, new FsPermission(USER_DIR_PERMISSION)); } stagingDir = StorageUtil.concatPath(userQueryDir, queryId.toString()); if (defaultFS.exists(stagingDir)) { throw new IOException("The staging directory " + stagingDir + "already exists. The directory must be unique to each query"); } else { defaultFS.mkdirs(stagingDir, new FsPermission(USER_DIR_PERMISSION)); } // Set the query id to the output table name conf.setOutputTable(queryId.toString()); } else { this.isCreateTableStmt = true; Path warehouseDir = new Path(conf.getVar(TajoConf.ConfVars.ROOT_DIR), TajoConstants.WAREHOUSE_DIR); stagingDir = new Path(warehouseDir, conf.getOutputTable()); FileSystem fs = warehouseDir.getFileSystem(conf); if (fs.exists(stagingDir)) { throw new IOException("The staging directory " + stagingDir + " already exists. The directory must be unique to each query"); } else { // TODO - should have appropriate permission fs.mkdirs(stagingDir, new FsPermission(USER_DIR_PERMISSION)); } } conf.setOutputPath(stagingDir); outputPath = stagingDir; LOG.info("Initialized Query Staging Dir: " + outputPath); }
From source file:org.apache.tajo.master.TaskRunnerLauncherImpl.java
License:Apache License
private ContainerLaunchContext createCommonContainerLaunchContext() { TajoConf conf = (TajoConf) getConfig(); ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class); try {/*w w w . j a v a 2 s . c o m*/ ctx.setUser(UserGroupInformation.getCurrentUser().getShortUserName()); } catch (IOException e) { e.printStackTrace(); } //////////////////////////////////////////////////////////////////////////// // Set the env variables to be setup //////////////////////////////////////////////////////////////////////////// LOG.info("Set the environment for the application master"); Map<String, String> environment = new HashMap<String, String>(); //String initialClassPath = getInitialClasspath(conf); environment.put(Environment.SHELL.name(), "/bin/bash"); environment.put(Environment.JAVA_HOME.name(), System.getenv(Environment.JAVA_HOME.name())); // TODO - to be improved with org.apache.tajo.sh shell script Properties prop = System.getProperties(); if (prop.getProperty("tajo.test", "FALSE").equalsIgnoreCase("TRUE")) { environment.put(Environment.CLASSPATH.name(), prop.getProperty("java.class.path", null)); } else { // Add AppMaster.jar location to classpath // At some point we should not be required to add // the hadoop specific classpaths to the env. // It should be provided out of the box. // For now setting all required classpaths including // the classpath to "." for the application jar StringBuilder classPathEnv = new StringBuilder("./"); //for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH)) { for (String c : YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH) { classPathEnv.append(':'); classPathEnv.append(c.trim()); } classPathEnv.append(":" + System.getenv("TAJO_BASE_CLASSPATH")); classPathEnv.append(":./log4j.properties:./*"); environment.put("HADOOP_HOME", System.getenv("HADOOP_HOME")); environment.put(Environment.HADOOP_COMMON_HOME.name(), System.getenv("HADOOP_HOME")); environment.put(Environment.HADOOP_HDFS_HOME.name(), System.getenv("HADOOP_HOME")); environment.put(Environment.HADOOP_YARN_HOME.name(), System.getenv("HADOOP_HOME")); environment.put("TAJO_BASE_CLASSPATH", System.getenv("TAJO_BASE_CLASSPATH")); environment.put(Environment.CLASSPATH.name(), classPathEnv.toString()); } ctx.setEnvironment(environment); //////////////////////////////////////////////////////////////////////////// // Set the local resources //////////////////////////////////////////////////////////////////////////// Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); FileSystem fs = null; LOG.info("defaultFS: " + conf.get("fs.default.name")); LOG.info("defaultFS: " + conf.get("fs.defaultFS")); try { fs = FileSystem.get(conf); } catch (IOException e) { e.printStackTrace(); } FileContext fsCtx = null; try { fsCtx = FileContext.getFileContext(getConfig()); } catch (UnsupportedFileSystemException e) { e.printStackTrace(); } LOG.info("Writing a QueryConf to HDFS and add to local environment"); Path queryConfPath = new Path(fs.getHomeDirectory(), QueryConf.FILENAME); try { writeConf(conf, queryConfPath); LocalResource queryConfSrc = createApplicationResource(fsCtx, queryConfPath, LocalResourceType.FILE); localResources.put(QueryConf.FILENAME, queryConfSrc); ctx.setLocalResources(localResources); } catch (IOException e) { e.printStackTrace(); } // Add shuffle token Map<String, ByteBuffer> serviceData = new HashMap<String, ByteBuffer>(); try { //LOG.info("Putting shuffle token in serviceData"); serviceData.put(PullServerAuxService.PULLSERVER_SERVICEID, PullServerAuxService.serializeMetaData(0)); } catch (IOException ioe) { LOG.error(ioe); } ctx.setServiceData(serviceData); return ctx; }
From source file:org.apache.tajo.yarn.command.LaunchCommand.java
License:Apache License
private void setupLocalResources(ContainerLaunchContext amContainer, FileSystem fs, ApplicationId appId) throws IOException { // set local resources for the application master // local files or archives as needed // In this scenario, the jar file for the application master is part of the local resources Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); LOG.info("Copy App Master jar from local filesystem and add to local environment"); // Copy the application master jar to the filesystem // Create a local resource to point to the destination jar path String appMasterJar = findContainingJar(ApplicationMaster.class); addToLocalResources(fs, appMasterJar, appMasterJarPath, appId.getId(), localResources, LocalResourceType.FILE);//from w w w . j ava2s. c o m addToLocalResources(fs, libDir, libDir, appId.getId(), localResources, LocalResourceType.FILE); addToLocalResources(fs, tajoArchive, "tajo", appId.getId(), localResources, LocalResourceType.ARCHIVE); // Set the log4j properties if needed if (!log4jPropFile.isEmpty()) { addToLocalResources(fs, log4jPropFile, log4jPath, appId.getId(), localResources, LocalResourceType.FILE); } // addToLocalResources(fs, confDir, "conf", appId.getId(), // localResources, LocalResourceType.FILE); // Tajo master conf Configuration tajoMasterConf = new Configuration(false); tajoMasterConf.addResource(new Path(confDir, "tajo-site.xml")); String suffix = appName + "/" + appId.getId() + "/master-conf"; Path dst = new Path(fs.getHomeDirectory(), suffix); fs.mkdirs(dst); Path confFile = new Path(dst, "tajo-site.xml"); FSDataOutputStream fdos = fs.create(confFile); tajoMasterConf.writeXml(fdos); fdos.close(); FileStatus scFileStatus = fs.getFileStatus(dst); LocalResource scRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(dst.toUri()), LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, scFileStatus.getLen(), scFileStatus.getModificationTime()); localResources.put("conf", scRsrc); amContainer.setLocalResources(localResources); }