List of usage examples for org.apache.hadoop.fs Path isAbsolute
public boolean isAbsolute()
From source file:org.apache.oozie.action.hadoop.MapReduceActionExecutor.java
License:Apache License
@Override @SuppressWarnings("unchecked") Configuration setupActionConf(Configuration actionConf, Context context, Element actionXml, Path appPath) throws ActionExecutorException { boolean regularMR = false; Namespace ns = actionXml.getNamespace(); if (actionXml.getChild("streaming", ns) != null) { Element streamingXml = actionXml.getChild("streaming", ns); String mapper = streamingXml.getChildTextTrim("mapper", ns); String reducer = streamingXml.getChildTextTrim("reducer", ns); String recordReader = streamingXml.getChildTextTrim("record-reader", ns); List<Element> list = (List<Element>) streamingXml.getChildren("record-reader-mapping", ns); String[] recordReaderMapping = new String[list.size()]; for (int i = 0; i < list.size(); i++) { recordReaderMapping[i] = list.get(i).getTextTrim(); }/*from ww w . ja v a 2s .c o m*/ list = (List<Element>) streamingXml.getChildren("env", ns); String[] env = new String[list.size()]; for (int i = 0; i < list.size(); i++) { env[i] = list.get(i).getTextTrim(); } setStreaming(actionConf, mapper, reducer, recordReader, recordReaderMapping, env); } else { if (actionXml.getChild("pipes", ns) != null) { Element pipesXml = actionXml.getChild("pipes", ns); String map = pipesXml.getChildTextTrim("map", ns); String reduce = pipesXml.getChildTextTrim("reduce", ns); String inputFormat = pipesXml.getChildTextTrim("inputformat", ns); String partitioner = pipesXml.getChildTextTrim("partitioner", ns); String writer = pipesXml.getChildTextTrim("writer", ns); String program = pipesXml.getChildTextTrim("program", ns); PipesMain.setPipes(actionConf, map, reduce, inputFormat, partitioner, writer, program, appPath); } else { regularMR = true; } } actionConf = super.setupActionConf(actionConf, context, actionXml, appPath); // For "regular" (not streaming or pipes) MR jobs if (regularMR) { // Resolve uber jar path (has to be done after super because oozie.mapreduce.uber.jar is under <configuration>) String uberJar = actionConf.get(MapReduceMain.OOZIE_MAPREDUCE_UBER_JAR); if (uberJar != null) { if (!ConfigurationService.getBoolean(OOZIE_MAPREDUCE_UBER_JAR_ENABLE)) { throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "MR003", "{0} property is not allowed. Set {1} to true in oozie-site to enable.", MapReduceMain.OOZIE_MAPREDUCE_UBER_JAR, OOZIE_MAPREDUCE_UBER_JAR_ENABLE); } String nameNode = actionXml.getChildTextTrim("name-node", ns); if (nameNode != null) { Path uberJarPath = new Path(uberJar); if (uberJarPath.toUri().getScheme() == null || uberJarPath.toUri().getAuthority() == null) { if (uberJarPath.isAbsolute()) { // absolute path without namenode --> prepend namenode Path nameNodePath = new Path(nameNode); String nameNodeSchemeAuthority = nameNodePath.toUri().getScheme() + "://" + nameNodePath.toUri().getAuthority(); actionConf.set(MapReduceMain.OOZIE_MAPREDUCE_UBER_JAR, new Path(nameNodeSchemeAuthority + uberJarPath).toString()); } else { // relative path --> prepend app path actionConf.set(MapReduceMain.OOZIE_MAPREDUCE_UBER_JAR, new Path(appPath, uberJarPath).toString()); } } } } } else { if (actionConf.get(MapReduceMain.OOZIE_MAPREDUCE_UBER_JAR) != null) { log.warn("The " + MapReduceMain.OOZIE_MAPREDUCE_UBER_JAR + " property is only applicable for MapReduce (not" + "streaming nor pipes) workflows, ignoring"); actionConf.set(MapReduceMain.OOZIE_MAPREDUCE_UBER_JAR, ""); } } // child job cancel delegation token for mapred action actionConf.setBoolean("mapreduce.job.complete.cancel.delegation.tokens", true); return actionConf; }
From source file:org.apache.oozie.action.hadoop.TestJavaActionExecutor.java
License:Apache License
public void testJobXmlAndNonDefaultNamenode() throws Exception { // By default the job.xml file is taken from the workflow application // namenode, regadless the namenode specified for the action. To specify // a job.xml on another namenode use a fully qualified file path. Path appPath = new Path(getFsTestCaseDir(), "app"); getFileSystem().mkdirs(appPath);/*from w ww.j a v a2s .c o m*/ Path jobXmlAbsolutePath = new Path(getFsTestCaseDir().toUri().getPath(), "jobxmlpath/job.xml"); assertTrue(jobXmlAbsolutePath.isAbsolute() && jobXmlAbsolutePath.toUri().getAuthority() == null); Path jobXmlAbsolutePath2 = new Path(getFsTestCaseDir().toUri().getPath(), "jobxmlpath/job3.xml"); assertTrue(jobXmlAbsolutePath2.isAbsolute() && jobXmlAbsolutePath2.toUri().getAuthority() == null); Path jobXmlQualifiedPath = new Path(getFs2TestCaseDir(), "jobxmlpath/job4.xml"); assertTrue(jobXmlQualifiedPath.toUri().getAuthority() != null); // Use non-default name node (second filesystem) and three job-xml configurations: // 1. Absolute (but not fully qualified) path located in the first filesystem // 2. Without path (fist filesystem) // 3. Absolute (but not fully qualified) path located in the both filesystems // (first should be used) // 4. Fully qualified path located in the second filesystem String str = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" + getNameNode2Uri() + "</name-node>" + "<job-xml>" + jobXmlAbsolutePath.toString() + "</job-xml>" + "<job-xml>job2.xml</job-xml>" + "<job-xml>" + jobXmlAbsolutePath2.toString() + "</job-xml>" + "<job-xml>" + jobXmlQualifiedPath.toString() + "</job-xml>" + "<configuration>" + "<property><name>p1</name><value>v1a</value></property>" + "<property><name>p2</name><value>v2</value></property>" + "</configuration>" + "</java>"; Element xml = XmlUtils.parseXml(str); XConfiguration jConf = new XConfiguration(); jConf.set("p1", "v1b"); jConf.set("p3", "v3a"); OutputStream os = getFileSystem().create(jobXmlAbsolutePath); jConf.writeXml(os); os.close(); jConf = new XConfiguration(); jConf.set("p4", "v4"); jConf.set("p3", "v3b"); os = getFileSystem().create(new Path(appPath, "job2.xml")); jConf.writeXml(os); os.close(); // This configuration is expected to be used jConf = new XConfiguration(); jConf.set("p5", "v5a"); jConf.set("p6", "v6a"); os = getFileSystem().create(jobXmlAbsolutePath2); jConf.writeXml(os); os.close(); // This configuration is expected to be ignored jConf = new XConfiguration(); jConf.set("p5", "v5b"); jConf.set("p6", "v6b"); os = getFileSystem2().create(new Path(jobXmlAbsolutePath2.toUri().getPath())); jConf.writeXml(os); os.close(); jConf = new XConfiguration(); jConf.set("p7", "v7a"); jConf.set("p8", "v8a"); os = getFileSystem2().create(jobXmlQualifiedPath); jConf.writeXml(os); os.close(); Context context = createContext("<java/>", null); Configuration conf = new JavaActionExecutor().createBaseHadoopConf(context, xml); int confSize0 = conf.size(); JavaActionExecutor.parseJobXmlAndConfiguration(context, xml, appPath, conf); assertEquals(confSize0 + 8, conf.size()); assertEquals("v1a", conf.get("p1")); assertEquals("v2", conf.get("p2")); assertEquals("v3b", conf.get("p3")); assertEquals("v4", conf.get("p4")); assertEquals("v5a", conf.get("p5")); assertEquals("v6a", conf.get("p6")); assertEquals("v7a", conf.get("p7")); assertEquals("v8a", conf.get("p8")); }
From source file:org.apache.pig.backend.hadoop.executionengine.physicalLayer.util.PlanHelper.java
License:Apache License
/** * Creates a relative path that can be used to build a temporary * place to store the output from a number of map-reduce tasks. *///from ww w .jav a2 s . c o m public static String makeStoreTmpPath(String orig) { Path path = new Path(orig); URI uri = path.toUri(); uri.normalize(); String pathStr = uri.getPath(); if (path.isAbsolute()) { return new Path("abs" + pathStr).toString(); } else { return new Path("rel/" + pathStr).toString(); } }
From source file:org.apache.pig.LoadFunc.java
License:Apache License
/** * Construct the absolute path from the file location and the current * directory. The current directory is either of the form * {code}hdfs://<nodename>:<nodeport>/<directory>{code} in Hadoop * MapReduce mode, or of the form /*from w w w . java 2 s . co m*/ * {code}file:///<directory>{code} in Hadoop local mode. * * @param location the location string specified in the load statement * @param curDir the current file system directory * @return the absolute path of file in the file system * @throws FrontendException if the scheme of the location is incompatible * with the scheme of the file system */ public static String getAbsolutePath(String location, Path curDir) throws FrontendException { if (location == null || curDir == null) { throw new FrontendException("location: " + location + " curDir: " + curDir); } URI fsUri = curDir.toUri(); String fsScheme = fsUri.getScheme(); if (fsScheme == null) { throw new FrontendException("curDir: " + curDir); } fsScheme = fsScheme.toLowerCase(); String authority = fsUri.getAuthority(); if (authority == null) { authority = ""; } Path rootDir = new Path(fsScheme, authority, "/"); ArrayList<String> pathStrings = new ArrayList<String>(); String[] fnames = getPathStrings(location); for (String fname : fnames) { // remove leading/trailing whitespace(s) fname = fname.trim(); Path p = new Path(fname); URI uri = p.toUri(); // if the supplied location has a scheme (i.e. uri is absolute) or // an absolute path, just use it. if (!(uri.isAbsolute() || p.isAbsolute())) { String scheme = uri.getScheme(); if (scheme != null) { scheme = scheme.toLowerCase(); } if (scheme != null && !scheme.equals(fsScheme)) { throw new FrontendException("Incompatible file URI scheme: " + scheme + " : " + fsScheme); } String path = uri.getPath(); fname = (p.isAbsolute()) ? new Path(rootDir, path).toString() : new Path(curDir, path).toString(); } fname = fname.replaceFirst("^file:/([^/])", "file:///$1"); // remove the trailing / fname = fname.replaceFirst("/$", ""); pathStrings.add(fname); } return join(pathStrings, ","); }
From source file:org.apache.sentry.binding.util.SentryAuthorizerUtil.java
License:Apache License
public static boolean isLocalUri(String uriString) throws URISyntaxException { URI uri = new URI(uriString); Path uriPath = new Path(uri); return ((uri.getScheme() != null && uri.getScheme().equalsIgnoreCase("file")) || (uri.getScheme() == null && uriPath.isAbsolute())); }
From source file:org.apache.sentry.core.common.utils.PathUtils.java
License:Apache License
/** * Parse a URI which can be HDFS, S3, SWIFT, WEBHDFS,etc. In either case it * should be on the same fs as the warehouse directory. *//*from w w w .j a va 2 s. c o m*/ public static String parseURI(String warehouseDir, String uri, boolean isLocal) throws URISyntaxException { Path warehouseDirPath = new Path(warehouseDir); Path uriPath = new Path(uri); if (uriPath.isAbsolute()) { // Merge warehouseDir and uri only when there is no scheme and authority // in uri. if (uriPath.isAbsoluteAndSchemeAuthorityNull()) { uriPath = uriPath.makeQualified(warehouseDirPath.toUri(), warehouseDirPath); } String uriScheme = uriPath.toUri().getScheme(); String uriAuthority = uriPath.toUri().getAuthority(); if (StringUtils.isEmpty(uriScheme) || isLocal) { uriScheme = LOCAL_FILE_SCHEMA; uriAuthority = ""; } uriPath = new Path(uriScheme + AUTHORITY_PREFIX + StringUtils.trimToEmpty(uriAuthority) + Path.getPathWithoutSchemeAndAuthority(uriPath)); } else { // don't support relative path throw new IllegalArgumentException("Invalid URI " + uri + "."); } return uriPath.toUri().toString(); }
From source file:org.apache.sentry.core.common.utils.PathUtils.java
License:Apache License
/** * Parse a URI which is on a local file system. *///from ww w . jav a 2s. c o m public static String parseLocalURI(String uri) throws URISyntaxException { Path uriPath = new Path(uri); if (uriPath.isAbsolute()) { uriPath = new Path( LOCAL_FILE_SCHEMA + AUTHORITY_PREFIX + StringUtils.trimToEmpty(uriPath.toUri().getAuthority()) + Path.getPathWithoutSchemeAndAuthority(uriPath)); } else { throw new IllegalArgumentException("Parse URI does not work on relative URI: " + uri); } return uriPath.toUri().toString(); }
From source file:org.apache.sentry.core.model.db.AccessURI.java
License:Apache License
/** * Wrap a URI which can be HDFS, S3, SWIFT, WEBHDFS,etc. Do the validation for the URI's format. *///from ww w. j av a2 s .c om public AccessURI(String uriName) { uriName = uriName == null ? "" : uriName; // Validating the URI format if (!uriName.equals(AccessConstants.ALL)) { Path uriPath = new Path(uriName); String schema = uriPath.toUri().getScheme(); if (StringUtils.isBlank(schema) || !uriPath.isAbsolute()) { throw new IllegalArgumentException( "URI '" + uriName + "' is invalid. Unsupported URI without schema or relative URI."); } if (!uriName.startsWith(schema + AUTHORITY_PREFIX)) { throw new IllegalArgumentException("URI '" + uriName + "' is invalid."); } } // ALL(*) represents all URIs. this.uriName = uriName; }
From source file:org.apache.solr.hadoop.PathArgumentType.java
License:Apache License
private void verifyIsAbsolute(ArgumentParser parser, Path file) throws ArgumentParserException { if (!file.isAbsolute()) { throw new ArgumentParserException("Not an absolute file: " + file, parser); }/*from w ww .j a v a 2s .c o m*/ }
From source file:org.apache.tajo.storage.s3.S3TableSpace.java
License:Apache License
private String pathToKey(Path path) { if (!path.isAbsolute()) { path = new Path(fs.getWorkingDirectory(), path); }/* w ww .j a v a 2 s . c o m*/ if (path.toUri().getScheme() != null && path.toUri().getPath().isEmpty()) { return ""; } return path.toUri().getPath().substring(1); }