List of usage examples for org.apache.hadoop.fs Path makeQualified
@InterfaceAudience.LimitedPrivate({ "HDFS", "MapReduce" })
public Path makeQualified(URI defaultUri, Path workingDir)
From source file:org.apache.nifi.processors.hadoop.AbstractFetchHDFSRecord.java
License:Apache License
@Override public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException { // do this before getting a flow file so that we always get a chance to attempt Kerberos relogin final FileSystem fileSystem = getFileSystem(); final Configuration configuration = getConfiguration(); final UserGroupInformation ugi = getUserGroupInformation(); if (configuration == null || fileSystem == null || ugi == null) { getLogger().error(/*from ww w . ja v a 2s . c o m*/ "Processor not configured properly because Configuration, FileSystem, or UserGroupInformation was null"); context.yield(); return; } final FlowFile originalFlowFile = session.get(); if (originalFlowFile == null) { context.yield(); return; } ugi.doAs((PrivilegedAction<Object>) () -> { FlowFile child = null; final String filenameValue = context.getProperty(FILENAME) .evaluateAttributeExpressions(originalFlowFile).getValue(); try { final Path path = new Path(filenameValue); final AtomicReference<Throwable> exceptionHolder = new AtomicReference<>(null); final AtomicReference<WriteResult> writeResult = new AtomicReference<>(); final RecordSetWriterFactory recordSetWriterFactory = context.getProperty(RECORD_WRITER) .asControllerService(RecordSetWriterFactory.class); final StopWatch stopWatch = new StopWatch(true); // use a child FlowFile so that if any error occurs we can route the original untouched FlowFile to retry/failure child = session.create(originalFlowFile); final AtomicReference<String> mimeTypeRef = new AtomicReference<>(); child = session.write(child, (final OutputStream rawOut) -> { try (final BufferedOutputStream out = new BufferedOutputStream(rawOut); final HDFSRecordReader recordReader = createHDFSRecordReader(context, originalFlowFile, configuration, path)) { Record record = recordReader.nextRecord(); final RecordSchema schema = recordSetWriterFactory.getSchema( originalFlowFile.getAttributes(), record == null ? null : record.getSchema()); try (final RecordSetWriter recordSetWriter = recordSetWriterFactory .createWriter(getLogger(), schema, out)) { recordSetWriter.beginRecordSet(); if (record != null) { recordSetWriter.write(record); } while ((record = recordReader.nextRecord()) != null) { recordSetWriter.write(record); } writeResult.set(recordSetWriter.finishRecordSet()); mimeTypeRef.set(recordSetWriter.getMimeType()); } } catch (Exception e) { exceptionHolder.set(e); } }); stopWatch.stop(); // if any errors happened within the session.write then throw the exception so we jump // into one of the appropriate catch blocks below if (exceptionHolder.get() != null) { throw exceptionHolder.get(); } FlowFile successFlowFile = postProcess(context, session, child, path); final Map<String, String> attributes = new HashMap<>(writeResult.get().getAttributes()); attributes.put(RECORD_COUNT_ATTR, String.valueOf(writeResult.get().getRecordCount())); attributes.put(CoreAttributes.MIME_TYPE.key(), mimeTypeRef.get()); successFlowFile = session.putAllAttributes(successFlowFile, attributes); final Path qualifiedPath = path.makeQualified(fileSystem.getUri(), fileSystem.getWorkingDirectory()); getLogger().info("Successfully received content from {} for {} in {} milliseconds", new Object[] { qualifiedPath, successFlowFile, stopWatch.getDuration() }); session.getProvenanceReporter().fetch(successFlowFile, qualifiedPath.toString(), stopWatch.getDuration(TimeUnit.MILLISECONDS)); session.transfer(successFlowFile, REL_SUCCESS); session.remove(originalFlowFile); return null; } catch (final FileNotFoundException | AccessControlException e) { getLogger().error("Failed to retrieve content from {} for {} due to {}; routing to failure", new Object[] { filenameValue, originalFlowFile, e }); final FlowFile failureFlowFile = session.putAttribute(originalFlowFile, FETCH_FAILURE_REASON_ATTR, e.getMessage() == null ? e.toString() : e.getMessage()); session.transfer(failureFlowFile, REL_FAILURE); } catch (final IOException | FlowFileAccessException e) { getLogger().error("Failed to retrieve content from {} for {} due to {}; routing to retry", new Object[] { filenameValue, originalFlowFile, e }); session.transfer(session.penalize(originalFlowFile), REL_RETRY); context.yield(); } catch (final Throwable t) { getLogger().error("Failed to retrieve content from {} for {} due to {}; routing to failure", new Object[] { filenameValue, originalFlowFile, t }); final FlowFile failureFlowFile = session.putAttribute(originalFlowFile, FETCH_FAILURE_REASON_ATTR, t.getMessage() == null ? t.toString() : t.getMessage()); session.transfer(failureFlowFile, REL_FAILURE); } // if we got this far then we weren't successful so we need to clean up the child flow file if it got initialized if (child != null) { session.remove(child); } return null; }); }
From source file:org.apache.nifi.processors.hadoop.AbstractPutHDFSRecord.java
License:Apache License
@Override public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException { // do this before getting a flow file so that we always get a chance to attempt Kerberos relogin final FileSystem fileSystem = getFileSystem(); final Configuration configuration = getConfiguration(); final UserGroupInformation ugi = getUserGroupInformation(); if (configuration == null || fileSystem == null || ugi == null) { getLogger().error(//from www. ja v a2 s. com "Processor not configured properly because Configuration, FileSystem, or UserGroupInformation was null"); context.yield(); return; } final FlowFile flowFile = session.get(); if (flowFile == null) { context.yield(); return; } ugi.doAs((PrivilegedAction<Object>) () -> { Path tempDotCopyFile = null; FlowFile putFlowFile = flowFile; try { final String filenameValue = putFlowFile.getAttribute(CoreAttributes.FILENAME.key()); // TODO codec extension final String directoryValue = context.getProperty(DIRECTORY) .evaluateAttributeExpressions(putFlowFile).getValue(); // create the directory if it doesn't exist final Path directoryPath = new Path(directoryValue); createDirectory(fileSystem, directoryPath, remoteOwner, remoteGroup); // write to tempFile first and on success rename to destFile final Path tempFile = new Path(directoryPath, "." + filenameValue); final Path destFile = new Path(directoryPath, filenameValue); final boolean destinationExists = fileSystem.exists(destFile) || fileSystem.exists(tempFile); final boolean shouldOverwrite = context.getProperty(OVERWRITE).asBoolean(); // if the tempFile or destFile already exist, and overwrite is set to false, then transfer to failure if (destinationExists && !shouldOverwrite) { session.transfer(session.penalize(putFlowFile), REL_FAILURE); getLogger().warn( "penalizing {} and routing to failure because file with same name already exists", new Object[] { putFlowFile }); return null; } final AtomicReference<Throwable> exceptionHolder = new AtomicReference<>(null); final AtomicReference<WriteResult> writeResult = new AtomicReference<>(); final RecordReaderFactory recordReaderFactory = context.getProperty(RECORD_READER) .asControllerService(RecordReaderFactory.class); final FlowFile flowFileIn = putFlowFile; final StopWatch stopWatch = new StopWatch(true); // Read records from the incoming FlowFile and write them the tempFile session.read(putFlowFile, (final InputStream rawIn) -> { RecordReader recordReader = null; HDFSRecordWriter recordWriter = null; try (final BufferedInputStream in = new BufferedInputStream(rawIn)) { // if we fail to create the RecordReader then we want to route to failure, so we need to // handle this separately from the other IOExceptions which normally route to retry try { recordReader = recordReaderFactory.createRecordReader(flowFileIn, in, getLogger()); } catch (Exception e) { final RecordReaderFactoryException rrfe = new RecordReaderFactoryException( "Unable to create RecordReader", e); exceptionHolder.set(rrfe); return; } final RecordSet recordSet = recordReader.createRecordSet(); recordWriter = createHDFSRecordWriter(context, flowFile, configuration, tempFile, recordReader.getSchema()); writeResult.set(recordWriter.write(recordSet)); } catch (Exception e) { exceptionHolder.set(e); } finally { IOUtils.closeQuietly(recordReader); IOUtils.closeQuietly(recordWriter); } }); stopWatch.stop(); final String dataRate = stopWatch.calculateDataRate(putFlowFile.getSize()); final long millis = stopWatch.getDuration(TimeUnit.MILLISECONDS); tempDotCopyFile = tempFile; // if any errors happened within the session.read then throw the exception so we jump // into one of the appropriate catch blocks below if (exceptionHolder.get() != null) { throw exceptionHolder.get(); } // Attempt to rename from the tempFile to destFile, and change owner if successfully renamed rename(fileSystem, tempFile, destFile); changeOwner(fileSystem, destFile, remoteOwner, remoteGroup); getLogger().info("Wrote {} to {} in {} milliseconds at a rate of {}", new Object[] { putFlowFile, destFile, millis, dataRate }); putFlowFile = postProcess(context, session, putFlowFile, destFile); final String newFilename = destFile.getName(); final String hdfsPath = destFile.getParent().toString(); // Update the filename and absolute path attributes final Map<String, String> attributes = new HashMap<>(writeResult.get().getAttributes()); attributes.put(CoreAttributes.FILENAME.key(), newFilename); attributes.put(ABSOLUTE_HDFS_PATH_ATTRIBUTE, hdfsPath); attributes.put(RECORD_COUNT_ATTR, String.valueOf(writeResult.get().getRecordCount())); putFlowFile = session.putAllAttributes(putFlowFile, attributes); // Send a provenance event and transfer to success final Path qualifiedPath = destFile.makeQualified(fileSystem.getUri(), fileSystem.getWorkingDirectory()); session.getProvenanceReporter().send(putFlowFile, qualifiedPath.toString()); session.transfer(putFlowFile, REL_SUCCESS); } catch (IOException | FlowFileAccessException e) { deleteQuietly(fileSystem, tempDotCopyFile); getLogger().error("Failed to write due to {}", new Object[] { e }); session.transfer(session.penalize(putFlowFile), REL_RETRY); context.yield(); } catch (Throwable t) { deleteQuietly(fileSystem, tempDotCopyFile); getLogger().error("Failed to write due to {}", new Object[] { t }); session.transfer(putFlowFile, REL_FAILURE); } return null; }); }
From source file:org.apache.nifi.processors.hadoop.DeleteHDFS.java
License:Apache License
@Override public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException { final FlowFile originalFlowFile = session.get(); // If this processor has an incoming connection, then do not run unless a // FlowFile is actually sent through if (originalFlowFile == null && context.hasIncomingConnection()) { context.yield();//from w w w. ja va 2s . co m return; } // We need a FlowFile to report provenance correctly. FlowFile flowFile = originalFlowFile != null ? originalFlowFile : session.create(); final String fileOrDirectoryName = context.getProperty(FILE_OR_DIRECTORY) .evaluateAttributeExpressions(flowFile).getValue(); final FileSystem fileSystem = getFileSystem(); try { // Check if the user has supplied a file or directory pattern List<Path> pathList = Lists.newArrayList(); if (GLOB_MATCHER.reset(fileOrDirectoryName).find()) { FileStatus[] fileStatuses = fileSystem.globStatus(new Path(fileOrDirectoryName)); if (fileStatuses != null) { for (FileStatus fileStatus : fileStatuses) { pathList.add(fileStatus.getPath()); } } } else { pathList.add(new Path(fileOrDirectoryName)); } int failedPath = 0; for (Path path : pathList) { if (fileSystem.exists(path)) { try { Map<String, String> attributes = Maps.newHashMapWithExpectedSize(2); attributes.put("hdfs.filename", path.getName()); attributes.put("hdfs.path", path.getParent().toString()); flowFile = session.putAllAttributes(flowFile, attributes); fileSystem.delete(path, context.getProperty(RECURSIVE).asBoolean()); getLogger().debug("For flowfile {} Deleted file at path {} with name {}", new Object[] { originalFlowFile, path.getParent().toString(), path.getName() }); final Path qualifiedPath = path.makeQualified(fileSystem.getUri(), fileSystem.getWorkingDirectory()); session.getProvenanceReporter().invokeRemoteProcess(flowFile, qualifiedPath.toString()); } catch (IOException ioe) { // One possible scenario is that the IOException is permissions based, however it would be impractical to check every possible // external HDFS authorization tool (Ranger, Sentry, etc). Local ACLs could be checked but the operation would be expensive. getLogger().warn("Failed to delete file or directory", ioe); Map<String, String> attributes = Maps.newHashMapWithExpectedSize(1); // The error message is helpful in understanding at a flowfile level what caused the IOException (which ACL is denying the operation, e.g.) attributes.put("hdfs.error.message", ioe.getMessage()); session.transfer(session.putAllAttributes(session.clone(flowFile), attributes), REL_FAILURE); failedPath++; } } } if (failedPath == 0) { session.transfer(flowFile, DeleteHDFS.REL_SUCCESS); } else { // If any path has been failed to be deleted, remove the FlowFile as it's been cloned and sent to failure. session.remove(flowFile); } } catch (IOException e) { getLogger().error("Error processing delete for flowfile {} due to {}", new Object[] { flowFile, e.getMessage() }, e); session.transfer(flowFile, DeleteHDFS.REL_FAILURE); } }
From source file:org.apache.nifi.processors.hadoop.FetchHDFS.java
License:Apache License
@Override public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException { FlowFile flowFile = session.get();/*from w ww . j a va2s .com*/ if (flowFile == null) { return; } final FileSystem hdfs = getFileSystem(); final UserGroupInformation ugi = getUserGroupInformation(); final String filenameValue = context.getProperty(FILENAME).evaluateAttributeExpressions(flowFile) .getValue(); final Path path; try { path = new Path(filenameValue); } catch (IllegalArgumentException e) { getLogger().error("Failed to retrieve content from {} for {} due to {}; routing to failure", new Object[] { filenameValue, flowFile, e }); flowFile = session.putAttribute(flowFile, "hdfs.failure.reason", e.getMessage()); flowFile = session.penalize(flowFile); session.transfer(flowFile, REL_FAILURE); return; } final StopWatch stopWatch = new StopWatch(true); final FlowFile finalFlowFile = flowFile; ugi.doAs(new PrivilegedAction<Object>() { @Override public Object run() { InputStream stream = null; CompressionCodec codec = null; Configuration conf = getConfiguration(); final CompressionCodecFactory compressionCodecFactory = new CompressionCodecFactory(conf); final CompressionType compressionType = CompressionType .valueOf(context.getProperty(COMPRESSION_CODEC).toString()); final boolean inferCompressionCodec = compressionType == CompressionType.AUTOMATIC; if (inferCompressionCodec) { codec = compressionCodecFactory.getCodec(path); } else if (compressionType != CompressionType.NONE) { codec = getCompressionCodec(context, getConfiguration()); } FlowFile flowFile = finalFlowFile; final Path qualifiedPath = path.makeQualified(hdfs.getUri(), hdfs.getWorkingDirectory()); try { final String outputFilename; final String originalFilename = path.getName(); stream = hdfs.open(path, 16384); // Check if compression codec is defined (inferred or otherwise) if (codec != null) { stream = codec.createInputStream(stream); outputFilename = StringUtils.removeEnd(originalFilename, codec.getDefaultExtension()); } else { outputFilename = originalFilename; } flowFile = session.importFrom(stream, finalFlowFile); flowFile = session.putAttribute(flowFile, CoreAttributes.FILENAME.key(), outputFilename); stopWatch.stop(); getLogger().info("Successfully received content from {} for {} in {}", new Object[] { qualifiedPath, flowFile, stopWatch.getDuration() }); session.getProvenanceReporter().fetch(flowFile, qualifiedPath.toString(), stopWatch.getDuration(TimeUnit.MILLISECONDS)); session.transfer(flowFile, REL_SUCCESS); } catch (final FileNotFoundException | AccessControlException e) { getLogger().error("Failed to retrieve content from {} for {} due to {}; routing to failure", new Object[] { qualifiedPath, flowFile, e }); flowFile = session.putAttribute(flowFile, "hdfs.failure.reason", e.getMessage()); flowFile = session.penalize(flowFile); session.transfer(flowFile, REL_FAILURE); } catch (final IOException e) { getLogger().error( "Failed to retrieve content from {} for {} due to {}; routing to comms.failure", new Object[] { qualifiedPath, flowFile, e }); flowFile = session.penalize(flowFile); session.transfer(flowFile, REL_COMMS_FAILURE); } finally { IOUtils.closeQuietly(stream); } return null; } }); }
From source file:org.apache.nifi.processors.hadoop.PutHDFS.java
License:Apache License
@Override public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException { final FlowFile flowFile = session.get(); if (flowFile == null) { return;//from www . j a v a2 s . co m } final FileSystem hdfs = getFileSystem(); final Configuration configuration = getConfiguration(); final UserGroupInformation ugi = getUserGroupInformation(); if (configuration == null || hdfs == null || ugi == null) { getLogger().error("HDFS not configured properly"); session.transfer(flowFile, REL_FAILURE); context.yield(); return; } ugi.doAs(new PrivilegedAction<Object>() { @Override public Object run() { Path tempDotCopyFile = null; FlowFile putFlowFile = flowFile; try { final String dirValue = context.getProperty(DIRECTORY).evaluateAttributeExpressions(putFlowFile) .getValue(); final Path configuredRootDirPath = new Path(dirValue); final String conflictResponse = context.getProperty(CONFLICT_RESOLUTION).getValue(); final Double blockSizeProp = context.getProperty(BLOCK_SIZE).asDataSize(DataUnit.B); final long blockSize = blockSizeProp != null ? blockSizeProp.longValue() : hdfs.getDefaultBlockSize(configuredRootDirPath); final Double bufferSizeProp = context.getProperty(BUFFER_SIZE).asDataSize(DataUnit.B); final int bufferSize = bufferSizeProp != null ? bufferSizeProp.intValue() : configuration.getInt(BUFFER_SIZE_KEY, BUFFER_SIZE_DEFAULT); final Integer replicationProp = context.getProperty(REPLICATION_FACTOR).asInteger(); final short replication = replicationProp != null ? replicationProp.shortValue() : hdfs.getDefaultReplication(configuredRootDirPath); final CompressionCodec codec = getCompressionCodec(context, configuration); final String filename = codec != null ? putFlowFile.getAttribute(CoreAttributes.FILENAME.key()) + codec.getDefaultExtension() : putFlowFile.getAttribute(CoreAttributes.FILENAME.key()); final Path tempCopyFile = new Path(configuredRootDirPath, "." + filename); final Path copyFile = new Path(configuredRootDirPath, filename); // Create destination directory if it does not exist try { if (!hdfs.getFileStatus(configuredRootDirPath).isDirectory()) { throw new IOException( configuredRootDirPath.toString() + " already exists and is not a directory"); } } catch (FileNotFoundException fe) { if (!hdfs.mkdirs(configuredRootDirPath)) { throw new IOException(configuredRootDirPath.toString() + " could not be created"); } changeOwner(context, hdfs, configuredRootDirPath, flowFile); } final boolean destinationExists = hdfs.exists(copyFile); // If destination file already exists, resolve that based on processor configuration if (destinationExists) { switch (conflictResponse) { case REPLACE_RESOLUTION: if (hdfs.delete(copyFile, false)) { getLogger().info("deleted {} in order to replace with the contents of {}", new Object[] { copyFile, putFlowFile }); } break; case IGNORE_RESOLUTION: session.transfer(putFlowFile, REL_SUCCESS); getLogger().info( "transferring {} to success because file with same name already exists", new Object[] { putFlowFile }); return null; case FAIL_RESOLUTION: session.transfer(session.penalize(putFlowFile), REL_FAILURE); getLogger().warn( "penalizing {} and routing to failure because file with same name already exists", new Object[] { putFlowFile }); return null; default: break; } } // Write FlowFile to temp file on HDFS final StopWatch stopWatch = new StopWatch(true); session.read(putFlowFile, new InputStreamCallback() { @Override public void process(InputStream in) throws IOException { OutputStream fos = null; Path createdFile = null; try { if (conflictResponse.equals(APPEND_RESOLUTION_AV.getValue()) && destinationExists) { fos = hdfs.append(copyFile, bufferSize); } else { fos = hdfs.create(tempCopyFile, true, bufferSize, replication, blockSize); } if (codec != null) { fos = codec.createOutputStream(fos); } createdFile = tempCopyFile; BufferedInputStream bis = new BufferedInputStream(in); StreamUtils.copy(bis, fos); bis = null; fos.flush(); } finally { try { if (fos != null) { fos.close(); } } catch (RemoteException re) { // when talking to remote HDFS clusters, we don't notice problems until fos.close() if (createdFile != null) { try { hdfs.delete(createdFile, false); } catch (Throwable ignore) { } } throw re; } catch (Throwable ignore) { } fos = null; } } }); stopWatch.stop(); final String dataRate = stopWatch.calculateDataRate(putFlowFile.getSize()); final long millis = stopWatch.getDuration(TimeUnit.MILLISECONDS); tempDotCopyFile = tempCopyFile; if (!conflictResponse.equals(APPEND_RESOLUTION_AV.getValue()) || (conflictResponse.equals(APPEND_RESOLUTION_AV.getValue()) && !destinationExists)) { boolean renamed = false; for (int i = 0; i < 10; i++) { // try to rename multiple times. if (hdfs.rename(tempCopyFile, copyFile)) { renamed = true; break;// rename was successful } Thread.sleep(200L);// try waiting to let whatever might cause rename failure to resolve } if (!renamed) { hdfs.delete(tempCopyFile, false); throw new ProcessException("Copied file to HDFS but could not rename dot file " + tempCopyFile + " to its final filename"); } changeOwner(context, hdfs, copyFile, flowFile); } getLogger().info("copied {} to HDFS at {} in {} milliseconds at a rate of {}", new Object[] { putFlowFile, copyFile, millis, dataRate }); final String newFilename = copyFile.getName(); final String hdfsPath = copyFile.getParent().toString(); putFlowFile = session.putAttribute(putFlowFile, CoreAttributes.FILENAME.key(), newFilename); putFlowFile = session.putAttribute(putFlowFile, ABSOLUTE_HDFS_PATH_ATTRIBUTE, hdfsPath); final Path qualifiedPath = copyFile.makeQualified(hdfs.getUri(), hdfs.getWorkingDirectory()); session.getProvenanceReporter().send(putFlowFile, qualifiedPath.toString()); session.transfer(putFlowFile, REL_SUCCESS); } catch (final Throwable t) { if (tempDotCopyFile != null) { try { hdfs.delete(tempDotCopyFile, false); } catch (Exception e) { getLogger().error("Unable to remove temporary file {} due to {}", new Object[] { tempDotCopyFile, e }); } } getLogger().error("Failed to write to HDFS due to {}", new Object[] { t }); session.transfer(session.penalize(putFlowFile), REL_FAILURE); context.yield(); } return null; } }); }
From source file:org.apache.oozie.util.ClasspathUtils.java
License:Apache License
private static void addToClasspathIfNotJar(Path[] paths, URI[] withLinks, Configuration conf, Map<String, String> environment, String classpathEnvVar) throws IOException { if (paths != null) { HashMap<Path, String> linkLookup = new HashMap<Path, String>(); if (withLinks != null) { for (URI u : withLinks) { Path p = new Path(u); FileSystem remoteFS = p.getFileSystem(conf); p = remoteFS.resolvePath(p.makeQualified(remoteFS.getUri(), remoteFS.getWorkingDirectory())); String name = (null == u.getFragment()) ? p.getName() : u.getFragment(); if (!name.toLowerCase(Locale.ENGLISH).endsWith(".jar")) { linkLookup.put(p, name); }//from www .ja v a 2 s . c o m } } for (Path p : paths) { FileSystem remoteFS = p.getFileSystem(conf); p = remoteFS.resolvePath(p.makeQualified(remoteFS.getUri(), remoteFS.getWorkingDirectory())); String name = linkLookup.get(p); if (name == null) { name = p.getName(); } if (!name.toLowerCase(Locale.ENGLISH).endsWith(".jar")) { MRApps.addToEnvironment(environment, classpathEnvVar, ApplicationConstants.Environment.PWD.$() + Path.SEPARATOR + name, conf); } } } }
From source file:org.apache.sentry.binding.hive.authz.HiveAuthzBindingHookBase.java
License:Apache License
@VisibleForTesting static public AccessURI parseURI(String uri, boolean isLocal) throws SemanticException { try {/* ww w.j a v a 2 s . c o m*/ HiveConf conf = SessionState.get().getConf(); String warehouseDir = conf.getVar(ConfVars.METASTOREWAREHOUSE); Path warehousePath = new Path(warehouseDir); // If warehousePath is an absolute path and a scheme is null and authority is null as well, // qualified it with default file system scheme and authority. if (warehousePath.isAbsoluteAndSchemeAuthorityNull()) { URI defaultUri = FileSystem.getDefaultUri(conf); warehousePath = warehousePath.makeQualified(defaultUri, warehousePath); warehouseDir = warehousePath.toUri().toString(); } return new AccessURI(PathUtils.parseURI(warehouseDir, uri, isLocal)); } catch (Exception e) { throw new SemanticException("Error parsing URI " + uri + ": " + e.getMessage(), e); } }
From source file:org.apache.sentry.binding.hive.HiveAuthzBindingHookBase.java
License:Apache License
@VisibleForTesting protected static AccessURI parseURI(String uri, boolean isLocal) throws SemanticException { try {/* w w w . j a va 2 s .c o m*/ HiveConf conf = SessionState.get().getConf(); String warehouseDir = conf.getVar(ConfVars.METASTOREWAREHOUSE); Path warehousePath = new Path(warehouseDir); // If warehousePath is an absolute path and a scheme is null and authority is null as well, // qualified it with default file system scheme and authority. if (warehousePath.isAbsoluteAndSchemeAuthorityNull()) { URI defaultUri = FileSystem.getDefaultUri(conf); warehousePath = warehousePath.makeQualified(defaultUri, warehousePath); warehouseDir = warehousePath.toUri().toString(); } return new AccessURI(PathUtils.parseURI(warehouseDir, uri, isLocal)); } catch (Exception e) { throw new SemanticException("Error parsing URI " + uri + ": " + e.getMessage(), e); } }
From source file:org.apache.sentry.core.common.utils.PathUtils.java
License:Apache License
/** * Make fully qualified URI if Scheme and/or Authority is missing, * based on the default file system Scheme and Authority. * Notes:/*from www . jav a2 s. c o m*/ * a) input URI path must be absolute; otherwise return null. * b) Path.makeQualified() provides no assurance that the * default file system Scheme and Authority values are not null. * * @param uriName The Uri name. * @return Returns the fully qualified URI or null if URI path is not absolute. * @throws IOException */ private static URI makeFullQualifiedURI(String uriName) throws IOException { Path uriPath = new Path(uriName); if (isNormalized(uriName) && uriPath.isUriPathAbsolute()) { // add scheme and/or authority if either is missing if ((uriPath.toUri().getScheme() == null || uriPath.toUri().getAuthority() == null)) { URI defaultUri = FileSystem.getDefaultUri(CONF); uriPath = uriPath.makeQualified(defaultUri, uriPath); } return uriPath.toUri(); } else { // relative URI path is unacceptable return null; } }
From source file:org.apache.sentry.core.common.utils.PathUtils.java
License:Apache License
/** * Parse a URI which can be HDFS, S3, SWIFT, WEBHDFS,etc. In either case it * should be on the same fs as the warehouse directory. *//*from w w w. j a va 2s .co m*/ public static String parseURI(String warehouseDir, String uri, boolean isLocal) throws URISyntaxException { Path warehouseDirPath = new Path(warehouseDir); Path uriPath = new Path(uri); if (uriPath.isAbsolute()) { // Merge warehouseDir and uri only when there is no scheme and authority // in uri. if (uriPath.isAbsoluteAndSchemeAuthorityNull()) { uriPath = uriPath.makeQualified(warehouseDirPath.toUri(), warehouseDirPath); } String uriScheme = uriPath.toUri().getScheme(); String uriAuthority = uriPath.toUri().getAuthority(); if (StringUtils.isEmpty(uriScheme) || isLocal) { uriScheme = LOCAL_FILE_SCHEMA; uriAuthority = ""; } uriPath = new Path(uriScheme + AUTHORITY_PREFIX + StringUtils.trimToEmpty(uriAuthority) + Path.getPathWithoutSchemeAndAuthority(uriPath)); } else { // don't support relative path throw new IllegalArgumentException("Invalid URI " + uri + "."); } return uriPath.toUri().toString(); }