Example usage for org.apache.hadoop.fs Path toUri

List of usage examples for org.apache.hadoop.fs Path toUri

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. Path
  6. toUri

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path toUri.

Prototype

public URI toUri()

Source Link

Document

Convert this Path to a URI.

Usage

From source file:com.thinkbiganalytics.kylo.catalog.azure.AzureNativeFileSystemProvider.java

License:Apache License

@Nonnull
@Override//  ww w .ja  v a 2s.c  om
@SuppressWarnings("squid:S1075")
public List<DataSetFile> listFiles(@Nonnull final Path path, @Nonnull final Configuration conf) {
    // Create Azure Blob client
    final URI uri = path.toUri();
    final CloudBlobClient client = createBlobClient(uri, conf);

    // List containers as data set files
    final String pathPrefix = uri.getScheme() + "://";
    final String pathSuffix = "@" + uri.getHost() + (uri.getPort() > -1 ? ":" + uri.getPort() : "") + "/";
    return StreamSupport.stream(listContainers(client).spliterator(), false).map(container -> {
        final DataSetFile file = new DataSetFile();
        file.setName(container.getName());
        file.setDirectory(true);
        file.setModificationTime(container.getProperties().getLastModified().getTime());
        file.setPath(pathPrefix + container.getName() + pathSuffix);
        return file;
    }).collect(Collectors.toList());
}

From source file:com.thinkbiganalytics.kylo.catalog.azure.AzureNativeFileSystemProvider.java

License:Apache License

@Override
public boolean supportsPath(@Nonnull Path path) {
    return StringUtils.equalsAnyIgnoreCase(path.toUri().getScheme(), "asv", "asvs", "wasb", "wasbs")
            && StringUtils.isEmpty(path.toUri().getUserInfo()) && StringUtils.isNotEmpty(path.toUri().getHost())
            && StringUtils.equalsAny(path.toUri().getPath(), null, "", "/");
}

From source file:com.thinkbiganalytics.kylo.catalog.file.DefaultCatalogFileManager.java

License:Apache License

/**
 * Executes the specified function in a separate class loader containing the jars of the specified template.
 *//* w w w .jav  a2 s.c  o  m*/
@VisibleForTesting
protected <R> R isolatedFunction(@Nonnull final DataSetTemplate template, @Nonnull final Path path,
        @Nonnull final FileSystemFunction<R> function) throws IOException {
    final Configuration conf = DataSetUtil.getConfiguration(template, defaultConf);
    try (final HadoopClassLoader classLoader = new HadoopClassLoader(conf)) {
        if (template.getJars() != null) {
            log.debug("Adding jars to HadoopClassLoader: {}", template.getJars());
            classLoader.addJars(template.getJars());
        }

        log.debug("Creating FileSystem from path: {}", path);
        try (final FileSystem fs = FileSystem.newInstance(path.toUri(), conf)) {
            return function.apply(fs);
        }
    }
}

From source file:com.thinkbiganalytics.kylo.catalog.file.PathValidator.java

License:Apache License

/**
 * Determines if the specified path is allowed for the specified data set and data source.
 *//*from   ww  w . ja v a2s . c  om*/
private boolean isPathAllowed(@Nonnull final Path path, @Nullable final String dataSetId,
        @Nonnull final DataSource dataSource) {
    final Optional<List<String>> dataSourcePaths = DataSourceUtil.getPaths(dataSource);
    if (dataSourcePaths.isPresent()) {
        final Stream<String> allowedPaths = dataSourcePaths.get().stream();
        final String pluginId = dataSource.getConnector().getPluginId();
        final Optional<ConnectorPlugin> plugin = this.pluginManager.getPlugin(pluginId);

        if (plugin.isPresent()) {
            if (ConnectorUtil.hasAnyTabSref(plugin.get().getDescriptor(), fileSystemSrefs)) {
                return isPathAllowed(path.toUri(), toURIs(allowedPaths));
            }
            if (dataSetId != null && ConnectorUtil.hasAnyTabSref(plugin.get().getDescriptor(), uploadSrefs)) {
                final Stream<String> uploadPaths = allowedPaths
                        .map(allowedPath -> allowedPath.endsWith(Path.SEPARATOR) ? allowedPath
                                : allowedPath + Path.SEPARATOR)
                        .map(allowedPath -> allowedPath + dataSetId + Path.SEPARATOR);
                return isPathAllowed(path.toUri(), toURIs(uploadPaths));
            }
        } else {
            return false;
        }
    }
    return true;
}

From source file:com.thinkbiganalytics.kylo.hadoop.MockFileSystem.java

License:Apache License

@Override
public File pathToFile(final Path path) {
    return new File(path.toUri().getPath());
}

From source file:com.toy.Client.java

License:Apache License

private static void registerLocalResource(Map<String, LocalResource> localResources, ApplicationId appId,
        FileSystem fs, Path src) throws IOException {
    String pathSuffix = Constants.TOY_PREFIX + appId.toString() + "/" + src.getName();
    Path dst = new Path(fs.getHomeDirectory(), pathSuffix);
    LOG.info("Copy {} from local filesystem to {} and add to local environment", src.getName(), dst.toUri());
    fs.copyFromLocalFile(false, true, src, dst);
    FileStatus destStatus = fs.getFileStatus(dst);
    LocalResource amJarRsrc = Records.newRecord(LocalResource.class);
    amJarRsrc.setType(LocalResourceType.FILE);
    amJarRsrc.setVisibility(LocalResourceVisibility.APPLICATION);
    amJarRsrc.setResource(ConverterUtils.getYarnUrlFromPath(dst));
    amJarRsrc.setTimestamp(destStatus.getModificationTime());
    amJarRsrc.setSize(destStatus.getLen());
    localResources.put(src.getName(), amJarRsrc);
}

From source file:com.toy.Client.java

License:Apache License

private void uploadDepAndRegister(Map<String, LocalResource> localResources, ApplicationId appId, FileSystem fs,
        String depname) throws IOException {
    File dep = new File(depname);
    if (!dep.exists())
        throw new IOException(dep.getAbsolutePath() + " does not exist");
    Path dst = new Path(fs.getHomeDirectory(), Constants.TOY_PREFIX + appId.toString() + "/" + dep.getName());
    LOG.info("Copy {} from local filesystem to {} and add to local environment", dep.getName(), dst.toUri());
    FileInputStream input = new FileInputStream(dep);
    final FSDataOutputStream outputStream = fs.create(dst, true);
    ByteStreams.copy(input, outputStream);
    input.close();/*from  www.ja  v  a 2  s .c om*/
    outputStream.close();
    LocalResource amJarRsrc = Records.newRecord(LocalResource.class);
    amJarRsrc.setType(LocalResourceType.FILE);
    amJarRsrc.setVisibility(LocalResourceVisibility.APPLICATION);
    amJarRsrc.setResource(ConverterUtils.getYarnUrlFromPath(dst));
    FileStatus destStatus = fs.getFileStatus(dst);
    amJarRsrc.setTimestamp(destStatus.getModificationTime());
    amJarRsrc.setSize(destStatus.getLen());
    localResources.put(dep.getName(), amJarRsrc);

}

From source file:com.trendmicro.hdfs.webdav.HDFSResource.java

License:Apache License

@Override
public void addMember(final DavResource resource, final InputContext context) throws DavException {
    // A PUT performed on an existing resource replaces the GET response entity
    // of the resource. Properties defined on the resource may be recomputed
    // during PUT processing but are not otherwise affected.
    final HDFSResource dfsResource = (HDFSResource) resource;
    final Path destPath = dfsResource.getPath();
    try {/*from w  ww. java 2  s.co  m*/
        if (dfsResource.isCollectionRequest) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Creating new directory '" + destPath.toUri().getPath() + "'");
            }
            boolean success = user.doAs(new PrivilegedExceptionAction<Boolean>() {
                public Boolean run() throws Exception {
                    return FileSystem.get(conf).mkdirs(destPath);
                }
            });
            if (!success) {
                throw new DavException(DavServletResponse.SC_CONFLICT);
            }
        } else {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Creating new file '" + destPath.toUri().getPath() + "'");
            }
            if (!context.hasStream() || context.getContentLength() < 0) {
                boolean success = user.doAs(new PrivilegedExceptionAction<Boolean>() {
                    public Boolean run() throws Exception {
                        return FileSystem.get(conf).createNewFile(destPath);
                    }
                });
                if (!success) {
                    throw new DavException(DavServletResponse.SC_CONFLICT);
                }
            } else {
                user.doAs(new PrivilegedExceptionAction<Void>() {
                    public Void run() throws Exception {
                        OutputStream out = FileSystem.get(conf).create(destPath);
                        InputStream in = context.getInputStream();
                        IOUtils.copyBytes(in, out, conf, true);
                        return null;
                    }
                });
            }
        }
    } catch (IOException e) {
        throw new RuntimeException(e);
    } catch (InterruptedException e) {
        throw new RuntimeException(e);
    }
}

From source file:com.trendmicro.hdfs.webdav.HDFSResource.java

License:Apache License

@Override
public void copy(final DavResource resource, final boolean shallow) throws DavException {
    if (!exists()) {
        throw new DavException(DavServletResponse.SC_NOT_FOUND);
    }/*  w  ww. j  a va  2 s.  c  o  m*/
    if (!shallow || !isCollection()) {
        final HDFSResource dfsResource = (HDFSResource) resource;
        final Path destPath = dfsResource.getPath();
        if (LOG.isDebugEnabled()) {
            LOG.debug("Copying '" + path.toUri().getPath() + "' to '" + destPath.toUri().getPath() + "'");
        }
        try {
            user.doAs(new PrivilegedExceptionAction<Void>() {
                public Void run() throws Exception {
                    FileSystem fs = FileSystem.get(conf);
                    FileUtil.copy(fs, path, fs, destPath, false, conf);
                    return null;
                }
            });
        } catch (IOException e) {
            throw new RuntimeException(e);
        } catch (InterruptedException e) {
            throw new RuntimeException(e);
        }
        return;
    }
    // TODO: Currently no support for shallow copy; however, this is
    // only relevant if the source resource is a collection
    throw new DavException(DavServletResponse.SC_FORBIDDEN, "Shallow copies are not supported");
}

From source file:com.trendmicro.hdfs.webdav.HDFSResource.java

License:Apache License

@Override
public DavResourceIterator getMembers() {
    List<DavResource> list = new ArrayList<DavResource>();
    try {/* w w  w  .j  ava2 s.  co  m*/
        FileStatus[] stat = user.doAs(new PrivilegedExceptionAction<FileStatus[]>() {
            public FileStatus[] run() throws Exception {
                return FileSystem.get(conf).listStatus(path);
            }
        });
        if (stat != null) {
            for (FileStatus s : stat) {
                Path p = s.getPath();
                DavResourceLocator resourceLocator = locator.getFactory().createResourceLocator(
                        locator.getPrefix(), locator.getWorkspacePath(), p.toString(), false);
                try {
                    HDFSResource resource = (HDFSResource) factory.createResource(resourceLocator,
                            getSession());
                    resource.user = this.user;
                    list.add(resource);
                } catch (DavException ex) {
                    LOG.warn("Exception adding resource '" + p.toUri().getPath() + "' to iterator");
                }
            }
        }
    } catch (IOException e) {
        throw new RuntimeException(e);
    } catch (InterruptedException e) {
        throw new RuntimeException(e);
    }
    return new DavResourceIteratorImpl(list);
}