Example usage for org.apache.hadoop.fs Path getPathWithoutSchemeAndAuthority

List of usage examples for org.apache.hadoop.fs Path getPathWithoutSchemeAndAuthority

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. Path
  6. getPathWithoutSchemeAndAuthority

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path getPathWithoutSchemeAndAuthority.

Prototype

public static Path getPathWithoutSchemeAndAuthority(Path path)

Source Link

Document

Return a version of the given Path without the scheme information.

Usage

From source file:com.uber.hoodie.hive.HoodieHiveClient.java

License:Apache License

/**
 * Iterate over the storage partitions and find if there are any new partitions that need to be
 * added or updated. Generate a list of PartitionEvent based on the changes required.
 *///from w w w . j  a va  2 s.c  o  m
List<PartitionEvent> getPartitionEvents(List<Partition> tablePartitions,
        List<String> partitionStoragePartitions) {
    Map<String, String> paths = Maps.newHashMap();
    for (Partition tablePartition : tablePartitions) {
        List<String> hivePartitionValues = tablePartition.getValues();
        Collections.sort(hivePartitionValues);
        String fullTablePartitionPath = Path
                .getPathWithoutSchemeAndAuthority(new Path(tablePartition.getSd().getLocation())).toUri()
                .getPath();
        paths.put(String.join(", ", hivePartitionValues), fullTablePartitionPath);
    }

    List<PartitionEvent> events = Lists.newArrayList();
    for (String storagePartition : partitionStoragePartitions) {
        String fullStoragePartitionPath = FSUtils.getPartitionPath(syncConfig.basePath, storagePartition)
                .toString();
        // Check if the partition values or if hdfs path is the same
        List<String> storagePartitionValues = partitionValueExtractor
                .extractPartitionValuesInPath(storagePartition);
        Collections.sort(storagePartitionValues);
        if (!storagePartitionValues.isEmpty()) {
            String storageValue = String.join(", ", storagePartitionValues);
            if (!paths.containsKey(storageValue)) {
                events.add(PartitionEvent.newPartitionAddEvent(storagePartition));
            } else if (!paths.get(storageValue).equals(fullStoragePartitionPath)) {
                events.add(PartitionEvent.newPartitionUpdateEvent(storagePartition));
            }
        }
    }
    return events;
}

From source file:com.uber.hoodie.hive.model.TablePartition.java

License:Apache License

public Path getLocation() {
    return Path.getPathWithoutSchemeAndAuthority(new Path(partition.getSd().getLocation()));
}

From source file:org.apache.ambari.view.filebrowser.HdfsApi.java

License:Apache License

/**
 * Converts a Hadoop <code>FileStatus</code> object into a JSON array object.
 * It replaces the <code>SCHEME://HOST:PORT</code> of the path with the
 * specified URL./*from   w ww. j av  a2s .c om*/
 * <p/>
 *
 * @param status
 *          Hadoop file status.
 * @return The JSON representation of the file status.
 */

public Map<String, Object> fileStatusToJSON(FileStatus status) {
    Map<String, Object> json = new LinkedHashMap<String, Object>();
    json.put("path", Path.getPathWithoutSchemeAndAuthority(status.getPath()).toString());
    json.put("replication", status.getReplication());
    json.put("isDirectory", status.isDirectory());
    json.put("len", status.getLen());
    json.put("owner", status.getOwner());
    json.put("group", status.getGroup());
    json.put("permission", permissionToString(status.getPermission()));
    json.put("accessTime", status.getAccessTime());
    json.put("modificationTime", status.getModificationTime());
    json.put("blockSize", status.getBlockSize());
    json.put("replication", status.getReplication());
    json.put("readAccess", checkAccessPermissions(status, FsAction.READ, ugi));
    json.put("writeAccess", checkAccessPermissions(status, FsAction.WRITE, ugi));
    json.put("executeAccess", checkAccessPermissions(status, FsAction.EXECUTE, ugi));
    return json;
}

From source file:org.apache.ambari.view.utils.hdfs.HdfsApi.java

License:Apache License

/**
 * Converts a Hadoop <code>FileStatus</code> object into a JSON array object.
 * It replaces the <code>SCHEME://HOST:PORT</code> of the path with the
 * specified URL.//from  w ww.j  a va  2  s. c  o  m
 * <p/>
 *
 * @param status
 *          Hadoop file status.
 * @return The JSON representation of the file status.
 */
public Map<String, Object> fileStatusToJSON(FileStatus status) {
    Map<String, Object> json = new LinkedHashMap<String, Object>();
    json.put("path", Path.getPathWithoutSchemeAndAuthority(status.getPath()).toString());
    json.put("replication", status.getReplication());
    json.put("isDirectory", status.isDirectory());
    json.put("len", status.getLen());
    json.put("owner", status.getOwner());
    json.put("group", status.getGroup());
    json.put("permission", permissionToString(status.getPermission()));
    json.put("accessTime", status.getAccessTime());
    json.put("modificationTime", status.getModificationTime());
    json.put("blockSize", status.getBlockSize());
    json.put("replication", status.getReplication());
    json.put("readAccess", checkAccessPermissions(status, FsAction.READ, ugi));
    json.put("writeAccess", checkAccessPermissions(status, FsAction.WRITE, ugi));
    json.put("executeAccess", checkAccessPermissions(status, FsAction.EXECUTE, ugi));
    return json;
}

From source file:org.apache.atlas.falcon.bridge.FalconBridge.java

License:Apache License

private static List<Referenceable> fillHDFSDataSet(final String pathUri, final String clusterName) {
    List<Referenceable> entities = new ArrayList<>();
    Referenceable ref = new Referenceable(HiveMetaStoreBridge.HDFS_PATH);
    ref.set("path", pathUri);
    //        Path path = new Path(pathUri);
    //        ref.set("name", path.getName());
    //TODO - Fix after ATLAS-542 to shorter Name
    Path path = new Path(pathUri);
    ref.set(AtlasClient.NAME, Path.getPathWithoutSchemeAndAuthority(path).toString().toLowerCase());
    ref.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, pathUri);
    ref.set(AtlasConstants.CLUSTER_NAME_ATTRIBUTE, clusterName);
    entities.add(ref);/*from  www.j a v  a 2 s.  c o m*/
    return entities;
}

From source file:org.apache.atlas.hive.HiveITBase.java

License:Apache License

protected void validateHDFSPaths(Referenceable processReference, String attributeName, String... testPaths)
        throws Exception {
    List<Id> hdfsPathRefs = (List<Id>) processReference.get(attributeName);

    for (String testPath : testPaths) {
        final Path path = new Path(testPath);
        final String testPathNormed = lower(path.toString());
        String hdfsPathId = assertHDFSPathIsRegistered(testPathNormed);
        Assert.assertEquals(hdfsPathRefs.get(0)._getId(), hdfsPathId);

        Referenceable hdfsPathRef = atlasClient.getEntity(hdfsPathId);
        Assert.assertEquals(hdfsPathRef.get("path"), testPathNormed);
        Assert.assertEquals(hdfsPathRef.get(NAME),
                Path.getPathWithoutSchemeAndAuthority(path).toString().toLowerCase());
        Assert.assertEquals(hdfsPathRef.get(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME), testPathNormed);
    }/*from ww  w . ja  va  2 s.c  om*/
}

From source file:org.apache.beam.sdk.io.hdfs.HadoopFileSystem.java

License:Apache License

/**
 * Renames a {@link List} of file-like resources from one location to another.
 *
 * <p>The number of source resources must equal the number of destination resources. Destination
 * resources will be created recursively.
 *
 * @param srcResourceIds the references of the source resources
 * @param destResourceIds the references of the destination resources
 * @throws FileNotFoundException if the source resources are missing. When rename throws, the
 *     state of the resources is unknown but safe: for every (source, destination) pair of
 *     resources, the following are possible: a) source exists, b) destination exists, c) source
 *     and destination both exist. Thus no data is lost, however, duplicated resource are
 *     possible. In such scenarios, callers can use {@code match()} to determine the state of the
 *     resource.//from   w ww  .j  a  v a2s .c o  m
 * @throws FileAlreadyExistsException if a target resource already exists and couldn't be
 *     overwritten.
 * @throws IOException if the underlying filesystem indicates the rename was not performed but no
 *     other errors were thrown.
 */
@Override
protected void rename(List<HadoopResourceId> srcResourceIds, List<HadoopResourceId> destResourceIds)
        throws IOException {
    for (int i = 0; i < srcResourceIds.size(); ++i) {

        final Path srcPath = srcResourceIds.get(i).toPath();
        final Path destPath = destResourceIds.get(i).toPath();

        // this enforces src and dest file systems to match
        final org.apache.hadoop.fs.FileSystem fs = srcPath.getFileSystem(configuration);

        // rename in HDFS requires the target directory to exist or silently fails (BEAM-4861)
        mkdirs(destPath);

        boolean success = fs.rename(srcPath, destPath);

        // If the failure was due to the file already existing, delete and retry (BEAM-5036).
        // This should be the exceptional case, so handle here rather than incur the overhead of
        // testing first
        if (!success && fs.exists(srcPath) && fs.exists(destPath)) {
            LOG.debug(
                    String.format(LOG_DELETING_EXISTING_FILE, Path.getPathWithoutSchemeAndAuthority(destPath)));
            fs.delete(destPath, false); // not recursive
            success = fs.rename(srcPath, destPath);
        }

        if (!success) {
            if (!fs.exists(srcPath)) {
                throw new FileNotFoundException(String
                        .format("Unable to rename resource %s to %s as source not found.", srcPath, destPath));

            } else if (fs.exists(destPath)) {
                throw new FileAlreadyExistsException(String.format(
                        "Unable to rename resource %s to %s as destination already exists and couldn't be deleted.",
                        srcPath, destPath));

            } else {
                throw new IOException(String.format(
                        "Unable to rename resource %s to %s. No further information provided by underlying filesystem.",
                        srcPath, destPath));
            }
        }
    }
}

From source file:org.apache.beam.sdk.io.hdfs.HadoopFileSystem.java

License:Apache License

/** Ensures that the target directory exists for the given filePath. */
private void mkdirs(Path filePath) throws IOException {
    final org.apache.hadoop.fs.FileSystem fs = filePath.getFileSystem(configuration);
    final Path targetDirectory = filePath.getParent();
    if (!fs.exists(targetDirectory)) {
        LOG.debug(String.format(LOG_CREATE_DIRECTORY, Path.getPathWithoutSchemeAndAuthority(targetDirectory)));
        if (!fs.mkdirs(targetDirectory)) {
            throw new IOException(String.format(
                    "Unable to create target directory %s. No further information provided by underlying filesystem.",
                    targetDirectory));/* ww w . j  a  v  a2s. c  o m*/
        }
    }
}

From source file:org.apache.carbondata.core.datastorage.store.filesystem.LocalCarbonFile.java

License:Apache License

public LocalCarbonFile(String filePath) {
    Path pathWithoutSchemeAndAuthority = Path.getPathWithoutSchemeAndAuthority(new Path(filePath));
    file = new File(pathWithoutSchemeAndAuthority.toString());
}

From source file:org.apache.carbondata.core.datastorage.store.impl.FileFactory.java

License:Apache License

/**
 * below method will be used to update the file path
 * for local type/*from   w w  w  .java  2 s  .c  o m*/
 * it removes the file:/ from the path
 *
 * @param filePath
 * @return updated file path without url for local
 */
public static String getUpdatedFilePath(String filePath) {
    FileType fileType = getFileType(filePath);
    switch (fileType) {
    case HDFS:
    case VIEWFS:
        return filePath;
    case LOCAL:
    default:
        Path pathWithoutSchemeAndAuthority = Path.getPathWithoutSchemeAndAuthority(new Path(filePath));
        return pathWithoutSchemeAndAuthority.toString();
    }
}