Example usage for org.apache.hadoop.fs Path getName

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path getName.

Prototype

public String getName()

Source Link

Document

Returns the final component of this path.

Usage

From source file:com.uber.hoodie.common.io.storage.HoodieWrapperFileSystem.java

License:Apache License

private FSDataOutputStream wrapOutputStream(final Path path, FSDataOutputStream fsDataOutputStream)
        throws IOException {
    if (fsDataOutputStream instanceof SizeAwareFSDataOutputStream) {
        return fsDataOutputStream;
    }/*from   w  w  w . j a va2s.c  o m*/

    SizeAwareFSDataOutputStream os = new SizeAwareFSDataOutputStream(path, fsDataOutputStream, consistencyGuard,
            () -> openStreams.remove(path.getName()));
    openStreams.put(path.getName(), os);
    return os;
}

From source file:com.uber.hoodie.common.io.storage.HoodieWrapperFileSystem.java

License:Apache License

public long getBytesWritten(Path file) {
    if (openStreams.containsKey(file.getName())) {
        return openStreams.get(file.getName()).getBytesWritten();
    }//  w  w w  .  j  a  va2  s.c o m
    // When the file is first written, we do not have a track of it
    throw new IllegalArgumentException(
            file.toString() + " does not have a open stream. Cannot get the bytes written on the stream");
}

From source file:com.uber.hoodie.common.model.HoodieTableMetadata.java

License:Apache License

/**
 * Scan the commit times (only choosing commit file with the given suffix)
 *//* w w w .  ja va  2  s  .  c  o  m*/
private List<String> scanCommits(final String commitFileSuffix) throws IOException {
    log.info("Attempting to load the commits under " + metadataFolder + " with suffix " + commitFileSuffix);
    final List<String> commitFiles = new ArrayList<>();
    fs.listStatus(metadataFolder, new PathFilter() {
        @Override
        public boolean accept(Path path) {
            if (path.getName().endsWith(commitFileSuffix)) {
                commitFiles.add(path.getName().split("\\.")[0]);
                return true;
            }
            return false;
        }
    });
    return commitFiles;
}

From source file:com.uber.hoodie.common.table.log.HoodieLogFormatAppendFailureTest.java

License:Apache License

@Test(timeout = 60000)
public void testFailedToGetAppendStreamFromHDFSNameNode()
        throws IOException, URISyntaxException, InterruptedException, TimeoutException {

    // Use some fs like LocalFileSystem, that does not support appends
    String uuid = UUID.randomUUID().toString();
    Path localPartitionPath = new Path("/tmp/");
    FileSystem fs = cluster.getFileSystem();
    Path testPath = new Path(localPartitionPath, uuid);
    fs.mkdirs(testPath);//from w ww.j a v  a  2  s  .c o m

    // Some data & append.
    List<IndexedRecord> records = SchemaTestUtil.generateTestRecords(0, 10);
    Map<HoodieLogBlock.HeaderMetadataType, String> header = Maps.newHashMap();
    header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, "100");
    header.put(HoodieLogBlock.HeaderMetadataType.SCHEMA, getSimpleSchema().toString());
    HoodieAvroDataBlock dataBlock = new HoodieAvroDataBlock(records, header);

    Writer writer = HoodieLogFormat.newWriterBuilder().onParentPath(testPath)
            .withFileExtension(HoodieArchivedLogFile.ARCHIVE_EXTENSION).withFileId("commits" + ".archive")
            .overBaseCommit("").withFs(fs).build();

    writer = writer.appendBlock(dataBlock);
    // get the current log file version to compare later
    int logFileVersion = writer.getLogFile().getLogVersion();
    Path logFilePath = writer.getLogFile().getPath();
    writer.close();

    // Wait for 3 times replication of file
    DFSTestUtil.waitReplication(fs, logFilePath, (short) 3);
    // Shut down all DNs that have the last block location for the file
    LocatedBlocks lbs = cluster.getFileSystem().getClient().getNamenode()
            .getBlockLocations("/tmp/" + uuid + "/" + logFilePath.getName(), 0, Long.MAX_VALUE);
    List<DataNode> dnsOfCluster = cluster.getDataNodes();
    DatanodeInfo[] dnsWithLocations = lbs.getLastLocatedBlock().getLocations();
    for (DataNode dn : dnsOfCluster) {
        for (DatanodeInfo loc : dnsWithLocations) {
            if (dn.getDatanodeId().equals(loc)) {
                dn.shutdown();
                cluster.stopDataNode(dn.getDisplayName());
                DFSTestUtil.waitForDatanodeDeath(dn);
            }
        }
    }
    // Wait for the replication of this file to go down to 0
    DFSTestUtil.waitReplication(fs, logFilePath, (short) 0);

    // Opening a new Writer right now will throw IOException. The code should handle this, rollover the logfile and
    // return a new writer with a bumped up logVersion
    writer = HoodieLogFormat.newWriterBuilder().onParentPath(testPath)
            .withFileExtension(HoodieArchivedLogFile.ARCHIVE_EXTENSION).withFileId("commits" + ".archive")
            .overBaseCommit("").withFs(fs).build();
    // The log version should be different for this new writer
    Assert.assertFalse(writer.getLogFile().getLogVersion() == logFileVersion);
}

From source file:com.uber.hoodie.common.util.DFSPropertiesConfiguration.java

License:Apache License

private void visitFile(Path file) {
    try {/*from   ww w. j a  v a 2  s .  co  m*/
        if (visitedFiles.contains(file.getName())) {
            throw new IllegalStateException("Loop detected; file " + file + " already referenced");
        }
        visitedFiles.add(file.getName());
        BufferedReader reader = new BufferedReader(new InputStreamReader(fs.open(file)));
        addProperties(reader);
    } catch (IOException ioe) {
        log.error("Error reading in properies from dfs", ioe);
        throw new IllegalArgumentException("Cannot read properties from dfs", ioe);
    }
}

From source file:com.uber.hoodie.common.util.FSUtils.java

License:Apache License

/**
 * Gets all partition paths assuming date partitioning (year, month, day) three levels down.
 *//*  w  ww .  ja v  a 2s  . com*/
public static List<String> getAllPartitionFoldersThreeLevelsDown(FileSystem fs, String basePath)
        throws IOException {
    List<String> datePartitions = new ArrayList<>();
    // Avoid listing and including any folders under the metafolder
    PathFilter filter = getExcludeMetaPathFilter();
    FileStatus[] folders = fs.globStatus(new Path(basePath + "/*/*/*"), filter);
    for (FileStatus status : folders) {
        Path path = status.getPath();
        datePartitions.add(String.format("%s/%s/%s", path.getParent().getParent().getName(),
                path.getParent().getName(), path.getName()));
    }
    return datePartitions;
}

From source file:com.uber.hoodie.common.util.FSUtils.java

License:Apache License

/**
 * Given a base partition and a partition path, return
 * relative path of partition path to the base path
 */// w  ww.  ja v  a  2s  . c o m
public static String getRelativePartitionPath(Path basePath, Path partitionPath) {
    basePath = Path.getPathWithoutSchemeAndAuthority(basePath);
    partitionPath = Path.getPathWithoutSchemeAndAuthority(partitionPath);
    String partitionFullPath = partitionPath.toString();
    int partitionStartIndex = partitionFullPath.indexOf(basePath.getName(),
            basePath.getParent() == null ? 0 : basePath.getParent().toString().length());
    // Partition-Path could be empty for non-partitioned tables
    return partitionStartIndex + basePath.getName().length() == partitionFullPath.length() ? ""
            : partitionFullPath.substring(partitionStartIndex + basePath.getName().length() + 1);
}

From source file:com.uber.hoodie.common.util.FSUtils.java

License:Apache License

/**
 * Obtain all the partition paths, that are present in this table, denoted by presence of {@link
 * com.uber.hoodie.common.model.HoodiePartitionMetadata#HOODIE_PARTITION_METAFILE}
 *///w w  w  . j  ava2  s .c o m
public static List<String> getAllFoldersWithPartitionMetaFile(FileSystem fs, String basePathStr)
        throws IOException {
    final Path basePath = new Path(basePathStr);
    final List<String> partitions = new ArrayList<>();
    processFiles(fs, basePathStr, (locatedFileStatus) -> {
        Path filePath = locatedFileStatus.getPath();
        if (filePath.getName().equals(HoodiePartitionMetadata.HOODIE_PARTITION_METAFILE)) {
            partitions.add(getRelativePartitionPath(basePath, filePath.getParent()));
        }
        return true;
    }, true);
    return partitions;
}

From source file:com.uber.hoodie.common.util.FSUtils.java

License:Apache License

/**
 * Get the file extension from the log file
 *//*w  w  w.j ava 2s. c  om*/
public static String getFileExtensionFromLog(Path logPath) {
    Matcher matcher = LOG_FILE_PATTERN.matcher(logPath.getName());
    if (!matcher.find()) {
        throw new InvalidHoodiePathException(logPath, "LogFile");
    }
    return matcher.group(3);
}

From source file:com.uber.hoodie.common.util.FSUtils.java

License:Apache License

/**
 * Get the first part of the file name in the log file. That will be the fileId. Log file do not
 * have commitTime in the file name./*from   www.jav a 2  s. c o  m*/
 */
public static String getFileIdFromLogPath(Path path) {
    Matcher matcher = LOG_FILE_PATTERN.matcher(path.getName());
    if (!matcher.find()) {
        throw new InvalidHoodiePathException(path, "LogFile");
    }
    return matcher.group(1);
}