Example usage for org.apache.hadoop.fs Path getName

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path getName.

Prototype

public String getName() 

Source Link

Document

Returns the final component of this path.

Usage

From source file:com.inmobi.conduit.distcp.tools.mapred.CopyMapper.java

License:Apache License

/**
 * Find entry from distributed cache//from w w  w  . jav  a 2 s  .c om
 *
 * @param cacheFiles - All localized cache files
 * @param fileName - fileName to search
 * @return Path of the filename if found, else null
 */
private Path findCacheFile(Path[] cacheFiles, String fileName) {
    if (cacheFiles != null && cacheFiles.length > 0) {
        for (Path file : cacheFiles) {
            if (file.getName().equals(fileName)) {
                return file;
            }
        }
    }
    return null;
}

---

From source file:com.inmobi.conduit.distcp.tools.mapred.CopyMapper.java

License:Apache License

/**
 * Implementation of the Mapper<>::map(). Does the copy.
 * @param relPath: The target path./* www .  j  a va  2s  . co m*/
 * @param sourceFileStatus: The source path.
 * @throws IOException
 */
@Override
public void map(Text relPath, FileStatus sourceFileStatus, Context context)
        throws IOException, InterruptedException {
    Path sourcePath = sourceFileStatus.getPath();
    Map<Long, Long> received = null;
    if (context.getConfiguration().getBoolean(ConduitConstants.AUDIT_ENABLED_KEY, true)) {
        received = new HashMap<Long, Long>();
    }
    if (LOG.isDebugEnabled())
        LOG.debug("DistCpMapper::map(): Received " + sourcePath + ", " + relPath);

    Path target = new Path(targetWorkPath.makeQualified(targetFS) + relPath.toString());

    EnumSet<DistCpOptions.FileAttribute> fileAttributes = getFileAttributeSettings(context);

    final String description = "Copying " + sourcePath + " to " + target;
    context.setStatus(description);

    LOG.info(description);

    try {
        FileStatus sourceCurrStatus;
        FileSystem sourceFS;
        try {
            sourceFS = sourcePath.getFileSystem(conf);
            sourceCurrStatus = sourceFS.getFileStatus(sourcePath);
        } catch (FileNotFoundException e) {
            throw new IOException(new RetriableFileCopyCommand.CopyReadException(e));
        }

        FileStatus targetStatus = null;

        try {
            targetStatus = targetFS.getFileStatus(target);
        } catch (FileNotFoundException ignore) {
        }

        if (targetStatus != null && (targetStatus.isDir() != sourceCurrStatus.isDir())) {
            throw new IOException("Can't replace " + target + ". Target is " + getFileType(targetStatus)
                    + ", Source is " + getFileType(sourceCurrStatus));
        }

        if (sourceCurrStatus.isDir()) {
            createTargetDirsWithRetry(description, target, context);
            return;
        }

        if (skipFile(sourceFS, sourceCurrStatus, target)) {
            LOG.info("Skipping copy of " + sourceCurrStatus.getPath() + " to " + target);
            updateSkipCounters(context, sourceCurrStatus);
        } else {
            String streamName = null;
            if (!relPath.toString().isEmpty()) {
                Path relativePath = new Path(relPath.toString());
                if (relativePath.depth() > 2) {
                    // path is for mirror service and is of format
                    // /conduit/streams/<streamName>/2013/09/12
                    Path tmpPath = relativePath;
                    while (tmpPath.getParent() != null && !tmpPath.getParent().getName().equals("streams")) {
                        tmpPath = tmpPath.getParent();
                    }
                    streamName = tmpPath.getName();
                } else {
                    // path is for merge service and of form /<stream name>/filename.gz
                    streamName = relativePath.getParent().getName();
                }
            }
            copyFileWithRetry(description, sourceCurrStatus, target, context, fileAttributes, received);
            // generate audit counters
            if (received != null) {
                for (Entry<Long, Long> entry : received.entrySet()) {
                    String counterNameValue = getCounterNameValue(streamName, sourcePath.getName(),
                            entry.getKey(), entry.getValue());
                    context.write(NullWritable.get(), new Text(counterNameValue));
                }
            }
        }

        DistCpUtils.preserve(target.getFileSystem(conf), target, sourceCurrStatus, fileAttributes);

    } catch (IOException exception) {
        handleFailures(exception, sourceFileStatus, target, context);
    }
}

---

From source file:com.inmobi.conduit.distcp.tools.mapred.lib.DynamicInputChunkSet.java

License:Apache License

/**
 * Constructor, to initialize the context in which DynamicInputChunks are
 * used.//from   w  ww  .j av a 2 s .c o  m
 * @param configuration The Configuration instance, as received from the
 * DynamicInputFormat or DynamicRecordReader.
 * @throws IOException Exception in case of failure.
 */
public DynamicInputChunkSet(Configuration configuration) throws IOException {
    this.configuration = configuration;
    Path listingFilePath = new Path(getListingFilePath(configuration));
    chunkRootPath = new Path(listingFilePath.getParent(), "chunkDir");
    fs = chunkRootPath.getFileSystem(configuration);
    chunkFilePrefix = listingFilePath.getName() + ".chunk.";
}

---

From source file:com.inmobi.conduit.local.CopyMapper.java

License:Apache License

@Override
public void map(Text key, FileStatus value, Context context) throws IOException, InterruptedException {
    Path src = value.getPath();
    String dest = key.toString();
    String collector = src.getParent().getName();
    String category = src.getParent().getParent().getName();
    Map<Long, Long> received = null;
    if (context.getConfiguration().getBoolean(ConduitConstants.AUDIT_ENABLED_KEY, true)) {
        received = new HashMap<Long, Long>();
    }/*w w  w.ja  v a  2  s .  com*/
    Configuration srcConf = new Configuration();
    srcConf.set(FS_DEFAULT_NAME_KEY, context.getConfiguration().get(SRC_FS_DEFAULT_NAME_KEY));

    FileSystem fs = FileSystem.get(srcConf);
    Path target = getTempPath(context, src, category, collector);
    if (FileUtil.gzip(src, target, srcConf, received)) {
        LOG.info("File " + src + " is empty hence returning without compressing");
        return;
    }
    // move to final destination
    fs.mkdirs(new Path(dest).makeQualified(fs));
    String destnFilename = collector + "-" + src.getName() + ".gz";
    Path destPath = new Path(dest + File.separator + destnFilename);
    LOG.info("Renaming file " + target + " to " + destPath);
    fs.rename(target, destPath);
    if (received != null) {

        for (Entry<Long, Long> entry : received.entrySet()) {
            String counterNameValue = getCounterNameValue(category, destnFilename, entry.getKey(),
                    entry.getValue());
            context.write(NullWritable.get(), new Text(counterNameValue));
        }
    }

}

---

From source file:com.inmobi.conduit.local.CopyMapper.java

License:Apache License

private Path getTempPath(Context context, Path src, String category, String collector) {
    Path tempPath = new Path(getTaskAttemptTmpDir(context),
            category + "-" + collector + "-" + src.getName() + ".gz");
    return tempPath;
}

---

From source file:com.inmobi.conduit.local.LocalStreamService.java

License:Apache License

private void populateCheckpointPathForCollector(Table<String, String, String> checkpointPaths,
        TreeMap<String, FileStatus> collectorPaths) {
    // Last file in sorted ascending order to be check-pointed for this
    // collector/*  w  w  w  . j  a v a 2  s. c  om*/
    if (collectorPaths != null && collectorPaths.size() > 0) {
        Entry<String, FileStatus> entry = collectorPaths.lastEntry();
        Path filePath = entry.getValue().getPath();
        String streamName = getCategoryFromSrcPath(filePath);
        String collectorName = filePath.getParent().getName();
        String checkpointPath = filePath.getName();
        checkpointPaths.put(streamName, collectorName, checkpointPath);
    }
}

---

From source file:com.inmobi.conduit.local.LocalStreamService.java

License:Apache License

public String getTopicNameFromDestnPath(Path destnPath) {
    String destnPathAsString = destnPath.toString();
    String destnDirAsString = new Path(srcCluster.getLocalFinalDestDirRoot()).toString();
    String pathWithoutRoot = destnPathAsString.substring(destnDirAsString.length());
    Path tmpPath = new Path(pathWithoutRoot);
    while (tmpPath.depth() != 1)
        tmpPath = tmpPath.getParent();/*  w  w  w . ja  v a  2s  .  c o  m*/
    return tmpPath.getName();
}

---

From source file:com.inmobi.conduit.purge.DataPurgerServiceTest.java

License:Apache License

private void createTestPurgePartitionFiles(FileSystem fs, Cluster cluster, Calendar date, Table table)
        throws Exception {
    for (String streamname : cluster.getSourceStreams()) {
        String[] files = new String[NUM_OF_FILES];
        String datapath = Cluster.getDateAsYYYYMMDDHHMNPath(date.getTime());
        String commitpath = cluster.getLocalFinalDestDirRoot() + File.separator + streamname + File.separator
                + datapath;//www. j  a  v a  2s.  co  m
        fs.mkdirs(new Path(commitpath));
        Map<String, String> partSpec = TestHCatUtil.getPartitionMap(date);
        LOG.info("Adding partition " + partSpec + " for stream " + streamname);
        TestHCatUtil.addPartition(table, partSpec);
        for (int j = 0; j < NUM_OF_FILES; ++j) {
            files[j] = new String(cluster.getName() + "-"
                    + TestLocalStreamService.getDateAsYYYYMMDDHHmm(new Date()) + "_" + idFormat.format(j));
            {
                Path path = new Path(commitpath + File.separator + files[j]);
                LOG.info("Creating streams_local File " + path.getName());
                FSDataOutputStream streamout = fs.create(path);
                streamout.writeBytes("Creating Test data for teststream " + path.toString());
                streamout.close();
                Assert.assertTrue(fs.exists(path));
            }
        }
    }
}

---

From source file:com.inmobi.conduit.purge.DataPurgerServiceTest.java

License:Apache License

private void createTestPurgefiles(FileSystem fs, Cluster cluster, Calendar date, boolean createEmptyDirs)
        throws Exception {
    for (String streamname : cluster.getSourceStreams()) {
        String[] files = new String[NUM_OF_FILES];
        String datapath = Cluster.getDateAsYYYYMMDDHHMNPath(date.getTime());
        String commitpath = cluster.getLocalFinalDestDirRoot() + File.separator + streamname + File.separator
                + datapath;/*from ww w .jav a  2 s.  c  o  m*/
        String mergecommitpath = cluster.getFinalDestDirRoot() + File.separator + streamname + File.separator
                + datapath;

        String trashpath = cluster.getTrashPath() + File.separator + CalendarHelper.getDateAsString(date)
                + File.separator;
        fs.mkdirs(new Path(commitpath));

        for (int j = 0; j < NUM_OF_FILES; ++j) {
            files[j] = new String(cluster.getName() + "-"
                    + TestLocalStreamService.getDateAsYYYYMMDDHHmm(new Date()) + "_" + idFormat.format(j));
            {
                Path path = new Path(commitpath + File.separator + files[j]);
                // LOG.info("Creating streams_local File " + path.getName());
                FSDataOutputStream streamout = fs.create(path);
                streamout.writeBytes("Creating Test data for teststream " + path.toString());
                streamout.close();
                Assert.assertTrue(fs.exists(path));
            }
            {
                Path path = new Path(mergecommitpath + File.separator + files[j]);
                // LOG.info("Creating streams File " + path.getName());
                FSDataOutputStream streamout = fs.create(path);
                streamout.writeBytes("Creating Test data for teststream " + path.toString());
                streamout.close();
                Assert.assertTrue(fs.exists(path));
            }

            {
                if (!createEmptyDirs) {
                    Path path = new Path(trashpath + File.separator
                            + String.valueOf(date.get(Calendar.HOUR_OF_DAY)) + File.separator + files[j]);
                    // LOG.info("Creating trash File " + path.toString());
                    FSDataOutputStream streamout = fs.create(path);
                    streamout.writeBytes("Creating Test trash data for teststream " + path.getName());
                    streamout.close();
                    Assert.assertTrue(fs.exists(path));
                }
            }

        }
        if (createEmptyDirs) {
            Path path = new Path(trashpath);
            if (!fs.exists(path))
                fs.mkdirs(path);
            Assert.assertTrue(fs.exists(path));
        }
    }

}

---

From source file:com.inmobi.databus.distcp.MergedStreamService.java

License:Apache License

private Map<String, Set<Path>> doLocalCommit(long commitTime, Map<String, List<Path>> categoriesToCommit)
        throws Exception {
    Map<String, Set<Path>> comittedPaths = new HashMap<String, Set<Path>>();
    Set<Map.Entry<String, List<Path>>> commitEntries = categoriesToCommit.entrySet();
    Iterator it = commitEntries.iterator();
    while (it.hasNext()) {
        Map.Entry<String, List<Path>> entry = (Map.Entry<String, List<Path>>) it.next();
        String category = entry.getKey();
        List<Path> filesInCategory = entry.getValue();
        for (Path filePath : filesInCategory) {
            Path destParentPath = new Path(getDestCluster().getFinalDestDir(category, commitTime));
            if (!getDestFs().exists(destParentPath)) {
                getDestFs().mkdirs(destParentPath);
            }/*  w w w. j a  v  a  2 s  .c o m*/
            LOG.debug("Moving from intermediatePath [" + filePath + "] to [" + destParentPath + "]");
            if (getDestFs().rename(filePath, destParentPath) == false) {
                LOG.warn("Rename failed, aborting transaction COMMIT to avoid "
                        + "dataloss. Partial data replay could happen in next run");
                throw new Exception("Abort transaction Commit. Rename failed from [" + filePath + "] to ["
                        + destParentPath + "]");
            }
            Path commitPath = new Path(destParentPath, filePath.getName());
            Set<Path> commitPaths = comittedPaths.get(category);
            if (commitPaths == null) {
                commitPaths = new HashSet<Path>();
            }
            commitPaths.add(commitPath);
            comittedPaths.put(category, commitPaths);
        }
    }
    return comittedPaths;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »