Example usage for org.apache.hadoop.fs Path getParent

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path getParent.

Prototype

public Path getParent() 

Source Link

Document

Returns the parent of a path or null if at root.

Usage

From source file:com.inmobi.conduit.distcp.tools.mapred.RetriableFileCopyCommand.java

License:Apache License

private void promoteTmpToTarget(Path tmpTarget, Path target, FileSystem fs) throws IOException {
    if ((fs.exists(target) && !fs.delete(target, false))
            || (!fs.exists(target.getParent()) && !fs.mkdirs(target.getParent()))
            || !fs.rename(tmpTarget, target)) {
        throw new IOException("Failed to promote tmp-file:" + tmpTarget + " to: " + target);
    }//from w  w w. j ava2 s  .co  m
}

---

From source file:com.inmobi.conduit.distcp.tools.mapred.RetriableFileCopyCommand.java

License:Apache License

private Path getTmpFile(Path target, Mapper.Context context) {
    Path targetWorkPath = new Path(
            HadoopCompat.getTaskConfiguration(context).get(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH));

    Path root = target.equals(targetWorkPath) ? targetWorkPath.getParent() : targetWorkPath;
    LOG.info("Creating temp file: " + new Path(root, ".distcp.tmp." + context.getTaskAttemptID().toString()));
    return new Path(root, ".distcp.tmp." + context.getTaskAttemptID().toString());
}

---

From source file:com.inmobi.conduit.distcp.tools.mapred.TestCopyMapper.java

License:Apache License

/**
 * If a single file is being copied to a location where the file (of the same
 * name) already exists, then the file shouldn't be skipped.
 *///from   w w  w.  j  a v  a  2s.  c o m
@Test
public void testSingleFileCopy() {
    try {
        deleteState();
        touchFile(SOURCE_PATH + "/1.gz");
        Path sourceFilePath = pathList.get(0);
        Path targetFilePath = new Path(sourceFilePath.toString().replaceAll(SOURCE_PATH, TARGET_PATH));
        touchFile(targetFilePath.toString());

        FileSystem fs = cluster.getFileSystem();
        CopyMapper copyMapper = new CopyMapper();
        StatusReporter reporter = new StubStatusReporter();
        InMemoryWriter writer = new InMemoryWriter();
        Mapper<Text, FileStatus, NullWritable, Text>.Context context = getMapperContext(copyMapper, reporter,
                writer);

        context.getConfiguration().set(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH,
                targetFilePath.getParent().toString()); // Parent directory.
        copyMapper.setup(context);

        final FileStatus sourceFileStatus = fs.getFileStatus(sourceFilePath);

        long before = fs.getFileStatus(targetFilePath).getModificationTime();
        copyMapper.map(new Text(DistCpUtils.getRelativePath(new Path(SOURCE_PATH), sourceFilePath)),
                sourceFileStatus, context);
        long after = fs.getFileStatus(targetFilePath).getModificationTime();

        Assert.assertTrue("File should have been skipped", before == after);

        context.getConfiguration().set(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH, targetFilePath.toString()); // Specify the file path.
        copyMapper.setup(context);

        before = fs.getFileStatus(targetFilePath).getModificationTime();
        try {
            Thread.sleep(2);
        } catch (Throwable ignore) {
        }
        copyMapper.map(new Text(DistCpUtils.getRelativePath(new Path(SOURCE_PATH), sourceFilePath)),
                sourceFileStatus, context);
        after = fs.getFileStatus(targetFilePath).getModificationTime();

        Assert.assertTrue("File should have been overwritten.", before < after);

    } catch (Exception exception) {
        Assert.fail("Unexpected exception: " + exception.getMessage());
        exception.printStackTrace();
    }
}

---

From source file:com.inmobi.conduit.distcp.tools.TestCopyListing.java

License:Apache License

@Test
public void testBuildListing() {
    FileSystem fs = null;/*from  ww w .  j  av  a 2 s .c om*/
    try {
        fs = FileSystem.get(getConf());
        List<Path> srcPaths = new ArrayList<Path>();
        Path p1 = new Path("/tmp/in/1");
        Path p2 = new Path("/tmp/in/2");
        Path p3 = new Path("/tmp/in2/2");
        Path target = new Path("/tmp/out/1");
        srcPaths.add(p1.getParent());
        srcPaths.add(p3.getParent());
        TestDistCpUtils.createFile(fs, "/tmp/in/1");
        TestDistCpUtils.createFile(fs, "/tmp/in/2");
        TestDistCpUtils.createFile(fs, "/tmp/in2/2");
        fs.mkdirs(target);
        OutputStream out = fs.create(p1);
        out.write("ABC".getBytes());
        out.close();

        out = fs.create(p2);
        out.write("DEF".getBytes());
        out.close();

        out = fs.create(p3);
        out.write("GHIJ".getBytes());
        out.close();

        Path listingFile = new Path("/tmp/file");

        DistCpOptions options = new DistCpOptions(srcPaths, target);
        options.setSyncFolder(true);
        CopyListing listing = new SimpleCopyListing(getConf(), CREDENTIALS);
        try {
            listing.buildListing(listingFile, options);
            Assert.fail("Duplicates not detected");
        } catch (DuplicateFileException ignore) {
        }
        Assert.assertEquals(listing.getBytesToCopy(), 10);
        Assert.assertEquals(listing.getNumberOfPaths(), 3);
        TestDistCpUtils.delete(fs, "/tmp");

        try {
            listing.buildListing(listingFile, options);
            Assert.fail("Invalid input not detected");
        } catch (InvalidInputException ignore) {
        }
        TestDistCpUtils.delete(fs, "/tmp");
    } catch (IOException e) {
        LOG.error("Exception encountered ", e);
        Assert.fail("Test build listing failed");
    } finally {
        TestDistCpUtils.delete(fs, "/tmp");
    }
}

---

From source file:com.inmobi.conduit.local.CopyMapper.java

License:Apache License

@Override
public void map(Text key, FileStatus value, Context context) throws IOException, InterruptedException {
    Path src = value.getPath();
    String dest = key.toString();
    String collector = src.getParent().getName();
    String category = src.getParent().getParent().getName();
    Map<Long, Long> received = null;
    if (context.getConfiguration().getBoolean(ConduitConstants.AUDIT_ENABLED_KEY, true)) {
        received = new HashMap<Long, Long>();
    }//w  ww  .  j  a v a 2 s .  c  o m
    Configuration srcConf = new Configuration();
    srcConf.set(FS_DEFAULT_NAME_KEY, context.getConfiguration().get(SRC_FS_DEFAULT_NAME_KEY));

    FileSystem fs = FileSystem.get(srcConf);
    Path target = getTempPath(context, src, category, collector);
    if (FileUtil.gzip(src, target, srcConf, received)) {
        LOG.info("File " + src + " is empty hence returning without compressing");
        return;
    }
    // move to final destination
    fs.mkdirs(new Path(dest).makeQualified(fs));
    String destnFilename = collector + "-" + src.getName() + ".gz";
    Path destPath = new Path(dest + File.separator + destnFilename);
    LOG.info("Renaming file " + target + " to " + destPath);
    fs.rename(target, destPath);
    if (received != null) {

        for (Entry<Long, Long> entry : received.entrySet()) {
            String counterNameValue = getCounterNameValue(category, destnFilename, entry.getKey(),
                    entry.getValue());
            context.write(NullWritable.get(), new Text(counterNameValue));
        }
    }

}

---

From source file:com.inmobi.conduit.local.LocalStreamService.java

License:Apache License

private void populateCheckpointPathForCollector(Table<String, String, String> checkpointPaths,
        TreeMap<String, FileStatus> collectorPaths) {
    // Last file in sorted ascending order to be check-pointed for this
    // collector/*from w  w w  . j  av  a 2 s  . c  om*/
    if (collectorPaths != null && collectorPaths.size() > 0) {
        Entry<String, FileStatus> entry = collectorPaths.lastEntry();
        Path filePath = entry.getValue().getPath();
        String streamName = getCategoryFromSrcPath(filePath);
        String collectorName = filePath.getParent().getName();
        String checkpointPath = filePath.getName();
        checkpointPaths.put(streamName, collectorName, checkpointPath);
    }
}

---

From source file:com.inmobi.conduit.local.LocalStreamService.java

License:Apache License

private String getCategoryFromSrcPath(Path src) {
    return src.getParent().getParent().getName();
}

---

From source file:com.inmobi.conduit.local.LocalStreamService.java

License:Apache License

public String getTopicNameFromDestnPath(Path destnPath) {
    String destnPathAsString = destnPath.toString();
    String destnDirAsString = new Path(srcCluster.getLocalFinalDestDirRoot()).toString();
    String pathWithoutRoot = destnPathAsString.substring(destnDirAsString.length());
    Path tmpPath = new Path(pathWithoutRoot);
    while (tmpPath.depth() != 1)
        tmpPath = tmpPath.getParent();
    return tmpPath.getName();
}

---

From source file:com.inmobi.conduit.utils.DatePathComparator.java

License:Apache License

@Override
public int compare(FileStatus fileStatus, FileStatus fileStatus1) {

    /*/* ww  w.  j  a  va2 s  .c  o m*/
    * Path eg:
    * <rootdir>/system/distcp_mirror_srcCluster_destCluster/conduit/streams
    * /<stream-Name>/2012/1/13/15/7/<hostname>-<streamName>-2012-01-16-07
    * -21_00000.gz
    *
    * in some cases paths can empty without files
    * eg:   /conduit/system/distcp_mirror_srcCluster_destCluster/conduit
    * /streams/<streamName>/2012/1/13/15/7/
    */

    Path streamDir = null;
    Path streamDir1 = null;
    Date streamDate1 = null;
    Date streamDate = null;

    if (fileStatus != null) {
        if (!fileStatus.isDir())
            streamDir = fileStatus.getPath().getParent();
        else
            streamDir = fileStatus.getPath();
        Path streamDirPrefix = streamDir.getParent().getParent().getParent().getParent().getParent();

        streamDate = CalendarHelper.getDateFromStreamDir(streamDirPrefix, streamDir);
    }

    if (fileStatus1 != null) {
        if (!fileStatus1.isDir())
            streamDir1 = fileStatus1.getPath().getParent();
        else
            streamDir1 = fileStatus1.getPath();

        Path streamDirPrefix1 = streamDir1.getParent().getParent().getParent().getParent().getParent();

        streamDate1 = CalendarHelper.getDateFromStreamDir(streamDirPrefix1, streamDir1);
    }

    if (streamDate != null && streamDate1 != null)
        return streamDate.compareTo(streamDate1);
    else
        return -1;

}

---

From source file:com.inmobi.databus.distcp.MirrorStreamService.java

License:Apache License

private void createCommitPaths(LinkedHashMap<FileStatus, Path> commitPaths, List<FileStatus> streamPaths) {
    /*  Path eg in streamPaths -
     *  /databus/system/distcp_mirror_<srcCluster>_<destCluster>/databus/streams
     *  /<streamName>/2012/1/13/15/7/<hostname>-<streamName>-2012-01-16-07
     *  -21_00000.gz//  ww  w  .j  a  va 2 s  .com
     *
     * or it could be an emptyDir like
     *  /* Path eg in streamPaths -
     *  /databus/system/distcp_mirror_<srcCluster>_<destCluster>/databus/streams
     *  /<streamName>/2012/1/13/15/7/
     *
     */

    for (FileStatus fileStatus : streamPaths) {
        String fileName = null;

        Path prefixDir = null;
        if (fileStatus.isDir()) {
            //empty directory
            prefixDir = fileStatus.getPath();
        } else {
            fileName = fileStatus.getPath().getName();
            prefixDir = fileStatus.getPath().getParent();
        }

        Path min = prefixDir;
        Path hr = min.getParent();
        Path day = hr.getParent();
        Path month = day.getParent();
        Path year = month.getParent();
        Path streamName = year.getParent();

        String finalPath = getDestCluster().getFinalDestDirRoot() + File.separator + streamName.getName()
                + File.separator + year.getName() + File.separator + month.getName() + File.separator
                + day.getName() + File.separator + hr.getName() + File.separator + min.getName();

        if (fileName != null) {
            finalPath += File.separator + fileName;
        }

        commitPaths.put(fileStatus, new Path(finalPath));
        LOG.debug("Going to commit [" + fileStatus.getPath() + "] to [" + finalPath + "]");
    }

}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »