Example usage for org.apache.hadoop.fs Path getParent

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path getParent.

Prototype

public Path getParent()

Source Link

Document

Returns the parent of a path or null if at root.

Usage

From source file:com.inmobi.databus.local.CopyMapper.java

License:Apache License

@Override
public void map(Text key, Text value, Context context) throws IOException, InterruptedException {
    Path src = new Path(key.toString());
    String dest = value.toString();
    String collector = src.getParent().getName();
    String category = src.getParent().getParent().getName();

    FileSystem fs = FileSystem.get(context.getConfiguration());
    Path target = getTempPath(context, src, category, collector);
    FileUtil.gzip(src, target, context.getConfiguration());
    // move to final destination
    fs.mkdirs(new Path(dest).makeQualified(fs));
    Path destPath = new Path(dest + File.separator + collector + "-" + src.getName() + ".gz");
    LOG.info("Renaming file " + target + " to " + destPath);
    fs.rename(target, destPath);/*from  w  ww  .  ja v a 2 s.c om*/

}

From source file:com.inmobi.databus.local.LocalStreamService.java

License:Apache License

private String getCategoryFromDestPath(Path dest) {
    return dest.getParent().getParent().getParent().getParent().getParent().getParent().getName();
}

From source file:com.inmobi.databus.partition.TestAbstractClusterReader.java

License:Apache License

public void prepareCheckpoint(StreamFile streamFile, int lineNum, Path databusFile,
        PartitionCheckpointList partitionCheckpointList) {
    Date date = DatabusStreamWaitingReader.getDateFromStreamDir(streamDir, databusFile.getParent());
    Calendar current = Calendar.getInstance();
    current.setTime(date);/*from  w  w w  .  j  a v a 2 s.  c o  m*/
    partitionCheckpointList.set(current.get(Calendar.MINUTE), new PartitionCheckpoint(streamFile, lineNum));
}

From source file:com.inmobi.databus.partition.TestClusterReaderMultipleCollectors.java

License:Apache License

public void prepareCheckpoint(StreamFile streamFile, int lineNum, Path databusFile,
        PartitionCheckpointList partitionCheckpointList) {
    Date date = DatabusStreamWaitingReader.getDateFromStreamDir(streamDir, databusFile.getParent());
    Calendar cal = Calendar.getInstance();
    cal.setTime(date);/*from w w  w . java 2 s .  c  o  m*/
    partitionCheckpointList.set(cal.get(Calendar.MINUTE), new PartitionCheckpoint(streamFile, lineNum));
}

From source file:com.inmobi.databus.readers.TestDatabusEmptyFolders.java

License:Apache License

private Path removeFilesIfAny() throws IOException {
    FileSystem fs = FileSystem.get(cluster.getHadoopConf());
    Path streamDir = DatabusUtil.getStreamDir(StreamType.LOCAL, new Path(cluster.getRootDir()), testStream);
    Path minuteDirPath = DatabusStreamReader.getMinuteDirPath(streamDir,
            modifyTime(new Date(), Calendar.MINUTE, -10));
    FileStatus[] fileStatuses = fs.listStatus(minuteDirPath.getParent());
    for (FileStatus folders : fileStatuses) {
        if (!folders.isDir()) {
            continue;
        }/*w ww  .  j  a va2  s  .com*/
        LOG.debug("Folder=" + folders.getPath().toString());
        FileStatus[] files = fs.listStatus(folders.getPath());
        for (FileStatus file : files) {
            if (file.isDir()) {
                continue;
            }
            fs.delete(file.getPath());
        }
    }
    Arrays.sort(fileStatuses, new java.util.Comparator<FileStatus>() {

        @Override
        public int compare(FileStatus o1, FileStatus o2) {
            try {
                return getDateFromFile(o1.getPath().toString()).before(getDateFromFile(o2.getPath().toString()))
                        ? -1
                        : 1;
            } catch (ParseException e) {
                e.printStackTrace();
            }
            return 0;
        }
    });
    return fileStatuses[fileStatuses.length - 1].getPath();
}

From source file:com.inmobi.databus.utils.DatePathComparator.java

License:Apache License

@Override
public int compare(FileStatus fileStatus, FileStatus fileStatus1) {

    /*//from  w w  w  .  j a v  a 2s  .  c o m
    * Path eg:
    * /databus/system/distcp_mirror_srcCluster_destCluster/databus/streams
    * /<stream-Name>/2012/1/13/15/7/<hostname>-<streamName>-2012-01-16-07
    * -21_00000.gz
    *
    * in some cases paths can empty without files
    * eg:   /databus/system/distcp_mirror_srcCluster_destCluster/databus
    * /streams/<streamName>/2012/1/13/15/7/
    */

    Path streamDir = null;
    Path streamDir1 = null;
    Date streamDate1 = null;
    Date streamDate = null;

    if (fileStatus != null) {
        if (!fileStatus.isDir())
            streamDir = fileStatus.getPath().getParent();
        else
            streamDir = fileStatus.getPath();
        Path streamDirPrefix = streamDir.getParent().getParent().getParent().getParent().getParent();

        streamDate = CalendarHelper.getDateFromStreamDir(streamDirPrefix, streamDir);
        LOG.debug("streamDate [" + streamDate + "]");
    }

    if (fileStatus1 != null) {
        if (!fileStatus1.isDir())
            streamDir1 = fileStatus1.getPath().getParent();
        else
            streamDir1 = fileStatus1.getPath();

        Path streamDirPrefix1 = streamDir1.getParent().getParent().getParent().getParent().getParent();

        streamDate1 = CalendarHelper.getDateFromStreamDir(streamDirPrefix1, streamDir1);
        LOG.debug("streamDate1 [" + streamDate1.toString() + "]");
    }

    if (streamDate != null && streamDate1 != null)
        return streamDate.compareTo(streamDate1);
    else
        return -1;

}

From source file:com.inmobi.messaging.consumer.hadoop.TestAbstractHadoopConsumer.java

License:Apache License

public void cleanup() throws IOException {
    FileSystem lfs = FileSystem.getLocal(conf);
    for (Path rootDir : rootDirs) {
        LOG.debug("Cleaning up the dir: " + rootDir.getParent());
        lfs.delete(rootDir.getParent(), true);
    }//www. jav  a2s.  com
    lfs.delete(new Path(chkpointPathPrefix).getParent(), true);
}

From source file:com.inmobi.messaging.consumer.hadoop.TestConsumerPartitionRetention.java

License:Apache License

@AfterTest
public void cleanup() throws Exception {
    FileSystem lfs = FileSystem.getLocal(conf);
    for (Path rootDir : rootDirs) {
        LOG.debug("Cleaning Up the dir: " + rootDir.getParent());
        lfs.delete(rootDir.getParent(), true);
    }/*from   w w  w  .ja va2  s.c om*/
    lfs.delete(new Path(chkpointPath).getParent(), true);
}

From source file:com.inmobi.messaging.consumer.hadoop.TestConsumerPartitionStartTime.java

License:Apache License

@AfterTest
public void cleanup() throws Exception {
    FileSystem lfs = FileSystem.getLocal(conf);
    for (Path rootDir : rootDirs) {
        LOG.debug("Cleaning up the dir: " + rootDir.getParent());
        lfs.delete(rootDir.getParent(), true);
    }/*from ww  w  .  j  a  v a2 s  .c  o  m*/
    lfs.delete(new Path(chkpointPath).getParent(), true);
}

From source file:com.inmobi.messaging.consumer.hadoop.TestHadoopConsumerWithPartitionList.java

License:Apache License

@AfterTest
public void cleanup() throws IOException {
    FileSystem lfs = FileSystem.getLocal(conf);
    for (Path rootDir : rootDirs) {
        LOG.debug("Cleaning up the dir: " + rootDir);
        lfs.delete(rootDir.getParent(), true);
    }/*ww  w  .j ava2s .c  o  m*/
    // delete checkpoint dir
    lfs.delete(new Path(ck1).getParent(), true);
}