Example usage for org.apache.hadoop.fs FileStatus getPath

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileStatus getPath.

Prototype

public Path getPath() 

Source Link

Usage

From source file:com.inmobi.databus.readers.LocalStreamCollectorReader.java

License:Apache License

@Override
protected void doRecursiveListing(Path dir, PathFilter pathFilter, FileMap<DatabusStreamFile> fmap)
        throws IOException {
    FileStatus[] fileStatuses = fsListFileStatus(dir, pathFilter);
    if (fileStatuses == null || fileStatuses.length == 0) {
        LOG.debug("No files in directory:" + dir);
    } else {//from ww  w.  j a  va2 s.  co m
        for (FileStatus file : fileStatuses) {
            if (file.isDir()) {
                doRecursiveListing(file.getPath(), pathFilter, fmap);
            } else {
                try {
                    Date currentTimeStamp = LocalStreamCollectorReader.getDateFromStreamFile(streamName,
                            file.getPath().getName());
                    if (stopTime != null && stopTime.before(currentTimeStamp)) {
                        LOG.info("stopTime [ " + stopTime + " ] " + "is beyond the"
                                + " current file timestamp [ " + currentTimeStamp + " ]");
                        stopListing();
                    } else {
                        fmap.addPath(file);
                    }
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
        }
    }
}

---

From source file:com.inmobi.databus.readers.LocalStreamCollectorReader.java

License:Apache License

protected DatabusStreamFile getStreamFile(FileStatus status) {
    return DatabusStreamFile.create(streamName, status.getPath().getName());
}

---

From source file:com.inmobi.databus.readers.LocalStreamCollectorReader.java

License:Apache License

public FileMap<DatabusStreamFile> createFileMap() throws IOException {
    return new FileMap<DatabusStreamFile>() {
        @Override/* ww  w . j  a va  2  s  . c  o m*/
        protected void buildList() throws IOException {
            buildListing(this, pathFilter);
        }

        @Override
        protected TreeMap<DatabusStreamFile, FileStatus> createFilesMap() {
            return new TreeMap<DatabusStreamFile, FileStatus>();
        }

        @Override
        protected DatabusStreamFile getStreamFile(String fileName) {
            return DatabusStreamFile.create(streamName, fileName);
        }

        @Override
        protected DatabusStreamFile getStreamFile(FileStatus file) {
            return DatabusStreamFile.create(streamName, file.getPath().getName());
        }

        @Override
        protected PathFilter createPathFilter() {
            return new PathFilter() {
                @Override
                public boolean accept(Path p) {
                    if (p.getName().startsWith(collector)) {
                        return true;
                    }
                    return false;
                }
            };
        }
    };
}

---

From source file:com.inmobi.databus.readers.LocalStreamCollectorReader.java

License:Apache License

protected Date getTimeStampFromCollectorStreamFile(FileStatus file) {
    try {//  ww w . j  a v  a2s  . c o m
        return LocalStreamCollectorReader.getDateFromStreamFile(streamName, file.getPath().getName());
    } catch (Exception exception) {
        LOG.info("Not able to get timestamp from " + file.getPath() + " file " + exception);
    }
    return null;
}

---

From source file:com.inmobi.databus.readers.StreamReader.java

License:Apache License

protected Path getLastFile() {
    FileStatus lastFile = fileMap.getLastFile();
    if (lastFile != null) {
        return lastFile.getPath();
    }/*from ww w  .  j  a v a 2  s.  c  om*/
    return null;
}

---

From source file:com.inmobi.databus.readers.TestDatabusEmptyFolders.java

License:Apache License

private Path removeFilesIfAny() throws IOException {
    FileSystem fs = FileSystem.get(cluster.getHadoopConf());
    Path streamDir = DatabusUtil.getStreamDir(StreamType.LOCAL, new Path(cluster.getRootDir()), testStream);
    Path minuteDirPath = DatabusStreamReader.getMinuteDirPath(streamDir,
            modifyTime(new Date(), Calendar.MINUTE, -10));
    FileStatus[] fileStatuses = fs.listStatus(minuteDirPath.getParent());
    for (FileStatus folders : fileStatuses) {
        if (!folders.isDir()) {
            continue;
        }/*from w  ww. j  a v a  2s.  c o  m*/
        LOG.debug("Folder=" + folders.getPath().toString());
        FileStatus[] files = fs.listStatus(folders.getPath());
        for (FileStatus file : files) {
            if (file.isDir()) {
                continue;
            }
            fs.delete(file.getPath());
        }
    }
    Arrays.sort(fileStatuses, new java.util.Comparator<FileStatus>() {

        @Override
        public int compare(FileStatus o1, FileStatus o2) {
            try {
                return getDateFromFile(o1.getPath().toString()).before(getDateFromFile(o2.getPath().toString()))
                        ? -1
                        : 1;
            } catch (ParseException e) {
                e.printStackTrace();
            }
            return 0;
        }
    });
    return fileStatuses[fileStatuses.length - 1].getPath();
}

---

From source file:com.inmobi.databus.utils.DatePathComparator.java

License:Apache License

@Override
public int compare(FileStatus fileStatus, FileStatus fileStatus1) {

    /*//from ww w  . ja v a  2s . c om
    * Path eg:
    * /databus/system/distcp_mirror_srcCluster_destCluster/databus/streams
    * /<stream-Name>/2012/1/13/15/7/<hostname>-<streamName>-2012-01-16-07
    * -21_00000.gz
    *
    * in some cases paths can empty without files
    * eg:   /databus/system/distcp_mirror_srcCluster_destCluster/databus
    * /streams/<streamName>/2012/1/13/15/7/
    */

    Path streamDir = null;
    Path streamDir1 = null;
    Date streamDate1 = null;
    Date streamDate = null;

    if (fileStatus != null) {
        if (!fileStatus.isDir())
            streamDir = fileStatus.getPath().getParent();
        else
            streamDir = fileStatus.getPath();
        Path streamDirPrefix = streamDir.getParent().getParent().getParent().getParent().getParent();

        streamDate = CalendarHelper.getDateFromStreamDir(streamDirPrefix, streamDir);
        LOG.debug("streamDate [" + streamDate + "]");
    }

    if (fileStatus1 != null) {
        if (!fileStatus1.isDir())
            streamDir1 = fileStatus1.getPath().getParent();
        else
            streamDir1 = fileStatus1.getPath();

        Path streamDirPrefix1 = streamDir1.getParent().getParent().getParent().getParent().getParent();

        streamDate1 = CalendarHelper.getDateFromStreamDir(streamDirPrefix1, streamDir1);
        LOG.debug("streamDate1 [" + streamDate1.toString() + "]");
    }

    if (streamDate != null && streamDate1 != null)
        return streamDate.compareTo(streamDate1);
    else
        return -1;

}

---

From source file:com.inmobi.grill.driver.hive.TestHiveDriver.java

License:Apache License

private void validatePersistentResult(GrillResultSet resultSet, String dataFile, String outptuDir,
        boolean formatNulls) throws Exception {
    assertTrue(resultSet instanceof HivePersistentResultSet);
    HivePersistentResultSet persistentResultSet = (HivePersistentResultSet) resultSet;
    String path = persistentResultSet.getOutputPath();
    QueryHandle handle = persistentResultSet.getQueryHandle();

    Path actualPath = new Path(path);
    FileSystem fs = actualPath.getFileSystem(conf);
    assertEquals(actualPath, fs.makeQualified(new Path(outptuDir, handle.toString())));
    List<String> actualRows = new ArrayList<String>();
    for (FileStatus stat : fs.listStatus(actualPath)) {
        FSDataInputStream in = fs.open(stat.getPath());
        BufferedReader br = null;
        try {/*w ww .  j av a  2  s  .  c  o m*/
            br = new BufferedReader(new InputStreamReader(in));
            String line = "";

            while ((line = br.readLine()) != null) {
                System.out.println("Actual:" + line);
                actualRows.add(line.trim());
            }
        } finally {
            if (br != null) {
                br.close();
            }
        }
    }

    BufferedReader br = null;
    List<String> expectedRows = new ArrayList<String>();

    try {
        br = new BufferedReader(new FileReader(new File(dataFile)));
        String line = "";
        while ((line = br.readLine()) != null) {
            String row = line.trim();
            if (formatNulls) {
                row += ",-NA-,";
                row += line.trim();
            }
            expectedRows.add(row);
        }
    } finally {
        if (br != null) {
            br.close();
        }
    }
    assertEquals(actualRows, expectedRows);
}

---

From source file:com.inmobi.grill.server.query.TestQueryService.java

License:Apache License

private void validatePersistentResult(PersistentQueryResult resultset, QueryHandle handle) throws IOException {
    Assert.assertTrue(resultset.getPersistedURI().endsWith(handle.toString()));
    Path actualPath = new Path(resultset.getPersistedURI());
    FileSystem fs = actualPath.getFileSystem(new Configuration());
    List<String> actualRows = new ArrayList<String>();
    for (FileStatus fstat : fs.listStatus(actualPath)) {
        FSDataInputStream in = fs.open(fstat.getPath());
        BufferedReader br = null;
        try {/*from  w  ww  .j av a 2  s .com*/
            br = new BufferedReader(new InputStreamReader(in));
            String line = "";

            while ((line = br.readLine()) != null) {
                actualRows.add(line);
            }
        } finally {
            if (br != null) {
                br.close();
            }
            if (in != null) {
                in.close();
            }
        }
    }
    Assert.assertEquals(actualRows.get(0), "1
one");
    Assert.assertEquals(actualRows.get(1), "\\N
two");
    Assert.assertEquals(actualRows.get(2), "3
\\N");
    Assert.assertEquals(actualRows.get(3), "\\N
\\N");
    Assert.assertEquals(actualRows.get(4), "5
");
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »