Example usage for org.apache.hadoop.fs Path getFileSystem

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path getFileSystem.

Prototype

public FileSystem getFileSystem(Configuration conf) throws IOException 

Source Link

Document

Return the FileSystem that owns this Path.

Usage

From source file:com.inmobi.conduit.distcp.tools.SimpleCopyListing.java

License:Apache License

private Path computeSourceRootPath(FileStatus sourceStatus, DistCpOptions options) throws IOException {

    Path target = options.getTargetPath();
    FileSystem targetFS = target.getFileSystem(getConf());

    boolean solitaryFile = options.getSourcePaths().size() == 1 && !sourceStatus.isDir();

    if (solitaryFile) {
        if (targetFS.isFile(target) || !targetFS.exists(target)) {
            return sourceStatus.getPath();
        } else {//from  w w w.j  a v  a 2s . c o m
            return sourceStatus.getPath().getParent();
        }
    } else {
        boolean specialHandling = (options.getSourcePaths().size() == 1 && !targetFS.exists(target))
                || options.shouldSyncFolder() || options.shouldOverwrite();

        return specialHandling && sourceStatus.isDir() ? sourceStatus.getPath()
                : sourceStatus.getPath().getParent();
    }
}

---

From source file:com.inmobi.conduit.distcp.tools.SimpleCopyListing.java

License:Apache License

private Path makeQualified(Path path) throws IOException {
    return path.makeQualified(path.getFileSystem(getConf()));
}

---

From source file:com.inmobi.conduit.distcp.tools.SimpleCopyListing.java

License:Apache License

private SequenceFile.Writer getWriter(Path pathToListFile) throws IOException {
    return SequenceFile.createWriter(pathToListFile.getFileSystem(getConf()), getConf(), pathToListFile,
            Text.class, FileStatus.class, SequenceFile.CompressionType.NONE);
}

---

From source file:com.inmobi.conduit.distcp.tools.SimpleCopyListing.java

License:Apache License

private void traverseNonEmptyDirectory(SequenceFile.Writer fileListWriter, FileStatus sourceStatus,
        Path sourcePathRoot, boolean localFile, DistCpOptions options) throws IOException {
    FileSystem sourceFS = sourcePathRoot.getFileSystem(getConf());
    Stack<FileStatus> pathStack = new Stack<FileStatus>();
    pathStack.push(sourceStatus);/*from   ww  w.  j a  v a 2 s.  c o m*/

    while (!pathStack.isEmpty()) {
        for (FileStatus child : getChildren(sourceFS, pathStack.pop())) {
            if (LOG.isDebugEnabled())
                LOG.debug("Recording source-path: " + sourceStatus.getPath() + " for copy.");
            writeToFileListing(fileListWriter, child, sourcePathRoot, localFile, options);
            if (isDirectoryAndNotEmpty(sourceFS, child)) {
                if (LOG.isDebugEnabled())
                    LOG.debug("Traversing non-empty source dir: " + sourceStatus.getPath());
                pathStack.push(child);
            }
        }
    }
}

---

From source file:com.inmobi.conduit.distcp.tools.TestDistCp.java

License:Apache License

@Test
public void testCleanup() {
    try {//from  w  w w .  ja v a 2 s.c  o m
        clearState();
        Path sourcePath = new Path("noscheme:///file");
        List<Path> sources = new ArrayList<Path>();
        sources.add(sourcePath);

        Path targetPath = new Path(TARGET_PATH).makeQualified(cluster.getFileSystem());
        DistCpOptions options = new DistCpOptions(sources, targetPath);
        options.setOutPutDirectory(counterOutputPath);

        Path stagingDir = null/* = JobSubmissionFiles.getStagingDir(
                              new JobClient(new JobConf(configuration)), configuration)*/;
        boolean hadoop1 = false;
        try {
            LOG.info("Trying to get staging path using hadoop-2");
            Class clusterClass = DistCp.class.getClassLoader().loadClass("org.apache.hadoop.mapreduce.Cluster");
            Method method = JobSubmissionFiles.class.getMethod("getStagingDir", clusterClass,
                    Configuration.class);
            Constructor constructor = clusterClass.getConstructor(Configuration.class);
            stagingDir = (Path) method.invoke(null, constructor.newInstance(configuration), configuration);
        } catch (Exception ignored) {
            // fallback to hadoop-1 API
            hadoop1 = true;
        }
        if (hadoop1) {
            try {
                LOG.info("Trying to get staging path using hadoop-1");
                Method method = JobSubmissionFiles.class.getMethod("getStagingDir", JobClient.class,
                        Configuration.class);
                stagingDir = (Path) method.invoke(null, new JobClient(new JobConf(configuration)),
                        configuration);
            } catch (Exception ignored) {
                // do nothing
            }
        }

        stagingDir.getFileSystem(configuration).mkdirs(stagingDir);

        try {
            new DistCp(configuration, options).execute();
        } catch (Throwable t) {
            Assert.assertEquals(stagingDir.getFileSystem(configuration).listStatus(stagingDir).length, 0);
        }
    } catch (Exception e) {
        LOG.error("Exception encountered ", e);
        Assert.fail("testCleanup failed " + e.getMessage());
    }
}

---

From source file:com.inmobi.conduit.distcp.tools.util.DistCpUtils.java

License:Apache License

/**
 * Retrieves size of the file at the specified path.
 * @param path: The path of the file whose size is sought.
 * @param configuration: Configuration, to retrieve the appropriate FileSystem.
 * @return The file-size, in number of bytes.
 * @throws IOException, on failure./*from ww  w . j a  v  a 2  s  . co  m*/
 */
public static long getFileSize(Path path, Configuration configuration) throws IOException {
    if (LOG.isDebugEnabled())
        LOG.debug("Retrieving file size for: " + path);
    return path.getFileSystem(configuration).getFileStatus(path).getLen();
}

---

From source file:com.inmobi.conduit.local.LocalStreamServiceTest.java

License:Apache License

@Test
public void testFileUtil() throws Exception {
    String streamName = "test1";
    Path rootDir = new Path("/tmp/localServiceTest/testcluster2/mergeservice");
    Path dataDir = new Path(rootDir, "data/test1/testcluster2");
    FileSystem fs = dataDir.getFileSystem(new Configuration());
    fs.mkdirs(dataDir);//w w  w .java2 s . c  om
    String filenameStr = new String(
            streamName + "-" + TestLocalStreamService.getDateAsYYYYMMDDHHmm(new Date()) + "_00001");
    Path src = new Path(dataDir, filenameStr);

    LOG.debug("Creating Test Data with filename [" + filenameStr + "]");
    FSDataOutputStream streamout = fs.create(src);
    String content = "Creating Test data for teststream";
    Message msg = new Message(content.getBytes());
    long currentTimestamp = new Date().getTime();
    AuditUtil.attachHeaders(msg, currentTimestamp);
    byte[] encodeMsg = Base64.encodeBase64(msg.getData().array());
    streamout.write(encodeMsg);
    streamout.write("\n".getBytes());
    streamout.write(encodeMsg);
    streamout.write("\n".getBytes());
    long nextMinuteTimeStamp = currentTimestamp + 60000;
    // Genearate a msg with different timestamp.  Default window period is 60sec
    AuditUtil.attachHeaders(msg, nextMinuteTimeStamp);
    encodeMsg = Base64.encodeBase64(msg.getData().array());
    streamout.write(encodeMsg);
    streamout.close();
    Map<Long, Long> received = new HashMap<Long, Long>();
    Path target = new Path(new Path(rootDir, "system/tmp/LocalStreamService_testcluster2_test1@/"
            + "job_local_0001/attempt_local_0001_m_000000_0/"), filenameStr + ".gz");
    FileUtil.gzip(src, target, new Configuration(), received);
    Assert.assertEquals(2, received.size());
    // current timestamp window = currentTimestamp - (currentTimestamp % 60000)
    Assert.assertTrue(2 == received.get(currentTimestamp - (currentTimestamp % 60000)));
    // next timestamp window = nextMinuteTimeStamp - (nextMinuteTimeStamp %60000)
    Assert.assertTrue(1 == received.get(nextMinuteTimeStamp - (nextMinuteTimeStamp % 60000)));
    fs.delete(rootDir, true);
}

---

From source file:com.inmobi.grid.fs.test.TestS3NwithSlash.java

License:Apache License

public static void main(String[] args) throws IOException {
    Configuration conf = new Configuration();
    //conf.set("fs.default.name", "s3n://AKIAJWJBBSUODWD7RMEA:LUnvKWWSeFuInkoOpYX%2FbJtDj080EovlioOchGwM@inmobi-grid-emr-dev");
    Path path = new Path(
            "s3n://AKIAJWJBBSUODWD7RMEA:LUnvKWWSeFuInkoOpYX/bJtDj080EovlioOchGwM@inmobi-grid-emr-dev/");
    FileSystem fs = path.getFileSystem(conf);
    FileStatus[] list = fs.listStatus(path);
    //System.out.println(list[0].getp);

}

---

From source file:com.inmobi.grill.driver.hive.HiveDriver.java

License:Apache License

void addPersistentPath(QueryContext context) throws IOException {
    String hiveQuery;/*ww  w.ja  v  a2 s.c  om*/
    if (context.isPersistent()
            && context.getConf().getBoolean(GrillConfConstants.GRILL_ADD_INSERT_OVEWRITE, true)) {
        // store persistent data into user specified location
        // If absent, take default home directory
        String resultSetParentDir = context.getResultSetParentDir();
        StringBuilder builder;
        Path resultSetPath;
        if (StringUtils.isNotBlank(resultSetParentDir)) {
            resultSetPath = new Path(resultSetParentDir, context.getQueryHandle().toString());
            // create query
            builder = new StringBuilder("INSERT OVERWRITE DIRECTORY ");
        } else {
            // Write to /tmp/grillreports
            resultSetPath = new Path(GrillConfConstants.GRILL_RESULT_SET_PARENT_DIR_DEFAULT,
                    context.getQueryHandle().toString());
            builder = new StringBuilder("INSERT OVERWRITE LOCAL DIRECTORY ");
        }
        context.setResultSetPath(
                resultSetPath.makeQualified(resultSetPath.getFileSystem(context.getConf())).toString());
        builder.append('"').append(resultSetPath).append("\" ");
        String outputDirFormat = context.getConf().get(GrillConfConstants.GRILL_OUTPUT_DIRECTORY_FORMAT);
        if (outputDirFormat != null) {
            builder.append(outputDirFormat);
        }
        builder.append(' ').append(context.getDriverQuery()).append(' ');
        hiveQuery = builder.toString();
    } else {
        hiveQuery = context.getDriverQuery();
    }
    LOG.info("Hive driver query:" + hiveQuery);
    context.setDriverQuery(hiveQuery);
}

---

From source file:com.inmobi.grill.driver.hive.TestHiveDriver.java

License:Apache License

private void validatePersistentResult(GrillResultSet resultSet, String dataFile, String outptuDir,
        boolean formatNulls) throws Exception {
    assertTrue(resultSet instanceof HivePersistentResultSet);
    HivePersistentResultSet persistentResultSet = (HivePersistentResultSet) resultSet;
    String path = persistentResultSet.getOutputPath();
    QueryHandle handle = persistentResultSet.getQueryHandle();

    Path actualPath = new Path(path);
    FileSystem fs = actualPath.getFileSystem(conf);
    assertEquals(actualPath, fs.makeQualified(new Path(outptuDir, handle.toString())));
    List<String> actualRows = new ArrayList<String>();
    for (FileStatus stat : fs.listStatus(actualPath)) {
        FSDataInputStream in = fs.open(stat.getPath());
        BufferedReader br = null;
        try {//w  ww  . j a  v a  2s  .c o  m
            br = new BufferedReader(new InputStreamReader(in));
            String line = "";

            while ((line = br.readLine()) != null) {
                System.out.println("Actual:" + line);
                actualRows.add(line.trim());
            }
        } finally {
            if (br != null) {
                br.close();
            }
        }
    }

    BufferedReader br = null;
    List<String> expectedRows = new ArrayList<String>();

    try {
        br = new BufferedReader(new FileReader(new File(dataFile)));
        String line = "";
        while ((line = br.readLine()) != null) {
            String row = line.trim();
            if (formatNulls) {
                row += ",-NA-,";
                row += line.trim();
            }
            expectedRows.add(row);
        }
    } finally {
        if (br != null) {
            br.close();
        }
    }
    assertEquals(actualRows, expectedRows);
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »