Example usage for org.apache.hadoop.fs FileSystem mkdirs

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem mkdirs.

Prototype

public boolean mkdirs(Path f) throws IOException

Source Link

Document

Call #mkdirs(Path,FsPermission) with default permission.

Usage

From source file:com.cloudera.crunch.impl.mr.exec.CrunchJob.java

License:Open Source License

private void handleMultiPaths() throws IOException {
    if (!multiPaths.isEmpty()) {
        // Need to handle moving the data from the output directory of the
        // job to the output locations specified in the paths.
        FileSystem fs = FileSystem.get(job.getConfiguration());
        for (int i = 0; i < multiPaths.size(); i++) {
            Path src = new Path(workingPath, PlanningParameters.MULTI_OUTPUT_PREFIX + i + "-*");
            Path[] srcs = FileUtil.stat2Paths(fs.globStatus(src), src);
            Path dst = multiPaths.get(i);
            if (!fs.exists(dst)) {
                fs.mkdirs(dst);
            }//w  w  w. ja  va2 s .  c  om
            int minPartIndex = getMinPartIndex(dst, fs);
            for (Path s : srcs) {
                fs.rename(s, getDestFile(s, dst, minPartIndex++));
            }
        }
    }
}

From source file:com.cloudera.crunch.impl.mr.MRPipeline.java

License:Open Source License

private static Path createTempDirectory(Configuration conf) throws IOException {
    FileSystem fs = FileSystem.get(conf);
    Path dir = new Path("/tmp/crunch" + RANDOM.nextInt());
    fs.mkdirs(dir);
    return dir;//from w  w  w  . j a  va 2s  . co m
}

From source file:com.cloudera.flume.handlers.hive.MarkerStore.java

License:Apache License

private boolean writeElasticSearchToMarkerFolder(StringBuilder httpQuery) {
    FileSystem hdfs;
    try {/*from w ww . j  ava2 s. com*/
        String markerFolder = conf.getElasticSearchDefaultMarkerFolder();
        dstPath = new Path(markerFolder);
        hdfs = dstPath.getFileSystem(conf);
        if (!hdfs.exists(dstPath)) {
            hdfs.mkdirs(dstPath);
        }

        dstPath = new Path(markerFolder + "/es-" + System.currentTimeMillis() + ".marker");
        System.out.println("creating file at: " + dstPath.toString());
        FSDataOutputStream writer_marker = hdfs.create(dstPath);
        writer_marker.writeBytes(httpQuery + "\n");
        writer_marker.close();
        dstPath = null;
        writer_marker = null;
    } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
        return false;
    }
    return true;
}

From source file:com.cloudera.flume.handlers.hive.MarkerStore.java

License:Apache License

public boolean writeHiveMarker(String hqlQuery, String filePath, String hiveMarkerFolder,
        String hiveMarkerPath) {//from ww w.  j av a 2  s  . c  o m
    LOG.debug("writing to hiveMarker: " + hiveMarkerFolder);

    FileSystem hdfs;
    dstPath = new Path(hiveMarkerFolder);
    try {
        hdfs = dstPath.getFileSystem(conf);

        if (!hdfs.exists(dstPath)) {
            hdfs.mkdirs(dstPath);
        }
        dstPath = new Path(hiveMarkerPath);
        FSDataOutputStream writer = hdfs.create(dstPath);
        writer.writeBytes(filePath + "\t" + hqlQuery + "\n");
        writer.close();
        dstPath = null;
        writer = null;

    } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    }

    /*
      dstPath = new Path(hiveMarkerPath);
      hdfs = dstPath.getFileSystem(conf);
            
      writer = hdfs.create(dstPath);      
      writer.writeUTF(hqlQuery);
      writer.close();
      writer = null;
     */
    return true;

}

From source file:com.cloudera.hadoop.hdfs.nfs.nfs4.handlers.CREATEHandler.java

License:Apache License

@Override
protected CREATEResponse doHandle(NFS4Handler server, Session session, CREATERequest request)
        throws NFS4Exception, IOException {
    if (session.getCurrentFileHandle() == null) {
        throw new NFS4Exception(NFS4ERR_NOFILEHANDLE);
    }/*w  w w . j  a v a  2s  .c  om*/
    if (request.getType() != NFS4_DIR) {
        throw new UnsupportedOperationException(
                "Create files of  type " + request.getType() + " is not supported.");
    }
    if ("".equals(request.getName())) {
        throw new NFS4Exception(NFS4ERR_INVAL);
    }
    Path parent = server.getPath(session.getCurrentFileHandle());
    Path path = new Path(parent, request.getName());
    FileSystem fs = session.getFileSystem();
    if (!fs.exists(parent)) {
        throw new NFS4Exception(NFS4ERR_STALE, "Parent " + parent + " does not exist");
    }
    if (fs.exists(path)) {
        throw new NFS4Exception(NFS4ERR_EXIST, "Path " + path + " already exists.");
    }
    long parentModTimeBefore = fs.getFileStatus(parent).getModificationTime();
    if (!fs.mkdirs(path)) {
        throw new NFS4Exception(NFS4ERR_IO);
    }
    long parentModTimeAfter = fs.getFileStatus(parent).getModificationTime();
    FileStatus fileStatus = fs.getFileStatus(path);
    ImmutableMap<Integer, Attribute> requestAttrs = request.getAttrValues();
    // TODO Handlers should have annotations so that setAttrs can throw an
    // error if they require the stateID to be set. 
    Bitmap responseAttrs = Attribute.setAttrs(server, session, request.getAttrs(), requestAttrs, fs, fileStatus,
            null);
    session.setCurrentFileHandle(server.createFileHandle(path));
    CREATEResponse response = createResponse();
    response.setChangeInfo(ChangeInfo.newChangeInfo(true, parentModTimeBefore, parentModTimeAfter));
    response.setStatus(NFS4_OK);
    response.setAttrs(responseAttrs);
    return response;
}

From source file:com.cloudera.hoop.client.fs.TestHoopFileSystem.java

License:Open Source License

private void testRename() throws Exception {
    FileSystem fs = FileSystem.get(getHadoopConf());
    Path path = new Path(getHadoopTestDir(), "foo");
    fs.mkdirs(path);
    fs.close();/*from  ww w  .  j a  v  a  2s.  c  o m*/
    Configuration conf = new Configuration();
    conf.set("fs.http.impl", HoopFileSystem.class.getName());
    fs = FileSystem.get(getJettyURL().toURI(), conf);
    Path oldPath = new Path(path.toUri().getPath());
    Path newPath = new Path(path.getParent(), "bar");
    fs.rename(oldPath, newPath);
    fs.close();
    fs = FileSystem.get(getHadoopConf());
    Assert.assertFalse(fs.exists(oldPath));
    Assert.assertTrue(fs.exists(newPath));
    fs.close();
}

From source file:com.cloudera.hoop.client.fs.TestHoopFileSystem.java

License:Open Source License

private void testDelete() throws Exception {
    Path foo = new Path(getHadoopTestDir(), "foo");
    Path bar = new Path(getHadoopTestDir(), "bar");
    Path foe = new Path(getHadoopTestDir(), "foe");
    FileSystem fs = FileSystem.get(getHadoopConf());
    fs.mkdirs(foo);
    fs.mkdirs(new Path(bar, "a"));
    fs.mkdirs(foe);//from  ww w. ja va  2  s  .  c  o m

    Configuration conf = new Configuration();
    conf.set("fs.http.impl", HoopFileSystem.class.getName());
    FileSystem hoopFs = FileSystem.get(getJettyURL().toURI(), conf);
    Assert.assertTrue(hoopFs.delete(new Path(foo.toUri().getPath()), false));
    Assert.assertFalse(fs.exists(foo));
    try {
        hoopFs.delete(new Path(bar.toUri().getPath()), false);
        Assert.fail();
    } catch (IOException ex) {
    } catch (Exception ex) {
        Assert.fail();
    }
    Assert.assertTrue(fs.exists(bar));
    Assert.assertTrue(hoopFs.delete(new Path(bar.toUri().getPath()), true));
    Assert.assertFalse(fs.exists(bar));

    Assert.assertTrue(fs.exists(foe));
    Assert.assertTrue(hoopFs.delete(foe));
    Assert.assertFalse(fs.exists(foe));

    hoopFs.close();
    fs.close();
}

From source file:com.cloudera.hoop.client.fs.TestHoopFileSystem.java

License:Open Source License

private void testMkdirs() throws Exception {
    Path path = new Path(getHadoopTestDir(), "foo");
    Configuration conf = new Configuration();
    conf.set("fs.http.impl", HoopFileSystem.class.getName());
    FileSystem fs = FileSystem.get(getJettyURL().toURI(), conf);
    fs.mkdirs(path);
    fs.close();/* w w  w  .ja  v a2s  .  c  o m*/
    fs = FileSystem.get(getHadoopConf());
    Assert.assertTrue(fs.exists(path));
    fs.close();
}

From source file:com.cloudera.impala.common.FileSystemUtil.java

License:Apache License

/**
 * Makes a temporary unique directory within the given directory.
 *//*from  w w  w  .j  a  v a 2 s . c  o m*/
public static Path makeTmpSubdirectory(Path directory) throws IOException {
    FileSystem fs = directory.getFileSystem(CONF);
    Path tmpDir = new Path(directory, ".tmp_" + UUID.randomUUID().toString());
    fs.mkdirs(tmpDir);
    return tmpDir;
}

From source file:com.cloudera.lib.service.hadoop.TestHadoopService.java

License:Open Source License

@Test
@TestDir/*  w w  w .  j  a  v  a  2s  . c o  m*/
@TestHadoop
public void createFileSystem() throws Exception {
    String dir = getTestDir().getAbsolutePath();
    String services = StringUtils.toString(
            Arrays.asList(InstrumentationService.class.getName(), HadoopService.class.getName()), ",");
    XConfiguration conf = new XConfiguration();
    conf.set("server.services", services);
    Server server = new Server("server", dir, dir, dir, dir, conf);
    server.init();
    Hadoop hadoop = server.get(Hadoop.class);
    FileSystem fs = hadoop.createFileSystem("u", getHadoopConf());
    Assert.assertNotNull(fs);
    fs.mkdirs(new Path("/tmp/foo"));
    hadoop.releaseFileSystem(fs);
    try {
        fs.mkdirs(new Path("/tmp/foo"));
        Assert.fail();
    } catch (IOException ex) {
    } catch (Exception ex) {
        Assert.fail();
    }
    server.destroy();
}