Example usage for org.apache.hadoop.fs FileSystem mkdirs

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem mkdirs.

Prototype

public boolean mkdirs(Path f) throws IOException

Source Link

Document

Call #mkdirs(Path,FsPermission) with default permission.

Usage

From source file:com.collective.celos.ci.deploy.HdfsDeployer.java

License:Apache License

public void deploy() throws Exception {

    FileSystem fs = context.getFileSystem();
    final String hdfsDirLocalPath = String.format(LOCAL_HDFS_PATTERN, context.getDeployDir());

    final File hdfsDirLocal = new File(hdfsDirLocalPath);
    if (!hdfsDirLocal.exists()) {
        throw new IllegalStateException(hdfsDirLocalPath + " not found local FS");
    }/*  w  w  w  .  j  a  v  a  2 s  .  c o m*/

    undeploy();

    Path dst = getDestinationHdfsPath();
    fs.mkdirs(dst);
    String[] childFiles = hdfsDirLocal.list();
    for (String child : childFiles) {
        fs.copyFromLocalFile(new Path(hdfsDirLocalPath, child), dst);
    }
}

From source file:com.collective.celos.ci.testing.fixtures.deploy.HdfsInputDeployer.java

License:Apache License

@Override
public void deploy(TestRun testRun) throws Exception {
    FileSystem fileSystem = testRun.getCiContext().getFileSystem();

    CollectFilesAndPathsProcessor pathToFile = new CollectFilesAndPathsProcessor();
    TreeObjectProcessor.process(fixObjectCreator.create(testRun), pathToFile);

    Path pathPrefixed = new Path(Util.augmentHdfsPath(testRun.getHdfsPrefix(), path.toString()));
    for (java.nio.file.Path childPath : pathToFile.pathToFiles.keySet()) {
        Path pathTo = new Path(pathPrefixed, childPath.toString());
        fileSystem.mkdirs(pathTo.getParent());

        FSDataOutputStream outputStream = fileSystem.create(pathTo);
        try {//from   w w  w.  j a v  a  2 s . c  o  m
            IOUtils.copy(pathToFile.pathToFiles.get(childPath).getContent(), outputStream);
        } finally {
            outputStream.flush();
            outputStream.close();
        }

    }
}

From source file:com.collective.celos.ci.testing.fixtures.deploy.hive.HiveTableDeployer.java

License:Apache License

private Path createTempHdfsFileForInsertion(FixTable fixTable, TestRun testRun) throws Exception {

    Path pathToParent = new Path(testRun.getHdfsPrefix(), ".hive");
    Path pathTo = new Path(pathToParent, UUID.randomUUID().toString());
    FileSystem fileSystem = testRun.getCiContext().getFileSystem();
    fileSystem.mkdirs(pathTo.getParent());
    FSDataOutputStream outputStream = fileSystem.create(pathTo);

    CSVWriter writer = new CSVWriter(new OutputStreamWriter(outputStream), '\t', CSVWriter.NO_QUOTE_CHARACTER);

    for (FixTable.FixRow fixRow : fixTable.getRows()) {
        List<String> rowData = Lists.newArrayList();
        for (String colName : fixTable.getColumnNames()) {
            rowData.add(fixRow.getCells().get(colName));
        }/*from www. j  a  v a2s  .  c  o m*/
        String[] dataArray = rowData.toArray(new String[rowData.size()]);
        writer.writeNext(dataArray);
    }

    writer.close();

    fileSystem.setPermission(pathToParent, new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL));
    fileSystem.setPermission(pathTo, new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL));
    return pathTo;
}

From source file:com.conversantmedia.mapreduce.example.PrepareInputsExample.java

License:Apache License

@DriverInit
public void copyFilesToWorking() throws IOException {
    // Copy the input files into the 'workingDir'
    FileSystem fs = FileSystem.get(getConf());

    this.workingDirectory = new Path("/tmp/" + UUID.randomUUID().toString());
    fs.mkdirs(workingDirectory);

    FileStatus[] files = fs.globStatus(new Path(context.getInput()));
    for (FileStatus file : files) {
        Path dest = new Path(workingDirectory, file.getPath().getName());
        FileUtil.copy(fs, file.getPath(), fs, dest, false, getConf());
    }// www  . jav a 2s.  c o  m
}

From source file:com.conversantmedia.mapreduce.tool.BaseTool.java

License:Apache License

/**
 * Moves our inputs into the 'archive' path for
 * long term storage, or perhaps further processing.
 * @param context      the job's driver context bean
 * @throws IOException   if the inputs cannot be moved to
 *          the archive path.//from w  w w .j a  va 2  s.  c o  m
 */
protected void archiveInputs(T context) throws IOException {
    FileSystem fs = FileSystem.get(getConf());
    fs.mkdirs(context.getArchive());

    for (Path input : context.getInput()) {
        List<FileStatus> status = getInputFiles(input);
        for (FileStatus file : status) {
            Path dest = new Path(context.getArchive(), file.getPath().getName());
            fs.rename(file.getPath(), dest);
            logger().debug("Moved [" + input + "] to [" + dest + "]");
        }
    }
}

From source file:com.cotdp.hadoop.ZipFileTest.java

License:Apache License

/**
 * Prepare the FileSystem and copy test files
 *//*from www.  j a v  a 2s .co m*/
@Override
protected void setUp() throws Exception {
    // One-off initialisation
    if (isInitialised == false) {
        LOG.info("setUp() called, preparing FileSystem for tests");

        // 
        FileSystem fs = FileSystem.get(conf);

        // Delete our working directory if it already exists
        LOG.info("   ... Deleting " + workingPath.toString());
        fs.delete(workingPath, true);

        // Copy the test files
        LOG.info("   ... Copying files");
        fs.mkdirs(inputPath);
        copyFile(fs, "zip-01.zip");
        copyFile(fs, "zip-02.zip");
        copyFile(fs, "zip-03.zip");
        copyFile(fs, "zip-04.dat");
        copyFile(fs, "random.dat");
        copyFile(fs, "encrypted.zip");
        copyFile(fs, "corrupt.zip");
        fs.close();

        //
        isInitialised = true;
    }

    // Reset ZipFileInputFormat leniency (false)
    ZipFileInputFormat.setLenient(false);
}

From source file:com.dasasian.chok.util.FileUtil.java

License:Apache License

public static void unzipInDfs(FileSystem fileSystem, final Path source, final Path target) {
    try {/*  w  w  w .  j a v a2 s.  c  o m*/
        FSDataInputStream dfsInputStream = fileSystem.open(source);
        fileSystem.mkdirs(target);
        final ZipInputStream zipInputStream = new ZipInputStream(dfsInputStream);
        ZipEntry entry;

        while ((entry = zipInputStream.getNextEntry()) != null) {
            final String entryPath = entry.getName();
            final int indexOf = entryPath.indexOf("/");
            final String cleanUpPath = entryPath.substring(indexOf + 1, entryPath.length());
            Path path = target;
            if (!cleanUpPath.equals("")) {
                path = new Path(target, cleanUpPath);
            }
            LOG.info("Extracting: " + entry + " to " + path);
            if (entry.isDirectory()) {
                fileSystem.mkdirs(path);
            } else {
                int count;
                final byte data[] = new byte[4096];
                FSDataOutputStream fsDataOutputStream = fileSystem.create(path);
                while ((count = zipInputStream.read(data, 0, 4096)) != -1) {
                    fsDataOutputStream.write(data, 0, count);
                }
                fsDataOutputStream.flush();
                fsDataOutputStream.close();
            }
        }
        zipInputStream.close();
    } catch (final Exception e) {
        LOG.error("can not open zip file", e);
        throw new RuntimeException("unable to expand upgrade files", e);
    }

}

From source file:com.datatorrent.stram.util.FSUtil.java

License:Apache License

public static boolean mkdirs(FileSystem fs, Path dest) throws IOException {
    try {//www .j  a va  2 s . co  m
        return fs.mkdirs(dest);
    } catch (IOException e) {
        // some file system (MapR) throw exception if folder exists
        if (!fs.exists(dest)) {
            throw e;
        } else {
            return false;
        }
    }
}

From source file:com.digitalpebble.behemoth.util.ContentExtractor.java

License:Apache License

private int generateDocs(String inputf, String outputf) throws IOException, ArchiveException {

    Path input = new Path(inputf);
    Path dirPath = new Path(outputf);

    FileSystem fsout = FileSystem.get(dirPath.toUri(), getConf());

    if (fsout.exists(dirPath) == false)
        fsout.mkdirs(dirPath);
    else {/* w  w w.j a  va  2s .  c om*/
        System.err.println("Output " + outputf + " already exists");
        return -1;
    }

    // index file
    Path indexPath = new Path(dirPath, "index");
    if (fsout.exists(indexPath) == false) {
        fsout.createNewFile(indexPath);
    }

    maxNumEntriesInArchive = getConf().getInt(numEntriesPerArchiveParamName, 10000);

    index = fsout.create(indexPath);

    createArchive(dirPath);

    FileSystem fs = input.getFileSystem(getConf());
    FileStatus[] statuses = fs.listStatus(input);
    int count[] = { 0 };
    for (int i = 0; i < statuses.length; i++) {
        FileStatus status = statuses[i];
        Path suPath = status.getPath();
        if (suPath.getName().equals("_SUCCESS"))
            continue;
        generateDocs(suPath, dirPath, count);
    }

    if (index != null)
        index.close();

    if (currentArchive != null) {
        currentArchive.finish();
        currentArchive.close();
    }

    return 0;
}

From source file:com.ebay.erl.mobius.core.fs.MobiusLocalFileSystem.java

License:Apache License

public static boolean mkdirs(FileSystem fs, Path dir, FsPermission permission) throws IOException {

    // create the directory using the default permission
    boolean result = fs.mkdirs(dir);
    // set its permission to be the supplied one
    //fs.setPermission(dir, permission);
    return result;
}