Example usage for org.apache.hadoop.fs FileSystem mkdirs

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem mkdirs.

Prototype

public boolean mkdirs(Path f) throws IOException 

Source Link

Document

Call #mkdirs(Path,FsPermission) with default permission.

Usage

From source file:edu.iu.lda.LDAMPCollectiveMapper.java

License:Apache License

private void printDocMap(LongArrayList[] docMap, Int2ObjectOpenHashMap<String> docIDMap, String folderPath,
        int selfID, Configuration congfiguration) throws IOException {
    FileSystem fs = FileSystem.get(congfiguration);
    Path folder = new Path(folderPath);
    if (!fs.exists(folder)) {
        fs.mkdirs(folder);
    }/*from ww w  .j a  v a 2  s .  c  o  m*/
    Path file = new Path(folderPath + "/" + selfID);
    PrintWriter writer = new PrintWriter(new BufferedWriter(new OutputStreamWriter(fs.create(file))));
    for (int i = 0; i < docMap.length; i++) {
        if (docMap[i] != null) {
            LongArrayList dRow = docMap[i];
            // Print real doc ID
            writer.print(docIDMap.get(i));
            // Print topic count
            for (int j = 0; j < dRow.size(); j++) {
                long t = dRow.getLong(j);
                writer.print(" " + (int) t + ":" + (int) (t >>> 32));
            }
            writer.println();
        }
    }
    writer.flush();
    writer.close();
}

---

From source file:edu.iu.lda.LDAMPCollectiveMapper.java

License:Apache License

private void saveLikelihood(double likelihood, String folderPath, int selfID, Configuration congfiguration)
        throws IOException {
    FileSystem fs = FileSystem.get(congfiguration);
    Path folder = new Path(folderPath);
    if (!fs.exists(folder)) {
        fs.mkdirs(folder);
    }//w w w  .j  a  v a 2s  .  com
    Path file = new Path(folderPath + "/evaluation");
    PrintWriter writer = new PrintWriter(new BufferedWriter(new OutputStreamWriter(fs.create(file))));
    writer.print(likelihood);
    writer.println();
    writer.flush();
    writer.close();
}

---

From source file:edu.iu.sgd.SGDCollectiveMapper.java

License:Apache License

private void saveModels(Table<DoubleArray>[] hTableMap, double[][] wMap, int K, double testRMSE,
        String folderPath, int selfID, Configuration congfiguration) throws IOException {
    FileSystem fs = FileSystem.get(congfiguration);
    Path folder = new Path(folderPath);
    if (!fs.exists(folder)) {
        fs.mkdirs(folder);
    }/*  w  w  w  .  j  a v a 2s  .c o  m*/

    //evaluation result
    if (this.isMaster()) {
        Path file = new Path(folderPath + "/evaluation");
        PrintWriter writer = new PrintWriter(new BufferedWriter(new OutputStreamWriter(fs.create(file))));
        writer.print(testRMSE);
        writer.println();
        writer.flush();
        writer.close();
    }

    //H model
    {
        Path file = new Path(folderPath + "/H-" + selfID);
        PrintWriter writer = new PrintWriter(new BufferedWriter(new OutputStreamWriter(fs.create(file))));
        for (Table<DoubleArray> hTable : hTableMap) {
            if (hTable == null)
                continue;
            for (Partition<DoubleArray> hPartition : hTable.getPartitions()) {
                int colID = hPartition.id();
                if (hPartition.get() != null) {
                    double[] hRow = hPartition.get().get();
                    // Print word
                    writer.print(colID + " :");
                    // Print topic count
                    for (int i = 0; i < K; i++) {
                        writer.print(" " + hRow[i]);
                    }
                    writer.println();
                }
            }
        }
        writer.flush();
        writer.close();
    }

    //W model
    {
        LOG.info("wMap length:" + wMap.length + " K:" + K);
        Path file = new Path(folderPath + "/W-" + selfID);
        PrintWriter writer = new PrintWriter(new BufferedWriter(new OutputStreamWriter(fs.create(file))));
        for (int i = 0; i < wMap.length; i++) {
            double[] wRow = wMap[i];
            if (wRow != null) {
                // Print word
                writer.print(i + " :");
                // Print topic count
                for (int j = 0; j < K; j++) {
                    writer.print(" " + wRow[j]);
                }
                writer.println();
            }
        }
        writer.flush();
        writer.close();
    }

}

---

From source file:edu.iu.sgd.SGDLauncher.java

License:Apache License

private void launch(String inputDirPath, int r, double lambda, double epsilon, int numIterations,
        int trainRatio, int numMapTasks, int numThreadsPerWorker, double scheduleRatio, int mem,
        String workDirPath, String testFilePath) throws IOException, URISyntaxException, InterruptedException,
        ExecutionException, ClassNotFoundException {
    Configuration configuration = getConf();
    FileSystem fs = FileSystem.get(configuration);
    Path inputDir = new Path(inputDirPath);
    Path workDir = new Path(workDirPath);
    if (fs.exists(workDir)) {
        fs.delete(workDir, true);/*from www .  j a  v a  2 s.  c  o m*/
        fs.mkdirs(workDir);
    }
    Path modelDir = new Path(workDirPath, "model");
    fs.mkdirs(modelDir);
    // Do not make output dir
    Path outputDir = new Path(workDirPath, "output");
    long startTime = System.currentTimeMillis();
    runSGD(inputDir, r, lambda, epsilon, numIterations, trainRatio, numMapTasks, numThreadsPerWorker,
            scheduleRatio, mem, modelDir, outputDir, testFilePath, configuration);
    long endTime = System.currentTimeMillis();
    System.out.println("Total SGD Execution Time: " + (endTime - startTime));
}

---

From source file:edu.iu.wdamds.MDSLauncher.java

License:Apache License

boolean launch(int numMapTasks, String inputFolder, String inputPrefix, String weightPrefix, String vPrefix,
        String idsFile, String labelsFile, double threshold, int d, double alpha, int n, int cgIter,
        int numThreads) throws IOException, URISyntaxException, InterruptedException, ExecutionException,
        ClassNotFoundException {/* ww w . j  a  v a 2 s.  c o  m*/
    Configuration configuration = getConf();
    FileSystem fs = FileSystem.get(configuration);
    // Generate working directory, for partition
    // files
    String workDirName = "wdamds" + System.currentTimeMillis();
    Path workDirPath = new Path(workDirName);
    // if (fs.exists(workDirPath)) {
    // fs.delete(workDirPath, true);
    // }
    fs.mkdirs(workDirPath);
    Path dataDirPath = new Path(workDirPath, "data");
    fs.mkdirs(dataDirPath);
    Path xDirPath = new Path(workDirPath, "x");
    fs.mkdirs(xDirPath);
    Path xFilePath = new Path(xDirPath, "x");
    Path xOutFilePath = new Path(xDirPath, "x_out");
    Path outDirPath = new Path(workDirPath, "out");
    // Generate initial mapping
    // Ignore initial mapping generating on HDFS
    // Generate in memory
    // DataGen.generateXData(n, d, xFilePath, fs);
    // Generate partition files
    // Each file contains a data partition and a
    // weight partition (or multiple)
    DataGen.generatePartitionFiles(inputFolder, inputPrefix, weightPrefix, vPrefix, dataDirPath, fs,
            numMapTasks);
    return runWDAMDS(numMapTasks, dataDirPath, xFilePath, xOutFilePath, outDirPath, idsFile, labelsFile,
            threshold, d, alpha, n, cgIter, numThreads);
}

---

From source file:edu.nyu.vida.data_polygamy.utils.FrameworkUtils.java

License:BSD License

public static void createDir(String dir, Configuration conf, boolean s3) throws IOException {
    if (s3) {//from   w  w  w  .  ja  va 2  s.c  o  m
        Path path = new Path(dir);
        FileSystem fs = FileSystem.get(path.toUri(), conf);

        if (!fs.exists(path))
            fs.mkdirs(path);

        fs.close();
    } else {
        FileSystem hdfs = FileSystem.get(new Configuration());
        Path hdfsFile = new Path(hdfs.getHomeDirectory() + "/" + dir);

        if (!hdfs.exists(hdfsFile))
            hdfs.mkdirs(hdfsFile);
    }
}

---

From source file:edu.purdue.cybercenter.dm.storage.HdfsStorageFileManager.java

private void transferFile(FileSystem sourceType, String source, FileSystem destFS, String dest, FileId fileId,
        boolean makeDir) throws IOException {
    if (makeDir) {
        destFS.mkdirs(new Path(dest.substring(0, dest.lastIndexOf("/"))));
    }//from  w ww. j  a  va  2 s.c o  m
    // FileUtil.copy(sourceType,new
    // Path("c://TEMP/springsource.exe"),destType,new
    // Path("Z://springsource.exe"),false,conf);
    boolean success = false;
    Exception e1 = null;
    try {
        FileUtil.copy(sourceType, new Path(source), destFS, new Path(dest), false, configuration);
        success = true;
        System.out.println("file: " + source + " transfered");
    } catch (Exception e) {
        success = false;
        e.printStackTrace();
        e1 = e;
    } finally {
        if (!success) {
            if (fileId != null) {
                removeStorageFileEntry(fileId.getFileId());
            }
            if (e1 != null) {
                throw new IOException(e1);
            } else {
                throw new IOException("File not transfered");
            }
        }
    }

}

---

From source file:edu.purdue.cybercenter.dm.storage.HdfsStorageFileManager.java

private void transferString(String toStore, FileSystem destFS, String dest) throws Exception {
    destFS.mkdirs(new Path(dest.substring(0, dest.lastIndexOf("//"))));
    BufferedWriter br = new BufferedWriter(new OutputStreamWriter(destFS.create(new Path(dest), true)));
    // TO append data to a file, use fs.append(Path f)
    br.write(toStore);/* ww  w . j av a2 s .  c  o m*/
    br.close();
    System.out.println("String transfered");
}

---

From source file:edu.purdue.cybercenter.dm.storage.HdfsStorageFileManager.java

private StorageFile moveFile(StorageFile file, Storage storage) throws Exception {
    FileSystem destFS = getFileSystemType(storage.getType());
    String destPath = storage.getLocation() + file.getLocation();
    String sourcePath = file.getStorageId().getLocation() + file.getLocation();
    FileSystem sourceFs = getFileSystemType(file.getStorageId().getType());
    destFS.mkdirs(new Path(destPath.substring(0, destPath.lastIndexOf("/"))));
    FileUtil.copy(getFileSystemType(file.getStorageId().getType()), new Path(sourcePath), destFS,
            new Path(destPath), false, configuration);
    FileUtil.fullyDelete(sourceFs, new Path(sourcePath));
    file.setStorageId(storage);//from   w w w.  ja  v a2 s. c om
    file.persist();
    System.out.println("move from ");
    System.out.println(sourcePath);
    System.out.println("To ");
    System.out.println(destPath);
    return file;
}

---

From source file:edu.uci.ics.hyracks.hdfs.dataflow.DataflowTest.java

License:Apache License

/**
 * Start the HDFS cluster and setup the data files
 * /*from   www . ja  v  a 2 s  .co  m*/
 * @throws IOException
 */
private void startHDFS() throws IOException {
    conf.addResource(new Path(PATH_TO_HADOOP_CONF + "/core-site.xml"));
    conf.addResource(new Path(PATH_TO_HADOOP_CONF + "/mapred-site.xml"));
    conf.addResource(new Path(PATH_TO_HADOOP_CONF + "/hdfs-site.xml"));

    FileSystem lfs = FileSystem.getLocal(new Configuration());
    lfs.delete(new Path("build"), true);
    System.setProperty("hadoop.log.dir", "logs");
    dfsCluster = new MiniDFSCluster(conf, numberOfNC, true, null);
    FileSystem dfs = FileSystem.get(conf);
    Path src = new Path(DATA_PATH);
    Path dest = new Path(HDFS_INPUT_PATH);
    Path result = new Path(HDFS_OUTPUT_PATH);
    dfs.mkdirs(dest);
    dfs.mkdirs(result);
    dfs.copyFromLocalFile(src, dest);

    DataOutputStream confOutput = new DataOutputStream(new FileOutputStream(new File(HADOOP_CONF_PATH)));
    conf.writeXml(confOutput);
    confOutput.flush();
    confOutput.close();
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• »