Example usage for org.apache.hadoop.fs Path getFileSystem

Introduction

In this page you can find the example usage for org.apache.hadoop.fs Path getFileSystem.

Prototype

public FileSystem getFileSystem(Configuration conf) throws IOException 

Source Link

Document

Return the FileSystem that owns this Path.

Usage

From source file:com.inmobi.grill.server.query.TestQueryService.java

License:Apache License

private void validatePersistentResult(PersistentQueryResult resultset, QueryHandle handle) throws IOException {
    Assert.assertTrue(resultset.getPersistedURI().endsWith(handle.toString()));
    Path actualPath = new Path(resultset.getPersistedURI());
    FileSystem fs = actualPath.getFileSystem(new Configuration());
    List<String> actualRows = new ArrayList<String>();
    for (FileStatus fstat : fs.listStatus(actualPath)) {
        FSDataInputStream in = fs.open(fstat.getPath());
        BufferedReader br = null;
        try {/*  ww w .j  a  v a 2  s. com*/
            br = new BufferedReader(new InputStreamReader(in));
            String line = "";

            while ((line = br.readLine()) != null) {
                actualRows.add(line);
            }
        } finally {
            if (br != null) {
                br.close();
            }
            if (in != null) {
                in.close();
            }
        }
    }
    Assert.assertEquals(actualRows.get(0), "1
one");
    Assert.assertEquals(actualRows.get(1), "\\N
two");
    Assert.assertEquals(actualRows.get(2), "3
\\N");
    Assert.assertEquals(actualRows.get(3), "\\N
\\N");
    Assert.assertEquals(actualRows.get(4), "5
");
}

---

From source file:com.inmobi.messaging.consumer.databus.TestAbstractDatabusConsumer.java

License:Apache License

public void cleanup() throws IOException {
    testConsumer.close();//from w ww  . j  a  v a  2 s  .  c  om
    for (Path p : rootDirs) {
        FileSystem fs = p.getFileSystem(conf);
        LOG.debug("Cleaning up the dir: " + p);
        fs.delete(p, true);
    }
    FileSystem lfs = new Path(chkpointPathPrefix).getFileSystem(conf);
    lfs.delete(new Path(chkpointPathPrefix).getParent(), true);
}

---

From source file:com.inmobi.messaging.consumer.util.HadoopUtil.java

License:Apache License

public static void setUpHadoopFiles(Path streamDirPrefix, Configuration conf, String[] files,
        String[] suffixDirs, Path[] finalFiles, boolean alternateEmptyFiles, Date minuteDirTimeStamp, int index,
        int startIndex) throws Exception {
    FileSystem fs = streamDirPrefix.getFileSystem(conf);
    Path rootDir = streamDirPrefix.getParent();
    Path tmpDataDir = new Path(rootDir, "data");
    boolean emptyFile = false;
    // setup data dirs
    if (files != null) {
        int i = startIndex;
        int j = index;
        for (String file : files) {
            if (alternateEmptyFiles && emptyFile) {
                MessageUtil.createEmptySequenceFile(file, fs, tmpDataDir, conf);
                emptyFile = false;//from  ww w .  ja  v  a2  s  .co  m
            } else {
                MessageUtil.createMessageSequenceFile(file, fs, tmpDataDir, i, conf);
                emptyFile = true;
                i += 100;
            }
            Path srcPath = new Path(tmpDataDir, file);
            Date commitTime = getCommitDateForFile(file, minuteDirTimeStamp);
            TestUtil.publishMissingPaths(fs, streamDirPrefix, lastCommitTime, commitTime);
            lastCommitTime = commitTime;
            Path targetDateDir = getTargetDateDir(streamDirPrefix, commitTime);
            List<Path> targetDirs = new ArrayList<Path>();
            if (suffixDirs != null) {
                for (String suffixDir : suffixDirs) {
                    targetDirs.add(new Path(targetDateDir, suffixDir));
                }
            } else {
                targetDirs.add(targetDateDir);
            }
            for (Path targetDir : targetDirs) {
                fs.mkdirs(targetDir);
                Path targetPath = new Path(targetDir, file);
                fs.copyFromLocalFile(srcPath, targetPath);
                LOG.info("Copied " + srcPath + " to " + targetPath);
                if (finalFiles != null) {
                    finalFiles[j] = targetPath;
                    j++;
                }
                Thread.sleep(1000);
            }
            fs.delete(srcPath, true);
        }
        TestUtil.publishLastPath(fs, streamDirPrefix, lastCommitTime);
    }
}

---

From source file:com.inmobi.messaging.consumer.util.HadoopUtil.java

License:Apache License

public static void setupHadoopCluster(Configuration conf, String[] files, String[] suffixDirs,
        Path[] finalFiles, Path finalDir, boolean withEmptyFiles, boolean createFilesInNextHour)
        throws Exception {
    FileSystem fs = finalDir.getFileSystem(conf);

    Path rootDir = finalDir.getParent();
    fs.delete(rootDir, true);/*from  www . j  av  a2  s .  com*/
    Path tmpDataDir = new Path(rootDir, "data");
    fs.mkdirs(tmpDataDir);

    if (!createFilesInNextHour) {
        setUpHadoopFiles(finalDir, conf, files, suffixDirs, finalFiles, withEmptyFiles, null, 0, 0);
    } else {
        // start from 1 hour back as we need files in two diff hours.
        Calendar cal = Calendar.getInstance();
        cal.setTime(startCommitTime);
        cal.add(Calendar.HOUR_OF_DAY, -1);

        setUpHadoopFiles(finalDir, conf, files, suffixDirs, finalFiles, withEmptyFiles, cal.getTime(), 0, 0);
        // go to next hour
        cal.add(Calendar.HOUR_OF_DAY, 1);
        int index = files.length;
        // find number of non empty(i.e. data) files in 1 hour
        int numberOfNonEmptyFiles = withEmptyFiles ? (int) Math.ceil(index / 2.0) : index;
        int startIndex = numberOfNonEmptyFiles * 100;
        setUpHadoopFiles(finalDir, conf, files, suffixDirs, finalFiles, withEmptyFiles, cal.getTime(), index,
                startIndex);
    }
}

---

From source file:com.ipcglobal.fredimportcdh.TsvsToImpala.java

License:Apache License

/**
 * Load properties.// w  w  w. j ava 2  s .c  om
 *
 * @param propFileNameExt the prop file name ext
 * @param conf the conf
 * @return the properties
 * @throws Exception the exception
 */
public static Properties loadProperties(String propFileNameExt, Configuration conf) throws Exception {
    InputStream input = null;
    Properties properties = new Properties();
    try {
        Path pathPropFile = new Path(propFileNameExt);
        FileSystem fsPropFile = pathPropFile.getFileSystem(conf);
        input = fsPropFile.open(pathPropFile);
        // input = new FileInputStream(propFileNameExt);
        properties.load(input);
        return properties;
    } catch (IOException ex) {
        ex.printStackTrace();
        throw ex;
    } finally {
        if (input != null) {
            try {
                input.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }
}

---

From source file:com.jackbe.mapreduce.LocalJobManager.java

License:Open Source License

public RunningJob startJob(String inputDir, String outputDir, String mapperScript, String reducerScript,
        String combinerScript) throws Exception {

    init();/*from ww w.j av  a2 s. c  o  m*/
    conf.setJobName("EMMLMapReduce");
    //conf.setSessionId(Long.toString(System.currentTimeMillis()));

    conf.set("MAPPER_SCRIPT", mapperScript);
    conf.set("REDUCER_SCRIPT", reducerScript);
    if (combinerScript != null) {
        conf.set("COMBINER_SCRIPT", combinerScript);
        conf.setCombinerClass(EMMLCombiner.class);
    }

    //      FileInputFormat.setInputPaths(conf, new Path(inputDir));
    FileInputFormat.setInputPaths(conf, new Path("hdfs://" + NAMENODE + "/" + inputDir));
    //      FileOutputFormat.setOutputPath(conf, new Path(outputDir));
    Path outputPath = new Path("hdfs://" + NAMENODE + "/" + outputDir);
    outputPath.getFileSystem(conf).delete(outputPath, true);
    FileOutputFormat.setOutputPath(conf, outputPath);
    RESTRegistrationJobCallback callback = new RESTRegistrationJobCallback(outputPath, outputDir, conf);

    RunningJob job = null;
    try {
        job = jobClient.submitJob(conf);
        this.registerJobCompleteCallback(job, callback);

        statusMap.put(job.getJobID(), job);
    } catch (IOException e) {
        e.printStackTrace();
        throw e;
    }
    jobClient.getSystemDir();
    return job;
}

---

From source file:com.jackbe.mapreduce.LocalJobManager.java

License:Open Source License

private void copyToLocal(String hdfsPath, String localPath) {
    Path hPath = new Path("hdfs://" + NAMENODE + "/" + hdfsPath);
    Path lPath = new Path(localPath);
    try {//from w w w .  j a va  2s  . com
        hPath.getFileSystem(conf).copyToLocalFile(hPath, lPath);
    } catch (IOException e) {
        log.error("Could not copy from " + hPath.toString() + " to " + lPath.toString() + ": " + e.getMessage(),
                e);
    }

}

---

From source file:com.jbw.tar.sf.TarOutputFormat.java

@Override
public RecordWriter<K, V> getRecordWriter(TaskAttemptContext context) throws IOException, InterruptedException {
    Configuration conf = context.getConfiguration();
    String extension = ".tar";

    Path file = getDefaultWorkFile(context, extension);
    FileSystem fs = file.getFileSystem(conf);
    OutputStream fileOut = fs.create(file, false);

    //tar?//from   w  w w.  j a  va  2s  . co m
    return new TarOutputWriter<>(fileOut);

}

---

From source file:com.jbw.taroutputformat.TarOutputFormat.java

@Override
public RecordWriter<K, V> getRecordWriter(TaskAttemptContext tac) throws IOException, InterruptedException {
    Configuration conf = tac.getConfiguration();
    String extension = ".tar";
    Path file = getDefaultWorkFile(tac, extension);
    FileSystem fs = file.getFileSystem(conf);
    OutputStream fileOut = fs.create(file, false);
    return new TarOutputWriter<>(fileOut);
}

---

From source file:com.jhkt.playgroundArena.hadoop.tasks.jobs.reducer.BloomFilterReducer.java

License:Apache License

protected void cleanup(Reducer<Text, BloomFilter, Text, Text>.Context context)
        throws IOException, InterruptedException {
    Configuration conf = context.getConfiguration();
    Path file = new Path(conf.get("mapred.output.dir") + "/bloomFilter");
    FSDataOutputStream out = file.getFileSystem(conf).create(file);
    _bFilter.write(out);//from  w  ww  .  ja v  a2s .  c  om
    out.close();
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »