Example usage for org.apache.hadoop.fs FileSystem open

List of usage examples for org.apache.hadoop.fs FileSystem open

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileSystem
  6. open

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem open.

Prototype

public FSDataInputStream open(PathHandle fd) throws IOException

Source Link

Document

Open an FSDataInputStream matching the PathHandle instance.

Usage

From source file:com.inmobi.grill.server.query.TestQueryService.java

License:Apache License

private void validatePersistentResult(PersistentQueryResult resultset, QueryHandle handle) throws IOException {
    Assert.assertTrue(resultset.getPersistedURI().endsWith(handle.toString()));
    Path actualPath = new Path(resultset.getPersistedURI());
    FileSystem fs = actualPath.getFileSystem(new Configuration());
    List<String> actualRows = new ArrayList<String>();
    for (FileStatus fstat : fs.listStatus(actualPath)) {
        FSDataInputStream in = fs.open(fstat.getPath());
        BufferedReader br = null;
        try {//from  ww w .java2  s  .  c o  m
            br = new BufferedReader(new InputStreamReader(in));
            String line = "";

            while ((line = br.readLine()) != null) {
                actualRows.add(line);
            }
        } finally {
            if (br != null) {
                br.close();
            }
            if (in != null) {
                in.close();
            }
        }
    }
    Assert.assertEquals(actualRows.get(0), "1one");
    Assert.assertEquals(actualRows.get(1), "\\Ntwo");
    Assert.assertEquals(actualRows.get(2), "3\\N");
    Assert.assertEquals(actualRows.get(3), "\\N\\N");
    Assert.assertEquals(actualRows.get(4), "5");
}

From source file:com.inmobi.messaging.consumer.util.FileUtil.java

License:Apache License

public static void gzip(Path src, Path target, Configuration conf) throws IOException {
    FileSystem fs = FileSystem.get(conf);
    FSDataOutputStream out = fs.create(target);
    GzipCodec gzipCodec = (GzipCodec) ReflectionUtils.newInstance(GzipCodec.class, conf);
    Compressor gzipCompressor = CodecPool.getCompressor(gzipCodec);
    OutputStream compressedOut = gzipCodec.createOutputStream(out, gzipCompressor);
    FSDataInputStream in = fs.open(src);
    try {/*  ww  w. ja  v a  2  s .c o  m*/
        IOUtils.copyBytes(in, compressedOut, conf);
    } catch (Exception e) {
        LOG.error("Error in compressing ", e);
    } finally {
        in.close();
        CodecPool.returnCompressor(gzipCompressor);
        compressedOut.close();
        out.close();
    }
}

From source file:com.intel.hibench.datagen.streaming.util.SourceFileReader.java

License:Apache License

static public BufferedReader getReader(Configuration dfsConf, String path, long offset) {
    BufferedReader reader = null;
    try {//from   w w  w . j a  v a  2  s. c om
        Path pt = new Path(path);
        FileSystem fs = FileSystem.get(dfsConf);
        InputStreamReader isr;
        if (fs.isDirectory(pt)) {
            //give path is an directory
            isr = new InputStreamReader(openMultipleParts(fs, pt, offset));
        } else {
            //give path is an file
            FSDataInputStream inputStream = fs.open(pt);
            if (offset > 0) {
                inputStream.seek(offset);
            }
            isr = new InputStreamReader(inputStream);
        }

        reader = new BufferedReader(isr);
    } catch (IOException e) {
        System.err.println("Fail to get reader from path: " + path);
        e.printStackTrace();
    }
    return reader;
}

From source file:com.intel.hibench.datagen.streaming.util.SourceFileReader.java

License:Apache License

static private InputStream openMultipleParts(FileSystem fs, Path pt, long offset) throws IOException {

    System.out.println("opening all parts in path: " + pt + ", from offset: " + offset);
    // list all files in given path
    RemoteIterator<LocatedFileStatus> rit = fs.listFiles(pt, false);
    Vector<FSDataInputStream> fileHandleList = new Vector<FSDataInputStream>();
    while (rit.hasNext()) {
        Path path = rit.next().getPath();

        // Only read those files start with "part-"
        if (path.getName().startsWith("part-")) {
            long fileSize = fs.getFileStatus(path).getLen();
            if (offset < fileSize) {
                FSDataInputStream inputStream = fs.open(path);
                if (offset > 0) {
                    inputStream.seek(offset);
                }/*from   ww w .ja v a2s .  c  om*/
                fileHandleList.add(inputStream);
            }
            offset -= fileSize;
        }
    }

    if (!fileHandleList.isEmpty()) {
        return new SequenceInputStream(fileHandleList.elements());
    } else {
        System.err.println("Error, no source file loaded. run genSeedDataset.sh first!");
        return null;
    }

}

From source file:com.intel.hibench.streambench.FileDataGenNew.java

License:Apache License

public BufferedReader loadDataFromFile(String filepath, long offset) {
    try {//www  .  j  av a 2  s .  c  om
        Path pt = new Path(filepath);
        FileSystem fs = FileSystem.get(fsConf);
        InputStreamReader isr;
        if (fs.isDirectory(pt)) { // multiple parts
            isr = new InputStreamReader(OpenMultiplePartsWithOffset(fs, pt, offset));
        } else { // single file
            FSDataInputStream fileHandler = fs.open(pt);
            if (offset > 0)
                fileHandler.seek(offset);
            isr = new InputStreamReader(fileHandler);
        }

        BufferedReader reader = new BufferedReader(isr);
        if (offset > 0)
            reader.readLine(); // skip first line in case of seek
        return reader;
    } catch (FileNotFoundException e) {
        e.printStackTrace();
    } catch (IOException e) {
        e.printStackTrace();
    }
    return null;
}

From source file:com.intel.hibench.streambench.FileDataGenNew.java

License:Apache License

private InputStream OpenMultiplePartsWithOffset(FileSystem fs, Path pt, long offset) throws IOException {
    System.out.println("Opening files, path:" + pt + " offset:" + offset);
    RemoteIterator<LocatedFileStatus> rit = fs.listFiles(pt, false);
    Vector<FSDataInputStream> fileHandleList = new Vector<FSDataInputStream>();
    while (rit.hasNext()) {
        Path path = rit.next().getPath();
        String filename = path.toString().substring(path.getParent().toString().length(),
                path.toString().length());

        if (filename.startsWith("/part-")) {
            long filesize = fs.getFileStatus(path).getLen();
            if (offset < filesize) {
                FSDataInputStream handle = fs.open(path);
                if (offset > 0) {
                    handle.seek(offset);
                }//from  w w  w .  j  a  va 2 s . c om
                fileHandleList.add(handle);
            }
            offset -= filesize;
        }
    }
    if (fileHandleList.size() == 1)
        return fileHandleList.get(0);
    else if (fileHandleList.size() > 1) {
        Enumeration<FSDataInputStream> enu = fileHandleList.elements();
        return new SequenceInputStream(enu);
    } else {
        System.err.println("Error, no source file loaded. run genSeedDataset.sh first!");
        return null;
    }
}

From source file:com.ipcglobal.fredimportcdh.TsvsToImpala.java

License:Apache License

/**
 * Load properties.//from  w w w  .ja v  a 2  s  .  c o  m
 *
 * @param propFileNameExt the prop file name ext
 * @param conf the conf
 * @return the properties
 * @throws Exception the exception
 */
public static Properties loadProperties(String propFileNameExt, Configuration conf) throws Exception {
    InputStream input = null;
    Properties properties = new Properties();
    try {
        Path pathPropFile = new Path(propFileNameExt);
        FileSystem fsPropFile = pathPropFile.getFileSystem(conf);
        input = fsPropFile.open(pathPropFile);
        // input = new FileInputStream(propFileNameExt);
        properties.load(input);
        return properties;
    } catch (IOException ex) {
        ex.printStackTrace();
        throw ex;
    } finally {
        if (input != null) {
            try {
                input.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }
}

From source file:com.jeffy.hdfs.compression.FileDecompressor.java

License:Apache License

/**
 * @param args//from ww w  .  ja v a2  s .  c  o m
 *            
 * @throws IOException
 */
public static void main(String[] args) throws IOException {
    //??
    Configuration conf = new Configuration();
    // ?
    CompressionCodecFactory factory = new CompressionCodecFactory(conf);
    for (String uri : args) {
        FileSystem fs = FileSystem.get(URI.create(uri), conf);
        Path inputPath = new Path(uri);
        // ??????io.compression.codecs
        CompressionCodec codec = factory.getCodec(inputPath);
        // ??
        if (codec == null) {
            System.err.println("No codec found for " + uri);
            continue;
        }
        String outputUri = CompressionCodecFactory.removeSuffix(uri, codec.getDefaultExtension());
        try (InputStream in = codec.createInputStream(fs.open(inputPath));
                OutputStream out = fs.create(new Path(outputUri))) {
            IOUtils.copyBytes(in, out, conf);
        }
    }
}

From source file:com.jeffy.hdfs.HDFSReadFile.java

License:Apache License

/**
 * HadoopFileSystem API??/*from w w w. j av a2  s  .co m*/
 * 
 * @param path
 * @throws IOException 
 */
public static void readDataUseFileSystem(String path) throws IOException {

    Configuration config = new Configuration();
    /**
     * ?FileSystem????????
     *  public static FileSystem get(Configuration conf) throws IOException
     *   public static FileSystem get(URI uri, Configuration conf) throws IOException
     *   public static FileSystem get(URI uri, Configuration conf, String user)
     *   throws IOException
     */
    FileSystem fs = FileSystem.get(URI.create(path), config);
    //??FSDataInputStream,DataInputStream,????
    //open4KB
    try (InputStream in = fs.open(new Path(path))) {
        IOUtils.copy(in, System.out);
    }
}

From source file:com.jkoolcloud.tnt4j.streams.inputs.HdfsFileLineStream.java

License:Apache License

private static int[] getFilesTotals(FileSystem fs, Path[] activityFiles) {
    int tbc = 0;/* ww  w  . j  a va2s.  co m*/
    int tlc = 0;
    if (ArrayUtils.isNotEmpty(activityFiles)) {
        for (Path f : activityFiles) {
            try {
                ContentSummary cSummary = fs.getContentSummary(f);
                tbc += cSummary.getLength();
                tlc += Utils.countLines(fs.open(f));
            } catch (IOException exc) {
            }
        }
    }

    return new int[] { tbc, tlc };
}