Example usage for org.apache.hadoop.io IOUtils closeStream

Introduction

In this page you can find the example usage for org.apache.hadoop.io IOUtils closeStream.

Prototype

public static void closeStream(java.io.Closeable stream) 

Source Link

Document

Closes the stream ignoring Throwable .

Usage

From source file:io.aos.hdfs.SequenceFileWriteDemo.java

License:Apache License

public static void main(String... args) throws IOException {
    String uri = args[0];//from www .  j  ava 2  s.c  o m
    Configuration conf = new Configuration();
    FileSystem fs = FileSystem.get(URI.create(uri), conf);
    Path path = new Path(uri);

    IntWritable key = new IntWritable();
    Text value = new Text();
    SequenceFile.Writer writer = null;
    try {
        writer = SequenceFile.createWriter(fs, conf, path, key.getClass(), value.getClass());

        for (int i = 0; i < 100; i++) {
            key.set(100 - i);
            value.set(DATA[i % DATA.length]);
            System.out.printf("[%s]\t%s\t%s\n", writer.getLength(), key, value);
            writer.append(key, value);
        }
    } finally {
        IOUtils.closeStream(writer);
    }
}

---

From source file:io.aos.hdfs.URLCat.java

License:Apache License

public static void main(String... args) throws Exception {
    InputStream in = null;// w ww  . ja  va 2 s . c o  m
    try {
        in = new URL(args[0]).openStream();
        IOUtils.copyBytes(in, System.out, 4096, false);
    } finally {
        IOUtils.closeStream(in);
    }
}

---

From source file:io.spring.batch.components.HdfsTextFileWriter.java

License:Apache License

@Override
public void close() {
    if (fsDataOutputStream != null) {
        IOUtils.closeStream(fsDataOutputStream);
    }
}

---

From source file:io.spring.batch.components.HdfsTextItemWriter.java

License:Apache License

private void closeStream() {
    logger.debug("Closing output stream");
    if (fsDataOutputStream != null) {
        try {// w  ww. j  a v  a  2 s. co  m
            fsDataOutputStream.close();
        } catch (IOException e) {
            IOUtils.closeStream(fsDataOutputStream);
            throw new RuntimeException("Error while closing stream", e);
        } finally {
            fsDataOutputStream = null;
        }
    }
}

---

From source file:kr.co.bitnine.octopus.util.VersionInfo.java

License:Apache License

private VersionInfo(String component) {
    info = new Properties();
    String versionInfoFile = component + "-version-info.properties";
    InputStream is = null;/*from  w w w . j  a  v  a2s  .  c  o m*/
    try {
        is = Thread.currentThread().getContextClassLoader().getResourceAsStream(versionInfoFile);
        if (is == null)
            throw new IOException("Resource not found");
        info.load(is);
    } catch (IOException e) {
        LOG.warn("Could not read '" + versionInfoFile + "', " + e.toString(), e);
    } finally {
        IOUtils.closeStream(is);
    }
}

---

From source file:madgik.exareme.worker.arm.storage.client.cluster.ClusterArmStorageClient.java

/**
 * @param src// w  ww  . j  av  a 2 s. c o  m
 * @param dest
 * @throws ArmStorageClientException
 */
@Override
public void put(String src, String dest) throws ArmStorageClientException {

    // connected ?
    if (this.fs == null)
        throw new ArmStorageClientException("Not connected!");

    // validate parameters
    if (src == null || src.isEmpty())
        throw new ArmStorageClientException("No valid src file path!");
    if (dest == null || dest.isEmpty())
        throw new ArmStorageClientException("No valid dest file path!");

    // src file exist?
    File srcFile = new File(src);
    if (!srcFile.exists() || !srcFile.isFile() || srcFile.length() == 0)
        throw new ArmStorageClientException("src file does not exist!");

    // copy !
    FSDataOutputStream out = null;
    InputStream in = null;
    try {
        in = new FileInputStream(srcFile);

        Path destPath = new Path(dest);
        fs.mkdirs(destPath.getParent());

        if (this.fs.exists(destPath))
            throw new ArmStorageClientException("dest file allready exist!");

        //long blocksize = ArmStorageClientUtils.roundFileLength(srcFile.length());
        long currentBlockSize = blocksize == -1 ? ArmStorageClientUtils.roundFileLength(srcFile.length())
                : blocksize;
        out = this.fs.create(destPath, true, buffersize, (short) replication, currentBlockSize);
        IOUtils.copyBytes(in, out, buffersize);

    } catch (IOException ex) {
        throw new ArmStorageClientException("Put failure : ", ex);
    } finally {
        IOUtils.closeStream(in);
        IOUtils.closeStream(out);
    }
}

---

From source file:madgik.exareme.worker.arm.storage.client.cluster.ClusterArmStorageClient.java

/**
 * @param src//from  w w w  .j a  v a  2  s.c o m
 * @param dest
 * @throws ArmStorageClientException
 */
@Override
public void fetch(String src, String dest) throws ArmStorageClientException {

    // connected ?
    if (this.fs == null)
        throw new ArmStorageClientException("Not conntected!");

    // validate parameters
    if (src == null || src.isEmpty())
        throw new ArmStorageClientException("No valid src file path!");
    if (dest == null || dest.isEmpty())
        throw new ArmStorageClientException("No valid remote file path!");

    // already exists? TODO use /tmp/adp/ location
    File destFile = new File(dest);
    if (destFile.exists())
        return;
    //throw new ArmStorageClientException("dest file allready exists!");

    // copy ! TODO create a link
    OutputStream out = null;
    FSDataInputStream in = null;
    try {
        out = new FileOutputStream(dest);
        in = this.fs.open(new Path(src));
        IOUtils.copyBytes(in, out, buffersize);

    } catch (IOException ex) {
        throw new ArmStorageClientException("Fetch failure : ", ex);
    } finally {
        IOUtils.closeStream(in);
        IOUtils.closeStream(out);
    }
}

---

From source file:madgik.exareme.worker.arm.storage.client.cluster.HDFSArmStorageClient.java

/**
 * @param src/*from   w  w w  .  j av a  2  s  . c  o  m*/
 * @param dest
 * @throws ArmStorageClientException
 */
@Override
public void put(String src, String dest) throws ArmStorageClientException {

    // connected ?
    if (this.fs == null)
        throw new ArmStorageClientException("Not connected.");

    // validate parameters
    if (src == null || src.isEmpty())
        throw new ArmStorageClientException("No valid src file path!");
    if (dest == null || dest.isEmpty())
        throw new ArmStorageClientException("No valid dest file path!");

    // src file exist?
    File srcFile = new File(src);
    if (!srcFile.exists() || !srcFile.isFile() || srcFile.length() == 0)
        throw new ArmStorageClientException("src file does not exist!");

    // copy !
    FSDataOutputStream out = null;
    InputStream in = null;
    try {
        in = new FileInputStream(srcFile);

        Path destPath = new Path(dest);
        fs.mkdirs(destPath.getParent());

        if (this.fs.exists(destPath))
            throw new ArmStorageClientException("dest file allready exist!");

        //long blocksize = ArmStorageClientUtils.roundFileLength(srcFile.length());
        long currentBlockSize = blocksize == -1 ? ArmStorageClientUtils.roundFileLength(srcFile.length())
                : blocksize;
        out = this.fs.create(destPath, true, buffersize, (short) replication, currentBlockSize);
        IOUtils.copyBytes(in, out, buffersize);

    } catch (IOException ex) {
        throw new ArmStorageClientException("Put failure : ", ex);
    } finally {
        IOUtils.closeStream(in);
        IOUtils.closeStream(out);
    }
}

---

From source file:madgik.exareme.worker.arm.storage.client.cluster.HDFSArmStorageClient.java

/**
 * @param src/*from  ww w.j a  v a2 s  .  c  o m*/
 * @param dest
 * @throws ArmStorageClientException
 */
@Override
public void fetch(String src, String dest) throws ArmStorageClientException {

    // connected ?
    if (this.fs == null)
        throw new ArmStorageClientException("Not connected.");

    // validate parameters
    if (src == null || src.isEmpty())
        throw new ArmStorageClientException("No valid src file path!");
    if (dest == null || dest.isEmpty())
        throw new ArmStorageClientException("No valid remote file path!");

    // already exists? TODO use /tmp/adp/ location
    File destFile = new File(dest);
    if (destFile.exists())
        return;
    //throw new ArmStorageClientException("dest file allready exists!");

    // copy ! TODO create a link
    OutputStream out = null;
    FSDataInputStream in = null;
    try {
        out = new FileOutputStream(dest);
        in = this.fs.open(new Path(src));
        IOUtils.copyBytes(in, out, buffersize);

    } catch (IOException ex) {
        throw new ArmStorageClientException("Fetch failure : ", ex);
    } finally {
        IOUtils.closeStream(in);
        IOUtils.closeStream(out);
    }
}

---

From source file:mapreduceindexfiles.WholeFileRecordReader.java

@Override
public boolean next(Text k, ArrayWritable v) throws IOException {
    //throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
    //if (!processed) {
    //byte[] contents = new byte[(int) fileSplit.getLength()];
    Path file = fileSplit.getPath();
    FileSystem fs = file.getFileSystem(conf);
    FSDataInputStream in = null;//from  w ww.ja  v a2s.c  o m
    try {
        in = fs.open(file);
        String linePosition = Long.toString(in.getPos());
        BufferedReader reader = new BufferedReader(new InputStreamReader(in));
        //IOUtils.readFully(in, contents, 0, contents.length);
        String line = "";
        while (reader.ready()) {
            line = reader.readLine();
        }
        Text values[] = new Text[2];
        values[0] = new Text(linePosition);
        values[1] = new Text(line);
        v.set(values);
        write = v;

    } finally {
        IOUtils.closeStream(in);
    }
    //processed = true;
    return true;
    // }
    //return false;

}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• »