List of usage examples for org.apache.hadoop.io IOUtils copyBytes
public static void copyBytes(InputStream in, OutputStream out, long count, boolean close) throws IOException
From source file:io.aos.hdfs.basics.AbstractHdfsFileTest.java
License:Apache License
private void testCreateHdfsFile(Path path) throws IOException { getLogger().info("Testing HDFS file creation."); byte[] buff = "The HDFS File content".getBytes(Charset.forName("UTF-8")); FSDataOutputStream outputStream = getFileSystem().create(path); outputStream.write(buff, 0, buff.length); outputStream.close();//from ww w . j a v a2 s. c om FSDataInputStream in = getFileSystem().open(path); IOUtils.copyBytes(in, System.out, 4096, false); in.seek(0); IOUtils.copyBytes(in, System.out, 4096, false); IOUtils.closeStream(in); }
From source file:io.aos.hdfs.basics.AbstractHdfsFileTest.java
License:Apache License
private void testCopyHdfsFile2(Path from, Path to) throws IOException { FSDataInputStream in = getFileSystem().open(from); OutputStream out = getFileSystem().create(to, new Progressable() { @Override//from w ww. ja v a 2 s . co m public void progress() { System.out.println("."); } }); IOUtils.copyBytes(in, out, 4096, true); FSDataOutputStream out2 = getFileSystem().create(to); out2.writeUTF(MESSAGE); out2.close(); FSDataInputStream in2 = getFileSystem().open(from); String messageIn = in2.readUTF(); System.out.print(messageIn); in2.close(); }
From source file:io.aos.hdfs.basics.AbstractHdfsFileTest.java
License:Apache License
private void test1() throws IOException { Path srcPath = new Path("./src/test/resources/log4j.properties"); Path dstPath = new Path("/log4j.properties"); getFileSystem().copyFromLocalFile(srcPath, dstPath); FSDataInputStream in = getFileSystem().open(new Path("./log4j.properties")); IOUtils.copyBytes(in, System.out, 4096, false); in.seek(0);//from w w w.j a v a2 s . co m IOUtils.copyBytes(in, System.out, 4096, false); IOUtils.closeStream(in); OutputStream out = getFileSystem().create(new Path("./log4j.properties"), new Progressable() { @Override public void progress() { System.out.println("."); } }); IOUtils.copyBytes(in, out, 4096, true); }
From source file:io.aos.hdfs.FileCatTest.java
License:Apache License
@Test @Ignore// w w w . j av a 2s. c o m public static void test() throws Exception { String uri = "your hadoop file uri..."; Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(URI.create(uri), conf); InputStream in = null; in = fs.open(new Path(uri)); IOUtils.copyBytes(in, System.out, 4096, false); // for very small files, try to copy in a String ByteArrayOutputStream baos = new ByteArrayOutputStream(); PrintStream ps = new PrintStream(baos); IOUtils.copyBytes(in, ps, 4096, false); String content = baos.toString("UTF-8"); System.out.println(content); }
From source file:io.aos.hdfs.FileCopyWithProgress.java
License:Apache License
public static void main(String... args) throws Exception { String localSrc = args[0];/* ww w .ja v a 2 s . c om*/ String dst = args[1]; InputStream in = new BufferedInputStream(new FileInputStream(localSrc)); Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(URI.create(dst), conf); OutputStream out = fs.create(new Path(dst), new Progressable() { public void progress() { System.out.print("."); } }); IOUtils.copyBytes(in, out, 4096, true); }
From source file:io.aos.hdfs.FileSystemDoubleCat.java
License:Apache License
public static void main(String... args) throws Exception { String uri = args[0];// w ww .j a v a2 s.c o m Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(URI.create(uri), conf); FSDataInputStream in = null; try { in = fs.open(new Path(uri)); IOUtils.copyBytes(in, System.out, 4096, false); in.seek(0); // go back to the start of the file IOUtils.copyBytes(in, System.out, 4096, false); } finally { IOUtils.closeStream(in); } }
From source file:io.aos.hdfs.PooledStreamCompressor.java
License:Apache License
public static void main(String... args) throws Exception { String codecClassname = args[0]; Class<?> codecClass = Class.forName(codecClassname); Configuration conf = new Configuration(); CompressionCodec codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, conf); /*[*/Compressor compressor = null; try {//from w ww . j a va 2 s .co m compressor = CodecPool.getCompressor(codec);/*]*/ CompressionOutputStream out = codec.createOutputStream(System.out, /*[*/compressor/*]*/); IOUtils.copyBytes(System.in, out, 4096, false); out.finish(); /*[*/} finally { CodecPool.returnCompressor(compressor); } /*]*/ }
From source file:io.aos.hdfs.StreamCompressor.java
License:Apache License
public static void main(String... args) throws Exception { String codecClassname = args[0]; Class<?> codecClass = Class.forName(codecClassname); Configuration conf = new Configuration(); CompressionCodec codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, conf); CompressionOutputStream out = codec.createOutputStream(System.out); IOUtils.copyBytes(System.in, out, 4096, false); out.finish();/*from w w w . j a v a2 s .c o m*/ }
From source file:io.aos.hdfs.URLCat.java
License:Apache License
public static void main(String... args) throws Exception { InputStream in = null;// w ww. j ava 2 s . com try { in = new URL(args[0]).openStream(); IOUtils.copyBytes(in, System.out, 4096, false); } finally { IOUtils.closeStream(in); } }
From source file:it.crs4.seal.read_sort.MergeAlignments.java
License:Open Source License
private void copyMerge(Path[] sources, OutputStream out) throws IOException { Configuration conf = getConf(); for (int i = 0; i < sources.length; ++i) { FileSystem fs = sources[i].getFileSystem(conf); InputStream in = fs.open(sources[i]); try {// w ww .j a v a 2 s . com IOUtils.copyBytes(in, out, conf, false); } finally { in.close(); } } }