List of usage examples for org.apache.hadoop.io IOUtils copyBytes
public static void copyBytes(InputStream in, OutputStream out, long count, boolean close) throws IOException
From source file:crunch.MaxTemperature.java
License:Apache License
public static void main(String[] args) throws Exception { String uri = args[0];//from w w w .j a va2 s. c o m Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(URI.create(uri), conf); InputStream in = null; try { in = fs.open(new Path(uri)); IOUtils.copyBytes(in, System.out, 4096, false); } finally { IOUtils.closeStream(in); } }
From source file:crunch.MaxTemperature.java
License:Apache License
public static void main(String[] args) throws Exception { String uri = args[0];//from w w w .j a va 2s. c o m Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(URI.create(uri), conf); FSDataInputStream in = null; // XXX FSDataInputStream is Seekable, pervious examples had InputStream (no seeking was done) try { in = fs.open(new Path(uri)); IOUtils.copyBytes(in, System.out, 4096, false); in.seek(0); // go back to the start of the file XXX IOUtils.copyBytes(in, System.out, 4096, false); } finally { IOUtils.closeStream(in); } }
From source file:crunch.MaxTemperature.java
License:Apache License
public static void main(String[] args) throws Exception { InputStream in = null;//from ww w .j a v a 2 s . c o m try { in = new URL(args[0]).openStream(); IOUtils.copyBytes(in, System.out, 4096, false); } finally { IOUtils.closeStream(in); } }
From source file:crunch.MaxTemperature.java
License:Apache License
public static void main(String[] args) throws Exception { String codecClassname = args[0]; Class<?> codecClass = Class.forName(codecClassname); Configuration conf = new Configuration(); CompressionCodec codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, conf); /*[*/Compressor compressor = null; try {//w w w . j av a 2s. c o m compressor = CodecPool.getCompressor(codec);/*]*/ CompressionOutputStream out = codec.createOutputStream(System.out, /*[*/compressor/*]*/); IOUtils.copyBytes(System.in, out, 4096, false); out.finish(); /*[*/} finally { CodecPool.returnCompressor(compressor); } /*]*/ }
From source file:crunch.MaxTemperature.java
License:Apache License
public static void main(String[] args) throws Exception { String codecClassname = args[0]; Class<?> codecClass = Class.forName(codecClassname); Configuration conf = new Configuration(); CompressionCodec codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, conf); CompressionOutputStream out = codec.createOutputStream(System.out); IOUtils.copyBytes(System.in, out, 4096, false); out.finish();//from w ww . j a v a2 s . com }
From source file:crunch.MaxTemperature.java
License:Apache License
@Test
public void decompressesGzippedFile() throws Exception {
File file = File.createTempFile("file", ".gz");
file.deleteOnExit();/*from www. j av a2 s . co m*/
InputStream in = this.getClass().getResourceAsStream("/file.gz");
IOUtils.copyBytes(in, new FileOutputStream(file), 4096, true);
String path = file.getAbsolutePath();
FileDecompressor.main(new String[] { path });
String decompressedPath = path.substring(0, path.length() - 3);
assertThat(readFile(new File(decompressedPath)), is("Text\n"));
}
From source file:dz.lab.hdfs.SeekReadFile.java
/** * @param args/*from w w w .ja v a2 s. c o m*/ */ public static void main(String[] args) throws IOException { Path fileToRead = new Path("/tmp/quotes.csv"); // read configuration from core-site.xml available in the classpath (under /resources) FileSystem fs = FileSystem.get(new Configuration()); FSDataInputStream input = null; try { // start at position 0 input = fs.open(fileToRead); System.out.print("start position=" + input.getPos() + ":"); IOUtils.copyBytes(input, System.out, 4096, false); // seek to position 11 input.seek(11); System.out.print("start position=" + input.getPos() + ":"); IOUtils.copyBytes(input, System.out, 4096, false); // seek back to position 0 input.seek(11); System.out.print("start position=" + input.getPos() + ":"); IOUtils.copyBytes(input, System.out, 4096, false); } finally { IOUtils.closeStream(input); } }
From source file:eml.studio.server.util.HDFSIO.java
License:Open Source License
/** * Write a file in hdfs//ww w.ja v a2s . c om * * @param uri target file uri * @param content file content * @throws IOException */ public static void upload(String uri, String content) throws IOException { Path path = new Path(Constants.NAME_NODE + "/" + uri); if (fs.exists(path)) fs.delete(path, true); OutputStream out = fs.create(path); InputStream rf = new ByteArrayInputStream(content.getBytes()); IOUtils.copyBytes(rf, out, 4096, true); out.close(); }
From source file:eml.studio.server.util.HDFSIO.java
License:Open Source License
/** * Create a new file on HDFS and add content, * it should be noted that the file did not exist * * @param uri HDFS uri//from ww w .j a v a2 s . c om * @param item InputStream item to upload * @throws IOException */ public static void uploadModel(String uri, InputStream item) throws IOException { FSDataOutputStream out = fs.create(new Path(Constants.NAME_NODE + "/" + uri)); // IOUtils.copyBytes method IOUtils.copyBytes(item, out, 4096, false); out.close(); }
From source file:eml.studio.server.util.HDFSIO.java
License:Open Source License
/** * Upload file to HDFS// w w w .j a v a2 s .c o m * @param uri target file path * @param item file to upload * @param name name of the new file * @throws IOException */ public static void uploadfile(String uri, FileItem item, String name) throws IOException { System.out.println("[dstPath]" + Constants.NAME_NODE + "/" + uri + name); FSDataOutputStream out = fs.create(new Path(Constants.NAME_NODE + uri + name)); IOUtils.copyBytes(item.getInputStream(), out, 4096, true); }