List of usage examples for org.apache.hadoop.io IOUtils copyBytes
public static void copyBytes(InputStream in, OutputStream out, long count, boolean close) throws IOException
From source file:it.isislab.sof.core.engine.hadoop.mapreduce.mason.util.SimulationMASON.java
License:Apache License
public static void unJar(File jarFile, File toDir, Pattern unpackRegex) throws IOException { JarFile jar = new JarFile(jarFile); try {/*from w w w .j av a2 s . c o m*/ Enumeration<JarEntry> entries = jar.entries(); while (entries.hasMoreElements()) { JarEntry entry = (JarEntry) entries.nextElement(); if (!entry.isDirectory() && unpackRegex.matcher(entry.getName()).matches()) { InputStream in = jar.getInputStream(entry); try { File file = new File(toDir, entry.getName()); ensureDirectory(file.getParentFile()); OutputStream out = new FileOutputStream(file); try { IOUtils.copyBytes(in, out, 8192, false); } finally { out.close(); } } finally { in.close(); } } } } finally { jar.close(); } }
From source file:ml.shifu.guagua.hadoop.util.HDPUtils.java
License:Apache License
/** * Copy local file to HDFS. This is used to set as classpath by distributed cache. */// www . ja v a2 s .c o m private static Path shipToHDFS(Configuration conf, String fileName) throws IOException { Path dst = new Path("tmp", fileName.substring(fileName.lastIndexOf(File.separator) + 1)); FileSystem fs = dst.getFileSystem(conf); OutputStream os = null; InputStream is = null; try { is = FileSystem.getLocal(conf).open(new Path(fileName)); os = fs.create(dst); IOUtils.copyBytes(is, os, 4096, true); } finally { org.apache.commons.io.IOUtils.closeQuietly(is); // IOUtils should not close stream to HDFS quietly if (os != null) { os.close(); } } return dst; }
From source file:net.anjackson.dpt.tools.toseq.SeqFileBuilder.java
License:Apache License
/** Reads all bytes from the input stream and * returns them as a byte array.//www .jav a 2s . com */ private byte[] getBytes(InputStream input, long size) throws Exception { if (size > Integer.MAX_VALUE) { throw new Exception("A file in the archive is too large."); } // Copy: BufferedInputStream bufIn = new BufferedInputStream(input); ByteArrayOutputStream bytes = new ByteArrayOutputStream(); IOUtils.copyBytes(bufIn, bytes, (int) size, false); bytes.close(); return bytes.toByteArray(); }
From source file:net.sf.jfilesync.plugins.net.items.THdfs_plugin.java
License:Apache License
@Override public void put(InputStream instream, String remoteFileName) throws IOException { synchronized (this) { abort = false;/*from w w w. ja va 2s.c o m*/ this.instream = instream; } try { OutputStream out = fs.create(new Path(remoteFileName)); IOUtils.copyBytes(instream, out, 4096, true); } catch (IOException e) { if (!abort) { throw e; } else { checkConnectionState(); } } }
From source file:net.sf.jfilesync.plugins.net.items.THdfs_plugin.java
License:Apache License
@Override public void get(String remoteFileName, OutputStream outstream) throws IOException { synchronized (this) { abort = false;/*w w w . j a v a 2 s. c o m*/ this.outstream = outstream; } InputStream in = null; try { in = fs.open(new Path(remoteFileName)); IOUtils.copyBytes(in, outstream, 4096, false); } catch (IOException e) { if (!abort) { throw e; } else { checkConnectionState(); } } }
From source file:nur.aini.hadoop.CopyMergeRegexToLocal.java
License:GNU General Public License
public void run(String srcf, String dst) { final Path srcPath = new Path("./" + srcf); final Path desPath = new Path(dst); try {//from ww w . j a v a 2 s . com Path[] srcs = FileUtil.stat2Paths(hdfs.globStatus(srcPath), srcPath); OutputStream out = FileSystem.getLocal(conf).create(desPath); for (int i = 0; i < srcs.length; i++) { System.out.println(srcs[i]); InputStream in = hdfs.open(srcs[i]); IOUtils.copyBytes(in, out, conf, false); in.close(); } out.close(); } catch (IOException ex) { System.err.print(ex.getMessage()); } }
From source file:org.apache.camel.component.hdfs.HdfsProducerFileWriteTest.java
License:Apache License
@Test public void testSimpleWriteFile() throws Exception { if (SKIP) {//w ww . ja va 2 s .com return; } final Path file = new Path(new File("target/test/test-camel-simple-write-file").getAbsolutePath()); deleteDirectory("target/file-batch1"); context.addRoutes(new RouteBuilder() { @Override public void configure() throws Exception { from("file://target/file-batch1?sortBy=file:name") .to("hdfs:///" + file.toUri() + "?fileSystemType=LOCAL"); } }); context.start(); NotifyBuilder nb = new NotifyBuilder(context).whenDone(10).create(); for (int i = 0; i < 10; ++i) { template.sendBodyAndHeader("file://target/file-batch1/", "CIAO", "CamelFileName", "CIAO" + i); } Assert.assertTrue("Timeout waiting for match" + nb.toString(), nb.matchesMockWaitTime()); context.stop(); InputStream in = null; try { in = new URL("file:///" + file.toUri()).openStream(); ByteArrayOutputStream bos = new ByteArrayOutputStream(); IOUtils.copyBytes(in, bos, 4096, false); Assert.assertEquals(40, bos.size()); } finally { IOUtils.closeStream(in); } }
From source file:org.apache.camel.component.hdfs.HdfsProducerTest.java
License:Apache License
@Test public void testWriteTextWithDynamicFilename() throws Exception { if (!canTest()) { return;//from w w w .j av a 2 s .c o m } for (int i = 0; i < 5; i++) { template.sendBodyAndHeader("direct:write_dynamic_filename", "CIAO" + i, Exchange.FILE_NAME, "file" + i); } for (int i = 0; i < 5; i++) { InputStream in = null; try { in = new URL("file:///" + TEMP_DIR.toUri() + "/test-camel-dynamic/file" + i).openStream(); ByteArrayOutputStream bos = new ByteArrayOutputStream(); IOUtils.copyBytes(in, bos, 4096, false); assertEquals("CIAO" + i, new String(bos.toByteArray())); } finally { IOHelper.close(in); } } }
From source file:org.apache.camel.component.hdfs.HdfsProducerTest.java
License:Apache License
@Test public void testWriteTextWithDynamicFilenameExpression() throws Exception { if (!canTest()) { return;/*ww w . j a v a 2s .c om*/ } for (int i = 0; i < 5; i++) { template.sendBodyAndHeader("direct:write_dynamic_filename", "CIAO" + i, Exchange.FILE_NAME, simple("file-${body}")); } for (int i = 0; i < 5; i++) { InputStream in = null; try { in = new URL("file:///" + TEMP_DIR.toUri() + "/test-camel-dynamic/file-CIAO" + i).openStream(); ByteArrayOutputStream bos = new ByteArrayOutputStream(); IOUtils.copyBytes(in, bos, 4096, false); assertEquals("CIAO" + i, new String(bos.toByteArray())); } finally { IOHelper.close(in); } } }
From source file:org.apache.falcon.extensions.store.ExtensionStore.java
License:Apache License
public String getExtensionResource(final String resourcePath) throws FalconException { if (StringUtils.isBlank(resourcePath)) { throw new StoreAccessException("Resource path cannot be null or empty"); }/*w w w .j ava 2s . c o m*/ try { Path resourceFile = new Path(resourcePath); InputStream data; ByteArrayOutputStream writer = new ByteArrayOutputStream(); if (resourcePath.startsWith("file")) { data = fs.open(resourceFile); IOUtils.copyBytes(data, writer, fs.getConf(), true); } else { FileSystem fileSystem = getHdfsFileSystem(resourcePath); data = fileSystem.open(resourceFile); IOUtils.copyBytes(data, writer, fileSystem.getConf(), true); } return writer.toString(); } catch (IOException e) { throw new StoreAccessException(e); } }