List of usage examples for org.apache.hadoop.io IOUtils copyBytes
public static void copyBytes(InputStream in, OutputStream out, long count, boolean close) throws IOException
From source file:com.sec.webs.classification.hdfs.HdfsFileOperation.java
License:Open Source License
/** * * appendLocalFile//from w w w . j av a2 s. c om * * @param fileName * @param appendFileName * void * @warning * @exception * @see Should add the following content in hdfs-site.xml <property> * <name>dfs.support.append</name> <value>true</value> </property> */ public void appendHdfsFileByAppend(String fileName, String appendFileName) { try { FSDataOutputStream outStream = fs.append(new Path(fileName)); FSDataInputStream inStream = fs.open(new Path(appendFileName)); IOUtils.copyBytes(inStream, outStream, 4096, true); } catch (IllegalArgumentException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } }
From source file:com.talis.hadoop.rdf.ZipUtils.java
License:Apache License
/** * Write a file to a zip output stream, removing leading path name components * from the actual file name when creating the zip file entry. * //from w w w. j a va 2 s .com * The entry placed in the zip file is <code>baseName</code>/ * <code>relativePath</code>, where <code>relativePath</code> is constructed * by removing a leading <code>root</code> from the path for * <code>itemToZip</code>. * * If <code>itemToZip</code> is an empty directory, it is ignored. If * <code>itemToZip</code> is a directory, the contents of the directory are * added recursively. * * @param zos The zip output stream * @param baseName The base name to use for the file name entry in the zip * file * @param root The path to remove from <code>itemToZip</code> to make a * relative path name * @param itemToZip The path to the file to be added to the zip file * @return the number of entries added * @throws IOException */ static public int zipDirectory(final Configuration conf, final ZipOutputStream zos, final String baseName, final String root, final Path itemToZip) throws IOException { LOG.info("zipDirectory: {} {} {}", new Object[] { baseName, root, itemToZip }); LocalFileSystem localFs = FileSystem.getLocal(conf); int count = 0; final FileStatus itemStatus = localFs.getFileStatus(itemToZip); if (itemStatus.isDir()) { final FileStatus[] statai = localFs.listStatus(itemToZip); // Add a directory entry to the zip file final String zipDirName = relativePathForZipEntry(itemToZip.toUri().getPath(), baseName, root); final ZipEntry dirZipEntry = new ZipEntry(zipDirName + Path.SEPARATOR_CHAR); LOG.info(String.format("Adding directory %s to zip", zipDirName)); zos.putNextEntry(dirZipEntry); zos.closeEntry(); count++; if (statai == null || statai.length == 0) { LOG.info(String.format("Skipping empty directory %s", itemToZip)); return count; } for (FileStatus status : statai) { count += zipDirectory(conf, zos, baseName, root, status.getPath()); } LOG.info(String.format("Wrote %d entries for directory %s", count, itemToZip)); return count; } final String inZipPath = relativePathForZipEntry(itemToZip.toUri().getPath(), baseName, root); if (inZipPath.length() == 0) { LOG.warn(String.format("Skipping empty zip file path for %s (%s %s)", itemToZip, root, baseName)); return 0; } // Take empty files in case the place holder is needed FSDataInputStream in = null; try { in = localFs.open(itemToZip); final ZipEntry ze = new ZipEntry(inZipPath); ze.setTime(itemStatus.getModificationTime()); // Comments confuse looking at the zip file // ze.setComment(itemToZip.toString()); zos.putNextEntry(ze); IOUtils.copyBytes(in, zos, conf, false); zos.closeEntry(); LOG.info(String.format("Wrote %d entries for file %s", count, itemToZip)); return 1; } finally { in.close(); } }
From source file:com.tomslabs.grid.avro.HadoopTestBase.java
License:Apache License
protected Path localResourceToPath(String path, String target) throws IOException { try {/*from w w w . j ava 2 s. c om*/ final InputStream resource = this.getClass().getResourceAsStream(path); if (resource == null) throw new IllegalArgumentException(path + " not found"); final Path targetPath = new Path(localFs.getWorkingDirectory(), "target/hadoop-test/imported/" + target) .makeQualified(localFs); localFs.delete(targetPath, true); OutputStream out = null; try { out = localFs.create(targetPath, true); IOUtils.copyBytes(resource, out, localConf, true); } catch (IOException e) { IOUtils.closeStream(out); IOUtils.closeStream(resource); throw e; } return targetPath; } catch (Exception e) { throw new IOException(e); } }
From source file:com.trendmicro.hdfs.webdav.HDFSResource.java
License:Apache License
@Override public void addMember(final DavResource resource, final InputContext context) throws DavException { // A PUT performed on an existing resource replaces the GET response entity // of the resource. Properties defined on the resource may be recomputed // during PUT processing but are not otherwise affected. final HDFSResource dfsResource = (HDFSResource) resource; final Path destPath = dfsResource.getPath(); try {//from w w w. j a v a 2 s .co m if (dfsResource.isCollectionRequest) { if (LOG.isDebugEnabled()) { LOG.debug("Creating new directory '" + destPath.toUri().getPath() + "'"); } boolean success = user.doAs(new PrivilegedExceptionAction<Boolean>() { public Boolean run() throws Exception { return FileSystem.get(conf).mkdirs(destPath); } }); if (!success) { throw new DavException(DavServletResponse.SC_CONFLICT); } } else { if (LOG.isDebugEnabled()) { LOG.debug("Creating new file '" + destPath.toUri().getPath() + "'"); } if (!context.hasStream() || context.getContentLength() < 0) { boolean success = user.doAs(new PrivilegedExceptionAction<Boolean>() { public Boolean run() throws Exception { return FileSystem.get(conf).createNewFile(destPath); } }); if (!success) { throw new DavException(DavServletResponse.SC_CONFLICT); } } else { user.doAs(new PrivilegedExceptionAction<Void>() { public Void run() throws Exception { OutputStream out = FileSystem.get(conf).create(destPath); InputStream in = context.getInputStream(); IOUtils.copyBytes(in, out, conf, true); return null; } }); } } } catch (IOException e) { throw new RuntimeException(e); } catch (InterruptedException e) { throw new RuntimeException(e); } }
From source file:com.trendmicro.hdfs.webdav.HDFSResource.java
License:Apache License
@Override public void spool(final OutputContext context) throws IOException { if (!isCollection()) try {/*from w w w . j a va 2s .c o m*/ user.doAs(new PrivilegedExceptionAction<Void>() { public Void run() throws Exception { InputStream input = FileSystem.get(conf).open(path); try { IOUtils.copyBytes(input, context.getOutputStream(), conf, false); } finally { input.close(); } return null; } }); } catch (InterruptedException e) { throw new IOException(e); } }
From source file:com.twitter.pig.backend.hadoop.executionengine.tez.TezJobControlCompiler.java
License:Apache License
/** * copy the file to hdfs in a temporary path * @param pigContext the pig context// w ww . j ava 2s . co m * @param conf the job conf * @param url the url to ship to hdfs * @return the location where it was shipped * @throws IOException */ private static Path shipToHDFS(PigContext pigContext, Configuration conf, URL url) throws IOException { String path = url.getPath(); int slash = path.lastIndexOf("/"); String suffix = slash == -1 ? path : path.substring(slash + 1); Path dst = new Path(FileLocalizer.getTemporaryPath(pigContext).toUri().getPath(), suffix); FileSystem fs = dst.getFileSystem(conf); OutputStream os = fs.create(dst); try { IOUtils.copyBytes(url.openStream(), os, 4096, true); } finally { // IOUtils can not close both the input and the output properly in a finally // as we can get an exception in between opening the stream and calling the method os.close(); } return dst; }
From source file:com.wipro.ats.bdre.dq.DQDriver.java
License:Apache License
private DQStats getQualityStats(Configuration conf, Path outputDir) throws IOException { int goodRecords = 0; int badRecords = 0; FileSystem destFs = outputDir.getFileSystem(getConf()); FSDataInputStream in = destFs.open(outputDir); OutputStream out = new OutputStream() { private StringBuilder string = new StringBuilder(); @Override//from w w w . j ava 2 s.co m public void write(int x) throws IOException { this.string.append((char) x); } @Override public String toString() { return this.string.toString(); } }; IOUtils.copyBytes(in, out, conf, false); String raw = out.toString(); for (String str : raw.split("\n")) { String[] tokens = str.toString().split("\t"); if (tokens[0].toString().equals(DQConstants.GOOD_RECORDS_FILE.trim())) { goodRecords = Integer.parseInt(tokens[1]); } else if (tokens[0].toString().equals(DQConstants.BAD_RECORDS_FILE.trim())) { badRecords = Integer.parseInt(tokens[1]); } } DQStats dqStats = new DQStats(); dqStats.setGoodPercent((goodRecords * 100.0F) / (goodRecords + badRecords)); dqStats.setNumBad(badRecords); dqStats.setNumGood(goodRecords); return dqStats; }
From source file:corner.hadoop.services.impl.HdfsAccessorProxy.java
License:Apache License
/** * @see corner.hadoop.services.DistributedResourceAccessor#putFile(java.lang.String, * java.io.InputStream)//from w ww. j a va 2s . c o m */ @Override public void putFile(String filePath, InputStream is) throws IOException { Path dstPath = new Path(filePath); FileSystem dstFs = dstPath.getFileSystem(getConf()); FSDataOutputStream out = dstFs.create(dstPath); try { IOUtils.copyBytes(is, out, getConf(), false); } finally { out.close(); } }
From source file:corner.services.hadoop.impl.HdfsAccessorProxy.java
License:Apache License
/** * @see corner.services.hadoop.DistributedResourceAccessor#putFile(java.lang.String, * java.io.InputStream)//from ww w.j a v a 2s . co m */ public void putFile(String filePath, InputStream is) throws IOException { Path dstPath = new Path(filePath); FileSystem dstFs = dstPath.getFileSystem(getConf()); FSDataOutputStream out = dstFs.create(dstPath); try { IOUtils.copyBytes(is, out, getConf(), false); } finally { out.close(); } }
From source file:crunch.MaxTemperature.java
License:Apache License
public static void main(String[] args) throws Exception { String localSrc = args[0];//www.j a v a 2 s . c o m String dst = args[1]; InputStream in = new BufferedInputStream(new FileInputStream(localSrc)); // XXX InputStream = BufferedInputStream(new FileInputStream()) Configuration conf = new Configuration(); // XXX hadoop.conf.Configuration FileSystem fs = FileSystem.get(URI.create(dst), conf); // XXX hadoop.fs.FileSystem from URI.create + hadoop.conf.Configuration OutputStream out = fs.create(new Path(dst), new Progressable() { // XXX fs.create -> stream hadoop.util.Progressable public void progress() { System.out.print("."); } }); IOUtils.copyBytes(in, out, 4096, true); // XXX hadoop.IOUtils.copyBytes - CLOSES in and out at the end }