List of usage examples for org.apache.hadoop.io IOUtils copyBytes
public static void copyBytes(InputStream in, OutputStream out, long count, boolean close) throws IOException
From source file:cn.lhfei.hadoop.ch03.URLCat.java
License:Apache License
public static void main(String[] args) { InputStream in = null;//from ww w . ja va 2 s. c o m try { in = new URL(args[0]).openStream(); IOUtils.copyBytes(in, System.out, 4096, false); } catch (MalformedURLException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } finally { IOUtils.closeStream(in); } }
From source file:cn.lhfei.hadoop.ch04.PooledStreamCompressor.java
License:Apache License
/** * use case: /*from w ww .ja v a2 s.c o m*/ * * @param args */ public static void main(String[] args) { String codecClassname = args[0]; Class<?> codecClass = null; CompressionOutputStream out = null; Compressor compressor = null; try { codecClass = Class.forName(codecClassname); Configuration conf = new Configuration(); CompressionCodec codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, conf); compressor = CodecPool.getCompressor(codec); out = codec.createOutputStream(System.out, compressor); IOUtils.copyBytes(System.in, out, 4096, false); out.finish(); } catch (ClassNotFoundException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } finally { CodecPool.returnCompressor(compressor); } }
From source file:cn.lhfei.hadoop.ch04.StreamCompressor.java
License:Apache License
/** * Use case: % echo "Text" | hadoop StreamCompressor org.apache.hadoop.io.compress.GzipCodec \ | gunzip - Text * @param args// ww w . j av a 2 s .c o m */ public static void main(String[] args) { String codecClassname = args[0]; try { Class<?> codecClass = Class.forName(codecClassname); Configuration conf = new Configuration(); CompressionCodec codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, conf); CompressionOutputStream out = codec.createOutputStream(System.out); IOUtils.copyBytes(System.in, out, 4096, false); out.finish(); } catch (ClassNotFoundException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } }
From source file:com.alexholmes.hdfsslurper.WorkerThread.java
License:Apache License
private void process(FileStatus srcFileStatus) throws IOException, InterruptedException { Path stagingFile = null;//from w w w. j av a 2 s. c om FileSystem destFs = null; String filenameBatchidDelimiter = config.getFileNameBatchIdDelimiter(); try { FileSystem srcFs = srcFileStatus.getPath().getFileSystem(config.getConfig()); // run a script which can change the name of the file as well as // write out a new version of the file // if (config.getWorkScript() != null) { Path newSrcFile = stageSource(srcFileStatus); srcFileStatus = srcFileStatus.getPath().getFileSystem(config.getConfig()).getFileStatus(newSrcFile); } Path srcFile = srcFileStatus.getPath(); // get the target HDFS file // Path destFile = getHdfsTargetPath(srcFileStatus); if (config.getCodec() != null) { String ext = config.getCodec().getDefaultExtension(); if (!destFile.getName().endsWith(ext)) { destFile = new Path(destFile.toString() + ext); } } destFs = destFile.getFileSystem(config.getConfig()); // get the staging HDFS file // stagingFile = fileSystemManager.getStagingFile(srcFileStatus, destFile); String batchId = srcFile.toString().substring( srcFile.toString().lastIndexOf(filenameBatchidDelimiter) + 1, srcFile.toString().length()); log.info("event#Copying source file '" + srcFile + "' to staging destination '" + stagingFile + "'" + "$batchId#" + batchId); // if the directory of the target file doesn't exist, attempt to // create it // Path destParentDir = destFile.getParent(); if (!destFs.exists(destParentDir)) { log.info("event#Attempting creation of target directory: " + destParentDir.toUri()); if (!destFs.mkdirs(destParentDir)) { throw new IOException("event#Failed to create target directory: " + destParentDir.toUri()); } } // if the staging directory doesn't exist, attempt to create it // Path destStagingParentDir = stagingFile.getParent(); if (!destFs.exists(destStagingParentDir)) { log.info("event#Attempting creation of staging directory: " + destStagingParentDir.toUri()); if (!destFs.mkdirs(destStagingParentDir)) { throw new IOException("event#Failed to create staging directory: " + destParentDir.toUri()); } } // copy the file // InputStream is = null; OutputStream os = null; CRC32 crc = new CRC32(); try { is = new BufferedInputStream(srcFs.open(srcFile)); if (config.isVerify()) { is = new CheckedInputStream(is, crc); } os = destFs.create(stagingFile); if (config.getCodec() != null) { os = config.getCodec().createOutputStream(os); } IOUtils.copyBytes(is, os, 4096, false); } finally { IOUtils.closeStream(is); IOUtils.closeStream(os); } long srcFileSize = srcFs.getFileStatus(srcFile).getLen(); long destFileSize = destFs.getFileStatus(stagingFile).getLen(); if (config.getCodec() == null && srcFileSize != destFileSize) { throw new IOException( "event#File sizes don't match, source = " + srcFileSize + ", dest = " + destFileSize); } log.info("event#Local file size = " + srcFileSize + ", HDFS file size = " + destFileSize + "$batchId#" + batchId); if (config.isVerify()) { verify(stagingFile, crc.getValue()); } if (destFs.exists(destFile)) { destFs.delete(destFile, false); } log.info("event#Moving staging file '" + stagingFile + "' to destination '" + destFile + "'" + "$batchId#" + batchId); if (!destFs.rename(stagingFile, destFile)) { throw new IOException("event#Failed to rename file"); } if (config.isCreateLzopIndex() && destFile.getName().endsWith(lzopExt)) { Path lzoIndexPath = new Path(destFile.toString() + LzoIndex.LZO_INDEX_SUFFIX); if (destFs.exists(lzoIndexPath)) { log.info("event#Deleting index file as it already exists"); destFs.delete(lzoIndexPath, false); } indexer.index(destFile); } fileSystemManager.fileCopyComplete(srcFileStatus); } catch (Throwable t) { log.error("event#Caught exception working on file " + srcFileStatus.getPath(), t); // delete the staging file if it still exists // try { if (destFs != null && destFs.exists(stagingFile)) { destFs.delete(stagingFile, false); } } catch (Throwable t2) { log.error("event#Failed to delete staging file " + stagingFile, t2); } fileSystemManager.fileCopyError(srcFileStatus); } }
From source file:com.asakusafw.directio.hive.parquet.ParquetFileFormatTest.java
License:Apache License
private <T> ModelInput<T> load(ParquetFileFormat<T> format, String name) throws IOException, InterruptedException { File target = folder.newFile(); try (InputStream in = getClass().getResourceAsStream(name)) { assertThat(in, is(notNullValue())); IOUtils.copyBytes(in, new FileOutputStream(target), 1024, true); }/* ww w . ja va 2 s .co m*/ FileSystem fs = FileSystem.getLocal(format.getConf()); return format.createInput(format.getSupportedType(), fs, new Path(target.toURI()), 0, -1, new Counter()); }
From source file:com.chinamobile.bcbsp.fault.tools.HdfsOperater.java
License:Apache License
/** * update file to hdfs//from w ww . j a v a 2 s. c o m * @param localPath * local file path to update * @param destPath * destination path in hdfs * @return upload result */ public static String uploadHdfs(String localPath, String destPath) { InputStream in = null; OutputStream out = null; try { String localSrc = localPath; File srcFile = new File(localSrc); if (srcFile.exists()) { // String dst = hostName + dirPath; in = new BufferedInputStream(new FileInputStream(localSrc)); // Configuration conf = new Configuration(); // FileSystem fs = FileSystem.get(URI.create(destPath), conf); BSPHdfs Hdfsup = new BSPHdfsImpl(); // out = fs.create(new Path(destPath), out = Hdfsup.hdfsOperater(destPath, Hdfsup.getConf()); // new Progressable() { // public void progress() { // // } // }); IOUtils.copyBytes(in, out, 4096, true); out.flush(); out.close(); in.close(); return destPath; } else { return "error"; } } catch (FileNotFoundException e) { LOG.error("[uploadHdfs]", e); return "error"; } catch (IOException e) { LOG.error("[uploadHdfs]", e); try { if (out != null) { out.flush(); } if (out != null) { out.close(); } if (in != null) { in.close(); } } catch (IOException e1) { LOG.error("[uploadHdfs]", e1); } return "error"; } }
From source file:com.ckelsel.hadoop.dfs.Test.Test.java
License:Open Source License
public static void main(String[] args) throws Exception { String uri = "hdfs://localhost:9000/"; Configuration config = new Configuration(); FileSystem fs = FileSystem.get(URI.create(uri), config); // hdfs/user/ckelsel/ FileStatus[] statuses = fs.listStatus(new Path("/user/ckelsel")); for (FileStatus status : statuses) { System.out.println(status); }//w ww .ja va2 s. com // hdfs/user/ckelsel FSDataOutputStream os = fs.create(new Path("/user/ckelsel/test.log")); os.write("Hello World!".getBytes()); os.flush(); os.close(); // hdfs/user/ckelsel InputStream is = fs.open(new Path("/user/ckelsel/test.log")); IOUtils.copyBytes(is, System.out, 1024, true); }
From source file:com.datasalt.pangool.solr.SolrRecordWriter.java
License:Apache License
/** * Write a file to a zip output stream, removing leading path name components from the actual file name when creating * the zip file entry.//from w w w. j a v a 2 s.c o m * * The entry placed in the zip file is <code>baseName</code>/ <code>relativePath</code>, where * <code>relativePath</code> is constructed by removing a leading <code>root</code> from the path for * <code>itemToZip</code>. * * If <code>itemToZip</code> is an empty directory, it is ignored. If <code>itemToZip</code> is a directory, the * contents of the directory are added recursively. * * @param zos * The zip output stream * @param baseName * The base name to use for the file name entry in the zip file * @param root * The path to remove from <code>itemToZip</code> to make a relative path name * @param itemToZip * The path to the file to be added to the zip file * @return the number of entries added * @throws IOException */ static public int zipDirectory(final Configuration conf, final ZipOutputStream zos, final String baseName, final String root, final Path itemToZip) throws IOException { LOG.info(String.format("zipDirectory: %s %s %s", baseName, root, itemToZip)); LocalFileSystem localFs = FileSystem.getLocal(conf); int count = 0; final FileStatus itemStatus = localFs.getFileStatus(itemToZip); if (itemStatus.isDir()) { final FileStatus[] statai = localFs.listStatus(itemToZip); // Add a directory entry to the zip file final String zipDirName = relativePathForZipEntry(itemToZip.toUri().getPath(), baseName, root); final ZipEntry dirZipEntry = new ZipEntry(zipDirName + Path.SEPARATOR_CHAR); LOG.info(String.format("Adding directory %s to zip", zipDirName)); zos.putNextEntry(dirZipEntry); zos.closeEntry(); count++; if (statai == null || statai.length == 0) { LOG.info(String.format("Skipping empty directory %s", itemToZip)); return count; } for (FileStatus status : statai) { count += zipDirectory(conf, zos, baseName, root, status.getPath()); } LOG.info(String.format("Wrote %d entries for directory %s", count, itemToZip)); return count; } final String inZipPath = relativePathForZipEntry(itemToZip.toUri().getPath(), baseName, root); if (inZipPath.length() == 0) { LOG.warn(String.format("Skipping empty zip file path for %s (%s %s)", itemToZip, root, baseName)); return 0; } // Take empty files in case the place holder is needed FSDataInputStream in = null; try { in = localFs.open(itemToZip); final ZipEntry ze = new ZipEntry(inZipPath); ze.setTime(itemStatus.getModificationTime()); // Comments confuse looking at the zip file // ze.setComment(itemToZip.toString()); zos.putNextEntry(ze); IOUtils.copyBytes(in, zos, conf, false); zos.closeEntry(); LOG.info(String.format("Wrote %d entries for file %s", count, itemToZip)); return 1; } finally { in.close(); } }
From source file:com.datatorrent.common.util.AsyncFSStorageAgent.java
License:Apache License
public void copyToHDFS(final int operatorId, final long windowId) throws IOException { if (this.localBasePath == null) { throw new AssertionError("save() was not called before copyToHDFS"); }/*from w w w .j ava 2 s. c om*/ String operatorIdStr = String.valueOf(operatorId); File directory = new File(localBasePath, operatorIdStr); String window = Long.toHexString(windowId); Path lPath = new Path(path + Path.SEPARATOR + operatorIdStr + Path.SEPARATOR + TMP_FILE); File srcFile = new File(directory, String.valueOf(windowId)); FSDataOutputStream stream = null; boolean stateSaved = false; try { // Create the temporary file with OverWrite option to avoid dangling lease issue and avoid exception if file already exists stream = fileContext.create(lPath, EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE), Options.CreateOpts.CreateParent.createParent()); InputStream in = null; try { in = new FileInputStream(srcFile); IOUtils.copyBytes(in, stream, conf, false); } finally { IOUtils.closeStream(in); } stateSaved = true; } catch (Throwable t) { logger.debug("while saving {} {}", operatorId, window, t); stateSaved = false; throw Throwables.propagate(t); } finally { try { if (stream != null) { stream.close(); } } catch (IOException ie) { stateSaved = false; throw new RuntimeException(ie); } finally { if (stateSaved) { fileContext.rename(lPath, new Path(path + Path.SEPARATOR + operatorIdStr + Path.SEPARATOR + window), Options.Rename.OVERWRITE); } FileUtil.fullyDelete(srcFile); } } }
From source file:com.davidgildeh.hadoop.utils.FileUtils.java
License:Apache License
/** * Merges a list of input files in a directory to a single file under the * outputpath with a specified filename// w w w.j a v a 2 s .c o m * * @param inputPath The input directory containing all the input files. E.g. /input/dir/on/hdfs/ * @param outputPath The output path to output the file. E.g. /output/dir/on/hdfs/filename * @throws IOException */ public static void mergeFiles(String inputPath, String outputPath) throws IOException { Path inputDir = new Path(inputPath); Path outputFile = new Path(outputPath); FileSystem fileSystem = getFileSystem(outputFile); checkFileExists(fileSystem, inputDir); // Check the input path is a directory if (!fileSystem.getFileStatus(inputDir).isDir()) { LOG.error("Path '" + inputDir.toString() + "' is not a directory."); throw new IOException("Path '" + inputDir.toString() + "' is not a directory."); } // Create Output File OutputStream out = fileSystem.create(outputFile); try { FileStatus contents[] = fileSystem.listStatus(inputDir); // Loop through all files in directory and merge them into one file for (int i = 0; i < contents.length; i++) { if (!contents[i].isDir()) { InputStream in = fileSystem.open(contents[i].getPath()); try { IOUtils.copyBytes(in, out, fileSystem.getConf(), false); } finally { in.close(); } } } } finally { out.close(); fileSystem.close(); LOG.info("Merged input files from '" + inputPath + "' to '" + outputPath + "'"); } }