List of usage examples for org.apache.hadoop.io.compress CompressionCodec createOutputStream
CompressionOutputStream createOutputStream(OutputStream out, Compressor compressor) throws IOException;
From source file:cn.lhfei.hadoop.ch04.PooledStreamCompressor.java
License:Apache License
/** * use case: /* w w w . j av a 2 s.co m*/ * * @param args */ public static void main(String[] args) { String codecClassname = args[0]; Class<?> codecClass = null; CompressionOutputStream out = null; Compressor compressor = null; try { codecClass = Class.forName(codecClassname); Configuration conf = new Configuration(); CompressionCodec codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, conf); compressor = CodecPool.getCompressor(codec); out = codec.createOutputStream(System.out, compressor); IOUtils.copyBytes(System.in, out, 4096, false); out.finish(); } catch (ClassNotFoundException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } finally { CodecPool.returnCompressor(compressor); } }
From source file:com.cloudera.flume.handlers.hdfs.CustomDfsSink.java
License:Apache License
/** * Hadoop Compression Codecs that use Native libs require an instance of a * Configuration Object. They require this due to some check against knowing * weather or not the native libs have been loaded. GzipCodec, LzoCodec, * LzopCodec are all codecs that require Native libs. GZipCodec has a slight * exception that if native libs are not accessible it will use Pure Java. * This results in no errors just notices. BZip2Codec is an example codec that * doesn't use native libs./*from w ww . j a v a 2s . c o m*/ */ @Override public void open() throws IOException { FlumeConfiguration conf = FlumeConfiguration.get(); FileSystem hdfs; String codecName = conf.getCollectorDfsCompressCodec(); CompressionCodec codec = getCodec(conf, codecName); if (codec == null) { dstPath = new Path(path); hdfs = dstPath.getFileSystem(conf); pathManager = new PathManager(hdfs, dstPath.getParent(), dstPath.getName()); writer = pathManager.open(); LOG.info("Creating HDFS file: " + pathManager.getOpenPath()); return; } Compressor cmp = codec.createCompressor(); dstPath = new Path(path + codec.getDefaultExtension()); hdfs = dstPath.getFileSystem(conf); pathManager = new PathManager(hdfs, dstPath.getParent(), dstPath.getName()); writer = pathManager.open(); try { writer = codec.createOutputStream(writer, cmp); } catch (NullPointerException npe) { // tries to find "native" version of codec, if that fails, then tries to // find java version. If there is no java version, the createOutputStream // exits via NPE. We capture this and convert it into a IOE with a more // useful error message. LOG.error("Unable to load compression codec " + codec); throw new IOException("Unable to load compression codec " + codec); } LOG.info("Creating " + codec + " compressed HDFS file: " + pathManager.getOpenPath()); }
From source file:com.flipkart.fdp.migration.distcp.core.MirrorUtils.java
License:Apache License
public static OutputStream getCodecOutputStream(Configuration conf, String codecName, OutputStream out) throws IOException { CompressionCodecFactory compressionCodecs = new CompressionCodecFactory(conf); String codecClassName = codecName; CodecType codecType = CodecType.getCodecType(codecName); if (codecType != null) { codecClassName = codecType.getIOCompressionCodecs(); }// w ww. j a v a2 s. c o m System.out.println("codec class : " + codecClassName); CompressionCodec codec = compressionCodecs.getCodecByName(codecClassName); if (codec == null) { return out; } System.out.println("Getting OutputStream : " + codec.getDefaultExtension()); System.out.println("Getting OutputStream : " + codec); Compressor compressor = codec.createCompressor(); return codec.createOutputStream(out, compressor); }
From source file:com.hadoop.mapreduce.TestLzoLazyLoading.java
License:Open Source License
public static Path writeFile(String name, String data) throws IOException { Path file = new Path(TEST_ROOT_DIR + "/" + name); localFs.delete(file, false);/*from w w w . jav a 2 s . c o m*/ CompressionCodec codec = new CompressionCodecFactory(conf).getCodec(file); OutputStream f; Compressor compressor = null; if (codec == null) { f = localFs.create(file); } else { compressor = CodecPool.getCompressor(codec); f = codec.createOutputStream(localFs.create(file), compressor); } f.write(data.getBytes()); f.close(); if (compressor != null) { CodecPool.returnCompressor(compressor); } return file; }
From source file:com.jeffy.hdfs.compression.FileCompressor.java
License:Apache License
/** * @param args//from ww w .ja v a 2 s. c o m * ?????? * ???? * @throws IOException */ public static void main(String[] args) throws IOException { Configuration conf = new Configuration(); //?? CompressionCodecFactory factory = new CompressionCodecFactory(conf); // For example for the 'GzipCodec' codec class name the alias are 'gzip' and 'gzipcodec'. CompressionCodec codec = factory.getCodecByName(args[0]); if (codec == null) {//??? System.err.println("Comperssion codec not found for " + args[0]); System.exit(1); } String ext = codec.getDefaultExtension(); Compressor compressor = null; try { //?CodecPool?Compressor compressor = CodecPool.getCompressor(codec); for (int i = 1; i < args.length; i++) { String filename = args[i] + ext; System.out.println("Compression the file " + filename); try (FileSystem outFs = FileSystem.get(URI.create(filename), conf); FileSystem inFs = FileSystem.get(URI.create(args[i]), conf); InputStream in = inFs.open(new Path(args[i]))) {// //Compressor? CompressionOutputStream out = codec.createOutputStream(outFs.create(new Path(filename)), compressor); //????? IOUtils.copy(in, out); out.finish();//?finish()?flush()??? compressor.reset(); //???????java.io.IOException: write beyond end of stream } } } finally {//?Compressor?? CodecPool.returnCompressor(compressor); } }
From source file:crunch.MaxTemperature.java
License:Apache License
public static void main(String[] args) throws Exception { String codecClassname = args[0]; Class<?> codecClass = Class.forName(codecClassname); Configuration conf = new Configuration(); CompressionCodec codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, conf); /*[*/Compressor compressor = null; try {//w w w . j a va 2 s .c o m compressor = CodecPool.getCompressor(codec);/*]*/ CompressionOutputStream out = codec.createOutputStream(System.out, /*[*/compressor/*]*/); IOUtils.copyBytes(System.in, out, 4096, false); out.finish(); /*[*/} finally { CodecPool.returnCompressor(compressor); } /*]*/ }
From source file:io.aos.hdfs.PooledStreamCompressor.java
License:Apache License
public static void main(String... args) throws Exception { String codecClassname = args[0]; Class<?> codecClass = Class.forName(codecClassname); Configuration conf = new Configuration(); CompressionCodec codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, conf); /*[*/Compressor compressor = null; try {/*from w w w. java2 s. c o m*/ compressor = CodecPool.getCompressor(codec);/*]*/ CompressionOutputStream out = codec.createOutputStream(System.out, /*[*/compressor/*]*/); IOUtils.copyBytes(System.in, out, 4096, false); out.finish(); /*[*/} finally { CodecPool.returnCompressor(compressor); } /*]*/ }
From source file:io.transwarp.flume.sink.HDFSCompressedDataStream.java
License:Apache License
@Override public void open(String filePath, CompressionCodec codec, CompressionType cType) throws IOException { Configuration conf = new Configuration(); Path dstPath = new Path(filePath); FileSystem hdfs = dstPath.getFileSystem(conf); if (useRawLocalFileSystem) { if (hdfs instanceof LocalFileSystem) { hdfs = ((LocalFileSystem) hdfs).getRaw(); } else {/* w w w .j a va 2s .com*/ logger.warn("useRawLocalFileSystem is set to true but file system " + "is not of type LocalFileSystem: " + hdfs.getClass().getName()); } } boolean appending = false; if (conf.getBoolean("hdfs.append.support", false) == true && hdfs.isFile(dstPath)) { fsOut = hdfs.append(dstPath); appending = true; } else { fsOut = hdfs.create(dstPath); } if (compressor == null) { compressor = CodecPool.getCompressor(codec, conf); } cmpOut = codec.createOutputStream(fsOut, compressor); serializer = EventSerializerFactory.getInstance(serializerType, serializerContext, cmpOut); if (appending && !serializer.supportsReopen()) { cmpOut.close(); serializer = null; throw new IOException("serializer (" + serializerType + ") does not support append"); } registerCurrentStream(fsOut, hdfs, dstPath); if (appending) { serializer.afterReopen(); } else { serializer.afterCreate(); } isFinished = false; }
From source file:org.apache.sqoop.connector.hdfs.hdfsWriter.HdfsTextWriter.java
License:Apache License
@Override public void initialize(Path filepath, Configuration conf, CompressionCodec codec) throws IOException { FileSystem fs = filepath.getFileSystem(conf); DataOutputStream filestream = fs.create(filepath, false); if (codec != null) { filewriter = new BufferedWriter(new OutputStreamWriter( codec.createOutputStream(filestream, codec.createCompressor()), Charsets.UTF_8)); } else {/*from w w w .j a v a 2 s .co m*/ filewriter = new BufferedWriter(new OutputStreamWriter(filestream, Charsets.UTF_8)); } }
From source file:org.apache.sqoop.connector.hdfs.TestHdfsBase.java
License:Apache License
protected void createTextInput(String indir, Class<? extends CompressionCodec> clz, int numberOfFiles, int numberOfRows) throws IOException, InstantiationException, IllegalAccessException { Configuration conf = new Configuration(); CompressionCodec codec = null; String extension = ""; if (clz != null) { codec = clz.newInstance();//from www .ja v a 2 s .c o m if (codec instanceof Configurable) { ((Configurable) codec).setConf(conf); } extension = codec.getDefaultExtension(); } int index = 1; for (int fi = 0; fi < numberOfFiles; fi++) { String fileName = indir + "/" + UUID.randomUUID() + extension; OutputStream filestream = FileUtils.create(fileName); BufferedWriter filewriter; if (codec != null) { filewriter = new BufferedWriter(new OutputStreamWriter( codec.createOutputStream(filestream, codec.createCompressor()), "UTF-8")); } else { filewriter = new BufferedWriter(new OutputStreamWriter(filestream, "UTF-8")); } for (int ri = 0; ri < numberOfRows; ri++) { String row = index + "," + (double) index + ",'" + index + "'"; filewriter.write(row + HdfsConstants.DEFAULT_RECORD_DELIMITER); index++; } filewriter.close(); } }