List of usage examples for org.apache.hadoop.io.compress CodecPool getCompressor
public static Compressor getCompressor(CompressionCodec codec)
From source file:cascading.tuple.hadoop.collect.HadoopSpillableTupleList.java
License:Open Source License
private Compressor getCompressor() { // some codecs are using direct memory, and the gc for direct memory cannot sometimes keep up // so we attempt to force a gc if we see a OOME once. try {/*from www . j a v a 2s. c o m*/ return CodecPool.getCompressor(codec); } catch (OutOfMemoryError error) { System.gc(); LOG.info("received OOME when allocating compressor for codec: {}, retrying once", codec.getClass().getCanonicalName(), error); return CodecPool.getCompressor(codec); } }
From source file:cn.lhfei.hadoop.ch04.PooledStreamCompressor.java
License:Apache License
/** * use case: /*from w w w . ja va 2s . co m*/ * * @param args */ public static void main(String[] args) { String codecClassname = args[0]; Class<?> codecClass = null; CompressionOutputStream out = null; Compressor compressor = null; try { codecClass = Class.forName(codecClassname); Configuration conf = new Configuration(); CompressionCodec codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, conf); compressor = CodecPool.getCompressor(codec); out = codec.createOutputStream(System.out, compressor); IOUtils.copyBytes(System.in, out, 4096, false); out.finish(); } catch (ClassNotFoundException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } finally { CodecPool.returnCompressor(compressor); } }
From source file:com.hadoop.mapreduce.TestLzoLazyLoading.java
License:Open Source License
public static Path writeFile(String name, String data) throws IOException { Path file = new Path(TEST_ROOT_DIR + "/" + name); localFs.delete(file, false);//from ww w . j a va 2 s. co m CompressionCodec codec = new CompressionCodecFactory(conf).getCodec(file); OutputStream f; Compressor compressor = null; if (codec == null) { f = localFs.create(file); } else { compressor = CodecPool.getCompressor(codec); f = codec.createOutputStream(localFs.create(file), compressor); } f.write(data.getBytes()); f.close(); if (compressor != null) { CodecPool.returnCompressor(compressor); } return file; }
From source file:com.inmobi.conduit.distcp.tools.mapred.TestCopyMapper.java
License:Apache License
private static void touchFile(String path) throws Exception { FileSystem fs;//ww w.j a va 2 s .c o m DataOutputStream outputStream = null; GzipCodec gzipCodec = ReflectionUtils.newInstance(GzipCodec.class, getConfiguration()); Compressor gzipCompressor = CodecPool.getCompressor(gzipCodec); OutputStream compressedOut = null; try { fs = cluster.getFileSystem(); final Path qualifiedPath = new Path(path).makeQualified(fs); final long blockSize = fs.getDefaultBlockSize() * 2; outputStream = fs.create(qualifiedPath, true, 0, (short) (fs.getDefaultReplication() * 2), blockSize); compressedOut = gzipCodec.createOutputStream(outputStream, gzipCompressor); Message msg = new Message("generating test data".getBytes()); AuditUtil.attachHeaders(msg, currentTimestamp); byte[] encodeMsg = Base64.encodeBase64(msg.getData().array()); compressedOut.write(encodeMsg); compressedOut.write("\n".getBytes()); compressedOut.write(encodeMsg); compressedOut.write("\n".getBytes()); // Genearate a msg with different timestamp. Default window period is 60sec AuditUtil.attachHeaders(msg, nextMinuteTimeStamp); encodeMsg = Base64.encodeBase64(msg.getData().array()); compressedOut.write(encodeMsg); compressedOut.write("\n".getBytes()); compressedOut.flush(); compressedOut.close(); pathList.add(qualifiedPath); ++nFiles; FileStatus fileStatus = fs.getFileStatus(qualifiedPath); System.out.println(fileStatus.getBlockSize()); System.out.println(fileStatus.getReplication()); } finally { compressedOut.close(); IOUtils.cleanup(null, outputStream); CodecPool.returnCompressor(gzipCompressor); } }
From source file:com.inmobi.conduit.distcp.tools.TestDistCp.java
License:Apache License
private static void touchFile(String path) throws Exception { FileSystem fs;/*from w w w .j a v a 2 s.co m*/ DataOutputStream outputStream = null; GzipCodec gzipCodec = ReflectionUtils.newInstance(GzipCodec.class, getConfigurationForCluster()); Compressor gzipCompressor = CodecPool.getCompressor(gzipCodec); OutputStream compressedOut = null; try { fs = cluster.getFileSystem(); final Path qualifiedPath = new Path(path).makeQualified(fs); final long blockSize = fs.getDefaultBlockSize() * 2; outputStream = fs.create(qualifiedPath, true, 0, (short) (fs.getDefaultReplication() * 2), blockSize); compressedOut = gzipCodec.createOutputStream(outputStream, gzipCompressor); compressedOut.write(new byte[FILE_SIZE]); compressedOut.write("\n".getBytes()); compressedOut.flush(); //outputStream.write(new byte[FILE_SIZE]); pathList.add(qualifiedPath); } finally { compressedOut.close(); IOUtils.cleanup(null, outputStream); CodecPool.returnCompressor(gzipCompressor); } }
From source file:com.inmobi.conduit.distcp.tools.TestIntegration.java
License:Apache License
private void createFileForAudit(String... entries) throws IOException { for (String entry : entries) { OutputStream out = fs.create(new Path(root + "/" + entry)); GzipCodec gzipCodec = ReflectionUtils.newInstance(GzipCodec.class, new Configuration()); Compressor gzipCompressor = CodecPool.getCompressor(gzipCodec); OutputStream compressedOut = null; try {/* w ww . j av a2 s. co m*/ compressedOut = gzipCodec.createOutputStream(out, gzipCompressor); Message msg = new Message((root + "/" + entry).getBytes()); long currentTimestamp = new Date().getTime(); AuditUtil.attachHeaders(msg, currentTimestamp); byte[] encodeMsg = Base64.encodeBase64(msg.getData().array()); compressedOut.write(encodeMsg); compressedOut.write("\n".getBytes()); compressedOut.write(encodeMsg); compressedOut.write("\n".getBytes()); /* add a different timestamp to the msg header. * The generation time stamp falls in diff window * Each file will be having two counters */ AuditUtil.attachHeaders(msg, currentTimestamp + 60001); encodeMsg = Base64.encodeBase64(msg.getData().array()); compressedOut.write(encodeMsg); compressedOut.write("\n".getBytes()); compressedOut.flush(); } finally { compressedOut.close(); out.close(); CodecPool.returnCompressor(gzipCompressor); } } }
From source file:com.inmobi.conduit.distcp.tools.TestIntegration.java
License:Apache License
private void createFiles(String... entries) throws IOException { for (String entry : entries) { OutputStream out = fs.create(new Path(root + "/" + entry)); GzipCodec gzipCodec = ReflectionUtils.newInstance(GzipCodec.class, new Configuration()); Compressor gzipCompressor = CodecPool.getCompressor(gzipCodec); OutputStream compressedOut = null; try {/*from w ww . ja va 2 s .c o m*/ compressedOut = gzipCodec.createOutputStream(out, gzipCompressor); compressedOut.write((root + "/" + entry).getBytes()); compressedOut.write("\n".getBytes()); } finally { compressedOut.close(); out.close(); CodecPool.returnCompressor(gzipCompressor); } } }
From source file:com.inmobi.messaging.consumer.util.FileUtil.java
License:Apache License
public static void gzip(Path src, Path target, Configuration conf) throws IOException { FileSystem fs = FileSystem.get(conf); FSDataOutputStream out = fs.create(target); GzipCodec gzipCodec = (GzipCodec) ReflectionUtils.newInstance(GzipCodec.class, conf); Compressor gzipCompressor = CodecPool.getCompressor(gzipCodec); OutputStream compressedOut = gzipCodec.createOutputStream(out, gzipCompressor); FSDataInputStream in = fs.open(src); try {/*from ww w . j a va 2 s . co m*/ IOUtils.copyBytes(in, compressedOut, conf); } catch (Exception e) { LOG.error("Error in compressing ", e); } finally { in.close(); CodecPool.returnCompressor(gzipCompressor); compressedOut.close(); out.close(); } }
From source file:com.jeffy.hdfs.compression.FileCompressor.java
License:Apache License
/** * @param args// w w w .java2 s.com * ?????? * ???? * @throws IOException */ public static void main(String[] args) throws IOException { Configuration conf = new Configuration(); //?? CompressionCodecFactory factory = new CompressionCodecFactory(conf); // For example for the 'GzipCodec' codec class name the alias are 'gzip' and 'gzipcodec'. CompressionCodec codec = factory.getCodecByName(args[0]); if (codec == null) {//??? System.err.println("Comperssion codec not found for " + args[0]); System.exit(1); } String ext = codec.getDefaultExtension(); Compressor compressor = null; try { //?CodecPool?Compressor compressor = CodecPool.getCompressor(codec); for (int i = 1; i < args.length; i++) { String filename = args[i] + ext; System.out.println("Compression the file " + filename); try (FileSystem outFs = FileSystem.get(URI.create(filename), conf); FileSystem inFs = FileSystem.get(URI.create(args[i]), conf); InputStream in = inFs.open(new Path(args[i]))) {// //Compressor? CompressionOutputStream out = codec.createOutputStream(outFs.create(new Path(filename)), compressor); //????? IOUtils.copy(in, out); out.finish();//?finish()?flush()??? compressor.reset(); //???????java.io.IOException: write beyond end of stream } } } finally {//?Compressor?? CodecPool.returnCompressor(compressor); } }
From source file:crunch.MaxTemperature.java
License:Apache License
public static void main(String[] args) throws Exception { String codecClassname = args[0]; Class<?> codecClass = Class.forName(codecClassname); Configuration conf = new Configuration(); CompressionCodec codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, conf); /*[*/Compressor compressor = null; try {// w ww. j ava2s.c o m compressor = CodecPool.getCompressor(codec);/*]*/ CompressionOutputStream out = codec.createOutputStream(System.out, /*[*/compressor/*]*/); IOUtils.copyBytes(System.in, out, 4096, false); out.finish(); /*[*/} finally { CodecPool.returnCompressor(compressor); } /*]*/ }