List of usage examples for org.apache.commons.compress.compressors.gzip GzipCompressorOutputStream write
public void write(byte[] b) throws IOException
From source file:divconq.util.IOUtil.java
public static byte[] charsEncodeAndCompress(CharSequence v) { try {/*from www . j a v a 2 s .co m*/ byte[] buffer = Utf8Encoder.encode(v); ByteArrayOutputStream bos = new ByteArrayOutputStream(); GzipCompressorOutputStream cos = new GzipCompressorOutputStream(bos); cos.write(buffer); cos.close(); bos.close(); return bos.toByteArray(); } catch (Exception x) { } return null; }
From source file:com.redhat.red.offliner.ftest.SinglePlaintextDownloadOfTarballFTest.java
@Test public void testGenericTarballDownload() throws Exception { // Generate some test content String path = contentGenerator.newArtifactPath("tar.gz"); Map<String, byte[]> entries = new HashMap<>(); entries.put(contentGenerator.newArtifactPath("jar"), contentGenerator.newBinaryContent(2400)); entries.put(contentGenerator.newArtifactPath("jar"), contentGenerator.newBinaryContent(2400)); final File tgz = makeTarball(entries); System.out.println("tar content array has length: " + tgz.length()); // We only need one repo server. ExpectationServer server = new ExpectationServer(); server.start();// w w w.j ava2 s . c o m String url = server.formatUrl(path); // Register the generated content by writing it to the path within the repo server's dir structure. // This way when the path is requested it can be downloaded instead of returning a 404. server.expect("GET", url, (req, resp) -> { // Content-Length: 47175 // Content-Type: application/x-gzip resp.setHeader("Content-Encoding", "x-gzip"); resp.setHeader("Content-Type", "application/x-gzip"); byte[] raw = FileUtils.readFileToByteArray(tgz); ByteArrayOutputStream baos = new ByteArrayOutputStream(); GzipCompressorOutputStream gzout = new GzipCompressorOutputStream(baos); gzout.write(raw); gzout.finish(); byte[] content = baos.toByteArray(); resp.setHeader("Content-Length", Long.toString(content.length)); OutputStream respStream = resp.getOutputStream(); respStream.write(content); respStream.flush(); System.out.println("Wrote content with length: " + content.length); }); final PoolingHttpClientConnectionManager ccm = new PoolingHttpClientConnectionManager(); ccm.setMaxTotal(1); final HttpClientBuilder builder = HttpClients.custom().setConnectionManager(ccm); CloseableHttpClient client = builder.build(); HttpGet get = new HttpGet(url); // get.setHeader( "Accept-Encoding", "gzip,deflate" ); Boolean result = client.execute(get, (response) -> { Arrays.stream(response.getAllHeaders()).forEach((h) -> System.out.println("Header:: " + h)); Header contentEncoding = response.getEntity().getContentEncoding(); if (contentEncoding == null) { contentEncoding = response.getFirstHeader("Content-Encoding"); } System.out.printf("Got content encoding: %s\n", contentEncoding == null ? "None" : contentEncoding.getValue()); byte[] content = IOUtils.toByteArray(response.getEntity().getContent()); try (TarArchiveInputStream tarIn = new TarArchiveInputStream( new GzipCompressorInputStream(new ByteArrayInputStream(content)))) { TarArchiveEntry entry = null; while ((entry = tarIn.getNextTarEntry()) != null) { System.out.printf("Got tar entry: %s\n", entry.getName()); byte[] entryData = new byte[(int) entry.getSize()]; int read = tarIn.read(entryData, 0, entryData.length); } } return false; }); }
From source file:com.redhat.red.offliner.ftest.SinglePlaintextDownloadOfTarballFTest.java
/** * In general, we should only have one test method per functional test. This allows for the best parallelism when we * execute the tests, especially if the setup takes some time. * * @throws Exception In case anything (anything at all) goes wrong! *///from w w w. j ava 2 s . co m @Test public void run() throws Exception { // Generate some test content String path = contentGenerator.newArtifactPath("tar.gz"); Map<String, byte[]> entries = new HashMap<>(); entries.put(contentGenerator.newArtifactPath("jar"), contentGenerator.newBinaryContent(2400)); entries.put(contentGenerator.newArtifactPath("jar"), contentGenerator.newBinaryContent(2400)); final File tgz = makeTarball(entries); System.out.println("tar content array has length: " + tgz.length()); // We only need one repo server. ExpectationServer server = new ExpectationServer(); server.start(); // Register the generated content by writing it to the path within the repo server's dir structure. // This way when the path is requested it can be downloaded instead of returning a 404. server.expect("GET", server.formatUrl(path), (req, resp) -> { // Content-Length: 47175 // Content-Type: application/x-gzip resp.setHeader("Content-Encoding", "gzip"); resp.setHeader("Content-Type", "application/x-gzip"); byte[] raw = FileUtils.readFileToByteArray(tgz); ByteArrayOutputStream baos = new ByteArrayOutputStream(); GzipCompressorOutputStream gzout = new GzipCompressorOutputStream(baos); gzout.write(raw); gzout.finish(); byte[] content = baos.toByteArray(); resp.setHeader("Content-Length", Long.toString(content.length)); OutputStream respStream = resp.getOutputStream(); respStream.write(content); respStream.flush(); System.out.println("Wrote content with length: " + content.length); }); byte[] content = FileUtils.readFileToByteArray(tgz); server.expect("GET", server.formatUrl(path + Main.SHA_SUFFIX), 200, sha1Hex(content)); server.expect("GET", server.formatUrl(path + Main.MD5_SUFFIX), 200, md5Hex(content)); // Write the plaintext file we'll use as input. File plaintextList = temporaryFolder.newFile("artifact-list." + getClass().getSimpleName() + ".txt"); String pathWithChecksum = contentGenerator.newPlaintextEntryWithChecksum(path, content); FileUtils.write(plaintextList, pathWithChecksum); Options opts = new Options(); opts.setBaseUrls(Collections.singletonList(server.getBaseUri())); // Capture the downloads here so we can verify the content. File downloads = temporaryFolder.newFolder(); opts.setDownloads(downloads); opts.setLocations(Collections.singletonList(plaintextList.getAbsolutePath())); opts.setConnections(1); // run `new Main(opts).run()` and return the Main instance so we can query it for errors, etc. Main finishedMain = run(opts); ConcurrentHashMap<String, Throwable> errors = finishedMain.getErrors(); System.out.printf("ERRORS:\n\n%s\n\n\n", StringUtils.join(errors.keySet().stream() .map(k -> "ERROR: " + k + ": " + errors.get(k).getMessage() + "\n " + StringUtils.join(errors.get(k).getStackTrace(), "\n ")) .collect(Collectors.toList()), "\n\n")); File downloaded = new File(downloads, path); assertThat("File: " + path + " doesn't seem to have been downloaded!", downloaded.exists(), equalTo(true)); // assertThat( "Downloaded file: " + path + " contains the wrong content!", // FileUtils.readFileToByteArray( downloaded ), equalTo( content ) ); File tarball = new File(downloads, path); System.out.println("Length of downloaded file: " + tarball.length()); File tmp = new File("/tmp/download.tar.gz"); File tmp2 = new File("/tmp/content.tar.gz"); FileUtils.writeByteArrayToFile(tmp2, content); FileUtils.copyFile(tarball, tmp); try (TarArchiveInputStream tarIn = new TarArchiveInputStream( new GzipCompressorInputStream(new FileInputStream(tarball)))) { TarArchiveEntry entry = null; while ((entry = tarIn.getNextTarEntry()) != null) { byte[] entryData = new byte[(int) entry.getSize()]; int read = tarIn.read(entryData, 0, entryData.length); assertThat("Not enough bytes read for: " + entry.getName(), read, equalTo((int) entry.getSize())); assertThat(entry.getName() + ": data doesn't match input", Arrays.equals(entries.get(entry.getName()), entryData), equalTo(true)); } } assertThat("Wrong number of downloads logged. Should have been 3 including checksums.", finishedMain.getDownloaded(), equalTo(3)); assertThat("Errors should be empty!", errors.isEmpty(), equalTo(true)); }
From source file:com.chenshu.compress.CompressOldTest.java
@Benchmark public int commonsGzipCompress() { ByteArrayOutputStream bout = null; GzipCompressorOutputStream gzout = null; try {/*from w w w . java 2 s . c o m*/ GzipParameters p = new GzipParameters(); p.setCompressionLevel(level); bout = new ByteArrayOutputStream(data.length); gzout = new GzipCompressorOutputStream(bout, p); gzout.write(data); } catch (IOException e) { e.printStackTrace(); } finally { if (gzout != null) { try { gzout.close(); } catch (IOException e) { e.printStackTrace(); } } if (bout != null) { try { bout.close(); } catch (IOException e) { e.printStackTrace(); } } } byte[] bs = bout.toByteArray(); return bs.length; }
From source file:org.apache.carbondata.core.datastore.compression.GzipCompressor.java
/** * This method takes the Byte Array data and Compresses in gzip format * * @param data Data Byte Array passed for compression * @return Compressed Byte Array//from w w w . j a v a 2 s .c o m */ private byte[] compressData(byte[] data) { int initialSize = (data.length / 2) == 0 ? data.length : data.length / 2; ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(initialSize); try { GzipCompressorOutputStream gzipCompressorOutputStream = new GzipCompressorOutputStream( byteArrayOutputStream); try { /** * Below api will write bytes from specified byte array to the gzipCompressorOutputStream * The output stream will compress the given byte array. */ gzipCompressorOutputStream.write(data); } catch (IOException e) { throw new RuntimeException("Error during Compression writing step ", e); } finally { gzipCompressorOutputStream.close(); } } catch (IOException e) { throw new RuntimeException("Error during Compression step ", e); } return byteArrayOutputStream.toByteArray(); }