List of usage examples for java.util.zip Deflater BEST_SPEED
int BEST_SPEED
To view the source code for java.util.zip Deflater BEST_SPEED.
Click Source Link
From source file:be.ibridge.kettle.job.entry.zipfile.JobEntryZipFile.java
public Result execute(Result prev_result, int nr, Repository rep, Job parentJob) { LogWriter log = LogWriter.getInstance(); Result result = new Result(nr); result.setResult(false);/* w ww . ja v a2 s. c o m*/ boolean Fileexists = false; String realZipfilename = StringUtil.environmentSubstitute(zipFilename); String realWildcard = StringUtil.environmentSubstitute(wildcard); String realWildcardExclude = StringUtil.environmentSubstitute(wildcardexclude); String realTargetdirectory = StringUtil.environmentSubstitute(sourcedirectory); String realMovetodirectory = StringUtil.environmentSubstitute(movetodirectory); if (realZipfilename != null) { FileObject fileObject = null; try { fileObject = KettleVFS.getFileObject(realZipfilename); // Check if Zip File exists if (fileObject.exists()) { Fileexists = true; log.logDebug(toString(), Messages.getString("JobZipFiles.Zip_FileExists1.Label") + realZipfilename + Messages.getString("JobZipFiles.Zip_FileExists2.Label")); } else { Fileexists = false; } // Let's start the process now if (ifzipfileexists == 3 && Fileexists) { // the zip file exists and user want to Fail result.setResult(false); result.setNrErrors(1); } else if (ifzipfileexists == 2 && Fileexists) { // the zip file exists and user want to do nothing result.setResult(true); } else if (afterzip == 2 && realMovetodirectory == null) { // After Zip, Move files..User must give a destination Folder result.setResult(false); result.setNrErrors(1); log.logError(toString(), Messages.getString("JobZipFiles.AfterZip_No_DestinationFolder_Defined.Label")); } else // After Zip, Move files..User must give a destination Folder { if (ifzipfileexists == 0 && Fileexists) { // the zip file exists and user want to create new one with unique name //Format Date DateFormat dateFormat = new SimpleDateFormat("hhmmss_mmddyyyy"); realZipfilename = realZipfilename + "_" + dateFormat.format(new Date()) + ".zip"; log.logDebug(toString(), Messages.getString("JobZipFiles.Zip_FileNameChange1.Label") + realZipfilename + Messages.getString("JobZipFiles.Zip_FileNameChange1.Label")); } else if (ifzipfileexists == 1 && Fileexists) { log.logDebug(toString(), Messages.getString("JobZipFiles.Zip_FileAppend1.Label") + realZipfilename + Messages.getString("JobZipFiles.Zip_FileAppend2.Label")); } // Get all the files in the directory... File f = new File(realTargetdirectory); String[] filelist = f.list(); log.logDetailed(toString(), Messages.getString("JobZipFiles.Files_Found1.Label") + filelist.length + Messages.getString("JobZipFiles.Files_Found2.Label") + realTargetdirectory + Messages.getString("JobZipFiles.Files_Found3.Label")); Pattern pattern = null; if (!Const.isEmpty(realWildcard)) { pattern = Pattern.compile(realWildcard); } Pattern patternexclude = null; if (!Const.isEmpty(realWildcardExclude)) { patternexclude = Pattern.compile(realWildcardExclude); } // Prepare Zip File byte[] buffer = new byte[18024]; FileOutputStream dest = new FileOutputStream(realZipfilename); BufferedOutputStream buff = new BufferedOutputStream(dest); ZipOutputStream out = new ZipOutputStream(buff); // Set the method out.setMethod(ZipOutputStream.DEFLATED); // Set the compression level if (compressionrate == 0) { out.setLevel(Deflater.NO_COMPRESSION); } else if (compressionrate == 1) { out.setLevel(Deflater.DEFAULT_COMPRESSION); } if (compressionrate == 2) { out.setLevel(Deflater.BEST_COMPRESSION); } if (compressionrate == 3) { out.setLevel(Deflater.BEST_SPEED); } // Specify Zipped files (After that we will move,delete them...) String[] ZippedFiles = new String[filelist.length]; int FileNum = 0; // Get the files in the list... for (int i = 0; i < filelist.length && !parentJob.isStopped(); i++) { boolean getIt = true; boolean getItexclude = false; // First see if the file matches the regular expression! if (pattern != null) { Matcher matcher = pattern.matcher(filelist[i]); getIt = matcher.matches(); } if (patternexclude != null) { Matcher matcherexclude = patternexclude.matcher(filelist[i]); getItexclude = matcherexclude.matches(); } // Get processing File String targetFilename = realTargetdirectory + Const.FILE_SEPARATOR + filelist[i]; File file = new File(targetFilename); if (getIt && !getItexclude && !file.isDirectory()) { // We can add the file to the Zip Archive log.logDebug(toString(), Messages.getString("JobZipFiles.Add_FilesToZip1.Label") + filelist[i] + Messages.getString("JobZipFiles.Add_FilesToZip2.Label") + realTargetdirectory + Messages.getString("JobZipFiles.Add_FilesToZip3.Label")); // Associate a file input stream for the current file FileInputStream in = new FileInputStream(targetFilename); // Add ZIP entry to output stream. out.putNextEntry(new ZipEntry(filelist[i])); int len; while ((len = in.read(buffer)) > 0) { out.write(buffer, 0, len); } out.closeEntry(); // Close the current file input stream in.close(); // Get Zipped File ZippedFiles[FileNum] = filelist[i]; FileNum = FileNum + 1; } } // Close the ZipOutPutStream out.close(); //-----Get the list of Zipped Files and Move or Delete Them if (afterzip == 1 || afterzip == 2) { // iterate through the array of Zipped files for (int i = 0; i < ZippedFiles.length; i++) { if (ZippedFiles[i] != null) { // Delete File FileObject fileObjectd = KettleVFS .getFileObject(realTargetdirectory + Const.FILE_SEPARATOR + ZippedFiles[i]); // Here we can move, delete files if (afterzip == 1) { // Delete File boolean deleted = fileObjectd.delete(); if (!deleted) { result.setResult(false); result.setNrErrors(1); log.logError(toString(), Messages.getString("JobZipFiles.Cant_Delete_File1.Label") + realTargetdirectory + Const.FILE_SEPARATOR + ZippedFiles[i] + Messages .getString("JobZipFiles.Cant_Delete_File2.Label")); } // File deleted log.logDebug(toString(), Messages.getString("JobZipFiles.File_Deleted1.Label") + realTargetdirectory + Const.FILE_SEPARATOR + ZippedFiles[i] + Messages.getString("JobZipFiles.File_Deleted2.Label")); } else if (afterzip == 2) { // Move File try { FileObject fileObjectm = KettleVFS.getFileObject( realMovetodirectory + Const.FILE_SEPARATOR + ZippedFiles[i]); fileObjectd.moveTo(fileObjectm); } catch (IOException e) { log.logError(toString(), Messages.getString("JobZipFiles.Cant_Move_File1.Label") + ZippedFiles[i] + Messages.getString("JobZipFiles.Cant_Move_File2.Label") + e.getMessage()); result.setResult(false); result.setNrErrors(1); } // File moved log.logDebug(toString(), Messages.getString("JobZipFiles.File_Moved1.Label") + ZippedFiles[i] + Messages.getString("JobZipFiles.File_Moved2.Label")); } } } } result.setResult(true); } } catch (IOException e) { log.logError(toString(), Messages.getString("JobZipFiles.Cant_CreateZipFile1.Label") + realZipfilename + Messages.getString("JobZipFiles.Cant_CreateZipFile2.Label") + e.getMessage()); result.setResult(false); result.setNrErrors(1); } finally { if (fileObject != null) { try { fileObject.close(); } catch (IOException ex) { } ; } } } else { result.setResult(false); result.setNrErrors(1); log.logError(toString(), Messages.getString("JobZipFiles.No_ZipFile_Defined.Label")); } return result; }
From source file:org.springframework.amqp.rabbit.core.BatchingRabbitTemplateTests.java
@Test public void testSimpleBatchGZipped() throws Exception { BatchingStrategy batchingStrategy = new SimpleBatchingStrategy(2, Integer.MAX_VALUE, 30000); BatchingRabbitTemplate template = new BatchingRabbitTemplate(batchingStrategy, this.scheduler); template.setConnectionFactory(this.connectionFactory); GZipPostProcessor gZipPostProcessor = new GZipPostProcessor(); assertEquals(Deflater.BEST_SPEED, getStreamLevel(gZipPostProcessor)); template.setBeforePublishPostProcessors(gZipPostProcessor); MessageProperties props = new MessageProperties(); Message message = new Message("foo".getBytes(), props); template.send("", ROUTE, message); message = new Message("bar".getBytes(), props); template.send("", ROUTE, message); message = receive(template);//from w w w.j a v a 2s .c o m assertEquals("gzip", message.getMessageProperties().getContentEncoding()); GUnzipPostProcessor unzipper = new GUnzipPostProcessor(); message = unzipper.postProcessMessage(message); assertEquals("\u0000\u0000\u0000\u0003foo\u0000\u0000\u0000\u0003bar", new String(message.getBody())); }
From source file:org.ajax4jsf.resource.ResourceBuilderImpl.java
protected byte[] encrypt(byte[] src) { try {/*from w ww . j av a 2 s .c om*/ Deflater compressor = new Deflater(Deflater.BEST_SPEED); byte[] compressed = new byte[src.length + 100]; compressor.setInput(src); compressor.finish(); int totalOut = compressor.deflate(compressed); byte[] zipsrc = new byte[totalOut]; System.arraycopy(compressed, 0, zipsrc, 0, totalOut); compressor.end(); return codec.encode(zipsrc); } catch (Exception e) { throw new FacesException("Error encode resource data", e); } }
From source file:org.springframework.amqp.rabbit.core.BatchingRabbitTemplateTests.java
@Test public void testSimpleBatchZippedWithEncoding() throws Exception { BatchingStrategy batchingStrategy = new SimpleBatchingStrategy(2, Integer.MAX_VALUE, 30000); BatchingRabbitTemplate template = new BatchingRabbitTemplate(batchingStrategy, this.scheduler); template.setConnectionFactory(this.connectionFactory); ZipPostProcessor zipPostProcessor = new ZipPostProcessor(); assertEquals(Deflater.BEST_SPEED, getStreamLevel(zipPostProcessor)); template.setBeforePublishPostProcessors(zipPostProcessor); MessageProperties props = new MessageProperties(); props.setContentEncoding("foo"); Message message = new Message("foo".getBytes(), props); template.send("", ROUTE, message); message = new Message("bar".getBytes(), props); template.send("", ROUTE, message); message = receive(template);/*from w w w. j a va2s .com*/ assertEquals("zip:foo", message.getMessageProperties().getContentEncoding()); UnzipPostProcessor unzipper = new UnzipPostProcessor(); message = unzipper.postProcessMessage(message); assertEquals("\u0000\u0000\u0000\u0003foo\u0000\u0000\u0000\u0003bar", new String(message.getBody())); }
From source file:com.panet.imeta.job.entries.zipfile.JobEntryZipFile.java
public boolean processRowFile(Job parentJob, Result result, String realZipfilename, String realWildcard, String realWildcardExclude, String realTargetdirectory, String realMovetodirectory, boolean createparentfolder) { LogWriter log = LogWriter.getInstance(); boolean Fileexists = false; File tempFile = null;/*from w ww . ja va2 s. c o m*/ File fileZip = null; boolean resultat = false; boolean renameOk = false; boolean orginexist = false; // Check if target file/folder exists! FileObject OriginFile = null; ZipInputStream zin = null; byte[] buffer = null; FileOutputStream dest = null; BufferedOutputStream buff = null; org.apache.tools.zip.ZipOutputStream out = null; org.apache.tools.zip.ZipEntry entry = null; try { OriginFile = KettleVFS.getFileObject(realTargetdirectory); orginexist = OriginFile.exists(); } catch (Exception e) { } finally { if (OriginFile != null) { try { OriginFile.close(); } catch (IOException ex) { } ; } } if (realZipfilename != null && orginexist) { FileObject fileObject = null; try { fileObject = KettleVFS.getFileObject(realZipfilename); // Check if Zip File exists if (fileObject.exists()) { Fileexists = true; if (log.isDebug()) log.logDebug(toString(), Messages.getString("JobZipFiles.Zip_FileExists1.Label") + realZipfilename + Messages.getString("JobZipFiles.Zip_FileExists2.Label")); } // Let's see if we need to create parent folder of destination // zip filename if (createparentfolder) { createParentFolder(realZipfilename); } // Let's start the process now if (ifzipfileexists == 3 && Fileexists) { // the zip file exists and user want to Fail resultat = false; } else if (ifzipfileexists == 2 && Fileexists) { // the zip file exists and user want to do nothing if (addfiletoresult) { // Add file to result files name ResultFile resultFile = new ResultFile(ResultFile.FILE_TYPE_GENERAL, KettleVFS.getFileObject(realZipfilename), parentJob.getJobname(), toString()); result.getResultFiles().put(resultFile.getFile().toString(), resultFile); } resultat = true; } else if (afterzip == 2 && realMovetodirectory == null) { // After Zip, Move files..User must give a destination // Folder resultat = false; log.logError(toString(), Messages.getString("JobZipFiles.AfterZip_No_DestinationFolder_Defined.Label")); } else // After Zip, Move files..User must give a destination Folder { // Let's see if we deal with file or folder String[] filelist = null; File f = new File(realTargetdirectory); if (f.isDirectory()) { // Target is a directory // Get all the files in the directory... filelist = f.list(); } else { // Target is a file filelist = new String[1]; filelist[0] = f.getName(); } if (filelist.length == 0) { resultat = false; log.logError(toString(), Messages.getString("JobZipFiles.Log.FolderIsEmpty", realTargetdirectory)); } else if (!checkContainsFile(realTargetdirectory, filelist)) { resultat = false; log.logError(toString(), Messages.getString("JobZipFiles.Log.NoFilesInFolder", realTargetdirectory)); } else { if (ifzipfileexists == 0 && Fileexists) { // the zip file exists and user want to create new // one with unique name // Format Date // do we have already a .zip at the end? if (realZipfilename.toLowerCase().endsWith(".zip")) { // strip this off realZipfilename = realZipfilename.substring(0, realZipfilename.length() - 4); } realZipfilename = realZipfilename + "_" + StringUtil.getFormattedDateTimeNow(true) + ".zip"; if (log.isDebug()) log.logDebug(toString(), Messages.getString("JobZipFiles.Zip_FileNameChange1.Label") + realZipfilename + Messages.getString("JobZipFiles.Zip_FileNameChange1.Label")); } else if (ifzipfileexists == 1 && Fileexists) { // the zip file exists and user want to append // get a temp file fileZip = new File(realZipfilename); tempFile = File.createTempFile(fileZip.getName(), null); // delete it, otherwise we cannot rename existing // zip to it. tempFile.delete(); renameOk = fileZip.renameTo(tempFile); if (!renameOk) { log.logError(toString(), Messages.getString("JobZipFiles.Cant_Rename_Temp1.Label") + fileZip.getAbsolutePath() + Messages.getString("JobZipFiles.Cant_Rename_Temp2.Label") + tempFile.getAbsolutePath() + Messages.getString("JobZipFiles.Cant_Rename_Temp3.Label")); } if (log.isDebug()) log.logDebug(toString(), Messages.getString("JobZipFiles.Zip_FileAppend1.Label") + realZipfilename + Messages.getString("JobZipFiles.Zip_FileAppend2.Label")); } if (log.isDetailed()) log.logDetailed(toString(), Messages.getString("JobZipFiles.Files_Found1.Label") + filelist.length + Messages.getString("JobZipFiles.Files_Found2.Label") + realTargetdirectory + Messages.getString("JobZipFiles.Files_Found3.Label")); Pattern pattern = null; Pattern patternexclude = null; // Let's prepare pattern..only if target is a folder ! if (f.isDirectory()) { if (!Const.isEmpty(realWildcard)) { pattern = Pattern.compile(realWildcard); } if (!Const.isEmpty(realWildcardExclude)) { patternexclude = Pattern.compile(realWildcardExclude); } } // Prepare Zip File buffer = new byte[18024]; dest = new FileOutputStream(realZipfilename); buff = new BufferedOutputStream(dest); out = new org.apache.tools.zip.ZipOutputStream(buff); HashSet<String> fileSet = new HashSet<String>(); if (renameOk) { // User want to append files to existing Zip file // The idea is to rename the existing zip file to a // temporary file // and then adds all entries in the existing zip // along with the new files, // excluding the zip entries that have the same name // as one of the new files. zin = new ZipInputStream(new FileInputStream(tempFile)); entry = (ZipEntry) zin.getNextEntry(); while (entry != null) { String name = entry.getName(); if (!fileSet.contains(name)) { // Add ZIP entry to output stream. out.putNextEntry(new ZipEntry(name)); // Transfer bytes from the ZIP file to the // output file int len; while ((len = zin.read(buffer)) > 0) { out.write(buffer, 0, len); } fileSet.add(name); } entry = (ZipEntry) zin.getNextEntry(); } // Close the streams zin.close(); } // Set the method out.setMethod(org.apache.tools.zip.ZipOutputStream.DEFLATED); // Set the compression level if (compressionrate == 0) { out.setLevel(Deflater.NO_COMPRESSION); } else if (compressionrate == 1) { out.setLevel(Deflater.DEFAULT_COMPRESSION); } if (compressionrate == 2) { out.setLevel(Deflater.BEST_COMPRESSION); } if (compressionrate == 3) { out.setLevel(Deflater.BEST_SPEED); } // Specify Zipped files (After that we will move,delete // them...) String[] ZippedFiles = new String[filelist.length]; int FileNum = 0; // Get the files in the list... for (int i = 0; i < filelist.length && !parentJob.isStopped(); i++) { boolean getIt = true; boolean getItexclude = false; // First see if the file matches the regular // expression! // ..only if target is a folder ! if (f.isDirectory()) { if (pattern != null) { Matcher matcher = pattern.matcher(filelist[i]); getIt = matcher.matches(); } if (patternexclude != null) { Matcher matcherexclude = patternexclude.matcher(filelist[i]); getItexclude = matcherexclude.matches(); } } // Get processing File String targetFilename = realTargetdirectory + Const.FILE_SEPARATOR + filelist[i]; if (f.isFile()) targetFilename = realTargetdirectory; File file = new File(targetFilename); if (getIt && !getItexclude && !file.isDirectory() && !fileSet.contains(filelist[i])) { // We can add the file to the Zip Archive if (log.isDebug()) log.logDebug(toString(), Messages.getString("JobZipFiles.Add_FilesToZip1.Label") + filelist[i] + Messages.getString("JobZipFiles.Add_FilesToZip2.Label") + realTargetdirectory + Messages.getString("JobZipFiles.Add_FilesToZip3.Label")); // Associate a file input stream for the current // file FileInputStream in = new FileInputStream(targetFilename); // Add ZIP entry to output stream. out.putNextEntry(new ZipEntry(filelist[i])); int len; while ((len = in.read(buffer)) > 0) { out.write(buffer, 0, len); } out.flush(); out.closeEntry(); // Close the current file input stream in.close(); // Get Zipped File ZippedFiles[FileNum] = filelist[i]; FileNum = FileNum + 1; } } // Close the ZipOutPutStream out.close(); buff.close(); dest.close(); if (log.isBasic()) log.logBasic(toString(), Messages.getString("JobZipFiles.Log.TotalZippedFiles", "" + ZippedFiles.length)); // Delete Temp File if (tempFile != null) { tempFile.delete(); } // -----Get the list of Zipped Files and Move or Delete // Them if (afterzip == 1 || afterzip == 2) { // iterate through the array of Zipped files for (int i = 0; i < ZippedFiles.length; i++) { if (ZippedFiles[i] != null) { // Delete File FileObject fileObjectd = KettleVFS.getFileObject( realTargetdirectory + Const.FILE_SEPARATOR + ZippedFiles[i]); if (f.isFile()) fileObjectd = KettleVFS.getFileObject(realTargetdirectory); // Here gc() is explicitly called if e.g. // createfile is used in the same // job for the same file. The problem is // that after creating the file the // file object is not properly garbaged // collected and thus the file cannot // be deleted anymore. This is a known // problem in the JVM. System.gc(); // Here we can move, delete files if (afterzip == 1) { // Delete File boolean deleted = fileObjectd.delete(); if (!deleted) { resultat = false; log.logError(toString(), Messages .getString("JobZipFiles.Cant_Delete_File1.Label") + realTargetdirectory + Const.FILE_SEPARATOR + ZippedFiles[i] + Messages.getString("JobZipFiles.Cant_Delete_File2.Label")); } // File deleted if (log.isDebug()) log.logDebug(toString(), Messages.getString("JobZipFiles.File_Deleted1.Label") + realTargetdirectory + Const.FILE_SEPARATOR + ZippedFiles[i] + Messages .getString("JobZipFiles.File_Deleted2.Label")); } else if (afterzip == 2) { // Move File try { FileObject fileObjectm = KettleVFS.getFileObject( realMovetodirectory + Const.FILE_SEPARATOR + ZippedFiles[i]); fileObjectd.moveTo(fileObjectm); } catch (IOException e) { log.logError(toString(), Messages.getString("JobZipFiles.Cant_Move_File1.Label") + ZippedFiles[i] + Messages .getString("JobZipFiles.Cant_Move_File2.Label") + e.getMessage()); resultat = false; } // File moved if (log.isDebug()) log.logDebug(toString(), Messages.getString("JobZipFiles.File_Moved1.Label") + ZippedFiles[i] + Messages.getString("JobZipFiles.File_Moved2.Label")); } } } } if (addfiletoresult) { // Add file to result files name ResultFile resultFile = new ResultFile(ResultFile.FILE_TYPE_GENERAL, KettleVFS.getFileObject(realZipfilename), parentJob.getJobname(), toString()); result.getResultFiles().put(resultFile.getFile().toString(), resultFile); } resultat = true; } } } catch (Exception e) { log.logError(toString(), Messages.getString("JobZipFiles.Cant_CreateZipFile1.Label") + realZipfilename + Messages.getString("JobZipFiles.Cant_CreateZipFile2.Label") + e.getMessage()); // result.setResult( false ); // result.setNrErrors(1); resultat = false; } finally { if (fileObject != null) { try { fileObject.close(); } catch (IOException ex) { } ; } // Close the ZipOutPutStream try { if (out != null) out.close(); if (buff != null) buff.close(); if (dest != null) dest.close(); if (zin != null) zin.close(); if (entry != null) entry = null; } catch (IOException ex) { } ; } } else { resultat = true; if (realZipfilename == null) log.logError(toString(), Messages.getString("JobZipFiles.No_ZipFile_Defined.Label")); if (!orginexist) log.logError(toString(), Messages.getString("JobZipFiles.No_FolderCible_Defined.Label", realTargetdirectory)); } // return a verifier return resultat; }
From source file:org.springframework.xd.dirt.integration.bus.rabbit.RabbitMessageBusTests.java
@SuppressWarnings("unchecked") @Test//from w ww .j a va2 s .c o m public void testBatchingAndCompression() throws Exception { RabbitTemplate template = new RabbitTemplate(this.rabbitAvailableRule.getResource()); MessageBus bus = getMessageBus(); Properties properties = new Properties(); properties.put("deliveryMode", "NON_PERSISTENT"); properties.put("batchingEnabled", "true"); properties.put("batchSize", "2"); properties.put("batchBufferLimit", "100000"); properties.put("batchTimeout", "30000"); properties.put("compress", "true"); DirectChannel output = new DirectChannel(); output.setBeanName("batchingProducer"); bus.bindProducer("batching.0", output, properties); while (template.receive("xdbus.batching.0") != null) { } Log logger = spy(TestUtils.getPropertyValue(bus, "messageBus.compressingPostProcessor.logger", Log.class)); new DirectFieldAccessor(TestUtils.getPropertyValue(bus, "messageBus.compressingPostProcessor")) .setPropertyValue("logger", logger); when(logger.isTraceEnabled()).thenReturn(true); assertEquals(Deflater.BEST_SPEED, TestUtils.getPropertyValue(bus, "messageBus.compressingPostProcessor.level")); output.send(new GenericMessage<>("foo".getBytes())); output.send(new GenericMessage<>("bar".getBytes())); Object out = spyOn("batching.0").receive(false); assertThat(out, instanceOf(byte[].class)); assertEquals("\u0000\u0000\u0000\u0003foo\u0000\u0000\u0000\u0003bar", new String((byte[]) out)); ArgumentCaptor<Object> captor = ArgumentCaptor.forClass(Object.class); verify(logger).trace(captor.capture()); assertThat(captor.getValue().toString(), containsString("Compressed 14 to ")); QueueChannel input = new QueueChannel(); input.setBeanName("batchingConsumer"); bus.bindConsumer("batching.0", input, null); output.send(new GenericMessage<>("foo".getBytes())); output.send(new GenericMessage<>("bar".getBytes())); Message<byte[]> in = (Message<byte[]>) input.receive(10000); assertNotNull(in); assertEquals("foo", new String(in.getPayload())); in = (Message<byte[]>) input.receive(10000); assertNotNull(in); assertEquals("bar", new String(in.getPayload())); assertNull(in.getHeaders().get(AmqpHeaders.DELIVERY_MODE)); bus.unbindProducers("batching.0"); bus.unbindConsumers("batching.0"); }
From source file:org.apache.hadoop.hive.ql.exec.Utilities.java
private static Path setBaseWork(Configuration conf, BaseWork w, Path hiveScratchDir, String name, boolean useCache) { Kryo kryo = SerializationUtilities.borrowKryo(); try {/*from w w w .j a va2 s .c o m*/ setPlanPath(conf, hiveScratchDir); Path planPath = getPlanPath(conf, name); setHasWork(conf, name); OutputStream out = null; final long serializedSize; final String planMode; if (HiveConf.getBoolVar(conf, ConfVars.HIVE_RPC_QUERY_PLAN)) { // add it to the conf ByteArrayOutputStream byteOut = new ByteArrayOutputStream(); try { out = new DeflaterOutputStream(byteOut, new Deflater(Deflater.BEST_SPEED)); SerializationUtilities.serializePlan(kryo, w, out); out.close(); out = null; } finally { IOUtils.closeStream(out); } final String serializedPlan = Base64.encodeBase64String(byteOut.toByteArray()); serializedSize = serializedPlan.length(); planMode = "RPC"; conf.set(planPath.toUri().getPath(), serializedPlan); } else { // use the default file system of the conf FileSystem fs = planPath.getFileSystem(conf); try { out = fs.create(planPath); SerializationUtilities.serializePlan(kryo, w, out); out.close(); out = null; long fileLen = fs.getFileStatus(planPath).getLen(); serializedSize = fileLen; planMode = "FILE"; } finally { IOUtils.closeStream(out); } // Serialize the plan to the default hdfs instance // Except for hadoop local mode execution where we should be // able to get the plan directly from the cache if (useCache && !ShimLoader.getHadoopShims().isLocalMode(conf)) { // Set up distributed cache if (!DistributedCache.getSymlink(conf)) { DistributedCache.createSymlink(conf); } String uriWithLink = planPath.toUri().toString() + "#" + name; DistributedCache.addCacheFile(new URI(uriWithLink), conf); // set replication of the plan file to a high number. we use the same // replication factor as used by the hadoop jobclient for job.xml etc. short replication = (short) conf.getInt("mapred.submit.replication", 10); fs.setReplication(planPath, replication); } } LOG.info("Serialized plan (via {}) - name: {} size: {}", planMode, w.getName(), humanReadableByteCount(serializedSize)); // Cache the plan in this process gWorkMap.get(conf).put(planPath, w); return planPath; } catch (Exception e) { String msg = "Error caching " + name + ": " + e; LOG.error(msg, e); throw new RuntimeException(msg, e); } finally { SerializationUtilities.releaseKryo(kryo); } }
From source file:org.pentaho.di.job.entries.zipfile.JobEntryZipFile.java
public boolean processRowFile(Job parentJob, Result result, String realZipfilename, String realWildcard, String realWildcardExclude, String realSourceDirectoryOrFile, String realMovetodirectory, boolean createparentfolder) { boolean Fileexists = false; File tempFile = null;//from ww w . j a v a 2 s .c o m File fileZip = null; boolean resultat = false; boolean renameOk = false; boolean orginExist = false; // Check if target file/folder exists! FileObject originFile = null; ZipInputStream zin = null; byte[] buffer = null; OutputStream dest = null; BufferedOutputStream buff = null; ZipOutputStream out = null; ZipEntry entry = null; String localSourceFilename = realSourceDirectoryOrFile; try { originFile = KettleVFS.getFileObject(realSourceDirectoryOrFile, this); localSourceFilename = KettleVFS.getFilename(originFile); orginExist = originFile.exists(); } catch (Exception e) { // Ignore errors } finally { if (originFile != null) { try { originFile.close(); } catch (IOException ex) { logError("Error closing file '" + originFile.toString() + "'", ex); } } } String localrealZipfilename = realZipfilename; if (realZipfilename != null && orginExist) { FileObject fileObject = null; try { fileObject = KettleVFS.getFileObject(localrealZipfilename, this); localrealZipfilename = KettleVFS.getFilename(fileObject); // Check if Zip File exists if (fileObject.exists()) { Fileexists = true; if (log.isDebug()) { logDebug(BaseMessages.getString(PKG, "JobZipFiles.Zip_FileExists1.Label") + localrealZipfilename + BaseMessages.getString(PKG, "JobZipFiles.Zip_FileExists2.Label")); } } // Let's see if we need to create parent folder of destination zip filename if (createparentfolder) { createParentFolder(localrealZipfilename); } // Let's start the process now if (ifZipFileExists == 3 && Fileexists) { // the zip file exists and user want to Fail resultat = false; } else if (ifZipFileExists == 2 && Fileexists) { // the zip file exists and user want to do nothing if (addFileToResult) { // Add file to result files name ResultFile resultFile = new ResultFile(ResultFile.FILE_TYPE_GENERAL, fileObject, parentJob.getJobname(), toString()); result.getResultFiles().put(resultFile.getFile().toString(), resultFile); } resultat = true; } else if (afterZip == 2 && realMovetodirectory == null) { // After Zip, Move files..User must give a destination Folder resultat = false; logError( BaseMessages.getString(PKG, "JobZipFiles.AfterZip_No_DestinationFolder_Defined.Label")); } else { // After Zip, Move files..User must give a destination Folder // Let's see if we deal with file or folder FileObject[] fileList = null; FileObject sourceFileOrFolder = KettleVFS.getFileObject(localSourceFilename); boolean isSourceDirectory = sourceFileOrFolder.getType().equals(FileType.FOLDER); final Pattern pattern; final Pattern patternexclude; if (isSourceDirectory) { // Let's prepare the pattern matcher for performance reasons. // We only do this if the target is a folder ! // if (!Const.isEmpty(realWildcard)) { pattern = Pattern.compile(realWildcard); } else { pattern = null; } if (!Const.isEmpty(realWildcardExclude)) { patternexclude = Pattern.compile(realWildcardExclude); } else { patternexclude = null; } // Target is a directory // Get all the files in the directory... // if (includingSubFolders) { fileList = sourceFileOrFolder.findFiles(new FileSelector() { public boolean traverseDescendents(FileSelectInfo fileInfo) throws Exception { return true; } public boolean includeFile(FileSelectInfo fileInfo) throws Exception { boolean include; // Only include files in the sub-folders... // When we include sub-folders we match the whole filename, not just the base-name // if (fileInfo.getFile().getType().equals(FileType.FILE)) { include = true; if (pattern != null) { String name = fileInfo.getFile().getName().getPath(); include = pattern.matcher(name).matches(); } if (include && patternexclude != null) { String name = fileInfo.getFile().getName().getPath(); include = !pattern.matcher(name).matches(); } } else { include = false; } return include; } }); } else { fileList = sourceFileOrFolder.getChildren(); } } else { pattern = null; patternexclude = null; // Target is a file fileList = new FileObject[] { sourceFileOrFolder }; } if (fileList.length == 0) { resultat = false; logError(BaseMessages.getString(PKG, "JobZipFiles.Log.FolderIsEmpty", localSourceFilename)); } else if (!checkContainsFile(localSourceFilename, fileList, isSourceDirectory)) { resultat = false; logError(BaseMessages.getString(PKG, "JobZipFiles.Log.NoFilesInFolder", localSourceFilename)); } else { if (ifZipFileExists == 0 && Fileexists) { // the zip file exists and user want to create new one with unique name // Format Date // do we have already a .zip at the end? if (localrealZipfilename.toLowerCase().endsWith(".zip")) { // strip this off localrealZipfilename = localrealZipfilename.substring(0, localrealZipfilename.length() - 4); } localrealZipfilename += "_" + StringUtil.getFormattedDateTimeNow(true) + ".zip"; if (log.isDebug()) { logDebug(BaseMessages.getString(PKG, "JobZipFiles.Zip_FileNameChange1.Label") + localrealZipfilename + BaseMessages.getString(PKG, "JobZipFiles.Zip_FileNameChange1.Label")); } } else if (ifZipFileExists == 1 && Fileexists) { // the zip file exists and user want to append // get a temp file fileZip = getFile(localrealZipfilename); tempFile = File.createTempFile(fileZip.getName(), null); // delete it, otherwise we cannot rename existing zip to it. tempFile.delete(); renameOk = fileZip.renameTo(tempFile); if (!renameOk) { logError(BaseMessages.getString(PKG, "JobZipFiles.Cant_Rename_Temp1.Label") + fileZip.getAbsolutePath() + BaseMessages.getString(PKG, "JobZipFiles.Cant_Rename_Temp2.Label") + tempFile.getAbsolutePath() + BaseMessages.getString(PKG, "JobZipFiles.Cant_Rename_Temp3.Label")); } if (log.isDebug()) { logDebug(BaseMessages.getString(PKG, "JobZipFiles.Zip_FileAppend1.Label") + localrealZipfilename + BaseMessages.getString(PKG, "JobZipFiles.Zip_FileAppend2.Label")); } } if (log.isDetailed()) { logDetailed( BaseMessages.getString(PKG, "JobZipFiles.Files_Found1.Label") + fileList.length + BaseMessages.getString(PKG, "JobZipFiles.Files_Found2.Label") + localSourceFilename + BaseMessages.getString(PKG, "JobZipFiles.Files_Found3.Label")); } // Prepare Zip File buffer = new byte[18024]; dest = KettleVFS.getOutputStream(localrealZipfilename, false); buff = new BufferedOutputStream(dest); out = new ZipOutputStream(buff); HashSet<String> fileSet = new HashSet<String>(); if (renameOk) { // User want to append files to existing Zip file // The idea is to rename the existing zip file to a temporary file // and then adds all entries in the existing zip along with the new files, // excluding the zip entries that have the same name as one of the new files. zin = new ZipInputStream(new FileInputStream(tempFile)); entry = zin.getNextEntry(); while (entry != null) { String name = entry.getName(); if (!fileSet.contains(name)) { // Add ZIP entry to output stream. out.putNextEntry(new ZipEntry(name)); // Transfer bytes from the ZIP file to the output file int len; while ((len = zin.read(buffer)) > 0) { out.write(buffer, 0, len); } fileSet.add(name); } entry = zin.getNextEntry(); } // Close the streams zin.close(); } // Set the method out.setMethod(ZipOutputStream.DEFLATED); // Set the compression level if (compressionRate == 0) { out.setLevel(Deflater.NO_COMPRESSION); } else if (compressionRate == 1) { out.setLevel(Deflater.DEFAULT_COMPRESSION); } if (compressionRate == 2) { out.setLevel(Deflater.BEST_COMPRESSION); } if (compressionRate == 3) { out.setLevel(Deflater.BEST_SPEED); } // Specify Zipped files (After that we will move,delete them...) FileObject[] zippedFiles = new FileObject[fileList.length]; int fileNum = 0; // Get the files in the list... for (int i = 0; i < fileList.length && !parentJob.isStopped(); i++) { boolean getIt = true; boolean getItexclude = false; // First see if the file matches the regular expression! // ..only if target is a folder ! if (isSourceDirectory) { // If we include sub-folders, we match on the whole name, not just the basename // String filename; if (includingSubFolders) { filename = fileList[i].getName().getPath(); } else { filename = fileList[i].getName().getBaseName(); } if (pattern != null) { // Matches the base name of the file (backward compatible!) // Matcher matcher = pattern.matcher(filename); getIt = matcher.matches(); } if (patternexclude != null) { Matcher matcherexclude = patternexclude.matcher(filename); getItexclude = matcherexclude.matches(); } } // Get processing File String targetFilename = KettleVFS.getFilename(fileList[i]); if (sourceFileOrFolder.getType().equals(FileType.FILE)) { targetFilename = localSourceFilename; } FileObject file = KettleVFS.getFileObject(targetFilename); boolean isTargetDirectory = file.exists() && file.getType().equals(FileType.FOLDER); if (getIt && !getItexclude && !isTargetDirectory && !fileSet.contains(targetFilename)) { // We can add the file to the Zip Archive if (log.isDebug()) { logDebug(BaseMessages.getString(PKG, "JobZipFiles.Add_FilesToZip1.Label") + fileList[i] + BaseMessages.getString(PKG, "JobZipFiles.Add_FilesToZip2.Label") + localSourceFilename + BaseMessages.getString(PKG, "JobZipFiles.Add_FilesToZip3.Label")); } // Associate a file input stream for the current file InputStream in = KettleVFS.getInputStream(file); // Add ZIP entry to output stream. // String relativeName; String fullName = fileList[i].getName().getPath(); String basePath = sourceFileOrFolder.getName().getPath(); if (isSourceDirectory) { if (fullName.startsWith(basePath)) { relativeName = fullName.substring(basePath.length() + 1); } else { relativeName = fullName; } } else if (isFromPrevious) { int depth = determineDepth(environmentSubstitute(storedSourcePathDepth)); relativeName = determineZipfilenameForDepth(fullName, depth); } else { relativeName = fileList[i].getName().getBaseName(); } out.putNextEntry(new ZipEntry(relativeName)); int len; while ((len = in.read(buffer)) > 0) { out.write(buffer, 0, len); } out.flush(); out.closeEntry(); // Close the current file input stream in.close(); // Get Zipped File zippedFiles[fileNum] = fileList[i]; fileNum = fileNum + 1; } } // Close the ZipOutPutStream out.close(); buff.close(); dest.close(); if (log.isBasic()) { logBasic(BaseMessages.getString(PKG, "JobZipFiles.Log.TotalZippedFiles", "" + zippedFiles.length)); } // Delete Temp File if (tempFile != null) { tempFile.delete(); } // -----Get the list of Zipped Files and Move or Delete Them if (afterZip == 1 || afterZip == 2) { // iterate through the array of Zipped files for (int i = 0; i < zippedFiles.length; i++) { if (zippedFiles[i] != null) { // Delete, Move File FileObject fileObjectd = zippedFiles[i]; if (!isSourceDirectory) { fileObjectd = KettleVFS.getFileObject(localSourceFilename); } // Here we can move, delete files if (afterZip == 1) { // Delete File boolean deleted = fileObjectd.delete(); if (!deleted) { resultat = false; logError(BaseMessages.getString(PKG, "JobZipFiles.Cant_Delete_File1.Label") + localSourceFilename + Const.FILE_SEPARATOR + zippedFiles[i] + BaseMessages .getString(PKG, "JobZipFiles.Cant_Delete_File2.Label")); } // File deleted if (log.isDebug()) { logDebug(BaseMessages.getString(PKG, "JobZipFiles.File_Deleted1.Label") + localSourceFilename + Const.FILE_SEPARATOR + zippedFiles[i] + BaseMessages.getString(PKG, "JobZipFiles.File_Deleted2.Label")); } } else if (afterZip == 2) { // Move File FileObject fileObjectm = null; try { fileObjectm = KettleVFS.getFileObject(realMovetodirectory + Const.FILE_SEPARATOR + fileObjectd.getName().getBaseName()); fileObjectd.moveTo(fileObjectm); } catch (IOException e) { logError( BaseMessages.getString(PKG, "JobZipFiles.Cant_Move_File1.Label") + zippedFiles[i] + BaseMessages.getString(PKG, "JobZipFiles.Cant_Move_File2.Label") + e.getMessage()); resultat = false; } finally { try { if (fileObjectm != null) { fileObjectm.close(); } } catch (Exception e) { if (fileObjectm != null) { logError("Error closing file '" + fileObjectm.toString() + "'", e); } } } // File moved if (log.isDebug()) { logDebug(BaseMessages.getString(PKG, "JobZipFiles.File_Moved1.Label") + zippedFiles[i] + BaseMessages.getString(PKG, "JobZipFiles.File_Moved2.Label")); } } } } } if (addFileToResult) { // Add file to result files name ResultFile resultFile = new ResultFile(ResultFile.FILE_TYPE_GENERAL, fileObject, parentJob.getJobname(), toString()); result.getResultFiles().put(resultFile.getFile().toString(), resultFile); } resultat = true; } } } catch (Exception e) { logError(BaseMessages.getString(PKG, "JobZipFiles.Cant_CreateZipFile1.Label") + localrealZipfilename + BaseMessages.getString(PKG, "JobZipFiles.Cant_CreateZipFile2.Label"), e); resultat = false; } finally { if (fileObject != null) { try { fileObject.close(); fileObject = null; } catch (IOException ex) { logError("Error closing file '" + fileObject.toString() + "'", ex); } } try { if (out != null) { out.close(); } if (buff != null) { buff.close(); } if (dest != null) { dest.close(); } if (zin != null) { zin.close(); } if (entry != null) { entry = null; } } catch (IOException ex) { logError("Error closing zip file entry for file '" + originFile.toString() + "'", ex); } } } else { resultat = true; if (localrealZipfilename == null) { logError(BaseMessages.getString(PKG, "JobZipFiles.No_ZipFile_Defined.Label")); } if (!orginExist) { logError(BaseMessages.getString(PKG, "JobZipFiles.No_FolderCible_Defined.Label", localSourceFilename)); } } // return a verifier return resultat; }
From source file:org.springframework.cloud.stream.binder.rabbit.RabbitBinderTests.java
@SuppressWarnings("unchecked") @Test/*from w w w .j a va 2s. c o m*/ public void testBatchingAndCompression() throws Exception { RabbitTestBinder binder = getBinder(); ExtendedProducerProperties<RabbitProducerProperties> producerProperties = createProducerProperties(); producerProperties.getExtension().setDeliveryMode(MessageDeliveryMode.NON_PERSISTENT); producerProperties.getExtension().setBatchingEnabled(true); producerProperties.getExtension().setBatchSize(2); producerProperties.getExtension().setBatchBufferLimit(100000); producerProperties.getExtension().setBatchTimeout(30000); producerProperties.getExtension().setCompress(true); producerProperties.setRequiredGroups("default"); DirectChannel output = createBindableChannel("input", createProducerBindingProperties(producerProperties)); output.setBeanName("batchingProducer"); Binding<MessageChannel> producerBinding = binder.bindProducer("batching.0", output, producerProperties); Log logger = spy(TestUtils.getPropertyValue(binder, "binder.compressingPostProcessor.logger", Log.class)); new DirectFieldAccessor(TestUtils.getPropertyValue(binder, "binder.compressingPostProcessor")) .setPropertyValue("logger", logger); when(logger.isTraceEnabled()).thenReturn(true); assertThat(TestUtils.getPropertyValue(binder, "binder.compressingPostProcessor.level")) .isEqualTo(Deflater.BEST_SPEED); output.send(new GenericMessage<>("foo".getBytes())); output.send(new GenericMessage<>("bar".getBytes())); Object out = spyOn("batching.0.default").receive(false); assertThat(out).isInstanceOf(byte[].class); assertThat(new String((byte[]) out)).isEqualTo("\u0000\u0000\u0000\u0003foo\u0000\u0000\u0000\u0003bar"); ArgumentCaptor<Object> captor = ArgumentCaptor.forClass(Object.class); verify(logger).trace(captor.capture()); assertThat(captor.getValue().toString()).contains(("Compressed 14 to ")); QueueChannel input = new QueueChannel(); input.setBeanName("batchingConsumer"); Binding<MessageChannel> consumerBinding = binder.bindConsumer("batching.0", "test", input, createConsumerProperties()); output.send(new GenericMessage<>("foo".getBytes())); output.send(new GenericMessage<>("bar".getBytes())); Message<byte[]> in = (Message<byte[]>) input.receive(10000); assertThat(in).isNotNull(); assertThat(new String(in.getPayload())).isEqualTo("foo"); in = (Message<byte[]>) input.receive(10000); assertThat(in).isNotNull(); assertThat(new String(in.getPayload())).isEqualTo("bar"); assertThat(in.getHeaders().get(AmqpHeaders.DELIVERY_MODE)).isNull(); producerBinding.unbind(); consumerBinding.unbind(); }
From source file:com.ichi2.anki.SyncClient.java
public static void fullSyncFromLocal(String password, String username, String deckName, String deckPath) { URL url;// w w w. j a v a 2 s . c om try { Log.i(AnkiDroidApp.TAG, "Fullup"); url = new URL(AnkiDroidProxy.SYNC_URL + "fullup"); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); conn.setDoInput(true); conn.setDoOutput(true); conn.setUseCaches(false); conn.setRequestMethod("POST"); conn.setRequestProperty("Connection", "close"); conn.setRequestProperty("Charset", "UTF-8"); // conn.setRequestProperty("Content-Length", "8494662"); conn.setRequestProperty("Content-Type", "multipart/form-data;boundary=" + MIME_BOUNDARY); conn.setRequestProperty("Host", AnkiDroidProxy.SYNC_HOST); DataOutputStream ds = new DataOutputStream(conn.getOutputStream()); Log.i(AnkiDroidApp.TAG, "Pass"); ds.writeBytes(TWO_HYPHENS + MIME_BOUNDARY + END); ds.writeBytes("Content-Disposition: form-data; name=\"p\"" + END + END + password + END); Log.i(AnkiDroidApp.TAG, "User"); ds.writeBytes(TWO_HYPHENS + MIME_BOUNDARY + END); ds.writeBytes("Content-Disposition: form-data; name=\"u\"" + END + END + username + END); Log.i(AnkiDroidApp.TAG, "DeckName"); ds.writeBytes(TWO_HYPHENS + MIME_BOUNDARY + END); ds.writeBytes("Content-Disposition: form-data; name=\"d\"" + END + END + deckName + END); Log.i(AnkiDroidApp.TAG, "Deck"); ds.writeBytes(TWO_HYPHENS + MIME_BOUNDARY + END); ds.writeBytes("Content-Disposition: form-data; name=\"deck\";filename=\"deck\"" + END); ds.writeBytes("Content-Type: application/octet-stream" + END); ds.writeBytes(END); FileInputStream fStream = new FileInputStream(deckPath); byte[] buffer = new byte[Utils.CHUNK_SIZE]; int length = -1; Deflater deflater = new Deflater(Deflater.BEST_SPEED); DeflaterOutputStream dos = new DeflaterOutputStream(ds, deflater); Log.i(AnkiDroidApp.TAG, "Writing buffer..."); while ((length = fStream.read(buffer)) != -1) { dos.write(buffer, 0, length); Log.i(AnkiDroidApp.TAG, "Length = " + length); } dos.finish(); fStream.close(); ds.writeBytes(END); ds.writeBytes(TWO_HYPHENS + MIME_BOUNDARY + TWO_HYPHENS + END); Log.i(AnkiDroidApp.TAG, "Closing streams..."); ds.flush(); ds.close(); // Ensure we got the HTTP 200 response code int responseCode = conn.getResponseCode(); if (responseCode != 200) { Log.i(AnkiDroidApp.TAG, "Response code = " + responseCode); // throw new Exception(String.format("Received the response code %d from the URL %s", responseCode, // url)); } else { Log.i(AnkiDroidApp.TAG, "Response code = 200"); } // Read the response InputStream is = conn.getInputStream(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); byte[] bytes = new byte[1024]; int bytesRead; while ((bytesRead = is.read(bytes)) != -1) { baos.write(bytes, 0, bytesRead); } byte[] bytesReceived = baos.toByteArray(); baos.close(); is.close(); String response = new String(bytesReceived); Log.i(AnkiDroidApp.TAG, "Finished!"); } catch (MalformedURLException e) { Log.i(AnkiDroidApp.TAG, "MalformedURLException = " + e.getMessage()); } catch (IOException e) { Log.i(AnkiDroidApp.TAG, "IOException = " + e.getMessage()); } }