List of usage examples for java.util.zip Deflater DEFAULT_COMPRESSION
int DEFAULT_COMPRESSION
To view the source code for java.util.zip Deflater DEFAULT_COMPRESSION.
Click Source Link
From source file:org.resthub.rpc.AMQPHessianProxy.java
/** * Create the request message body//from w w w. j a v a2 s . c o m * @param method * @param args * @return * @throws IOException */ private byte[] createRequestBody(Method method, Object[] args) throws IOException { String methodName = method.getName(); if (_factory.isOverloadEnabled() && args != null && args.length > 0) { methodName = AbstractSkeleton.mangleName(method, false); } ByteArrayOutputStream payload = new ByteArrayOutputStream(256); OutputStream os; if (_factory.isCompressed()) { Deflater deflater = new Deflater(Deflater.DEFAULT_COMPRESSION, true); os = new DeflaterOutputStream(payload, deflater); } else { os = payload; } AbstractHessianOutput out = _factory.getHessianOutput(os); out.call(methodName, args); if (os instanceof DeflaterOutputStream) { ((DeflaterOutputStream) os).finish(); } out.flush(); return payload.toByteArray(); }
From source file:gridool.db.partitioning.phihash.csv.distmm.InMemoryIndexHelper.java
private static OutputStream prepareFkOutputStream(final String fkIdxName, final int bucket, final Map<String, OutputStream> outputMap) throws IOException { String fname = fkIdxName + bucket + ".fk.gz"; OutputStream out = outputMap.get(fname); if (out == null) { File file = getFkIndexFile(fname); FileOutputStream fos = new FileOutputStream(file, true); DeflaterOutputStream zos = new DeflaterOutputStream(fos, new Deflater(Deflater.DEFAULT_COMPRESSION, false), COMPRESSOR_BUFSIZE); out = new FastBufferedOutputStream(zos, FKCHUNK_IO_BUFSIZE); outputMap.put(fname, out);//from w ww .j ava 2 s. c o m } return out; }
From source file:de.unikassel.puma.openaccess.sword.SwordService.java
/** * collects all informations to send Documents with metadata to repository * @throws SwordException //from w w w. j a v a 2 s. c om */ public void submitDocument(PumaData<?> pumaData, User user) throws SwordException { log.info("starting sword"); DepositResponse depositResponse = new DepositResponse(999); File swordZipFile = null; Post<?> post = pumaData.getPost(); // ------------------------------------------------------------------------------- /* * retrieve ZIP-FILE */ if (post.getResource() instanceof BibTex) { // fileprefix String fileID = HashUtils.getMD5Hash(user.getName().getBytes()) + "_" + post.getResource().getIntraHash(); // Destination directory File destinationDirectory = new File(repositoryConfig.getDirTemp() + "/" + fileID); // zip-filename swordZipFile = new File(destinationDirectory.getAbsoluteFile() + "/" + fileID + ".zip"); byte[] buffer = new byte[18024]; log.info("getIntraHash = " + post.getResource().getIntraHash()); /* * get documents */ // At the moment, there are no Documents delivered by method parameter post. // retrieve list of documents from database - workaround // get documents for post and insert documents into post ((BibTex) post.getResource()) .setDocuments(retrieveDocumentsFromDatabase(user, post.getResource().getIntraHash())); if (((BibTex) post.getResource()).getDocuments().isEmpty()) { // Wenn kein PDF da, dann Fehlermeldung ausgeben!! log.info("throw SwordException: noPDFattached"); throw new SwordException("error.sword.noPDFattached"); } try { // create directory boolean mkdir_success = (new File(destinationDirectory.getAbsolutePath())).mkdir(); if (mkdir_success) { log.info("Directory: " + destinationDirectory.getAbsolutePath() + " created"); } // open zip archive to add files to log.info("zipFilename: " + swordZipFile); ZipOutputStream zipOutputStream = new ZipOutputStream(new FileOutputStream(swordZipFile)); ArrayList<String> fileList = new ArrayList<String>(); for (final Document document : ((BibTex) post.getResource()).getDocuments()) { //getpostdetails // get file and store it in hard coded folder "/tmp/" //final Document document2 = logic.getDocument(user.getName(), post.getResource().getIntraHash(), document.getFileName()); // move file to user folder with username_resource-hash as folder name // File (or directory) to be copied //File fileToZip = new File(document.getFileHash()); fileList.add(document.getFileName()); // Move file to new directory //boolean rename_success = fileToCopy.renameTo(new File(destinationDirectory, fileToMove.getName())); /* if (!rename_success) { // File was not successfully moved } log.info("File was not successfully moved: "+fileToMove.getName()); } */ ZipEntry zipEntry = new ZipEntry(document.getFileName()); // Set the compression ratio zipOutputStream.setLevel(Deflater.DEFAULT_COMPRESSION); String inputFilePath = projectDocumentPath + document.getFileHash().substring(0, 2) + "/" + document.getFileHash(); FileInputStream in = new FileInputStream(inputFilePath); // Add ZIP entry to output stream. zipOutputStream.putNextEntry(zipEntry); // Transfer bytes from the current file to the ZIP file //out.write(buffer, 0, in.read(buffer)); int len; while ((len = in.read(buffer)) > 0) { zipOutputStream.write(buffer, 0, len); } zipOutputStream.closeEntry(); // Close the current file input stream in.close(); } // write meta data into zip archive ZipEntry zipEntry = new ZipEntry("mets.xml"); zipOutputStream.putNextEntry(zipEntry); // create XML-Document // PrintWriter from a Servlet MetsBibTexMLGenerator metsBibTexMLGenerator = new MetsBibTexMLGenerator(urlRenderer); metsBibTexMLGenerator.setUser(user); metsBibTexMLGenerator.setFilenameList(fileList); //metsGenerator.setMetadata(metadataMap); metsBibTexMLGenerator.setMetadata((PumaData<BibTex>) pumaData); // PumaPost additionalMetadata = new PumaPost(); // additionalMetadata.setExaminstitution(null); // additionalMetadata.setAdditionaltitle(null); // additionalMetadata.setExamreferee(null); // additionalMetadata.setPhdoralexam(null); // additionalMetadata.setSponsors(null); // additionalMetadata.setAdditionaltitle(null); // metsBibTexMLGenerator.setMetadata((Post<BibTex>) post); //StreamResult streamResult = new StreamResult(zipOutputStream); zipOutputStream.write(metsBibTexMLGenerator.generateMets().getBytes("UTF-8")); zipOutputStream.closeEntry(); // close zip archive zipOutputStream.close(); log.debug("saved to " + swordZipFile.getPath()); } catch (MalformedURLException e) { // e.printStackTrace(); log.info("MalformedURLException! " + e.getMessage()); } catch (IOException e) { //e.printStackTrace(); log.info("IOException! " + e.getMessage()); } catch (ResourceNotFoundException e) { // e.printStackTrace(); log.warn("ResourceNotFoundException! SwordService-retrievePost"); } } /* * end of retrieve ZIP-FILE */ //--------------------------------------------------- /* * do the SWORD stuff */ if (null != swordZipFile) { // get an instance of SWORD-Client Client swordClient = new Client(); PostMessage swordMessage = new PostMessage(); // create sword post message // message file // create directory in temp-folder // store post documents there // store meta data there in format http://purl.org/net/sword-types/METSDSpaceSIP // delete post document files and meta data file // add files to zip archive // -- send zip archive // -- delete zip archive swordClient.setServer(repositoryConfig.getHttpServer(), repositoryConfig.getHttpPort()); swordClient.setUserAgent(repositoryConfig.getHttpUserAgent()); swordClient.setCredentials(repositoryConfig.getAuthUsername(), repositoryConfig.getAuthPassword()); // message meta swordMessage.setNoOp(false); swordMessage.setUserAgent(repositoryConfig.getHttpUserAgent()); swordMessage.setFilepath(swordZipFile.getAbsolutePath()); swordMessage.setFiletype("application/zip"); swordMessage.setFormatNamespace("http://purl.org/net/sword-types/METSDSpaceSIP"); // sets packaging! swordMessage.setVerbose(false); try { // check depositurl against service document if (checkServicedokument(retrieveServicedocument(), repositoryConfig.getHttpServicedocumentUrl(), SWORDFILETYPE, SWORDFORMAT)) { // transmit sword message (zip file with document metadata and document files swordMessage.setDestination(repositoryConfig.getHttpDepositUrl()); depositResponse = swordClient.postFile(swordMessage); /* * 200 OK Used in response to successful GET operations and * to Media Resource Creation operations where X-No-Op is * set to true and the server supports this header. * * 201 Created * * 202 Accepted - One of these MUST be used to indicate that * a deposit was successful. 202 Accepted is used when * processing of the data is not yet complete. * * * 400 Bad Request - used to indicate that there is some * problem with the request where there is no more * appropriate 4xx code. * * 401 Unauthorized - In addition to the usage described in * HTTP, servers that support mediated deposit SHOULD use * this status code when the server does not understand the * value given in the X-Behalf-Of header. In this case a * human-readable explanation MUST be provided. * * 403 Forbidden - indicates that there was a problem making * the deposit, it may be that the depositor is not * authorised to deposit on behalf of the target owner, or * the target owner does not have permission to deposit into * the specified collection. * * 412 Precondition failed - MUST be returned by server * implementations if a calculated checksum does not match a * value provided by the client in the Content-MD5 header. * * 415 Unsupported Media Type - MUST be used to indicate * that the format supplied in either a Content-Type header * or in an X-Packaging header or the combination of the two * is not accepted by the server. */ log.info("throw SwordException: errcode" + depositResponse.getHttpResponse()); throw new SwordException("error.sword.errcode" + depositResponse.getHttpResponse()); } } catch (SWORDClientException e) { log.warn("SWORDClientException: " + e.getMessage() + "\n" + e.getCause() + " / " + swordMessage.getDestination()); throw new SwordException("error.sword.urlnotaccessable"); } } }
From source file:com.thoughtworks.go.util.ZipUtilTest.java
@Test void shouldPreserveFileTimestampWhileGeneratingTheZipFile() throws Exception { File file = temporaryFolder.newFile("foo.txt"); file.setLastModified(1297989100000L); // Set this to any date in the past which is greater than the epoch File zip = zipUtil.zip(file, temporaryFolder.newFile("foo.zip"), Deflater.DEFAULT_COMPRESSION); ZipFile actualZip = new ZipFile(zip.getAbsolutePath()); ZipEntry entry = actualZip.getEntry(file.getName()); assertThat(entry.getTime()).isEqualTo(file.lastModified()); }
From source file:com.lizardtech.expresszip.model.Job.java
private void writeZipFile(File baseDir, File archive, List<String> files) throws FileNotFoundException, IOException { FilterOutputStream out = null; ZipOutputStream stream = new ZipOutputStream(new FileOutputStream(archive)); stream.setLevel(Deflater.DEFAULT_COMPRESSION); out = stream;// ww w .java2 s .co m byte data[] = new byte[18024]; for (String f : files) { logger.info(String.format("Writing %s to ZIP archive %s", f, archive)); ((ZipOutputStream) out).putNextEntry(new ZipEntry(f)); BufferedInputStream in = new BufferedInputStream(new FileInputStream(new File(baseDir, f))); int len = 0; while ((len = in.read(data)) > 0) { out.write(data, 0, len); } out.flush(); in.close(); } out.close(); }
From source file:be.ibridge.kettle.job.entry.zipfile.JobEntryZipFile.java
public Result execute(Result prev_result, int nr, Repository rep, Job parentJob) { LogWriter log = LogWriter.getInstance(); Result result = new Result(nr); result.setResult(false);//from w ww . j a v a2 s.c o m boolean Fileexists = false; String realZipfilename = StringUtil.environmentSubstitute(zipFilename); String realWildcard = StringUtil.environmentSubstitute(wildcard); String realWildcardExclude = StringUtil.environmentSubstitute(wildcardexclude); String realTargetdirectory = StringUtil.environmentSubstitute(sourcedirectory); String realMovetodirectory = StringUtil.environmentSubstitute(movetodirectory); if (realZipfilename != null) { FileObject fileObject = null; try { fileObject = KettleVFS.getFileObject(realZipfilename); // Check if Zip File exists if (fileObject.exists()) { Fileexists = true; log.logDebug(toString(), Messages.getString("JobZipFiles.Zip_FileExists1.Label") + realZipfilename + Messages.getString("JobZipFiles.Zip_FileExists2.Label")); } else { Fileexists = false; } // Let's start the process now if (ifzipfileexists == 3 && Fileexists) { // the zip file exists and user want to Fail result.setResult(false); result.setNrErrors(1); } else if (ifzipfileexists == 2 && Fileexists) { // the zip file exists and user want to do nothing result.setResult(true); } else if (afterzip == 2 && realMovetodirectory == null) { // After Zip, Move files..User must give a destination Folder result.setResult(false); result.setNrErrors(1); log.logError(toString(), Messages.getString("JobZipFiles.AfterZip_No_DestinationFolder_Defined.Label")); } else // After Zip, Move files..User must give a destination Folder { if (ifzipfileexists == 0 && Fileexists) { // the zip file exists and user want to create new one with unique name //Format Date DateFormat dateFormat = new SimpleDateFormat("hhmmss_mmddyyyy"); realZipfilename = realZipfilename + "_" + dateFormat.format(new Date()) + ".zip"; log.logDebug(toString(), Messages.getString("JobZipFiles.Zip_FileNameChange1.Label") + realZipfilename + Messages.getString("JobZipFiles.Zip_FileNameChange1.Label")); } else if (ifzipfileexists == 1 && Fileexists) { log.logDebug(toString(), Messages.getString("JobZipFiles.Zip_FileAppend1.Label") + realZipfilename + Messages.getString("JobZipFiles.Zip_FileAppend2.Label")); } // Get all the files in the directory... File f = new File(realTargetdirectory); String[] filelist = f.list(); log.logDetailed(toString(), Messages.getString("JobZipFiles.Files_Found1.Label") + filelist.length + Messages.getString("JobZipFiles.Files_Found2.Label") + realTargetdirectory + Messages.getString("JobZipFiles.Files_Found3.Label")); Pattern pattern = null; if (!Const.isEmpty(realWildcard)) { pattern = Pattern.compile(realWildcard); } Pattern patternexclude = null; if (!Const.isEmpty(realWildcardExclude)) { patternexclude = Pattern.compile(realWildcardExclude); } // Prepare Zip File byte[] buffer = new byte[18024]; FileOutputStream dest = new FileOutputStream(realZipfilename); BufferedOutputStream buff = new BufferedOutputStream(dest); ZipOutputStream out = new ZipOutputStream(buff); // Set the method out.setMethod(ZipOutputStream.DEFLATED); // Set the compression level if (compressionrate == 0) { out.setLevel(Deflater.NO_COMPRESSION); } else if (compressionrate == 1) { out.setLevel(Deflater.DEFAULT_COMPRESSION); } if (compressionrate == 2) { out.setLevel(Deflater.BEST_COMPRESSION); } if (compressionrate == 3) { out.setLevel(Deflater.BEST_SPEED); } // Specify Zipped files (After that we will move,delete them...) String[] ZippedFiles = new String[filelist.length]; int FileNum = 0; // Get the files in the list... for (int i = 0; i < filelist.length && !parentJob.isStopped(); i++) { boolean getIt = true; boolean getItexclude = false; // First see if the file matches the regular expression! if (pattern != null) { Matcher matcher = pattern.matcher(filelist[i]); getIt = matcher.matches(); } if (patternexclude != null) { Matcher matcherexclude = patternexclude.matcher(filelist[i]); getItexclude = matcherexclude.matches(); } // Get processing File String targetFilename = realTargetdirectory + Const.FILE_SEPARATOR + filelist[i]; File file = new File(targetFilename); if (getIt && !getItexclude && !file.isDirectory()) { // We can add the file to the Zip Archive log.logDebug(toString(), Messages.getString("JobZipFiles.Add_FilesToZip1.Label") + filelist[i] + Messages.getString("JobZipFiles.Add_FilesToZip2.Label") + realTargetdirectory + Messages.getString("JobZipFiles.Add_FilesToZip3.Label")); // Associate a file input stream for the current file FileInputStream in = new FileInputStream(targetFilename); // Add ZIP entry to output stream. out.putNextEntry(new ZipEntry(filelist[i])); int len; while ((len = in.read(buffer)) > 0) { out.write(buffer, 0, len); } out.closeEntry(); // Close the current file input stream in.close(); // Get Zipped File ZippedFiles[FileNum] = filelist[i]; FileNum = FileNum + 1; } } // Close the ZipOutPutStream out.close(); //-----Get the list of Zipped Files and Move or Delete Them if (afterzip == 1 || afterzip == 2) { // iterate through the array of Zipped files for (int i = 0; i < ZippedFiles.length; i++) { if (ZippedFiles[i] != null) { // Delete File FileObject fileObjectd = KettleVFS .getFileObject(realTargetdirectory + Const.FILE_SEPARATOR + ZippedFiles[i]); // Here we can move, delete files if (afterzip == 1) { // Delete File boolean deleted = fileObjectd.delete(); if (!deleted) { result.setResult(false); result.setNrErrors(1); log.logError(toString(), Messages.getString("JobZipFiles.Cant_Delete_File1.Label") + realTargetdirectory + Const.FILE_SEPARATOR + ZippedFiles[i] + Messages .getString("JobZipFiles.Cant_Delete_File2.Label")); } // File deleted log.logDebug(toString(), Messages.getString("JobZipFiles.File_Deleted1.Label") + realTargetdirectory + Const.FILE_SEPARATOR + ZippedFiles[i] + Messages.getString("JobZipFiles.File_Deleted2.Label")); } else if (afterzip == 2) { // Move File try { FileObject fileObjectm = KettleVFS.getFileObject( realMovetodirectory + Const.FILE_SEPARATOR + ZippedFiles[i]); fileObjectd.moveTo(fileObjectm); } catch (IOException e) { log.logError(toString(), Messages.getString("JobZipFiles.Cant_Move_File1.Label") + ZippedFiles[i] + Messages.getString("JobZipFiles.Cant_Move_File2.Label") + e.getMessage()); result.setResult(false); result.setNrErrors(1); } // File moved log.logDebug(toString(), Messages.getString("JobZipFiles.File_Moved1.Label") + ZippedFiles[i] + Messages.getString("JobZipFiles.File_Moved2.Label")); } } } } result.setResult(true); } } catch (IOException e) { log.logError(toString(), Messages.getString("JobZipFiles.Cant_CreateZipFile1.Label") + realZipfilename + Messages.getString("JobZipFiles.Cant_CreateZipFile2.Label") + e.getMessage()); result.setResult(false); result.setNrErrors(1); } finally { if (fileObject != null) { try { fileObject.close(); } catch (IOException ex) { } ; } } } else { result.setResult(false); result.setNrErrors(1); log.logError(toString(), Messages.getString("JobZipFiles.No_ZipFile_Defined.Label")); } return result; }
From source file:com.gargoylesoftware.htmlunit.WebClient3Test.java
private void doTestDeflateCompression(final boolean gzipCompatibleCompression) throws Exception { final byte[] input = "document.title = 'modified';".getBytes("UTF-8"); final byte[] buffer = new byte[100]; final Deflater deflater = new Deflater(Deflater.DEFAULT_COMPRESSION, gzipCompatibleCompression); deflater.setInput(input);//from w w w . j a v a 2 s . co m deflater.finish(); final int compressedDataLength = deflater.deflate(buffer); final byte[] content = new byte[compressedDataLength]; System.arraycopy(buffer, 0, content, 0, compressedDataLength); final List<NameValuePair> headers = new ArrayList<>(); headers.add(new NameValuePair("Content-Encoding", "deflate")); headers.add(new NameValuePair("Content-Length", String.valueOf(compressedDataLength))); final MockWebConnection conn = getMockWebConnection(); conn.setResponse(URL_SECOND, content, 200, "OK", "text/javascript", headers); final String html = "<html><head>" + "<title>Hello world</title>" + "<script src='" + URL_SECOND + "'></script>" + "</head><body><script>alert(document.title)</script></body></html>"; loadPageWithAlerts2(html); }
From source file:com.panet.imeta.job.entries.zipfile.JobEntryZipFile.java
public boolean processRowFile(Job parentJob, Result result, String realZipfilename, String realWildcard, String realWildcardExclude, String realTargetdirectory, String realMovetodirectory, boolean createparentfolder) { LogWriter log = LogWriter.getInstance(); boolean Fileexists = false; File tempFile = null;/* w w w .ja va2s .c o m*/ File fileZip = null; boolean resultat = false; boolean renameOk = false; boolean orginexist = false; // Check if target file/folder exists! FileObject OriginFile = null; ZipInputStream zin = null; byte[] buffer = null; FileOutputStream dest = null; BufferedOutputStream buff = null; org.apache.tools.zip.ZipOutputStream out = null; org.apache.tools.zip.ZipEntry entry = null; try { OriginFile = KettleVFS.getFileObject(realTargetdirectory); orginexist = OriginFile.exists(); } catch (Exception e) { } finally { if (OriginFile != null) { try { OriginFile.close(); } catch (IOException ex) { } ; } } if (realZipfilename != null && orginexist) { FileObject fileObject = null; try { fileObject = KettleVFS.getFileObject(realZipfilename); // Check if Zip File exists if (fileObject.exists()) { Fileexists = true; if (log.isDebug()) log.logDebug(toString(), Messages.getString("JobZipFiles.Zip_FileExists1.Label") + realZipfilename + Messages.getString("JobZipFiles.Zip_FileExists2.Label")); } // Let's see if we need to create parent folder of destination // zip filename if (createparentfolder) { createParentFolder(realZipfilename); } // Let's start the process now if (ifzipfileexists == 3 && Fileexists) { // the zip file exists and user want to Fail resultat = false; } else if (ifzipfileexists == 2 && Fileexists) { // the zip file exists and user want to do nothing if (addfiletoresult) { // Add file to result files name ResultFile resultFile = new ResultFile(ResultFile.FILE_TYPE_GENERAL, KettleVFS.getFileObject(realZipfilename), parentJob.getJobname(), toString()); result.getResultFiles().put(resultFile.getFile().toString(), resultFile); } resultat = true; } else if (afterzip == 2 && realMovetodirectory == null) { // After Zip, Move files..User must give a destination // Folder resultat = false; log.logError(toString(), Messages.getString("JobZipFiles.AfterZip_No_DestinationFolder_Defined.Label")); } else // After Zip, Move files..User must give a destination Folder { // Let's see if we deal with file or folder String[] filelist = null; File f = new File(realTargetdirectory); if (f.isDirectory()) { // Target is a directory // Get all the files in the directory... filelist = f.list(); } else { // Target is a file filelist = new String[1]; filelist[0] = f.getName(); } if (filelist.length == 0) { resultat = false; log.logError(toString(), Messages.getString("JobZipFiles.Log.FolderIsEmpty", realTargetdirectory)); } else if (!checkContainsFile(realTargetdirectory, filelist)) { resultat = false; log.logError(toString(), Messages.getString("JobZipFiles.Log.NoFilesInFolder", realTargetdirectory)); } else { if (ifzipfileexists == 0 && Fileexists) { // the zip file exists and user want to create new // one with unique name // Format Date // do we have already a .zip at the end? if (realZipfilename.toLowerCase().endsWith(".zip")) { // strip this off realZipfilename = realZipfilename.substring(0, realZipfilename.length() - 4); } realZipfilename = realZipfilename + "_" + StringUtil.getFormattedDateTimeNow(true) + ".zip"; if (log.isDebug()) log.logDebug(toString(), Messages.getString("JobZipFiles.Zip_FileNameChange1.Label") + realZipfilename + Messages.getString("JobZipFiles.Zip_FileNameChange1.Label")); } else if (ifzipfileexists == 1 && Fileexists) { // the zip file exists and user want to append // get a temp file fileZip = new File(realZipfilename); tempFile = File.createTempFile(fileZip.getName(), null); // delete it, otherwise we cannot rename existing // zip to it. tempFile.delete(); renameOk = fileZip.renameTo(tempFile); if (!renameOk) { log.logError(toString(), Messages.getString("JobZipFiles.Cant_Rename_Temp1.Label") + fileZip.getAbsolutePath() + Messages.getString("JobZipFiles.Cant_Rename_Temp2.Label") + tempFile.getAbsolutePath() + Messages.getString("JobZipFiles.Cant_Rename_Temp3.Label")); } if (log.isDebug()) log.logDebug(toString(), Messages.getString("JobZipFiles.Zip_FileAppend1.Label") + realZipfilename + Messages.getString("JobZipFiles.Zip_FileAppend2.Label")); } if (log.isDetailed()) log.logDetailed(toString(), Messages.getString("JobZipFiles.Files_Found1.Label") + filelist.length + Messages.getString("JobZipFiles.Files_Found2.Label") + realTargetdirectory + Messages.getString("JobZipFiles.Files_Found3.Label")); Pattern pattern = null; Pattern patternexclude = null; // Let's prepare pattern..only if target is a folder ! if (f.isDirectory()) { if (!Const.isEmpty(realWildcard)) { pattern = Pattern.compile(realWildcard); } if (!Const.isEmpty(realWildcardExclude)) { patternexclude = Pattern.compile(realWildcardExclude); } } // Prepare Zip File buffer = new byte[18024]; dest = new FileOutputStream(realZipfilename); buff = new BufferedOutputStream(dest); out = new org.apache.tools.zip.ZipOutputStream(buff); HashSet<String> fileSet = new HashSet<String>(); if (renameOk) { // User want to append files to existing Zip file // The idea is to rename the existing zip file to a // temporary file // and then adds all entries in the existing zip // along with the new files, // excluding the zip entries that have the same name // as one of the new files. zin = new ZipInputStream(new FileInputStream(tempFile)); entry = (ZipEntry) zin.getNextEntry(); while (entry != null) { String name = entry.getName(); if (!fileSet.contains(name)) { // Add ZIP entry to output stream. out.putNextEntry(new ZipEntry(name)); // Transfer bytes from the ZIP file to the // output file int len; while ((len = zin.read(buffer)) > 0) { out.write(buffer, 0, len); } fileSet.add(name); } entry = (ZipEntry) zin.getNextEntry(); } // Close the streams zin.close(); } // Set the method out.setMethod(org.apache.tools.zip.ZipOutputStream.DEFLATED); // Set the compression level if (compressionrate == 0) { out.setLevel(Deflater.NO_COMPRESSION); } else if (compressionrate == 1) { out.setLevel(Deflater.DEFAULT_COMPRESSION); } if (compressionrate == 2) { out.setLevel(Deflater.BEST_COMPRESSION); } if (compressionrate == 3) { out.setLevel(Deflater.BEST_SPEED); } // Specify Zipped files (After that we will move,delete // them...) String[] ZippedFiles = new String[filelist.length]; int FileNum = 0; // Get the files in the list... for (int i = 0; i < filelist.length && !parentJob.isStopped(); i++) { boolean getIt = true; boolean getItexclude = false; // First see if the file matches the regular // expression! // ..only if target is a folder ! if (f.isDirectory()) { if (pattern != null) { Matcher matcher = pattern.matcher(filelist[i]); getIt = matcher.matches(); } if (patternexclude != null) { Matcher matcherexclude = patternexclude.matcher(filelist[i]); getItexclude = matcherexclude.matches(); } } // Get processing File String targetFilename = realTargetdirectory + Const.FILE_SEPARATOR + filelist[i]; if (f.isFile()) targetFilename = realTargetdirectory; File file = new File(targetFilename); if (getIt && !getItexclude && !file.isDirectory() && !fileSet.contains(filelist[i])) { // We can add the file to the Zip Archive if (log.isDebug()) log.logDebug(toString(), Messages.getString("JobZipFiles.Add_FilesToZip1.Label") + filelist[i] + Messages.getString("JobZipFiles.Add_FilesToZip2.Label") + realTargetdirectory + Messages.getString("JobZipFiles.Add_FilesToZip3.Label")); // Associate a file input stream for the current // file FileInputStream in = new FileInputStream(targetFilename); // Add ZIP entry to output stream. out.putNextEntry(new ZipEntry(filelist[i])); int len; while ((len = in.read(buffer)) > 0) { out.write(buffer, 0, len); } out.flush(); out.closeEntry(); // Close the current file input stream in.close(); // Get Zipped File ZippedFiles[FileNum] = filelist[i]; FileNum = FileNum + 1; } } // Close the ZipOutPutStream out.close(); buff.close(); dest.close(); if (log.isBasic()) log.logBasic(toString(), Messages.getString("JobZipFiles.Log.TotalZippedFiles", "" + ZippedFiles.length)); // Delete Temp File if (tempFile != null) { tempFile.delete(); } // -----Get the list of Zipped Files and Move or Delete // Them if (afterzip == 1 || afterzip == 2) { // iterate through the array of Zipped files for (int i = 0; i < ZippedFiles.length; i++) { if (ZippedFiles[i] != null) { // Delete File FileObject fileObjectd = KettleVFS.getFileObject( realTargetdirectory + Const.FILE_SEPARATOR + ZippedFiles[i]); if (f.isFile()) fileObjectd = KettleVFS.getFileObject(realTargetdirectory); // Here gc() is explicitly called if e.g. // createfile is used in the same // job for the same file. The problem is // that after creating the file the // file object is not properly garbaged // collected and thus the file cannot // be deleted anymore. This is a known // problem in the JVM. System.gc(); // Here we can move, delete files if (afterzip == 1) { // Delete File boolean deleted = fileObjectd.delete(); if (!deleted) { resultat = false; log.logError(toString(), Messages .getString("JobZipFiles.Cant_Delete_File1.Label") + realTargetdirectory + Const.FILE_SEPARATOR + ZippedFiles[i] + Messages.getString("JobZipFiles.Cant_Delete_File2.Label")); } // File deleted if (log.isDebug()) log.logDebug(toString(), Messages.getString("JobZipFiles.File_Deleted1.Label") + realTargetdirectory + Const.FILE_SEPARATOR + ZippedFiles[i] + Messages .getString("JobZipFiles.File_Deleted2.Label")); } else if (afterzip == 2) { // Move File try { FileObject fileObjectm = KettleVFS.getFileObject( realMovetodirectory + Const.FILE_SEPARATOR + ZippedFiles[i]); fileObjectd.moveTo(fileObjectm); } catch (IOException e) { log.logError(toString(), Messages.getString("JobZipFiles.Cant_Move_File1.Label") + ZippedFiles[i] + Messages .getString("JobZipFiles.Cant_Move_File2.Label") + e.getMessage()); resultat = false; } // File moved if (log.isDebug()) log.logDebug(toString(), Messages.getString("JobZipFiles.File_Moved1.Label") + ZippedFiles[i] + Messages.getString("JobZipFiles.File_Moved2.Label")); } } } } if (addfiletoresult) { // Add file to result files name ResultFile resultFile = new ResultFile(ResultFile.FILE_TYPE_GENERAL, KettleVFS.getFileObject(realZipfilename), parentJob.getJobname(), toString()); result.getResultFiles().put(resultFile.getFile().toString(), resultFile); } resultat = true; } } } catch (Exception e) { log.logError(toString(), Messages.getString("JobZipFiles.Cant_CreateZipFile1.Label") + realZipfilename + Messages.getString("JobZipFiles.Cant_CreateZipFile2.Label") + e.getMessage()); // result.setResult( false ); // result.setNrErrors(1); resultat = false; } finally { if (fileObject != null) { try { fileObject.close(); } catch (IOException ex) { } ; } // Close the ZipOutPutStream try { if (out != null) out.close(); if (buff != null) buff.close(); if (dest != null) dest.close(); if (zin != null) zin.close(); if (entry != null) entry = null; } catch (IOException ex) { } ; } } else { resultat = true; if (realZipfilename == null) log.logError(toString(), Messages.getString("JobZipFiles.No_ZipFile_Defined.Label")); if (!orginexist) log.logError(toString(), Messages.getString("JobZipFiles.No_FolderCible_Defined.Label", realTargetdirectory)); } // return a verifier return resultat; }
From source file:com.twinsoft.convertigo.engine.servlets.ReverseProxyServlet.java
/** * Executes the {@link HttpMethod} passed in and sends the proxy response * back to the client via the given {@link HttpServletResponse} * // w ww.j ava 2 s .c om * @param httpMethodProxyRequest * An object representing the proxy request to be made * @param httpServletResponse * An object by which we can send the proxied response back to * the client * @throws IOException * Can be thrown by the {@link HttpClient}.executeMethod * @throws ServletException * Can be thrown to indicate that another error has occurred * @throws EngineException */ private void doRequest(HttpMethodType httpMethodType, HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse) throws IOException, ServletException { try { Engine.logEngine.debug("(ReverseProxyServlet) Starting request handling"); if (Boolean.parseBoolean(EnginePropertiesManager.getProperty(PropertyName.SSL_DEBUG))) { System.setProperty("javax.net.debug", "all"); Engine.logEngine.trace("(ReverseProxyServlet) Enabling SSL debug mode"); } else { System.setProperty("javax.net.debug", ""); Engine.logEngine.debug("(ReverseProxyServlet) Disabling SSL debug mode"); } String baseUrl; String projectName; String connectorName; String contextName; String extraPath; { String requestURI = httpServletRequest.getRequestURI(); Engine.logEngine.trace("(ReverseProxyServlet) Requested URI : " + requestURI); Matcher m = reg_fields.matcher(requestURI); if (m.matches() && m.groupCount() >= 5) { baseUrl = m.group(1); projectName = m.group(2); connectorName = m.group(3); contextName = m.group(4); extraPath = m.group(5); } else { throw new MalformedURLException( "The request doesn't contains needed fields : projectName, connectorName and contextName"); } } String sessionID = httpServletRequest.getSession().getId(); Engine.logEngine.debug("(ReverseProxyServlet) baseUrl : " + baseUrl + " ; projectName : " + projectName + " ; connectorName : " + connectorName + " ; contextName : " + contextName + " ; extraPath : " + extraPath + " ; sessionID : " + sessionID); Context context = Engine.theApp.contextManager.get(null, contextName, sessionID, null, projectName, connectorName, null); Project project = Engine.theApp.databaseObjectsManager.getProjectByName(projectName); context.projectName = projectName; context.project = project; ProxyHttpConnector proxyHttpConnector = (ProxyHttpConnector) project.getConnectorByName(connectorName); context.connector = proxyHttpConnector; context.connectorName = proxyHttpConnector.getName(); HostConfiguration hostConfiguration = proxyHttpConnector.hostConfiguration; // Proxy configuration String proxyServer = Engine.theApp.proxyManager.getProxyServer(); String proxyUser = Engine.theApp.proxyManager.getProxyUser(); String proxyPassword = Engine.theApp.proxyManager.getProxyPassword(); int proxyPort = Engine.theApp.proxyManager.getProxyPort(); if (!proxyServer.equals("")) { hostConfiguration.setProxy(proxyServer, proxyPort); Engine.logEngine.debug("(ReverseProxyServlet) Using proxy: " + proxyServer + ":" + proxyPort); } else { // Remove old proxy configuration hostConfiguration.setProxyHost(null); } String targetHost = proxyHttpConnector.getServer(); Engine.logEngine.debug("(ReverseProxyServlet) Target host: " + targetHost); int targetPort = proxyHttpConnector.getPort(); Engine.logEngine.debug("(ReverseProxyServlet) Target port: " + targetPort); // Configuration SSL Engine.logEngine.debug("(ReverseProxyServlet) Https: " + proxyHttpConnector.isHttps()); CertificateManager certificateManager = proxyHttpConnector.certificateManager; boolean trustAllServerCertificates = proxyHttpConnector.isTrustAllServerCertificates(); if (proxyHttpConnector.isHttps()) { Engine.logEngine.debug("(ReverseProxyServlet) Setting up SSL properties"); certificateManager.collectStoreInformation(context); Engine.logEngine.debug( "(ReverseProxyServlet) CertificateManager has changed: " + certificateManager.hasChanged); if (certificateManager.hasChanged || (!targetHost.equalsIgnoreCase(hostConfiguration.getHost())) || (hostConfiguration.getPort() != targetPort)) { Engine.logEngine .debug("(ReverseProxyServlet) Using MySSLSocketFactory for creating the SSL socket"); Protocol myhttps = new Protocol("https", MySSLSocketFactory.getSSLSocketFactory(certificateManager.keyStore, certificateManager.keyStorePassword, certificateManager.trustStore, certificateManager.trustStorePassword, trustAllServerCertificates), targetPort); hostConfiguration.setHost(targetHost, targetPort, myhttps); } Engine.logEngine.debug("(ReverseProxyServlet) Updated host configuration for SSL purposes"); } else { hostConfiguration.setHost(targetHost, targetPort); } HttpMethod httpMethodProxyRequest; String targetPath = proxyHttpConnector.getBaseDir() + extraPath; // Handle the query string if (httpServletRequest.getQueryString() != null) { targetPath += "?" + httpServletRequest.getQueryString(); } Engine.logEngine.debug("(ReverseProxyServlet) Target path: " + targetPath); Engine.logEngine.debug("(ReverseProxyServlet) Requested method: " + httpMethodType); if (httpMethodType == HttpMethodType.GET) { // Create a GET request httpMethodProxyRequest = new GetMethod(); } else if (httpMethodType == HttpMethodType.POST) { // Create a standard POST request httpMethodProxyRequest = new PostMethod(); ((PostMethod) httpMethodProxyRequest) .setRequestEntity(new InputStreamRequestEntity(httpServletRequest.getInputStream())); } else { throw new IllegalArgumentException("Unknown HTTP method: " + httpMethodType); } String charset = httpMethodProxyRequest.getParams().getUriCharset(); URI targetURI; try { targetURI = new URI(targetPath, true, charset); } catch (URIException e) { // Bugfix #1484 String newTargetPath = ""; for (String part : targetPath.split("&")) { if (!newTargetPath.equals("")) { newTargetPath += "&"; } String[] pair = part.split("="); try { newTargetPath += URLDecoder.decode(pair[0], "UTF-8") + "=" + (pair.length > 1 ? URLEncoder.encode(URLDecoder.decode(pair[1], "UTF-8"), "UTF-8") : ""); } catch (UnsupportedEncodingException ee) { newTargetPath = targetPath; } } targetURI = new URI(newTargetPath, true, charset); } httpMethodProxyRequest.setURI(targetURI); // Tells the method to automatically handle authentication. httpMethodProxyRequest.setDoAuthentication(true); HttpState httpState = getHttpState(proxyHttpConnector, context); String basicUser = proxyHttpConnector.getAuthUser(); String basicPassword = proxyHttpConnector.getAuthPassword(); String givenBasicUser = proxyHttpConnector.getGivenAuthUser(); String givenBasicPassword = proxyHttpConnector.getGivenAuthPassword(); // Basic authentication configuration String realm = null; if (!basicUser.equals("") || (basicUser.equals("") && (givenBasicUser != null))) { String userName = ((givenBasicUser == null) ? basicUser : givenBasicUser); String userPassword = ((givenBasicPassword == null) ? basicPassword : givenBasicPassword); httpState.setCredentials(new AuthScope(targetHost, targetPort, realm), new UsernamePasswordCredentials(userName, userPassword)); Engine.logEngine.debug("(ReverseProxyServlet) Credentials: " + userName + ":******"); } // Setting basic authentication for proxy if (!proxyServer.equals("") && !proxyUser.equals("")) { httpState.setProxyCredentials(new AuthScope(proxyServer, proxyPort), new UsernamePasswordCredentials(proxyUser, proxyPassword)); Engine.logEngine.debug("(ReverseProxyServlet) Proxy credentials: " + proxyUser + ":******"); } // Forward the request headers setProxyRequestHeaders(httpServletRequest, httpMethodProxyRequest, proxyHttpConnector); // Use the CEMS HttpClient HttpClient httpClient = Engine.theApp.httpClient; httpMethodProxyRequest.setFollowRedirects(false); // Execute the request int intProxyResponseCode = httpClient.executeMethod(hostConfiguration, httpMethodProxyRequest, httpState); // Check if the proxy response is a redirect // The following code is adapted from // org.tigris.noodle.filters.CheckForRedirect // Hooray for open source software if (intProxyResponseCode >= HttpServletResponse.SC_MULTIPLE_CHOICES /* 300 */ && intProxyResponseCode < HttpServletResponse.SC_NOT_MODIFIED /* 304 */) { String stringStatusCode = Integer.toString(intProxyResponseCode); String stringLocation = httpMethodProxyRequest.getResponseHeader(STRING_LOCATION_HEADER).getValue(); if (stringLocation == null) { throw new ServletException("Received status code: " + stringStatusCode + " but no " + STRING_LOCATION_HEADER + " header was found in the response"); } // Modify the redirect to go to this proxy servlet rather that // the // proxied host String redirect = handleRedirect(stringLocation, baseUrl, proxyHttpConnector); httpServletResponse.sendRedirect(redirect); Engine.logEngine.debug("(ReverseProxyServlet) Send redirect (" + redirect + ")"); return; } else if (intProxyResponseCode == HttpServletResponse.SC_NOT_MODIFIED) { // 304 needs special handling. See: // http://www.ics.uci.edu/pub/ietf/http/rfc1945.html#Code304 // We get a 304 whenever passed an 'If-Modified-Since' // header and the data on disk has not changed; server // responds w/ a 304 saying I'm not going to send the // body because the file has not changed. httpServletResponse.setIntHeader(STRING_CONTENT_LENGTH_HEADER_NAME, 0); httpServletResponse.setStatus(HttpServletResponse.SC_NOT_MODIFIED); Engine.logEngine.debug("(ReverseProxyServlet) NOT MODIFIED (304)"); return; } // Pass the response code back to the client httpServletResponse.setStatus(intProxyResponseCode); // Pass response headers back to the client Engine.logEngine.debug("(ReverseProxyServlet) Response headers back to the client:"); Header[] headerArrayResponse = httpMethodProxyRequest.getResponseHeaders(); for (Header header : headerArrayResponse) { String headerName = header.getName(); String headerValue = header.getValue(); if (!headerName.equalsIgnoreCase("Transfer-Encoding") && !headerName.equalsIgnoreCase("Set-Cookie")) { httpServletResponse.setHeader(headerName, headerValue); Engine.logEngine.debug(" " + headerName + "=" + headerValue); } } String contentType = null; Header[] contentTypeHeaders = httpMethodProxyRequest.getResponseHeaders("Content-Type"); for (Header contentTypeHeader : contentTypeHeaders) { contentType = contentTypeHeader.getValue(); break; } String pageCharset = "UTF-8"; if (contentType != null) { int iCharset = contentType.indexOf("charset="); if (iCharset != -1) { pageCharset = contentType.substring(iCharset + "charset=".length()).trim(); } Engine.logEngine.debug("(ReverseProxyServlet) Using charset: " + pageCharset); } InputStream siteIn = httpMethodProxyRequest.getResponseBodyAsStream(); // Handle gzipped content Header[] contentEncodingHeaders = httpMethodProxyRequest.getResponseHeaders("Content-Encoding"); boolean bGZip = false, bDeflate = false; for (Header contentEncodingHeader : contentEncodingHeaders) { HeaderElement[] els = contentEncodingHeader.getElements(); for (int j = 0; j < els.length; j++) { if ("gzip".equals(els[j].getName())) { Engine.logBeans.debug("(ReverseProxyServlet) Decode GZip stream"); siteIn = new GZIPInputStream(siteIn); bGZip = true; } else if ("deflate".equals(els[j].getName())) { Engine.logBeans.debug("(ReverseProxyServlet) Decode Deflate stream"); siteIn = new InflaterInputStream(siteIn, new Inflater(true)); bDeflate = true; } } } byte[] bytesDataResult; ByteArrayOutputStream baos = new ByteArrayOutputStream(2048); // String resourceUrl = projectName + targetPath; String t = context.statistics.start(EngineStatistics.APPLY_USER_REQUEST); try { // Read either from the cache, either from the remote server // InputStream is = proxyCacheManager.getResource(resourceUrl); // if (is != null) { // Engine.logEngine.debug("(ReverseProxyServlet) Getting data from cache"); // siteIn = is; // } int c = siteIn.read(); while (c > -1) { baos.write(c); c = siteIn.read(); } // if (is != null) is.close(); } finally { context.statistics.stop(t, true); } bytesDataResult = baos.toByteArray(); baos.close(); Engine.logEngine.debug("(ReverseProxyServlet) Data retrieved!"); // if (isDynamicContent(httpServletRequest.getPathInfo(), // proxyHttpConnector.getDynamicContentFiles())) { Engine.logEngine.debug("(ReverseProxyServlet) Dynamic content"); bytesDataResult = handleStringReplacements(baseUrl, contentType, pageCharset, proxyHttpConnector, bytesDataResult); String billingClassName = context.getConnector().getBillingClassName(); if (billingClassName != null) { try { Engine.logContext.debug("Billing class name required: " + billingClassName); AbstractBiller biller = (AbstractBiller) Class.forName(billingClassName).newInstance(); Engine.logContext.debug("Executing the biller"); biller.insertBilling(context); } catch (Throwable e) { Engine.logContext.warn("Unable to execute the biller (the billing is thus ignored): [" + e.getClass().getName() + "] " + e.getMessage()); } } // } // else { // Engine.logEngine.debug("(ReverseProxyServlet) Static content: " + // contentType); // // // Determine if the resource has already been cached or not // CacheEntry cacheEntry = // proxyCacheManager.getCacheEntry(resourceUrl); // if (cacheEntry instanceof FileCacheEntry) { // FileCacheEntry fileCacheEntry = (FileCacheEntry) cacheEntry; // File file = new File(fileCacheEntry.fileName); // if (!file.exists()) // proxyCacheManager.removeCacheEntry(cacheEntry); // cacheEntry = null; // } // if (cacheEntry == null) { // bytesDataResult = handleStringReplacements(contentType, // proxyHttpConnector, bytesDataResult); // // if (intProxyResponseCode == 200) { // Engine.logEngine.debug("(ReverseProxyServlet) Resource stored: " // + resourceUrl); // cacheEntry = proxyCacheManager.storeResponse(resourceUrl, // bytesDataResult); // cacheEntry.contentLength = bytesDataResult.length; // cacheEntry.contentType = contentType; // Engine.logEngine.debug("(ReverseProxyServlet) Cache entry: " + // cacheEntry); // } // } // } // Send the content to the client if (Engine.logEngine.isDebugEnabled() && MimeType.Html.is(contentType)) { Engine.logEngine.debug("Data proxied:\n" + new String(bytesDataResult, pageCharset)); } if (bGZip || bDeflate) { baos = new ByteArrayOutputStream(); OutputStream compressedOutputStream = bGZip ? new GZIPOutputStream(baos) : new DeflaterOutputStream(baos, new Deflater(Deflater.DEFAULT_COMPRESSION | Deflater.DEFAULT_STRATEGY, true)); compressedOutputStream.write(bytesDataResult); compressedOutputStream.close(); bytesDataResult = baos.toByteArray(); baos.close(); } httpServletResponse.setContentLength(bytesDataResult.length); OutputStream outputStreamClientResponse = httpServletResponse.getOutputStream(); outputStreamClientResponse.write(bytesDataResult); Engine.logEngine.debug("(ReverseProxyServlet) End of document retransmission"); } catch (Exception e) { Engine.logEngine.error("Error while trying to proxy page", e); throw new ServletException("Error while trying to proxy page", e); } }
From source file:com.twinsoft.convertigo.beans.connectors.SiteClipperConnector.java
private void doProcessRequest(Shuttle shuttle) throws IOException, ServletException, EngineException { shuttle.statisticsTaskID = context.statistics.start(EngineStatistics.GET_DOCUMENT); try {/* w ww . j av a2s . c o m*/ shuttle.sharedScope = context.getSharedScope(); String domain = shuttle.getRequest(QueryPart.host) + shuttle.getRequest(QueryPart.port); Engine.logSiteClipper.trace("(SiteClipperConnector) Prepare the request for the domain " + domain); if (!shouldRewrite(domain)) { Engine.logSiteClipper.info( "(SiteClipperConnector) The domain " + domain + " is not allowed with this connector"); shuttle.response.sendError(HttpServletResponse.SC_FORBIDDEN, "The domain " + domain + " is not allowed with this connector"); return; } String uri = shuttle.getRequest(QueryPart.uri); Engine.logSiteClipper.info("Preparing " + shuttle.request.getMethod() + " " + shuttle.getRequestUrl()); HttpMethod httpMethod = null; XulRecorder xulRecorder = context.getXulRecorder(); if (xulRecorder != null) { httpMethod = shuttle.httpMethod = xulRecorder.getRecord(shuttle.getRequestUrlAndQuery()); } if (httpMethod == null) { try { switch (shuttle.getRequestHttpMethodType()) { case GET: httpMethod = new GetMethod(uri); break; case POST: httpMethod = new PostMethod(uri); ((PostMethod) httpMethod) .setRequestEntity(new InputStreamRequestEntity(shuttle.request.getInputStream())); break; case PUT: httpMethod = new PutMethod(uri); ((PutMethod) httpMethod) .setRequestEntity(new InputStreamRequestEntity(shuttle.request.getInputStream())); break; case DELETE: httpMethod = new DeleteMethod(uri); break; case HEAD: httpMethod = new HeadMethod(uri); break; case OPTIONS: httpMethod = new OptionsMethod(uri); break; case TRACE: httpMethod = new TraceMethod(uri); break; default: throw new ServletException( "(SiteClipperConnector) unknown http method " + shuttle.request.getMethod()); } httpMethod.setFollowRedirects(false); } catch (Exception e) { throw new ServletException( "(SiteClipperConnector) unexpected exception will building the http method : " + e.getMessage()); } shuttle.httpMethod = httpMethod; SiteClipperScreenClass screenClass = getCurrentScreenClass(); Engine.logSiteClipper.info("Request screen class: " + screenClass.getName()); for (String name : Collections .list(GenericUtils.<Enumeration<String>>cast(shuttle.request.getHeaderNames()))) { if (requestHeadersToIgnore.contains(HeaderName.parse(name))) { Engine.logSiteClipper.trace("(SiteClipperConnector) Ignoring request header " + name); } else { String value = shuttle.request.getHeader(name); Engine.logSiteClipper .trace("(SiteClipperConnector) Copying request header " + name + "=" + value); shuttle.setRequestCustomHeader(name, value); } } Engine.logSiteClipper.debug("(SiteClipperConnector) applying request rules for the screenclass " + screenClass.getName()); for (IRequestRule rule : screenClass.getRequestRules()) { if (rule.isEnabled()) { Engine.logSiteClipper .trace("(SiteClipperConnector) applying request rule " + rule.getName()); rule.fireEvents(); boolean done = rule.applyOnRequest(shuttle); Engine.logSiteClipper.debug("(SiteClipperConnector) the request rule " + rule.getName() + " is " + (done ? "well" : "not") + " applied"); } else { Engine.logSiteClipper .trace("(SiteClipperConnector) skip the disabled request rule " + rule.getName()); } } for (Entry<String, String> header : shuttle.requestCustomHeaders.entrySet()) { Engine.logSiteClipper.trace("(SiteClipperConnector) Push request header " + header.getKey() + "=" + header.getValue()); httpMethod.addRequestHeader(header.getKey(), header.getValue()); } String queryString = shuttle.request.getQueryString(); if (queryString != null) { try { // Fake test in order to check query string validity new URI("http://localhost/index?" + queryString, true, httpMethod.getParams().getUriCharset()); } catch (URIException e) { // Bugfix #2103 StringBuffer newQuery = new StringBuffer(); for (String part : RegexpUtils.pattern_and.split(queryString)) { String[] pair = RegexpUtils.pattern_equals.split(part, 2); try { newQuery.append('&') .append(URLEncoder.encode(URLDecoder.decode(pair[0], "UTF-8"), "UTF-8")); if (pair.length > 1) { newQuery.append('=').append( URLEncoder.encode(URLDecoder.decode(pair[1], "UTF-8"), "UTF-8")); } } catch (UnsupportedEncodingException ee) { Engine.logSiteClipper .trace("(SiteClipperConnector) failed to encode query part : " + part); } } queryString = newQuery.length() > 0 ? newQuery.substring(1) : newQuery.toString(); Engine.logSiteClipper.trace("(SiteClipperConnector) re-encode query : " + queryString); } } Engine.logSiteClipper.debug("(SiteClipperConnector) Copying the query string : " + queryString); httpMethod.setQueryString(queryString); // if (context.httpState == null) { // Engine.logSiteClipper.debug("(SiteClipperConnector) Creating new HttpState for context id " + context.contextID); // context.httpState = new HttpState(); // } else { // Engine.logSiteClipper.debug("(SiteClipperConnector) Using HttpState of context id " + context.contextID); // } getHttpState(shuttle); HostConfiguration hostConfiguration = getHostConfiguration(shuttle); HttpMethodParams httpMethodParams = httpMethod.getParams(); httpMethodParams.setBooleanParameter("http.connection.stalecheck", true); httpMethodParams.setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler(3, true)); Engine.logSiteClipper.info("Requesting " + httpMethod.getName() + " " + hostConfiguration.getHostURL() + httpMethod.getURI().toString()); HttpClient httpClient = context.getHttpClient3(shuttle.getHttpPool()); HttpUtils.logCurrentHttpConnection(httpClient, hostConfiguration, shuttle.getHttpPool()); httpClient.executeMethod(hostConfiguration, httpMethod, context.httpState); } else { Engine.logSiteClipper.info("Retrieve recorded response from Context"); } int status = httpMethod.getStatusCode(); shuttle.processState = ProcessState.response; Engine.logSiteClipper.info("Request terminated with status " + status); shuttle.response.setStatus(status); if (Engine.isStudioMode() && status == HttpServletResponse.SC_OK && shuttle.getResponseMimeType().startsWith("text/")) { fireDataChanged(new ConnectorEvent(this, shuttle.getResponseAsString())); } SiteClipperScreenClass screenClass = getCurrentScreenClass(); Engine.logSiteClipper.info("Response screen class: " + screenClass.getName()); if (Engine.isStudioMode()) { Engine.theApp.fireObjectDetected(new EngineEvent(screenClass)); } for (Header header : httpMethod.getResponseHeaders()) { String name = header.getName(); if (responseHeadersToIgnore.contains(HeaderName.parse(name))) { Engine.logSiteClipper.trace("(SiteClipperConnector) Ignoring response header " + name); } else { String value = header.getValue(); Engine.logSiteClipper .trace("(SiteClipperConnector) Copying response header " + name + "=" + value); shuttle.responseCustomHeaders.put(name, value); } } Engine.logSiteClipper.debug( "(SiteClipperConnector) applying response rules for the screenclass " + screenClass.getName()); for (IResponseRule rule : screenClass.getResponseRules()) { if (rule.isEnabled()) { Engine.logSiteClipper.trace("(SiteClipperConnector) applying response rule " + rule.getName()); rule.fireEvents(); boolean done = rule.applyOnResponse(shuttle); Engine.logSiteClipper.debug("(SiteClipperConnector) the response rule " + rule.getName() + " is " + (done ? "well" : "not") + " applied"); } else { Engine.logSiteClipper .trace("(SiteClipperConnector) skip the disabled response rule " + rule.getName()); } } for (Entry<String, String> header : shuttle.responseCustomHeaders.entrySet()) { Engine.logSiteClipper.trace( "(SiteClipperConnector) Push request header " + header.getKey() + "=" + header.getValue()); shuttle.response.addHeader(header.getKey(), header.getValue()); } if (shuttle.postInstructions != null) { JSONArray instructions = new JSONArray(); for (IClientInstruction instruction : shuttle.postInstructions) { try { instructions.put(instruction.getInstruction()); } catch (JSONException e) { Engine.logSiteClipper.error( "(SiteClipperConnector) Failed to add a post instruction due to a JSONException", e); } } String codeToInject = "<script>C8O_postInstructions = " + instructions.toString() + "</script>\n" + "<script src=\"" + shuttle.getRequest(QueryPart.full_convertigo_path) + "/scripts/jquery.min.js\"></script>\n" + "<script src=\"" + shuttle.getRequest(QueryPart.full_convertigo_path) + "/scripts/siteclipper.js\"></script>\n"; String content = shuttle.getResponseAsString(); Matcher matcher = HtmlLocation.head_top.matcher(content); String newContent = RegexpUtils.inject(matcher, codeToInject); if (newContent == null) { matcher = HtmlLocation.body_top.matcher(content); newContent = RegexpUtils.inject(matcher, codeToInject); } if (newContent != null) { shuttle.setResponseAsString(newContent); } else { Engine.logSiteClipper.info( "(SiteClipperConnector) Failed to find a head or body tag in the response content"); Engine.logSiteClipper.trace("(SiteClipperConnector) Response content : \"" + content + "\""); } } long nbBytes = 0L; if (shuttle.responseAsString != null && shuttle.responseAsString.hashCode() != shuttle.responseAsStringOriginal.hashCode()) { OutputStream os = shuttle.response.getOutputStream(); switch (shuttle.getResponseContentEncoding()) { case gzip: os = new GZIPOutputStream(os); break; case deflate: os = new DeflaterOutputStream(os, new Deflater(Deflater.DEFAULT_COMPRESSION | Deflater.DEFAULT_STRATEGY, true)); break; default: break; } nbBytes = shuttle.responseAsByte.length; IOUtils.write(shuttle.responseAsString, os, shuttle.getResponseCharset()); os.close(); } else { InputStream is = (shuttle.responseAsByte == null) ? httpMethod.getResponseBodyAsStream() : new ByteArrayInputStream(shuttle.responseAsByte); if (is != null) { nbBytes = 0; OutputStream os = shuttle.response.getOutputStream(); int read = is.read(); while (read >= 0) { os.write(read); os.flush(); read = is.read(); nbBytes++; } is.close(); // nbBytes = IOUtils.copyLarge(is, shuttle.response.getOutputStream()); Engine.logSiteClipper .trace("(SiteClipperConnector) Response body copyied (" + nbBytes + " bytes)"); } } shuttle.response.getOutputStream().close(); shuttle.score = getScore(nbBytes); Engine.logSiteClipper .debug("(SiteClipperConnector) Request terminated with a score of " + shuttle.score); } finally { long duration = context.statistics.stop(shuttle.statisticsTaskID); if (context.requestedObject != null) { try { Engine.theApp.billingManager.insertBilling(context, Long.valueOf(duration), Long.valueOf(shuttle.score)); } catch (Exception e) { Engine.logContext.warn("Unable to insert billing ticket (the billing is thus ignored): [" + e.getClass().getName() + "] " + e.getMessage()); } } } }