List of usage examples for org.apache.commons.vfs FileObject toString
public String toString()
From source file:org.pentaho.di.job.entries.xmlwellformed.JobEntryXMLWellFormed.java
private boolean processFileFolder(String sourcefilefoldername, String wildcard, Job parentJob, Result result) { boolean entrystatus = false; FileObject sourcefilefolder = null; FileObject CurrentFile = null; // Get real source file and wilcard String realSourceFilefoldername = environmentSubstitute(sourcefilefoldername); if (Const.isEmpty(realSourceFilefoldername)) { logError(BaseMessages.getString(PKG, "JobXMLWellFormed.log.FileFolderEmpty", sourcefilefoldername)); // Update Errors updateErrors();/*w w w.ja v a2 s .c om*/ return entrystatus; } String realWildcard = environmentSubstitute(wildcard); try { sourcefilefolder = KettleVFS.getFileObject(realSourceFilefoldername, this); if (sourcefilefolder.exists()) { if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobXMLWellFormed.Log.FileExists", sourcefilefolder.toString())); } if (sourcefilefolder.getType() == FileType.FILE) { entrystatus = checkOneFile(sourcefilefolder, result, parentJob); } else if (sourcefilefolder.getType() == FileType.FOLDER) { FileObject[] fileObjects = sourcefilefolder.findFiles(new AllFileSelector() { public boolean traverseDescendents(FileSelectInfo info) { return true; } public boolean includeFile(FileSelectInfo info) { FileObject fileObject = info.getFile(); try { if (fileObject == null) { return false; } if (fileObject.getType() != FileType.FILE) { return false; } } catch (Exception ex) { // Upon error don't process the file. return false; } finally { if (fileObject != null) { try { fileObject.close(); } catch (IOException ex) { /* Ignore */ } } } return true; } }); if (fileObjects != null) { for (int j = 0; j < fileObjects.length && !parentJob.isStopped(); j++) { if (successConditionBroken) { if (!successConditionBrokenExit) { logError(BaseMessages.getString(PKG, "JobXMLWellFormed.Error.SuccessConditionbroken", "" + NrAllErrors)); successConditionBrokenExit = true; } return false; } // Fetch files in list one after one ... CurrentFile = fileObjects[j]; if (!CurrentFile.getParent().toString().equals(sourcefilefolder.toString())) { // Not in the Base Folder..Only if include sub folders if (include_subfolders) { if (GetFileWildcard(CurrentFile.toString(), realWildcard)) { checkOneFile(CurrentFile, result, parentJob); } } } else { // In the base folder if (GetFileWildcard(CurrentFile.toString(), realWildcard)) { checkOneFile(CurrentFile, result, parentJob); } } } } } else { logError(BaseMessages.getString(PKG, "JobXMLWellFormed.Error.UnknowFileFormat", sourcefilefolder.toString())); // Update Errors updateErrors(); } } else { logError(BaseMessages.getString(PKG, "JobXMLWellFormed.Error.SourceFileNotExists", realSourceFilefoldername)); // Update Errors updateErrors(); } } catch (Exception e) { logError(BaseMessages.getString(PKG, "JobXMLWellFormed.Error.Exception.Processing", realSourceFilefoldername.toString(), e)); // Update Errors updateErrors(); } finally { if (sourcefilefolder != null) { try { sourcefilefolder.close(); } catch (IOException ex) { /* Ignore */ } } if (CurrentFile != null) { try { CurrentFile.close(); } catch (IOException ex) { /* Ignore */ } } } return entrystatus; }
From source file:org.pentaho.di.job.entries.xmlwellformed.JobEntryXMLWellFormed.java
private boolean checkOneFile(FileObject file, Result result, Job parentJob) throws KettleException { boolean retval = false; try {//from www . j a v a2 s .co m // We deal with a file..so let's check if it's well formed boolean retformed = CheckFile(file); if (!retformed) { logError(BaseMessages.getString(PKG, "JobXMLWellFormed.Error.FileBadFormed", file.toString())); // Update Bad formed files number updateBadFormed(); if (resultfilenames.equals(ADD_ALL_FILENAMES) || resultfilenames.equals(ADD_BAD_FORMED_FILES_ONLY)) { addFileToResultFilenames(KettleVFS.getFilename(file), result, parentJob); } } else { if (log.isDetailed()) { logDetailed("---------------------------"); logDetailed( BaseMessages.getString(PKG, "JobXMLWellFormed.Error.FileWellFormed", file.toString())); } // Update Well formed files number updateWellFormed(); if (resultfilenames.equals(ADD_ALL_FILENAMES) || resultfilenames.equals(ADD_WELL_FORMED_FILES_ONLY)) { addFileToResultFilenames(KettleVFS.getFilename(file), result, parentJob); } } } catch (Exception e) { throw new KettleException("Unable to verify file '" + file + "'", e); } return retval; }
From source file:org.pentaho.di.job.entries.zipfile.JobEntryZipFile.java
public boolean processRowFile(Job parentJob, Result result, String realZipfilename, String realWildcard, String realWildcardExclude, String realSourceDirectoryOrFile, String realMovetodirectory, boolean createparentfolder) { boolean Fileexists = false; File tempFile = null;/*from www . j a v a2 s.c o m*/ File fileZip = null; boolean resultat = false; boolean renameOk = false; boolean orginExist = false; // Check if target file/folder exists! FileObject originFile = null; ZipInputStream zin = null; byte[] buffer = null; OutputStream dest = null; BufferedOutputStream buff = null; ZipOutputStream out = null; ZipEntry entry = null; String localSourceFilename = realSourceDirectoryOrFile; try { originFile = KettleVFS.getFileObject(realSourceDirectoryOrFile, this); localSourceFilename = KettleVFS.getFilename(originFile); orginExist = originFile.exists(); } catch (Exception e) { // Ignore errors } finally { if (originFile != null) { try { originFile.close(); } catch (IOException ex) { logError("Error closing file '" + originFile.toString() + "'", ex); } } } String localrealZipfilename = realZipfilename; if (realZipfilename != null && orginExist) { FileObject fileObject = null; try { fileObject = KettleVFS.getFileObject(localrealZipfilename, this); localrealZipfilename = KettleVFS.getFilename(fileObject); // Check if Zip File exists if (fileObject.exists()) { Fileexists = true; if (log.isDebug()) { logDebug(BaseMessages.getString(PKG, "JobZipFiles.Zip_FileExists1.Label") + localrealZipfilename + BaseMessages.getString(PKG, "JobZipFiles.Zip_FileExists2.Label")); } } // Let's see if we need to create parent folder of destination zip filename if (createparentfolder) { createParentFolder(localrealZipfilename); } // Let's start the process now if (ifZipFileExists == 3 && Fileexists) { // the zip file exists and user want to Fail resultat = false; } else if (ifZipFileExists == 2 && Fileexists) { // the zip file exists and user want to do nothing if (addFileToResult) { // Add file to result files name ResultFile resultFile = new ResultFile(ResultFile.FILE_TYPE_GENERAL, fileObject, parentJob.getJobname(), toString()); result.getResultFiles().put(resultFile.getFile().toString(), resultFile); } resultat = true; } else if (afterZip == 2 && realMovetodirectory == null) { // After Zip, Move files..User must give a destination Folder resultat = false; logError( BaseMessages.getString(PKG, "JobZipFiles.AfterZip_No_DestinationFolder_Defined.Label")); } else { // After Zip, Move files..User must give a destination Folder // Let's see if we deal with file or folder FileObject[] fileList = null; FileObject sourceFileOrFolder = KettleVFS.getFileObject(localSourceFilename); boolean isSourceDirectory = sourceFileOrFolder.getType().equals(FileType.FOLDER); final Pattern pattern; final Pattern patternexclude; if (isSourceDirectory) { // Let's prepare the pattern matcher for performance reasons. // We only do this if the target is a folder ! // if (!Const.isEmpty(realWildcard)) { pattern = Pattern.compile(realWildcard); } else { pattern = null; } if (!Const.isEmpty(realWildcardExclude)) { patternexclude = Pattern.compile(realWildcardExclude); } else { patternexclude = null; } // Target is a directory // Get all the files in the directory... // if (includingSubFolders) { fileList = sourceFileOrFolder.findFiles(new FileSelector() { public boolean traverseDescendents(FileSelectInfo fileInfo) throws Exception { return true; } public boolean includeFile(FileSelectInfo fileInfo) throws Exception { boolean include; // Only include files in the sub-folders... // When we include sub-folders we match the whole filename, not just the base-name // if (fileInfo.getFile().getType().equals(FileType.FILE)) { include = true; if (pattern != null) { String name = fileInfo.getFile().getName().getPath(); include = pattern.matcher(name).matches(); } if (include && patternexclude != null) { String name = fileInfo.getFile().getName().getPath(); include = !pattern.matcher(name).matches(); } } else { include = false; } return include; } }); } else { fileList = sourceFileOrFolder.getChildren(); } } else { pattern = null; patternexclude = null; // Target is a file fileList = new FileObject[] { sourceFileOrFolder }; } if (fileList.length == 0) { resultat = false; logError(BaseMessages.getString(PKG, "JobZipFiles.Log.FolderIsEmpty", localSourceFilename)); } else if (!checkContainsFile(localSourceFilename, fileList, isSourceDirectory)) { resultat = false; logError(BaseMessages.getString(PKG, "JobZipFiles.Log.NoFilesInFolder", localSourceFilename)); } else { if (ifZipFileExists == 0 && Fileexists) { // the zip file exists and user want to create new one with unique name // Format Date // do we have already a .zip at the end? if (localrealZipfilename.toLowerCase().endsWith(".zip")) { // strip this off localrealZipfilename = localrealZipfilename.substring(0, localrealZipfilename.length() - 4); } localrealZipfilename += "_" + StringUtil.getFormattedDateTimeNow(true) + ".zip"; if (log.isDebug()) { logDebug(BaseMessages.getString(PKG, "JobZipFiles.Zip_FileNameChange1.Label") + localrealZipfilename + BaseMessages.getString(PKG, "JobZipFiles.Zip_FileNameChange1.Label")); } } else if (ifZipFileExists == 1 && Fileexists) { // the zip file exists and user want to append // get a temp file fileZip = getFile(localrealZipfilename); tempFile = File.createTempFile(fileZip.getName(), null); // delete it, otherwise we cannot rename existing zip to it. tempFile.delete(); renameOk = fileZip.renameTo(tempFile); if (!renameOk) { logError(BaseMessages.getString(PKG, "JobZipFiles.Cant_Rename_Temp1.Label") + fileZip.getAbsolutePath() + BaseMessages.getString(PKG, "JobZipFiles.Cant_Rename_Temp2.Label") + tempFile.getAbsolutePath() + BaseMessages.getString(PKG, "JobZipFiles.Cant_Rename_Temp3.Label")); } if (log.isDebug()) { logDebug(BaseMessages.getString(PKG, "JobZipFiles.Zip_FileAppend1.Label") + localrealZipfilename + BaseMessages.getString(PKG, "JobZipFiles.Zip_FileAppend2.Label")); } } if (log.isDetailed()) { logDetailed( BaseMessages.getString(PKG, "JobZipFiles.Files_Found1.Label") + fileList.length + BaseMessages.getString(PKG, "JobZipFiles.Files_Found2.Label") + localSourceFilename + BaseMessages.getString(PKG, "JobZipFiles.Files_Found3.Label")); } // Prepare Zip File buffer = new byte[18024]; dest = KettleVFS.getOutputStream(localrealZipfilename, false); buff = new BufferedOutputStream(dest); out = new ZipOutputStream(buff); HashSet<String> fileSet = new HashSet<String>(); if (renameOk) { // User want to append files to existing Zip file // The idea is to rename the existing zip file to a temporary file // and then adds all entries in the existing zip along with the new files, // excluding the zip entries that have the same name as one of the new files. zin = new ZipInputStream(new FileInputStream(tempFile)); entry = zin.getNextEntry(); while (entry != null) { String name = entry.getName(); if (!fileSet.contains(name)) { // Add ZIP entry to output stream. out.putNextEntry(new ZipEntry(name)); // Transfer bytes from the ZIP file to the output file int len; while ((len = zin.read(buffer)) > 0) { out.write(buffer, 0, len); } fileSet.add(name); } entry = zin.getNextEntry(); } // Close the streams zin.close(); } // Set the method out.setMethod(ZipOutputStream.DEFLATED); // Set the compression level if (compressionRate == 0) { out.setLevel(Deflater.NO_COMPRESSION); } else if (compressionRate == 1) { out.setLevel(Deflater.DEFAULT_COMPRESSION); } if (compressionRate == 2) { out.setLevel(Deflater.BEST_COMPRESSION); } if (compressionRate == 3) { out.setLevel(Deflater.BEST_SPEED); } // Specify Zipped files (After that we will move,delete them...) FileObject[] zippedFiles = new FileObject[fileList.length]; int fileNum = 0; // Get the files in the list... for (int i = 0; i < fileList.length && !parentJob.isStopped(); i++) { boolean getIt = true; boolean getItexclude = false; // First see if the file matches the regular expression! // ..only if target is a folder ! if (isSourceDirectory) { // If we include sub-folders, we match on the whole name, not just the basename // String filename; if (includingSubFolders) { filename = fileList[i].getName().getPath(); } else { filename = fileList[i].getName().getBaseName(); } if (pattern != null) { // Matches the base name of the file (backward compatible!) // Matcher matcher = pattern.matcher(filename); getIt = matcher.matches(); } if (patternexclude != null) { Matcher matcherexclude = patternexclude.matcher(filename); getItexclude = matcherexclude.matches(); } } // Get processing File String targetFilename = KettleVFS.getFilename(fileList[i]); if (sourceFileOrFolder.getType().equals(FileType.FILE)) { targetFilename = localSourceFilename; } FileObject file = KettleVFS.getFileObject(targetFilename); boolean isTargetDirectory = file.exists() && file.getType().equals(FileType.FOLDER); if (getIt && !getItexclude && !isTargetDirectory && !fileSet.contains(targetFilename)) { // We can add the file to the Zip Archive if (log.isDebug()) { logDebug(BaseMessages.getString(PKG, "JobZipFiles.Add_FilesToZip1.Label") + fileList[i] + BaseMessages.getString(PKG, "JobZipFiles.Add_FilesToZip2.Label") + localSourceFilename + BaseMessages.getString(PKG, "JobZipFiles.Add_FilesToZip3.Label")); } // Associate a file input stream for the current file InputStream in = KettleVFS.getInputStream(file); // Add ZIP entry to output stream. // String relativeName; String fullName = fileList[i].getName().getPath(); String basePath = sourceFileOrFolder.getName().getPath(); if (isSourceDirectory) { if (fullName.startsWith(basePath)) { relativeName = fullName.substring(basePath.length() + 1); } else { relativeName = fullName; } } else if (isFromPrevious) { int depth = determineDepth(environmentSubstitute(storedSourcePathDepth)); relativeName = determineZipfilenameForDepth(fullName, depth); } else { relativeName = fileList[i].getName().getBaseName(); } out.putNextEntry(new ZipEntry(relativeName)); int len; while ((len = in.read(buffer)) > 0) { out.write(buffer, 0, len); } out.flush(); out.closeEntry(); // Close the current file input stream in.close(); // Get Zipped File zippedFiles[fileNum] = fileList[i]; fileNum = fileNum + 1; } } // Close the ZipOutPutStream out.close(); buff.close(); dest.close(); if (log.isBasic()) { logBasic(BaseMessages.getString(PKG, "JobZipFiles.Log.TotalZippedFiles", "" + zippedFiles.length)); } // Delete Temp File if (tempFile != null) { tempFile.delete(); } // -----Get the list of Zipped Files and Move or Delete Them if (afterZip == 1 || afterZip == 2) { // iterate through the array of Zipped files for (int i = 0; i < zippedFiles.length; i++) { if (zippedFiles[i] != null) { // Delete, Move File FileObject fileObjectd = zippedFiles[i]; if (!isSourceDirectory) { fileObjectd = KettleVFS.getFileObject(localSourceFilename); } // Here we can move, delete files if (afterZip == 1) { // Delete File boolean deleted = fileObjectd.delete(); if (!deleted) { resultat = false; logError(BaseMessages.getString(PKG, "JobZipFiles.Cant_Delete_File1.Label") + localSourceFilename + Const.FILE_SEPARATOR + zippedFiles[i] + BaseMessages .getString(PKG, "JobZipFiles.Cant_Delete_File2.Label")); } // File deleted if (log.isDebug()) { logDebug(BaseMessages.getString(PKG, "JobZipFiles.File_Deleted1.Label") + localSourceFilename + Const.FILE_SEPARATOR + zippedFiles[i] + BaseMessages.getString(PKG, "JobZipFiles.File_Deleted2.Label")); } } else if (afterZip == 2) { // Move File FileObject fileObjectm = null; try { fileObjectm = KettleVFS.getFileObject(realMovetodirectory + Const.FILE_SEPARATOR + fileObjectd.getName().getBaseName()); fileObjectd.moveTo(fileObjectm); } catch (IOException e) { logError( BaseMessages.getString(PKG, "JobZipFiles.Cant_Move_File1.Label") + zippedFiles[i] + BaseMessages.getString(PKG, "JobZipFiles.Cant_Move_File2.Label") + e.getMessage()); resultat = false; } finally { try { if (fileObjectm != null) { fileObjectm.close(); } } catch (Exception e) { if (fileObjectm != null) { logError("Error closing file '" + fileObjectm.toString() + "'", e); } } } // File moved if (log.isDebug()) { logDebug(BaseMessages.getString(PKG, "JobZipFiles.File_Moved1.Label") + zippedFiles[i] + BaseMessages.getString(PKG, "JobZipFiles.File_Moved2.Label")); } } } } } if (addFileToResult) { // Add file to result files name ResultFile resultFile = new ResultFile(ResultFile.FILE_TYPE_GENERAL, fileObject, parentJob.getJobname(), toString()); result.getResultFiles().put(resultFile.getFile().toString(), resultFile); } resultat = true; } } } catch (Exception e) { logError(BaseMessages.getString(PKG, "JobZipFiles.Cant_CreateZipFile1.Label") + localrealZipfilename + BaseMessages.getString(PKG, "JobZipFiles.Cant_CreateZipFile2.Label"), e); resultat = false; } finally { if (fileObject != null) { try { fileObject.close(); fileObject = null; } catch (IOException ex) { logError("Error closing file '" + fileObject.toString() + "'", ex); } } try { if (out != null) { out.close(); } if (buff != null) { buff.close(); } if (dest != null) { dest.close(); } if (zin != null) { zin.close(); } if (entry != null) { entry = null; } } catch (IOException ex) { logError("Error closing zip file entry for file '" + originFile.toString() + "'", ex); } } } else { resultat = true; if (localrealZipfilename == null) { logError(BaseMessages.getString(PKG, "JobZipFiles.No_ZipFile_Defined.Label")); } if (!orginExist) { logError(BaseMessages.getString(PKG, "JobZipFiles.No_FolderCible_Defined.Label", localSourceFilename)); } } // return a verifier return resultat; }
From source file:org.pentaho.di.repository.filerep.KettleFileRepository.java
public ObjectId renameRepositoryDirectory(ObjectId id, RepositoryDirectoryInterface newParentDir, String newName) throws KettleException { if (newParentDir != null || newName != null) { try {//from www . j a va 2 s . c o m // In case of a root object, the ID is the same as the relative filename... RepositoryDirectoryInterface tree = loadRepositoryDirectoryTree(); RepositoryDirectoryInterface dir = tree.findDirectory(id); if (dir == null) { throw new KettleException("Could not find folder [" + id + "]"); } // If newName is null, keep the current name newName = (newName != null) ? newName : dir.getName(); FileObject folder = KettleVFS.getFileObject(dir.getPath()); String newFolderName = null; if (newParentDir != null) { FileObject newParentFolder = KettleVFS.getFileObject(newParentDir.getPath()); newFolderName = newParentFolder.toString() + "/" + newName; } else { newFolderName = folder.getParent().toString() + "/" + newName; } FileObject newFolder = KettleVFS.getFileObject(newFolderName); folder.moveTo(newFolder); return new StringObjectId(dir.getObjectId()); } catch (Exception e) { throw new KettleException("Unable to rename directory folder to [" + id + "]"); } } return (id); }
From source file:org.pentaho.di.trans.steps.blockingstep.BlockingStep.java
private Object[] getBuffer() { Object[] retval;/*from www. j a v a 2 s . c om*/ // Open all files at once and read one row from each file... if (data.files.size() > 0 && (data.dis.size() == 0 || data.fis.size() == 0)) { if (log.isBasic()) { logBasic(BaseMessages.getString(PKG, "BlockingStep.Log.Openfiles")); } try { FileObject fileObject = data.files.get(0); String filename = KettleVFS.getFilename(fileObject); if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "BlockingStep.Log.Openfilename1") + filename + BaseMessages.getString(PKG, "BlockingStep.Log.Openfilename2")); } InputStream fi = KettleVFS.getInputStream(fileObject); DataInputStream di; data.fis.add(fi); if (meta.getCompress()) { GZIPInputStream gzfi = new GZIPInputStream(new BufferedInputStream(fi)); di = new DataInputStream(gzfi); data.gzis.add(gzfi); } else { di = new DataInputStream(fi); } data.dis.add(di); // How long is the buffer? int buffersize = di.readInt(); if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "BlockingStep.Log.BufferSize1") + filename + BaseMessages.getString(PKG, "BlockingStep.Log.BufferSize2") + buffersize + " " + BaseMessages.getString(PKG, "BlockingStep.Log.BufferSize3")); } if (buffersize > 0) { // Read a row from temp-file data.rowbuffer.add(data.outputRowMeta.readData(di)); } } catch (Exception e) { logError(BaseMessages.getString(PKG, "BlockingStepMeta.ErrorReadingFile") + e.toString()); logError(Const.getStackTracker(e)); } } if (data.files.size() == 0) { if (data.buffer.size() > 0) { retval = data.buffer.get(0); data.buffer.remove(0); } else { retval = null; } } else { if (data.rowbuffer.size() == 0) { retval = null; } else { retval = data.rowbuffer.get(0); data.rowbuffer.remove(0); // now get another FileObject file = data.files.get(0); DataInputStream di = data.dis.get(0); InputStream fi = data.fis.get(0); GZIPInputStream gzfi = (meta.getCompress()) ? data.gzis.get(0) : null; try { data.rowbuffer.add(0, data.outputRowMeta.readData(di)); } catch (SocketTimeoutException e) { logError(BaseMessages.getString(PKG, "System.Log.UnexpectedError") + " : " + e.toString()); logError(Const.getStackTracker(e)); setErrors(1); stopAll(); } catch (KettleFileException fe) { // empty file or EOF mostly try { di.close(); fi.close(); if (gzfi != null) { gzfi.close(); } file.delete(); } catch (IOException e) { logError( BaseMessages.getString(PKG, "BlockingStepMeta.UnableDeleteFile") + file.toString()); setErrors(1); stopAll(); return null; } data.files.remove(0); data.dis.remove(0); data.fis.remove(0); if (gzfi != null) { data.gzis.remove(0); } } } } return retval; }
From source file:org.pentaho.di.trans.steps.mail.Mail.java
private void setAttachedFilesList(Object[] r, LogChannelInterface log) throws Exception { String realSourceFileFoldername = null; String realSourceWildcard = null; FileObject sourcefile = null; FileObject file = null;//from w w w. j av a2 s. co m ZipOutputStream zipOutputStream = null; File masterZipfile = null; if (meta.isZipFilenameDynamic()) { data.ZipFilename = data.previousRowMeta.getString(r, data.indexOfDynamicZipFilename); } try { if (meta.isDynamicFilename()) { // dynamic attached filenames if (data.indexOfSourceFilename > -1) { realSourceFileFoldername = data.previousRowMeta.getString(r, data.indexOfSourceFilename); } if (data.indexOfSourceWildcard > -1) { realSourceWildcard = data.previousRowMeta.getString(r, data.indexOfSourceWildcard); } } else { // static attached filenames realSourceFileFoldername = data.realSourceFileFoldername; realSourceWildcard = data.realSourceWildcard; } if (!Const.isEmpty(realSourceFileFoldername)) { sourcefile = KettleVFS.getFileObject(realSourceFileFoldername, getTransMeta()); if (sourcefile.exists()) { long FileSize = 0; FileObject[] list = null; if (sourcefile.getType() == FileType.FILE) { list = new FileObject[1]; list[0] = sourcefile; } else { list = sourcefile .findFiles(new TextFileSelector(sourcefile.toString(), realSourceWildcard)); } if (list.length > 0) { boolean zipFiles = meta.isZipFiles(); if (zipFiles && data.zipFileLimit == 0) { masterZipfile = new File( System.getProperty("java.io.tmpdir") + Const.FILE_SEPARATOR + data.ZipFilename); zipOutputStream = new ZipOutputStream(new FileOutputStream(masterZipfile)); } for (int i = 0; i < list.length; i++) { file = KettleVFS.getFileObject(KettleVFS.getFilename(list[i]), getTransMeta()); if (zipFiles) { if (data.zipFileLimit == 0) { ZipEntry zipEntry = new ZipEntry(file.getName().getBaseName()); zipOutputStream.putNextEntry(zipEntry); // Now put the content of this file into this archive... BufferedInputStream inputStream = new BufferedInputStream( file.getContent().getInputStream()); int c; while ((c = inputStream.read()) >= 0) { zipOutputStream.write(c); } inputStream.close(); zipOutputStream.closeEntry(); } else { FileSize += file.getContent().getSize(); } } else { addAttachedFilePart(file); } } // end for if (zipFiles) { if (isDebug()) { logDebug(BaseMessages.getString(PKG, "Mail.Log.FileSize", "" + FileSize)); } if (isDebug()) { logDebug(BaseMessages.getString(PKG, "Mail.Log.LimitSize", "" + data.zipFileLimit)); } if (data.zipFileLimit > 0 && FileSize > data.zipFileLimit) { masterZipfile = new File(System.getProperty("java.io.tmpdir") + Const.FILE_SEPARATOR + data.ZipFilename); zipOutputStream = new ZipOutputStream(new FileOutputStream(masterZipfile)); for (int i = 0; i < list.length; i++) { file = KettleVFS.getFileObject(KettleVFS.getFilename(list[i]), getTransMeta()); ZipEntry zipEntry = new ZipEntry(file.getName().getBaseName()); zipOutputStream.putNextEntry(zipEntry); // Now put the content of this file into this archive... BufferedInputStream inputStream = new BufferedInputStream( file.getContent().getInputStream()); int c; while ((c = inputStream.read()) >= 0) { zipOutputStream.write(c); } inputStream.close(); zipOutputStream.closeEntry(); } } if (data.zipFileLimit > 0 && FileSize > data.zipFileLimit || data.zipFileLimit == 0) { file = KettleVFS.getFileObject(masterZipfile.getAbsolutePath(), getTransMeta()); addAttachedFilePart(file); } } } } else { logError(BaseMessages.getString(PKG, "Mail.Error.SourceFileFolderNotExists", realSourceFileFoldername)); } } } catch (Exception e) { logError(e.getMessage()); } finally { if (sourcefile != null) { try { sourcefile.close(); } catch (Exception e) { // Ignore errors } } if (file != null) { try { file.close(); } catch (Exception e) { // Ignore errors } } if (zipOutputStream != null) { try { zipOutputStream.finish(); zipOutputStream.close(); } catch (IOException e) { logError("Unable to close attachement zip file archive : " + e.toString()); } } } }
From source file:org.pentaho.di.trans.steps.processfiles.ProcessFiles.java
public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException { meta = (ProcessFilesMeta) smi;/*from w w w .j a v a2 s. c o m*/ data = (ProcessFilesData) sdi; Object[] r = getRow(); // Get row from input rowset & set row busy! if (r == null) { // no more input to be expected... setOutputDone(); return false; } if (first) { first = false; // Check is source filename field is provided if (Const.isEmpty(meta.getDynamicSourceFileNameField())) { throw new KettleException( BaseMessages.getString(PKG, "ProcessFiles.Error.SourceFilenameFieldMissing")); } // Check is target filename field is provided if (meta.getOperationType() != ProcessFilesMeta.OPERATION_TYPE_DELETE && Const.isEmpty(meta.getDynamicTargetFileNameField())) { throw new KettleException( BaseMessages.getString(PKG, "ProcessFiles.Error.TargetFilenameFieldMissing")); } // cache the position of the source filename field if (data.indexOfSourceFilename < 0) { data.indexOfSourceFilename = getInputRowMeta().indexOfValue(meta.getDynamicSourceFileNameField()); if (data.indexOfSourceFilename < 0) { // The field is unreachable ! throw new KettleException(BaseMessages.getString(PKG, "ProcessFiles.Exception.CouldnotFindField", meta.getDynamicSourceFileNameField())); } } // cache the position of the source filename field if (meta.getOperationType() != ProcessFilesMeta.OPERATION_TYPE_DELETE && data.indexOfTargetFilename < 0) { data.indexOfTargetFilename = getInputRowMeta().indexOfValue(meta.getDynamicTargetFileNameField()); if (data.indexOfTargetFilename < 0) { // The field is unreachable ! throw new KettleException(BaseMessages.getString(PKG, "ProcessFiles.Exception.CouldnotFindField", meta.getDynamicTargetFileNameField())); } } if (meta.simulate) { if (log.isBasic()) { logBasic(BaseMessages.getString(PKG, "ProcessFiles.Log.SimulationModeON")); } } } // End If first try { // get source filename String sourceFilename = getInputRowMeta().getString(r, data.indexOfSourceFilename); if (Const.isEmpty(sourceFilename)) { logError(BaseMessages.getString(PKG, "ProcessFiles.Error.SourceFileEmpty")); throw new KettleException(BaseMessages.getString(PKG, "ProcessFiles.Error.SourceFileEmpty")); } data.sourceFile = KettleVFS.getFileObject(sourceFilename, getTransMeta()); if (!data.sourceFile.exists()) { logError(BaseMessages.getString(PKG, "ProcessFiles.Error.SourceFileNotExist", sourceFilename)); throw new KettleException( BaseMessages.getString(PKG, "ProcessFiles.Error.SourceFileNotExist", sourceFilename)); } if (data.sourceFile.getType() != FileType.FILE) { logError(BaseMessages.getString(PKG, "ProcessFiles.Error.SourceFileNotFile", sourceFilename)); throw new KettleException( BaseMessages.getString(PKG, "ProcessFiles.Error.SourceFileNotFile", sourceFilename)); } String targetFilename = null; if (meta.getOperationType() != ProcessFilesMeta.OPERATION_TYPE_DELETE) { // get value for target filename targetFilename = getInputRowMeta().getString(r, data.indexOfTargetFilename); if (Const.isEmpty(targetFilename)) { logError(BaseMessages.getString(PKG, "ProcessFiles.Error.TargetFileEmpty")); throw new KettleException(BaseMessages.getString(PKG, "ProcessFiles.Error.TargetFileEmpty")); } data.targetFile = KettleVFS.getFileObject(targetFilename, getTransMeta()); if (data.targetFile.exists()) { if (log.isDetailed()) { logDetailed( BaseMessages.getString(PKG, "ProcessFiles.Log.TargetFileExists", targetFilename)); } // check if target is really a file otherwise it could overwrite a complete folder by copy or move operations if (data.targetFile.getType() != FileType.FILE) { logError(BaseMessages.getString(PKG, "ProcessFiles.Error.TargetFileNotFile", targetFilename)); throw new KettleException(BaseMessages.getString(PKG, "ProcessFiles.Error.TargetFileNotFile", targetFilename)); } } else { // let's check parent folder FileObject parentFolder = data.targetFile.getParent(); if (!parentFolder.exists()) { if (!meta.isCreateParentFolder()) { throw new KettleException(BaseMessages.getString(PKG, "ProcessFiles.Error.TargetParentFolderNotExists", parentFolder.toString())); } else { parentFolder.createFolder(); } } if (parentFolder != null) { parentFolder.close(); } } } switch (meta.getOperationType()) { case ProcessFilesMeta.OPERATION_TYPE_COPY: if (((meta.isOverwriteTargetFile() && data.targetFile.exists()) || !data.targetFile.exists()) && !meta.simulate) { data.targetFile.copyFrom(data.sourceFile, new TextOneToOneFileSelector()); if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "ProcessFiles.Log.SourceFileCopied", sourceFilename, targetFilename)); } } else { if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "ProcessFiles.Log.TargetNotOverwritten", sourceFilename, targetFilename)); } } break; case ProcessFilesMeta.OPERATION_TYPE_MOVE: if (((meta.isOverwriteTargetFile() && data.targetFile.exists()) || !data.targetFile.exists()) && !meta.simulate) { data.sourceFile.moveTo(KettleVFS.getFileObject(targetFilename, getTransMeta())); if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "ProcessFiles.Log.SourceFileMoved", sourceFilename, targetFilename)); } } else { if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "ProcessFiles.Log.TargetNotOverwritten", sourceFilename, targetFilename)); } } break; case ProcessFilesMeta.OPERATION_TYPE_DELETE: if (!meta.simulate) { if (!data.sourceFile.delete()) { throw new KettleException(BaseMessages.getString(PKG, "ProcessFiles.Error.CanNotDeleteFile", data.sourceFile.toString())); } } if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "ProcessFiles.Log.SourceFileDeleted", sourceFilename)); } break; default: break; } // add filename to result filenames? if (meta.isaddTargetFileNametoResult() && meta.getOperationType() != ProcessFilesMeta.OPERATION_TYPE_DELETE && data.sourceFile.getType() == FileType.FILE) { // Add this to the result file names... ResultFile resultFile = new ResultFile(ResultFile.FILE_TYPE_GENERAL, data.targetFile, getTransMeta().getName(), getStepname()); resultFile.setComment(BaseMessages.getString(PKG, "ProcessFiles.Log.FileAddedResult")); addResultFile(resultFile); if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "ProcessFiles.Log.FilenameAddResult", data.sourceFile.toString())); } } putRow(getInputRowMeta(), r); // copy row to possible alternate rowset(s). if (checkFeedback(getLinesRead())) { if (log.isBasic()) { logBasic(BaseMessages.getString(PKG, "ProcessFiles.LineNumber") + getLinesRead()); } } } catch (Exception e) { boolean sendToErrorRow = false; String errorMessage = null; if (getStepMeta().isDoingErrorHandling()) { sendToErrorRow = true; errorMessage = e.toString(); } else { logError(BaseMessages.getString(PKG, "ProcessFiles.ErrorInStepRunning") + e.getMessage()); setErrors(1); stopAll(); setOutputDone(); // signal end to receiver(s) return false; } if (sendToErrorRow) { // Simply add this row to the error row putError(getInputRowMeta(), r, 1, errorMessage, null, "ProcessFiles001"); } } return true; }
From source file:org.pentaho.di.trans.steps.sort.SortRows.java
Object[] getBuffer() throws KettleValueException { Object[] retval;/* ww w . j a v a2s. c om*/ // Open all files at once and read one row from each file... if (data.files.size() > 0 && (data.dis.size() == 0 || data.fis.size() == 0)) { if (log.isBasic()) { logBasic(BaseMessages.getString(PKG, "SortRows.Basic.OpeningTempFiles", data.files.size())); } try { for (int f = 0; f < data.files.size() && !isStopped(); f++) { FileObject fileObject = data.files.get(f); String filename = KettleVFS.getFilename(fileObject); if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "SortRows.Detailed.OpeningTempFile", filename)); } InputStream fi = KettleVFS.getInputStream(fileObject); DataInputStream di; data.fis.add(fi); if (data.compressFiles) { di = getDataInputStream(new GZIPInputStream(new BufferedInputStream(fi))); } else { di = new DataInputStream(new BufferedInputStream(fi, 50000)); } data.dis.add(di); // How long is the buffer? int buffersize = data.bufferSizes.get(f); if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "SortRows.Detailed.FromFileExpectingRows", filename, buffersize)); } if (buffersize > 0) { Object[] row = data.outputRowMeta.readData(di); data.rowbuffer.add(row); // new row from input stream data.tempRows.add(new RowTempFile(row, f)); } } // Sort the data row buffer Collections.sort(data.tempRows, data.comparator); } catch (Exception e) { logError(BaseMessages.getString(PKG, "SortRows.Error.ErrorReadingBackTempFiles"), e); } } if (data.files.size() == 0) { // read from in-memory processing if (data.getBufferIndex < data.buffer.size()) { retval = data.buffer.get(data.getBufferIndex); data.getBufferIndex++; } else { retval = null; } } else { // read from disk processing if (data.rowbuffer.size() == 0) { retval = null; } else { // We now have "filenr" rows waiting: which one is the smallest? // if (log.isRowLevel()) { for (int i = 0; i < data.rowbuffer.size() && !isStopped(); i++) { Object[] b = data.rowbuffer.get(i); logRowlevel(BaseMessages.getString(PKG, "SortRows.RowLevel.PrintRow", i, data.outputRowMeta.getString(b))); } } RowTempFile rowTempFile = data.tempRows.remove(0); retval = rowTempFile.row; int smallest = rowTempFile.fileNumber; // now get another Row for position smallest FileObject file = data.files.get(smallest); DataInputStream di = data.dis.get(smallest); InputStream fi = data.fis.get(smallest); try { Object[] row2 = data.outputRowMeta.readData(di); RowTempFile extra = new RowTempFile(row2, smallest); int index = Collections.binarySearch(data.tempRows, extra, data.comparator); if (index < 0) { data.tempRows.add(index * (-1) - 1, extra); } else { data.tempRows.add(index, extra); } } catch (KettleFileException fe) { // empty file or EOF mostly GZIPInputStream gzfi = (data.compressFiles) ? data.gzis.get(smallest) : null; try { di.close(); fi.close(); if (gzfi != null) { gzfi.close(); } file.delete(); } catch (IOException e) { logError(BaseMessages.getString(PKG, "SortRows.Error.UnableToCloseFile", smallest, file.toString())); setErrors(1); stopAll(); return null; } data.files.remove(smallest); data.dis.remove(smallest); data.fis.remove(smallest); if (gzfi != null) { data.gzis.remove(smallest); } // Also update all file numbers in in data.tempRows if they are larger // than smallest. // for (RowTempFile rtf : data.tempRows) { if (rtf.fileNumber > smallest) { rtf.fileNumber--; } } } catch (SocketTimeoutException e) { throw new KettleValueException(e); // should never happen on local files } } } return retval; }
From source file:org.pentaho.di.trans.steps.zipfile.ZipFile.java
public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException { meta = (ZipFileMeta) smi;//from w w w . ja v a 2s. co m data = (ZipFileData) sdi; Object[] r = getRow(); // Get row from input rowset & set row busy! if (r == null) { // no more input to be expected... setOutputDone(); return false; } if (first) { first = false; data.outputRowMeta = getInputRowMeta().clone(); meta.getFields(data.outputRowMeta, getStepname(), null, null, this, getTrans().getRepository(), getTrans().getMetaStore()); // Check is source filename field is provided if (Const.isEmpty(meta.getDynamicSourceFileNameField())) { throw new KettleException(BaseMessages.getString(PKG, "ZipFile.Error.SourceFilenameFieldMissing")); } // Check is target filename field is provided if (Const.isEmpty(meta.getDynamicTargetFileNameField())) { throw new KettleException(BaseMessages.getString(PKG, "ZipFile.Error.TargetFilenameFieldMissing")); } // cache the position of the source filename field if (data.indexOfSourceFilename < 0) { data.indexOfSourceFilename = getInputRowMeta().indexOfValue(meta.getDynamicSourceFileNameField()); if (data.indexOfSourceFilename < 0) { // The field is unreachable ! throw new KettleException(BaseMessages.getString(PKG, "ZipFile.Exception.CouldnotFindField", meta.getDynamicSourceFileNameField())); } } data.indexOfZipFilename = getInputRowMeta().indexOfValue(meta.getDynamicTargetFileNameField()); if (data.indexOfZipFilename < 0) { // The field is unreachable ! throw new KettleException(BaseMessages.getString(PKG, "ZipFile.Exception.CouldnotFindField", meta.getDynamicTargetFileNameField())); } if (meta.isKeepSouceFolder()) { if (!Const.isEmpty(meta.getBaseFolderField())) { // cache the position of the source filename field data.indexOfBaseFolder = getInputRowMeta().indexOfValue(meta.getBaseFolderField()); if (data.indexOfBaseFolder < 0) { // The field is unreachable ! throw new KettleException(BaseMessages.getString(PKG, "ZipFile.Exception.CouldnotFindField", meta.getBaseFolderField())); } } } // Move to folder if (meta.getOperationType() == ZipFileMeta.OPERATION_TYPE_MOVE) { if (Const.isEmpty(meta.getMoveToFolderField())) { throw new KettleException(BaseMessages.getString(PKG, "ZipFile.Exception.EmptyMovetoFolder")); } data.indexOfMoveToFolder = getInputRowMeta().indexOfValue(meta.getMoveToFolderField()); if (data.indexOfMoveToFolder < 0) { // The field is unreachable ! throw new KettleException(BaseMessages.getString(PKG, "ZipFile.Exception.CouldnotFindField", meta.getMoveToFolderField())); } } } // End If first boolean sendToErrorRow = false; String errorMessage = null; try { // get source filename String sourceFilename = getInputRowMeta().getString(r, data.indexOfSourceFilename); if (Const.isEmpty(sourceFilename)) { log.logError(toString(), BaseMessages.getString(PKG, "ZipFile.Error.SourceFileEmpty")); throw new KettleException(BaseMessages.getString(PKG, "ZipFile.Error.SourceFileEmpty")); } data.sourceFile = KettleVFS.getFileObject(sourceFilename); // Check sourcefile boolean skip = false; if (!data.sourceFile.exists()) { log.logError(toString(), BaseMessages.getString(PKG, "ZipFile.Error.SourceFileNotExist", sourceFilename)); throw new KettleException( BaseMessages.getString(PKG, "ZipFile.Error.SourceFileNotExist", sourceFilename)); } else { if (data.sourceFile.getType() != FileType.FILE) { log.logError(toString(), BaseMessages.getString(PKG, "ZipFile.Error.SourceFileNotFile", sourceFilename)); throw new KettleException( BaseMessages.getString(PKG, "ZipFile.Error.SourceFileNotFile", sourceFilename)); } } // get basefolder if (data.indexOfBaseFolder > -1) { data.baseFolder = getInputRowMeta().getString(r, data.indexOfBaseFolder); } // get destination folder String moveToFolder = null; if (data.indexOfMoveToFolder > -1) { moveToFolder = getInputRowMeta().getString(r, data.indexOfMoveToFolder); if (Const.isEmpty(moveToFolder)) { throw new KettleException(BaseMessages.getString(PKG, "ZipFile.Error.EmptyMoveToFolder")); } } if (!skip) { // get value for target filename String targetFilename = getInputRowMeta().getString(r, data.indexOfZipFilename); if (Const.isEmpty(targetFilename)) { log.logError(toString(), BaseMessages.getString(PKG, "ZipFile.Error.TargetFileEmpty")); throw new KettleException(BaseMessages.getString(PKG, "ZipFile.Error.TargetFileEmpty")); } data.zipFile = KettleVFS.getFileObject(targetFilename); if (data.zipFile.exists()) { if (log.isDetailed()) { log.logDetailed(toString(), BaseMessages.getString(PKG, "ZipFile.Log.TargetFileExists", targetFilename)); } } else { // let's check parent folder FileObject parentFolder = data.zipFile.getParent(); if (!parentFolder.exists()) { if (!meta.isCreateParentFolder()) { // Parent folder not exist // So we will fail throw new KettleException(BaseMessages.getString(PKG, "ZipFile.Error.TargetParentFolderNotExists", parentFolder.toString())); } else { // Create parent folder parentFolder.createFolder(); } } if (parentFolder != null) { parentFolder.close(); } } // Let's zip zipFile(); // file was zipped, let's see if we need to move or delete it processFile(moveToFolder); // add filename to result filenames? addFilenameToResult(); } getLinesInput(); putRow(data.outputRowMeta, r); // copy row to output rowset(s); if (checkFeedback(getLinesRead())) { if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "ZipFile.LineNumber", "" + getLinesRead())); } } } catch (Exception e) { if (getStepMeta().isDoingErrorHandling()) { sendToErrorRow = true; errorMessage = e.toString(); } else { logError(BaseMessages.getString(PKG, "ZipFile.ErrorInStepRunning") + e.getMessage()); setErrors(1); stopAll(); setOutputDone(); // signal end to receiver(s) return false; } if (sendToErrorRow) { // Simply add this row to the error row putError(getInputRowMeta(), r, 1, errorMessage, null, "ZipFile001"); } } finally { try { if (data.sourceFile != null) { data.sourceFile.close(); } if (data.zipFile != null) { data.zipFile.close(); } } catch (Exception e) { /* Ignore */ } } return true; }
From source file:org.sonatype.gshell.vfs.provider.truezip.TruezipFileObject.java
/** * Rename this file// w w w . j a va 2 s . com */ protected void doRename(FileObject newfile) throws Exception { if (!file.renameTo(((TruezipFileObject) newfile).getLocalFile())) { throw new FileSystemException("vfs.provider.truezip/rename-file.error", new String[] { file.toString(), newfile.toString() }); } }