List of usage examples for org.apache.commons.vfs FileObject findFiles
public FileObject[] findFiles(FileSelector selector) throws FileSystemException;
From source file:org.pentaho.di.job.entries.unzip.JobEntryUnZip.java
private boolean unzipFile(FileObject sourceFileObject, String realTargetdirectory, String realWildcard, String realWildcardExclude, Result result, Job parentJob, FileObject fileObject, FileObject movetodir, String realMovetodirectory) { boolean retval = false; String unzipToFolder = realTargetdirectory; try {// w ww. j a v a 2 s .co m if (log.isDetailed()) { logDetailed( BaseMessages.getString(PKG, "JobUnZip.Log.ProcessingFile", sourceFileObject.toString())); } // Do you create a root folder? // if (rootzip) { String shortSourceFilename = sourceFileObject.getName().getBaseName(); int lenstring = shortSourceFilename.length(); int lastindexOfDot = shortSourceFilename.lastIndexOf('.'); if (lastindexOfDot == -1) { lastindexOfDot = lenstring; } String foldername = realTargetdirectory + "/" + shortSourceFilename.substring(0, lastindexOfDot); FileObject rootfolder = KettleVFS.getFileObject(foldername, this); if (!rootfolder.exists()) { try { rootfolder.createFolder(); if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobUnZip.Log.RootFolderCreated", foldername)); } } catch (Exception e) { throw new Exception( BaseMessages.getString(PKG, "JobUnZip.Error.CanNotCreateRootFolder", foldername), e); } } unzipToFolder = foldername; } // Try to read the entries from the VFS object... // String zipFilename = "zip:" + sourceFileObject.getName().getFriendlyURI(); FileObject zipFile = KettleVFS.getFileObject(zipFilename, this); FileObject[] items = zipFile.findFiles(new AllFileSelector() { public boolean traverseDescendents(FileSelectInfo info) { return true; } public boolean includeFile(FileSelectInfo info) { // Never return the parent directory of a file list. if (info.getDepth() == 0) { return false; } FileObject fileObject = info.getFile(); return fileObject != null; } }); Pattern pattern = null; if (!Const.isEmpty(realWildcard)) { pattern = Pattern.compile(realWildcard); } Pattern patternexclude = null; if (!Const.isEmpty(realWildcardExclude)) { patternexclude = Pattern.compile(realWildcardExclude); } for (FileObject item : items) { if (successConditionBroken) { if (!successConditionBrokenExit) { logError(BaseMessages.getString(PKG, "JobUnZip.Error.SuccessConditionbroken", "" + NrErrors)); successConditionBrokenExit = true; } return false; } synchronized (KettleVFS.getInstance().getFileSystemManager()) { FileObject newFileObject = null; try { if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobUnZip.Log.ProcessingZipEntry", item.getName().getURI(), sourceFileObject.toString())); } // get real destination filename // String newFileName = unzipToFolder + Const.FILE_SEPARATOR + getTargetFilename(item); newFileObject = KettleVFS.getFileObject(newFileName, this); if (item.getType().equals(FileType.FOLDER)) { // Directory // if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobUnZip.CreatingDirectory.Label", newFileName)); } // Create Directory if necessary ... // if (!newFileObject.exists()) { newFileObject.createFolder(); } } else { // File // boolean getIt = true; boolean getItexclude = false; // First see if the file matches the regular expression! // if (pattern != null) { Matcher matcher = pattern.matcher(item.getName().getURI()); getIt = matcher.matches(); } if (patternexclude != null) { Matcher matcherexclude = patternexclude.matcher(item.getName().getURI()); getItexclude = matcherexclude.matches(); } boolean take = takeThisFile(item, newFileName); if (getIt && !getItexclude && take) { if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobUnZip.ExtractingEntry.Label", item.getName().getURI(), newFileName)); } if (iffileexist == IF_FILE_EXISTS_UNIQ) { // Create file with unique name int lenstring = newFileName.length(); int lastindexOfDot = newFileName.lastIndexOf('.'); if (lastindexOfDot == -1) { lastindexOfDot = lenstring; } newFileName = newFileName.substring(0, lastindexOfDot) + StringUtil.getFormattedDateTimeNow(true) + newFileName.substring(lastindexOfDot, lenstring); if (log.isDebug()) { logDebug(BaseMessages.getString(PKG, "JobUnZip.Log.CreatingUniqFile", newFileName)); } } // See if the folder to the target file exists... // if (!newFileObject.getParent().exists()) { newFileObject.getParent().createFolder(); // creates the whole path. } InputStream is = null; OutputStream os = null; try { is = KettleVFS.getInputStream(item); os = KettleVFS.getOutputStream(newFileObject, false); if (is != null) { byte[] buff = new byte[2048]; int len; while ((len = is.read(buff)) > 0) { os.write(buff, 0, len); } // Add filename to result filenames addFilenameToResultFilenames(result, parentJob, newFileName); } } finally { if (is != null) { is.close(); } if (os != null) { os.close(); } } } // end if take } } catch (Exception e) { updateErrors(); logError(BaseMessages.getString(PKG, "JobUnZip.Error.CanNotProcessZipEntry", item.getName().getURI(), sourceFileObject.toString()), e); } finally { if (newFileObject != null) { try { newFileObject.close(); if (setOriginalModificationDate) { // Change last modification date newFileObject.getContent() .setLastModifiedTime(item.getContent().getLastModifiedTime()); } } catch (Exception e) { /* Ignore */ } // ignore this } // Close file object // close() does not release resources! KettleVFS.getInstance().getFileSystemManager().closeFileSystem(item.getFileSystem()); if (items != null) { items = null; } } } // Synchronized block on KettleVFS.getInstance().getFileSystemManager() } // End for // Here gc() is explicitly called if e.g. createfile is used in the same // job for the same file. The problem is that after creating the file the // file object is not properly garbaged collected and thus the file cannot // be deleted anymore. This is a known problem in the JVM. // System.gc(); // Unzip done... if (afterunzip == 1) { // delete zip file boolean deleted = fileObject.delete(); if (!deleted) { updateErrors(); logError(BaseMessages.getString(PKG, "JobUnZip.Cant_Delete_File.Label", sourceFileObject.toString())); } // File deleted if (log.isDebug()) { logDebug(BaseMessages.getString(PKG, "JobUnZip.File_Deleted.Label", sourceFileObject.toString())); } } else if (afterunzip == 2) { FileObject destFile = null; // Move File try { String destinationFilename = movetodir + Const.FILE_SEPARATOR + fileObject.getName().getBaseName(); destFile = KettleVFS.getFileObject(destinationFilename, this); fileObject.moveTo(destFile); // File moved if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobUnZip.Log.FileMovedTo", sourceFileObject.toString(), realMovetodirectory)); } } catch (Exception e) { updateErrors(); logError(BaseMessages.getString(PKG, "JobUnZip.Cant_Move_File.Label", sourceFileObject.toString(), realMovetodirectory, e.getMessage())); } finally { if (destFile != null) { try { destFile.close(); } catch (IOException ex) { /* Ignore */ } } } } retval = true; } catch (Exception e) { updateErrors(); log.logError(BaseMessages.getString(PKG, "JobUnZip.Error.Label"), BaseMessages.getString(PKG, "JobUnZip.ErrorUnzip.Label", sourceFileObject.toString(), e.getMessage()), e); } return retval; }
From source file:org.pentaho.di.job.entries.xmlwellformed.JobEntryXMLWellFormed.java
private boolean processFileFolder(String sourcefilefoldername, String wildcard, Job parentJob, Result result) { boolean entrystatus = false; FileObject sourcefilefolder = null; FileObject CurrentFile = null; // Get real source file and wilcard String realSourceFilefoldername = environmentSubstitute(sourcefilefoldername); if (Const.isEmpty(realSourceFilefoldername)) { logError(BaseMessages.getString(PKG, "JobXMLWellFormed.log.FileFolderEmpty", sourcefilefoldername)); // Update Errors updateErrors();//from w ww .j a v a2 s . c o m return entrystatus; } String realWildcard = environmentSubstitute(wildcard); try { sourcefilefolder = KettleVFS.getFileObject(realSourceFilefoldername, this); if (sourcefilefolder.exists()) { if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobXMLWellFormed.Log.FileExists", sourcefilefolder.toString())); } if (sourcefilefolder.getType() == FileType.FILE) { entrystatus = checkOneFile(sourcefilefolder, result, parentJob); } else if (sourcefilefolder.getType() == FileType.FOLDER) { FileObject[] fileObjects = sourcefilefolder.findFiles(new AllFileSelector() { public boolean traverseDescendents(FileSelectInfo info) { return true; } public boolean includeFile(FileSelectInfo info) { FileObject fileObject = info.getFile(); try { if (fileObject == null) { return false; } if (fileObject.getType() != FileType.FILE) { return false; } } catch (Exception ex) { // Upon error don't process the file. return false; } finally { if (fileObject != null) { try { fileObject.close(); } catch (IOException ex) { /* Ignore */ } } } return true; } }); if (fileObjects != null) { for (int j = 0; j < fileObjects.length && !parentJob.isStopped(); j++) { if (successConditionBroken) { if (!successConditionBrokenExit) { logError(BaseMessages.getString(PKG, "JobXMLWellFormed.Error.SuccessConditionbroken", "" + NrAllErrors)); successConditionBrokenExit = true; } return false; } // Fetch files in list one after one ... CurrentFile = fileObjects[j]; if (!CurrentFile.getParent().toString().equals(sourcefilefolder.toString())) { // Not in the Base Folder..Only if include sub folders if (include_subfolders) { if (GetFileWildcard(CurrentFile.toString(), realWildcard)) { checkOneFile(CurrentFile, result, parentJob); } } } else { // In the base folder if (GetFileWildcard(CurrentFile.toString(), realWildcard)) { checkOneFile(CurrentFile, result, parentJob); } } } } } else { logError(BaseMessages.getString(PKG, "JobXMLWellFormed.Error.UnknowFileFormat", sourcefilefolder.toString())); // Update Errors updateErrors(); } } else { logError(BaseMessages.getString(PKG, "JobXMLWellFormed.Error.SourceFileNotExists", realSourceFilefoldername)); // Update Errors updateErrors(); } } catch (Exception e) { logError(BaseMessages.getString(PKG, "JobXMLWellFormed.Error.Exception.Processing", realSourceFilefoldername.toString(), e)); // Update Errors updateErrors(); } finally { if (sourcefilefolder != null) { try { sourcefilefolder.close(); } catch (IOException ex) { /* Ignore */ } } if (CurrentFile != null) { try { CurrentFile.close(); } catch (IOException ex) { /* Ignore */ } } } return entrystatus; }
From source file:org.pentaho.di.job.entries.zipfile.JobEntryZipFile.java
public boolean processRowFile(Job parentJob, Result result, String realZipfilename, String realWildcard, String realWildcardExclude, String realSourceDirectoryOrFile, String realMovetodirectory, boolean createparentfolder) { boolean Fileexists = false; File tempFile = null;/*from ww w . ja va 2s.c o m*/ File fileZip = null; boolean resultat = false; boolean renameOk = false; boolean orginExist = false; // Check if target file/folder exists! FileObject originFile = null; ZipInputStream zin = null; byte[] buffer = null; OutputStream dest = null; BufferedOutputStream buff = null; ZipOutputStream out = null; ZipEntry entry = null; String localSourceFilename = realSourceDirectoryOrFile; try { originFile = KettleVFS.getFileObject(realSourceDirectoryOrFile, this); localSourceFilename = KettleVFS.getFilename(originFile); orginExist = originFile.exists(); } catch (Exception e) { // Ignore errors } finally { if (originFile != null) { try { originFile.close(); } catch (IOException ex) { logError("Error closing file '" + originFile.toString() + "'", ex); } } } String localrealZipfilename = realZipfilename; if (realZipfilename != null && orginExist) { FileObject fileObject = null; try { fileObject = KettleVFS.getFileObject(localrealZipfilename, this); localrealZipfilename = KettleVFS.getFilename(fileObject); // Check if Zip File exists if (fileObject.exists()) { Fileexists = true; if (log.isDebug()) { logDebug(BaseMessages.getString(PKG, "JobZipFiles.Zip_FileExists1.Label") + localrealZipfilename + BaseMessages.getString(PKG, "JobZipFiles.Zip_FileExists2.Label")); } } // Let's see if we need to create parent folder of destination zip filename if (createparentfolder) { createParentFolder(localrealZipfilename); } // Let's start the process now if (ifZipFileExists == 3 && Fileexists) { // the zip file exists and user want to Fail resultat = false; } else if (ifZipFileExists == 2 && Fileexists) { // the zip file exists and user want to do nothing if (addFileToResult) { // Add file to result files name ResultFile resultFile = new ResultFile(ResultFile.FILE_TYPE_GENERAL, fileObject, parentJob.getJobname(), toString()); result.getResultFiles().put(resultFile.getFile().toString(), resultFile); } resultat = true; } else if (afterZip == 2 && realMovetodirectory == null) { // After Zip, Move files..User must give a destination Folder resultat = false; logError( BaseMessages.getString(PKG, "JobZipFiles.AfterZip_No_DestinationFolder_Defined.Label")); } else { // After Zip, Move files..User must give a destination Folder // Let's see if we deal with file or folder FileObject[] fileList = null; FileObject sourceFileOrFolder = KettleVFS.getFileObject(localSourceFilename); boolean isSourceDirectory = sourceFileOrFolder.getType().equals(FileType.FOLDER); final Pattern pattern; final Pattern patternexclude; if (isSourceDirectory) { // Let's prepare the pattern matcher for performance reasons. // We only do this if the target is a folder ! // if (!Const.isEmpty(realWildcard)) { pattern = Pattern.compile(realWildcard); } else { pattern = null; } if (!Const.isEmpty(realWildcardExclude)) { patternexclude = Pattern.compile(realWildcardExclude); } else { patternexclude = null; } // Target is a directory // Get all the files in the directory... // if (includingSubFolders) { fileList = sourceFileOrFolder.findFiles(new FileSelector() { public boolean traverseDescendents(FileSelectInfo fileInfo) throws Exception { return true; } public boolean includeFile(FileSelectInfo fileInfo) throws Exception { boolean include; // Only include files in the sub-folders... // When we include sub-folders we match the whole filename, not just the base-name // if (fileInfo.getFile().getType().equals(FileType.FILE)) { include = true; if (pattern != null) { String name = fileInfo.getFile().getName().getPath(); include = pattern.matcher(name).matches(); } if (include && patternexclude != null) { String name = fileInfo.getFile().getName().getPath(); include = !pattern.matcher(name).matches(); } } else { include = false; } return include; } }); } else { fileList = sourceFileOrFolder.getChildren(); } } else { pattern = null; patternexclude = null; // Target is a file fileList = new FileObject[] { sourceFileOrFolder }; } if (fileList.length == 0) { resultat = false; logError(BaseMessages.getString(PKG, "JobZipFiles.Log.FolderIsEmpty", localSourceFilename)); } else if (!checkContainsFile(localSourceFilename, fileList, isSourceDirectory)) { resultat = false; logError(BaseMessages.getString(PKG, "JobZipFiles.Log.NoFilesInFolder", localSourceFilename)); } else { if (ifZipFileExists == 0 && Fileexists) { // the zip file exists and user want to create new one with unique name // Format Date // do we have already a .zip at the end? if (localrealZipfilename.toLowerCase().endsWith(".zip")) { // strip this off localrealZipfilename = localrealZipfilename.substring(0, localrealZipfilename.length() - 4); } localrealZipfilename += "_" + StringUtil.getFormattedDateTimeNow(true) + ".zip"; if (log.isDebug()) { logDebug(BaseMessages.getString(PKG, "JobZipFiles.Zip_FileNameChange1.Label") + localrealZipfilename + BaseMessages.getString(PKG, "JobZipFiles.Zip_FileNameChange1.Label")); } } else if (ifZipFileExists == 1 && Fileexists) { // the zip file exists and user want to append // get a temp file fileZip = getFile(localrealZipfilename); tempFile = File.createTempFile(fileZip.getName(), null); // delete it, otherwise we cannot rename existing zip to it. tempFile.delete(); renameOk = fileZip.renameTo(tempFile); if (!renameOk) { logError(BaseMessages.getString(PKG, "JobZipFiles.Cant_Rename_Temp1.Label") + fileZip.getAbsolutePath() + BaseMessages.getString(PKG, "JobZipFiles.Cant_Rename_Temp2.Label") + tempFile.getAbsolutePath() + BaseMessages.getString(PKG, "JobZipFiles.Cant_Rename_Temp3.Label")); } if (log.isDebug()) { logDebug(BaseMessages.getString(PKG, "JobZipFiles.Zip_FileAppend1.Label") + localrealZipfilename + BaseMessages.getString(PKG, "JobZipFiles.Zip_FileAppend2.Label")); } } if (log.isDetailed()) { logDetailed( BaseMessages.getString(PKG, "JobZipFiles.Files_Found1.Label") + fileList.length + BaseMessages.getString(PKG, "JobZipFiles.Files_Found2.Label") + localSourceFilename + BaseMessages.getString(PKG, "JobZipFiles.Files_Found3.Label")); } // Prepare Zip File buffer = new byte[18024]; dest = KettleVFS.getOutputStream(localrealZipfilename, false); buff = new BufferedOutputStream(dest); out = new ZipOutputStream(buff); HashSet<String> fileSet = new HashSet<String>(); if (renameOk) { // User want to append files to existing Zip file // The idea is to rename the existing zip file to a temporary file // and then adds all entries in the existing zip along with the new files, // excluding the zip entries that have the same name as one of the new files. zin = new ZipInputStream(new FileInputStream(tempFile)); entry = zin.getNextEntry(); while (entry != null) { String name = entry.getName(); if (!fileSet.contains(name)) { // Add ZIP entry to output stream. out.putNextEntry(new ZipEntry(name)); // Transfer bytes from the ZIP file to the output file int len; while ((len = zin.read(buffer)) > 0) { out.write(buffer, 0, len); } fileSet.add(name); } entry = zin.getNextEntry(); } // Close the streams zin.close(); } // Set the method out.setMethod(ZipOutputStream.DEFLATED); // Set the compression level if (compressionRate == 0) { out.setLevel(Deflater.NO_COMPRESSION); } else if (compressionRate == 1) { out.setLevel(Deflater.DEFAULT_COMPRESSION); } if (compressionRate == 2) { out.setLevel(Deflater.BEST_COMPRESSION); } if (compressionRate == 3) { out.setLevel(Deflater.BEST_SPEED); } // Specify Zipped files (After that we will move,delete them...) FileObject[] zippedFiles = new FileObject[fileList.length]; int fileNum = 0; // Get the files in the list... for (int i = 0; i < fileList.length && !parentJob.isStopped(); i++) { boolean getIt = true; boolean getItexclude = false; // First see if the file matches the regular expression! // ..only if target is a folder ! if (isSourceDirectory) { // If we include sub-folders, we match on the whole name, not just the basename // String filename; if (includingSubFolders) { filename = fileList[i].getName().getPath(); } else { filename = fileList[i].getName().getBaseName(); } if (pattern != null) { // Matches the base name of the file (backward compatible!) // Matcher matcher = pattern.matcher(filename); getIt = matcher.matches(); } if (patternexclude != null) { Matcher matcherexclude = patternexclude.matcher(filename); getItexclude = matcherexclude.matches(); } } // Get processing File String targetFilename = KettleVFS.getFilename(fileList[i]); if (sourceFileOrFolder.getType().equals(FileType.FILE)) { targetFilename = localSourceFilename; } FileObject file = KettleVFS.getFileObject(targetFilename); boolean isTargetDirectory = file.exists() && file.getType().equals(FileType.FOLDER); if (getIt && !getItexclude && !isTargetDirectory && !fileSet.contains(targetFilename)) { // We can add the file to the Zip Archive if (log.isDebug()) { logDebug(BaseMessages.getString(PKG, "JobZipFiles.Add_FilesToZip1.Label") + fileList[i] + BaseMessages.getString(PKG, "JobZipFiles.Add_FilesToZip2.Label") + localSourceFilename + BaseMessages.getString(PKG, "JobZipFiles.Add_FilesToZip3.Label")); } // Associate a file input stream for the current file InputStream in = KettleVFS.getInputStream(file); // Add ZIP entry to output stream. // String relativeName; String fullName = fileList[i].getName().getPath(); String basePath = sourceFileOrFolder.getName().getPath(); if (isSourceDirectory) { if (fullName.startsWith(basePath)) { relativeName = fullName.substring(basePath.length() + 1); } else { relativeName = fullName; } } else if (isFromPrevious) { int depth = determineDepth(environmentSubstitute(storedSourcePathDepth)); relativeName = determineZipfilenameForDepth(fullName, depth); } else { relativeName = fileList[i].getName().getBaseName(); } out.putNextEntry(new ZipEntry(relativeName)); int len; while ((len = in.read(buffer)) > 0) { out.write(buffer, 0, len); } out.flush(); out.closeEntry(); // Close the current file input stream in.close(); // Get Zipped File zippedFiles[fileNum] = fileList[i]; fileNum = fileNum + 1; } } // Close the ZipOutPutStream out.close(); buff.close(); dest.close(); if (log.isBasic()) { logBasic(BaseMessages.getString(PKG, "JobZipFiles.Log.TotalZippedFiles", "" + zippedFiles.length)); } // Delete Temp File if (tempFile != null) { tempFile.delete(); } // -----Get the list of Zipped Files and Move or Delete Them if (afterZip == 1 || afterZip == 2) { // iterate through the array of Zipped files for (int i = 0; i < zippedFiles.length; i++) { if (zippedFiles[i] != null) { // Delete, Move File FileObject fileObjectd = zippedFiles[i]; if (!isSourceDirectory) { fileObjectd = KettleVFS.getFileObject(localSourceFilename); } // Here we can move, delete files if (afterZip == 1) { // Delete File boolean deleted = fileObjectd.delete(); if (!deleted) { resultat = false; logError(BaseMessages.getString(PKG, "JobZipFiles.Cant_Delete_File1.Label") + localSourceFilename + Const.FILE_SEPARATOR + zippedFiles[i] + BaseMessages .getString(PKG, "JobZipFiles.Cant_Delete_File2.Label")); } // File deleted if (log.isDebug()) { logDebug(BaseMessages.getString(PKG, "JobZipFiles.File_Deleted1.Label") + localSourceFilename + Const.FILE_SEPARATOR + zippedFiles[i] + BaseMessages.getString(PKG, "JobZipFiles.File_Deleted2.Label")); } } else if (afterZip == 2) { // Move File FileObject fileObjectm = null; try { fileObjectm = KettleVFS.getFileObject(realMovetodirectory + Const.FILE_SEPARATOR + fileObjectd.getName().getBaseName()); fileObjectd.moveTo(fileObjectm); } catch (IOException e) { logError( BaseMessages.getString(PKG, "JobZipFiles.Cant_Move_File1.Label") + zippedFiles[i] + BaseMessages.getString(PKG, "JobZipFiles.Cant_Move_File2.Label") + e.getMessage()); resultat = false; } finally { try { if (fileObjectm != null) { fileObjectm.close(); } } catch (Exception e) { if (fileObjectm != null) { logError("Error closing file '" + fileObjectm.toString() + "'", e); } } } // File moved if (log.isDebug()) { logDebug(BaseMessages.getString(PKG, "JobZipFiles.File_Moved1.Label") + zippedFiles[i] + BaseMessages.getString(PKG, "JobZipFiles.File_Moved2.Label")); } } } } } if (addFileToResult) { // Add file to result files name ResultFile resultFile = new ResultFile(ResultFile.FILE_TYPE_GENERAL, fileObject, parentJob.getJobname(), toString()); result.getResultFiles().put(resultFile.getFile().toString(), resultFile); } resultat = true; } } } catch (Exception e) { logError(BaseMessages.getString(PKG, "JobZipFiles.Cant_CreateZipFile1.Label") + localrealZipfilename + BaseMessages.getString(PKG, "JobZipFiles.Cant_CreateZipFile2.Label"), e); resultat = false; } finally { if (fileObject != null) { try { fileObject.close(); fileObject = null; } catch (IOException ex) { logError("Error closing file '" + fileObject.toString() + "'", ex); } } try { if (out != null) { out.close(); } if (buff != null) { buff.close(); } if (dest != null) { dest.close(); } if (zin != null) { zin.close(); } if (entry != null) { entry = null; } } catch (IOException ex) { logError("Error closing zip file entry for file '" + originFile.toString() + "'", ex); } } } else { resultat = true; if (localrealZipfilename == null) { logError(BaseMessages.getString(PKG, "JobZipFiles.No_ZipFile_Defined.Label")); } if (!orginExist) { logError(BaseMessages.getString(PKG, "JobZipFiles.No_FolderCible_Defined.Label", localSourceFilename)); } } // return a verifier return resultat; }
From source file:org.pentaho.di.repository.KettleDatabaseRepositoryTest.java
protected void verifyJobSamples(RepositoryDirectoryInterface samplesDirectory) throws Exception { FileObject jobSamplesFolder = KettleVFS.getFileObject("samples/jobs/"); FileObject[] files = jobSamplesFolder.findFiles(new FileSelector() { @Override// w ww . j av a 2 s .co m public boolean traverseDescendents(FileSelectInfo arg0) throws Exception { return true; } @Override public boolean includeFile(FileSelectInfo info) throws Exception { return info.getFile().getName().getExtension().equalsIgnoreCase("kjb"); } }); List<FileObject> filesList = Arrays.asList(files); Collections.sort(filesList, new Comparator<FileObject>() { @Override public int compare(FileObject o1, FileObject o2) { return o1.getName().getPath().compareTo(o2.getName().getPath()); } }); for (FileObject file : filesList) { String jobFilename = file.getName().getPath(); System.out.println("Storing/Loading/validating job '" + jobFilename + "'"); // Load the JobMeta object... // JobMeta jobMeta = new JobMeta(jobFilename, repository); if (Const.isEmpty(jobMeta.getName())) { jobMeta.setName(Const.createName(file.getName().getBaseName())); } // Save it in the repository in the samples folder // jobMeta.setRepositoryDirectory(samplesDirectory); repository.save(jobMeta, "unit testing"); assertNotNull(jobMeta.getObjectId()); // Load it back up again... // JobMeta repJobMeta = repository.loadJob(jobMeta.getObjectId(), null); String oneXml = repJobMeta.getXML(); // Save & load it again // repository.save(jobMeta, "unit testing"); repJobMeta = repository.loadJob(jobMeta.getObjectId(), null); String twoXml = repJobMeta.getXML(); // The XML needs to be identical after loading // // storeFile(oneXml, "/tmp/one.ktr"); // storeFile(twoXml, "/tmp/two.ktr"); // assertEquals(oneXml, twoXml); } // Verify the number of stored files, see if we can find them all again. // System.out.println("Stored " + files.length + " job samples in folder " + samplesDirectory.getPath()); String[] jobNames = repository.getJobNames(samplesDirectory.getObjectId(), false); assertEquals(files.length, jobNames.length); }
From source file:org.pentaho.di.repository.KettleFileRepositoryTest.java
private void verifyJobSamples(RepositoryDirectoryInterface samplesDirectory) throws Exception { FileObject jobSamplesFolder = KettleVFS.getFileObject("samples/jobs/"); FileObject[] files = jobSamplesFolder.findFiles(new FileSelector() { @Override//w w w.j ava 2 s . c om public boolean traverseDescendents(FileSelectInfo arg0) throws Exception { return true; } @Override public boolean includeFile(FileSelectInfo info) throws Exception { return info.getFile().getName().getExtension().equalsIgnoreCase("kjb"); } }); List<FileObject> filesList = Arrays.asList(files); Collections.sort(filesList, new Comparator<FileObject>() { @Override public int compare(FileObject o1, FileObject o2) { return o1.getName().getPath().compareTo(o2.getName().getPath()); } }); for (FileObject file : filesList) { String jobFilename = file.getName().getPath(); System.out.println("Storing/Loading/validating job '" + jobFilename + "'"); // Load the JobMeta object... // JobMeta jobMeta = new JobMeta(jobFilename, repository); jobMeta.setFilename(null); // The name is sometimes empty in the file, duplicates are present too... // Replaces slashes and the like as well... // jobMeta.setName(Const.createName(file.getName().getBaseName())); jobMeta.setName(jobMeta.getName().replace('/', '-')); if (Const.isEmpty(jobMeta.getName())) { jobMeta.setName(Const.createName(file.getName().getBaseName())); } if (jobMeta.getName().contains("/")) { jobMeta.setName(jobMeta.getName().replace('/', '-')); } // Save it in the repository in the samples folder // jobMeta.setRepositoryDirectory(samplesDirectory); repository.save(jobMeta, "unit testing"); assertNotNull(jobMeta.getObjectId()); // Load it back up again... // JobMeta repJobMeta = repository.loadJob(jobMeta.getObjectId(), null); String oneXml = repJobMeta.getXML(); // Save & load it again // repository.save(jobMeta, "unit testing"); repJobMeta = repository.loadJob(jobMeta.getObjectId(), null); String twoXml = repJobMeta.getXML(); // The XML needs to be identical after loading // // storeFile(oneXml, "/tmp/one.ktr"); // storeFile(twoXml, "/tmp/two.ktr"); // assertEquals(oneXml, twoXml); } // Verify the number of stored files, see if we can find them all again. // System.out.println("Stored " + files.length + " job samples in folder " + samplesDirectory.getPath()); String[] jobNames = repository.getJobNames(samplesDirectory.getObjectId(), false); assertEquals(files.length, jobNames.length); }
From source file:org.pentaho.di.trans.steps.mail.Mail.java
private void setAttachedFilesList(Object[] r, LogChannelInterface log) throws Exception { String realSourceFileFoldername = null; String realSourceWildcard = null; FileObject sourcefile = null; FileObject file = null;//from w ww . j a va 2 s. c o m ZipOutputStream zipOutputStream = null; File masterZipfile = null; if (meta.isZipFilenameDynamic()) { data.ZipFilename = data.previousRowMeta.getString(r, data.indexOfDynamicZipFilename); } try { if (meta.isDynamicFilename()) { // dynamic attached filenames if (data.indexOfSourceFilename > -1) { realSourceFileFoldername = data.previousRowMeta.getString(r, data.indexOfSourceFilename); } if (data.indexOfSourceWildcard > -1) { realSourceWildcard = data.previousRowMeta.getString(r, data.indexOfSourceWildcard); } } else { // static attached filenames realSourceFileFoldername = data.realSourceFileFoldername; realSourceWildcard = data.realSourceWildcard; } if (!Const.isEmpty(realSourceFileFoldername)) { sourcefile = KettleVFS.getFileObject(realSourceFileFoldername, getTransMeta()); if (sourcefile.exists()) { long FileSize = 0; FileObject[] list = null; if (sourcefile.getType() == FileType.FILE) { list = new FileObject[1]; list[0] = sourcefile; } else { list = sourcefile .findFiles(new TextFileSelector(sourcefile.toString(), realSourceWildcard)); } if (list.length > 0) { boolean zipFiles = meta.isZipFiles(); if (zipFiles && data.zipFileLimit == 0) { masterZipfile = new File( System.getProperty("java.io.tmpdir") + Const.FILE_SEPARATOR + data.ZipFilename); zipOutputStream = new ZipOutputStream(new FileOutputStream(masterZipfile)); } for (int i = 0; i < list.length; i++) { file = KettleVFS.getFileObject(KettleVFS.getFilename(list[i]), getTransMeta()); if (zipFiles) { if (data.zipFileLimit == 0) { ZipEntry zipEntry = new ZipEntry(file.getName().getBaseName()); zipOutputStream.putNextEntry(zipEntry); // Now put the content of this file into this archive... BufferedInputStream inputStream = new BufferedInputStream( file.getContent().getInputStream()); int c; while ((c = inputStream.read()) >= 0) { zipOutputStream.write(c); } inputStream.close(); zipOutputStream.closeEntry(); } else { FileSize += file.getContent().getSize(); } } else { addAttachedFilePart(file); } } // end for if (zipFiles) { if (isDebug()) { logDebug(BaseMessages.getString(PKG, "Mail.Log.FileSize", "" + FileSize)); } if (isDebug()) { logDebug(BaseMessages.getString(PKG, "Mail.Log.LimitSize", "" + data.zipFileLimit)); } if (data.zipFileLimit > 0 && FileSize > data.zipFileLimit) { masterZipfile = new File(System.getProperty("java.io.tmpdir") + Const.FILE_SEPARATOR + data.ZipFilename); zipOutputStream = new ZipOutputStream(new FileOutputStream(masterZipfile)); for (int i = 0; i < list.length; i++) { file = KettleVFS.getFileObject(KettleVFS.getFilename(list[i]), getTransMeta()); ZipEntry zipEntry = new ZipEntry(file.getName().getBaseName()); zipOutputStream.putNextEntry(zipEntry); // Now put the content of this file into this archive... BufferedInputStream inputStream = new BufferedInputStream( file.getContent().getInputStream()); int c; while ((c = inputStream.read()) >= 0) { zipOutputStream.write(c); } inputStream.close(); zipOutputStream.closeEntry(); } } if (data.zipFileLimit > 0 && FileSize > data.zipFileLimit || data.zipFileLimit == 0) { file = KettleVFS.getFileObject(masterZipfile.getAbsolutePath(), getTransMeta()); addAttachedFilePart(file); } } } } else { logError(BaseMessages.getString(PKG, "Mail.Error.SourceFileFolderNotExists", realSourceFileFoldername)); } } } catch (Exception e) { logError(e.getMessage()); } finally { if (sourcefile != null) { try { sourcefile.close(); } catch (Exception e) { // Ignore errors } } if (file != null) { try { file.close(); } catch (Exception e) { // Ignore errors } } if (zipOutputStream != null) { try { zipOutputStream.finish(); zipOutputStream.close(); } catch (IOException e) { logError("Unable to close attachement zip file archive : " + e.toString()); } } } }
From source file:org.pentaho.di.ui.i18n.MessagesSourceCrawler.java
public void crawl() throws Exception { for (final String sourceDirectory : sourceDirectories) { FileObject folder = KettleVFS.getFileObject(sourceDirectory); FileObject[] javaFiles = folder.findFiles(new FileSelector() { @Override//from w ww. j av a 2 s .c o m public boolean traverseDescendents(FileSelectInfo info) throws Exception { return true; } @Override public boolean includeFile(FileSelectInfo info) throws Exception { return info.getFile().getName().getExtension().equals("java"); } }); for (FileObject javaFile : javaFiles) { /** * We don't want the Messages.java files, there is nothing in there for us. */ boolean skip = false; for (String filename : filesToAvoid) { if (javaFile.getName().getBaseName().equals(filename)) { skip = true; } } if (skip) { continue; // don't process this file. } // For each of these files we look for keys... // lookForOccurrencesInFile(sourceDirectory, javaFile); } } // Also search for keys in the XUL files... // for (SourceCrawlerXMLFolder xmlFolder : xmlFolders) { String[] xmlDirs = { xmlFolder.getFolder(), }; String[] xmlMasks = { xmlFolder.getWildcard(), }; String[] xmlReq = { "N", }; boolean[] xmlSubdirs = { true, }; // search sub-folders too FileInputList xulFileInputList = FileInputList.createFileList(new Variables(), xmlDirs, xmlMasks, xmlReq, xmlSubdirs); for (FileObject fileObject : xulFileInputList.getFiles()) { try { Document doc = XMLHandler.loadXMLFile(fileObject); // Scan for elements and tags in this file... // for (SourceCrawlerXMLElement xmlElement : xmlFolder.getElements()) { addLabelOccurrences(xmlFolder.getDefaultSourceFolder(), fileObject, doc.getElementsByTagName(xmlElement.getSearchElement()), xmlFolder.getKeyPrefix(), xmlElement.getKeyTag(), xmlElement.getKeyAttribute(), xmlFolder.getDefaultPackage(), xmlFolder.getPackageExceptions()); } } catch (KettleXMLException e) { log.logError("Unable to open XUL / XML document: " + fileObject); } } } }
From source file:org.sonatype.gshell.commands.vfs.FindCommand.java
private void find(final CommandContext context, final FileObject file, final FileSelector selector) throws FileSystemException { assert context != null; assert file != null; assert selector != null; FileObject[] files = file.findFiles(selector); if (files != null && files.length != 0) { for (FileObject child : files) { display(context, child, file); }/*from w ww . java2s. c om*/ } }
From source file:org.sonatype.gshell.commands.vfs.ListDirectoryCommand.java
private void listChildren(final IO io, final FileObject dir) throws Exception { assert io != null; assert dir != null; FileObject[] files;/*from ww w. j ava2s . co m*/ if (includeHidden) { files = dir.getChildren(); } else { FileFilter filter = new FileFilter() { public boolean accept(final FileSelectInfo selection) { assert selection != null; try { return !selection.getFile().isHidden(); } catch (FileSystemException e) { throw new RuntimeException(e); } } }; files = dir.findFiles(new FileFilterSelector(filter)); } ConsoleReader reader = new ConsoleReader(io.streams.in, io.out, io.getTerminal()); reader.setPaginationEnabled(false); List<String> names = new ArrayList<String>(files.length); List<FileObject> dirs = new LinkedList<FileObject>(); for (FileObject file : files) { String fileName = file.getName().getBaseName(); if (FileObjects.hasChildren(file)) { fileName += FileName.SEPARATOR; if (recursive) { dirs.add(file); } } names.add(fileName); file.close(); } if (longList) { for (String name : names) { io.out.println(name); } } else { reader.printColumns(names); } if (!dirs.isEmpty()) { for (FileObject subdir : dirs) { io.out.println(); io.out.print(subdir.getName().getBaseName()); io.out.print(":"); listChildren(io, subdir); } } dir.close(); }
From source file:org.vivoweb.harvester.util.FileAide.java
/** * Deletes the file at the given path//ww w .j a va 2 s. co m * @param path the path to delete * @return true if deleted, false otherwise * @throws IOException error resolving path */ public static boolean delete(String path) throws IOException { FileObject file = getFileObject(path); if (!file.exists()) { return true; } if (isFolder(path)) { try { for (FileObject subFile : file.findFiles(new AllFileSelector())) { try { String subPath = subFile.getName().getPath(); if (!subPath.equals(path)) { delete(subPath); } } catch (FileSystemException e) { //log.trace(e.getMessage(), e); } } file.delete(new AllFileSelector()); } catch (FileSystemException e) { //log.trace(e.getMessage(), e); throw new IOException("Error deleting file!"); } } return file.delete(); }