List of usage examples for org.apache.commons.vfs FileObject delete
public boolean delete() throws FileSystemException;
From source file:org.pentaho.di.cluster.PartitioningTest.java
/** * This test reads a CSV file in parallel on the cluster, one copy per slave.<br> * It then partitions the data on id in 12 partitions (4 per slave).<br> * After that it re-partitions the data in 9 partitions (3 per slave).<br> * As such we expect 9 result files on disk.<br> * File: "partitioning-repartitioning-on-cluster.ktr"<br> */// w w w . j a v a 2 s . c om public void testPartitioningRepartitioningOnCluster() throws Exception { init(); ClusterGenerator clusterGenerator = new ClusterGenerator(); try { clusterGenerator.launchSlaveServers(); TransMeta transMeta = loadAndModifyTestTransformation(clusterGenerator, "test/org/pentaho/di/cluster/partitioning-repartitioning-on-cluster.ktr"); TransExecutionConfiguration config = createClusteredTransExecutionConfiguration(); TransSplitter transSplitter = Trans.executeClustered(transMeta, config); long nrErrors = Trans.monitorClusteredTransformation( new LogChannel("cluster unit test <testParallelFileReadOnMaster>"), transSplitter, null, 1); assertEquals(0L, nrErrors); String[] results = new String[] { "8", "9", "9", "9", "9", "8", "8", "8", "8", "8", "8", "8", }; String[] files = new String[] { "000", "001", "002", "003", "004", "005", "006", "007", "008", "009", "010", "011", }; for (int i = 0; i < results.length; i++) { String filename = "${java.io.tmpdir}/partitioning-repartitioning-on-cluster-" + files[i] + ".txt"; String result = loadFileContent(transMeta, filename); assertEqualsIgnoreWhitespacesAndCase(results[i], result); // Remove the output file : we don't want to leave too much clutter around // FileObject file = KettleVFS.getFileObject(transMeta.environmentSubstitute(filename)); file.delete(); } } catch (Exception e) { e.printStackTrace(); fail(e.toString()); } finally { try { clusterGenerator.stopSlaveServers(); } catch (Exception e) { e.printStackTrace(); fail(e.toString()); } } }
From source file:org.pentaho.di.core.vfs.VfsCoreTest.java
public void testWriteReadFile() throws Exception { // Write a text //// ww w .j av a 2 s . c o m FileObject tempFile = KettleVFS.createTempFile("prefix", "suffix", tmpDir); OutputStream outputStream = KettleVFS.getOutputStream(tempFile, false); OutputStreamWriter writer = new OutputStreamWriter(outputStream); writer.write(content); writer.close(); outputStream.close(); // Read it back... // InputStream inputStream = KettleVFS.getInputStream(tempFile); StringBuffer buffer = new StringBuffer(); int c; while ((c = inputStream.read()) >= 0) { buffer.append((char) c); } inputStream.close(); assertEquals(content, buffer.toString()); // Now open the data as a regular file... // String url = tempFile.getName().getURI(); String textFileContent = KettleVFS.getTextFileContent(url, Const.XML_ENCODING); // Now delete the file... // tempFile.delete(); assertEquals(false, tempFile.exists()); assertEquals(content, textFileContent); }
From source file:org.pentaho.di.job.entries.copyfiles.JobEntryCopyFiles.java
private boolean ProcessFileFolder(String sourcefilefoldername, String destinationfilefoldername, String wildcard, Job parentJob, Result result) { boolean entrystatus = false; FileObject sourcefilefolder = null; FileObject destinationfilefolder = null; // Clear list files to remove after copy process // This list is also added to result files name list_files_remove.clear();/*from w w w. j a v a 2 s.c o m*/ list_add_result.clear(); // Get real source, destination file and wildcard String realSourceFilefoldername = environmentSubstitute(sourcefilefoldername); String realDestinationFilefoldername = environmentSubstitute(destinationfilefoldername); String realWildcard = environmentSubstitute(wildcard); try { sourcefilefolder = KettleVFS.getFileObject(realSourceFilefoldername, this); destinationfilefolder = KettleVFS.getFileObject(realDestinationFilefoldername, this); if (sourcefilefolder.exists()) { // Check if destination folder/parent folder exists ! // If user wanted and if destination folder does not exist // PDI will create it if (CreateDestinationFolder(destinationfilefolder)) { // Basic Tests if (sourcefilefolder.getType().equals(FileType.FOLDER) && destination_is_a_file) { // Source is a folder, destination is a file // WARNING !!! CAN NOT COPY FOLDER TO FILE !!! logError(BaseMessages.getString(PKG, "JobCopyFiles.Log.CanNotCopyFolderToFile", realSourceFilefoldername, realDestinationFilefoldername)); NbrFail++; } else { if (destinationfilefolder.getType().equals(FileType.FOLDER) && sourcefilefolder.getType().equals(FileType.FILE)) { // Source is a file, destination is a folder // Copy the file to the destination folder destinationfilefolder.copyFrom(sourcefilefolder.getParent(), new TextOneFileSelector(sourcefilefolder.getParent().toString(), sourcefilefolder.getName().getBaseName(), destinationfilefolder.toString())); if (isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobCopyFiles.Log.FileCopied", sourcefilefolder.getName().toString(), destinationfilefolder.getName().toString())); } } else if (sourcefilefolder.getType().equals(FileType.FILE) && destination_is_a_file) { // Source is a file, destination is a file destinationfilefolder.copyFrom(sourcefilefolder, new TextOneToOneFileSelector(destinationfilefolder)); } else { // Both source and destination are folders if (isDetailed()) { logDetailed(" "); logDetailed(BaseMessages.getString(PKG, "JobCopyFiles.Log.FetchFolder", sourcefilefolder.toString())); } TextFileSelector textFileSelector = new TextFileSelector(sourcefilefolder, destinationfilefolder, realWildcard, parentJob); try { destinationfilefolder.copyFrom(sourcefilefolder, textFileSelector); } finally { textFileSelector.shutdown(); } } // Remove Files if needed if (remove_source_files && !list_files_remove.isEmpty()) { String sourceFilefoldername = sourcefilefolder.toString(); int trimPathLength = sourceFilefoldername.length() + 1; FileObject removeFile; for (Iterator<String> iter = list_files_remove.iterator(); iter.hasNext() && !parentJob.isStopped();) { String fileremoventry = iter.next(); removeFile = null; // re=null each iteration // Try to get the file relative to the existing connection if (fileremoventry.startsWith(sourceFilefoldername)) { if (trimPathLength < fileremoventry.length()) { removeFile = sourcefilefolder .getChild(fileremoventry.substring(trimPathLength)); } } // Unable to retrieve file through existing connection; Get the file through a new VFS connection if (removeFile == null) { removeFile = KettleVFS.getFileObject(fileremoventry, this); } // Remove ONLY Files if (removeFile.getType() == FileType.FILE) { boolean deletefile = removeFile.delete(); logBasic(" ------ "); if (!deletefile) { logError(" " + BaseMessages.getString(PKG, "JobCopyFiles.Error.Exception.CanRemoveFileFolder", fileremoventry)); } else { if (isDetailed()) { logDetailed(" " + BaseMessages.getString(PKG, "JobCopyFiles.Log.FileFolderRemoved", fileremoventry)); } } } } } // Add files to result files name if (add_result_filesname && !list_add_result.isEmpty()) { String destinationFilefoldername = destinationfilefolder.toString(); int trimPathLength = destinationFilefoldername.length() + 1; FileObject addFile; for (Iterator<String> iter = list_add_result.iterator(); iter.hasNext();) { String fileaddentry = iter.next(); addFile = null; // re=null each iteration // Try to get the file relative to the existing connection if (fileaddentry.startsWith(destinationFilefoldername)) { if (trimPathLength < fileaddentry.length()) { addFile = destinationfilefolder .getChild(fileaddentry.substring(trimPathLength)); } } // Unable to retrieve file through existing connection; Get the file through a new VFS connection if (addFile == null) { addFile = KettleVFS.getFileObject(fileaddentry, this); } // Add ONLY Files if (addFile.getType() == FileType.FILE) { ResultFile resultFile = new ResultFile(ResultFile.FILE_TYPE_GENERAL, addFile, parentJob.getJobname(), toString()); result.getResultFiles().put(resultFile.getFile().toString(), resultFile); if (isDetailed()) { logDetailed(" ------ "); logDetailed(" " + BaseMessages.getString(PKG, "JobCopyFiles.Log.FileAddedToResultFilesName", fileaddentry)); } } } } } entrystatus = true; } else { // Destination Folder or Parent folder is missing logError(BaseMessages.getString(PKG, "JobCopyFiles.Error.DestinationFolderNotFound", realDestinationFilefoldername)); } } else { logError(BaseMessages.getString(PKG, "JobCopyFiles.Error.SourceFileNotExists", realSourceFilefoldername)); } } catch (FileSystemException fse) { logError(BaseMessages.getString(PKG, "JobCopyFiles.Error.Exception.CopyProcessFileSystemException", fse.getMessage())); Throwable throwable = fse.getCause(); while (throwable != null) { logError(BaseMessages.getString(PKG, "JobCopyFiles.Log.CausedBy", throwable.getMessage())); throwable = throwable.getCause(); } } catch (Exception e) { logError(BaseMessages.getString(PKG, "JobCopyFiles.Error.Exception.CopyProcess", realSourceFilefoldername, realDestinationFilefoldername, e.getMessage()), e); } finally { if (sourcefilefolder != null) { try { sourcefilefolder.close(); sourcefilefolder = null; } catch (IOException ex) { /* Ignore */ } } if (destinationfilefolder != null) { try { destinationfilefolder.close(); destinationfilefolder = null; } catch (IOException ex) { /* Ignore */ } } } return entrystatus; }
From source file:org.pentaho.di.job.entries.copymoveresultfilenames.JobEntryCopyMoveResultFilenames.java
private boolean processFile(FileObject sourcefile, String destinationFolder, Result result, Job parentJob, boolean deleteFile) { boolean retval = false; try {//from w w w.j av a 2 s.co m if (deleteFile) { // delete file if (sourcefile.delete()) { if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobEntryCopyMoveResultFilenames.log.DeletedFile", sourcefile.toString())); } // Remove source file from result files list result.getResultFiles().remove(sourcefile.toString()); if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobEntryCopyMoveResultFilenames.RemovedFileFromResult", sourcefile.toString())); } } else { logError(BaseMessages.getString(PKG, "JobEntryCopyMoveResultFilenames.CanNotDeletedFile", sourcefile.toString())); } } else { // return destination short filename String shortfilename = getDestinationFilename(sourcefile.getName().getBaseName()); // build full destination filename String destinationFilename = destinationFolder + Const.FILE_SEPARATOR + shortfilename; FileObject destinationfile = KettleVFS.getFileObject(destinationFilename, this); boolean filexists = destinationfile.exists(); if (filexists) { if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobEntryCopyMoveResultFilenames.Log.FileExists", destinationFilename)); } } if ((!filexists) || (filexists && isOverwriteFile())) { if (getAction().equals("copy")) { // Copy file FileUtil.copyContent(sourcefile, destinationfile); if (log.isDetailed()) { logDetailed( BaseMessages.getString(PKG, "JobEntryCopyMoveResultFilenames.log.CopiedFile", sourcefile.toString(), destinationFolder)); } } else { // Move file sourcefile.moveTo(destinationfile); if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobEntryCopyMoveResultFilenames.log.MovedFile", sourcefile.toString(), destinationFolder)); } } if (isRemovedSourceFilename()) { // Remove source file from result files list result.getResultFiles().remove(sourcefile.toString()); if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobEntryCopyMoveResultFilenames.RemovedFileFromResult", sourcefile.toString())); } } if (isAddDestinationFilename()) { // Add destination filename to Resultfilenames ... ResultFile resultFile = new ResultFile(ResultFile.FILE_TYPE_GENERAL, KettleVFS.getFileObject(destinationfile.toString(), this), parentJob.getJobname(), toString()); result.getResultFiles().put(resultFile.getFile().toString(), resultFile); if (log.isDetailed()) { logDetailed( BaseMessages.getString(PKG, "JobEntryCopyMoveResultFilenames.AddedFileToResult", destinationfile.toString())); } } } } retval = true; } catch (Exception e) { logError(BaseMessages.getString(PKG, "JobEntryCopyMoveResultFilenames.Log.ErrorProcessing", e.toString())); } return retval; }
From source file:org.pentaho.di.job.entries.deletefile.JobEntryDeleteFile.java
public Result execute(Result previousResult, int nr) { Result result = previousResult; result.setResult(false);//from ww w . j av a 2 s . com if (filename != null) { String realFilename = getRealFilename(); FileObject fileObject = null; try { fileObject = KettleVFS.getFileObject(realFilename, this); if (!fileObject.exists()) { if (isFailIfFileNotExists()) { // File doesn't exist and fail flag is on. result.setResult(false); logError(BaseMessages.getString(PKG, "JobEntryDeleteFile.ERROR_0004_File_Does_Not_Exist", realFilename)); } else { // File already deleted, no reason to try to delete it result.setResult(true); if (log.isBasic()) { logBasic(BaseMessages.getString(PKG, "JobEntryDeleteFile.File_Already_Deleted", realFilename)); } } } else { boolean deleted = fileObject.delete(); if (!deleted) { logError(BaseMessages.getString(PKG, "JobEntryDeleteFile.ERROR_0005_Could_Not_Delete_File", realFilename)); result.setResult(false); result.setNrErrors(1); } if (log.isBasic()) { logBasic(BaseMessages.getString(PKG, "JobEntryDeleteFile.File_Deleted", realFilename)); } result.setResult(true); } } catch (Exception e) { logError(BaseMessages.getString(PKG, "JobEntryDeleteFile.ERROR_0006_Exception_Deleting_File", realFilename, e.getMessage()), e); result.setResult(false); result.setNrErrors(1); } finally { if (fileObject != null) { try { fileObject.close(); fileObject = null; } catch (IOException ex) { /* Ignore */ } } } } else { logError(BaseMessages.getString(PKG, "JobEntryDeleteFile.ERROR_0007_No_Filename_Is_Defined")); } return result; }
From source file:org.pentaho.di.job.entries.dostounix.JobEntryDosToUnix.java
private boolean convert(FileObject file, boolean toUnix) { boolean retval = false; // CR = CR/* w ww .j av a2 s.c o m*/ // LF = LF try { String localfilename = KettleVFS.getFilename(file); File source = new File(localfilename); if (isDetailed()) { if (toUnix) { logDetailed(BaseMessages.getString(PKG, "JobDosToUnix.Log.ConvertingFileToUnix", source.getAbsolutePath())); } else { logDetailed(BaseMessages.getString(PKG, "JobDosToUnix.Log.ConvertingFileToDos", source.getAbsolutePath())); } } File tempFile = new File(tempFolder, source.getName() + ".tmp"); if (isDebug()) { logDebug(BaseMessages.getString(PKG, "JobDosToUnix.Log.CreatingTempFile", tempFile.getAbsolutePath())); } FileOutputStream out = new FileOutputStream(tempFile); FileInputStream in = new FileInputStream(localfilename); if (toUnix) { // Dos to Unix while (in.available() > 0) { int b1 = in.read(); if (b1 == CR) { int b2 = in.read(); if (b2 == LF) { out.write(LF); } else { out.write(b1); out.write(b2); } } else { out.write(b1); } } } else { // Unix to Dos while (in.available() > 0) { int b1 = in.read(); if (b1 == LF) { out.write(CR); out.write(LF); } else { out.write(b1); } } } in.close(); out.close(); if (isDebug()) { logDebug(BaseMessages.getString(PKG, "JobDosToUnix.Log.DeletingSourceFile", localfilename)); } file.delete(); if (isDebug()) { logDebug(BaseMessages.getString(PKG, "JobDosToUnix.Log.RenamingTempFile", tempFile.getAbsolutePath(), source.getAbsolutePath())); } tempFile.renameTo(source); retval = true; } catch (Exception e) { logError(BaseMessages.getString(PKG, "JobDosToUnix.Log.ErrorConvertingFile", file.toString(), e.getMessage())); } return retval; }
From source file:org.pentaho.di.job.entries.movefiles.JobEntryMoveFiles.java
private boolean MoveFile(String shortfilename, FileObject sourcefilename, FileObject destinationfilename, FileObject movetofolderfolder, Job parentJob, Result result) { FileObject destinationfile = null; boolean retval = false; try {//from w w w . j av a 2 s . com if (!destinationfilename.exists()) { if (!simulate) { sourcefilename.moveTo(destinationfilename); } if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobMoveFiles.Log.FileMoved", sourcefilename.getName().toString(), destinationfilename.getName().toString())); } // add filename to result filename if (add_result_filesname && !iffileexists.equals("fail") && !iffileexists.equals("do_nothing")) { addFileToResultFilenames(destinationfilename.toString(), result, parentJob); } updateSuccess(); retval = true; } else { if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobMoveFiles.Log.FileExists", destinationfilename.toString())); } if (iffileexists.equals("overwrite_file")) { if (!simulate) { sourcefilename.moveTo(destinationfilename); } if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobMoveFiles.Log.FileOverwrite", destinationfilename.getName().toString())); } // add filename to result filename if (add_result_filesname && !iffileexists.equals("fail") && !iffileexists.equals("do_nothing")) { addFileToResultFilenames(destinationfilename.toString(), result, parentJob); } updateSuccess(); retval = true; } else if (iffileexists.equals("unique_name")) { String short_filename = shortfilename; // return destination short filename try { short_filename = getMoveDestinationFilename(short_filename, "ddMMyyyy_HHmmssSSS"); } catch (Exception e) { logError(BaseMessages.getString(PKG, BaseMessages.getString(PKG, "JobMoveFiles.Error.GettingFilename", short_filename)), e); return retval; } String movetofilenamefull = destinationfilename.getParent().toString() + Const.FILE_SEPARATOR + short_filename; destinationfile = KettleVFS.getFileObject(movetofilenamefull, this); if (!simulate) { sourcefilename.moveTo(destinationfile); } if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobMoveFiles.Log.FileMoved", sourcefilename.getName().toString(), destinationfile.getName().toString())); } // add filename to result filename if (add_result_filesname && !iffileexists.equals("fail") && !iffileexists.equals("do_nothing")) { addFileToResultFilenames(destinationfile.toString(), result, parentJob); } updateSuccess(); retval = true; } else if (iffileexists.equals("delete_file")) { if (!simulate) { destinationfilename.delete(); } if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobMoveFiles.Log.FileDeleted", destinationfilename.getName().toString())); } } else if (iffileexists.equals("move_file")) { String short_filename = shortfilename; // return destination short filename try { short_filename = getMoveDestinationFilename(short_filename, null); } catch (Exception e) { logError(BaseMessages.getString(PKG, BaseMessages.getString(PKG, "JobMoveFiles.Error.GettingFilename", short_filename)), e); return retval; } String movetofilenamefull = movetofolderfolder.toString() + Const.FILE_SEPARATOR + short_filename; destinationfile = KettleVFS.getFileObject(movetofilenamefull, this); if (!destinationfile.exists()) { if (!simulate) { sourcefilename.moveTo(destinationfile); } if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobMoveFiles.Log.FileMoved", sourcefilename.getName().toString(), destinationfile.getName().toString())); } // add filename to result filename if (add_result_filesname && !iffileexists.equals("fail") && !iffileexists.equals("do_nothing")) { addFileToResultFilenames(destinationfile.toString(), result, parentJob); } } else { if (ifmovedfileexists.equals("overwrite_file")) { if (!simulate) { sourcefilename.moveTo(destinationfile); } if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobMoveFiles.Log.FileOverwrite", destinationfile.getName().toString())); } // add filename to result filename if (add_result_filesname && !iffileexists.equals("fail") && !iffileexists.equals("do_nothing")) { addFileToResultFilenames(destinationfile.toString(), result, parentJob); } updateSuccess(); retval = true; } else if (ifmovedfileexists.equals("unique_name")) { SimpleDateFormat daf = new SimpleDateFormat(); Date now = new Date(); daf.applyPattern("ddMMyyyy_HHmmssSSS"); String dt = daf.format(now); short_filename += "_" + dt; String destinationfilenamefull = movetofolderfolder.toString() + Const.FILE_SEPARATOR + short_filename; destinationfile = KettleVFS.getFileObject(destinationfilenamefull, this); if (!simulate) { sourcefilename.moveTo(destinationfile); } if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobMoveFiles.Log.FileMoved", destinationfile.getName().toString())); } // add filename to result filename if (add_result_filesname && !iffileexists.equals("fail") && !iffileexists.equals("do_nothing")) { addFileToResultFilenames(destinationfile.toString(), result, parentJob); } updateSuccess(); retval = true; } else if (ifmovedfileexists.equals("fail")) { // Update Errors updateErrors(); } } } else if (iffileexists.equals("fail")) { // Update Errors updateErrors(); } } } catch (Exception e) { logError(BaseMessages.getString(PKG, "JobMoveFiles.Error.Exception.MoveProcessError", sourcefilename.toString(), destinationfilename.toString(), e.getMessage())); updateErrors(); } finally { if (destinationfile != null) { try { destinationfile.close(); } catch (IOException ex) { /* Ignore */ } } } return retval; }
From source file:org.pentaho.di.job.entries.pgpdecryptfiles.JobEntryPGPDecryptFiles.java
private boolean DecryptFile(String shortfilename, FileObject sourcefilename, String passPharse, FileObject destinationfilename, FileObject movetofolderfolder, Job parentJob, Result result) { FileObject destinationfile = null; boolean retval = false; try {/*from ww w. j ava 2 s. c om*/ if (!destinationfilename.exists()) { gpg.decryptFile(sourcefilename, passPharse, destinationfilename); if (isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobPGPDecryptFiles.Log.FileDecrypted", sourcefilename.getName().toString(), destinationfilename.getName().toString())); } // add filename to result filename if (add_result_filesname && !iffileexists.equals("fail") && !iffileexists.equals("do_nothing")) { addFileToResultFilenames(destinationfilename.toString(), result, parentJob); } updateSuccess(); } else { if (isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobPGPDecryptFiles.Log.FileExists", destinationfilename.toString())); } if (iffileexists.equals("overwrite_file")) { gpg.decryptFile(sourcefilename, passPharse, destinationfilename); if (isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobPGPDecryptFiles.Log.FileOverwrite", destinationfilename.getName().toString())); } // add filename to result filename if (add_result_filesname && !iffileexists.equals("fail") && !iffileexists.equals("do_nothing")) { addFileToResultFilenames(destinationfilename.toString(), result, parentJob); } updateSuccess(); } else if (iffileexists.equals("unique_name")) { String short_filename = shortfilename; // return destination short filename try { short_filename = getMoveDestinationFilename(short_filename, "ddMMyyyy_HHmmssSSS"); } catch (Exception e) { logError(BaseMessages.getString(PKG, "JobPGPDecryptFiles.Error.GettingFilename", short_filename), e); return retval; } String movetofilenamefull = destinationfilename.getParent().toString() + Const.FILE_SEPARATOR + short_filename; destinationfile = KettleVFS.getFileObject(movetofilenamefull); gpg.decryptFile(sourcefilename, passPharse, destinationfile); if (isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobPGPDecryptFiles.Log.FileDecrypted", sourcefilename.getName().toString(), destinationfile.getName().toString())); } // add filename to result filename if (add_result_filesname && !iffileexists.equals("fail") && !iffileexists.equals("do_nothing")) { addFileToResultFilenames(destinationfile.toString(), result, parentJob); } updateSuccess(); } else if (iffileexists.equals("delete_file")) { destinationfilename.delete(); if (isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobPGPDecryptFiles.Log.FileDeleted", destinationfilename.getName().toString())); } } else if (iffileexists.equals("move_file")) { String short_filename = shortfilename; // return destination short filename try { short_filename = getMoveDestinationFilename(short_filename, null); } catch (Exception e) { logError(BaseMessages.getString(PKG, "JobPGPDecryptFiles.Error.GettingFilename", short_filename), e); return retval; } String movetofilenamefull = movetofolderfolder.toString() + Const.FILE_SEPARATOR + short_filename; destinationfile = KettleVFS.getFileObject(movetofilenamefull); if (!destinationfile.exists()) { sourcefilename.moveTo(destinationfile); if (isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobPGPDecryptFiles.Log.FileDecrypted", sourcefilename.getName().toString(), destinationfile.getName().toString())); } // add filename to result filename if (add_result_filesname && !iffileexists.equals("fail") && !iffileexists.equals("do_nothing")) { addFileToResultFilenames(destinationfile.toString(), result, parentJob); } } else { if (ifmovedfileexists.equals("overwrite_file")) { sourcefilename.moveTo(destinationfile); if (isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobPGPDecryptFiles.Log.FileOverwrite", destinationfile.getName().toString())); } // add filename to result filename if (add_result_filesname && !iffileexists.equals("fail") && !iffileexists.equals("do_nothing")) { addFileToResultFilenames(destinationfile.toString(), result, parentJob); } updateSuccess(); } else if (ifmovedfileexists.equals("unique_name")) { SimpleDateFormat daf = new SimpleDateFormat(); Date now = new Date(); daf.applyPattern("ddMMyyyy_HHmmssSSS"); String dt = daf.format(now); short_filename += "_" + dt; String destinationfilenamefull = movetofolderfolder.toString() + Const.FILE_SEPARATOR + short_filename; destinationfile = KettleVFS.getFileObject(destinationfilenamefull); sourcefilename.moveTo(destinationfile); if (isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobPGPDecryptFiles.Log.FileDecrypted", destinationfile.getName().toString())); } // add filename to result filename if (add_result_filesname && !iffileexists.equals("fail") && !iffileexists.equals("do_nothing")) { addFileToResultFilenames(destinationfile.toString(), result, parentJob); } updateSuccess(); } else if (ifmovedfileexists.equals("fail")) { // Update Errors updateErrors(); } } } else if (iffileexists.equals("fail")) { // Update Errors updateErrors(); } } } catch (Exception e) { updateErrors(); logError(BaseMessages.getString(PKG, "JobPGPDecryptFiles.Error.Exception.MoveProcessError", sourcefilename.toString(), destinationfilename.toString(), e.getMessage())); } finally { if (destinationfile != null) { try { destinationfile.close(); } catch (IOException ex) { /* Ignore */ } } } return retval; }
From source file:org.pentaho.di.job.entries.pgpencryptfiles.JobEntryPGPEncryptFiles.java
private boolean EncryptFile(int actionType, String shortfilename, FileObject sourcefilename, String userID, FileObject destinationfilename, FileObject movetofolderfolder, Job parentJob, Result result) { FileObject destinationfile = null; boolean retval = false; try {// ww w . j a va2 s . co m if (!destinationfilename.exists()) { doJob(actionType, sourcefilename, userID, destinationfilename); if (isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobPGPEncryptFiles.Log.FileEncrypted", sourcefilename.getName().toString(), destinationfilename.getName().toString())); } // add filename to result filename if (add_result_filesname && !iffileexists.equals("fail") && !iffileexists.equals("do_nothing")) { addFileToResultFilenames(destinationfilename.toString(), result, parentJob); } updateSuccess(); } else { if (isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobPGPEncryptFiles.Log.FileExists", destinationfilename.toString())); } if (iffileexists.equals("overwrite_file")) { doJob(actionType, sourcefilename, userID, destinationfilename); if (isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobPGPEncryptFiles.Log.FileOverwrite", destinationfilename.getName().toString())); } // add filename to result filename if (add_result_filesname && !iffileexists.equals("fail") && !iffileexists.equals("do_nothing")) { addFileToResultFilenames(destinationfilename.toString(), result, parentJob); } updateSuccess(); } else if (iffileexists.equals("unique_name")) { String short_filename = shortfilename; // return destination short filename try { short_filename = getMoveDestinationFilename(short_filename, "ddMMyyyy_HHmmssSSS"); } catch (Exception e) { logError(BaseMessages.getString(PKG, "JobPGPEncryptFiles.Error.GettingFilename", short_filename), e); return retval; } String movetofilenamefull = destinationfilename.getParent().toString() + Const.FILE_SEPARATOR + short_filename; destinationfile = KettleVFS.getFileObject(movetofilenamefull); doJob(actionType, sourcefilename, userID, destinationfilename); if (isDetailed()) { logDetailed(toString(), BaseMessages.getString(PKG, "JobPGPEncryptFiles.Log.FileEncrypted", sourcefilename.getName().toString(), destinationfile.getName().toString())); } // add filename to result filename if (add_result_filesname && !iffileexists.equals("fail") && !iffileexists.equals("do_nothing")) { addFileToResultFilenames(destinationfile.toString(), result, parentJob); } updateSuccess(); } else if (iffileexists.equals("delete_file")) { destinationfilename.delete(); if (isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobPGPEncryptFiles.Log.FileDeleted", destinationfilename.getName().toString())); } } else if (iffileexists.equals("move_file")) { String short_filename = shortfilename; // return destination short filename try { short_filename = getMoveDestinationFilename(short_filename, null); } catch (Exception e) { logError(BaseMessages.getString(PKG, "JobPGPEncryptFiles.Error.GettingFilename", short_filename), e); return retval; } String movetofilenamefull = movetofolderfolder.toString() + Const.FILE_SEPARATOR + short_filename; destinationfile = KettleVFS.getFileObject(movetofilenamefull); if (!destinationfile.exists()) { sourcefilename.moveTo(destinationfile); if (isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobPGPEncryptFiles.Log.FileEncrypted", sourcefilename.getName().toString(), destinationfile.getName().toString())); } // add filename to result filename if (add_result_filesname && !iffileexists.equals("fail") && !iffileexists.equals("do_nothing")) { addFileToResultFilenames(destinationfile.toString(), result, parentJob); } } else { if (ifmovedfileexists.equals("overwrite_file")) { sourcefilename.moveTo(destinationfile); if (isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobPGPEncryptFiles.Log.FileOverwrite", destinationfile.getName().toString())); } // add filename to result filename if (add_result_filesname && !iffileexists.equals("fail") && !iffileexists.equals("do_nothing")) { addFileToResultFilenames(destinationfile.toString(), result, parentJob); } updateSuccess(); } else if (ifmovedfileexists.equals("unique_name")) { SimpleDateFormat daf = new SimpleDateFormat(); Date now = new Date(); daf.applyPattern("ddMMyyyy_HHmmssSSS"); String dt = daf.format(now); short_filename += "_" + dt; String destinationfilenamefull = movetofolderfolder.toString() + Const.FILE_SEPARATOR + short_filename; destinationfile = KettleVFS.getFileObject(destinationfilenamefull); sourcefilename.moveTo(destinationfile); if (isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobPGPEncryptFiles.Log.FileEncrypted", destinationfile.getName().toString())); } // add filename to result filename if (add_result_filesname && !iffileexists.equals("fail") && !iffileexists.equals("do_nothing")) { addFileToResultFilenames(destinationfile.toString(), result, parentJob); } updateSuccess(); } else if (ifmovedfileexists.equals("fail")) { // Update Errors updateErrors(); } } } else if (iffileexists.equals("fail")) { // Update Errors updateErrors(); } } } catch (Exception e) { updateErrors(); logError(BaseMessages.getString(PKG, "JobPGPEncryptFiles.Error.Exception.MoveProcessError", sourcefilename.toString(), destinationfilename.toString(), e.getMessage())); } finally { if (destinationfile != null) { try { destinationfile.close(); } catch (IOException ex) { /* Ignore */ } } } return retval; }
From source file:org.pentaho.di.job.entries.sftpput.JobEntrySFTPPUT.java
public Result execute(Result previousResult, int nr) throws KettleException { Result result = previousResult; List<RowMetaAndData> rows = result.getRows(); result.setResult(false);/*from w ww .j a v a 2 s. co m*/ if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobSFTPPUT.Log.StartJobEntry")); } ArrayList<FileObject> myFileList = new ArrayList<FileObject>(); if (copyprevious) { if (rows.size() == 0) { if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobSFTPPUT.ArgsFromPreviousNothing")); } result.setResult(true); return result; } try { RowMetaAndData resultRow = null; // Copy the input row to the (command line) arguments for (int iteration = 0; iteration < rows.size(); iteration++) { resultRow = rows.get(iteration); // Get file names String file_previous = resultRow.getString(0, null); if (!Const.isEmpty(file_previous)) { FileObject file = KettleVFS.getFileObject(file_previous, this); if (!file.exists()) { logError(BaseMessages.getString(PKG, "JobSFTPPUT.Log.FilefromPreviousNotFound", file_previous)); } else { myFileList.add(file); if (log.isDebug()) { logDebug(BaseMessages.getString(PKG, "JobSFTPPUT.Log.FilenameFromResult", file_previous)); } } } } } catch (Exception e) { logError(BaseMessages.getString(PKG, "JobSFTPPUT.Error.ArgFromPrevious")); result.setNrErrors(1); // free resource myFileList = null; return result; } } if (copypreviousfiles) { List<ResultFile> resultFiles = result.getResultFilesList(); if (resultFiles == null || resultFiles.size() == 0) { if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobSFTPPUT.ArgsFromPreviousNothingFiles")); } result.setResult(true); return result; } try { for (Iterator<ResultFile> it = resultFiles.iterator(); it.hasNext() && !parentJob.isStopped();) { ResultFile resultFile = it.next(); FileObject file = resultFile.getFile(); if (file != null) { if (!file.exists()) { logError(BaseMessages.getString(PKG, "JobSFTPPUT.Log.FilefromPreviousNotFound", file.toString())); } else { myFileList.add(file); if (log.isDebug()) { logDebug(BaseMessages.getString(PKG, "JobSFTPPUT.Log.FilenameFromResult", file.toString())); } } } } } catch (Exception e) { logError(BaseMessages.getString(PKG, "JobSFTPPUT.Error.ArgFromPrevious")); result.setNrErrors(1); // free resource myFileList = null; return result; } } SFTPClient sftpclient = null; // String substitution.. String realServerName = environmentSubstitute(serverName); String realServerPort = environmentSubstitute(serverPort); String realUsername = environmentSubstitute(userName); String realPassword = Encr.decryptPasswordOptionallyEncrypted(environmentSubstitute(password)); String realSftpDirString = environmentSubstitute(sftpDirectory); String realWildcard = environmentSubstitute(wildcard); String realLocalDirectory = environmentSubstitute(localDirectory); String realKeyFilename = null; String realPassPhrase = null; // Destination folder (Move to) String realDestinationFolder = environmentSubstitute(getDestinationFolder()); try { // Let's perform some checks before starting if (getAfterFTPS() == AFTER_FTPSPUT_MOVE) { if (Const.isEmpty(realDestinationFolder)) { logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.DestinatFolderMissing")); result.setNrErrors(1); return result; } else { FileObject folder = null; try { folder = KettleVFS.getFileObject(realDestinationFolder, this); // Let's check if folder exists... if (!folder.exists()) { // Do we need to create it? if (createDestinationFolder) { folder.createFolder(); } else { logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.DestinatFolderNotExist", realDestinationFolder)); result.setNrErrors(1); return result; } } realDestinationFolder = KettleVFS.getFilename(folder); } catch (Exception e) { throw new KettleException(e); } finally { if (folder != null) { try { folder.close(); } catch (Exception e) { /* Ignore */ } } } } } if (isUseKeyFile()) { // We must have here a private keyfilename realKeyFilename = environmentSubstitute(getKeyFilename()); if (Const.isEmpty(realKeyFilename)) { // Error..Missing keyfile logError(BaseMessages.getString(PKG, "JobSFTP.Error.KeyFileMissing")); result.setNrErrors(1); return result; } if (!KettleVFS.fileExists(realKeyFilename)) { // Error.. can not reach keyfile logError(BaseMessages.getString(PKG, "JobSFTP.Error.KeyFileNotFound")); result.setNrErrors(1); return result; } realPassPhrase = environmentSubstitute(getKeyPassPhrase()); } // Create sftp client to host ... sftpclient = new SFTPClient(InetAddress.getByName(realServerName), Const.toInt(realServerPort, 22), realUsername, realKeyFilename, realPassPhrase); if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobSFTPPUT.Log.OpenedConnection", realServerName, "" + realServerPort, realUsername)); } // Set compression sftpclient.setCompression(getCompression()); // Set proxy? String realProxyHost = environmentSubstitute(getProxyHost()); if (!Const.isEmpty(realProxyHost)) { // Set proxy sftpclient.setProxy(realProxyHost, environmentSubstitute(getProxyPort()), environmentSubstitute(getProxyUsername()), environmentSubstitute(getProxyPassword()), getProxyType()); } // login to ftp host ... sftpclient.login(realPassword); // Don't show the password in the logs, it's not good for security audits // logDetailed("logged in using password "+realPassword); // Logging this seems a bad idea! Oh well. // move to spool dir ... if (!Const.isEmpty(realSftpDirString)) { boolean existfolder = sftpclient.folderExists(realSftpDirString); if (!existfolder) { if (!isCreateRemoteFolder()) { throw new KettleException(BaseMessages.getString(PKG, "JobSFTPPUT.Error.CanNotFindRemoteFolder", realSftpDirString)); } if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobSFTPPUT.Error.CanNotFindRemoteFolder", realSftpDirString)); } // Let's create folder sftpclient.createFolder(realSftpDirString); if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobSFTPPUT.Log.RemoteFolderCreated", realSftpDirString)); } } sftpclient.chdir(realSftpDirString); if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobSFTPPUT.Log.ChangedDirectory", realSftpDirString)); } } // end if if (!copyprevious && !copypreviousfiles) { // Get all the files in the local directory... myFileList = new ArrayList<FileObject>(); FileObject localFiles = KettleVFS.getFileObject(realLocalDirectory, this); FileObject[] children = localFiles.getChildren(); if (children != null) { for (int i = 0; i < children.length; i++) { // Get filename of file or directory if (children[i].getType().equals(FileType.FILE)) { // myFileList.add(children[i].getAbsolutePath()); myFileList.add(children[i]); } } // end for } } if (myFileList == null || myFileList.size() == 0) { if (isSuccessWhenNoFile()) { // Just warn user if (isBasic()) { logBasic(BaseMessages.getString(PKG, "JobSFTPPUT.Error.NoFileToSend")); } } else { // Fail logError(BaseMessages.getString(PKG, "JobSFTPPUT.Error.NoFileToSend")); result.setNrErrors(1); return result; } } if (log.isDetailed()) { logDetailed( BaseMessages.getString(PKG, "JobSFTPPUT.Log.RowsFromPreviousResult", myFileList.size())); } Pattern pattern = null; if (!copyprevious && !copypreviousfiles) { if (!Const.isEmpty(realWildcard)) { pattern = Pattern.compile(realWildcard); } } // Get the files in the list and execute sftp.put() for each file Iterator<FileObject> it = myFileList.iterator(); while (it.hasNext() && !parentJob.isStopped()) { FileObject myFile = it.next(); try { String localFilename = myFile.toString(); String destinationFilename = myFile.getName().getBaseName(); boolean getIt = true; // First see if the file matches the regular expression! if (pattern != null) { Matcher matcher = pattern.matcher(destinationFilename); getIt = matcher.matches(); } if (getIt) { if (log.isDebug()) { logDebug(BaseMessages.getString(PKG, "JobSFTPPUT.Log.PuttingFile", localFilename, realSftpDirString)); } sftpclient.put(myFile, destinationFilename); if (log.isDetailed()) { logDetailed( BaseMessages.getString(PKG, "JobSFTPPUT.Log.TransferedFile", localFilename)); } // We successfully uploaded the file // what's next ... switch (getAfterFTPS()) { case AFTER_FTPSPUT_DELETE: myFile.delete(); if (log.isDetailed()) { logDetailed( BaseMessages.getString(PKG, "JobSFTPPUT.Log.DeletedFile", localFilename)); } break; case AFTER_FTPSPUT_MOVE: FileObject destination = null; try { destination = KettleVFS.getFileObject(realDestinationFolder + Const.FILE_SEPARATOR + myFile.getName().getBaseName(), this); myFile.moveTo(destination); if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobSFTPPUT.Log.FileMoved", myFile, destination)); } } finally { if (destination != null) { destination.close(); } } break; default: if (addFilenameResut) { // Add to the result files... ResultFile resultFile = new ResultFile(ResultFile.FILE_TYPE_GENERAL, myFile, parentJob.getJobname(), toString()); result.getResultFiles().put(resultFile.getFile().toString(), resultFile); if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobSFTPPUT.Log.FilenameAddedToResultFilenames", localFilename)); } } break; } } } finally { if (myFile != null) { myFile.close(); } } } // end for result.setResult(true); // JKU: no idea if this is needed...! // result.setNrFilesRetrieved(filesRetrieved); } catch (Exception e) { result.setNrErrors(1); logError(BaseMessages.getString(PKG, "JobSFTPPUT.Exception", e.getMessage())); logError(Const.getStackTracker(e)); } finally { // close connection, if possible try { if (sftpclient != null) { sftpclient.disconnect(); } } catch (Exception e) { // just ignore this, makes no big difference } // end catch myFileList = null; } // end finally return result; }