List of usage examples for org.apache.commons.vfs FileObject toString
public String toString()
From source file:org.pentaho.di.job.entries.sftpput.JobEntrySFTPPUT.java
public Result execute(Result previousResult, int nr) throws KettleException { Result result = previousResult; List<RowMetaAndData> rows = result.getRows(); result.setResult(false);/*ww w. j a v a 2 s .co m*/ if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobSFTPPUT.Log.StartJobEntry")); } ArrayList<FileObject> myFileList = new ArrayList<FileObject>(); if (copyprevious) { if (rows.size() == 0) { if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobSFTPPUT.ArgsFromPreviousNothing")); } result.setResult(true); return result; } try { RowMetaAndData resultRow = null; // Copy the input row to the (command line) arguments for (int iteration = 0; iteration < rows.size(); iteration++) { resultRow = rows.get(iteration); // Get file names String file_previous = resultRow.getString(0, null); if (!Const.isEmpty(file_previous)) { FileObject file = KettleVFS.getFileObject(file_previous, this); if (!file.exists()) { logError(BaseMessages.getString(PKG, "JobSFTPPUT.Log.FilefromPreviousNotFound", file_previous)); } else { myFileList.add(file); if (log.isDebug()) { logDebug(BaseMessages.getString(PKG, "JobSFTPPUT.Log.FilenameFromResult", file_previous)); } } } } } catch (Exception e) { logError(BaseMessages.getString(PKG, "JobSFTPPUT.Error.ArgFromPrevious")); result.setNrErrors(1); // free resource myFileList = null; return result; } } if (copypreviousfiles) { List<ResultFile> resultFiles = result.getResultFilesList(); if (resultFiles == null || resultFiles.size() == 0) { if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobSFTPPUT.ArgsFromPreviousNothingFiles")); } result.setResult(true); return result; } try { for (Iterator<ResultFile> it = resultFiles.iterator(); it.hasNext() && !parentJob.isStopped();) { ResultFile resultFile = it.next(); FileObject file = resultFile.getFile(); if (file != null) { if (!file.exists()) { logError(BaseMessages.getString(PKG, "JobSFTPPUT.Log.FilefromPreviousNotFound", file.toString())); } else { myFileList.add(file); if (log.isDebug()) { logDebug(BaseMessages.getString(PKG, "JobSFTPPUT.Log.FilenameFromResult", file.toString())); } } } } } catch (Exception e) { logError(BaseMessages.getString(PKG, "JobSFTPPUT.Error.ArgFromPrevious")); result.setNrErrors(1); // free resource myFileList = null; return result; } } SFTPClient sftpclient = null; // String substitution.. String realServerName = environmentSubstitute(serverName); String realServerPort = environmentSubstitute(serverPort); String realUsername = environmentSubstitute(userName); String realPassword = Encr.decryptPasswordOptionallyEncrypted(environmentSubstitute(password)); String realSftpDirString = environmentSubstitute(sftpDirectory); String realWildcard = environmentSubstitute(wildcard); String realLocalDirectory = environmentSubstitute(localDirectory); String realKeyFilename = null; String realPassPhrase = null; // Destination folder (Move to) String realDestinationFolder = environmentSubstitute(getDestinationFolder()); try { // Let's perform some checks before starting if (getAfterFTPS() == AFTER_FTPSPUT_MOVE) { if (Const.isEmpty(realDestinationFolder)) { logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.DestinatFolderMissing")); result.setNrErrors(1); return result; } else { FileObject folder = null; try { folder = KettleVFS.getFileObject(realDestinationFolder, this); // Let's check if folder exists... if (!folder.exists()) { // Do we need to create it? if (createDestinationFolder) { folder.createFolder(); } else { logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.DestinatFolderNotExist", realDestinationFolder)); result.setNrErrors(1); return result; } } realDestinationFolder = KettleVFS.getFilename(folder); } catch (Exception e) { throw new KettleException(e); } finally { if (folder != null) { try { folder.close(); } catch (Exception e) { /* Ignore */ } } } } } if (isUseKeyFile()) { // We must have here a private keyfilename realKeyFilename = environmentSubstitute(getKeyFilename()); if (Const.isEmpty(realKeyFilename)) { // Error..Missing keyfile logError(BaseMessages.getString(PKG, "JobSFTP.Error.KeyFileMissing")); result.setNrErrors(1); return result; } if (!KettleVFS.fileExists(realKeyFilename)) { // Error.. can not reach keyfile logError(BaseMessages.getString(PKG, "JobSFTP.Error.KeyFileNotFound")); result.setNrErrors(1); return result; } realPassPhrase = environmentSubstitute(getKeyPassPhrase()); } // Create sftp client to host ... sftpclient = new SFTPClient(InetAddress.getByName(realServerName), Const.toInt(realServerPort, 22), realUsername, realKeyFilename, realPassPhrase); if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobSFTPPUT.Log.OpenedConnection", realServerName, "" + realServerPort, realUsername)); } // Set compression sftpclient.setCompression(getCompression()); // Set proxy? String realProxyHost = environmentSubstitute(getProxyHost()); if (!Const.isEmpty(realProxyHost)) { // Set proxy sftpclient.setProxy(realProxyHost, environmentSubstitute(getProxyPort()), environmentSubstitute(getProxyUsername()), environmentSubstitute(getProxyPassword()), getProxyType()); } // login to ftp host ... sftpclient.login(realPassword); // Don't show the password in the logs, it's not good for security audits // logDetailed("logged in using password "+realPassword); // Logging this seems a bad idea! Oh well. // move to spool dir ... if (!Const.isEmpty(realSftpDirString)) { boolean existfolder = sftpclient.folderExists(realSftpDirString); if (!existfolder) { if (!isCreateRemoteFolder()) { throw new KettleException(BaseMessages.getString(PKG, "JobSFTPPUT.Error.CanNotFindRemoteFolder", realSftpDirString)); } if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobSFTPPUT.Error.CanNotFindRemoteFolder", realSftpDirString)); } // Let's create folder sftpclient.createFolder(realSftpDirString); if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobSFTPPUT.Log.RemoteFolderCreated", realSftpDirString)); } } sftpclient.chdir(realSftpDirString); if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobSFTPPUT.Log.ChangedDirectory", realSftpDirString)); } } // end if if (!copyprevious && !copypreviousfiles) { // Get all the files in the local directory... myFileList = new ArrayList<FileObject>(); FileObject localFiles = KettleVFS.getFileObject(realLocalDirectory, this); FileObject[] children = localFiles.getChildren(); if (children != null) { for (int i = 0; i < children.length; i++) { // Get filename of file or directory if (children[i].getType().equals(FileType.FILE)) { // myFileList.add(children[i].getAbsolutePath()); myFileList.add(children[i]); } } // end for } } if (myFileList == null || myFileList.size() == 0) { if (isSuccessWhenNoFile()) { // Just warn user if (isBasic()) { logBasic(BaseMessages.getString(PKG, "JobSFTPPUT.Error.NoFileToSend")); } } else { // Fail logError(BaseMessages.getString(PKG, "JobSFTPPUT.Error.NoFileToSend")); result.setNrErrors(1); return result; } } if (log.isDetailed()) { logDetailed( BaseMessages.getString(PKG, "JobSFTPPUT.Log.RowsFromPreviousResult", myFileList.size())); } Pattern pattern = null; if (!copyprevious && !copypreviousfiles) { if (!Const.isEmpty(realWildcard)) { pattern = Pattern.compile(realWildcard); } } // Get the files in the list and execute sftp.put() for each file Iterator<FileObject> it = myFileList.iterator(); while (it.hasNext() && !parentJob.isStopped()) { FileObject myFile = it.next(); try { String localFilename = myFile.toString(); String destinationFilename = myFile.getName().getBaseName(); boolean getIt = true; // First see if the file matches the regular expression! if (pattern != null) { Matcher matcher = pattern.matcher(destinationFilename); getIt = matcher.matches(); } if (getIt) { if (log.isDebug()) { logDebug(BaseMessages.getString(PKG, "JobSFTPPUT.Log.PuttingFile", localFilename, realSftpDirString)); } sftpclient.put(myFile, destinationFilename); if (log.isDetailed()) { logDetailed( BaseMessages.getString(PKG, "JobSFTPPUT.Log.TransferedFile", localFilename)); } // We successfully uploaded the file // what's next ... switch (getAfterFTPS()) { case AFTER_FTPSPUT_DELETE: myFile.delete(); if (log.isDetailed()) { logDetailed( BaseMessages.getString(PKG, "JobSFTPPUT.Log.DeletedFile", localFilename)); } break; case AFTER_FTPSPUT_MOVE: FileObject destination = null; try { destination = KettleVFS.getFileObject(realDestinationFolder + Const.FILE_SEPARATOR + myFile.getName().getBaseName(), this); myFile.moveTo(destination); if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobSFTPPUT.Log.FileMoved", myFile, destination)); } } finally { if (destination != null) { destination.close(); } } break; default: if (addFilenameResut) { // Add to the result files... ResultFile resultFile = new ResultFile(ResultFile.FILE_TYPE_GENERAL, myFile, parentJob.getJobname(), toString()); result.getResultFiles().put(resultFile.getFile().toString(), resultFile); if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobSFTPPUT.Log.FilenameAddedToResultFilenames", localFilename)); } } break; } } } finally { if (myFile != null) { myFile.close(); } } } // end for result.setResult(true); // JKU: no idea if this is needed...! // result.setNrFilesRetrieved(filesRetrieved); } catch (Exception e) { result.setNrErrors(1); logError(BaseMessages.getString(PKG, "JobSFTPPUT.Exception", e.getMessage())); logError(Const.getStackTracker(e)); } finally { // close connection, if possible try { if (sftpclient != null) { sftpclient.disconnect(); } } catch (Exception e) { // just ignore this, makes no big difference } // end catch myFileList = null; } // end finally return result; }
From source file:org.pentaho.di.job.entries.shell.JobEntryShell.java
private void executeShell(Result result, List<RowMetaAndData> cmdRows, String[] args) { FileObject fileObject = null; String realScript = null;/*from w ww. jav a 2s . c om*/ FileObject tempFile = null; try { // What's the exact command? String[] base = null; List<String> cmds = new ArrayList<String>(); if (log.isBasic()) { logBasic(BaseMessages.getString(PKG, "JobShell.RunningOn", Const.getOS())); } if (insertScript) { realScript = environmentSubstitute(script); } else { String realFilename = environmentSubstitute(getFilename()); fileObject = KettleVFS.getFileObject(realFilename, this); } if (Const.getOS().equals("Windows 95")) { base = new String[] { "command.com", "/C" }; if (insertScript) { tempFile = KettleVFS.createTempFile("kettle", "shell.bat", environmentSubstitute(workDirectory), this); fileObject = createTemporaryShellFile(tempFile, realScript); } } else if (Const.getOS().startsWith("Windows")) { base = new String[] { "cmd.exe", "/C" }; if (insertScript) { tempFile = KettleVFS.createTempFile("kettle", "shell.bat", environmentSubstitute(workDirectory), this); fileObject = createTemporaryShellFile(tempFile, realScript); } } else { if (insertScript) { tempFile = KettleVFS.createTempFile("kettle", "shell", environmentSubstitute(workDirectory), this); fileObject = createTemporaryShellFile(tempFile, realScript); } base = new String[] { KettleVFS.getFilename(fileObject) }; } // Construct the arguments... if (argFromPrevious && cmdRows != null) { // Add the base command... for (int i = 0; i < base.length; i++) { cmds.add(base[i]); } if (Const.getOS().equals("Windows 95") || Const.getOS().startsWith("Windows")) { // for windows all arguments including the command itself // need to be // included in 1 argument to cmd/command. StringBuffer cmdline = new StringBuffer(300); cmdline.append('"'); cmdline.append(Const.optionallyQuoteStringByOS(KettleVFS.getFilename(fileObject))); // Add the arguments from previous results... for (int i = 0; i < cmdRows.size(); i++) { // Normally just one row, but once in a while to remain compatible we have multiple. RowMetaAndData r = cmdRows.get(i); for (int j = 0; j < r.size(); j++) { cmdline.append(' '); cmdline.append(Const.optionallyQuoteStringByOS(r.getString(j, null))); } } cmdline.append('"'); cmds.add(cmdline.toString()); } else { // Add the arguments from previous results... for (int i = 0; i < cmdRows.size(); i++) { // Normally just one row, but once in a while to remain compatible we have multiple. RowMetaAndData r = cmdRows.get(i); for (int j = 0; j < r.size(); j++) { cmds.add(Const.optionallyQuoteStringByOS(r.getString(j, null))); } } } } else if (args != null) { // Add the base command... for (int i = 0; i < base.length; i++) { cmds.add(base[i]); } if (Const.getOS().equals("Windows 95") || Const.getOS().startsWith("Windows")) { // for windows all arguments including the command itself // need to be // included in 1 argument to cmd/command. StringBuffer cmdline = new StringBuffer(300); cmdline.append('"'); cmdline.append(Const.optionallyQuoteStringByOS(KettleVFS.getFilename(fileObject))); for (int i = 0; i < args.length; i++) { cmdline.append(' '); cmdline.append(Const.optionallyQuoteStringByOS(args[i])); } cmdline.append('"'); cmds.add(cmdline.toString()); } else { for (int i = 0; i < args.length; i++) { cmds.add(args[i]); } } } StringBuffer command = new StringBuffer(); Iterator<String> it = cmds.iterator(); boolean first = true; while (it.hasNext()) { if (!first) { command.append(' '); } else { first = false; } command.append(it.next()); } if (log.isBasic()) { logBasic(BaseMessages.getString(PKG, "JobShell.ExecCommand", command.toString())); } // Build the environment variable list... ProcessBuilder procBuilder = new ProcessBuilder(cmds); Map<String, String> env = procBuilder.environment(); String[] variables = listVariables(); for (int i = 0; i < variables.length; i++) { env.put(variables[i], getVariable(variables[i])); } if (getWorkDirectory() != null && !Const.isEmpty(Const.rtrim(getWorkDirectory()))) { String vfsFilename = environmentSubstitute(getWorkDirectory()); File file = new File(KettleVFS.getFilename(KettleVFS.getFileObject(vfsFilename, this))); procBuilder.directory(file); } Process proc = procBuilder.start(); // any error message? StreamLogger errorLogger = new StreamLogger(log, proc.getErrorStream(), "(stderr)", true); // any output? StreamLogger outputLogger = new StreamLogger(log, proc.getInputStream(), "(stdout)"); // kick them off Thread errorLoggerThread = new Thread(errorLogger); errorLoggerThread.start(); Thread outputLoggerThread = new Thread(outputLogger); outputLoggerThread.start(); proc.waitFor(); if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobShell.CommandFinished", command.toString())); } // What's the exit status? result.setExitStatus(proc.exitValue()); if (result.getExitStatus() != 0) { if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobShell.ExitStatus", environmentSubstitute(getFilename()), "" + result.getExitStatus())); } result.setNrErrors(1); } // wait until loggers read all data from stdout and stderr errorLoggerThread.join(); outputLoggerThread.join(); // close the streams // otherwise you get "Too many open files, java.io.IOException" after a lot of iterations proc.getErrorStream().close(); proc.getOutputStream().close(); } catch (IOException ioe) { logError(BaseMessages.getString(PKG, "JobShell.ErrorRunningShell", environmentSubstitute(getFilename()), ioe.toString()), ioe); result.setNrErrors(1); } catch (InterruptedException ie) { logError(BaseMessages.getString(PKG, "JobShell.Shellinterupted", environmentSubstitute(getFilename()), ie.toString()), ie); result.setNrErrors(1); } catch (Exception e) { logError(BaseMessages.getString(PKG, "JobShell.UnexpectedError", environmentSubstitute(getFilename()), e.toString()), e); result.setNrErrors(1); } finally { // If we created a temporary file, remove it... // if (tempFile != null) { try { tempFile.delete(); } catch (Exception e) { BaseMessages.getString(PKG, "JobShell.UnexpectedError", tempFile.toString(), e.toString()); } } } if (result.getNrErrors() > 0) { result.setResult(false); } else { result.setResult(true); } }
From source file:org.pentaho.di.job.entries.ssh2get.JobEntrySSH2GET.java
private boolean CreateFolder(String filefolder) { FileObject folder = null; try {/* ww w.j a v a2 s. c om*/ folder = KettleVFS.getFileObject(filefolder, this); if (!folder.exists()) { if (createtargetfolder) { folder.createFolder(); if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobSSH2GET.Log.FolderCreated", folder.toString())); } } else { return false; } } return true; } catch (Exception e) { logError(BaseMessages.getString(PKG, "JobSSH2GET.Log.CanNotCreateFolder", folder.toString()), e); } finally { if (folder != null) { try { folder.close(); } catch (Exception ex) { /* Ignore */ } } } return false; }
From source file:org.pentaho.di.job.entries.ssh2put.JobEntrySSH2PUT.java
public Result execute(Result previousResult, int nr) { Result result = previousResult; result.setResult(false);//w ww. j av a 2 s . c o m try { // Get real variable value String realServerName = environmentSubstitute(serverName); int realServerPort = Const.toInt(environmentSubstitute(serverPort), 22); String realUserName = environmentSubstitute(userName); String realServerPassword = Encr.decryptPasswordOptionallyEncrypted(environmentSubstitute(password)); // Proxy Host String realProxyHost = environmentSubstitute(httpproxyhost); int realProxyPort = Const.toInt(environmentSubstitute(httpproxyport), 22); String realproxyUserName = environmentSubstitute(httpproxyusername); String realProxyPassword = Encr .decryptPasswordOptionallyEncrypted(environmentSubstitute(httpProxyPassword)); // Key file String realKeyFilename = environmentSubstitute(keyFilename); String relKeyFilepass = environmentSubstitute(keyFilePass); // Source files String realLocalDirectory = environmentSubstitute(localDirectory); String realwildcard = environmentSubstitute(wildcard); // Remote destination String realftpDirectory = environmentSubstitute(ftpDirectory); // Destination folder (Move to) String realDestinationFolder = environmentSubstitute(destinationfolder); try { // Remote source realftpDirectory = FTPUtils.normalizePath(realftpDirectory); // Destination folder (Move to) realDestinationFolder = FTPUtils.normalizePath(realDestinationFolder); } catch (Exception e) { logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.CanNotNormalizePath", e.getMessage())); result.setNrErrors(1); return result; } // Check for mandatory fields boolean mandatoryok = true; if (Const.isEmpty(realServerName)) { mandatoryok = false; logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.ServernameMissing")); } if (usehttpproxy) { if (Const.isEmpty(realProxyHost)) { mandatoryok = false; logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.HttpProxyhostMissing")); } } if (publicpublickey) { if (Const.isEmpty(realKeyFilename)) { mandatoryok = false; logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.KeyFileMissing")); } else { // Let's check if folder exists... if (!KettleVFS.fileExists(realKeyFilename, this)) { mandatoryok = false; logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.KeyFileNotExist")); } } } if (Const.isEmpty(realLocalDirectory)) { mandatoryok = false; logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.LocalFolderMissing")); } if (afterFtpPut.equals("move_file")) { if (Const.isEmpty(realDestinationFolder)) { mandatoryok = false; logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.DestinatFolderMissing")); } else { FileObject folder = null; try { folder = KettleVFS.getFileObject(realDestinationFolder, this); // Let's check if folder exists... if (!folder.exists()) { // Do we need to create it? if (createDestinationFolder) { folder.createFolder(); } else { logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.DestinatFolderNotExist", realDestinationFolder)); } } } catch (Exception e) { throw new KettleException(e); } finally { if (folder != null) { try { folder.close(); folder = null; } catch (Exception e) { /* Ignore */ } } } } } if (mandatoryok) { Connection conn = null; SFTPv3Client client = null; boolean good = true; int nbfilestoput = 0; int nbput = 0; int nbrerror = 0; try { // Create a connection instance conn = getConnection(realServerName, realServerPort, realProxyHost, realProxyPort, realproxyUserName, realProxyPassword); if (timeout > 0) { // Use timeout // Cache Host Key if (cachehostkey) { conn.connect(new SimpleVerifier(database), 0, timeout * 1000); } else { conn.connect(null, 0, timeout * 1000); } } else { // Cache Host Key if (cachehostkey) { conn.connect(new SimpleVerifier(database)); } else { conn.connect(); } } // Authenticate boolean isAuthenticated = false; if (publicpublickey) { String keyContent = KettleVFS.getTextFileContent(realKeyFilename, this, Const.XML_ENCODING); isAuthenticated = conn.authenticateWithPublicKey(realUserName, keyContent.toCharArray(), relKeyFilepass); } else { isAuthenticated = conn.authenticateWithPassword(realUserName, realServerPassword); } // LET'S CHECK AUTHENTICATION ... if (isAuthenticated == false) { logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.AuthenticationFailed")); } else { if (log.isBasic()) { logBasic(BaseMessages.getString(PKG, "JobSSH2PUT.Log.Connected", serverName, userName)); } client = new SFTPv3Client(conn); if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobSSH2PUT.Log.ProtocolVersion", "" + client.getProtocolVersion())); } // Check if remote directory exists if (!Const.isEmpty(realftpDirectory)) { if (!sshDirectoryExists(client, realftpDirectory)) { good = false; if (createRemoteFolder) { good = CreateRemoteFolder(client, realftpDirectory); if (good) { logBasic(BaseMessages.getString(PKG, "JobSSH2PUT.Log.RemoteDirectoryCreated")); } } else { logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.RemoteDirectoryNotExist", realftpDirectory)); } } else if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobSSH2PUT.Log.RemoteDirectoryExist", realftpDirectory)); } } if (good) { // Get files list from local folder (source) List<FileObject> myFileList = getFiles(realLocalDirectory); // Prepare Pattern for wildcard Pattern pattern = null; if (!Const.isEmpty(realwildcard)) { pattern = Pattern.compile(realwildcard); } // Let's put files now ... // Get the files in the list for (int i = 0; i < myFileList.size() && !parentJob.isStopped(); i++) { FileObject myFile = myFileList.get(i); String localFilename = myFile.toString(); String remoteFilename = myFile.getName().getBaseName(); boolean getIt = true; // First see if the file matches the regular expression! if (pattern != null) { Matcher matcher = pattern.matcher(remoteFilename); getIt = matcher.matches(); } // do we have a target directory? if (!Const.isEmpty(realftpDirectory)) { remoteFilename = realftpDirectory + FTPUtils.FILE_SEPARATOR + remoteFilename; } if (onlyGettingNewFiles) { // We get only new files // ie not exist on the remote server getIt = !sshFileExists(client, remoteFilename); } if (getIt) { nbfilestoput++; boolean putok = putFile(myFile, remoteFilename, client); if (!putok) { nbrerror++; logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.Error.CanNotPutFile", localFilename)); } else { nbput++; } if (putok && !afterFtpPut.equals("do_nothing")) { deleteOrMoveFiles(myFile, realDestinationFolder); } } } /******************************** RESULT ********************/ if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobSSH2PUT.Log.Result.JobEntryEnd1")); logDetailed(BaseMessages.getString(PKG, "JobSSH2PUT.Log.Result.TotalFiles", "" + nbfilestoput)); logDetailed(BaseMessages.getString(PKG, "JobSSH2PUT.Log.Result.TotalFilesPut", "" + nbput)); logDetailed(BaseMessages.getString(PKG, "JobSSH2PUT.Log.Result.TotalFilesError", "" + nbrerror)); logDetailed(BaseMessages.getString(PKG, "JobSSH2PUT.Log.Result.JobEntryEnd2")); } if (nbrerror == 0) { result.setResult(true); /******************************** RESULT ********************/ } } } } catch (Exception e) { result.setNrErrors(nbrerror); logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.Error.ErrorFTP", e.getMessage())); } finally { if (conn != null) { conn.close(); } if (client != null) { client.close(); } } } } catch (Exception e) { result.setResult(false); result.setNrErrors(1L); logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.Error.UnexpectedError"), e); } return result; }
From source file:org.pentaho.di.job.entries.ssh2put.JobEntrySSH2PUT.java
private boolean putFile(FileObject localFile, String remotefilename, SFTPv3Client sftpClient) { long filesize = -1; InputStream in = null;//from w ww.j av a 2 s . c o m BufferedInputStream inBuf = null; SFTPv3FileHandle sftpFileHandle = null; boolean retval = false; try { // Put file in the folder sftpFileHandle = sftpClient.createFileTruncate(remotefilename); // Associate a file input stream for the current local file in = KettleVFS.getInputStream(localFile); inBuf = new BufferedInputStream(in); byte[] buf = new byte[2048]; long offset = 0; long length = localFile.getContent().getSize(); if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobSSH2PUT.Log.SendingFile", localFile.toString(), "" + length, remotefilename)); } // Write to remote file while (true) { int len = in.read(buf, 0, buf.length); if (len <= 0) { break; } sftpClient.write(sftpFileHandle, offset, buf, 0, len); offset += len; } // Get File size filesize = getFileSize(sftpClient, remotefilename); if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobSSH2PUT.Log.FileOnRemoteHost", remotefilename, "" + filesize)); } retval = true; } catch (Exception e) { // We failed to put files logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.ErrorCopyingFile", localFile.toString()) + ":" + e.getMessage()); } finally { if (in != null) { try { in.close(); in = null; } catch (Exception ex) { // Ignore errors } } if (inBuf != null) { try { inBuf.close(); inBuf = null; } catch (Exception ex) { // Ignore errors } } if (sftpFileHandle != null) { try { sftpClient.closeFile(sftpFileHandle); sftpFileHandle = null; } catch (Exception ex) { // Ignore errors } } } return retval; }
From source file:org.pentaho.di.job.entries.ssh2put.JobEntrySSH2PUT.java
private boolean deleteOrMoveFiles(FileObject file, String destinationFolder) throws KettleException { try {//from w w w . j a v a 2s .com boolean retval = false; // Delete the file if this is needed! // if (afterFtpPut.equals("delete_file")) { file.delete(); retval = true; if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobSSH2PUT.Log.DeletedFile", file.toString())); } } else if (afterFtpPut.equals("move_file")) { // Move File FileObject destination = null; FileObject source = null; try { destination = KettleVFS.getFileObject( destinationFolder + Const.FILE_SEPARATOR + file.getName().getBaseName(), this); file.moveTo(destination); retval = true; } catch (Exception e) { logError(BaseMessages.getString(PKG, "JobSSH2PUT.Cant_Move_File.Label", file.toString(), destinationFolder, e.getMessage())); } finally { if (destination != null) { try { destination.close(); } catch (Exception ex) { /* Ignore */ } } if (source != null) { try { source.close(); } catch (Exception ex) { /* Ignore */ } } } if (log.isDetailed()) { logDetailed( BaseMessages.getString(PKG, "JobSSH2PUT.Log.MovedFile", file.toString(), ftpDirectory)); } } return retval; } catch (Exception e) { throw new KettleException(e); } }
From source file:org.pentaho.di.job.entries.talendjobexec.JobEntryTalendJobExec.java
private Result executeTalenJob(FileObject file, Result result, int nr) throws Exception { ClassLoader classLoader = null; try {/*from w w w. j a v a 2 s . co m*/ classLoader = classLoaderCache.get(file.toString()); if (classLoader == null) { // Find the jar files in the ZIP file... // final URL[] jarFiles = prepareJarFiles(file); // Create a new class loader with the extracted jar files. // classLoader = new KettleURLClassLoader(jarFiles, getClass().getClassLoader()); Runtime.getRuntime().addShutdownHook(new Thread() { @Override public void run() { try { cleanupJarFiles(jarFiles); } catch (Exception e) { System.err.println("Error cleaning up temporary Talend jar file extracts: " + Const.getStackTracker(e)); } } }); classLoaderCache.put(file.toString(), classLoader); } Class<?> clazz = classLoader.loadClass(environmentSubstitute(getClassName())); Object jobObject = clazz.newInstance(); Method runJob = clazz.getMethod("runJobInTOS", String[].class); // TODO: consider passing something of some sort in this next method: // variables, arguments... // int returnCode = (Integer) runJob.invoke(jobObject, (Object) new String[] {}); result.setExitStatus(returnCode); result.setResult(true); result.setNrErrors(0); } catch (Exception e) { throw new KettleException( BaseMessages.getString(PKG, "JobEntryTalendJobExec.ERROR_0006_ExceptionExecutingTalenJob"), e); } return result; }
From source file:org.pentaho.di.job.entries.talendjobexec.JobEntryTalendJobExec.java
private URL[] prepareJarFiles(FileObject zipFile) throws Exception { // zip:file:///tmp/foo.zip FileInputList fileList = FileInputList.createFileList(this, new String[] { "zip:" + zipFile.toString(), }, new String[] { ".*\\.jar$", }, // Include mask: only jar files new String[] { ".*classpath\\.jar$", }, // Exclude mask: only jar files new String[] { "Y", }, // File required new boolean[] { true, }); // Search sub-directories List<URL> files = new ArrayList<URL>(); // Copy the jar files in the temp folder... ////from w ww . ja v a2s .c om for (FileObject file : fileList.getFiles()) { FileObject jarfilecopy = KettleVFS.createTempFile(file.getName().getBaseName(), ".jar", environmentSubstitute("${java.io.tmpdir}")); jarfilecopy.copyFrom(file, new AllFileSelector()); files.add(jarfilecopy.getURL()); } return files.toArray(new URL[files.size()]); }
From source file:org.pentaho.di.job.entries.unzip.JobEntryUnZip.java
private boolean unzipFile(FileObject sourceFileObject, String realTargetdirectory, String realWildcard, String realWildcardExclude, Result result, Job parentJob, FileObject fileObject, FileObject movetodir, String realMovetodirectory) { boolean retval = false; String unzipToFolder = realTargetdirectory; try {/*from w ww . j av a 2s . com*/ if (log.isDetailed()) { logDetailed( BaseMessages.getString(PKG, "JobUnZip.Log.ProcessingFile", sourceFileObject.toString())); } // Do you create a root folder? // if (rootzip) { String shortSourceFilename = sourceFileObject.getName().getBaseName(); int lenstring = shortSourceFilename.length(); int lastindexOfDot = shortSourceFilename.lastIndexOf('.'); if (lastindexOfDot == -1) { lastindexOfDot = lenstring; } String foldername = realTargetdirectory + "/" + shortSourceFilename.substring(0, lastindexOfDot); FileObject rootfolder = KettleVFS.getFileObject(foldername, this); if (!rootfolder.exists()) { try { rootfolder.createFolder(); if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobUnZip.Log.RootFolderCreated", foldername)); } } catch (Exception e) { throw new Exception( BaseMessages.getString(PKG, "JobUnZip.Error.CanNotCreateRootFolder", foldername), e); } } unzipToFolder = foldername; } // Try to read the entries from the VFS object... // String zipFilename = "zip:" + sourceFileObject.getName().getFriendlyURI(); FileObject zipFile = KettleVFS.getFileObject(zipFilename, this); FileObject[] items = zipFile.findFiles(new AllFileSelector() { public boolean traverseDescendents(FileSelectInfo info) { return true; } public boolean includeFile(FileSelectInfo info) { // Never return the parent directory of a file list. if (info.getDepth() == 0) { return false; } FileObject fileObject = info.getFile(); return fileObject != null; } }); Pattern pattern = null; if (!Const.isEmpty(realWildcard)) { pattern = Pattern.compile(realWildcard); } Pattern patternexclude = null; if (!Const.isEmpty(realWildcardExclude)) { patternexclude = Pattern.compile(realWildcardExclude); } for (FileObject item : items) { if (successConditionBroken) { if (!successConditionBrokenExit) { logError(BaseMessages.getString(PKG, "JobUnZip.Error.SuccessConditionbroken", "" + NrErrors)); successConditionBrokenExit = true; } return false; } synchronized (KettleVFS.getInstance().getFileSystemManager()) { FileObject newFileObject = null; try { if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobUnZip.Log.ProcessingZipEntry", item.getName().getURI(), sourceFileObject.toString())); } // get real destination filename // String newFileName = unzipToFolder + Const.FILE_SEPARATOR + getTargetFilename(item); newFileObject = KettleVFS.getFileObject(newFileName, this); if (item.getType().equals(FileType.FOLDER)) { // Directory // if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobUnZip.CreatingDirectory.Label", newFileName)); } // Create Directory if necessary ... // if (!newFileObject.exists()) { newFileObject.createFolder(); } } else { // File // boolean getIt = true; boolean getItexclude = false; // First see if the file matches the regular expression! // if (pattern != null) { Matcher matcher = pattern.matcher(item.getName().getURI()); getIt = matcher.matches(); } if (patternexclude != null) { Matcher matcherexclude = patternexclude.matcher(item.getName().getURI()); getItexclude = matcherexclude.matches(); } boolean take = takeThisFile(item, newFileName); if (getIt && !getItexclude && take) { if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobUnZip.ExtractingEntry.Label", item.getName().getURI(), newFileName)); } if (iffileexist == IF_FILE_EXISTS_UNIQ) { // Create file with unique name int lenstring = newFileName.length(); int lastindexOfDot = newFileName.lastIndexOf('.'); if (lastindexOfDot == -1) { lastindexOfDot = lenstring; } newFileName = newFileName.substring(0, lastindexOfDot) + StringUtil.getFormattedDateTimeNow(true) + newFileName.substring(lastindexOfDot, lenstring); if (log.isDebug()) { logDebug(BaseMessages.getString(PKG, "JobUnZip.Log.CreatingUniqFile", newFileName)); } } // See if the folder to the target file exists... // if (!newFileObject.getParent().exists()) { newFileObject.getParent().createFolder(); // creates the whole path. } InputStream is = null; OutputStream os = null; try { is = KettleVFS.getInputStream(item); os = KettleVFS.getOutputStream(newFileObject, false); if (is != null) { byte[] buff = new byte[2048]; int len; while ((len = is.read(buff)) > 0) { os.write(buff, 0, len); } // Add filename to result filenames addFilenameToResultFilenames(result, parentJob, newFileName); } } finally { if (is != null) { is.close(); } if (os != null) { os.close(); } } } // end if take } } catch (Exception e) { updateErrors(); logError(BaseMessages.getString(PKG, "JobUnZip.Error.CanNotProcessZipEntry", item.getName().getURI(), sourceFileObject.toString()), e); } finally { if (newFileObject != null) { try { newFileObject.close(); if (setOriginalModificationDate) { // Change last modification date newFileObject.getContent() .setLastModifiedTime(item.getContent().getLastModifiedTime()); } } catch (Exception e) { /* Ignore */ } // ignore this } // Close file object // close() does not release resources! KettleVFS.getInstance().getFileSystemManager().closeFileSystem(item.getFileSystem()); if (items != null) { items = null; } } } // Synchronized block on KettleVFS.getInstance().getFileSystemManager() } // End for // Here gc() is explicitly called if e.g. createfile is used in the same // job for the same file. The problem is that after creating the file the // file object is not properly garbaged collected and thus the file cannot // be deleted anymore. This is a known problem in the JVM. // System.gc(); // Unzip done... if (afterunzip == 1) { // delete zip file boolean deleted = fileObject.delete(); if (!deleted) { updateErrors(); logError(BaseMessages.getString(PKG, "JobUnZip.Cant_Delete_File.Label", sourceFileObject.toString())); } // File deleted if (log.isDebug()) { logDebug(BaseMessages.getString(PKG, "JobUnZip.File_Deleted.Label", sourceFileObject.toString())); } } else if (afterunzip == 2) { FileObject destFile = null; // Move File try { String destinationFilename = movetodir + Const.FILE_SEPARATOR + fileObject.getName().getBaseName(); destFile = KettleVFS.getFileObject(destinationFilename, this); fileObject.moveTo(destFile); // File moved if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobUnZip.Log.FileMovedTo", sourceFileObject.toString(), realMovetodirectory)); } } catch (Exception e) { updateErrors(); logError(BaseMessages.getString(PKG, "JobUnZip.Cant_Move_File.Label", sourceFileObject.toString(), realMovetodirectory, e.getMessage())); } finally { if (destFile != null) { try { destFile.close(); } catch (IOException ex) { /* Ignore */ } } } } retval = true; } catch (Exception e) { updateErrors(); log.logError(BaseMessages.getString(PKG, "JobUnZip.Error.Label"), BaseMessages.getString(PKG, "JobUnZip.ErrorUnzip.Label", sourceFileObject.toString(), e.getMessage()), e); } return retval; }
From source file:org.pentaho.di.job.entries.xmlwellformed.JobEntryXMLWellFormed.java
private boolean CheckFile(FileObject file) { boolean retval = false; try {/*from w ww.j a v a 2 s . c o m*/ retval = XMLCheck.isXMLFileWellFormed(file); } catch (Exception e) { logError(BaseMessages.getString(PKG, "JobXMLWellFormed.Log.ErrorCheckingFile", file.toString(), e.getMessage())); } return retval; }