Java tutorial
/* Copyright (c) 2007 Pentaho Corporation. All rights reserved. * This software was developed by Pentaho Corporation and is provided under the terms * of the GNU Lesser General Public License, Version 2.1. You may not use * this file except in compliance with the license. If you need a copy of the license, * please go to http://www.gnu.org/licenses/lgpl-2.1.txt. The Original Code is Pentaho * Data Integration. The Initial Developer is Pentaho Corporation. * * Software distributed under the GNU Lesser Public License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. Please refer to * the license for the specific language governing your rights and limitations.*/ package com.panet.imeta.job.entries.sftpput; import static com.panet.imeta.job.entry.validator.AndValidator.putValidators; import static com.panet.imeta.job.entry.validator.JobEntryValidatorUtils.andValidator; import static com.panet.imeta.job.entry.validator.JobEntryValidatorUtils.fileExistsValidator; import static com.panet.imeta.job.entry.validator.JobEntryValidatorUtils.integerValidator; import static com.panet.imeta.job.entry.validator.JobEntryValidatorUtils.notBlankValidator; import static com.panet.imeta.job.entry.validator.JobEntryValidatorUtils.notNullValidator; import java.net.InetAddress; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.commons.vfs.FileObject; import org.apache.commons.vfs.FileType; import org.w3c.dom.Node; import com.panet.imeta.cluster.SlaveServer; import com.panet.imeta.core.CheckResultInterface; import com.panet.imeta.core.Const; import com.panet.imeta.core.Result; import com.panet.imeta.core.ResultFile; import com.panet.imeta.core.RowMetaAndData; import com.panet.imeta.core.database.DatabaseMeta; import com.panet.imeta.core.encryption.Encr; import com.panet.imeta.core.exception.KettleDatabaseException; import com.panet.imeta.core.exception.KettleException; import com.panet.imeta.core.exception.KettleXMLException; import com.panet.imeta.core.logging.LogWriter; import com.panet.imeta.core.vfs.KettleVFS; import com.panet.imeta.core.xml.XMLHandler; import com.panet.imeta.job.Job; import com.panet.imeta.job.JobEntryType; import com.panet.imeta.job.JobMeta; import com.panet.imeta.job.entries.sftp.SFTPClient; import com.panet.imeta.job.entry.JobEntryBase; import com.panet.imeta.job.entry.JobEntryInterface; import com.panet.imeta.repository.Repository; import com.panet.imeta.resource.ResourceEntry; import com.panet.imeta.resource.ResourceReference; import com.panet.imeta.resource.ResourceEntry.ResourceType; import com.panet.imeta.shared.SharedObjectInterface; /** * This defines an SFTP put job entry. * * @author Matt * @since 05-11-2003 * */ public class JobEntrySFTPPUT extends JobEntryBase implements Cloneable, JobEntryInterface { private String serverName; private String serverPort; private String userName; private String password; private String sftpDirectory; private String localDirectory; private String wildcard; private boolean remove; private boolean copyprevious; private boolean addFilenameResut; public JobEntrySFTPPUT(String n) { super(n, ""); serverName = null; serverPort = "22"; copyprevious = false; addFilenameResut = false; setID(-1L); setJobEntryType(JobEntryType.SFTPPUT); } public JobEntrySFTPPUT() { this(""); } public JobEntrySFTPPUT(JobEntryBase jeb) { super(jeb); } public Object clone() { JobEntrySFTPPUT je = (JobEntrySFTPPUT) super.clone(); return je; } public String getXML() { StringBuffer retval = new StringBuffer(300); retval.append(super.getXML()); retval.append(" ").append(XMLHandler.addTagValue("servername", serverName)); retval.append(" ").append(XMLHandler.addTagValue("serverport", serverPort)); retval.append(" ").append(XMLHandler.addTagValue("username", userName)); retval.append(" ") .append(XMLHandler.addTagValue("password", Encr.encryptPasswordIfNotUsingVariables(password))); retval.append(" ").append(XMLHandler.addTagValue("sftpdirectory", sftpDirectory)); retval.append(" ").append(XMLHandler.addTagValue("localdirectory", localDirectory)); retval.append(" ").append(XMLHandler.addTagValue("wildcard", wildcard)); retval.append(" ").append(XMLHandler.addTagValue("remove", remove)); retval.append(" ").append(XMLHandler.addTagValue("copyprevious", copyprevious)); retval.append(" ").append(XMLHandler.addTagValue("addFilenameResut", addFilenameResut)); return retval.toString(); } public void loadXML(Node entrynode, List<DatabaseMeta> databases, List<SlaveServer> slaveServers, Repository rep) throws KettleXMLException { try { super.loadXML(entrynode, databases, slaveServers); serverName = XMLHandler.getTagValue(entrynode, "servername"); serverPort = XMLHandler.getTagValue(entrynode, "serverport"); userName = XMLHandler.getTagValue(entrynode, "username"); password = Encr.decryptPasswordOptionallyEncrypted(XMLHandler.getTagValue(entrynode, "password")); sftpDirectory = XMLHandler.getTagValue(entrynode, "sftpdirectory"); localDirectory = XMLHandler.getTagValue(entrynode, "localdirectory"); wildcard = XMLHandler.getTagValue(entrynode, "wildcard"); remove = "Y".equalsIgnoreCase(XMLHandler.getTagValue(entrynode, "remove")); copyprevious = "Y".equalsIgnoreCase(XMLHandler.getTagValue(entrynode, "copyprevious")); addFilenameResut = "Y".equalsIgnoreCase(XMLHandler.getTagValue(entrynode, "addFilenameResut")); } catch (KettleXMLException xe) { throw new KettleXMLException("Unable to load job entry of type 'SFTPPUT' from XML node", xe); } } public void setInfo(Map<String, String[]> p, String id, List<? extends SharedObjectInterface> databases) { serverName = JobEntryBase.parameterToString(p.get(id + ".serverName")); serverPort = JobEntryBase.parameterToString(p.get(id + ".serverPort")); userName = JobEntryBase.parameterToString(p.get(id + ".userName")); password = JobEntryBase.parameterToString(p.get(id + ".password")); sftpDirectory = JobEntryBase.parameterToString(p.get(id + ".sftpDirectory")); localDirectory = JobEntryBase.parameterToString(p.get(id + ".localDirectory")); remove = JobEntryBase.parameterToBoolean(p.get(id + ".remove")); copyprevious = JobEntryBase.parameterToBoolean(p.get(id + ".copyprevious")); addFilenameResut = JobEntryBase.parameterToBoolean(p.get(id + ".addFilenameResut")); wildcard = JobEntryBase.parameterToString(p.get(id + ".wildcard")); } public void loadRep(Repository rep, long id_jobentry, List<DatabaseMeta> databases, List<SlaveServer> slaveServers) throws KettleException { try { super.loadRep(rep, id_jobentry, databases, slaveServers); serverName = rep.getJobEntryAttributeString(id_jobentry, "servername"); int intServerPort = (int) rep.getJobEntryAttributeInteger(id_jobentry, "serverport"); serverPort = rep.getJobEntryAttributeString(id_jobentry, "serverport"); // backward compatible. if (intServerPort > 0 && Const.isEmpty(serverPort)) serverPort = Integer.toString(intServerPort); userName = rep.getJobEntryAttributeString(id_jobentry, "username"); password = Encr .decryptPasswordOptionallyEncrypted(rep.getJobEntryAttributeString(id_jobentry, "password")); sftpDirectory = rep.getJobEntryAttributeString(id_jobentry, "sftpdirectory"); localDirectory = rep.getJobEntryAttributeString(id_jobentry, "localdirectory"); wildcard = rep.getJobEntryAttributeString(id_jobentry, "wildcard"); remove = rep.getJobEntryAttributeBoolean(id_jobentry, "remove"); copyprevious = rep.getJobEntryAttributeBoolean(id_jobentry, "copyprevious"); addFilenameResut = rep.getJobEntryAttributeBoolean(id_jobentry, "addFilenameResut"); } catch (KettleException dbe) { throw new KettleException( "Unable to load job entry of type 'SFTPPUT' from the repository for id_jobentry=" + id_jobentry, dbe); } } public void saveRep(Repository rep, long id_job) throws KettleException { try { super.saveRep(rep, id_job); rep.saveJobEntryAttribute(id_job, getID(), "servername", serverName); rep.saveJobEntryAttribute(id_job, getID(), "serverport", serverPort); rep.saveJobEntryAttribute(id_job, getID(), "username", userName); rep.saveJobEntryAttribute(id_job, getID(), "password", Encr.encryptPasswordIfNotUsingVariables(password)); rep.saveJobEntryAttribute(id_job, getID(), "sftpdirectory", sftpDirectory); rep.saveJobEntryAttribute(id_job, getID(), "localdirectory", localDirectory); rep.saveJobEntryAttribute(id_job, getID(), "wildcard", wildcard); rep.saveJobEntryAttribute(id_job, getID(), "remove", remove); rep.saveJobEntryAttribute(id_job, getID(), "copyprevious", copyprevious); rep.saveJobEntryAttribute(id_job, getID(), "addFilenameResut", addFilenameResut); } catch (KettleDatabaseException dbe) { throw new KettleException( "Unable to load job entry of type 'SFTPPUT' to the repository for id_job=" + id_job, dbe); } } /** * @return Returns the directory. */ public String getScpDirectory() { return sftpDirectory; } /** * @param directory * The directory to set. */ public void setScpDirectory(String directory) { this.sftpDirectory = directory; } /** * @return Returns the password. */ public String getPassword() { return password; } /** * @param password * The password to set. */ public void setPassword(String password) { this.password = password; } /** * @return Returns the serverName. */ public String getServerName() { return serverName; } /** * @param serverName * The serverName to set. */ public void setServerName(String serverName) { this.serverName = serverName; } /** * @return Returns the userName. */ public String getUserName() { return userName; } /** * @param userName * The userName to set. */ public void setUserName(String userName) { this.userName = userName; } /** * @return Returns the wildcard. */ public String getWildcard() { return wildcard; } /** * @param wildcard * The wildcard to set. */ public void setWildcard(String wildcard) { this.wildcard = wildcard; } /** * @return Returns the localdirectory. */ public String getLocalDirectory() { return localDirectory; } /** * @param localDirectory * The localDirectory to set. */ public void setLocalDirectory(String localDirectory) { this.localDirectory = localDirectory; } /** * @param remove * The remove to set. */ public void setRemove(boolean remove) { this.remove = remove; } /** * @return Returns the remove. */ public boolean getRemove() { return remove; } public boolean isCopyPrevious() { return copyprevious; } public void setCopyPrevious(boolean copyprevious) { this.copyprevious = copyprevious; } public boolean isAddFilenameResut() { return addFilenameResut; } public void setAddFilenameResut(boolean addFilenameResut) { this.addFilenameResut = addFilenameResut; } public String getServerPort() { return serverPort; } public void setServerPort(String serverPort) { this.serverPort = serverPort; } public Result execute(Result previousResult, int nr, Repository rep, Job parentJob) throws KettleException { LogWriter log = LogWriter.getInstance(); Result result = previousResult; List<RowMetaAndData> rows = result.getRows(); result.setResult(false); if (log.isDetailed()) log.logDetailed(toString(), Messages.getString("JobSFTPPUT.Log.StartJobEntry")); ArrayList<FileObject> myFileList = new ArrayList<FileObject>(); if (copyprevious) { if (rows.size() == 0) { if (log.isDetailed()) log.logDetailed(toString(), Messages.getString("JobSFTPPUT.ArgsFromPreviousNothing")); result.setResult(true); return result; } try { RowMetaAndData resultRow = null; // Copy the input row to the (command line) arguments for (int iteration = 0; iteration < rows.size(); iteration++) { resultRow = rows.get(iteration); // Get file names String file_previous = resultRow.getString(0, null); if (!Const.isEmpty(file_previous)) { FileObject file = KettleVFS.getFileObject(file_previous); if (!file.exists()) log.logError(toString(), Messages.getString("JobSFTPPUT.Log.FilefromPreviousNotFound", file_previous)); else { myFileList.add(file); if (log.isDebug()) log.logDebug(toString(), Messages.getString("JobSFTPPUT.Log.FilenameFromResult", file_previous)); } } } } catch (Exception e) { log.logError(toString(), Messages.getString("JobSFTPPUT.Error.ArgFromPrevious")); result.setNrErrors(1); return result; } } SFTPClient sftpclient = null; // String substitution.. String realServerName = environmentSubstitute(serverName); String realServerPort = environmentSubstitute(serverPort); String realUsername = environmentSubstitute(userName); String realPassword = environmentSubstitute(password); String realSftpDirString = environmentSubstitute(sftpDirectory); String realWildcard = environmentSubstitute(wildcard); String realLocalDirectory = environmentSubstitute(localDirectory); try { // Create sftp client to host ... sftpclient = new SFTPClient(InetAddress.getByName(realServerName), Const.toInt(realServerPort, 22), realUsername); if (log.isDetailed()) log.logDetailed(toString(), Messages.getString("JobSFTPPUT.Log.OpenedConnection", realServerName, "" + realServerPort, realUsername)); // login to ftp host ... sftpclient.login(realPassword); // Don't show the password in the logs, it's not good for security // audits // log.logDetailed(toString(), "logged in using password // "+realPassword); // Logging this seems a bad idea! Oh well. // move to spool dir ... if (!Const.isEmpty(realSftpDirString)) { sftpclient.chdir(realSftpDirString); if (log.isDetailed()) log.logDetailed(toString(), Messages.getString("JobSFTPPUT.Log.ChangedDirectory", realSftpDirString)); } // end if if (!copyprevious) { // Get all the files in the local directory... myFileList = new ArrayList<FileObject>(); FileObject localFiles = KettleVFS.getFileObject(realLocalDirectory); FileObject[] children = localFiles.getChildren(); if (children != null) { for (int i = 0; i < children.length; i++) { // Get filename of file or directory if (children[i].getType().equals(FileType.FILE)) { // myFileList.add(children[i].getAbsolutePath()); myFileList.add(children[i]); } } // end for } } if (myFileList == null || myFileList.size() == 0) { log.logError(toString(), Messages.getString("JobSFTPPUT.Error.NoFileToSend")); result.setNrErrors(1); return result; } if (log.isDetailed()) log.logDetailed(toString(), Messages.getString("JobSFTPPUT.Log.RowsFromPreviousResult", "" + myFileList.size())); Pattern pattern = null; if (!copyprevious) { if (!Const.isEmpty(realWildcard)) { pattern = Pattern.compile(realWildcard); } } // Get the files in the list and execute sftp.put() for each file for (int i = 0; i < myFileList.size() && !parentJob.isStopped(); i++) { FileObject myFile = myFileList.get(i); String localFilename = myFile.toString(); String destinationFilename = myFile.getName().getBaseName(); boolean getIt = true; // First see if the file matches the regular expression! if (pattern != null) { Matcher matcher = pattern.matcher(destinationFilename); getIt = matcher.matches(); } if (getIt) { if (log.isDebug()) log.logDebug(toString(), Messages.getString("JobSFTPPUT.Log.PuttingFile", localFilename, realSftpDirString)); sftpclient.put(myFile, destinationFilename); if (log.isDetailed()) log.logDetailed(toString(), Messages.getString("JobSFTPPUT.Log.TransferedFile", localFilename)); // Delete the file if this is needed! if (remove) { myFile.delete(); if (log.isDetailed()) log.logDetailed(toString(), Messages.getString("JobSFTPPUT.Log.DeletedFile", localFilename)); } else { if (addFilenameResut) { // Add to the result files... ResultFile resultFile = new ResultFile(ResultFile.FILE_TYPE_GENERAL, myFile, parentJob.getJobname(), toString()); result.getResultFiles().put(resultFile.getFile().toString(), resultFile); if (log.isDetailed()) log.logDetailed(toString(), Messages .getString("JobSFTPPUT.Log.FilenameAddedToResultFilenames", localFilename)); } } } } // end for result.setResult(true); // JKU: no idea if this is needed...! // result.setNrFilesRetrieved(filesRetrieved); } // end try catch (Exception e) { result.setNrErrors(1); log.logError(toString(), Messages.getString("JobSFTPPUT.Exception", e.getMessage())); log.logError(toString(), Const.getStackTracker(e)); } finally { // close connection, if possible try { if (sftpclient != null) sftpclient.disconnect(); } catch (Exception e) { // just ignore this, makes no big difference } // end catch } // end finallly return result; } // JKU: end function execute() public boolean evaluates() { return true; } public List<ResourceReference> getResourceDependencies(JobMeta jobMeta) { List<ResourceReference> references = super.getResourceDependencies(jobMeta); if (!Const.isEmpty(serverName)) { String realServerName = jobMeta.environmentSubstitute(serverName); ResourceReference reference = new ResourceReference(this); reference.getEntries().add(new ResourceEntry(realServerName, ResourceType.SERVER)); references.add(reference); } return references; } @Override public void check(List<CheckResultInterface> remarks, JobMeta jobMeta) { andValidator().validate(this, "serverName", remarks, putValidators(notBlankValidator())); //$NON-NLS-1$ andValidator().validate(this, "localDirectory", remarks, //$NON-NLS-1$ putValidators(notBlankValidator(), fileExistsValidator())); andValidator().validate(this, "userName", remarks, putValidators(notBlankValidator())); //$NON-NLS-1$ andValidator().validate(this, "password", remarks, putValidators(notNullValidator())); //$NON-NLS-1$ andValidator().validate(this, "serverPort", remarks, putValidators(integerValidator())); //$NON-NLS-1$ } }