List of usage examples for org.apache.commons.vfs FileObject toString
public String toString()
From source file:com.nary.util.LogFileObject.java
public static void println(FileObject outFile, String _Line) { String key = outFile.toString(); if (!logfiles.containsKey(key)) { try {//from w w w . jav a2 s. c o m logfiles.put(key, new LogFileObject(outFile)); } catch (Exception E) { return; } } ((LogFileObject) (logfiles.get(key))).println(_Line); }
From source file:com.panet.imeta.trans.steps.processfiles.ProcessFiles.java
public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException { meta = (ProcessFilesMeta) smi;/*from w ww.j av a 2s.com*/ data = (ProcessFilesData) sdi; Object[] r = getRow(); // Get row from input rowset & set row busy! if (r == null) // no more input to be expected... { setOutputDone(); return false; } if (first) { first = false; // Check is source filename field is provided if (Const.isEmpty(meta.getDynamicSourceFileNameField())) { throw new KettleException(Messages.getString("ProcessFiles.Error.SourceFilenameFieldMissing")); } // Check is target filename field is provided if (meta.getOperationType() != ProcessFilesMeta.OPERATION_TYPE_DELETE && Const.isEmpty(meta.getDynamicTargetFileNameField())) { throw new KettleException(Messages.getString("ProcessFiles.Error.TargetFilenameFieldMissing")); } // cache the position of the source filename field if (data.indexOfSourceFilename < 0) { data.indexOfSourceFilename = getInputRowMeta().indexOfValue(meta.getDynamicSourceFileNameField()); if (data.indexOfSourceFilename < 0) { // The field is unreachable ! throw new KettleException(Messages.getString("ProcessFiles.Exception.CouldnotFindField", //$NON-NLS-1$ meta.getDynamicSourceFileNameField())); //$NON-NLS-2$ } } // cache the position of the source filename field if (meta.getOperationType() != ProcessFilesMeta.OPERATION_TYPE_DELETE && data.indexOfTargetFilename < 0) { data.indexOfTargetFilename = getInputRowMeta().indexOfValue(meta.getDynamicTargetFileNameField()); if (data.indexOfTargetFilename < 0) { // The field is unreachable ! throw new KettleException(Messages.getString("ProcessFiles.Exception.CouldnotFindField", //$NON-NLS-1$ meta.getDynamicTargetFileNameField())); //$NON-NLS-2$ } } if (meta.simulate) { if (log.isBasic()) log.logBasic(toString(), Messages.getString("ProcessFiles.Log.SimulationModeON")); } } // End If first try { // get source filename String sourceFilename = getInputRowMeta().getString(r, data.indexOfSourceFilename); if (Const.isEmpty(sourceFilename)) { log.logError(toString(), Messages.getString("ProcessFiles.Error.SourceFileEmpty")); throw new KettleException(Messages.getString("ProcessFiles.Error.SourceFileEmpty")); } data.sourceFile = KettleVFS.getFileObject(sourceFilename); boolean targetFileExists = false; if (!data.sourceFile.exists()) { log.logError(toString(), Messages.getString("ProcessFiles.Error.SourceFileNotExist", sourceFilename)); throw new KettleException( Messages.getString("ProcessFiles.Error.SourceFileNotExist", sourceFilename)); } if (data.sourceFile.getType() != FileType.FILE) { log.logError(toString(), Messages.getString("ProcessFiles.Error.SourceFileNotFile", sourceFilename)); throw new KettleException( Messages.getString("ProcessFiles.Error.SourceFileNotFile", sourceFilename)); } String targetFilename = null; if (meta.getOperationType() != ProcessFilesMeta.OPERATION_TYPE_DELETE) { // get value for target filename targetFilename = getInputRowMeta().getString(r, data.indexOfTargetFilename); if (Const.isEmpty(targetFilename)) { log.logError(toString(), Messages.getString("ProcessFiles.Error.TargetFileEmpty")); throw new KettleException(Messages.getString("ProcessFiles.Error.TargetFileEmpty")); } data.targetFile = KettleVFS.getFileObject(targetFilename); if (data.targetFile.exists()) { if (log.isDetailed()) log.logDetailed(toString(), Messages.getString("ProcessFiles.Log.TargetFileExists", targetFilename)); } else { // let's check parent folder FileObject parentFolder = data.targetFile.getParent(); if (!parentFolder.exists()) { if (!meta.isCreateParentFolder()) throw new KettleException(Messages.getString( "ProcessFiles.Error.TargetParentFolderNotExists", parentFolder.toString())); else parentFolder.createFolder(); } if (parentFolder != null) parentFolder.close(); } } switch (meta.getOperationType()) { case ProcessFilesMeta.OPERATION_TYPE_COPY: if (((meta.isOverwriteTargetFile() && targetFileExists) || !targetFileExists) && !meta.simulate) data.targetFile.copyFrom(data.sourceFile, new TextOneToOneFileSelector()); if (log.isDetailed()) log.logDetailed(toString(), Messages.getString("ProcessFiles.Log.SourceFileCopied", sourceFilename, targetFilename)); break; case ProcessFilesMeta.OPERATION_TYPE_MOVE: if (((meta.isOverwriteTargetFile() && targetFileExists) || !targetFileExists) && !meta.simulate) data.sourceFile.moveTo(KettleVFS.getFileObject(targetFilename)); if (log.isDetailed()) log.logDetailed(toString(), Messages.getString("ProcessFiles.Log.SourceFileMoved", sourceFilename, targetFilename)); break; case ProcessFilesMeta.OPERATION_TYPE_DELETE: if (!meta.simulate) { if (!data.sourceFile.delete()) throw new KettleException(Messages.getString("ProcessFiles.Error.CanNotDeleteFile", data.sourceFile.toString())); } if (log.isDetailed()) log.logDetailed(toString(), Messages.getString("ProcessFiles.Log.SourceFileDeleted", sourceFilename)); break; default: break; } // add filename to result filenames? if (meta.isaddTargetFileNametoResult() && meta.getOperationType() != ProcessFilesMeta.OPERATION_TYPE_DELETE && data.sourceFile.getType() == FileType.FILE) { // Add this to the result file names... ResultFile resultFile = new ResultFile(ResultFile.FILE_TYPE_GENERAL, data.targetFile, getTransMeta().getName(), getStepname()); resultFile.setComment(Messages.getString("ProcessFiles.Log.FileAddedResult")); addResultFile(resultFile); if (log.isDetailed()) log.logDetailed(toString(), Messages.getString("ProcessFiles.Log.FilenameAddResult", data.sourceFile.toString())); } putRow(getInputRowMeta(), r); // copy row to possible alternate rowset(s). if (checkFeedback(getLinesRead())) { if (log.isBasic()) logBasic(Messages.getString("ProcessFiles.LineNumber") + getLinesRead()); //$NON-NLS-1$ } } catch (Exception e) { boolean sendToErrorRow = false; String errorMessage = null; if (getStepMeta().isDoingErrorHandling()) { sendToErrorRow = true; errorMessage = e.toString(); } else { logError(Messages.getString("ProcessFiles.ErrorInStepRunning") + e.getMessage()); //$NON-NLS-1$ setErrors(1); stopAll(); setOutputDone(); // signal end to receiver(s) return false; } if (sendToErrorRow) { // Simply add this row to the error row putError(getInputRowMeta(), r, 1, errorMessage, null, "ProcessFiles001"); } } return true; }
From source file:com.panet.imeta.job.entries.sftpput.JobEntrySFTPPUT.java
public Result execute(Result previousResult, int nr, Repository rep, Job parentJob) throws KettleException { LogWriter log = LogWriter.getInstance(); Result result = previousResult; List<RowMetaAndData> rows = result.getRows(); result.setResult(false);//from w ww . j a v a 2 s. c om if (log.isDetailed()) log.logDetailed(toString(), Messages.getString("JobSFTPPUT.Log.StartJobEntry")); ArrayList<FileObject> myFileList = new ArrayList<FileObject>(); if (copyprevious) { if (rows.size() == 0) { if (log.isDetailed()) log.logDetailed(toString(), Messages.getString("JobSFTPPUT.ArgsFromPreviousNothing")); result.setResult(true); return result; } try { RowMetaAndData resultRow = null; // Copy the input row to the (command line) arguments for (int iteration = 0; iteration < rows.size(); iteration++) { resultRow = rows.get(iteration); // Get file names String file_previous = resultRow.getString(0, null); if (!Const.isEmpty(file_previous)) { FileObject file = KettleVFS.getFileObject(file_previous); if (!file.exists()) log.logError(toString(), Messages.getString("JobSFTPPUT.Log.FilefromPreviousNotFound", file_previous)); else { myFileList.add(file); if (log.isDebug()) log.logDebug(toString(), Messages.getString("JobSFTPPUT.Log.FilenameFromResult", file_previous)); } } } } catch (Exception e) { log.logError(toString(), Messages.getString("JobSFTPPUT.Error.ArgFromPrevious")); result.setNrErrors(1); return result; } } SFTPClient sftpclient = null; // String substitution.. String realServerName = environmentSubstitute(serverName); String realServerPort = environmentSubstitute(serverPort); String realUsername = environmentSubstitute(userName); String realPassword = environmentSubstitute(password); String realSftpDirString = environmentSubstitute(sftpDirectory); String realWildcard = environmentSubstitute(wildcard); String realLocalDirectory = environmentSubstitute(localDirectory); try { // Create sftp client to host ... sftpclient = new SFTPClient(InetAddress.getByName(realServerName), Const.toInt(realServerPort, 22), realUsername); if (log.isDetailed()) log.logDetailed(toString(), Messages.getString("JobSFTPPUT.Log.OpenedConnection", realServerName, "" + realServerPort, realUsername)); // login to ftp host ... sftpclient.login(realPassword); // Don't show the password in the logs, it's not good for security // audits // log.logDetailed(toString(), "logged in using password // "+realPassword); // Logging this seems a bad idea! Oh well. // move to spool dir ... if (!Const.isEmpty(realSftpDirString)) { sftpclient.chdir(realSftpDirString); if (log.isDetailed()) log.logDetailed(toString(), Messages.getString("JobSFTPPUT.Log.ChangedDirectory", realSftpDirString)); } // end if if (!copyprevious) { // Get all the files in the local directory... myFileList = new ArrayList<FileObject>(); FileObject localFiles = KettleVFS.getFileObject(realLocalDirectory); FileObject[] children = localFiles.getChildren(); if (children != null) { for (int i = 0; i < children.length; i++) { // Get filename of file or directory if (children[i].getType().equals(FileType.FILE)) { // myFileList.add(children[i].getAbsolutePath()); myFileList.add(children[i]); } } // end for } } if (myFileList == null || myFileList.size() == 0) { log.logError(toString(), Messages.getString("JobSFTPPUT.Error.NoFileToSend")); result.setNrErrors(1); return result; } if (log.isDetailed()) log.logDetailed(toString(), Messages.getString("JobSFTPPUT.Log.RowsFromPreviousResult", "" + myFileList.size())); Pattern pattern = null; if (!copyprevious) { if (!Const.isEmpty(realWildcard)) { pattern = Pattern.compile(realWildcard); } } // Get the files in the list and execute sftp.put() for each file for (int i = 0; i < myFileList.size() && !parentJob.isStopped(); i++) { FileObject myFile = myFileList.get(i); String localFilename = myFile.toString(); String destinationFilename = myFile.getName().getBaseName(); boolean getIt = true; // First see if the file matches the regular expression! if (pattern != null) { Matcher matcher = pattern.matcher(destinationFilename); getIt = matcher.matches(); } if (getIt) { if (log.isDebug()) log.logDebug(toString(), Messages.getString("JobSFTPPUT.Log.PuttingFile", localFilename, realSftpDirString)); sftpclient.put(myFile, destinationFilename); if (log.isDetailed()) log.logDetailed(toString(), Messages.getString("JobSFTPPUT.Log.TransferedFile", localFilename)); // Delete the file if this is needed! if (remove) { myFile.delete(); if (log.isDetailed()) log.logDetailed(toString(), Messages.getString("JobSFTPPUT.Log.DeletedFile", localFilename)); } else { if (addFilenameResut) { // Add to the result files... ResultFile resultFile = new ResultFile(ResultFile.FILE_TYPE_GENERAL, myFile, parentJob.getJobname(), toString()); result.getResultFiles().put(resultFile.getFile().toString(), resultFile); if (log.isDetailed()) log.logDetailed(toString(), Messages .getString("JobSFTPPUT.Log.FilenameAddedToResultFilenames", localFilename)); } } } } // end for result.setResult(true); // JKU: no idea if this is needed...! // result.setNrFilesRetrieved(filesRetrieved); } // end try catch (Exception e) { result.setNrErrors(1); log.logError(toString(), Messages.getString("JobSFTPPUT.Exception", e.getMessage())); log.logError(toString(), Const.getStackTracker(e)); } finally { // close connection, if possible try { if (sftpclient != null) sftpclient.disconnect(); } catch (Exception e) { // just ignore this, makes no big difference } // end catch } // end finallly return result; }
From source file:com.panet.imeta.job.entries.folderisempty.JobEntryFolderIsEmpty.java
public Result execute(Result previousResult, int nr, Repository rep, Job parentJob) { LogWriter log = LogWriter.getInstance(); Result result = previousResult; result.setResult(false);// w w w .j a v a2 s. c o m result.setNrErrors(1); filescount = 0; folderscount = 0; pattern = null; if (!Const.isEmpty(getWildcard())) pattern = Pattern.compile(getRealWildcard()); if (foldername != null) { String realFoldername = getRealFoldername(); FileObject FolderObject = null; try { FolderObject = KettleVFS.getFileObject(realFoldername); if (FolderObject.exists()) { //Check if it's a folder if (FolderObject.getType() == FileType.FOLDER) { // File provided is a folder, so we can process ... FolderObject.findFiles(new TextFileSelector(FolderObject.toString())); if (log.isBasic()) log.logBasic("Total files", "We found : " + filescount + " file(s)"); if (filescount == 0) { result.setResult(true); result.setNrErrors(0); } } else { // Not a folder, fail log.logError("Found files", "[" + realFoldername + "] is not a folder, failing."); } } else { // No Folder found if (log.isBasic()) log.logBasic(toString(), "we can not find [" + realFoldername + "] !"); } } catch (IOException e) { log.logError(toString(), "Could not create Folder [" + realFoldername + "], exception: " + e.getMessage()); result.setResult(false); result.setNrErrors(1); } finally { if (FolderObject != null) { try { FolderObject.close(); } catch (IOException ex) { } ; } } } else { log.logError(toString(), "No Foldername is defined."); } return result; }
From source file:com.panet.imeta.job.entries.xmlwellformed.JobEntryXMLWellFormed.java
private boolean checkOneFile(FileObject file, LogWriter log, Result result, Job parentJob) { boolean retval = false; try {/*from w ww.j a v a 2s . c o m*/ // We deal with a file..so let's check if it's well formed boolean retformed = CheckWellFormed(file, log); if (!retformed) { log.logError(toString(), Messages.getString("JobXMLWellFormed.Error.FileBadFormed", file.toString())); // Update Bad formed files number updateBadFormed(); if (resultfilenames.equals(ADD_ALL_FILENAMES) || resultfilenames.equals(ADD_BAD_FORMED_FILES_ONLY)) addFileToResultFilenames(KettleVFS.getFilename(file), log, result, parentJob); } else { if (log.isDetailed()) { log.logDetailed(toString(), "---------------------------"); log.logDetailed(toString(), Messages.getString("JobXMLWellFormed.Error.FileWellFormed", file.toString())); } // Update Well formed files number updateWellFormed(); if (resultfilenames.equals(ADD_ALL_FILENAMES) || resultfilenames.equals(ADD_WELL_FORMED_FILES_ONLY)) addFileToResultFilenames(KettleVFS.getFilename(file), log, result, parentJob); } } catch (Exception e) { } return retval; }
From source file:com.panet.imeta.job.entries.deleteresultfilenames.JobEntryDeleteResultFilenames.java
public Result execute(Result previousResult, int nr, Repository rep, Job parentJob) { LogWriter log = LogWriter.getInstance(); Result result = previousResult; result.setResult(false);//from w w w . j a v a 2s . co m if (previousResult != null) { try { int size = previousResult.getResultFiles().size(); if (log.isBasic()) log.logBasic(toString(), Messages.getString("JobEntryDeleteResultFilenames.log.FilesFound", "" + size)); if (!specifywildcard) { // Delete all files previousResult.getResultFiles().clear(); if (log.isDetailed()) log.logDetailed(toString(), Messages.getString("JobEntryDeleteResultFilenames.log.DeletedFiles", "" + size)); } else { List<ResultFile> resultFiles = result.getResultFilesList(); if (resultFiles != null && resultFiles.size() > 0) { for (Iterator<ResultFile> it = resultFiles.iterator(); it.hasNext() && !parentJob.isStopped();) { ResultFile resultFile = (ResultFile) it.next(); FileObject file = resultFile.getFile(); if (file != null && file.exists()) { if (CheckFileWildcard(file.getName().getBaseName(), environmentSubstitute(wildcard), true) && !CheckFileWildcard(file.getName().getBaseName(), environmentSubstitute(wildcardexclude), false)) { // Remove file from result files list result.getResultFiles().remove(resultFile.getFile().toString()); if (log.isDetailed()) log.logDetailed(toString(), Messages.getString( "JobEntryDeleteResultFilenames.log.DeletedFile", file.toString())); } } } } } result.setResult(true); } catch (Exception e) { log.logError(toString(), Messages.getString("JobEntryDeleteResultFilenames.Error", e.toString())); } } return result; }
From source file:com.panet.imeta.job.entries.xmlwellformed.JobEntryXMLWellFormed.java
private boolean CheckWellFormed(FileObject file, LogWriter log) { boolean retval = false; try {/*from w w w .j a va 2 s. c o m*/ SAXParserFactory factory = SAXParserFactory.newInstance(); XMLTreeHandler handler = new XMLTreeHandler(); // Parse the input. SAXParser saxParser = factory.newSAXParser(); saxParser.parse(new File(KettleVFS.getFilename(file)), handler); retval = true; } catch (Exception e) { log.logError(toString(), Messages.getString("JobXMLWellFormed.Log.ErrorCheckingFile", file.toString(), e.getMessage())); } return retval; }
From source file:be.ibridge.kettle.trans.step.sortrows.SortRows.java
private Row getBuffer() { int i, f;/*from w w w . ja v a2s . co m*/ int smallest; Row r1, r2; Row retval; // Open all files at once and read one row from each file... if (data.files.size() > 0 && (data.dis.size() == 0 || data.fis.size() == 0)) { logBasic("Opening " + data.files.size() + " tmp-files..."); try { for (f = 0; f < data.files.size() && !isStopped(); f++) { FileObject fileObject = (FileObject) data.files.get(f); String filename = KettleVFS.getFilename(fileObject); if (log.isDetailed()) logDetailed("Opening tmp-file: [" + filename + "]"); InputStream fi = fileObject.getContent().getInputStream(); DataInputStream di; data.fis.add(fi); if (meta.getCompress()) { GZIPInputStream gzfi = new GZIPInputStream(new BufferedInputStream(fi)); di = new DataInputStream(gzfi); data.gzis.add(gzfi); } else { di = new DataInputStream(fi); } data.dis.add(di); // How long is the buffer? int buffersize = di.readInt(); if (log.isDetailed()) logDetailed("[" + filename + "] expecting " + buffersize + " rows..."); if (buffersize > 0) { // Read a row from each temp-file Row metadata = (Row) data.rowMeta.get(f); data.rowbuffer.add(new Row(di, metadata.size(), metadata)); // new row } } } catch (Exception e) { logError("Error reading back tmp-files : " + e.toString()); e.printStackTrace(); } } if (data.files.size() == 0) { if (data.buffer.size() > 0) { retval = (Row) data.buffer.get(0); data.buffer.remove(0); } else { retval = null; } } else { if (data.rowbuffer.size() == 0) { retval = null; } else { // We now have "filenr" rows waiting: which one is the smallest? // for (i = 0; i < data.rowbuffer.size() && !isStopped(); i++) { Row b = (Row) data.rowbuffer.get(i); if (log.isRowLevel()) logRowlevel("--BR#" + i + ": " + b.toString()); } // smallest = 0; r1 = (Row) data.rowbuffer.get(smallest); for (f = 1; f < data.rowbuffer.size() && !isStopped(); f++) { r2 = (Row) data.rowbuffer.get(f); if (r2 != null && r2.compare(r1, data.fieldnrs, meta.getAscending()) < 0) { smallest = f; r1 = (Row) data.rowbuffer.get(smallest); } } retval = r1; data.rowbuffer.remove(smallest); if (log.isRowLevel()) logRowlevel("Smallest row selected on [" + smallest + "] : " + retval); // now get another Row for position smallest FileObject file = (FileObject) data.files.get(smallest); DataInputStream di = (DataInputStream) data.dis.get(smallest); InputStream fi = (InputStream) data.fis.get(smallest); GZIPInputStream gzfi = (meta.getCompress()) ? (GZIPInputStream) data.gzis.get(smallest) : null; try { Row metadata = (Row) data.rowMeta.get(smallest); data.rowbuffer.add(smallest, new Row(di, metadata.size(), metadata)); } catch (KettleFileException fe) // empty file or EOF mostly { try { di.close(); fi.close(); if (gzfi != null) gzfi.close(); file.delete(); } catch (IOException e) { logError("Unable to close/delete file #" + smallest + " --> " + file.toString()); setErrors(1); stopAll(); return null; } data.files.remove(smallest); data.dis.remove(smallest); data.fis.remove(smallest); if (gzfi != null) data.gzis.remove(smallest); data.rowMeta.remove(smallest); } } } return retval; }
From source file:com.panet.imeta.job.entries.ssh2put.JobEntrySSH2PUT.java
public Result execute(Result previousResult, int nr, Repository rep, Job parentJob) { LogWriter log = LogWriter.getInstance(); Result result = previousResult; result.setResult(false);//from w w w . jav a2 s. c om try { // Get real variable value String realServerName = environmentSubstitute(serverName); int realServerPort = Const.toInt(environmentSubstitute(serverPort), 22); String realUserName = environmentSubstitute(userName); String realServerPassword = environmentSubstitute(password); // Proxy Host String realProxyHost = environmentSubstitute(httpproxyhost); int realProxyPort = Const.toInt(environmentSubstitute(httpproxyport), 22); String realproxyUserName = environmentSubstitute(httpproxyusername); String realProxyPassword = environmentSubstitute(httpProxyPassword); // Key file String realKeyFilename = environmentSubstitute(keyFilename); String relKeyFilepass = environmentSubstitute(keyFilePass); // Source files String realLocalDirectory = environmentSubstitute(localDirectory); String realwildcard = environmentSubstitute(wildcard); // Remote destination String realftpDirectory = environmentSubstitute(ftpDirectory); // Destination folder (Move to) String realDestinationFolder = environmentSubstitute(destinationfolder); try { // Remote source realftpDirectory = FTPUtils.normalizePath(realftpDirectory); // Destination folder (Move to) realDestinationFolder = FTPUtils.normalizePath(realDestinationFolder); } catch (Exception e) { log.logError(toString(), Messages.getString("JobSSH2PUT.Log.CanNotNormalizePath", e.getMessage())); result.setNrErrors(1); return result; } // Check for mandatory fields boolean mandatoryok = true; if (Const.isEmpty(realServerName)) { mandatoryok = false; log.logError(toString(), Messages.getString("JobSSH2PUT.Log.ServernameMissing")); } if (usehttpproxy) { if (Const.isEmpty(realProxyHost)) { mandatoryok = false; log.logError(toString(), Messages.getString("JobSSH2PUT.Log.HttpProxyhostMissing")); } } if (publicpublickey) { if (Const.isEmpty(realKeyFilename)) { mandatoryok = false; log.logError(toString(), Messages.getString("JobSSH2PUT.Log.KeyFileMissing")); } else { // Let's check if folder exists... if (!KettleVFS.fileExists(realKeyFilename)) { mandatoryok = false; log.logError(toString(), Messages.getString("JobSSH2PUT.Log.KeyFileNotExist")); } } } if (Const.isEmpty(realLocalDirectory)) { mandatoryok = false; log.logError(toString(), Messages.getString("JobSSH2PUT.Log.LocalFolderMissing")); } if (afterFtpPut.equals("move_file")) { if (Const.isEmpty(realDestinationFolder)) { mandatoryok = false; log.logError(toString(), Messages.getString("JobSSH2PUT.Log.DestinatFolderMissing")); } else { // Let's check if folder exists... if (!KettleVFS.fileExists(realDestinationFolder)) { mandatoryok = false; log.logError(toString(), Messages.getString("JobSSH2PUT.Log.DestinatFolderNotExist", realDestinationFolder)); } } } if (mandatoryok) { Connection conn = null; SFTPv3Client client = null; boolean good = true; int nbfilestoput = 0; int nbput = 0; int nbrerror = 0; try { // Create a connection instance conn = getConnection(realServerName, realServerPort, realProxyHost, realProxyPort, realproxyUserName, realProxyPassword); if (timeout > 0) { // Use timeout // Cache Host Key if (cachehostkey) conn.connect(new SimpleVerifier(database), 0, timeout * 1000); else conn.connect(null, 0, timeout * 1000); } else { // Cache Host Key if (cachehostkey) conn.connect(new SimpleVerifier(database)); else conn.connect(); } // Authenticate boolean isAuthenticated = false; if (publicpublickey) { String keyContent = KettleVFS.getTextFileContent(realKeyFilename, Const.XML_ENCODING); isAuthenticated = conn.authenticateWithPublicKey(realUserName, keyContent.toCharArray(), relKeyFilepass); } else { isAuthenticated = conn.authenticateWithPassword(realUserName, realServerPassword); } // LET'S CHECK AUTHENTICATION ... if (isAuthenticated == false) log.logError(toString(), Messages.getString("JobSSH2PUT.Log.AuthenticationFailed")); else { if (log.isBasic()) log.logBasic(toString(), Messages.getString("JobSSH2PUT.Log.Connected", serverName, userName)); client = new SFTPv3Client(conn); if (log.isDetailed()) log.logDetailed(toString(), Messages.getString("JobSSH2PUT.Log.ProtocolVersion", "" + client.getProtocolVersion())); // Check if remote directory exists if (!Const.isEmpty(realftpDirectory)) { if (!sshDirectoryExists(client, realftpDirectory)) { good = false; if (createRemoteFolder) { good = CreateRemoteFolder(client, realftpDirectory); if (good) log.logBasic(toString(), Messages.getString("JobSSH2PUT.Log.RemoteDirectoryCreated")); } else log.logError(toString(), Messages .getString("JobSSH2PUT.Log.RemoteDirectoryNotExist", realftpDirectory)); } else if (log.isDetailed()) log.logDetailed(toString(), Messages .getString("JobSSH2PUT.Log.RemoteDirectoryExist", realftpDirectory)); } if (good) { // Get files list from local folder (source) List<FileObject> myFileList = getFiles(realLocalDirectory); // Prepare Pattern for wildcard Pattern pattern = null; if (!Const.isEmpty(realwildcard)) pattern = Pattern.compile(realwildcard); // Let's put files now ... // Get the files in the list for (int i = 0; i < myFileList.size() && !parentJob.isStopped(); i++) { FileObject myFile = myFileList.get(i); String localFilename = myFile.toString(); String remoteFilename = myFile.getName().getBaseName(); // do we have a target directory? if (!Const.isEmpty(realftpDirectory)) remoteFilename = realftpDirectory + FTPUtils.FILE_SEPARATOR + remoteFilename; boolean getIt = true; // First see if the file matches the regular expression! if (pattern != null) { Matcher matcher = pattern.matcher(remoteFilename); getIt = matcher.matches(); } if (getIt) { nbfilestoput++; boolean putok = true; if ((onlyGettingNewFiles == false) || (onlyGettingNewFiles == true) && !sshFileExists(client, remoteFilename)) { putok = putFile(myFile, remoteFilename, client); if (!putok) { nbrerror++; log.logError(toString(), Messages.getString( "JobSSH2PUT.Log.Error.CanNotPutFile", localFilename)); } else { nbput++; } } if (putok && !afterFtpPut.equals("do_nothing")) { deleteOrMoveFiles(myFile, realDestinationFolder); } } } /********************************RESULT ********************/ if (log.isDetailed()) { log.logDetailed(toString(), Messages.getString("JobSSH2PUT.Log.Result.JobEntryEnd1")); log.logDetailed(toString(), Messages.getString("JobSSH2PUT.Log.Result.TotalFiles", "" + nbfilestoput)); log.logDetailed(toString(), Messages.getString("JobSSH2PUT.Log.Result.TotalFilesPut", "" + nbput)); log.logDetailed(toString(), Messages.getString("JobSSH2PUT.Log.Result.TotalFilesError", "" + nbrerror)); log.logDetailed(toString(), Messages.getString("JobSSH2PUT.Log.Result.JobEntryEnd2")); } if (nbrerror == 0) result.setResult(true); /********************************RESULT ********************/ } } } catch (Exception e) { result.setNrErrors(nbrerror); log.logError(toString(), Messages.getString("JobSSH2PUT.Log.Error.ErrorFTP", e.getMessage())); } finally { if (conn != null) conn.close(); if (client != null) client.close(); } } } catch (Exception e) { result.setResult(false); result.setNrErrors(1L); log.logError(toString(), Messages.getString("JobSSH2PUT.Log.Error.UnexpectedError"), e); } return result; }
From source file:com.panet.imeta.trans.steps.blockingstep.BlockingStep.java
private Object[] getBuffer() { Object[] retval;// w w w .j ava2 s . c om // Open all files at once and read one row from each file... if (data.files.size() > 0 && (data.dis.size() == 0 || data.fis.size() == 0)) { if (log.isBasic()) logBasic(Messages.getString("BlockingStep.Log.Openfiles")); try { FileObject fileObject = (FileObject) data.files.get(0); String filename = KettleVFS.getFilename(fileObject); if (log.isDetailed()) logDetailed(Messages.getString("BlockingStep.Log.Openfilename1") + filename + Messages.getString("BlockingStep.Log.Openfilename2")); InputStream fi = KettleVFS.getInputStream(fileObject); DataInputStream di; data.fis.add(fi); if (meta.getCompress()) { GZIPInputStream gzfi = new GZIPInputStream(new BufferedInputStream(fi)); di = new DataInputStream(gzfi); data.gzis.add(gzfi); } else { di = new DataInputStream(fi); } data.dis.add(di); // How long is the buffer? int buffersize = di.readInt(); if (log.isDetailed()) logDetailed(Messages.getString("BlockingStep.Log.BufferSize1") + filename + Messages.getString("BlockingStep.Log.BufferSize2") + buffersize + " " + Messages.getString("BlockingStep.Log.BufferSize3")); if (buffersize > 0) { // Read a row from temp-file data.rowbuffer.add(data.outputRowMeta.readData(di)); } } catch (Exception e) { logError(Messages.getString("BlockingStepMeta.ErrorReadingFile") + e.toString()); logError(Const.getStackTracker(e)); } } if (data.files.size() == 0) { if (data.buffer.size() > 0) { retval = (Object[]) data.buffer.get(0); data.buffer.remove(0); } else { retval = null; } } else { if (data.rowbuffer.size() == 0) { retval = null; } else { retval = (Object[]) data.rowbuffer.get(0); data.rowbuffer.remove(0); // now get another FileObject file = (FileObject) data.files.get(0); DataInputStream di = (DataInputStream) data.dis.get(0); InputStream fi = (InputStream) data.fis.get(0); GZIPInputStream gzfi = (meta.getCompress()) ? (GZIPInputStream) data.gzis.get(0) : null; try { data.rowbuffer.add(0, data.outputRowMeta.readData(di)); } catch (SocketTimeoutException e) { logError(Messages.getString("System.Log.UnexpectedError") + " : " + e.toString()); //$NON-NLS-1$ //$NON-NLS-2$ logError(Const.getStackTracker(e)); setErrors(1); stopAll(); } catch (KettleFileException fe) // empty file or EOF mostly { try { di.close(); fi.close(); if (gzfi != null) gzfi.close(); file.delete(); } catch (IOException e) { logError(Messages.getString("BlockingStepMeta.UnableDeleteFile") + file.toString()); setErrors(1); stopAll(); return null; } data.files.remove(0); data.dis.remove(0); data.fis.remove(0); if (gzfi != null) data.gzis.remove(0); } } } return retval; }