List of usage examples for org.apache.commons.vfs FileObject exists
public boolean exists() throws FileSystemException;
From source file:org.pentaho.di.job.entries.filecompare.JobEntryFileCompare.java
public Result execute(Result previousResult, int nr) { Result result = previousResult; result.setResult(false);//ww w . j a va 2s. c o m String realFilename1 = getRealFilename1(); String realFilename2 = getRealFilename2(); FileObject file1 = null; FileObject file2 = null; try { if (filename1 != null && filename2 != null) { file1 = KettleVFS.getFileObject(realFilename1, this); file2 = KettleVFS.getFileObject(realFilename2, this); if (file1.exists() && file2.exists()) { if (equalFileContents(file1, file2)) { result.setResult(true); } else { result.setResult(false); } // add filename to result filenames if (addFilenameToResult && file1.getType() == FileType.FILE && file2.getType() == FileType.FILE) { ResultFile resultFile = new ResultFile(ResultFile.FILE_TYPE_GENERAL, file1, parentJob.getJobname(), toString()); resultFile.setComment(BaseMessages.getString(PKG, "JobWaitForFile.FilenameAdded")); result.getResultFiles().put(resultFile.getFile().toString(), resultFile); resultFile = new ResultFile(ResultFile.FILE_TYPE_GENERAL, file2, parentJob.getJobname(), toString()); resultFile.setComment(BaseMessages.getString(PKG, "JobWaitForFile.FilenameAdded")); result.getResultFiles().put(resultFile.getFile().toString(), resultFile); } } else { if (!file1.exists()) { logError(BaseMessages.getString(PKG, "JobEntryFileCompare.ERROR_0004_File1_Does_Not_Exist", realFilename1)); } if (!file2.exists()) { logError(BaseMessages.getString(PKG, "JobEntryFileCompare.ERROR_0005_File2_Does_Not_Exist", realFilename2)); } result.setResult(false); result.setNrErrors(1); } } else { logError(BaseMessages.getString(PKG, "JobEntryFileCompare.ERROR_0006_Need_Two_Filenames")); } } catch (Exception e) { result.setResult(false); result.setNrErrors(1); logError(BaseMessages.getString(PKG, "JobEntryFileCompare.ERROR_0007_Comparing_Files", realFilename2, realFilename2, e.getMessage())); } finally { try { if (file1 != null) { file1.close(); file1 = null; } if (file2 != null) { file2.close(); file2 = null; } } catch (IOException e) { // Ignore errors } } return result; }
From source file:org.pentaho.di.job.entries.fileexists.JobEntryFileExists.java
public Result execute(Result previousResult, int nr) { Result result = previousResult; result.setResult(false);/*from w ww .j a v a 2s . co m*/ result.setNrErrors(0); if (filename != null) { String realFilename = getRealFilename(); try { FileObject file = KettleVFS.getFileObject(realFilename, this); if (file.exists() && file.isReadable()) { logDetailed(BaseMessages.getString(PKG, "JobEntryFileExists.File_Exists", realFilename)); result.setResult(true); } else { logDetailed( BaseMessages.getString(PKG, "JobEntryFileExists.File_Does_Not_Exist", realFilename)); } } catch (Exception e) { result.setNrErrors(1); logError(BaseMessages.getString(PKG, "JobEntryFileExists.ERROR_0004_IO_Exception", e.getMessage()), e); } } else { result.setNrErrors(1); logError(BaseMessages.getString(PKG, "JobEntryFileExists.ERROR_0005_No_Filename_Defined")); } return result; }
From source file:org.pentaho.di.job.entries.fileexists.JobEntryFileExists.java
/** * Exports the object to a flat-file system, adding content with filename keys to a set of definitions. The supplied * resource naming interface allows the object to name appropriately without worrying about those parts of the * implementation specific details.//from w ww. ja v a 2 s . c o m * * @param space * The variable space to resolve (environment) variables with. * @param definitions * The map containing the filenames and content * @param namingInterface * The resource naming interface allows the object to be named appropriately * @param repository * The repository to load resources from * @param metaStore * the metaStore to load external metadata from * * @return The filename for this object. (also contained in the definitions map) * @throws KettleException * in case something goes wrong during the export */ public String exportResources(VariableSpace space, Map<String, ResourceDefinition> definitions, ResourceNamingInterface namingInterface, Repository repository, IMetaStore metaStore) throws KettleException { try { // The object that we're modifying here is a copy of the original! // So let's change the filename from relative to absolute by grabbing the file object... // In case the name of the file comes from previous steps, forget about this! // if (!Const.isEmpty(filename)) { // From : ${FOLDER}/../foo/bar.csv // To : /home/matt/test/files/foo/bar.csv // FileObject fileObject = KettleVFS.getFileObject(space.environmentSubstitute(filename), space); // If the file doesn't exist, forget about this effort too! // if (fileObject.exists()) { // Convert to an absolute path... // filename = namingInterface.nameResource(fileObject, space, true); return filename; } } return null; } catch (Exception e) { throw new KettleException(e); } }
From source file:org.pentaho.di.job.entries.filesexist.JobEntryFilesExist.java
public Result execute(Result previousResult, int nr) { Result result = previousResult; result.setResult(false);//from www. j a v a2s . c o m result.setNrErrors(0); int missingfiles = 0; int nrErrors = 0; // see PDI-10270 for details boolean oldBehavior = "Y" .equalsIgnoreCase(getVariable(Const.KETTLE_COMPATIBILITY_SET_ERROR_ON_SPECIFIC_JOB_ENTRIES, "N")); if (arguments != null) { for (int i = 0; i < arguments.length && !parentJob.isStopped(); i++) { FileObject file = null; try { String realFilefoldername = environmentSubstitute(arguments[i]); file = KettleVFS.getFileObject(realFilefoldername, this); if (file.exists() && file.isReadable()) // TODO: is it needed to check file for readability? { if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobEntryFilesExist.File_Exists", realFilefoldername)); } } else { missingfiles++; if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobEntryFilesExist.File_Does_Not_Exist", realFilefoldername)); } } } catch (Exception e) { nrErrors++; missingfiles++; logError( BaseMessages.getString(PKG, "JobEntryFilesExist.ERROR_0004_IO_Exception", e.toString()), e); } finally { if (file != null) { try { file.close(); file = null; } catch (IOException ex) { /* Ignore */ } } } } } result.setNrErrors(nrErrors); if (oldBehavior) { result.setNrErrors(missingfiles); } if (missingfiles == 0) { result.setResult(true); } return result; }
From source file:org.pentaho.di.job.entries.folderisempty.JobEntryFolderIsEmpty.java
public Result execute(Result previousResult, int nr) { // see PDI-10270 for details boolean oldBehavior = "Y" .equalsIgnoreCase(getVariable(Const.KETTLE_COMPATIBILITY_SET_ERROR_ON_SPECIFIC_JOB_ENTRIES, "N")); Result result = previousResult; result.setResult(false);//from ww w .j a v a2 s. co m result.setNrErrors(oldBehavior ? 1 : 0); filescount = 0; folderscount = 0; pattern = null; if (!Const.isEmpty(getWildcard())) { pattern = Pattern.compile(getRealWildcard()); } if (foldername != null) { String realFoldername = getRealFoldername(); FileObject folderObject = null; try { folderObject = KettleVFS.getFileObject(realFoldername, this); if (folderObject.exists()) { // Check if it's a folder if (folderObject.getType() == FileType.FOLDER) { // File provided is a folder, so we can process ... try { folderObject.findFiles(new TextFileSelector(folderObject.toString())); } catch (Exception ex) { if (!(ex.getCause() instanceof ExpectedException)) { throw ex; } } if (log.isBasic()) { log.logBasic("Total files", "We found : " + filescount + " file(s)"); } if (filescount == 0) { result.setResult(true); result.setNrLinesInput(folderscount); } } else { // Not a folder, fail log.logError("[" + realFoldername + "] is not a folder, failing."); result.setNrErrors(1); } } else { // No Folder found if (log.isBasic()) { logBasic("we can not find [" + realFoldername + "] !"); } result.setNrErrors(1); } } catch (Exception e) { logError("Error checking folder [" + realFoldername + "]", e); result.setResult(false); result.setNrErrors(1); } finally { if (folderObject != null) { try { folderObject.close(); folderObject = null; } catch (IOException ex) { /* Ignore */ } } } } else { logError("No Foldername is defined."); result.setNrErrors(1); } return result; }
From source file:org.pentaho.di.job.entries.folderscompare.JobEntryFoldersCompare.java
public Result execute(Result previousResult, int nr) { Result result = previousResult; result.setResult(false);/* ww w . ja v a 2s . c om*/ boolean ok = true; String realFilename1 = getRealFilename1(); String realFilename2 = getRealFilename2(); FileObject folder1 = null; FileObject folder2 = null; FileObject filefolder1 = null; FileObject filefolder2 = null; try { if (filename1 != null && filename2 != null) { // Get Folders/Files to compare folder1 = KettleVFS.getFileObject(realFilename1, this); folder2 = KettleVFS.getFileObject(realFilename2, this); if (folder1.exists() && folder2.exists()) { if (!folder1.getType().equals(folder2.getType())) { // pb...we try to compare file with folder !!! logError(BaseMessages.getString(PKG, "JobFoldersCompare.Log.CanNotCompareFilesFolders")); if (folder1.getType() == FileType.FILE) { logError(BaseMessages.getString(PKG, "JobFoldersCompare.Log.IsAFile", realFilename1)); } else if (folder1.getType() == FileType.FOLDER) { logError(BaseMessages.getString(PKG, "JobFoldersCompare.Log.IsAFolder", realFilename1)); } else { logError(BaseMessages.getString(PKG, "JobFoldersCompare.Log.IsUnknownFileType", realFilename1)); } if (folder2.getType() == FileType.FILE) { logError(BaseMessages.getString(PKG, "JobFoldersCompare.Log.IsAFile", realFilename2)); } else if (folder2.getType() == FileType.FOLDER) { logError(BaseMessages.getString(PKG, "JobFoldersCompare.Log.IsAFolder", realFilename2)); } else { logError(BaseMessages.getString(PKG, "JobFoldersCompare.Log.IsUnknownFileType", realFilename2)); } } else { if (folder1.getType() == FileType.FILE) { // simply compare 2 files .. if (equalFileContents(folder1, folder2)) { result.setResult(true); } else { result.setResult(false); } } else if (folder1.getType() == FileType.FOLDER) { // We compare 2 folders ... FileObject[] list1 = folder1.findFiles(new TextFileSelector(folder1.toString())); FileObject[] list2 = folder2.findFiles(new TextFileSelector(folder2.toString())); int lenList1 = list1.length; int lenList2 = list2.length; if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobFoldersCompare.Log.FolderContains", realFilename1, "" + lenList1)); logDetailed(BaseMessages.getString(PKG, "JobFoldersCompare.Log.FolderContains", realFilename2, "" + lenList2)); } if (lenList1 == lenList2) { HashMap<String, String> collection1 = new HashMap<String, String>(); HashMap<String, String> collection2 = new HashMap<String, String>(); for (int i = 0; i < list1.length; i++) { // Put files list1 in TreeMap collection1 collection1.put(list1[i].getName().getBaseName(), list1[i].toString()); } for (int i = 0; i < list2.length; i++) { // Put files list2 in TreeMap collection2 collection2.put(list2[i].getName().getBaseName(), list2[i].toString()); } // Let's now fetch Folder1 // and for each entry, we will search it in Folder2 // if the entry exists..we will compare file entry (file or folder?) // if the 2 entry are file (not folder), we will compare content Set<Map.Entry<String, String>> entrees = collection1.entrySet(); Iterator<Map.Entry<String, String>> iterateur = entrees.iterator(); while (iterateur.hasNext()) { Map.Entry<String, String> entree = iterateur.next(); if (!collection2.containsKey(entree.getKey())) { ok = false; if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobFoldersCompare.Log.FileCanNotBeFoundIn", entree.getKey().toString(), realFilename2)); } } else { if (log.isDebug()) { logDebug(BaseMessages.getString(PKG, "JobFoldersCompare.Log.FileIsFoundIn", entree.getKey().toString(), realFilename2)); } filefolder1 = KettleVFS.getFileObject(entree.getValue().toString(), this); filefolder2 = KettleVFS .getFileObject(collection2.get(entree.getKey()).toString(), this); if (!filefolder2.getType().equals(filefolder1.getType())) { // The file1 exist in the folder2..but they don't have the same type ok = false; if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobFoldersCompare.Log.FilesNotSameType", filefolder1.toString(), filefolder2.toString())); } if (filefolder1.getType() == FileType.FILE) { logError(BaseMessages.getString(PKG, "JobFoldersCompare.Log.IsAFile", filefolder1.toString())); } else if (filefolder1.getType() == FileType.FOLDER) { logError(BaseMessages.getString(PKG, "JobFoldersCompare.Log.IsAFolder", filefolder1.toString())); } else { logError(BaseMessages.getString(PKG, "JobFoldersCompare.Log.IsUnknownFileType", filefolder1.toString())); } if (filefolder2.getType() == FileType.FILE) { logError(BaseMessages.getString(PKG, "JobFoldersCompare.Log.IsAFile", filefolder2.toString())); } else if (filefolder2.getType() == FileType.FOLDER) { logError(BaseMessages.getString(PKG, "JobFoldersCompare.Log.IsAFolder", filefolder2.toString())); } else { logError(BaseMessages.getString(PKG, "JobFoldersCompare.Log.IsUnknownFileType", filefolder2.toString())); } } else { // Files are the same type ... if (filefolder2.getType() == FileType.FILE) { // Let's compare file size if (comparefilesize) { long filefolder1_size = filefolder1.getContent().getSize(); long filefolder2_size = filefolder2.getContent().getSize(); if (filefolder1_size != filefolder2_size) { ok = false; if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobFoldersCompare.Log.FilesNotSameSize", filefolder1.toString(), filefolder2.toString())); logDetailed(BaseMessages.getString(PKG, "JobFoldersCompare.Log.SizeFileIs", filefolder1.toString(), "" + filefolder1_size)); logDetailed(BaseMessages.getString(PKG, "JobFoldersCompare.Log.SizeFileIs", filefolder2.toString(), "" + filefolder2_size)); } } } if (ok) { // Let's compare files content.. if (comparefilecontent) { if (!equalFileContents(filefolder1, filefolder2)) { ok = false; if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobFoldersCompare.Log.FilesNotSameContent", filefolder1.toString(), filefolder2.toString())); } } } } } } } // logBasic(entree.getKey() + " - " + entree.getValue()); } result.setResult(ok); } else { // The 2 folders don't have the same files number if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobFoldersCompare.Log.FoldersDifferentFiles", realFilename1.toString(), realFilename2.toString())); } } } // else: File type unknown !! } } else { if (!folder1.exists()) { logError(BaseMessages.getString(PKG, "JobFileCompare.Log.FileNotExist", realFilename1)); } if (!folder2.exists()) { logError(BaseMessages.getString(PKG, "JobFileCompare.Log.FileNotExist", realFilename2)); } result.setResult(false); result.setNrErrors(1); } } else { logError(BaseMessages.getString(PKG, "JobFoldersCompare.Log.Need2Files")); } } catch (Exception e) { result.setResult(false); result.setNrErrors(1); logError(BaseMessages.getString(PKG, "JobFoldersCompare.Log.ErrorComparing", realFilename2, realFilename2, e.getMessage())); } finally { try { if (folder1 != null) { folder1.close(); folder1 = null; } if (folder2 != null) { folder2.close(); folder2 = null; } if (filefolder1 != null) { filefolder1.close(); filefolder1 = null; } if (filefolder2 != null) { filefolder2.close(); filefolder2 = null; } } catch (IOException e) { // Ignore errors } } return result; }
From source file:org.pentaho.di.job.entries.getpop.JobEntryGetPOP.java
public Result execute(Result previousResult, int nr) throws KettleException { Result result = previousResult; result.setResult(false);/*from w w w .ja va 2 s .c om*/ //FileObject fileObject = null; MailConnection mailConn = null; Date beginDate = null; Date endDate = null; SimpleDateFormat df = new SimpleDateFormat(DATE_PATTERN); try { boolean usePOP3 = getProtocol().equals(MailConnectionMeta.PROTOCOL_STRING_POP3); boolean moveafter = false; int nbrmailtoretrieve = usePOP3 ? getRetrievemails() == 2 ? Const.toInt(getFirstMails(), 0) : 0 : Const.toInt(getFirstIMAPMails(), 0); String realOutputFolder = getRealOutputDirectory(); String targetAttachmentFolder = null; if (getActionType() == MailConnectionMeta.ACTION_TYPE_GET) { FileObject fileObject = KettleVFS.getFileObject(realOutputFolder, this); // Check if output folder exists if (fileObject.exists()) { if (fileObject.getType() != FileType.FOLDER) { throw new KettleException(BaseMessages.getString(PKG, "JobGetMailsFromPOP.Error.NotAFolderNot", realOutputFolder)); } if (isDebug()) { logDebug(BaseMessages.getString(PKG, "JobGetMailsFromPOP.Log.OutputFolderExists", realOutputFolder)); } } else { if (isCreateLocalFolder()) { if (isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobGetMailsFromPOP.Log.OutputFolderNotExist", realOutputFolder)); } // create folder fileObject.createFolder(); } else { throw new KettleException( BaseMessages.getString(PKG, "JobGetMailsFromPOP.FolderNotExists1.Label") + realOutputFolder + BaseMessages.getString(PKG, "JobGetMailsFromPOP.FolderNotExists2.Label")); } } targetAttachmentFolder = KettleVFS.getFilename(fileObject); // check for attachment folder boolean useDifferentFolderForAttachment = (isSaveAttachment() && isDifferentFolderForAttachment()); if (useDifferentFolderForAttachment) { String realFolderAttachment = environmentSubstitute(getAttachmentFolder()); if (Const.isEmpty(realFolderAttachment)) { throw new KettleException( BaseMessages.getString(PKG, "JobGetMailsFromPOP.Error.AttachmentFolderEmpty")); } //close old file object try { fileObject.close(); } catch (IOException ex) { /* Ignore */ } //reuse old link fileObject = KettleVFS.getFileObject(realFolderAttachment, this); if (!fileObject.exists()) { throw new KettleException(BaseMessages.getString(PKG, "JobGetMailsFromPOP.Error.AttachmentFolderNotExist", realFolderAttachment)); } if (fileObject.getType() != FileType.FOLDER) { throw new KettleException(BaseMessages.getString(PKG, "JobGetMailsFromPOP.Error.AttachmentFolderNotAFolder", realFolderAttachment)); } targetAttachmentFolder = KettleVFS.getFilename(fileObject); } // Close fileObject! we don't need it anymore ... try { fileObject.close(); } catch (IOException ex) { /* Ignore */ } } // end if get // Check destination folder String realMoveToIMAPFolder = environmentSubstitute(getMoveToIMAPFolder()); if (getProtocol().equals(MailConnectionMeta.PROTOCOL_STRING_IMAP) && (getActionType() == MailConnectionMeta.ACTION_TYPE_MOVE) || (getActionType() == MailConnectionMeta.ACTION_TYPE_GET && getAfterGetIMAP() == MailConnectionMeta.AFTER_GET_IMAP_MOVE)) { if (Const.isEmpty(realMoveToIMAPFolder)) { throw new KettleException( BaseMessages.getString(PKG, "JobGetMailsFromPOP.Error.MoveToIMAPFolderEmpty")); } moveafter = true; } // check search terms // Received Date switch (getConditionOnReceivedDate()) { case MailConnectionMeta.CONDITION_DATE_EQUAL: case MailConnectionMeta.CONDITION_DATE_GREATER: case MailConnectionMeta.CONDITION_DATE_SMALLER: String realBeginDate = environmentSubstitute(getReceivedDate1()); if (Const.isEmpty(realBeginDate)) { throw new KettleException( BaseMessages.getString(PKG, "JobGetMailsFromPOP.Error.ReceivedDateSearchTermEmpty")); } beginDate = df.parse(realBeginDate); break; case MailConnectionMeta.CONDITION_DATE_BETWEEN: realBeginDate = environmentSubstitute(getReceivedDate1()); if (Const.isEmpty(realBeginDate)) { throw new KettleException( BaseMessages.getString(PKG, "JobGetMailsFromPOP.Error.ReceivedDatesSearchTermEmpty")); } beginDate = df.parse(realBeginDate); String realEndDate = environmentSubstitute(getReceivedDate2()); if (Const.isEmpty(realEndDate)) { throw new KettleException( BaseMessages.getString(PKG, "JobGetMailsFromPOP.Error.ReceivedDatesSearchTermEmpty")); } endDate = df.parse(realEndDate); break; default: break; } String realserver = getRealServername(); String realusername = getRealUsername(); String realpassword = getRealPassword(); String realFilenamePattern = getRealFilenamePattern(); int realport = Const.toInt(environmentSubstitute(sslport), -1); String realIMAPFolder = environmentSubstitute(getIMAPFolder()); String realProxyUsername = getRealProxyUsername(); initVariables(); // create a mail connection object mailConn = new MailConnection(log, MailConnectionMeta.getProtocolFromString(getProtocol(), MailConnectionMeta.PROTOCOL_IMAP), realserver, realport, realusername, realpassword, isUseSSL(), isUseProxy(), realProxyUsername); // connect mailConn.connect(); if (moveafter) { // Set destination folder // Check if folder exists mailConn.setDestinationFolder(realMoveToIMAPFolder, isCreateMoveToFolder()); } // apply search term? String realSearchSender = environmentSubstitute(getSenderSearchTerm()); if (!Const.isEmpty(realSearchSender)) { // apply FROM mailConn.setSenderTerm(realSearchSender, isNotTermSenderSearch()); } String realSearchReceipient = environmentSubstitute(getReceipientSearch()); if (!Const.isEmpty(realSearchReceipient)) { // apply TO mailConn.setReceipientTerm(realSearchReceipient); } String realSearchSubject = environmentSubstitute(getSubjectSearch()); if (!Const.isEmpty(realSearchSubject)) { // apply Subject mailConn.setSubjectTerm(realSearchSubject, isNotTermSubjectSearch()); } String realSearchBody = environmentSubstitute(getBodySearch()); if (!Const.isEmpty(realSearchBody)) { // apply body mailConn.setBodyTerm(realSearchBody, isNotTermBodySearch()); } // Received Date switch (getConditionOnReceivedDate()) { case MailConnectionMeta.CONDITION_DATE_EQUAL: mailConn.setReceivedDateTermEQ(beginDate); break; case MailConnectionMeta.CONDITION_DATE_GREATER: mailConn.setReceivedDateTermGT(beginDate); break; case MailConnectionMeta.CONDITION_DATE_SMALLER: mailConn.setReceivedDateTermLT(beginDate); break; case MailConnectionMeta.CONDITION_DATE_BETWEEN: mailConn.setReceivedDateTermBetween(beginDate, endDate); break; default: break; } // set FlagTerm? if (usePOP3) { // retrieve messages if (getRetrievemails() == 1) { // New messages // POP doesn't support the concept of "new" messages! mailConn.setFlagTermUnread(); } } else { switch (getValueImapList()) { case MailConnectionMeta.VALUE_IMAP_LIST_NEW: mailConn.setFlagTermNew(); break; case MailConnectionMeta.VALUE_IMAP_LIST_OLD: mailConn.setFlagTermOld(); break; case MailConnectionMeta.VALUE_IMAP_LIST_READ: mailConn.setFlagTermRead(); break; case MailConnectionMeta.VALUE_IMAP_LIST_UNREAD: mailConn.setFlagTermUnread(); break; case MailConnectionMeta.VALUE_IMAP_LIST_FLAGGED: mailConn.setFlagTermFlagged(); break; case MailConnectionMeta.VALUE_IMAP_LIST_NOT_FLAGGED: mailConn.setFlagTermNotFlagged(); break; case MailConnectionMeta.VALUE_IMAP_LIST_DRAFT: mailConn.setFlagTermDraft(); break; case MailConnectionMeta.VALUE_IMAP_LIST_NOT_DRAFT: mailConn.setFlagTermNotDraft(); break; default: break; } } // open folder and retrieve messages fetchOneFolder(mailConn, usePOP3, realIMAPFolder, realOutputFolder, targetAttachmentFolder, realMoveToIMAPFolder, realFilenamePattern, nbrmailtoretrieve, df); if (isIncludeSubFolders()) { // Fetch also sub folders? if (isDebug()) { logDebug(BaseMessages.getString(PKG, "JobGetPOP.FetchingSubFolders")); } String[] subfolders = mailConn.returnAllFolders(); if (subfolders.length == 0) { if (isDebug()) { logDebug(BaseMessages.getString(PKG, "JobGetPOP.NoSubFolders")); } } else { for (int i = 0; i < subfolders.length; i++) { fetchOneFolder(mailConn, usePOP3, subfolders[i], realOutputFolder, targetAttachmentFolder, realMoveToIMAPFolder, realFilenamePattern, nbrmailtoretrieve, df); } } } result.setResult(true); result.setNrFilesRetrieved(mailConn.getSavedAttachedFilesCounter()); result.setNrLinesWritten(mailConn.getSavedMessagesCounter()); result.setNrLinesDeleted(mailConn.getDeletedMessagesCounter()); result.setNrLinesUpdated(mailConn.getMovedMessagesCounter()); if (isDetailed()) { logDetailed("======================================="); logDetailed(BaseMessages.getString(PKG, "JobGetPOP.Log.Info.SavedMessages", "" + mailConn.getSavedMessagesCounter())); logDetailed(BaseMessages.getString(PKG, "JobGetPOP.Log.Info.DeletedMessages", "" + mailConn.getDeletedMessagesCounter())); logDetailed(BaseMessages.getString(PKG, "JobGetPOP.Log.Info.MovedMessages", "" + mailConn.getMovedMessagesCounter())); if (getActionType() == MailConnectionMeta.ACTION_TYPE_GET && isSaveAttachment()) { logDetailed(BaseMessages.getString(PKG, "JobGetPOP.Log.Info.AttachedMessagesSuccess", "" + mailConn.getSavedAttachedFilesCounter())); } logDetailed("======================================="); } } catch (Exception e) { result.setNrErrors(1); logError("Unexpected error: " + e.getMessage()); logError(Const.getStackTracker(e)); } finally { try { if (mailConn != null) { mailConn.disconnect(); mailConn = null; } } catch (Exception e) { /* Ignore */ } } return result; }
From source file:org.pentaho.di.job.entries.hadooptransjobexecutor.DistributedCacheUtil.java
public void stagePluginsForCache(FileSystem fs, Path pluginsDir, boolean overwrite, List<FileObject> pluginDirectories) throws KettleFileException, IOException { if (pluginDirectories == null) { throw new IllegalArgumentException("plugins required"); }//from w w w .ja v a2 s. c o m if (!fs.exists(pluginsDir)) { fs.mkdirs(pluginsDir); } for (FileObject localPluginDir : pluginDirectories) { if (!localPluginDir.exists()) { throw new KettleFileException(BaseMessages.getString(DistributedCacheUtil.class, "DistributedCacheUtil.PluginDirectoryNotFound", localPluginDir)); } Path pluginDir = new Path(pluginsDir, localPluginDir.getName().getBaseName()); if (!overwrite && fs.exists(pluginDir)) { // skip installing this plugin, it already exists continue; } stageForCache(localPluginDir, fs, pluginDir, true); } }
From source file:org.pentaho.di.job.entries.hadooptransjobexecutor.DistributedCacheUtil.java
/** * Stages the source file or folder to a Hadoop file system and sets their permission and replication value appropriately * to be used with the Distributed Cache. WARNING: This will delete the contents of dest before staging the archive. * * @param source File or folder to copy to the file system. If it is a folder all contents will be copied into dest. * @param fs Hadoop file system to store the contents of the archive in * @param dest Destination to copy source into. If source is a file, the new file name will be exactly dest. If source * is a folder its contents will be copied into dest. For more info see * {@link FileSystem#copyFromLocalFile(org.apache.hadoop.fs.Path, org.apache.hadoop.fs.Path)}. * @param overwrite Should an existing file or folder be overwritten? If not an exception will be thrown. * @throws IOException Destination exists is not a directory * @throws KettleFileException Source does not exist or destination exists and overwrite is false. */// w w w. j a v a2s .c o m public void stageForCache(FileObject source, FileSystem fs, Path dest, boolean overwrite) throws IOException, KettleFileException { if (!source.exists()) { throw new KettleFileException(BaseMessages.getString(DistributedCacheUtil.class, "DistributedCacheUtil.SourceDoesNotExist", source)); } if (fs.exists(dest)) { if (overwrite) { // It is a directory, clear it out fs.delete(dest, true); } else { throw new KettleFileException(BaseMessages.getString(DistributedCacheUtil.class, "DistributedCacheUtil.DestinationExists", dest.toUri().getPath())); } } // Use the same replication we'd use for submitting jobs short replication = (short) fs.getConf().getInt("mapred.submit.replication", 10); Path local = new Path(source.getURL().getPath()); fs.copyFromLocalFile(local, dest); fs.setPermission(dest, CACHED_FILE_PERMISSION); fs.setReplication(dest, replication); }
From source file:org.pentaho.di.job.entries.hadooptransjobexecutor.DistributedCacheUtil.java
/** * Delete a directory and all of its contents * * @param dir Directory to delete/*from ww w . ja v a 2 s . co m*/ * @return True if the directory was deleted successfully */ public boolean deleteDirectory(FileObject dir) throws FileSystemException { dir.delete(new AllFileSelector()); return !dir.exists(); }