List of usage examples for org.apache.commons.vfs2 FileSystemException getMessage
@Override
public String getMessage()
From source file:com.streamsets.pipeline.stage.origin.remote.FTPRemoteDownloadSourceDelegate.java
@Override void queueFiles(FileQueueChecker fqc, NavigableSet<RemoteFile> fileQueue, FileFilter fileFilter) throws IOException, StageException { boolean done = false; int retryCounter = 0; while (!done && retryCounter < MAX_RETRIES) { try {/*from ww w . j a va 2 s . com*/ remoteDir.refresh(); // A call to getChildren() and then refresh() is needed in order to properly refresh if files were updated // A possible bug in VFS? remoteDir.getChildren(); remoteDir.refresh(); done = true; } catch (FileSystemException fse) { // Refresh can fail due to session is down, a timeout, etc; so try getting a new connection if (retryCounter < MAX_RETRIES - 1) { LOG.info("Got FileSystemException when trying to refresh remote directory. '{}'", fse.getMessage(), fse); LOG.warn("Retrying connection to remote directory"); remoteDir = VFS.getManager().resolveFile(remoteURI.toString(), options); } else { throw new StageException(Errors.REMOTE_18, fse.getMessage(), fse); } } retryCounter++; } FileObject[] theFiles = remoteDir.getChildren(); if (conf.processSubDirectories) { theFiles = ArrayUtils.addAll(theFiles, remoteDir.findFiles(fileFilter)); } for (FileObject file : theFiles) { String path = relativizeToRoot(file.getName().getPath()); LOG.debug("Checking {}", path); if (file.getType() != FileType.FILE) { LOG.trace("Skipping {} because it is not a file", path); continue; } //check if base name matches - not full path. Matcher matcher = fileFilter.getRegex().matcher(file.getName().getBaseName()); if (!matcher.matches()) { LOG.trace("Skipping {} because it does not match the regex", path); continue; } RemoteFile tempFile = new FTPRemoteFile(path, getModTime(file), file); if (fqc.shouldQueue(tempFile)) { LOG.debug("Queuing file {} with modtime {}", tempFile.getFilePath(), tempFile.getLastModified()); // If we are done with all files, the files with the final mtime might get re-ingested over and over. // So if it is the one of those, don't pull it in. fileQueue.add(tempFile); } } }
From source file:fr.cls.atoll.motu.library.misc.vfs.provider.gsiftp.GsiFtpFileObject.java
/** * Called when the children of this file change. * /*from w ww . j av a 2 s.c om*/ * @param child the child * @param newType the new type */ @Override protected void onChildrenChanged(FileName child, FileType newType) { if (children != null && newType.equals(FileType.IMAGINARY)) { try { children.remove(UriParser.decode(child.getBaseName())); } catch (FileSystemException e) { throw new RuntimeException(e.getMessage()); } } else { // if child was added we have to rescan the children // TODO - get rid of this children = null; } }
From source file:com.yenlo.synapse.transport.vfs.VFSTransportListener.java
@Override protected void doInit() throws AxisFault { super.doInit(); try {/* w w w .ja v a 2s . c o m*/ StandardFileSystemManager fsm = new StandardFileSystemManager(); fsm.setConfiguration(getClass().getClassLoader().getResource("providers.xml")); fsm.init(); this.workerPool = super.workerPool; fsManager = fsm; Parameter lockFlagParam = getTransportInDescription().getParameter(VFSConstants.TRANSPORT_FILE_LOCKING); if (lockFlagParam != null) { String strLockingFlag = lockFlagParam.getValue().toString(); // by-default enabled, if explicitly specified as "disable" make it disable if (VFSConstants.TRANSPORT_FILE_LOCKING_DISABLED.equals(strLockingFlag)) { globalFileLockingFlag = false; } } } catch (FileSystemException e) { handleException("Error initializing the file transport : " + e.getMessage(), e); } }
From source file:com.yenlo.synapse.transport.vfs.VFSTransportListener.java
/** * Search for files that match the given regex pattern and create a list * Then process each of these files and update the status of the scan on * the poll table/*from w w w . j ava2 s.c om*/ * @param entry the poll table entry for the scan * @param fileURI the file or directory to be scanned */ private void scanFileOrDirectory(final PollTableEntry entry, String fileURI) { FileObject fileObject = null; //TODO : Trying to make the correct URL out of the malformed one. if (fileURI.contains("vfs:")) { fileURI = fileURI.substring(fileURI.indexOf("vfs:") + 4); } if (log.isDebugEnabled()) { log.debug("Scanning directory or file : " + VFSUtils.maskURLPassword(fileURI)); } boolean wasError = true; int retryCount = 0; int maxRetryCount = entry.getMaxRetryCount(); long reconnectionTimeout = entry.getReconnectTimeout(); while (wasError) { try { retryCount++; fileObject = fsManager.resolveFile(fileURI); if (fileObject == null) { log.error("fileObject is null"); throw new FileSystemException("fileObject is null"); } wasError = false; } catch (FileSystemException e) { if (retryCount >= maxRetryCount) { processFailure( "Repeatedly failed to resolve the file URI: " + VFSUtils.maskURLPassword(fileURI), e, entry); return; } else { log.warn("Failed to resolve the file URI: " + VFSUtils.maskURLPassword(fileURI) + ", in attempt " + retryCount + ", " + e.getMessage() + " Retrying in " + reconnectionTimeout + " milliseconds."); } } if (wasError) { try { Thread.sleep(reconnectionTimeout); } catch (InterruptedException e2) { log.error("Thread was interrupted while waiting to reconnect.", e2); } } } try { if (fileObject.exists() && fileObject.isReadable()) { entry.setLastPollState(PollTableEntry.NONE); FileObject[] children = null; try { children = fileObject.getChildren(); } catch (FileNotFolderException ignored) { } catch (FileSystemException ex) { log.error(ex.getMessage(), ex); } // if this is a file that would translate to a single message if (children == null || children.length == 0) { boolean isFailedRecord = false; if (entry.getMoveAfterMoveFailure() != null) { isFailedRecord = isFailedRecord(fileObject, entry); } if (fileObject.getType() == FileType.FILE && !isFailedRecord) { if (!entry.isFileLockingEnabled() || (entry.isFileLockingEnabled() && VFSUtils.acquireLock(fsManager, fileObject))) { try { processFile(entry, fileObject); entry.setLastPollState(PollTableEntry.SUCCSESSFUL); metrics.incrementMessagesReceived(); } catch (AxisFault e) { logException("Error processing File URI : " + fileObject.getName(), e); entry.setLastPollState(PollTableEntry.FAILED); metrics.incrementFaultsReceiving(); } try { moveOrDeleteAfterProcessing(entry, fileObject); } catch (AxisFault axisFault) { logException("File object '" + fileObject.getURL().toString() + "' " + "cloud not be moved", axisFault); entry.setLastPollState(PollTableEntry.FAILED); String timeStamp = VFSUtils.getSystemTime(entry.getFailedRecordTimestampFormat()); addFailedRecord(entry, fileObject, timeStamp); } if (entry.isFileLockingEnabled()) { VFSUtils.releaseLock(fsManager, fileObject); if (log.isDebugEnabled()) { log.debug("Removed the lock file '" + fileObject.toString() + ".lock' of the file '" + fileObject.toString()); } } } else if (log.isDebugEnabled()) { log.debug("Couldn't get the lock for processing the file : " + fileObject.getName()); } else if (isFailedRecord) { if (entry.isFileLockingEnabled()) { VFSUtils.releaseLock(fsManager, fileObject); } // schedule a cleanup task if the file is there if (fsManager.resolveFile(fileObject.getURL().toString()) != null && removeTaskState == STATE_STOPPED && entry.getMoveAfterMoveFailure() != null) { workerPool.execute(new FileRemoveTask(entry, fileObject)); } if (log.isDebugEnabled()) { log.debug("File '" + fileObject.getURL() + "' has been marked as a failed" + " record, it will not process"); } } } } else { int failCount = 0; int successCount = 0; int processCount = 0; Integer iFileProcessingInterval = entry.getFileProcessingInterval(); Integer iFileProcessingCount = entry.getFileProcessingCount(); if (log.isDebugEnabled()) { log.debug("File name pattern : " + entry.getFileNamePattern()); } for (FileObject child : children) { //skipping *.lock file if (child.getName().getBaseName().endsWith(".lock")) { continue; } boolean isFailedRecord = false; if (entry.getMoveAfterMoveFailure() != null) { isFailedRecord = isFailedRecord(child, entry); } if (entry.getFileNamePattern() != null && child.getName().getBaseName().matches(entry.getFileNamePattern())) { //child's file name matches the file name pattern //now we try to get the lock and process if (log.isDebugEnabled()) { log.debug("Matching file : " + child.getName().getBaseName()); } if ((!entry.isFileLockingEnabled() || (entry.isFileLockingEnabled() && VFSUtils.acquireLock(fsManager, child))) && !isFailedRecord) { //process the file try { if (log.isDebugEnabled()) { log.debug("Processing file :" + child); } processCount++; processFile(entry, child); successCount++; // tell moveOrDeleteAfterProcessing() file was success entry.setLastPollState(PollTableEntry.SUCCSESSFUL); metrics.incrementMessagesReceived(); } catch (Exception e) { logException("Error processing File URI : " + child.getName(), e); failCount++; // tell moveOrDeleteAfterProcessing() file failed entry.setLastPollState(PollTableEntry.FAILED); metrics.incrementFaultsReceiving(); } //skipping un-locking file if failed to do delete/move after process boolean skipUnlock = false; try { moveOrDeleteAfterProcessing(entry, child); } catch (AxisFault axisFault) { logException( "File object '" + child.getURL().toString() + "'cloud not be moved, will remain in \"locked\" state", axisFault); skipUnlock = true; failCount++; entry.setLastPollState(PollTableEntry.FAILED); String timeStamp = VFSUtils .getSystemTime(entry.getFailedRecordTimestampFormat()); addFailedRecord(entry, child, timeStamp); } // if there is a failure or not we'll try to release the lock if (entry.isFileLockingEnabled() && !skipUnlock) { VFSUtils.releaseLock(fsManager, child); } } } else if (entry.getFileNamePattern() != null && !child.getName().getBaseName().matches(entry.getFileNamePattern())) { //child's file name does not match the file name pattern if (log.isDebugEnabled()) { log.debug("Non-Matching file : " + child.getName().getBaseName()); } } else if (isFailedRecord) { //it is a failed record if (entry.isFileLockingEnabled()) { VFSUtils.releaseLock(fsManager, child); VFSUtils.releaseLock(fsManager, fileObject); } if (fsManager.resolveFile(child.getURL().toString()) != null && removeTaskState == STATE_STOPPED && entry.getMoveAfterMoveFailure() != null) { workerPool.execute(new FileRemoveTask(entry, child)); } if (log.isDebugEnabled()) { log.debug("File '" + fileObject.getURL() + "' has been marked as a failed record, it will not " + "process"); } } if (iFileProcessingInterval != null && iFileProcessingInterval > 0) { try { if (log.isDebugEnabled()) { log.debug("Put the VFS processor to sleep for : " + iFileProcessingInterval); } Thread.sleep(iFileProcessingInterval); } catch (InterruptedException ie) { log.error("Unable to set the interval between file processors." + ie); } } else if (iFileProcessingCount != null && iFileProcessingCount <= processCount) { break; } } if (failCount == 0 && successCount > 0) { entry.setLastPollState(PollTableEntry.SUCCSESSFUL); } else if (successCount == 0 && failCount > 0) { entry.setLastPollState(PollTableEntry.FAILED); } else { entry.setLastPollState(PollTableEntry.WITH_ERRORS); } } // processing of this poll table entry is complete long now = System.currentTimeMillis(); entry.setLastPollTime(now); entry.setNextPollTime(now + entry.getPollInterval()); } else if (log.isDebugEnabled()) { log.debug("Unable to access or read file or directory : " + VFSUtils.maskURLPassword(fileURI) + "." + " Reason: " + (fileObject.exists() ? (fileObject.isReadable() ? "Unknown reason" : "The file can not be read!") : "The file does not exists!")); } onPollCompletion(entry); } catch (FileSystemException e) { processFailure("Error checking for existence and readability : " + VFSUtils.maskURLPassword(fileURI), e, entry); } }
From source file:maspack.fileutil.FileManager.java
/** * Fetches a hash file from a remote location. The hash file is assumed to * have the form <uri>.sha1 * //from w ww . j a va 2 s .c o m * @param uri * the URI of the file to obtain the hash * @return the hex-encoded hash value string */ public String getRemoteHash(URIx uri) throws FileTransferException { uri = getAbsoluteURI(uri); // construct a uri for the remote sha1 hash URIx hashURI = mergeExtension(uri, ".sha1"); // create input stream and download hash InputStream in = null; String sha1 = null; try { cacher.initialize(); } catch (FileSystemException e) { cacher.release(); throw new FileTransferException("Failed to initialize FileCacher", e); } try { in = cacher.getInputStream(hashURI); byte[] hash = new byte[40]; in.read(hash, 0, hash.length); sha1 = new String(hash); } catch (FileSystemException e) { String msg = decodeVFSMessage(e); throw new FileTransferException("Cannot obtain remote hash: " + uncap(msg), e); } catch (IOException e) { throw new FileTransferException("Cannot read hash from input stream: " + uncap(e.getMessage()), e); } finally { closeQuietly(in); cacher.release(); } return sha1; }
From source file:com.streamsets.pipeline.lib.remote.FTPRemoteConnector.java
@Override protected void initAndConnect(List<Stage.ConfigIssue> issues, ConfigIssueContext context, URI remoteURI, Label remoteGroup, Label credGroup) { options = new FileSystemOptions(); this.remoteURI = remoteURI; if (remoteConfig.strictHostChecking) { issues.add(context.createConfigIssue(credGroup.getLabel(), CONF_PREFIX + "strictHostChecking", Errors.REMOTE_07));//from ww w. j a v a 2s . c o m } switch (remoteConfig.auth) { case PRIVATE_KEY: issues.add( context.createConfigIssue(credGroup.getLabel(), CONF_PREFIX + "privateKey", Errors.REMOTE_06)); break; case PASSWORD: String username = resolveUsername(remoteURI, issues, context, credGroup); String password = resolvePassword(remoteURI, issues, context, credGroup); if (remoteURI.getUserInfo() != null) { remoteURI = UriBuilder.fromUri(remoteURI).userInfo(null).build(); } StaticUserAuthenticator authenticator = new StaticUserAuthenticator(remoteURI.getHost(), username, password); try { DefaultFileSystemConfigBuilder.getInstance().setUserAuthenticator(options, authenticator); } catch (FileSystemException e) { // This method doesn't actually ever throw a FileSystemException, it simply declares that it does... // This shouldn't happen, but just in case, we'll log it LOG.error("Unexpected Exception", e); } break; case NONE: break; default: break; } FtpFileSystemConfigBuilder configBuilder = FtpFileSystemConfigBuilder.getInstance(); if (remoteURI.getScheme().equals(FTPS_SCHEME)) { configBuilder = FtpsFileSystemConfigBuilder.getInstance(); FtpsFileSystemConfigBuilder.getInstance().setFtpsMode(options, remoteConfig.ftpsMode.getMode()); FtpsFileSystemConfigBuilder.getInstance().setDataChannelProtectionLevel(options, remoteConfig.ftpsDataChannelProtectionLevel.getLevel()); if (remoteConfig.useFTPSClientCert) { setFtpsUserKeyManagerOrTrustManager(remoteConfig.ftpsClientCertKeystoreFile, CONF_PREFIX + "ftpsClientCertKeystoreFile", remoteConfig.ftpsClientCertKeystorePassword, CONF_PREFIX + "ftpsClientCertKeystorePassword", remoteConfig.ftpsClientCertKeystoreType, true, issues, context, credGroup); } switch (remoteConfig.ftpsTrustStoreProvider) { case FILE: setFtpsUserKeyManagerOrTrustManager(remoteConfig.ftpsTruststoreFile, CONF_PREFIX + "ftpsTruststoreFile", remoteConfig.ftpsTruststorePassword, CONF_PREFIX + "ftpsTruststorePassword", remoteConfig.ftpsTruststoreType, false, issues, context, credGroup); break; case JVM_DEFAULT: try { FtpsFileSystemConfigBuilder.getInstance().setTrustManager(options, TrustManagerUtils.getDefaultTrustManager(null)); } catch (GeneralSecurityException e) { issues.add(context.createConfigIssue(credGroup.getLabel(), CONF_PREFIX + "ftpsTruststoreFile", Errors.REMOTE_14, "trust", "JVM", e.getMessage(), e)); } break; case ALLOW_ALL: // fall through default: FtpsFileSystemConfigBuilder.getInstance().setTrustManager(options, TrustManagerUtils.getAcceptAllTrustManager()); break; } } configBuilder.setPassiveMode(options, true); configBuilder.setUserDirIsRoot(options, remoteConfig.userDirIsRoot); // Only actually try to connect and authenticate if there were no issues if (issues.isEmpty()) { LOG.info("Connecting to {}", remoteURI.toString()); try { remoteDir = VFS.getManager().resolveFile(remoteURI.toString(), options); remoteDir.refresh(); if (remoteConfig.createPathIfNotExists && !remoteDir.exists()) { remoteDir.createFolder(); } remoteDir.getChildren(); } catch (FileSystemException e) { LOG.error(Errors.REMOTE_11.getMessage(), remoteURI.toString(), e.getMessage(), e); issues.add(context.createConfigIssue(remoteGroup.getLabel(), CONF_PREFIX + "remoteAddress", Errors.REMOTE_11, remoteURI.toString(), e.getMessage(), e)); } } }
From source file:org.apache.metron.common.dsl.functions.resolver.ClasspathFunctionResolver.java
@Override public void initialize(Context context) { super.initialize(context); if (context != null) { Optional<Object> optional = context.getCapability(STELLAR_CONFIG, false); if (optional.isPresent()) { Map<String, Object> stellarConfig = (Map<String, Object>) optional.get(); if (LOG.isDebugEnabled()) { LOG.debug("Setting up classloader using the following config: " + stellarConfig); }//from w w w .java2s . c om include(STELLAR_SEARCH_INCLUDES_KEY.get(stellarConfig, String.class).split(STELLAR_SEARCH_DELIMS)); exclude(STELLAR_SEARCH_EXCLUDES_KEY.get(stellarConfig, String.class).split(STELLAR_SEARCH_DELIMS)); Optional<ClassLoader> vfsLoader = null; try { vfsLoader = VFSClassloaderUtil .configureClassloader(STELLAR_VFS_PATHS.get(stellarConfig, String.class)); if (vfsLoader.isPresent()) { LOG.debug("CLASSLOADER LOADED WITH: " + STELLAR_VFS_PATHS.get(stellarConfig, String.class)); if (LOG.isDebugEnabled()) { for (FileObject fo : ((VFSClassLoader) vfsLoader.get()).getFileObjects()) { LOG.error(fo.getURL() + " - " + fo.exists()); } } classLoaders(vfsLoader.get()); } } catch (FileSystemException e) { LOG.error("Unable to process filesystem: " + e.getMessage(), e); } } else { LOG.info("No stellar config set; I'm reverting to the context classpath with no restrictions."); if (LOG.isDebugEnabled()) { try { throw new IllegalStateException("No config set, stacktrace follows."); } catch (IllegalStateException ise) { LOG.error(ise.getMessage(), ise); } } } } else { throw new IllegalStateException("CONTEXT IS NULL!"); } }
From source file:org.apache.metron.stellar.dsl.functions.resolver.ClasspathFunctionResolver.java
@Override public void initialize(Context context) { super.initialize(context); if (context != null) { Optional<Object> optional = context.getCapability(STELLAR_CONFIG, false); if (optional.isPresent()) { Map<String, Object> stellarConfig = (Map<String, Object>) optional.get(); if (LOG.isDebugEnabled()) { LOG.debug("Setting up classloader using the following config: {}", stellarConfig); }/*from w ww. j a v a 2 s. c om*/ include(STELLAR_SEARCH_INCLUDES_KEY.get(stellarConfig, String.class).split(STELLAR_SEARCH_DELIMS)); exclude(STELLAR_SEARCH_EXCLUDES_KEY.get(stellarConfig, String.class).split(STELLAR_SEARCH_DELIMS)); Optional<ClassLoader> vfsLoader = Optional.empty(); try { vfsLoader = VFSClassloaderUtil .configureClassloader(STELLAR_VFS_PATHS.get(stellarConfig, String.class)); if (vfsLoader.isPresent()) { LOG.debug("CLASSLOADER LOADED WITH: {}", STELLAR_VFS_PATHS.get(stellarConfig, String.class)); if (LOG.isDebugEnabled()) { for (FileObject fo : ((VFSClassLoader) vfsLoader.get()).getFileObjects()) { LOG.error("{} - {}", fo.getURL(), fo.exists()); } } classLoaders(vfsLoader.get()); } } catch (FileSystemException e) { LOG.error("Unable to process filesystem: {}", e.getMessage(), e); } } else { LOG.info("No stellar config set; I'm reverting to the context classpath with no restrictions."); if (LOG.isDebugEnabled()) { try { throw new IllegalStateException("No config set, stacktrace follows."); } catch (IllegalStateException ise) { LOG.error(ise.getMessage(), ise); } } } } else { throw new IllegalStateException("CONTEXT IS NULL!"); } }
From source file:org.apache.synapse.transport.vfs.VFSTransportListener.java
/** * Search for files that match the given regex pattern and create a list * Then process each of these files and update the status of the scan on * the poll table//w w w . j av a 2 s . c om * @param entry the poll table entry for the scan * @param fileURI the file or directory to be scanned */ private void scanFileOrDirectory(final PollTableEntry entry, String fileURI) { FileSystemOptions fso = null; setFileSystemClosed(false); try { fso = VFSUtils.attachFileSystemOptions(entry.getVfsSchemeProperties(), fsManager); } catch (Exception e) { log.error("Error while attaching VFS file system properties. " + e.getMessage()); } FileObject fileObject = null; //TODO : Trying to make the correct URL out of the malformed one. if (fileURI.contains("vfs:")) { fileURI = fileURI.substring(fileURI.indexOf("vfs:") + 4); } if (log.isDebugEnabled()) { log.debug("Scanning directory or file : " + VFSUtils.maskURLPassword(fileURI)); } boolean wasError = true; int retryCount = 0; int maxRetryCount = entry.getMaxRetryCount(); long reconnectionTimeout = entry.getReconnectTimeout(); while (wasError) { try { retryCount++; fileObject = fsManager.resolveFile(fileURI, fso); if (fileObject == null) { log.error("fileObject is null"); throw new FileSystemException("fileObject is null"); } wasError = false; } catch (FileSystemException e) { if (retryCount >= maxRetryCount) { processFailure( "Repeatedly failed to resolve the file URI: " + VFSUtils.maskURLPassword(fileURI), e, entry); closeFileSystem(fileObject); return; } else { log.warn("Failed to resolve the file URI: " + VFSUtils.maskURLPassword(fileURI) + ", in attempt " + retryCount + ", " + e.getMessage() + " Retrying in " + reconnectionTimeout + " milliseconds."); } } if (wasError) { try { Thread.sleep(reconnectionTimeout); } catch (InterruptedException e2) { log.error("Thread was interrupted while waiting to reconnect.", e2); } } } try { if (fileObject.exists() && fileObject.isReadable()) { entry.setLastPollState(PollTableEntry.NONE); FileObject[] children = null; try { children = fileObject.getChildren(); } catch (FileNotFolderException ignored) { } catch (FileSystemException ex) { log.error(ex.getMessage(), ex); } // if this is a file that would translate to a single message if (children == null || children.length == 0) { boolean isFailedRecord = false; if (entry.getMoveAfterMoveFailure() != null) { isFailedRecord = isFailedRecord(fileObject, entry); } if (fileObject.getType() == FileType.FILE && !isFailedRecord) { boolean runPostProcess = true; if (!entry.isFileLockingEnabled() || (entry.isFileLockingEnabled() && acquireLock(fsManager, fileObject, entry, fso))) { try { if (fileObject.getType() == FileType.FILE) { processFile(entry, fileObject); entry.setLastPollState(PollTableEntry.SUCCSESSFUL); metrics.incrementMessagesReceived(); } else { runPostProcess = false; } } catch (AxisFault e) { if (e.getCause() instanceof FileNotFoundException) { log.warn("Error processing File URI : " + VFSUtils.maskURLPassword(fileObject.getName().toString()) + ". This can be due to file moved from another process."); runPostProcess = false; } else { logException("Error processing File URI : " + VFSUtils.maskURLPassword(fileObject.getName().getURI()), e); entry.setLastPollState(PollTableEntry.FAILED); metrics.incrementFaultsReceiving(); } } if (runPostProcess) { try { moveOrDeleteAfterProcessing(entry, fileObject, fso); } catch (AxisFault axisFault) { logException("File object '" + VFSUtils.maskURLPassword(fileObject.getURL().toString()) + "' " + "cloud not be moved", axisFault); entry.setLastPollState(PollTableEntry.FAILED); String timeStamp = VFSUtils .getSystemTime(entry.getFailedRecordTimestampFormat()); addFailedRecord(entry, fileObject, timeStamp); } } if (entry.isFileLockingEnabled()) { VFSUtils.releaseLock(fsManager, fileObject, fso); if (log.isDebugEnabled()) { log.debug("Removed the lock file '" + VFSUtils.maskURLPassword(fileObject.toString()) + ".lock' of the file '" + VFSUtils.maskURLPassword(fileObject.toString())); } } } else if (log.isDebugEnabled()) { log.debug("Couldn't get the lock for processing the file : " + VFSUtils.maskURLPassword(fileObject.getName().getURI())); } else if (isFailedRecord) { if (entry.isFileLockingEnabled()) { VFSUtils.releaseLock(fsManager, fileObject, fso); } // schedule a cleanup task if the file is there if (fsManager.resolveFile(fileObject.getURL().toString(), fso) != null && removeTaskState == STATE_STOPPED && entry.getMoveAfterMoveFailure() != null) { workerPool.execute(new FileRemoveTask(entry, fileObject, fso)); } if (log.isDebugEnabled()) { log.debug("File '" + VFSUtils.maskURLPassword(fileObject.getURL().toString()) + "' has been marked as a failed" + " record, it will not process"); } } } } else { int failCount = 0; int successCount = 0; int processCount = 0; Integer iFileProcessingInterval = entry.getFileProcessingInterval(); Integer iFileProcessingCount = entry.getFileProcessingCount(); if (log.isDebugEnabled()) { log.debug("File name pattern : " + entry.getFileNamePattern()); } // Sort the files String strSortParam = entry.getFileSortParam(); if (strSortParam != null) { log.debug("Start Sorting the files."); boolean bSortOrderAsscending = entry.isFileSortAscending(); if (log.isDebugEnabled()) { log.debug( "Sorting the files by : " + strSortParam + ". (" + bSortOrderAsscending + ")"); } if (strSortParam.equals(VFSConstants.FILE_SORT_VALUE_NAME) && bSortOrderAsscending) { Arrays.sort(children, new FileNameAscComparator()); } else if (strSortParam.equals(VFSConstants.FILE_SORT_VALUE_NAME) && !bSortOrderAsscending) { Arrays.sort(children, new FileNameDesComparator()); } else if (strSortParam.equals(VFSConstants.FILE_SORT_VALUE_SIZE) && bSortOrderAsscending) { Arrays.sort(children, new FileSizeAscComparator()); } else if (strSortParam.equals(VFSConstants.FILE_SORT_VALUE_SIZE) && !bSortOrderAsscending) { Arrays.sort(children, new FileSizeDesComparator()); } else if (strSortParam.equals(VFSConstants.FILE_SORT_VALUE_LASTMODIFIEDTIMESTAMP) && bSortOrderAsscending) { Arrays.sort(children, new FileLastmodifiedtimestampAscComparator()); } else if (strSortParam.equals(VFSConstants.FILE_SORT_VALUE_LASTMODIFIEDTIMESTAMP) && !bSortOrderAsscending) { Arrays.sort(children, new FileLastmodifiedtimestampDesComparator()); } log.debug("End Sorting the files."); } for (FileObject child : children) { //skipping *.lock file if (child.getName().getBaseName().endsWith(".lock")) { continue; } boolean isFailedRecord = false; if (entry.getMoveAfterMoveFailure() != null) { isFailedRecord = isFailedRecord(child, entry); } if (entry.getFileNamePattern() != null && child.getName().getBaseName().matches(entry.getFileNamePattern())) { //child's file name matches the file name pattern //now we try to get the lock and process if (log.isDebugEnabled()) { log.debug("Matching file : " + child.getName().getBaseName()); } boolean runPostProcess = true; if ((!entry.isFileLockingEnabled() || (entry.isFileLockingEnabled() && VFSUtils.acquireLock(fsManager, child, fso))) && !isFailedRecord) { //process the file try { if (log.isDebugEnabled()) { log.debug("Processing file :" + VFSUtils.maskURLPassword(child.toString())); } processCount++; if (child.getType() == FileType.FILE) { processFile(entry, child); successCount++; // tell moveOrDeleteAfterProcessing() file was success entry.setLastPollState(PollTableEntry.SUCCSESSFUL); metrics.incrementMessagesReceived(); } else { runPostProcess = false; } } catch (Exception e) { if (e.getCause() instanceof FileNotFoundException) { log.warn("Error processing File URI : " + VFSUtils.maskURLPassword(child.getName().toString()) + ". This can be due to file moved from another process."); runPostProcess = false; } else { logException("Error processing File URI : " + VFSUtils.maskURLPassword(child.getName().getURI()), e); failCount++; // tell moveOrDeleteAfterProcessing() file failed entry.setLastPollState(PollTableEntry.FAILED); metrics.incrementFaultsReceiving(); } } //skipping un-locking file if failed to do delete/move after process boolean skipUnlock = false; if (runPostProcess) { try { moveOrDeleteAfterProcessing(entry, child, fso); } catch (AxisFault axisFault) { logException( "File object '" + VFSUtils.maskURLPassword(child.getURL().toString()) + "'cloud not be moved, will remain in \"locked\" state", axisFault); skipUnlock = true; failCount++; entry.setLastPollState(PollTableEntry.FAILED); String timeStamp = VFSUtils .getSystemTime(entry.getFailedRecordTimestampFormat()); addFailedRecord(entry, child, timeStamp); } } // if there is a failure or not we'll try to release the lock if (entry.isFileLockingEnabled() && !skipUnlock) { VFSUtils.releaseLock(fsManager, child, fso); } } } else if (entry.getFileNamePattern() != null && !child.getName().getBaseName().matches(entry.getFileNamePattern())) { //child's file name does not match the file name pattern if (log.isDebugEnabled()) { log.debug("Non-Matching file : " + child.getName().getBaseName()); } } else if (isFailedRecord) { //it is a failed record if (entry.isFileLockingEnabled()) { VFSUtils.releaseLock(fsManager, child, fso); VFSUtils.releaseLock(fsManager, fileObject, fso); } if (fsManager.resolveFile(child.getURL().toString()) != null && removeTaskState == STATE_STOPPED && entry.getMoveAfterMoveFailure() != null) { workerPool.execute(new FileRemoveTask(entry, child, fso)); } if (log.isDebugEnabled()) { log.debug("File '" + VFSUtils.maskURLPassword(fileObject.getURL().toString()) + "' has been marked as a failed record, it will not " + "process"); } } if (iFileProcessingInterval != null && iFileProcessingInterval > 0) { try { if (log.isDebugEnabled()) { log.debug("Put the VFS processor to sleep for : " + iFileProcessingInterval); } Thread.sleep(iFileProcessingInterval); } catch (InterruptedException ie) { log.error("Unable to set the interval between file processors." + ie); } } else if (iFileProcessingCount != null && iFileProcessingCount <= processCount) { break; } } if (failCount == 0 && successCount > 0) { entry.setLastPollState(PollTableEntry.SUCCSESSFUL); } else if (successCount == 0 && failCount > 0) { entry.setLastPollState(PollTableEntry.FAILED); } else { entry.setLastPollState(PollTableEntry.WITH_ERRORS); } } // processing of this poll table entry is complete long now = System.currentTimeMillis(); entry.setLastPollTime(now); entry.setNextPollTime(now + entry.getPollInterval()); } else if (log.isDebugEnabled()) { log.debug("Unable to access or read file or directory : " + VFSUtils.maskURLPassword(fileURI) + "." + " Reason: " + (fileObject.exists() ? (fileObject.isReadable() ? "Unknown reason" : "The file can not be read!") : "The file does not exists!")); } onPollCompletion(entry); } catch (FileSystemException e) { processFailure("Error checking for existence and readability : " + VFSUtils.maskURLPassword(fileURI), e, entry); } catch (Exception ex) { processFailure("Un-handled exception thrown when processing the file : ", ex, entry); } finally { closeFileSystem(fileObject); } }
From source file:org.cloudifysource.esc.installer.filetransfer.SftpFileTransfer.java
@Override protected void initVFSManager(final InstallationDetails details, final long endTimeMillis) throws InstallerException { try {// www .j a v a 2 s .co m this.opts = new FileSystemOptions(); SftpFileSystemConfigBuilder.getInstance().setStrictHostKeyChecking(opts, "no"); SftpFileSystemConfigBuilder.getInstance().setUserDirIsRoot(opts, false); final Object preferredAuthenticationMethods = details.getCustomData() .get(CloudifyConstants.INSTALLER_CUSTOM_DATA_SFTP_PREFERRED_AUTHENTICATION_METHODS_KEY); if (preferredAuthenticationMethods != null) { if (String.class.isInstance(preferredAuthenticationMethods)) { SftpFileSystemConfigBuilder.getInstance().setPreferredAuthentications(opts, (String) preferredAuthenticationMethods); } else { throw new IllegalArgumentException("Was expecti`ng a string value for custom data field '" + CloudifyConstants.INSTALLER_CUSTOM_DATA_SFTP_PREFERRED_AUTHENTICATION_METHODS_KEY + "', got a; " + preferredAuthenticationMethods.getClass().getName()); } } final String keyFile = details.getKeyFile(); if (keyFile != null && !keyFile.isEmpty()) { final File temp = new File(keyFile); if (!temp.exists()) { throw new InstallerException("Could not find key file: " + temp + ". KeyFile " + keyFile + " that was passed in the installation Details does not exist"); } SftpFileSystemConfigBuilder.getInstance().setIdentities(opts, new File[] { temp }); } SftpFileSystemConfigBuilder.getInstance().setTimeout(opts, installerConfiguration.getFileTransferConnectionTimeoutMillis()); this.fileSystemManager = VFS.getManager(); } catch (final FileSystemException e) { throw new InstallerException("Failed to set up file transfer: " + e.getMessage(), e); } }