Java tutorial
package com.clickha.nifi.processors.util; /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ import java.io.File; import java.io.IOException; import java.io.InputStream; import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.Proxy; import java.net.UnknownHostException; import java.nio.file.Path; import java.nio.file.Paths; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.Locale; import java.util.concurrent.TimeUnit; import java.util.regex.Pattern; import org.apache.commons.net.ftp.FTPClient; import org.apache.commons.net.ftp.FTPClientConfig; import org.apache.commons.net.ftp.FTPFile; import org.apache.commons.net.ftp.FTPHTTPClient; import org.apache.commons.net.ftp.FTPReply; import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.flowfile.FlowFile; import org.apache.nifi.logging.ComponentLog; import org.apache.nifi.processor.ProcessContext; import org.apache.nifi.processor.exception.ProcessException; import org.apache.nifi.processor.util.StandardValidators; import com.clickha.nifi.processors.util.SocksProxySocketFactory; public class FTPTransferV2 implements FileTransferV2 { public static final String CONNECTION_MODE_ACTIVE = "Active"; public static final String CONNECTION_MODE_PASSIVE = "Passive"; // adjusted to handle Unix SunOS // FTPClientConfig config = new FTPClientConfig(FTPClientConfig.SYST_UNIX); // client.configure(config); public static final String FTP_CLIENT_CONFIG_SYST_UNIX = FTPClientConfig.SYST_UNIX; public static final String FTP_CLIENT_CONFIG_SYST_NT = FTPClientConfig.SYST_NT; public static final String TRANSFER_MODE_ASCII = "ASCII"; public static final String TRANSFER_MODE_BINARY = "Binary"; public static final String FTP_TIMEVAL_FORMAT = "yyyyMMddHHmmss"; public static final String PROXY_TYPE_DIRECT = Proxy.Type.DIRECT.name(); public static final String PROXY_TYPE_HTTP = Proxy.Type.HTTP.name(); public static final String PROXY_TYPE_SOCKS = Proxy.Type.SOCKS.name(); public static final PropertyDescriptor CONNECTION_MODE = new PropertyDescriptor.Builder() .name("Connection Mode").description("The FTP Connection Mode") .allowableValues(CONNECTION_MODE_ACTIVE, CONNECTION_MODE_PASSIVE).defaultValue(CONNECTION_MODE_PASSIVE) .build(); public static final PropertyDescriptor FTP_CLIENT_CONFIG_SYST = new PropertyDescriptor.Builder() .name("Client Configuration").description("FTP Client Configuration") .allowableValues(FTP_CLIENT_CONFIG_SYST_UNIX, FTP_CLIENT_CONFIG_SYST_NT) //.defaultValue(CONNECTION_MODE_PASSIVE) .build(); public static final PropertyDescriptor TRANSFER_MODE = new PropertyDescriptor.Builder().name("Transfer Mode") .description("The FTP Transfer Mode").allowableValues(TRANSFER_MODE_BINARY, TRANSFER_MODE_ASCII) .defaultValue(TRANSFER_MODE_BINARY).build(); public static final PropertyDescriptor PORT = new PropertyDescriptor.Builder().name("Port") .description("The port that the remote system is listening on for file transfers") .addValidator(StandardValidators.PORT_VALIDATOR).required(true).defaultValue("21") .expressionLanguageSupported(true).build(); public static final PropertyDescriptor PROXY_TYPE = new PropertyDescriptor.Builder().name("Proxy Type") .description("Proxy type used for file transfers") .allowableValues(PROXY_TYPE_DIRECT, PROXY_TYPE_HTTP, PROXY_TYPE_SOCKS).defaultValue(PROXY_TYPE_DIRECT) .build(); public static final PropertyDescriptor PROXY_HOST = new PropertyDescriptor.Builder().name("Proxy Host") .description("The fully qualified hostname or IP address of the proxy server") .addValidator(StandardValidators.NON_EMPTY_VALIDATOR).build(); public static final PropertyDescriptor PROXY_PORT = new PropertyDescriptor.Builder().name("Proxy Port") .description("The port of the proxy server").addValidator(StandardValidators.PORT_VALIDATOR).build(); public static final PropertyDescriptor HTTP_PROXY_USERNAME = new PropertyDescriptor.Builder() .name("Http Proxy Username").description("Http Proxy Username") .addValidator(StandardValidators.NON_EMPTY_VALIDATOR).required(false).build(); public static final PropertyDescriptor HTTP_PROXY_PASSWORD = new PropertyDescriptor.Builder() .name("Http Proxy Password").description("Http Proxy Password") .addValidator(StandardValidators.NON_EMPTY_VALIDATOR).required(false).sensitive(true).build(); private final ComponentLog logger; private final ProcessContext ctx; private boolean closed = true; private FTPClient client; private FTPClientConfig ftpConfig; private String homeDirectory; private String remoteHostName; public FTPTransferV2(final ProcessContext context, final ComponentLog logger) { this.ctx = context; this.logger = logger; } @Override public String getProtocolName() { return "ftp"; } @Override public void close() throws IOException { if (closed) { return; } closed = true; try { if (null != client) { client.disconnect(); } } catch (final Exception ex) { logger.warn("Failed to close FTPClient due to {}", new Object[] { ex.toString() }, ex); } client = null; } @Override public String getHomeDirectory(final FlowFile flowFile) throws IOException { getClient(flowFile); return homeDirectory; } @Override public List<FileInfo> getListing() throws IOException { final String path = ctx.getProperty(FileTransferV2.REMOTE_PATH).evaluateAttributeExpressions().getValue(); final int depth = 0; final int maxResults = ctx.getProperty(FileTransferV2.REMOTE_POLL_BATCH_SIZE).asInteger(); return getListing(path, depth, maxResults); } private List<FileInfo> getListing(final String path, final int depth, final int maxResults) throws IOException { final List<FileInfo> listing = new ArrayList<>(); if (maxResults < 1) { return listing; } if (depth >= 100) { logger.warn(this + " had to stop recursively searching directories at a recursive depth of " + depth + " to avoid memory issues"); return listing; } final boolean ignoreDottedFiles = ctx.getProperty(FileTransferV2.IGNORE_DOTTED_FILES).asBoolean(); final boolean recurse = ctx.getProperty(FileTransferV2.RECURSIVE_SEARCH).asBoolean(); final String fileFilterRegex = ctx.getProperty(FileTransferV2.FILE_FILTER_REGEX).getValue(); final Pattern pattern = (fileFilterRegex == null) ? null : Pattern.compile(fileFilterRegex); final String pathFilterRegex = ctx.getProperty(FileTransferV2.PATH_FILTER_REGEX).getValue(); final Pattern pathPattern = (!recurse || pathFilterRegex == null) ? null : Pattern.compile(pathFilterRegex); final String remotePath = ctx.getProperty(FileTransferV2.REMOTE_PATH).evaluateAttributeExpressions() .getValue(); // check if this directory path matches the PATH_FILTER_REGEX boolean pathFilterMatches = true; if (pathPattern != null) { Path reldir = path == null ? Paths.get(".") : Paths.get(path); if (remotePath != null) { reldir = Paths.get(remotePath).relativize(reldir); } if (reldir != null && !reldir.toString().isEmpty()) { if (!pathPattern.matcher(reldir.toString().replace("\\", "/")).matches()) { pathFilterMatches = false; } } } final FTPClient client = getClient(null); int count = 0; final FTPFile[] files; if (path == null || path.trim().isEmpty()) { files = client.listFiles("."); } else { files = client.listFiles(path); } if (files.length == 0 && path != null && !path.trim().isEmpty()) { // throw exception if directory doesn't exist final boolean cdSuccessful = setWorkingDirectory(path); if (!cdSuccessful) { throw new IOException("Cannot list files for non-existent directory " + path); } } for (final FTPFile file : files) { final String filename = file.getName(); if (filename.equals(".") || filename.equals("..")) { continue; } if (ignoreDottedFiles && filename.startsWith(".")) { continue; } final File newFullPath = new File(path, filename); final String newFullForwardPath = newFullPath.getPath().replace("\\", "/"); if (recurse && file.isDirectory()) { try { listing.addAll(getListing(newFullForwardPath, depth + 1, maxResults - count)); } catch (final IOException e) { logger.error( "Unable to get listing from " + newFullForwardPath + "; skipping this subdirectory"); } } // if is not a directory and is not a link and it matches // FILE_FILTER_REGEX - then let's add it if (!file.isDirectory() && !file.isSymbolicLink() && pathFilterMatches) { if (pattern == null || pattern.matcher(filename).matches()) { listing.add(newFileInfo(file, path)); count++; } } if (count >= maxResults) { break; } } return listing; } private FileInfo newFileInfo(final FTPFile file, String path) { if (file == null) { return null; } final File newFullPath = new File(path, file.getName()); final String newFullForwardPath = newFullPath.getPath().replace("\\", "/"); StringBuilder perms = new StringBuilder(); perms.append(file.hasPermission(FTPFile.USER_ACCESS, FTPFile.READ_PERMISSION) ? "r" : "-"); perms.append(file.hasPermission(FTPFile.USER_ACCESS, FTPFile.WRITE_PERMISSION) ? "w" : "-"); perms.append(file.hasPermission(FTPFile.USER_ACCESS, FTPFile.EXECUTE_PERMISSION) ? "x" : "-"); perms.append(file.hasPermission(FTPFile.GROUP_ACCESS, FTPFile.READ_PERMISSION) ? "r" : "-"); perms.append(file.hasPermission(FTPFile.GROUP_ACCESS, FTPFile.WRITE_PERMISSION) ? "w" : "-"); perms.append(file.hasPermission(FTPFile.GROUP_ACCESS, FTPFile.EXECUTE_PERMISSION) ? "x" : "-"); perms.append(file.hasPermission(FTPFile.WORLD_ACCESS, FTPFile.READ_PERMISSION) ? "r" : "-"); perms.append(file.hasPermission(FTPFile.WORLD_ACCESS, FTPFile.WRITE_PERMISSION) ? "w" : "-"); perms.append(file.hasPermission(FTPFile.WORLD_ACCESS, FTPFile.EXECUTE_PERMISSION) ? "x" : "-"); FileInfo.Builder builder = new FileInfo.Builder().filename(file.getName()) .fullPathFileName(newFullForwardPath).directory(file.isDirectory()).size(file.getSize()) .lastModifiedTime(file.getTimestamp().getTimeInMillis()).permissions(perms.toString()) .owner(file.getUser()).group(file.getGroup()); return builder.build(); } @Override public InputStream getInputStream(String remoteFileName) throws IOException { return getInputStream(remoteFileName, null); } @Override public InputStream getInputStream(final String remoteFileName, final FlowFile flowFile) throws IOException { final FTPClient client = getClient(null); InputStream in = client.retrieveFileStream(remoteFileName); if (in == null) { throw new IOException(client.getReplyString()); } return in; } @Override public void flush() throws IOException { final FTPClient client = getClient(null); client.completePendingCommand(); } @Override public FileInfo getRemoteFileInfo(final FlowFile flowFile, String path, String remoteFileName) throws IOException { final FTPClient client = getClient(flowFile); if (path == null) { int slashpos = remoteFileName.lastIndexOf('/'); if (slashpos >= 0 && !remoteFileName.endsWith("/")) { path = remoteFileName.substring(0, slashpos); remoteFileName = remoteFileName.substring(slashpos + 1); } else { path = ""; } } final FTPFile[] files = client.listFiles(path); FTPFile matchingFile = null; for (final FTPFile file : files) { if (file.getName().equalsIgnoreCase(remoteFileName)) { matchingFile = file; break; } } if (matchingFile == null) { return null; } return newFileInfo(matchingFile, path); } @Override public void ensureDirectoryExists(final FlowFile flowFile, final File directoryName) throws IOException { if (directoryName.getParent() != null && !directoryName.getParentFile().equals(new File(File.separator))) { ensureDirectoryExists(flowFile, directoryName.getParentFile()); } final String remoteDirectory = directoryName.getAbsolutePath().replace("\\", "/").replaceAll("^.\\:", ""); final FTPClient client = getClient(flowFile); final boolean cdSuccessful = setWorkingDirectory(remoteDirectory); if (!cdSuccessful) { logger.debug("Remote Directory {} does not exist; creating it", new Object[] { remoteDirectory }); if (client.makeDirectory(remoteDirectory)) { logger.debug("Created {}", new Object[] { remoteDirectory }); } else { throw new IOException("Failed to create remote directory " + remoteDirectory); } } } private String setAndGetWorkingDirectory(final String path) throws IOException { client.changeWorkingDirectory(homeDirectory); if (!client.changeWorkingDirectory(path)) { throw new ProcessException("Unable to change working directory to " + path); } return client.printWorkingDirectory(); } private boolean setWorkingDirectory(final String path) throws IOException { client.changeWorkingDirectory(homeDirectory); return client.changeWorkingDirectory(path); } private boolean resetWorkingDirectory() throws IOException { return client.changeWorkingDirectory(homeDirectory); } @Override public String put(final FlowFile flowFile, final String path, final String filename, final InputStream content) throws IOException { final FTPClient client = getClient(flowFile); final String fullPath; if (path == null) { fullPath = filename; } else { final String workingDir = setAndGetWorkingDirectory(path); fullPath = workingDir.endsWith("/") ? workingDir + filename : workingDir + "/" + filename; } String tempFilename = ctx.getProperty(TEMP_FILENAME).evaluateAttributeExpressions(flowFile).getValue(); if (tempFilename == null) { final boolean dotRename = ctx.getProperty(DOT_RENAME).asBoolean(); tempFilename = dotRename ? "." + filename : filename; } final boolean storeSuccessful = client.storeFile(tempFilename, content); if (!storeSuccessful) { throw new IOException("Failed to store file " + tempFilename + " to " + fullPath + " due to: " + client.getReplyString()); } final String lastModifiedTime = ctx.getProperty(LAST_MODIFIED_TIME).evaluateAttributeExpressions(flowFile) .getValue(); if (lastModifiedTime != null && !lastModifiedTime.trim().isEmpty()) { try { final DateFormat informat = new SimpleDateFormat(FILE_MODIFY_DATE_ATTR_FORMAT, Locale.US); final Date fileModifyTime = informat.parse(lastModifiedTime); final DateFormat outformat = new SimpleDateFormat(FTP_TIMEVAL_FORMAT, Locale.US); final String time = outformat.format(fileModifyTime); if (!client.setModificationTime(tempFilename, time)) { // FTP server probably doesn't support MFMT command logger.warn("Could not set lastModifiedTime on {} to {}", new Object[] { flowFile, lastModifiedTime }); } } catch (final Exception e) { logger.error("Failed to set lastModifiedTime on {} to {} due to {}", new Object[] { flowFile, lastModifiedTime, e }); } } final String permissions = ctx.getProperty(PERMISSIONS).evaluateAttributeExpressions(flowFile).getValue(); if (permissions != null && !permissions.trim().isEmpty()) { try { int perms = numberPermissions(permissions); if (perms >= 0) { if (!client.sendSiteCommand("chmod " + Integer.toOctalString(perms) + " " + tempFilename)) { logger.warn("Could not set permission on {} to {}", new Object[] { flowFile, permissions }); } } } catch (final Exception e) { logger.error("Failed to set permission on {} to {} due to {}", new Object[] { flowFile, permissions, e }); } } if (!filename.equals(tempFilename)) { try { logger.debug("Renaming remote path from {} to {} for {}", new Object[] { tempFilename, filename, flowFile }); final boolean renameSuccessful = client.rename(tempFilename, filename); if (!renameSuccessful) { throw new IOException("Failed to rename temporary file " + tempFilename + " to " + fullPath + " due to: " + client.getReplyString()); } } catch (final IOException e) { try { client.deleteFile(tempFilename); throw e; } catch (final IOException e1) { throw new IOException("Failed to rename temporary file " + tempFilename + " to " + fullPath + " and failed to delete it when attempting to clean up", e1); } } } return fullPath; } @Override public void rename(final String source, final String target) throws IOException { final FTPClient client = getClient(null); final boolean renameSuccessful = client.rename(source, target); if (!renameSuccessful) { throw new IOException("Failed to rename temporary file " + source + " to " + target + " due to: " + client.getReplyString()); } } @Override public void deleteFile(final String path, final String remoteFileName) throws IOException { final FTPClient client = getClient(null); if (path != null) { setWorkingDirectory(path); } if (!client.deleteFile(remoteFileName)) { throw new IOException("Failed to remove file " + remoteFileName + " due to " + client.getReplyString()); } } @Override public void deleteDirectory(final String remoteDirectoryName) throws IOException { final FTPClient client = getClient(null); final boolean success = client.removeDirectory(remoteDirectoryName); if (!success) { throw new IOException( "Failed to remove directory " + remoteDirectoryName + " due to " + client.getReplyString()); } } @Override public boolean isClosed() { return closed; } public void sendCommands(final List<String> commands, final FlowFile flowFile) throws IOException { if (commands.isEmpty()) { return; } final FTPClient client = getClient(flowFile); for (String cmd : commands) { if (!cmd.isEmpty()) { int result; result = client.sendCommand(cmd); logger.debug(this + " sent command to the FTP server: " + cmd + " for " + flowFile); if (FTPReply.isNegativePermanent(result) || FTPReply.isNegativeTransient(result)) { throw new IOException(this + " negative reply back from FTP server cmd: " + cmd + " reply:" + result + ": " + client.getReplyString() + " for " + flowFile); } } } } private FTPClient getClient(final FlowFile flowFile) throws IOException { if (client != null) { String desthost = ctx.getProperty(HOSTNAME).evaluateAttributeExpressions(flowFile).getValue(); if (remoteHostName.equals(desthost)) { // destination matches so we can keep our current session resetWorkingDirectory(); return client; } else { // this flowFile is going to a different destination, reset session close(); } } final Proxy.Type proxyType = Proxy.Type.valueOf(ctx.getProperty(PROXY_TYPE).getValue()); final String proxyHost = ctx.getProperty(PROXY_HOST).getValue(); final Integer proxyPort = ctx.getProperty(PROXY_PORT).asInteger(); FTPClient client; if (proxyType == Proxy.Type.HTTP) { client = new FTPHTTPClient(proxyHost, proxyPort, ctx.getProperty(HTTP_PROXY_USERNAME).getValue(), ctx.getProperty(HTTP_PROXY_PASSWORD).getValue()); } else { client = new FTPClient(); if (proxyType == Proxy.Type.SOCKS) { client.setSocketFactory(new SocksProxySocketFactory( new Proxy(proxyType, new InetSocketAddress(proxyHost, proxyPort)))); } } this.client = client; client.setDataTimeout(ctx.getProperty(DATA_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue()); client.setDefaultTimeout( ctx.getProperty(CONNECTION_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue()); client.setRemoteVerificationEnabled(false); final String remoteHostname = ctx.getProperty(HOSTNAME).evaluateAttributeExpressions(flowFile).getValue(); this.remoteHostName = remoteHostname; InetAddress inetAddress = null; try { inetAddress = InetAddress.getByAddress(remoteHostname, null); } catch (final UnknownHostException uhe) { } if (inetAddress == null) { inetAddress = InetAddress.getByName(remoteHostname); } client.connect(inetAddress, ctx.getProperty(PORT).evaluateAttributeExpressions(flowFile).asInteger()); this.closed = false; client.setDataTimeout(ctx.getProperty(DATA_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue()); client.setSoTimeout(ctx.getProperty(CONNECTION_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue()); final String username = ctx.getProperty(USERNAME).evaluateAttributeExpressions(flowFile).getValue(); final String password = ctx.getProperty(PASSWORD).evaluateAttributeExpressions(flowFile).getValue(); final boolean loggedIn = client.login(username, password); if (!loggedIn) { throw new IOException("Could not login for user '" + username + "'"); } final String connectionMode = ctx.getProperty(CONNECTION_MODE).getValue(); if (connectionMode.equalsIgnoreCase(CONNECTION_MODE_ACTIVE)) { client.enterLocalActiveMode(); } else { client.enterLocalPassiveMode(); } // additional FTPClientConfig ftpConfig = new FTPClientConfig(FTPClientConfig.SYST_UNIX); final String ftpClientConfig = ctx.getProperty(FTP_CLIENT_CONFIG_SYST).getValue(); if (ftpClientConfig.equalsIgnoreCase(FTP_CLIENT_CONFIG_SYST_UNIX)) { this.ftpConfig = ftpConfig; client.configure(ftpConfig); } else if (ftpClientConfig.equalsIgnoreCase(FTP_CLIENT_CONFIG_SYST_NT)) { this.ftpConfig = ftpConfig; client.configure(ftpConfig); } final String transferMode = ctx.getProperty(TRANSFER_MODE).getValue(); final int fileType = (transferMode.equalsIgnoreCase(TRANSFER_MODE_ASCII)) ? FTPClient.ASCII_FILE_TYPE : FTPClient.BINARY_FILE_TYPE; if (!client.setFileType(fileType)) { throw new IOException("Unable to set transfer mode to type " + transferMode); } this.homeDirectory = client.printWorkingDirectory(); return client; } protected int numberPermissions(String perms) { int number = -1; final Pattern rwxPattern = Pattern.compile("^[rwx-]{9}$"); final Pattern numPattern = Pattern.compile("\\d+"); if (rwxPattern.matcher(perms).matches()) { number = 0; if (perms.charAt(0) == 'r') { number |= 0x100; } if (perms.charAt(1) == 'w') { number |= 0x80; } if (perms.charAt(2) == 'x') { number |= 0x40; } if (perms.charAt(3) == 'r') { number |= 0x20; } if (perms.charAt(4) == 'w') { number |= 0x10; } if (perms.charAt(5) == 'x') { number |= 0x8; } if (perms.charAt(6) == 'r') { number |= 0x4; } if (perms.charAt(7) == 'w') { number |= 0x2; } if (perms.charAt(8) == 'x') { number |= 0x1; } } else if (numPattern.matcher(perms).matches()) { try { number = Integer.parseInt(perms, 8); } catch (NumberFormatException ignore) { } } return number; } }