List of usage examples for org.apache.commons.net.ftp FTPClient listFiles
public FTPFile[] listFiles(String pathname) throws IOException
From source file:fr.ibp.nifi.processors.IBPFTPTransfer.java
private List<FileInfo> getListing(final String path, final int depth, final int maxResults, final Integer maxDepth) throws IOException { final List<FileInfo> listing = new ArrayList<>(); if (maxResults < 1) { return listing; }//from w ww.j ava 2 s .c o m if (depth >= 100) { logger.warn(this + " had to stop recursively searching directories at a recursive depth of " + depth + " to avoid memory issues"); return listing; } final boolean ignoreDottedFiles = ctx.getProperty(FileTransfer.IGNORE_DOTTED_FILES).asBoolean(); final boolean recurse = ctx.getProperty(FileTransfer.RECURSIVE_SEARCH).asBoolean(); final String fileFilterRegex = ctx.getProperty(FileTransfer.FILE_FILTER_REGEX).getValue(); final Pattern pattern = (fileFilterRegex == null) ? null : Pattern.compile(fileFilterRegex); final String pathFilterRegex = ctx.getProperty(FileTransfer.PATH_FILTER_REGEX).getValue(); final Pattern pathPattern = (!recurse || pathFilterRegex == null) ? null : Pattern.compile(pathFilterRegex); final String remotePath = ctx.getProperty(FileTransfer.REMOTE_PATH).evaluateAttributeExpressions() .getValue(); logger.info(String.format("path : %s", path)); logger.info(String.format("pathPattern : %s", pathPattern)); // check if this directory path matches the PATH_FILTER_REGEX boolean pathFilterMatches = true; if (pathPattern != null) { Path reldir = path == null ? Paths.get(".") : Paths.get(path); if (remotePath != null) { reldir = Paths.get(remotePath).relativize(reldir); } if (reldir != null && !reldir.toString().isEmpty()) { if (!pathPattern.matcher(reldir.toString().replace("\\", "/")).matches()) { pathFilterMatches = false; } } } logger.info(String.format("pathFilterMatches : %s", pathFilterMatches)); final FTPClient client = getClient(null); int count = 0; final FTPFile[] files; if (path == null || path.trim().isEmpty()) { files = client.listFiles("."); } else { files = client.listFiles(path); } if (files.length == 0 && path != null && !path.trim().isEmpty()) { // throw exception if directory doesn't exist final boolean cdSuccessful = setWorkingDirectory(path); if (!cdSuccessful) { throw new IOException("Cannot list files for non-existent directory " + path); } } for (final FTPFile file : files) { final String filename = file.getName(); if (filename.equals(".") || filename.equals("..")) { continue; } if (ignoreDottedFiles && filename.startsWith(".")) { continue; } final File newFullPath = new File(path, filename); final String newFullForwardPath = newFullPath.getPath().replace("\\", "/"); if (file.isDirectory()) { logger.info("PATH: {} ", new Object[] { newFullForwardPath }); // Repertoire if (maxDepth != null) { // Si la profondeur de recherche est dfinieoct@ve12 int level = maxDepth.intValue() - 1; if (depth == level) { logger.info("depth == level"); boolean matches = true; if (pathPattern != null) { Path reldir = path == null ? Paths.get(".") : Paths.get(newFullForwardPath); if (remotePath != null) { reldir = Paths.get(remotePath).relativize(reldir); } if (reldir != null && !reldir.toString().isEmpty()) { if (!pathPattern.matcher(reldir.toString().replace("\\", "/")).matches()) { matches = false; } } } if (pathPattern == null || matches) { try { logger.info("going into depth depth:{} and maxDepth: {} ", new Object[] { depth, maxDepth }); listing.addAll( getListing(newFullForwardPath, depth + 1, maxResults - count, maxDepth)); } catch (final IOException e) { logger.error("Unable to get listing from " + newFullForwardPath + "; skipping this subdirectory"); throw e; } } } else if (depth < level) { logger.info("depth < level"); try { logger.info("going into depth depth:{} and maxDepth: {} ", new Object[] { depth, maxDepth }); listing.addAll(getListing(newFullForwardPath, depth + 1, maxResults - count, maxDepth)); } catch (final IOException e) { logger.error("Unable to get listing from " + newFullForwardPath + "; skipping this subdirectory"); throw e; } } } else if (recurse) { logger.info("MAxDepth null and recurse = true"); try { logger.info("Recurse mode depth depth:{}", new Object[] { depth }); listing.addAll(getListing(newFullForwardPath, depth + 1, maxResults - count, maxDepth)); } catch (final IOException e) { logger.error("Unable to get listing from " + newFullForwardPath + "; skipping this subdirectory"); throw e; } } } /* * if ((file.isDirectory() && pathFilterMatches && maxDepth!=null && * depth<=maxDepth)) { try { logger.info( * "going into depth depth:{} and maxDepth: {} ",new * Object[]{depth,maxDepth}); * listing.addAll(getListing(newFullForwardPath, depth + 1, * maxResults - count, maxDepth)); } catch (final IOException e) { * logger.error("Unable to get listing from " + newFullForwardPath + * "; skipping this subdirectory"); throw e; } }else{ if ((recurse * && file.isDirectory()) ) { try { logger.info( * "Recurse mode depth depth:{}",new Object[]{depth}); * listing.addAll(getListing(newFullForwardPath, depth + 1, * maxResults - count, maxDepth)); } catch (final IOException e) { * logger.error("Unable to get listing from " + newFullForwardPath + * "; skipping this subdirectory"); throw e; } } } */ // if is not a directory and is not a link and it matches // FILE_FILTER_REGEX - then let's add it if (!file.isDirectory() && !file.isSymbolicLink() && pathFilterMatches) { if (pattern == null || pattern.matcher(filename).matches()) { logger.info(String.format("Ajout du fichier %s/%s", path, file.getName())); listing.add(newFileInfo(file, path)); count++; logger.info(String.format("Nb fichiers retenus %s", count)); } } if (count >= maxResults) { break; } } return listing; }
From source file:net.siegmar.japtproxy.fetcher.FetcherFtp.java
/** * {@inheritDoc}//from w w w .ja va2 s . co m */ @Override public FetchedResourceFtp fetch(final URL targetResource, final long lastModified, final String originalUserAgent) throws IOException, ResourceUnavailableException { final FTPClient ftpClient = new FTPClient(); ftpClient.setSoTimeout(socketTimeout); ftpClient.setDataTimeout(dataTimeout); try { final String host = targetResource.getHost(); final String resourceName = targetResource.getPath(); LOG.debug("Configured FetcherFtp: Host '{}', Resource '{}'", host, resourceName); ftpClient.connect(host); ftpClient.enterLocalPassiveMode(); if (!ftpClient.login("anonymous", "japt-proxy")) { throw new IOException("Can't login to FTP server"); } ftpClient.setFileType(FTP.BINARY_FILE_TYPE); final FTPFile[] files = ftpClient.listFiles(resourceName); if (files.length == 0) { throw new ResourceUnavailableException("Resource '" + resourceName + "' not found"); } if (files.length > 1) { throw new IOException("Multiple files found"); } final FTPFile file = files[0]; final FetchedResourceFtp fetchedResourceFtp = new FetchedResourceFtp(ftpClient, file); fetchedResourceFtp .setModified(lastModified == 0 || lastModified < file.getTimestamp().getTimeInMillis()); return fetchedResourceFtp; } catch (final IOException e) { // Closing only in case of an exception - otherwise closed by FetchedResourceFtp if (ftpClient.isConnected()) { ftpClient.disconnect(); } throw e; } }
From source file:com.clickha.nifi.processors.util.FTPTransferV2.java
private List<FileInfo> getListing(final String path, final int depth, final int maxResults) throws IOException { final List<FileInfo> listing = new ArrayList<>(); if (maxResults < 1) { return listing; }// w w w .j av a2s .c o m if (depth >= 100) { logger.warn(this + " had to stop recursively searching directories at a recursive depth of " + depth + " to avoid memory issues"); return listing; } final boolean ignoreDottedFiles = ctx.getProperty(FileTransferV2.IGNORE_DOTTED_FILES).asBoolean(); final boolean recurse = ctx.getProperty(FileTransferV2.RECURSIVE_SEARCH).asBoolean(); final String fileFilterRegex = ctx.getProperty(FileTransferV2.FILE_FILTER_REGEX).getValue(); final Pattern pattern = (fileFilterRegex == null) ? null : Pattern.compile(fileFilterRegex); final String pathFilterRegex = ctx.getProperty(FileTransferV2.PATH_FILTER_REGEX).getValue(); final Pattern pathPattern = (!recurse || pathFilterRegex == null) ? null : Pattern.compile(pathFilterRegex); final String remotePath = ctx.getProperty(FileTransferV2.REMOTE_PATH).evaluateAttributeExpressions() .getValue(); // check if this directory path matches the PATH_FILTER_REGEX boolean pathFilterMatches = true; if (pathPattern != null) { Path reldir = path == null ? Paths.get(".") : Paths.get(path); if (remotePath != null) { reldir = Paths.get(remotePath).relativize(reldir); } if (reldir != null && !reldir.toString().isEmpty()) { if (!pathPattern.matcher(reldir.toString().replace("\\", "/")).matches()) { pathFilterMatches = false; } } } final FTPClient client = getClient(null); int count = 0; final FTPFile[] files; if (path == null || path.trim().isEmpty()) { files = client.listFiles("."); } else { files = client.listFiles(path); } if (files.length == 0 && path != null && !path.trim().isEmpty()) { // throw exception if directory doesn't exist final boolean cdSuccessful = setWorkingDirectory(path); if (!cdSuccessful) { throw new IOException("Cannot list files for non-existent directory " + path); } } for (final FTPFile file : files) { final String filename = file.getName(); if (filename.equals(".") || filename.equals("..")) { continue; } if (ignoreDottedFiles && filename.startsWith(".")) { continue; } final File newFullPath = new File(path, filename); final String newFullForwardPath = newFullPath.getPath().replace("\\", "/"); if (recurse && file.isDirectory()) { try { listing.addAll(getListing(newFullForwardPath, depth + 1, maxResults - count)); } catch (final IOException e) { logger.error( "Unable to get listing from " + newFullForwardPath + "; skipping this subdirectory"); } } // if is not a directory and is not a link and it matches // FILE_FILTER_REGEX - then let's add it if (!file.isDirectory() && !file.isSymbolicLink() && pathFilterMatches) { if (pattern == null || pattern.matcher(filename).matches()) { listing.add(newFileInfo(file, path)); count++; } } if (count >= maxResults) { break; } } return listing; }
From source file:com.seajas.search.contender.service.modifier.ArchiveModifierService.java
/** * Test an archive connection by logging onto the given URL and retrieving a directory listing. * //from ww w . j a v a 2 s . c om * @param uri * @return boolean */ public boolean testConnection(final URI uri) { if (!uri.getScheme().equalsIgnoreCase("ftp") && !uri.getScheme().equalsIgnoreCase("ftps")) { logger.error("Archive URL " + uri + " using protocol '" + uri.getScheme() + "' is not supported"); return false; } FTPClient ftpClient = retrieveFtpClient(uri); if (ftpClient != null) { try { logger.info("Retrieving testing content for archive with URI " + uri); // Retrieve a file listing ftpClient.listFiles(uri.getPath()); // Disconnect the client ftpClient.disconnect(); } catch (IOException e) { logger.error("Could not perform all FTP operations during archive testing", e); return false; } return true; } else return false; }
From source file:lucee.runtime.tag.Ftp.java
/** * List data of a ftp connection// w ww.ja va 2 s. c om * @return FTPCLient * @throws PageException * @throws IOException */ private FTPClient actionListDir() throws PageException, IOException { required("name", name); required("directory", directory); FTPClient client = getClient(); FTPFile[] files = client.listFiles(directory); if (files == null) files = new FTPFile[0]; String[] cols = new String[] { "attributes", "isdirectory", "lastmodified", "length", "mode", "name", "path", "url", "type", "raw" }; String[] types = new String[] { "VARCHAR", "BOOLEAN", "DATE", "DOUBLE", "VARCHAR", "VARCHAR", "VARCHAR", "VARCHAR", "VARCHAR", "VARCHAR" }; lucee.runtime.type.Query query = new QueryImpl(cols, types, 0, "query"); // translate directory path for display if (directory.length() == 0) directory = "/"; else if (directory.startsWith("./")) directory = directory.substring(1); else if (directory.charAt(0) != '/') directory = '/' + directory; if (directory.charAt(directory.length() - 1) != '/') directory = directory + '/'; pageContext.setVariable(name, query); int row = 0; for (int i = 0; i < files.length; i++) { FTPFile file = files[i]; if (file.getName().equals(".") || file.getName().equals("..")) continue; query.addRow(); row++; query.setAt("attributes", row, ""); query.setAt("isdirectory", row, Caster.toBoolean(file.isDirectory())); query.setAt("lastmodified", row, new DateTimeImpl(file.getTimestamp())); query.setAt("length", row, Caster.toDouble(file.getSize())); query.setAt("mode", row, FTPConstant.getPermissionASInteger(file)); query.setAt("type", row, FTPConstant.getTypeAsString(file.getType())); //query.setAt("permission",row,FTPConstant.getPermissionASInteger(file)); query.setAt("raw", row, file.getRawListing()); query.setAt("name", row, file.getName()); query.setAt("path", row, directory + file.getName()); query.setAt("url", row, "ftp://" + client.getRemoteAddress().getHostName() + "" + directory + file.getName()); } writeCfftp(client); return client; }
From source file:com.clickha.nifi.processors.util.FTPTransferV2.java
@Override public FileInfo getRemoteFileInfo(final FlowFile flowFile, String path, String remoteFileName) throws IOException { final FTPClient client = getClient(flowFile); if (path == null) { int slashpos = remoteFileName.lastIndexOf('/'); if (slashpos >= 0 && !remoteFileName.endsWith("/")) { path = remoteFileName.substring(0, slashpos); remoteFileName = remoteFileName.substring(slashpos + 1); } else {//from w w w. j av a2 s . co m path = ""; } } final FTPFile[] files = client.listFiles(path); FTPFile matchingFile = null; for (final FTPFile file : files) { if (file.getName().equalsIgnoreCase(remoteFileName)) { matchingFile = file; break; } } if (matchingFile == null) { return null; } return newFileInfo(matchingFile, path); }
From source file:fr.bmartel.speedtest.SpeedTestTask.java
/** * Get FTP file size./*www . j a va2 s . c om*/ * * @param ftpClient ftp client * @param filePath remote file path * @return file size * @throws Exception file read/write IOException */ private long getFileSize(final FTPClient ftpClient, final String filePath) throws IOException { long fileSize = 0; final FTPFile[] files = ftpClient.listFiles(filePath); if (files.length == 1 && files[0].isFile()) { fileSize = files[0].getSize(); } return fileSize; }
From source file:lucee.runtime.tag.Ftp.java
private FTPFile existsFile(FTPClient client, String strPath, boolean isFile) throws PageException, IOException { strPath = strPath.trim();/* w w w . j av a 2 s . c om*/ if (strPath.equals("/")) { FTPFile file = new FTPFile(); file.setName("/"); file.setType(FTPFile.DIRECTORY_TYPE); return file; } // get parent path FTPPath path = new FTPPath(client.printWorkingDirectory(), strPath); String p = path.getPath(); String n = path.getName(); strPath = p; if ("//".equals(p)) strPath = "/"; if (isFile) strPath += n; // when directory FTPFile[] files = null; try { files = client.listFiles(strPath); } catch (IOException e) { } if (files != null) { for (int i = 0; i < files.length; i++) { if (files[i].getName().equalsIgnoreCase(n)) { return files[i]; } } } return null; }
From source file:base.BasePlayer.AddGenome.java
static void updateEnsemblList() { try {//from www .j a va2 s .c o m menu = new JPopupMenu(); area = new JTextArea(); menuscroll = new JScrollPane(); area.setFont(Main.menuFont); menu.add(menuscroll); //menu.setPreferredSize(new Dimension(menu.getFontMetrics(Main.menuFont).stringWidth("0000000000000000000000000000000000000000000000000")+Main.defaultFontSize*10, (int)menu.getFontMetrics(Main.menuFont).getHeight()*4)); menu.setPreferredSize(new Dimension(300, 200)); //area.setMaximumSize(new Dimension(300, 600)); //area.setLineWrap(true); //area.setWrapStyleWord(true); //area.setPreferredSize(new Dimension(300,200)); area.revalidate(); menuscroll.getViewport().add(area); menu.pack(); menu.show(AddGenome.treescroll, 0, 0); /* area.addMouseListener(new MouseListener() { @Override public void mouseClicked(MouseEvent arg0) { // TODO Auto-generated method stub } @Override public void mouseEntered(MouseEvent arg0) { // TODO Auto-generated method stub } @Override public void mouseExited(MouseEvent arg0) { // TODO Auto-generated method stub } @Override public void mousePressed(MouseEvent arg0) { StringBuffer buf = new StringBuffer(""); for(int i= 0; i<(int)(Math.random()*100); i++) { buf.append("O"); } AddGenome.area.append(buf.toString() +"\n"); AddGenome.area.setCaretPosition(AddGenome.area.getText().length()); AddGenome.area.revalidate(); } @Override public void mouseReleased(MouseEvent arg0) { // TODO Auto-generated method stub } });*/ FTPClient f = new FTPClient(); news = new ArrayList<String[]>(); area.append("Connecting to Ensembl...\n"); //System.out.println("Connecting to Ensembl..."); f.connect("ftp.ensembl.org"); f.enterLocalPassiveMode(); f.login("anonymous", ""); //System.out.println("Connected."); area.append("Connected.\n"); FTPFile[] files = f.listFiles("pub"); String releasedir = ""; String releasenro; for (int i = 0; i < files.length; i++) { if (files[i].isDirectory() && files[i].getName().contains("release")) { releasedir = files[i].getName(); } } files = f.listFiles("pub/" + releasedir + "/fasta/"); releasenro = releasedir.substring(releasedir.indexOf("-") + 1); area.append("Searching for new genomes"); for (int i = 0; i < files.length; i++) { if (files[i].isDirectory()) { FTPFile[] fastafiles = f .listFiles("pub/" + releasedir + "/fasta/" + files[i].getName() + "/dna/"); String[] urls = new String[5]; for (int j = 0; j < fastafiles.length; j++) { if (fastafiles[j].getName().contains(".dna.toplevel.")) { urls[0] = "ftp://ftp.ensembl.org/pub/" + releasedir + "/fasta/" + files[i].getName() + "/dna/" + fastafiles[j].getName(); String filePath = "/pub/" + releasedir + "/fasta/" + files[i].getName() + "/dna/" + fastafiles[j].getName(); f.sendCommand("SIZE", filePath); String reply = f.getReplyString().split("\\s+")[1]; urls[1] = reply; break; } } if (urls[0] == null) { continue; } FTPFile[] annofiles = f.listFiles("pub/" + releasedir + "/gff3/" + files[i].getName()); for (int j = 0; j < annofiles.length; j++) { if (annofiles[j].getName().contains("." + releasenro + ".gff3.gz")) { urls[2] = "ftp://ftp.ensembl.org/pub/" + releasedir + "/gff3/" + files[i].getName() + "/" + annofiles[j].getName(); String filePath = "/pub/" + releasedir + "/gff3/" + files[i].getName() + "/" + annofiles[j].getName(); f.sendCommand("SIZE", filePath); String reply = f.getReplyString().split("\\s+")[1]; urls[3] = reply; break; } } if (urls[2] == null) { continue; } if (files[i].getName().contains("homo_sapiens")) { urls[4] = "http://hgdownload.cse.ucsc.edu/goldenPath/hg19/database/cytoBand.txt.gz"; } else if (files[i].getName().contains("mus_musculus")) { urls[4] = "http://hgdownload.cse.ucsc.edu/goldenPath/mm10/database/cytoBand.txt.gz"; } String name = urls[0].substring(urls[0].lastIndexOf("/") + 1, urls[0].indexOf(".dna.")); //System.out.print(urls[0]+"\t" +urls[1] +"\t" +urls[2] +"\t" +urls[3]); if (genomeHash.containsKey(name) || AddGenome.removables.contains(name)) { //System.out.println(name +" already in the list."); area.append("."); } else { area.append("\nNew genome " + name + " added.\n"); AddGenome.area.setCaretPosition(AddGenome.area.getText().length()); AddGenome.area.revalidate(); //System.out.println("New reference " +name +" found."); organisms.add(name); news.add(urls); if (urls[4] != null) { //System.out.println(urls[0] +" " + urls[2] +" " +urls[4]); URL[] newurls = { new URL(urls[0]), new URL(urls[2]), new URL(urls[4]) }; genomeHash.put(name, newurls); } else { URL[] newurls = { new URL(urls[0]), new URL(urls[2]) }; genomeHash.put(name, newurls); } Integer[] sizes = { Integer.parseInt(urls[1]), Integer.parseInt(urls[3]) }; sizeHash.put(name, sizes); } /*if(urls[4] != null) { System.out.print("\t" +urls[4]); } System.out.println(); */ } } checkGenomes(); if (news.size() > 0) { try { //File file = new File(); FileWriter fw = new FileWriter(Main.genomeDir.getCanonicalPath() + "/ensembl_fetched.txt"); BufferedWriter bw = new BufferedWriter(fw); for (int i = 0; i < news.size(); i++) { for (int j = 0; j < news.get(i).length; j++) { if (news.get(i)[j] == null) { break; } if (j > 0) { bw.write("\t"); } bw.write(news.get(i)[j]); } bw.write("\n"); } bw.close(); fw.close(); } catch (IOException e) { e.printStackTrace(); } } } catch (Exception e) { Main.showError(e.getMessage(), "Error"); e.printStackTrace(); } }
From source file:co.cask.hydrator.action.ftp.FTPCopyAction.java
@Override public void run(ActionContext context) throws Exception { Path destination = new Path(config.getDestDirectory()); FileSystem fileSystem = FileSystem.get(new Configuration()); destination = fileSystem.makeQualified(destination); if (!fileSystem.exists(destination)) { fileSystem.mkdirs(destination);// ww w . j a va 2s . c om } FTPClient ftp; if ("ftp".equals(config.getProtocol().toLowerCase())) { ftp = new FTPClient(); } else { ftp = new FTPSClient(); } ftp.setControlKeepAliveTimeout(5); // UNIX type server FTPClientConfig ftpConfig = new FTPClientConfig(); // Set additional parameters required for the ftp // for example config.setServerTimeZoneId("Pacific/Pitcairn") ftp.configure(ftpConfig); try { ftp.connect(config.getHost(), config.getPort()); ftp.enterLocalPassiveMode(); String replyString = ftp.getReplyString(); LOG.info("Connected to server {} and port {} with reply from connect as {}.", config.getHost(), config.getPort(), replyString); // Check the reply code for actual success int replyCode = ftp.getReplyCode(); if (!FTPReply.isPositiveCompletion(replyCode)) { ftp.disconnect(); throw new RuntimeException(String.format("FTP server refused connection with code %s and reply %s.", replyCode, replyString)); } if (!ftp.login(config.getUserName(), config.getPassword())) { LOG.error("login command reply code {}, {}", ftp.getReplyCode(), ftp.getReplyString()); ftp.logout(); throw new RuntimeException(String.format( "Login to the FTP server %s and port %s failed. " + "Please check user name and password.", config.getHost(), config.getPort())); } FTPFile[] ftpFiles = ftp.listFiles(config.getSrcDirectory()); LOG.info("listFiles command reply code: {}, {}.", ftp.getReplyCode(), ftp.getReplyString()); // Check the reply code for listFiles call. // If its "522 Data connections must be encrypted" then it means data channel also need to be encrypted if (ftp.getReplyCode() == 522 && "sftp".equalsIgnoreCase(config.getProtocol())) { // encrypt data channel and listFiles again ((FTPSClient) ftp).execPROT("P"); LOG.info("Attempting command listFiles on encrypted data channel."); ftpFiles = ftp.listFiles(config.getSrcDirectory()); } for (FTPFile file : ftpFiles) { String source = config.getSrcDirectory() + "/" + file.getName(); LOG.info("Current file {}, source {}", file.getName(), source); if (config.getExtractZipFiles() && file.getName().endsWith(".zip")) { copyZip(ftp, source, fileSystem, destination); } else { Path destinationPath = fileSystem.makeQualified(new Path(destination, file.getName())); LOG.debug("Downloading {} to {}", file.getName(), destinationPath.toString()); try (OutputStream output = fileSystem.create(destinationPath)) { InputStream is = ftp.retrieveFileStream(source); ByteStreams.copy(is, output); } } if (!ftp.completePendingCommand()) { LOG.error("Error completing command."); } } ftp.logout(); } finally { if (ftp.isConnected()) { try { ftp.disconnect(); } catch (Throwable e) { LOG.error("Failure to disconnect the ftp connection.", e); } } } }