List of usage examples for org.apache.commons.net.ftp FTPFile getTimestamp
public Calendar getTimestamp()
From source file:org.paxle.crawler.ftp.impl.FtpUrlConnection.java
private static void formatStdDirlisting(final OutputStream into, final FTPListParseEngine fileParseEngine) { final Formatter writer = new Formatter(into); FTPFile[] files;/* www .j ava 2s . c om*/ while (fileParseEngine.hasNext()) { files = fileParseEngine.getNext(16); for (final FTPFile file : files) { if (file == null) continue; // directory char c; switch (file.getType()) { case FTPFile.DIRECTORY_TYPE: c = 'd'; break; case FTPFile.SYMBOLIC_LINK_TYPE: c = 's'; break; default: c = '-'; break; } writer.format("%c", Character.valueOf(c)); // permissions for (final int access : FILE_ACCESS_MODES) { writer.format("%c%c%c", Character.valueOf(file.hasPermission(access, FTPFile.READ_PERMISSION) ? 'r' : '-'), Character.valueOf(file.hasPermission(access, FTPFile.WRITE_PERMISSION) ? 'w' : '-'), Character.valueOf(file.hasPermission(access, FTPFile.EXECUTE_PERMISSION) ? 'x' : '-')); } // other information writer.format(" %2d", Integer.valueOf(file.getHardLinkCount())); writer.format(" %8s", file.getUser()); writer.format(" %8s", file.getGroup()); writer.format(" %12d", Long.valueOf(file.getSize())); writer.format(" %1$tY-%1$tm-%1$td %1$tH:%1$tM", Long.valueOf(file.getTimestamp().getTimeInMillis())); writer.format(" %s", file.getName()); writer.format("%s", System.getProperty("line.separator")); } } writer.flush(); }
From source file:org.paxle.crawler.ftp.impl.FtpUrlConnection.java
@Override public long getLastModified() { if (!this.client.isConnected()) return 0l; long lastModTimestamp = 0l; try {//from www.j ava2s . c om if (this.isDirectory) { Calendar lastMod = null; FTPFile[] files = client.listFiles(path); for (FTPFile nextFile : files) { Calendar fileDate = nextFile.getTimestamp(); lastMod = (lastMod == null || fileDate.after(lastMod)) ? fileDate : lastMod; } if (lastMod != null) { lastModTimestamp = lastMod.getTimeInMillis(); } } else { FTPFile file = this.findFile(); if (file != null) { lastModTimestamp = file.getTimestamp().getTimeInMillis(); } } } catch (IOException e) { e.printStackTrace(); } return lastModTimestamp; }
From source file:org.punksearch.crawler.adapters.FtpAdapterTest.java
public void testGetModificationTime() { FTPFile file = getSomeFile(); assertEquals(file.getTimestamp().getTimeInMillis(), adapter.getModificationTime(new FtpItem(file, null))); }
From source file:org.ramadda.repository.type.FtpTypeHandler.java
/** * _more_/*from ww w . j a v a2s . c o m*/ * * @param request _more_ * @param parentEntry _more_ * @param id _more_ * * @return _more_ * * @throws Exception _more_ */ public Entry makeSynthEntry(Request request, Entry parentEntry, String id) throws Exception { Object[] values = parentEntry.getValues(); if (values == null) { return null; } String baseDir = (String) values[COL_BASEDIR]; String server = (String) values[COL_SERVER]; MyFTPFile myFtpFile = getFileFromId(parentEntry, id, baseDir); if (myFtpFile == null) { return null; } if (myFtpFile.path.equals(baseDir)) { return parentEntry; } FTPFile ftpFile = myFtpFile.ftpFile; TypeHandler handler = (ftpFile.isDirectory() ? getRepository().getTypeHandler(TypeHandler.TYPE_GROUP) : getRepository().getTypeHandler(TypeHandler.TYPE_FILE)); handler = this; String synthId = Repository.ID_PREFIX_SYNTH + parentEntry.getId() + ":" + id; boolean isDir = ftpFile.isDirectory(); Entry entry = (isDir ? (Entry) new Entry(synthId, handler, true) : new Entry(synthId, handler)); String name = IOUtil.getFileTail(ftpFile.getName()); entry.setIsLocalFile(true); Entry parent; if (myFtpFile.path.equals(baseDir)) { parent = (Entry) parentEntry; } else { File tmp = new File(myFtpFile.path); String parentPath = tmp.getParent().replace("\\", "/"); String parentId = getSynthId(parentEntry, baseDir, parentPath); if (parentPath.equals(baseDir)) { parent = (Entry) parentEntry; } else { parent = (Entry) getEntryManager().getEntry(request, parentId, false, false); } } double maxSize = 0; if (values[COL_MAXSIZE] != null) { maxSize = ((Double) values[COL_MAXSIZE]).doubleValue(); } long dttm = ftpFile.getTimestamp().getTime().getTime(); /* String datePattern = (String)values[COL_DATE_PATTERN]; //TODO cache the compiled patterns if(datePattern!=null && datePattern.length()>0) { Pattern p = Pattern.compile(datePattern); Matcher m = p.matcher(path); if(m.matches()) { String dateString = m.group(1); String dateFormat = (String)values[COL_DATE_FORMAT]; Date date=null; if(dateFormat!=null && dateFormat.length()==0) { date = new SimpleDateFormat(dateFormat).parse(dateString); } else { date = DateUtil.parse(dateString); } dttm = date.getTime(); } } */ Resource resource; if (isDir) { resource = new Resource("ftp://" + server + myFtpFile.path, Resource.TYPE_URL); } else { if (ftpFile.getSize() > 1000000 * maxSize) { resource = new Resource("ftp://" + server + myFtpFile.path, Resource.TYPE_URL); } else { resource = new Resource(name, Resource.TYPE_REMOTE_FILE); } resource.setFileSize(ftpFile.getSize()); } entry.initEntry(name, "", parent, getUserManager().getLocalFileUser(), resource, "", dttm, dttm, dttm, dttm, null); return entry; }
From source file:org.runmyprocess.sec.FTP.java
private void listFiles(JSONObject jsonObject) throws Exception { // get all files from server and store them in an array of // FTPFiles/*from w ww.j a v a2 s . c o m*/ JSONArray fileInfo = new JSONArray(); LOG.log("LIST directories: " + jsonObject.getString("path"), Level.INFO); String[] directories = this.client.listNames(jsonObject.getString("path")); LOG.log("LIST files: " + jsonObject.getString("path"), Level.INFO); FTPFile[] files = this.client.listFiles(jsonObject.getString("path")); LOG.log("LIST DONE", Level.INFO); for (FTPFile file : files) { if (file.getType() == FTPFile.FILE_TYPE) { JSONObject fileObject = new JSONObject(); fileObject.put("Name", file.getName()); fileObject.put("Size", FileUtils.byteCountToDisplaySize(file.getSize())); fileObject.put("Timestamp", file.getTimestamp().getTime().toString()); fileInfo.add(fileObject); } } response.setStatus(200);//sets the return status to 200 JSONObject resp = new JSONObject(); resp.put("files", fileInfo);//sends the info inside an object resp.put("directories", directories); response.setData(resp); }
From source file:org.shept.util.FtpFileCopy.java
/** * Compare the source path and the destination path for filenames with the filePattern * e.g. *.bak, *tmp and copy these files to the destination dir if they are not present at the * destination or if they have changed (by modifactionDate). * Copying is done through a retrieve operation. * /*from w w w . j a va 2s . c om*/ * @param ftpSource * @param localDestPat * @param filePattern * @return the number of files being copied * @throws IOException */ public Integer syncPull(FTPClient ftpSource, String localDestPat, String filePattern) throws IOException { // check for new files since the last check which need to be copied Integer number = 0; SortedMap<FileNameDate, File> destMap = FileUtils.fileMapByNameAndDate(localDestPat, filePattern); SortedMap<FileNameDate, FTPFile> sourceMap = fileMapByNameAndDate(ftpSource, filePattern); // identify the list of source files different from their destinations for (FileNameDate fk : destMap.keySet()) { sourceMap.remove(fk); } // copy the list of files from source to destination for (FTPFile file : sourceMap.values()) { logger.debug("Copying file " + file.getName() + ": " + file.getTimestamp()); File copy = new File(localDestPat, file.getName()); try { if (!copy.exists()) { // copy to tmp file first to avoid clashes during lengthy copy action File tmp = File.createTempFile("vrs", ".tmp", copy.getParentFile()); FileOutputStream writeStream = new FileOutputStream(tmp); boolean rc = ftpSource.retrieveFile(file.getName(), writeStream); writeStream.close(); if (rc) { rc = tmp.renameTo(copy); number++; } if (!rc) { tmp.delete(); // cleanup if we fail } } } catch (IOException ex) { logger.error("Ftp FileCopy did not succeed (using " + file.getName() + ")" + " FTP reported error " + ftpSource.getReplyString(), ex); } } return number; }
From source file:org.shept.util.FtpFileCopy.java
/** * List the file directory as specified by the name filter and the path directory * The maps keys contain name and modification date for comparison * @throws IOException /*w w w. jav a 2 s .com*/ */ public SortedMap<FileNameDate, FTPFile> fileMapByNameAndDate(FTPClient ftp, String filePattern) throws IOException { SortedMap<FileNameDate, FTPFile> newFileMap = new TreeMap<FileNameDate, FTPFile>(); FTPFile[] files = ftp.listFiles(); for (FTPFile ftpFile : files) { if (acceptFile(ftpFile, filePattern)) { ftpFile.getTimestamp(); FileNameDate fk = new FileNameDate(ftpFile.getName(), ftpFile.getTimestamp().getTimeInMillis()); newFileMap.put(fk, ftpFile); } } return newFileMap; }
From source file:org.sofun.platform.opta.ftp.FTPFileFilterImpl.java
@Override public boolean accept(FTPFile file) { if (file == null || filePattern == null) { return false; }/* w ww . j a v a 2s .c o m*/ final Pattern p = Pattern.compile(filePattern); final Matcher m = p.matcher(file.getName()); if (!m.find()) { // Filename does not match pattern. Do not include. return false; } if (service == null) { // No service available to filter further. Filename matches pattern: // do include. return true; } OptaProcessedFeed pfeed = service.getProcessedFeedFor(file.getName()); if (pfeed == null) { // File has not been processed yet: do include return true; } else { Date formerTimestamp = pfeed.getTimestamp(); if (formerTimestamp == null) { // No record of last processing: do include. return true; } if (formerTimestamp.compareTo(file.getTimestamp().getTime()) < 0) { // Former time stamp earlier than new one: do include return true; } } return false; }
From source file:org.sofun.platform.opta.impl.OptaServiceImpl.java
@Override public void syncFLiveFeeds() throws OptaException { final String liveFeedPattern = "srml-.*-.*-f.*-matchresults.xml"; int processed = 0; final FTPClientWrapper opta = getFTPClientWrapper(); try {/*from ww w . ja va 2 s . c o m*/ opta.connect(); final FTPFileFilter filter = new FTPFileFilterImpl(liveFeedPattern, this); for (FTPFile ftpFile : opta.getClient().listFiles(null, filter)) { if (processed > 50) { // batch to keep transactions small. // mostly an issue while bootstrapping new competitions with // lots of pending feeds to process. log.info("End of Football Live Feeds processing batch. Will resume in 5 minutes."); break; } File feed = getFileFromFTP(opta, ftpFile.getName()); F07Parser f7p = f7Sync(feed); if (f7p.getGame() != null) { log.info("Syncing file with name=" + ftpFile.getName()); setFileAsProcessed(ftpFile.getName(), ftpFile.getTimestamp().getTime()); processed += 1; } else { log.warn("Could not find game for live feed with name=" + ftpFile.getName()); } } } catch (IOException e) { throw new OptaException(e.getMessage()); } finally { opta.disconnect(); } }
From source file:org.sofun.platform.opta.impl.OptaServiceImpl.java
/** * Synchronize all F feeds for a given competition's season (F1, F3 and F40) * //from www.j a va2 s . c o m * <p> * * This method is responsible for the actual update of feeds if they do * require it. * * @param competitionId: the competition ID * @param season: the season label. * @return the amount processed files. * @throws OptaException */ private int syncFComp(String competitionId, String season) throws OptaException { final String f1 = "srml-" + competitionId + "-" + season + "-results.xml"; final String f3 = "srml-" + competitionId + "-" + season + "-standings.xml"; final String f40 = "srml-" + competitionId + "-" + season + "-squads.xml"; int processed = 0; final FTPClientWrapper opta = getFTPClientWrapper(); try { opta.connect(); // Processing F40 (order matters for first run) OptaProcessedFeed pfeed = getProcessedFeedFor(f40); FTPFile ftpFile = null; boolean process = false; FTPFile[] ftpfiles; try { ftpfiles = opta.getClient().listFiles(f40); } catch (IOException e) { throw new OptaException(e.getMessage()); } if (ftpfiles.length == 1) { ftpFile = ftpfiles[0]; } if (pfeed == null) { process = true; } else { if (ftpFile != null) { final Date formerFeedTimestamp = pfeed.getTimestamp(); final Date newFeedTimestamp = ftpFile.getTimestamp().getTime(); if (newFeedTimestamp == null || newFeedTimestamp.compareTo(formerFeedTimestamp) > 0) { process = true; } else { log.debug("Feed has not changed. No need to reprocess feed=" + f40); } } else { log.warn("Feed with name=" + f40 + " cannot be found"); } } if (process && ftpFile != null) { File feed = getFileFromFTP(opta, f40); if (feed != null) { log.info("Syncing file with name=" + f40); f40Sync(feed); setFileAsProcessed(f40, ftpFile.getTimestamp().getTime()); processed += 1; } else { log.warn("Feed not available:" + f40); } } // Processing F1 pfeed = getProcessedFeedFor(f1); ftpFile = null; process = false; try { ftpfiles = opta.getClient().listFiles(f1); } catch (IOException e) { throw new OptaException(e.getMessage()); } if (ftpfiles.length == 1) { ftpFile = ftpfiles[0]; } if (pfeed == null) { process = true; } else { if (ftpFile != null) { final Date formerFeedTimestamp = pfeed.getTimestamp(); final Date newFeedTimestamp = ftpFile.getTimestamp().getTime(); if (newFeedTimestamp == null || newFeedTimestamp.compareTo(formerFeedTimestamp) > 0) { process = true; } else { log.debug("Feed has not changed. No need to reprocess feed=" + f1); } } else { log.warn("Feed with name=" + f1 + " cannot be found"); } } if (process && ftpFile != null) { File feed = getFileFromFTP(opta, f1); if (feed != null) { log.info("Syncing file with name=" + f1); f1Sync(feed); setFileAsProcessed(f1, ftpFile.getTimestamp().getTime()); processed += 1; } else { log.warn("Feed not available:" + f1); } } // Processing F3 pfeed = getProcessedFeedFor(f3); process = false; ftpFile = null; try { ftpfiles = opta.getClient().listFiles(f3); } catch (IOException e) { throw new OptaException(e.getMessage()); } if (ftpfiles.length == 1) { ftpFile = ftpfiles[0]; } if (pfeed == null) { process = true; } else { if (ftpFile != null) { final Date formerFeedTimestamp = pfeed.getTimestamp(); final Date newFeedTimestamp = ftpFile.getTimestamp().getTime(); if (newFeedTimestamp == null || newFeedTimestamp.compareTo(formerFeedTimestamp) > 0) { process = true; } else { log.debug("Feed has not changed. No need to reprocess feed=" + f3); } } else { log.warn("Feed with name=" + f3 + " cannot be found"); } } if (process && ftpFile != null) { File feed = getFileFromFTP(opta, f3); if (feed != null) { log.info("Syncing file with name=" + f3); f3Sync(feed); setFileAsProcessed(f3, ftpFile.getTimestamp().getTime()); processed += 1; } else { log.warn("Feed not available:" + f3); } } } finally { opta.disconnect(); } return processed; }