Java tutorial
/** * Copyright (C) 2011 Brian Ferris <bdferris@onebusaway.org> * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.onebusaway.transit_data_federation.impl.realtime.orbcad; import java.io.IOException; import java.io.InputStream; import java.net.SocketException; import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import org.apache.commons.net.ftp.FTPClient; import org.apache.commons.net.ftp.FTPFile; import org.apache.commons.net.ftp.FTPListParseEngine; import org.apache.commons.net.ftp.FTPReply; import org.onebusaway.csv_entities.CsvEntityReader; import org.onebusaway.csv_entities.schema.AnnotationDrivenEntitySchemaFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.jmx.export.annotation.ManagedAttribute; import org.springframework.jmx.export.annotation.ManagedResource; @ManagedResource("org.onebusaway.transit_data_federation.impl.realtime.orbcad:name=OrbcadRecordFtpSource") public class OrbcadRecordFtpSource extends AbstractOrbcadRecordSource { private static final int TIMEOUT_IN_SECONDS = 10; private static Logger _log = LoggerFactory.getLogger(OrbcadRecordFtpSource.class); private Set<String> _paths = new HashSet<String>(); private CsvEntityReader _reader; private FTPClient _ftpClient = null; private FtpDataSource _dataSource; private String _dataDirectory; private int _maxDownloadCount = 1; private transient int _totalFtpFiles = 0; private transient int _newFtpFiles = 0; public void setDataSource(FtpDataSource dataSource) { _dataSource = dataSource; } public void setDataDirectory(String dataDirectory) { _dataDirectory = dataDirectory; } /**** * JMX Attributes ***/ @ManagedAttribute public int getTotalFtpFiles() { return _totalFtpFiles; } @ManagedAttribute public int getNewFtpFiles() { return _newFtpFiles; } /**** * Setup and Teardown ****/ @PostConstruct public void start() throws SocketException, IOException { _log.info("starting orbcad ftp download client"); super.start(); } @PreDestroy public void stop() throws IOException { _log.info("stopping orbcad ftp download client"); super.stop(); if (_ftpClient != null) _ftpClient.disconnect(); } /**** * Private Methods ****/ @Override protected void setup() { _reader = new CsvEntityReader(); AnnotationDrivenEntitySchemaFactory entitySchemaFactory = new AnnotationDrivenEntitySchemaFactory(); entitySchemaFactory.addEntityClass(OrbcadRecord.class); _reader.setEntitySchemaFactory(entitySchemaFactory); _reader.addEntityHandler(new RecordHandler()); } @Override protected synchronized void handleRefresh() throws IOException { try { if (_ftpClient == null) reconnectFtp(); List<String> toDownload = getUpdatedFilesToDownload(); downloadUpdatedFiles(toDownload); } catch (IOException ex) { _log.error("error refreshing avl files", ex); disconnectFtpClient(); } } private void reconnectFtp() throws SocketException, IOException { _log.info("attempting to establish ftp connection"); disconnectFtpClient(); _ftpClient = new FTPClient(); _ftpClient.setConnectTimeout(TIMEOUT_IN_SECONDS * 1000); _ftpClient.setDataTimeout(TIMEOUT_IN_SECONDS * 1000); _ftpClient.setDefaultTimeout(TIMEOUT_IN_SECONDS * 1000); _ftpClient.connect(_dataSource.getServername(), _dataSource.getPort()); _ftpClient.login(_dataSource.getUsername(), _dataSource.getPassword()); _ftpClient.enterLocalPassiveMode(); _log.info("ftp connection established"); } private List<String> getUpdatedFilesToDownload() throws IOException { long t1 = System.currentTimeMillis(); FTPListParseEngine engine = _ftpClient.initiateListParsing(_dataDirectory); Set<String> paths = new HashSet<String>(); List<String> toDownload = new ArrayList<String>(); while (engine.hasNext()) { FTPFile[] files = engine.getNext(25); // "page size" you want for (FTPFile file : files) { String path = _dataDirectory + "/" + file.getName(); paths.add(path); if (!_paths.contains(path)) toDownload.add(path); } } _totalFtpFiles = paths.size(); _newFtpFiles = toDownload.size(); long t2 = System.currentTimeMillis(); if (_log.isDebugEnabled()) _log.debug("file listing time: " + (t2 - t1) + " totalFiles: " + paths.size() + " newFiles: " + toDownload.size()); _paths = paths; if (_maxDownloadCount > 0 && toDownload.size() > _maxDownloadCount) { List<String> reduced = new ArrayList<String>(_maxDownloadCount); for (int i = 0; i < _maxDownloadCount; i++) reduced.add(toDownload.get(toDownload.size() - _maxDownloadCount + i)); toDownload = reduced; } return toDownload; } private void downloadUpdatedFiles(List<String> toDownload) throws IOException { for (String path : toDownload) { _log.debug("downloading path: {}", path); long t3 = System.currentTimeMillis(); InputStream in = _ftpClient.retrieveFileStream(path); if (!FTPReply.isPositivePreliminary(_ftpClient.getReplyCode())) { _log.warn("error initiating file transfer: " + _ftpClient.getReplyCode() + " " + _ftpClient.getReplyString()); continue; } _reader.readEntities(OrbcadRecord.class, in); in.close(); if (!_ftpClient.completePendingCommand()) { _log.warn("error completing file transfer: " + _ftpClient.getReplyCode() + " " + _ftpClient.getReplyString()); continue; } long t4 = System.currentTimeMillis(); if (_log.isDebugEnabled()) _log.info("file download time: " + (t4 - t3)); } } private void disconnectFtpClient() { try { if (_ftpClient != null) _ftpClient.disconnect(); } catch (Throwable t) { } finally { _ftpClient = null; } } }