Java tutorial
/* * #%L * Netarchivesuite - common * %% * Copyright (C) 2005 - 2014 The Royal Danish Library, the Danish State and University Library, * the National Library of France and the Austrian National Library. * %% * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License as * published by the Free Software Foundation, either version 2.1 of the * License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Lesser Public License for more details. * * You should have received a copy of the GNU General Lesser Public * License along with this program. If not, see * <http://www.gnu.org/licenses/lgpl-2.1.html>. * #L% */ package dk.netarkivet.common.distribute; import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.security.DigestInputStream; import java.security.DigestOutputStream; import java.util.Date; import java.util.Random; import org.apache.commons.io.IOUtils; import org.apache.commons.net.io.CopyStreamException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import dk.netarkivet.common.CommonSettings; import dk.netarkivet.common.exceptions.ArgumentNotValid; import dk.netarkivet.common.exceptions.IOFailure; import dk.netarkivet.common.utils.ChecksumCalculator; import dk.netarkivet.common.utils.FileUtils; import dk.netarkivet.common.utils.NotificationType; import dk.netarkivet.common.utils.NotificationsFactory; import dk.netarkivet.common.utils.Settings; /** * Class encapsulating upload to & download from an ftp-server. * <p> * Transfers are done using binary type and passive mode, if available. */ @SuppressWarnings({ "serial" }) public final class FTPRemoteFile extends AbstractRemoteFile { /** A named logger for this class. */ private static final transient Logger log = LoggerFactory.getLogger(FTPRemoteFile.class); /** * How many times we will retry upload, download, and logon. */ public static int FTP_RETRIES = Settings.getInt(CommonSettings.FTP_RETRIES_SETTINGS); /** * How large a data timeout on our FTP connections. */ public static int FTP_DATATIMEOUT = Settings.getInt(CommonSettings.FTP_DATATIMEOUT_SETTINGS); /** The default place in classpath where the settings file can be found. */ private static final String DEFAULT_SETTINGS_CLASSPATH = "dk/netarkivet/common/distribute/FTPRemoteFileSettings.xml"; /* * The static initialiser is called when the class is loaded. It will add default values for all settings defined in * this class, by loading them from a settings.xml file in classpath. */ static { Settings.addDefaultClasspathSettings(DEFAULT_SETTINGS_CLASSPATH); } /** * Ftp-connection information. FTP-related settings are by default read from settings, unless connectionParameters * are given in the constructor. */ private String ftpServerName; /** The ftp-server port. */ private final int ftpServerPort; /** The username used to connect to the ftp-server. */ private final String ftpUserName; /** The password used to connect to the ftp-server. */ private final String ftpUserPassword; /** The name that we use for the file on the FTP server. This is only for internal use. */ private final String ftpFileName; /** If useChecksums is true, contains the file checksum. */ protected final String checksum; private FTPConnectionManager cm; // NOTE: The constants defining setting names below are left non-final on // purpose! Otherwise, the static initialiser that loads default values // will not run. /** * Private constructor used by getInstance() static-method Tries to generate unique name on ftp-server. * * @param localFile File used to create new file on ftp-server. * @param useChecksums If true, checksums will be used to check transfers. * @param fileDeletable If true, this file will be deleted after upload to FTP. * @param multipleDownloads If true, the file will not be removed from FTP server automatically after first * download. * @param connectionParams If not null, contains connection parameters to the FTP-server desired by the user * @throws IOFailure if MD5 checksum fails, or ftp fails * @throws ArgumentNotValid if the local file cannot be read. */ private FTPRemoteFile(File localFile, boolean useChecksums, boolean fileDeletable, boolean multipleDownloads, RemoteFileSettings connectionParams) throws IOFailure { super(localFile, useChecksums, fileDeletable, multipleDownloads); if (connectionParams != null) { // use the connection parameters desired by the user. this.ftpServerName = connectionParams.getServerName(); this.ftpServerPort = connectionParams.getServerPort(); this.ftpUserName = connectionParams.getUserName(); this.ftpUserPassword = connectionParams.getUserPassword(); } else { // use the connection parameters specified by the settings. this.ftpServerName = Settings.get(CommonSettings.FTP_SERVER_NAME); this.ftpServerPort = Settings.getInt(CommonSettings.FTP_SERVER_PORT); this.ftpUserName = Settings.get(CommonSettings.FTP_USER_NAME); this.ftpUserPassword = Settings.get(CommonSettings.FTP_USER_PASSWORD); } this.cm = new FTPConnectionManager(ftpUserName, ftpUserPassword, ftpServerName, ftpServerPort, Settings.getInt(CommonSettings.FTP_RETRIES_SETTINGS), Settings.getInt(CommonSettings.FTP_DATATIMEOUT_SETTINGS)); if (filesize == 0) { if (useChecksums) { checksum = ChecksumCalculator.calculateMd5(file); } else { checksum = null; } ftpFileName = "-"; } else { // A large enough number to make it unlikely that two files are // created with the same FTP server name. Already the millisecond // datestamp reduces the likelihood, with this even if two // processes/threads try to upload the same file in the same // millisecond (very unlikely) they have only .01% chance of // clashing. final int aMagicNumber = 100000; ftpFileName = file.getName() + "-" + new Random().nextInt(aMagicNumber) + "-" + new Date().getTime(); InputStream in; try { in = new FileInputStream(localFile); } catch (FileNotFoundException e) { final String message = "Couldn't prepare file '" + localFile + "' for remote access. File not found."; log.debug(message, e); throw new IOFailure(message, e); } log.debug("Writing '{}' as '{}' on ftp-server {}", file.getName(), ftpFileName, cm.getFtpServer()); // Writing inlined in constructor to allow the checksum field to // be final (and thus must be set in constructor). try { cm.logOn(); if (useChecksums) { in = new DigestInputStream(in, ChecksumCalculator.getMessageDigest(ChecksumCalculator.MD5)); } boolean success = false; int tried = 0; String message = null; while (!success && tried < FTP_RETRIES) { tried++; try { success = cm.getFTPClient().storeFile(ftpFileName, in); if (!success) { log.debug("FTP store failed attempt '{}' of {}: {}", tried, FTP_RETRIES, cm.getFtpErrorMessage()); } } catch (IOException e) { message = "Write operation to '" + ftpFileName + "' failed on attempt " + tried + " of " + FTP_RETRIES; if (e instanceof CopyStreamException) { CopyStreamException realException = (CopyStreamException) e; message += "(real cause = " + realException.getIOException() + ")"; } log.debug(message, e); } } if (!success) { final String msg = "Failed to upload '" + localFile + "' after " + tried + " attempts. Reason for last failure: " + message; log.warn(msg); // Send an Notification because of this NotificationsFactory.getInstance().notify(msg, NotificationType.ERROR); throw new IOFailure(msg); } log.debug("Completed writing the file '{}'", ftpFileName); if (useChecksums) { checksum = ChecksumCalculator.toHex(((DigestInputStream) in).getMessageDigest().digest()); log.debug("Checksum of '{}' is:{}", ftpFileName, checksum); } else { checksum = null; } } finally { IOUtils.closeQuietly(in); cm.logOut(); log.debug("Ftp logout"); } } if (fileDeletable) { try { FileUtils.removeRecursively(localFile); } catch (IOFailure e) { // Not fatal log.warn("Couldn't remove tmp file {}", localFile, e); } } } /** * Create a remote file that handles the transport of the remote file data. This method is used by the sender to * prepare the transport. * * @param localFile File object for the remote file * @param useChecksums If true, checksums will be used to check transfers. * @param fileDeletable If true, this file will be deleted after upload to FTP. * @param multipleDownloads If true, the file will not be removed from FTP server automatically after first * download. * @return FTPRemoteFile object * @throws IOFailure if FTPRemoteFile creation fails */ public static RemoteFile getInstance(File localFile, Boolean useChecksums, Boolean fileDeletable, Boolean multipleDownloads) throws IOFailure { ArgumentNotValid.checkNotNull(localFile, "File remoteFile"); return new FTPRemoteFile(localFile, useChecksums, fileDeletable, multipleDownloads, null); } public static RemoteFile getInstance(File localFile, Boolean useChecksums, Boolean fileDeletable, Boolean multipleDownloads, RemoteFileSettings connectionParams) throws IOFailure { ArgumentNotValid.checkNotNull(localFile, "File remoteFile"); return new FTPRemoteFile(localFile, useChecksums, fileDeletable, multipleDownloads, connectionParams); } /** * An implementation of the getInputStream operation that works with FTP. Notice that most of the special work * (logging out and checking MD5) happens in the close() method of the returned InputStream, since that is the only * place where we can know we're done. * * @return An InputStream that will deliver the data transferred by FTP. Holding on to this for long periods without * reading any data might cause a timeout. */ @Override public InputStream getInputStream() { if (filesize == 0) { return new ByteArrayInputStream(new byte[] {}); } try { cm.logOn(); InputStream in = cm.getFTPClient().retrieveFileStream(ftpFileName); if (in == null) { throw new IOFailure("Unable to retrieve input stream:" + cm.getFtpErrorMessage()); } if (useChecksums) { in = new DigestInputStream(in, ChecksumCalculator.getMessageDigest(ChecksumCalculator.MD5)); } return new FilterInputStream(in) { public void close() throws IOException { try { super.close(); if (useChecksums) { String newChecksum = ChecksumCalculator .toHex(((DigestInputStream) in).getMessageDigest().digest()); if (!newChecksum.equals(checksum)) { final String msg = "Checksums of '" + ftpFileName + "' do not match! " + "Should be " + checksum + " but was " + newChecksum; log.warn(msg); throw new IOFailure(msg); } } } finally { cm.logOut(); if (!multipleDownloads) { cleanup(); } } } }; } catch (IOException e) { String msg = "Creating inputstream from '" + ftpFileName + "' failed "; if (e instanceof CopyStreamException) { CopyStreamException realException = (CopyStreamException) e; msg += "(real cause = " + realException.getIOException() + ")"; } log.warn(msg, e); throw new IOFailure(msg, e); } } /** * Write the contents of this ftp remote file to an output stream. Notice that while the checksum of the transferred * data is checked, no retries are performed, and in case of failure, there is no guarantee that any data have been * transferred. * * @param out OutputStream that the data will be written to. This stream will not be closed by this operation. * @throws IOFailure If append operation fails */ @Override public void appendTo(OutputStream out) { ArgumentNotValid.checkNotNull(out, "OutputStream out"); if (filesize == 0) { return; } try { cm.logOn(); if (useChecksums) { out = new DigestOutputStream(out, ChecksumCalculator.getMessageDigest(ChecksumCalculator.MD5)); } if (!cm.getFTPClient().retrieveFile(ftpFileName, out)) { final String msg = "Append operation from '" + ftpFileName + "' failed: " + cm.getFtpErrorMessage(); log.warn(msg); throw new IOFailure(msg); } out.flush(); if (useChecksums) { String newChecksum = ChecksumCalculator .toHex(((DigestOutputStream) out).getMessageDigest().digest()); if (checksum != null && !checksum.equals(newChecksum)) { final String msg = "Checksums of '" + ftpFileName + "' do not match! Should be " + checksum + " but was " + newChecksum; log.warn(msg); throw new IOFailure(msg); } } } catch (IOException e) { String msg = "Append operation from '" + ftpFileName + "' failed "; if (e instanceof CopyStreamException) { CopyStreamException realException = (CopyStreamException) e; msg += "(real cause = " + realException.getIOException() + ")"; } log.warn(msg, e); throw new IOFailure(msg, e); } finally { cm.logOut(); if (!multipleDownloads) { cleanup(); } } } /** * Cleanup will delete the file on the FTP server. This method should never throw exceptions. It is idempotent, * meaning it can be called twice without trouble. */ @Override public void cleanup() { if (filesize == 0) { return; } log.debug("Deleting file '{}' from ftp server", ftpFileName); try { cm.logOn(); cm.getFTPClient().deleteFile(ftpFileName); } catch (Exception e) { log.warn("Error while deleting ftp file '{}' for file '{}'", ftpFileName, file.getName(), e); } finally { // try to disconnect before returning from method try { cm.logOut(); } catch (Exception e) { log.warn("Unexpected error while logging out ", e); } } log.debug("File '{}' deleted from ftp server. Cleanup finished.", ftpFileName); } /** * Return a human-readable description of the object. * * @return description of object -- not machine readable */ public String toString() { return "RemoteFile '" + file.getName() + "' (#" + checksum + ")"; } /** * Get checksum for file, or null if checksums were not requested. * * @return checksum for file, or null if checksums were not requested. */ public String getChecksum() { return checksum; } /** * Retrieval of the number of retries for retrieving a file from a FTP server. Returns the setting for number of * retries. * * @return The number of retries for the FTP connection, defined in settings. */ @Override public int getNumberOfRetries() { return FTP_RETRIES; } public static RemoteFileSettings getRemoteFileSettings() { return new RemoteFileSettings(Settings.get(CommonSettings.FTP_SERVER_NAME), Settings.getInt(CommonSettings.FTP_SERVER_PORT), Settings.get(CommonSettings.FTP_USER_NAME), Settings.get(CommonSettings.FTP_USER_PASSWORD)); } }