Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.nifi.file; import java.io.BufferedInputStream; import java.io.Closeable; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.FilenameFilter; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.nio.channels.FileChannel; import java.nio.channels.FileLock; import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.Random; import org.apache.commons.codec.digest.DigestUtils; import org.slf4j.Logger; /** * A utility class containing a few useful static methods to do typical IO * operations. * * @author unattributed */ public class FileUtils { public static final long TRANSFER_CHUNK_SIZE_BYTES = 1024 * 1024 * 8; //8 MB chunks public static final long MILLIS_BETWEEN_ATTEMPTS = 50L; /** * Closes the given closeable quietly - no logging, no exceptions... * * @param closeable */ public static void closeQuietly(final Closeable closeable) { if (null != closeable) { try { closeable.close(); } catch (final IOException io) {/*IGNORE*/ } } } /** * Releases the given lock quietly - no logging, no exception * * @param lock */ public static void releaseQuietly(final FileLock lock) { if (null != lock) { try { lock.release(); } catch (final IOException io) { /*IGNORE*/ } } } public static void ensureDirectoryExistAndCanAccess(final File dir) throws IOException { if (dir.exists() && !dir.isDirectory()) { throw new IOException(dir.getAbsolutePath() + " is not a directory"); } else if (!dir.exists()) { final boolean made = dir.mkdirs(); if (!made) { throw new IOException(dir.getAbsolutePath() + " could not be created"); } } if (!(dir.canRead() && dir.canWrite())) { throw new IOException(dir.getAbsolutePath() + " directory does not have read/write privilege"); } } /** * Deletes the given file. If the given file exists but could not be deleted * this will be printed as a warning to the given logger * * @param file * @param logger * @return */ public static boolean deleteFile(final File file, final Logger logger) { return FileUtils.deleteFile(file, logger, 1); } /** * Deletes the given file. If the given file exists but could not be deleted * this will be printed as a warning to the given logger * * @param file * @param logger * @param attempts indicates how many times an attempt to delete should be * made * @return true if given file no longer exists */ public static boolean deleteFile(final File file, final Logger logger, final int attempts) { if (file == null) { return false; } boolean isGone = false; try { if (file.exists()) { final int effectiveAttempts = Math.max(1, attempts); for (int i = 0; i < effectiveAttempts && !isGone; i++) { isGone = file.delete() || !file.exists(); if (!isGone && (effectiveAttempts - i) > 1) { FileUtils.sleepQuietly(MILLIS_BETWEEN_ATTEMPTS); } } if (!isGone && logger != null) { logger.warn("File appears to exist but unable to delete file: " + file.getAbsolutePath()); } } } catch (final Throwable t) { if (logger != null) { logger.warn("Unable to delete file: '" + file.getAbsolutePath() + "' due to " + t); } } return isGone; } /** * Deletes all of the given files. If any exist and cannot be deleted that * will be printed at warn to the given logger. * * @param files can be null * @param logger can be null */ public static void deleteFile(final List<File> files, final Logger logger) { FileUtils.deleteFile(files, logger, 1); } /** * Deletes all of the given files. If any exist and cannot be deleted that * will be printed at warn to the given logger. * * @param files can be null * @param logger can be null * @param attempts indicates how many times an attempt should be made to * delete each file */ public static void deleteFile(final List<File> files, final Logger logger, final int attempts) { if (null == files || files.isEmpty()) { return; } final int effectiveAttempts = Math.max(1, attempts); for (final File file : files) { try { boolean isGone = false; for (int i = 0; i < effectiveAttempts && !isGone; i++) { isGone = file.delete() || !file.exists(); if (!isGone && (effectiveAttempts - i) > 1) { FileUtils.sleepQuietly(MILLIS_BETWEEN_ATTEMPTS); } } if (!isGone && logger != null) { logger.warn("File appears to exist but unable to delete file: " + file.getAbsolutePath()); } } catch (final Throwable t) { if (null != logger) { logger.warn("Unable to delete file given from path: '" + file.getPath() + "' due to " + t); } } } } /** * Deletes all files (not directories..) in the given directory (non * recursive) that match the given filename filter. If any file cannot be * deleted then this is printed at warn to the given logger. * * @param directory * @param filter if null then no filter is used * @param logger */ public static void deleteFilesInDir(final File directory, final FilenameFilter filter, final Logger logger) { FileUtils.deleteFilesInDir(directory, filter, logger, false); } /** * Deletes all files (not directories) in the given directory (recursive) * that match the given filename filter. If any file cannot be deleted then * this is printed at warn to the given logger. * * @param directory * @param filter if null then no filter is used * @param logger * @param recurse */ public static void deleteFilesInDir(final File directory, final FilenameFilter filter, final Logger logger, final boolean recurse) { FileUtils.deleteFilesInDir(directory, filter, logger, recurse, false); } /** * Deletes all files (not directories) in the given directory (recursive) * that match the given filename filter. If any file cannot be deleted then * this is printed at warn to the given logger. * * @param directory * @param filter if null then no filter is used * @param logger * @param recurse * @param deleteEmptyDirectories default is false; if true will delete * directories found that are empty */ public static void deleteFilesInDir(final File directory, final FilenameFilter filter, final Logger logger, final boolean recurse, final boolean deleteEmptyDirectories) { // ensure the specified directory is actually a directory and that it exists if (null != directory && directory.isDirectory()) { final File ingestFiles[] = directory.listFiles(); for (File ingestFile : ingestFiles) { boolean process = (filter == null) ? true : filter.accept(directory, ingestFile.getName()); if (ingestFile.isFile() && process) { FileUtils.deleteFile(ingestFile, logger, 3); } if (ingestFile.isDirectory() && recurse) { FileUtils.deleteFilesInDir(ingestFile, filter, logger, recurse, deleteEmptyDirectories); if (deleteEmptyDirectories && ingestFile.list().length == 0) { FileUtils.deleteFile(ingestFile, logger, 3); } } } } } /** * Deletes given files. * * @param files * @param recurse will recurse * @throws IOException */ public static void deleteFiles(final Collection<File> files, final boolean recurse) throws IOException { for (final File file : files) { FileUtils.deleteFile(file, recurse); } } public static void deleteFile(final File file, final boolean recurse) throws IOException { if (file.isDirectory() && recurse) { FileUtils.deleteFiles(Arrays.asList(file.listFiles()), recurse); } //now delete the file itself regardless of whether it is plain file or a directory if (!FileUtils.deleteFile(file, null, 5)) { throw new IOException("Unable to delete " + file.getAbsolutePath()); } } /** * Randomly generates a sequence of bytes and overwrites the contents of the * file a number of times. The file is then deleted. * * @param file File to be overwritten a number of times and, ultimately, * deleted * @param passes Number of times file should be overwritten * @throws IOException if something makes shredding or deleting a problem */ public static void shredFile(final File file, final int passes) throws IOException { final Random generator = new Random(); final long fileLength = file.length(); final int byteArraySize = (int) Math.min(fileLength, 1048576); // 1MB final byte[] b = new byte[byteArraySize]; final long numOfRandomWrites = (fileLength / b.length) + 1; final FileOutputStream fos = new FileOutputStream(file); try { // Over write file contents (passes) times final FileChannel channel = fos.getChannel(); for (int i = 0; i < passes; i++) { generator.nextBytes(b); for (int j = 0; j <= numOfRandomWrites; j++) { fos.write(b); } fos.flush(); channel.position(0); } // Write out "0" for each byte in the file Arrays.fill(b, (byte) 0); for (int j = 0; j < numOfRandomWrites; j++) { fos.write(b); } fos.flush(); fos.close(); // Try to delete the file a few times if (!FileUtils.deleteFile(file, null, 5)) { throw new IOException("Failed to delete file after shredding"); } } finally { FileUtils.closeQuietly(fos); } } public static long copy(final InputStream in, final OutputStream out) throws IOException { final byte[] buffer = new byte[65536]; long copied = 0L; int len; while ((len = in.read(buffer)) > 0) { out.write(buffer, 0, len); copied += len; } return copied; } public static long copyBytes(final byte[] bytes, final File destination, final boolean lockOutputFile) throws FileNotFoundException, IOException { FileOutputStream fos = null; FileLock outLock = null; long fileSize = 0L; try { fos = new FileOutputStream(destination); final FileChannel out = fos.getChannel(); if (lockOutputFile) { outLock = out.tryLock(0, Long.MAX_VALUE, false); if (null == outLock) { throw new IOException( "Unable to obtain exclusive file lock for: " + destination.getAbsolutePath()); } } fos.write(bytes); fos.flush(); fileSize = bytes.length; } finally { FileUtils.releaseQuietly(outLock); FileUtils.closeQuietly(fos); } return fileSize; } /** * Copies the given source file to the given destination file. The given * destination will be overwritten if it already exists. * * @param source * @param destination * @param lockInputFile if true will lock input file during copy; if false * will not * @param lockOutputFile if true will lock output file during copy; if false * will not * @param move if true will perform what is effectively a move operation * rather than a pure copy. This allows for potentially highly efficient * movement of the file but if not possible this will revert to a copy then * delete behavior. If false, then the file is copied and the source file is * retained. If a true rename/move occurs then no lock is held during that * time. * @param logger if failures occur, they will be logged to this logger if * possible. If this logger is null, an IOException will instead be thrown, * indicating the problem. * @return long number of bytes copied * @throws FileNotFoundException if the source file could not be found * @throws IOException * @throws SecurityException if a security manager denies the needed file * operations */ public static long copyFile(final File source, final File destination, final boolean lockInputFile, final boolean lockOutputFile, final boolean move, final Logger logger) throws FileNotFoundException, IOException { FileInputStream fis = null; FileOutputStream fos = null; FileLock inLock = null; FileLock outLock = null; long fileSize = 0L; if (!source.canRead()) { throw new IOException("Must at least have read permission"); } if (move && source.renameTo(destination)) { fileSize = destination.length(); } else { try { fis = new FileInputStream(source); fos = new FileOutputStream(destination); final FileChannel in = fis.getChannel(); final FileChannel out = fos.getChannel(); if (lockInputFile) { inLock = in.tryLock(0, Long.MAX_VALUE, true); if (null == inLock) { throw new IOException("Unable to obtain shared file lock for: " + source.getAbsolutePath()); } } if (lockOutputFile) { outLock = out.tryLock(0, Long.MAX_VALUE, false); if (null == outLock) { throw new IOException( "Unable to obtain exclusive file lock for: " + destination.getAbsolutePath()); } } long bytesWritten = 0; do { bytesWritten += out.transferFrom(in, bytesWritten, TRANSFER_CHUNK_SIZE_BYTES); fileSize = in.size(); } while (bytesWritten < fileSize); out.force(false); FileUtils.closeQuietly(fos); FileUtils.closeQuietly(fis); fos = null; fis = null; if (move && !FileUtils.deleteFile(source, null, 5)) { if (logger == null) { FileUtils.deleteFile(destination, null, 5); throw new IOException("Could not remove file " + source.getAbsolutePath()); } else { logger.warn( "Configured to delete source file when renaming/move not successful. However, unable to delete file at: " + source.getAbsolutePath()); } } } finally { FileUtils.releaseQuietly(inLock); FileUtils.releaseQuietly(outLock); FileUtils.closeQuietly(fos); FileUtils.closeQuietly(fis); } } return fileSize; } /** * Copies the given source file to the given destination file. The given * destination will be overwritten if it already exists. * * @param source * @param destination * @param lockInputFile if true will lock input file during copy; if false * will not * @param lockOutputFile if true will lock output file during copy; if false * will not * @param logger * @return long number of bytes copied * @throws FileNotFoundException if the source file could not be found * @throws IOException * @throws SecurityException if a security manager denies the needed file * operations */ public static long copyFile(final File source, final File destination, final boolean lockInputFile, final boolean lockOutputFile, final Logger logger) throws FileNotFoundException, IOException { return FileUtils.copyFile(source, destination, lockInputFile, lockOutputFile, false, logger); } public static long copyFile(final File source, final OutputStream stream, final boolean closeOutputStream, final boolean lockInputFile) throws FileNotFoundException, IOException { FileInputStream fis = null; FileLock inLock = null; long fileSize = 0L; try { fis = new FileInputStream(source); final FileChannel in = fis.getChannel(); if (lockInputFile) { inLock = in.tryLock(0, Long.MAX_VALUE, true); if (inLock == null) { throw new IOException("Unable to obtain exclusive file lock for: " + source.getAbsolutePath()); } } byte[] buffer = new byte[1 << 18]; //256 KB int bytesRead = -1; while ((bytesRead = fis.read(buffer)) != -1) { stream.write(buffer, 0, bytesRead); } in.force(false); stream.flush(); fileSize = in.size(); } finally { FileUtils.releaseQuietly(inLock); FileUtils.closeQuietly(fis); if (closeOutputStream) { FileUtils.closeQuietly(stream); } } return fileSize; } public static long copyFile(final InputStream stream, final File destination, final boolean closeInputStream, final boolean lockOutputFile) throws FileNotFoundException, IOException { final Path destPath = destination.toPath(); final long size = Files.copy(stream, destPath); if (closeInputStream) { stream.close(); } return size; } /** * Renames the given file from the source path to the destination path. This * handles multiple attempts. This should only be used to rename within a * given directory. Renaming across directories might not work well. See the * <code>File.renameTo</code> for more information. * * @param source the file to rename * @param destination the file path to rename to * @param maxAttempts the max number of attempts to attempt the rename * @throws IOException if rename isn't successful */ public static void renameFile(final File source, final File destination, final int maxAttempts) throws IOException { FileUtils.renameFile(source, destination, maxAttempts, false); } /** * Renames the given file from the source path to the destination path. This * handles multiple attempts. This should only be used to rename within a * given directory. Renaming across directories might not work well. See the * <code>File.renameTo</code> for more information. * * @param source the file to rename * @param destination the file path to rename to * @param maxAttempts the max number of attempts to attempt the rename * @param replace if true and a rename attempt fails will check if a file is * already at the destination path. If so it will delete that file and * attempt the rename according the remaining maxAttempts. If false, any * conflicting files will be left as they were and the rename attempts will * fail if conflicting. * @throws IOException if rename isn't successful */ public static void renameFile(final File source, final File destination, final int maxAttempts, final boolean replace) throws IOException { final int attempts = (replace || maxAttempts < 1) ? Math.max(2, maxAttempts) : maxAttempts; boolean renamed = false; for (int i = 0; i < attempts; i++) { renamed = source.renameTo(destination); if (!renamed) { FileUtils.deleteFile(destination, null, 5); } else { break; //rename has succeeded } } if (!renamed) { throw new IOException("Attempted " + maxAttempts + " times but unable to rename from \'" + source.getPath() + "\' to \'" + destination.getPath() + "\'"); } } public static void sleepQuietly(final long millis) { try { Thread.sleep(millis); } catch (final InterruptedException ex) { /* do nothing */ } } /** * Syncs a primary copy of a file with the copy in the restore directory. If * the restore directory does not have a file and the primary has a file, * the the primary's file is copied to the restore directory. Else if the * restore directory has a file, but the primary does not, then the * restore's file is copied to the primary directory. Else if the primary * file is different than the restore file, then an IllegalStateException is * thrown. Otherwise, if neither file exists, then no syncing is performed. * * @param primaryFile the primary file * @param restoreFile the restore file * @param logger a logger * @throws IOException if an I/O problem was encountered during syncing * @throws IllegalStateException if the primary and restore copies exist but * are different */ public static void syncWithRestore(final File primaryFile, final File restoreFile, final Logger logger) throws IOException { if (primaryFile.exists() && !restoreFile.exists()) { // copy primary file to restore copyFile(primaryFile, restoreFile, false, false, logger); } else if (restoreFile.exists() && !primaryFile.exists()) { // copy restore file to primary copyFile(restoreFile, primaryFile, false, false, logger); } else if (primaryFile.exists() && restoreFile.exists() && !isSame(primaryFile, restoreFile)) { throw new IllegalStateException(String.format("Primary file '%s' is different than restore file '%s'", primaryFile.getAbsoluteFile(), restoreFile.getAbsolutePath())); } } /** * Returns true if the given files are the same according to their MD5 hash. * * @param file1 a file * @param file2 a file * @return true if the files are the same; false otherwise * @throws IOException if the MD5 hash could not be computed */ public static boolean isSame(final File file1, final File file2) throws IOException { return Arrays.equals(computeMd5Digest(file1), computeMd5Digest(file2)); } /** * Returns the MD5 hash of the given file. * * @param file a file * @return the MD5 hash * @throws IOException if the MD5 hash could not be computed */ public static byte[] computeMd5Digest(final File file) throws IOException { BufferedInputStream bis = null; try { bis = new BufferedInputStream(new FileInputStream(file)); return DigestUtils.md5(bis); } finally { FileUtils.closeQuietly(bis); } } }