Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.fs; import java.io.Closeable; import java.io.FileNotFoundException; import java.io.IOException; import java.lang.ref.WeakReference; import java.lang.ref.ReferenceQueue; import java.net.URI; import java.net.URISyntaxException; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.Collection; import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; import java.util.IdentityHashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.NoSuchElementException; import java.util.ServiceConfigurationError; import java.util.ServiceLoader; import java.util.Set; import java.util.Stack; import java.util.TreeSet; import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.GlobalStorageStatistics.StorageStatisticsProvider; import org.apache.hadoop.fs.Options.ChecksumOpt; import org.apache.hadoop.fs.Options.HandleOpt; import org.apache.hadoop.fs.Options.Rename; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclStatus; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsCreateModes; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.io.MultipleIOException; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.DelegationTokenIssuer; import org.apache.hadoop.util.ClassUtil; import org.apache.hadoop.util.DataChecksum; import org.apache.hadoop.util.Progressable; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.ShutdownHookManager; import org.apache.hadoop.util.StringUtils; import org.apache.htrace.core.Tracer; import org.apache.htrace.core.TraceScope; import com.google.common.base.Preconditions; import com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import static com.google.common.base.Preconditions.checkArgument; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.*; /**************************************************************** * An abstract base class for a fairly generic filesystem. It * may be implemented as a distributed filesystem, or as a "local" * one that reflects the locally-connected disk. The local version * exists for small Hadoop instances and for testing. * * <p> * * All user code that may potentially use the Hadoop Distributed * File System should be written to use a FileSystem object or its * successor, {@link FileContext}. * * <p> * The local implementation is {@link LocalFileSystem} and distributed * implementation is DistributedFileSystem. There are other implementations * for object stores and (outside the Apache Hadoop codebase), * third party filesystems. * <p> * Notes * <ol> * <li>The behaviour of the filesystem is * <a href="https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-common/filesystem/filesystem.html"> * specified in the Hadoop documentation. </a> * However, the normative specification of the behavior of this class is * actually HDFS: if HDFS does not behave the way these Javadocs or * the specification in the Hadoop documentations define, assume that * the documentation is incorrect. * </li> * <li>The term {@code FileSystem} refers to an instance of this class.</li> * <li>The acronym "FS" is used as an abbreviation of FileSystem.</li> * <li>The term {@code filesystem} refers to the distributed/local filesystem * itself, rather than the class used to interact with it.</li> * <li>The term "file" refers to a file in the remote filesystem, * rather than instances of {@code java.io.File}.</li> * </ol> *****************************************************************/ @SuppressWarnings("DeprecatedIsStillUsed") @InterfaceAudience.Public @InterfaceStability.Stable public abstract class FileSystem extends Configured implements Closeable, DelegationTokenIssuer { public static final String FS_DEFAULT_NAME_KEY = CommonConfigurationKeys.FS_DEFAULT_NAME_KEY; public static final String DEFAULT_FS = CommonConfigurationKeys.FS_DEFAULT_NAME_DEFAULT; /** * This log is widely used in the org.apache.hadoop.fs code and tests, * so must be considered something to only be changed with care. */ @InterfaceAudience.Private public static final Log LOG = LogFactory.getLog(FileSystem.class); /** * The SLF4J logger to use in logging within the FileSystem class itself. */ private static final Logger LOGGER = LoggerFactory.getLogger(FileSystem.class); /** * Priority of the FileSystem shutdown hook: {@value}. */ public static final int SHUTDOWN_HOOK_PRIORITY = 10; /** * Prefix for trash directory: {@value}. */ public static final String TRASH_PREFIX = ".Trash"; public static final String USER_HOME_PREFIX = "/user"; /** FileSystem cache. */ static final Cache CACHE = new Cache(); /** The key this instance is stored under in the cache. */ private Cache.Key key; /** Recording statistics per a FileSystem class. */ private static final Map<Class<? extends FileSystem>, Statistics> statisticsTable = new IdentityHashMap<>(); /** * The statistics for this file system. */ protected Statistics statistics; /** * A cache of files that should be deleted when the FileSystem is closed * or the JVM is exited. */ private final Set<Path> deleteOnExit = new TreeSet<>(); /** * Should symbolic links be resolved by {@link FileSystemLinkResolver}. * Set to the value of * {@link CommonConfigurationKeysPublic#FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY} */ boolean resolveSymlinks; /** * This method adds a FileSystem instance to the cache so that it can * be retrieved later. It is only for testing. * @param uri the uri to store it under * @param conf the configuration to store it under * @param fs the FileSystem to store * @throws IOException if the current user cannot be determined. */ @VisibleForTesting static void addFileSystemForTesting(URI uri, Configuration conf, FileSystem fs) throws IOException { CACHE.map.put(new Cache.Key(uri, conf), fs); } /** * Get a FileSystem instance based on the uri, the passed in * configuration and the user. * @param uri of the filesystem * @param conf the configuration to use * @param user to perform the get as * @return the filesystem instance * @throws IOException failure to load * @throws InterruptedException If the {@code UGI.doAs()} call was * somehow interrupted. */ public static FileSystem get(final URI uri, final Configuration conf, final String user) throws IOException, InterruptedException { String ticketCachePath = conf.get(CommonConfigurationKeys.KERBEROS_TICKET_CACHE_PATH); UserGroupInformation ugi = UserGroupInformation.getBestUGI(ticketCachePath, user); return ugi.doAs(new PrivilegedExceptionAction<FileSystem>() { @Override public FileSystem run() throws IOException { return get(uri, conf); } }); } /** * Returns the configured FileSystem implementation. * @param conf the configuration to use */ public static FileSystem get(Configuration conf) throws IOException { return get(getDefaultUri(conf), conf); } /** * Get the default FileSystem URI from a configuration. * @param conf the configuration to use * @return the uri of the default filesystem */ public static URI getDefaultUri(Configuration conf) { URI uri = URI.create(fixName(conf.get(FS_DEFAULT_NAME_KEY, DEFAULT_FS))); if (uri.getScheme() == null) { throw new IllegalArgumentException("No scheme in default FS: " + uri); } return uri; } /** * Set the default FileSystem URI in a configuration. * @param conf the configuration to alter * @param uri the new default filesystem uri */ public static void setDefaultUri(Configuration conf, URI uri) { conf.set(FS_DEFAULT_NAME_KEY, uri.toString()); } /** Set the default FileSystem URI in a configuration. * @param conf the configuration to alter * @param uri the new default filesystem uri */ public static void setDefaultUri(Configuration conf, String uri) { setDefaultUri(conf, URI.create(fixName(uri))); } /** * Initialize a FileSystem. * * Called after the new FileSystem instance is constructed, and before it * is ready for use. * * FileSystem implementations overriding this method MUST forward it to * their superclass, though the order in which it is done, and whether * to alter the configuration before the invocation are options of the * subclass. * @param name a URI whose authority section names the host, port, etc. * for this FileSystem * @param conf the configuration * @throws IOException on any failure to initialize this instance. * @throws IllegalArgumentException if the URI is considered invalid. */ public void initialize(URI name, Configuration conf) throws IOException { final String scheme; if (name.getScheme() == null || name.getScheme().isEmpty()) { scheme = getDefaultUri(conf).getScheme(); } else { scheme = name.getScheme(); } statistics = getStatistics(scheme, getClass()); resolveSymlinks = conf.getBoolean(CommonConfigurationKeysPublic.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY, CommonConfigurationKeysPublic.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_DEFAULT); } /** * Return the protocol scheme for this FileSystem. * <p> * This implementation throws an <code>UnsupportedOperationException</code>. * * @return the protocol scheme for this FileSystem. * @throws UnsupportedOperationException if the operation is unsupported * (default). */ public String getScheme() { throw new UnsupportedOperationException( "Not implemented by the " + getClass().getSimpleName() + " FileSystem implementation"); } /** * Returns a URI which identifies this FileSystem. * * @return the URI of this filesystem. */ public abstract URI getUri(); /** * Return a canonicalized form of this FileSystem's URI. * * The default implementation simply calls {@link #canonicalizeUri(URI)} * on the filesystem's own URI, so subclasses typically only need to * implement that method. * * @see #canonicalizeUri(URI) */ protected URI getCanonicalUri() { return canonicalizeUri(getUri()); } /** * Canonicalize the given URI. * * This is implementation-dependent, and may for example consist of * canonicalizing the hostname using DNS and adding the default * port if not specified. * * The default implementation simply fills in the default port if * not specified and if {@link #getDefaultPort()} returns a * default port. * * @return URI * @see NetUtils#getCanonicalUri(URI, int) */ protected URI canonicalizeUri(URI uri) { if (uri.getPort() == -1 && getDefaultPort() > 0) { // reconstruct the uri with the default port set try { uri = new URI(uri.getScheme(), uri.getUserInfo(), uri.getHost(), getDefaultPort(), uri.getPath(), uri.getQuery(), uri.getFragment()); } catch (URISyntaxException e) { // Should never happen! throw new AssertionError("Valid URI became unparseable: " + uri); } } return uri; } /** * Get the default port for this FileSystem. * @return the default port or 0 if there isn't one */ protected int getDefaultPort() { return 0; } protected static FileSystem getFSofPath(final Path absOrFqPath, final Configuration conf) throws UnsupportedFileSystemException, IOException { absOrFqPath.checkNotSchemeWithRelative(); absOrFqPath.checkNotRelative(); // Uses the default FileSystem if not fully qualified return get(absOrFqPath.toUri(), conf); } /** * Get a canonical service name for this FileSystem. * The token cache is the only user of the canonical service name, * and uses it to lookup this FileSystem's service tokens. * If the file system provides a token of its own then it must have a * canonical name, otherwise the canonical name can be null. * * Default implementation: If the FileSystem has child file systems * (such as an embedded file system) then it is assumed that the FS has no * tokens of its own and hence returns a null name; otherwise a service * name is built using Uri and port. * * @return a service string that uniquely identifies this file system, null * if the filesystem does not implement tokens * @see SecurityUtil#buildDTServiceName(URI, int) */ @InterfaceAudience.Public @InterfaceStability.Evolving @Override public String getCanonicalServiceName() { return (getChildFileSystems() == null) ? SecurityUtil.buildDTServiceName(getUri(), getDefaultPort()) : null; } /** @deprecated call {@link #getUri()} instead.*/ @Deprecated public String getName() { return getUri().toString(); } /** @deprecated call {@link #get(URI, Configuration)} instead. */ @Deprecated public static FileSystem getNamed(String name, Configuration conf) throws IOException { return get(URI.create(fixName(name)), conf); } /** Update old-format filesystem names, for back-compatibility. This should * eventually be replaced with a checkName() method that throws an exception * for old-format names. */ private static String fixName(String name) { // convert old-format name to new-format name if (name.equals("local")) { // "local" is now "file:///". LOGGER.warn("\"local\" is a deprecated filesystem name." + " Use \"file:///\" instead."); name = "file:///"; } else if (name.indexOf('/') == -1) { // unqualified is "hdfs://" LOGGER.warn( "\"" + name + "\" is a deprecated filesystem name." + " Use \"hdfs://" + name + "/\" instead."); name = "hdfs://" + name; } return name; } /** * Get the local FileSystem. * @param conf the configuration to configure the FileSystem with * if it is newly instantiated. * @return a LocalFileSystem * @throws IOException if somehow the local FS cannot be instantiated. */ public static LocalFileSystem getLocal(Configuration conf) throws IOException { return (LocalFileSystem) get(LocalFileSystem.NAME, conf); } /** * Get a FileSystem for this URI's scheme and authority. * <ol> * <li> * If the configuration has the property * {@code "fs.$SCHEME.impl.disable.cache"} set to true, * a new instance will be created, initialized with the supplied URI and * configuration, then returned without being cached. * </li> * <li> * If the there is a cached FS instance matching the same URI, it will * be returned. * </li> * <li> * Otherwise: a new FS instance will be created, initialized with the * configuration and URI, cached and returned to the caller. * </li> * </ol> * @throws IOException if the FileSystem cannot be instantiated. */ public static FileSystem get(URI uri, Configuration conf) throws IOException { String scheme = uri.getScheme(); String authority = uri.getAuthority(); if (scheme == null && authority == null) { // use default FS return get(conf); } if (scheme != null && authority == null) { // no authority URI defaultUri = getDefaultUri(conf); if (scheme.equals(defaultUri.getScheme()) // if scheme matches default && defaultUri.getAuthority() != null) { // & default has authority return get(defaultUri, conf); // return default } } String disableCacheName = String.format("fs.%s.impl.disable.cache", scheme); if (conf.getBoolean(disableCacheName, false)) { LOGGER.debug("Bypassing cache to create filesystem {}", uri); return createFileSystem(uri, conf); } return CACHE.get(uri, conf); } /** * Returns the FileSystem for this URI's scheme and authority and the * given user. Internally invokes {@link #newInstance(URI, Configuration)} * @param uri of the filesystem * @param conf the configuration to use * @param user to perform the get as * @return filesystem instance * @throws IOException if the FileSystem cannot be instantiated. * @throws InterruptedException If the {@code UGI.doAs()} call was * somehow interrupted. */ public static FileSystem newInstance(final URI uri, final Configuration conf, final String user) throws IOException, InterruptedException { String ticketCachePath = conf.get(CommonConfigurationKeys.KERBEROS_TICKET_CACHE_PATH); UserGroupInformation ugi = UserGroupInformation.getBestUGI(ticketCachePath, user); return ugi.doAs(new PrivilegedExceptionAction<FileSystem>() { @Override public FileSystem run() throws IOException { return newInstance(uri, conf); } }); } /** * Returns the FileSystem for this URI's scheme and authority. * The entire URI is passed to the FileSystem instance's initialize method. * This always returns a new FileSystem object. * @param uri FS URI * @param config configuration to use * @return the new FS instance * @throws IOException FS creation or initialization failure. */ public static FileSystem newInstance(URI uri, Configuration config) throws IOException { String scheme = uri.getScheme(); String authority = uri.getAuthority(); if (scheme == null) { // no scheme: use default FS return newInstance(config); } if (authority == null) { // no authority URI defaultUri = getDefaultUri(config); if (scheme.equals(defaultUri.getScheme()) // if scheme matches default && defaultUri.getAuthority() != null) { // & default has authority return newInstance(defaultUri, config); // return default } } return CACHE.getUnique(uri, config); } /** * Returns a unique configured FileSystem implementation for the default * filesystem of the supplied configuration. * This always returns a new FileSystem object. * @param conf the configuration to use * @return the new FS instance * @throws IOException FS creation or initialization failure. */ public static FileSystem newInstance(Configuration conf) throws IOException { return newInstance(getDefaultUri(conf), conf); } /** * Get a unique local FileSystem object. * @param conf the configuration to configure the FileSystem with * @return a new LocalFileSystem object. * @throws IOException FS creation or initialization failure. */ public static LocalFileSystem newInstanceLocal(Configuration conf) throws IOException { return (LocalFileSystem) newInstance(LocalFileSystem.NAME, conf); } /** * Close all cached FileSystem instances. After this operation, they * may not be used in any operations. * * @throws IOException a problem arose closing one or more filesystem. */ public static void closeAll() throws IOException { CACHE.closeAll(); } /** * Close all cached FileSystem instances for a given UGI. * Be sure those filesystems are not used anymore. * @param ugi user group info to close * @throws IOException a problem arose closing one or more filesystem. */ public static void closeAllForUGI(UserGroupInformation ugi) throws IOException { CACHE.closeAll(ugi); } /** * Qualify a path to one which uses this FileSystem and, if relative, * made absolute. * @param path to qualify. * @return this path if it contains a scheme and authority and is absolute, or * a new path that includes a path and authority and is fully qualified * @see Path#makeQualified(URI, Path) * @throws IllegalArgumentException if the path has a schema/URI different * from this FileSystem. */ public Path makeQualified(Path path) { checkPath(path); return path.makeQualified(this.getUri(), this.getWorkingDirectory()); } /** * Get a new delegation token for this FileSystem. * This is an internal method that should have been declared protected * but wasn't historically. * Callers should use {@link #addDelegationTokens(String, Credentials)} * * @param renewer the account name that is allowed to renew the token. * @return a new delegation token or null if the FS does not support tokens. * @throws IOException on any problem obtaining a token */ @InterfaceAudience.Private() @Override public Token<?> getDelegationToken(String renewer) throws IOException { return null; } /** * Get all the immediate child FileSystems embedded in this FileSystem. * It does not recurse and get grand children. If a FileSystem * has multiple child FileSystems, then it must return a unique list * of those FileSystems. Default is to return null to signify no children. * * @return FileSystems that are direct children of this FileSystem, * or null for "no children" */ @InterfaceAudience.LimitedPrivate({ "HDFS" }) @VisibleForTesting public FileSystem[] getChildFileSystems() { return null; } @InterfaceAudience.Private @Override public DelegationTokenIssuer[] getAdditionalTokenIssuers() throws IOException { return getChildFileSystems(); } /** * Create a file with the provided permission. * * The permission of the file is set to be the provided permission as in * setPermission, not permission&~umask * * The HDFS implementation is implemented using two RPCs. * It is understood that it is inefficient, * but the implementation is thread-safe. The other option is to change the * value of umask in configuration to be 0, but it is not thread-safe. * * @param fs FileSystem * @param file the name of the file to be created * @param permission the permission of the file * @return an output stream * @throws IOException IO failure */ public static FSDataOutputStream create(FileSystem fs, Path file, FsPermission permission) throws IOException { // create the file with default permission FSDataOutputStream out = fs.create(file); // set its permission to the supplied one fs.setPermission(file, permission); return out; } /** * Create a directory with the provided permission. * The permission of the directory is set to be the provided permission as in * setPermission, not permission&~umask * * @see #create(FileSystem, Path, FsPermission) * * @param fs FileSystem handle * @param dir the name of the directory to be created * @param permission the permission of the directory * @return true if the directory creation succeeds; false otherwise * @throws IOException A problem creating the directories. */ public static boolean mkdirs(FileSystem fs, Path dir, FsPermission permission) throws IOException { // create the directory using the default permission boolean result = fs.mkdirs(dir); // set its permission to be the supplied one fs.setPermission(dir, permission); return result; } /////////////////////////////////////////////////////////////// // FileSystem /////////////////////////////////////////////////////////////// protected FileSystem() { super(null); } /** * Check that a Path belongs to this FileSystem. * * The base implementation performs case insensitive equality checks * of the URIs' schemes and authorities. Subclasses may implement slightly * different checks. * @param path to check * @throws IllegalArgumentException if the path is not considered to be * part of this FileSystem. * */ protected void checkPath(Path path) { URI uri = path.toUri(); String thatScheme = uri.getScheme(); if (thatScheme == null) // fs is relative return; URI thisUri = getCanonicalUri(); String thisScheme = thisUri.getScheme(); //authority and scheme are not case sensitive if (thisScheme.equalsIgnoreCase(thatScheme)) {// schemes match String thisAuthority = thisUri.getAuthority(); String thatAuthority = uri.getAuthority(); if (thatAuthority == null && // path's authority is null thisAuthority != null) { // fs has an authority URI defaultUri = getDefaultUri(getConf()); if (thisScheme.equalsIgnoreCase(defaultUri.getScheme())) { uri = defaultUri; // schemes match, so use this uri instead } else { uri = null; // can't determine auth of the path } } if (uri != null) { // canonicalize uri before comparing with this fs uri = canonicalizeUri(uri); thatAuthority = uri.getAuthority(); if (thisAuthority == thatAuthority || // authorities match (thisAuthority != null && thisAuthority.equalsIgnoreCase(thatAuthority))) return; } } throw new IllegalArgumentException("Wrong FS: " + path + ", expected: " + this.getUri()); } /** * Return an array containing hostnames, offset and size of * portions of the given file. For nonexistent * file or regions, {@code null} is returned. * * <pre> * if f == null : * result = null * elif f.getLen() <= start: * result = [] * else result = [ locations(FS, b) for b in blocks(FS, p, s, s+l)] * </pre> * This call is most helpful with and distributed filesystem * where the hostnames of machines that contain blocks of the given file * can be determined. * * The default implementation returns an array containing one element: * <pre> * BlockLocation( { "localhost:9866" }, { "localhost" }, 0, file.getLen()) * </pre> * * In HDFS, if file is three-replicated, the returned array contains * elements like: * <pre> * BlockLocation(offset: 0, length: BLOCK_SIZE, * hosts: {"host1:9866", "host2:9866, host3:9866"}) * BlockLocation(offset: BLOCK_SIZE, length: BLOCK_SIZE, * hosts: {"host2:9866", "host3:9866, host4:9866"}) * </pre> * * And if a file is erasure-coded, the returned BlockLocation are logical * block groups. * * Suppose we have a RS_3_2 coded file (3 data units and 2 parity units). * 1. If the file size is less than one stripe size, say 2 * CELL_SIZE, then * there will be one BlockLocation returned, with 0 offset, actual file size * and 4 hosts (2 data blocks and 2 parity blocks) hosting the actual blocks. * 3. If the file size is less than one group size but greater than one * stripe size, then there will be one BlockLocation returned, with 0 offset, * actual file size with 5 hosts (3 data blocks and 2 parity blocks) hosting * the actual blocks. * 4. If the file size is greater than one group size, 3 * BLOCK_SIZE + 123 * for example, then the result will be like: * <pre> * BlockLocation(offset: 0, length: 3 * BLOCK_SIZE, hosts: {"host1:9866", * "host2:9866","host3:9866","host4:9866","host5:9866"}) * BlockLocation(offset: 3 * BLOCK_SIZE, length: 123, hosts: {"host1:9866", * "host4:9866", "host5:9866"}) * </pre> * * @param file FilesStatus to get data from * @param start offset into the given file * @param len length for which to get locations for * @throws IOException IO failure */ public BlockLocation[] getFileBlockLocations(FileStatus file, long start, long len) throws IOException { if (file == null) { return null; } if (start < 0 || len < 0) { throw new IllegalArgumentException("Invalid start or len parameter"); } if (file.getLen() <= start) { return new BlockLocation[0]; } String[] name = { "localhost:9866" }; String[] host = { "localhost" }; return new BlockLocation[] { new BlockLocation(name, host, 0, file.getLen()) }; } /** * Return an array containing hostnames, offset and size of * portions of the given file. For a nonexistent * file or regions, {@code null} is returned. * * This call is most helpful with location-aware distributed * filesystems, where it returns hostnames of machines that * contain the given file. * * A FileSystem will normally return the equivalent result * of passing the {@code FileStatus} of the path to * {@link #getFileBlockLocations(FileStatus, long, long)} * * @param p path is used to identify an FS since an FS could have * another FS that it could be delegating the call to * @param start offset into the given file * @param len length for which to get locations for * @throws FileNotFoundException when the path does not exist * @throws IOException IO failure */ public BlockLocation[] getFileBlockLocations(Path p, long start, long len) throws IOException { if (p == null) { throw new NullPointerException(); } FileStatus file = getFileStatus(p); return getFileBlockLocations(file, start, len); } /** * Return a set of server default configuration values. * @return server default configuration values * @throws IOException IO failure * @deprecated use {@link #getServerDefaults(Path)} instead */ @Deprecated public FsServerDefaults getServerDefaults() throws IOException { Configuration config = getConf(); // CRC32 is chosen as default as it is available in all // releases that support checksum. // The client trash configuration is ignored. return new FsServerDefaults(getDefaultBlockSize(), config.getInt("io.bytes.per.checksum", 512), 64 * 1024, getDefaultReplication(), config.getInt(IO_FILE_BUFFER_SIZE_KEY, IO_FILE_BUFFER_SIZE_DEFAULT), false, FS_TRASH_INTERVAL_DEFAULT, DataChecksum.Type.CRC32, ""); } /** * Return a set of server default configuration values. * @param p path is used to identify an FS since an FS could have * another FS that it could be delegating the call to * @return server default configuration values * @throws IOException IO failure */ public FsServerDefaults getServerDefaults(Path p) throws IOException { return getServerDefaults(); } /** * Return the fully-qualified path of path, resolving the path * through any symlinks or mount point. * @param p path to be resolved * @return fully qualified path * @throws FileNotFoundException if the path is not present * @throws IOException for any other error */ public Path resolvePath(final Path p) throws IOException { checkPath(p); return getFileStatus(p).getPath(); } /** * Opens an FSDataInputStream at the indicated Path. * @param f the file name to open * @param bufferSize the size of the buffer to be used. * @throws IOException IO failure */ public abstract FSDataInputStream open(Path f, int bufferSize) throws IOException; /** * Opens an FSDataInputStream at the indicated Path. * @param f the file to open * @throws IOException IO failure */ public FSDataInputStream open(Path f) throws IOException { return open(f, getConf().getInt(IO_FILE_BUFFER_SIZE_KEY, IO_FILE_BUFFER_SIZE_DEFAULT)); } /** * Open an FSDataInputStream matching the PathHandle instance. The * implementation may encode metadata in PathHandle to address the * resource directly and verify that the resource referenced * satisfies constraints specified at its construciton. * @param fd PathHandle object returned by the FS authority. * @throws InvalidPathHandleException If {@link PathHandle} constraints are * not satisfied * @throws IOException IO failure * @throws UnsupportedOperationException If {@link #open(PathHandle, int)} * not overridden by subclass */ public FSDataInputStream open(PathHandle fd) throws IOException { return open(fd, getConf().getInt(IO_FILE_BUFFER_SIZE_KEY, IO_FILE_BUFFER_SIZE_DEFAULT)); } /** * Open an FSDataInputStream matching the PathHandle instance. The * implementation may encode metadata in PathHandle to address the * resource directly and verify that the resource referenced * satisfies constraints specified at its construciton. * @param fd PathHandle object returned by the FS authority. * @param bufferSize the size of the buffer to use * @throws InvalidPathHandleException If {@link PathHandle} constraints are * not satisfied * @throws IOException IO failure * @throws UnsupportedOperationException If not overridden by subclass */ public FSDataInputStream open(PathHandle fd, int bufferSize) throws IOException { throw new UnsupportedOperationException(); } /** * Create a durable, serializable handle to the referent of the given * entity. * @param stat Referent in the target FileSystem * @param opt If absent, assume {@link HandleOpt#path()}. * @throws IllegalArgumentException If the FileStatus does not belong to * this FileSystem * @throws UnsupportedOperationException If {@link #createPathHandle} * not overridden by subclass. * @throws UnsupportedOperationException If this FileSystem cannot enforce * the specified constraints. */ public final PathHandle getPathHandle(FileStatus stat, HandleOpt... opt) { // method is final with a default so clients calling getPathHandle(stat) // get the same semantics for all FileSystem implementations if (null == opt || 0 == opt.length) { return createPathHandle(stat, HandleOpt.path()); } return createPathHandle(stat, opt); } /** * Hook to implement support for {@link PathHandle} operations. * @param stat Referent in the target FileSystem * @param opt Constraints that determine the validity of the * {@link PathHandle} reference. */ protected PathHandle createPathHandle(FileStatus stat, HandleOpt... opt) { throw new UnsupportedOperationException(); } /** * Create an FSDataOutputStream at the indicated Path. * Files are overwritten by default. * @param f the file to create * @throws IOException IO failure */ public FSDataOutputStream create(Path f) throws IOException { return create(f, true); } /** * Create an FSDataOutputStream at the indicated Path. * @param f the file to create * @param overwrite if a file with this name already exists, then if true, * the file will be overwritten, and if false an exception will be thrown. * @throws IOException IO failure */ public FSDataOutputStream create(Path f, boolean overwrite) throws IOException { return create(f, overwrite, getConf().getInt(IO_FILE_BUFFER_SIZE_KEY, IO_FILE_BUFFER_SIZE_DEFAULT), getDefaultReplication(f), getDefaultBlockSize(f)); } /** * Create an FSDataOutputStream at the indicated Path with write-progress * reporting. * Files are overwritten by default. * @param f the file to create * @param progress to report progress * @throws IOException IO failure */ public FSDataOutputStream create(Path f, Progressable progress) throws IOException { return create(f, true, getConf().getInt(IO_FILE_BUFFER_SIZE_KEY, IO_FILE_BUFFER_SIZE_DEFAULT), getDefaultReplication(f), getDefaultBlockSize(f), progress); } /** * Create an FSDataOutputStream at the indicated Path. * Files are overwritten by default. * @param f the file to create * @param replication the replication factor * @throws IOException IO failure */ public FSDataOutputStream create(Path f, short replication) throws IOException { return create(f, true, getConf().getInt(IO_FILE_BUFFER_SIZE_KEY, IO_FILE_BUFFER_SIZE_DEFAULT), replication, getDefaultBlockSize(f)); } /** * Create an FSDataOutputStream at the indicated Path with write-progress * reporting. * Files are overwritten by default. * @param f the file to create * @param replication the replication factor * @param progress to report progress * @throws IOException IO failure */ public FSDataOutputStream create(Path f, short replication, Progressable progress) throws IOException { return create(f, true, getConf().getInt(IO_FILE_BUFFER_SIZE_KEY, IO_FILE_BUFFER_SIZE_DEFAULT), replication, getDefaultBlockSize(f), progress); } /** * Create an FSDataOutputStream at the indicated Path. * @param f the file to create * @param overwrite if a path with this name already exists, then if true, * the file will be overwritten, and if false an error will be thrown. * @param bufferSize the size of the buffer to be used. * @throws IOException IO failure */ public FSDataOutputStream create(Path f, boolean overwrite, int bufferSize) throws IOException { return create(f, overwrite, bufferSize, getDefaultReplication(f), getDefaultBlockSize(f)); } /** * Create an {@link FSDataOutputStream} at the indicated Path * with write-progress reporting. * * The frequency of callbacks is implementation-specific; it may be "none". * @param f the path of the file to open * @param overwrite if a file with this name already exists, then if true, * the file will be overwritten, and if false an error will be thrown. * @param bufferSize the size of the buffer to be used. * @throws IOException IO failure */ public FSDataOutputStream create(Path f, boolean overwrite, int bufferSize, Progressable progress) throws IOException { return create(f, overwrite, bufferSize, getDefaultReplication(f), getDefaultBlockSize(f), progress); } /** * Create an FSDataOutputStream at the indicated Path. * @param f the file name to open * @param overwrite if a file with this name already exists, then if true, * the file will be overwritten, and if false an error will be thrown. * @param bufferSize the size of the buffer to be used. * @param replication required block replication for the file. * @throws IOException IO failure */ public FSDataOutputStream create(Path f, boolean overwrite, int bufferSize, short replication, long blockSize) throws IOException { return create(f, overwrite, bufferSize, replication, blockSize, null); } /** * Create an FSDataOutputStream at the indicated Path with write-progress * reporting. * @param f the file name to open * @param overwrite if a file with this name already exists, then if true, * the file will be overwritten, and if false an error will be thrown. * @param bufferSize the size of the buffer to be used. * @param replication required block replication for the file. * @throws IOException IO failure */ public FSDataOutputStream create(Path f, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { return this.create(f, FsCreateModes.applyUMask(FsPermission.getFileDefault(), FsPermission.getUMask(getConf())), overwrite, bufferSize, replication, blockSize, progress); } /** * Create an FSDataOutputStream at the indicated Path with write-progress * reporting. * @param f the file name to open * @param permission file permission * @param overwrite if a file with this name already exists, then if true, * the file will be overwritten, and if false an error will be thrown. * @param bufferSize the size of the buffer to be used. * @param replication required block replication for the file. * @param blockSize block size * @param progress the progress reporter * @throws IOException IO failure * @see #setPermission(Path, FsPermission) */ public abstract FSDataOutputStream create(Path f, FsPermission permission, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException; /** * Create an FSDataOutputStream at the indicated Path with write-progress * reporting. * @param f the file name to open * @param permission file permission * @param flags {@link CreateFlag}s to use for this stream. * @param bufferSize the size of the buffer to be used. * @param replication required block replication for the file. * @param blockSize block size * @param progress the progress reporter * @throws IOException IO failure * @see #setPermission(Path, FsPermission) */ public FSDataOutputStream create(Path f, FsPermission permission, EnumSet<CreateFlag> flags, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { return create(f, permission, flags, bufferSize, replication, blockSize, progress, null); } /** * Create an FSDataOutputStream at the indicated Path with a custom * checksum option. * @param f the file name to open * @param permission file permission * @param flags {@link CreateFlag}s to use for this stream. * @param bufferSize the size of the buffer to be used. * @param replication required block replication for the file. * @param blockSize block size * @param progress the progress reporter * @param checksumOpt checksum parameter. If null, the values * found in conf will be used. * @throws IOException IO failure * @see #setPermission(Path, FsPermission) */ public FSDataOutputStream create(Path f, FsPermission permission, EnumSet<CreateFlag> flags, int bufferSize, short replication, long blockSize, Progressable progress, ChecksumOpt checksumOpt) throws IOException { // Checksum options are ignored by default. The file systems that // implement checksum need to override this method. The full // support is currently only available in DFS. return create(f, permission, flags.contains(CreateFlag.OVERWRITE), bufferSize, replication, blockSize, progress); } /** * This create has been added to support the FileContext that processes * the permission with umask before calling this method. * This a temporary method added to support the transition from FileSystem * to FileContext for user applications. * @throws IOException IO failure */ @Deprecated protected FSDataOutputStream primitiveCreate(Path f, FsPermission absolutePermission, EnumSet<CreateFlag> flag, int bufferSize, short replication, long blockSize, Progressable progress, ChecksumOpt checksumOpt) throws IOException { boolean pathExists = exists(f); CreateFlag.validate(f, pathExists, flag); // Default impl assumes that permissions do not matter and // nor does the bytesPerChecksum hence // calling the regular create is good enough. // FSs that implement permissions should override this. if (pathExists && flag.contains(CreateFlag.APPEND)) { return append(f, bufferSize, progress); } return this.create(f, absolutePermission, flag.contains(CreateFlag.OVERWRITE), bufferSize, replication, blockSize, progress); } /** * This version of the mkdirs method assumes that the permission is absolute. * It has been added to support the FileContext that processes the permission * with umask before calling this method. * This a temporary method added to support the transition from FileSystem * to FileContext for user applications. * @param f path * @param absolutePermission permissions * @return true if the directory was actually created. * @throws IOException IO failure * @see #mkdirs(Path, FsPermission) */ @Deprecated protected boolean primitiveMkdir(Path f, FsPermission absolutePermission) throws IOException { return this.mkdirs(f, absolutePermission); } /** * This version of the mkdirs method assumes that the permission is absolute. * It has been added to support the FileContext that processes the permission * with umask before calling this method. * This a temporary method added to support the transition from FileSystem * to FileContext for user applications. */ @Deprecated protected void primitiveMkdir(Path f, FsPermission absolutePermission, boolean createParent) throws IOException { if (!createParent) { // parent must exist. // since the this.mkdirs makes parent dirs automatically // we must throw exception if parent does not exist. final FileStatus stat = getFileStatus(f.getParent()); if (stat == null) { throw new FileNotFoundException("Missing parent:" + f); } if (!stat.isDirectory()) { throw new ParentNotDirectoryException("parent is not a dir"); } // parent does exist - go ahead with mkdir of leaf } // Default impl is to assume that permissions do not matter and hence // calling the regular mkdirs is good enough. // FSs that implement permissions should override this. if (!this.mkdirs(f, absolutePermission)) { throw new IOException("mkdir of " + f + " failed"); } } /** * Opens an FSDataOutputStream at the indicated Path with write-progress * reporting. Same as create(), except fails if parent directory doesn't * already exist. * @param f the file name to open * @param overwrite if a file with this name already exists, then if true, * the file will be overwritten, and if false an error will be thrown. * @param bufferSize the size of the buffer to be used. * @param replication required block replication for the file. * @param blockSize block size * @param progress the progress reporter * @throws IOException IO failure * @see #setPermission(Path, FsPermission) */ public FSDataOutputStream createNonRecursive(Path f, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { return this.createNonRecursive(f, FsPermission.getFileDefault(), overwrite, bufferSize, replication, blockSize, progress); } /** * Opens an FSDataOutputStream at the indicated Path with write-progress * reporting. Same as create(), except fails if parent directory doesn't * already exist. * @param f the file name to open * @param permission file permission * @param overwrite if a file with this name already exists, then if true, * the file will be overwritten, and if false an error will be thrown. * @param bufferSize the size of the buffer to be used. * @param replication required block replication for the file. * @param blockSize block size * @param progress the progress reporter * @throws IOException IO failure * @see #setPermission(Path, FsPermission) */ public FSDataOutputStream createNonRecursive(Path f, FsPermission permission, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { return createNonRecursive(f, permission, overwrite ? EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE) : EnumSet.of(CreateFlag.CREATE), bufferSize, replication, blockSize, progress); } /** * Opens an FSDataOutputStream at the indicated Path with write-progress * reporting. Same as create(), except fails if parent directory doesn't * already exist. * @param f the file name to open * @param permission file permission * @param flags {@link CreateFlag}s to use for this stream. * @param bufferSize the size of the buffer to be used. * @param replication required block replication for the file. * @param blockSize block size * @param progress the progress reporter * @throws IOException IO failure * @see #setPermission(Path, FsPermission) */ public FSDataOutputStream createNonRecursive(Path f, FsPermission permission, EnumSet<CreateFlag> flags, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { throw new IOException("createNonRecursive unsupported for this filesystem " + this.getClass()); } /** * Creates the given Path as a brand-new zero-length file. If * create fails, or if it already existed, return false. * <i>Important: the default implementation is not atomic</i> * @param f path to use for create * @throws IOException IO failure */ public boolean createNewFile(Path f) throws IOException { if (exists(f)) { return false; } else { create(f, false, getConf().getInt(IO_FILE_BUFFER_SIZE_KEY, IO_FILE_BUFFER_SIZE_DEFAULT)).close(); return true; } } /** * Append to an existing file (optional operation). * Same as * {@code append(f, getConf().getInt(IO_FILE_BUFFER_SIZE_KEY, * IO_FILE_BUFFER_SIZE_DEFAULT), null)} * @param f the existing file to be appended. * @throws IOException IO failure * @throws UnsupportedOperationException if the operation is unsupported * (default). */ public FSDataOutputStream append(Path f) throws IOException { return append(f, getConf().getInt(IO_FILE_BUFFER_SIZE_KEY, IO_FILE_BUFFER_SIZE_DEFAULT), null); } /** * Append to an existing file (optional operation). * Same as append(f, bufferSize, null). * @param f the existing file to be appended. * @param bufferSize the size of the buffer to be used. * @throws IOException IO failure * @throws UnsupportedOperationException if the operation is unsupported * (default). */ public FSDataOutputStream append(Path f, int bufferSize) throws IOException { return append(f, bufferSize, null); } /** * Append to an existing file (optional operation). * @param f the existing file to be appended. * @param bufferSize the size of the buffer to be used. * @param progress for reporting progress if it is not null. * @throws IOException IO failure * @throws UnsupportedOperationException if the operation is unsupported * (default). */ public abstract FSDataOutputStream append(Path f, int bufferSize, Progressable progress) throws IOException; /** * Concat existing files together. * @param trg the path to the target destination. * @param psrcs the paths to the sources to use for the concatenation. * @throws IOException IO failure * @throws UnsupportedOperationException if the operation is unsupported * (default). */ public void concat(final Path trg, final Path[] psrcs) throws IOException { throw new UnsupportedOperationException( "Not implemented by the " + getClass().getSimpleName() + " FileSystem implementation"); } /** * Get the replication factor. * * @deprecated Use {@link #getFileStatus(Path)} instead * @param src file name * @return file replication * @throws FileNotFoundException if the path does not resolve. * @throws IOException an IO failure */ @Deprecated public short getReplication(Path src) throws IOException { return getFileStatus(src).getReplication(); } /** * Set the replication for an existing file. * If a filesystem does not support replication, it will always * return true: the check for a file existing may be bypassed. * This is the default behavior. * @param src file name * @param replication new replication * @throws IOException * @return true if successful, or the feature in unsupported; * false if replication is supported but the file does not exist, * or is a directory */ public boolean setReplication(Path src, short replication) throws IOException { return true; } /** * Renames Path src to Path dst. * @param src path to be renamed * @param dst new path after rename * @throws IOException on failure * @return true if rename is successful */ public abstract boolean rename(Path src, Path dst) throws IOException; /** * Renames Path src to Path dst * <ul> * <li>Fails if src is a file and dst is a directory.</li> * <li>Fails if src is a directory and dst is a file.</li> * <li>Fails if the parent of dst does not exist or is a file.</li> * </ul> * <p> * If OVERWRITE option is not passed as an argument, rename fails * if the dst already exists. * <p> * If OVERWRITE option is passed as an argument, rename overwrites * the dst if it is a file or an empty directory. Rename fails if dst is * a non-empty directory. * <p> * Note that atomicity of rename is dependent on the file system * implementation. Please refer to the file system documentation for * details. This default implementation is non atomic. * <p> * This method is deprecated since it is a temporary method added to * support the transition from FileSystem to FileContext for user * applications. * * @param src path to be renamed * @param dst new path after rename * @throws FileNotFoundException src path does not exist, or the parent * path of dst does not exist. * @throws FileAlreadyExistsException dest path exists and is a file * @throws ParentNotDirectoryException if the parent path of dest is not * a directory * @throws IOException on failure */ @Deprecated protected void rename(final Path src, final Path dst, final Rename... options) throws IOException { // Default implementation final FileStatus srcStatus = getFileLinkStatus(src); if (srcStatus == null) { throw new FileNotFoundException("rename source " + src + " not found."); } boolean overwrite = false; if (null != options) { for (Rename option : options) { if (option == Rename.OVERWRITE) { overwrite = true; } } } FileStatus dstStatus; try { dstStatus = getFileLinkStatus(dst); } catch (IOException e) { dstStatus = null; } if (dstStatus != null) { if (srcStatus.isDirectory() != dstStatus.isDirectory()) { throw new IOException( "Source " + src + " Destination " + dst + " both should be either file or directory"); } if (!overwrite) { throw new FileAlreadyExistsException("rename destination " + dst + " already exists."); } // Delete the destination that is a file or an empty directory if (dstStatus.isDirectory()) { FileStatus[] list = listStatus(dst); if (list != null && list.length != 0) { throw new IOException("rename cannot overwrite non empty destination directory " + dst); } } delete(dst, false); } else { final Path parent = dst.getParent(); final FileStatus parentStatus = getFileStatus(parent); if (parentStatus == null) { throw new FileNotFoundException("rename destination parent " + parent + " not found."); } if (!parentStatus.isDirectory()) { throw new ParentNotDirectoryException("rename destination parent " + parent + " is a file."); } } if (!rename(src, dst)) { throw new IOException("rename from " + src + " to " + dst + " failed."); } } /** * Truncate the file in the indicated path to the indicated size. * <ul> * <li>Fails if path is a directory.</li> * <li>Fails if path does not exist.</li> * <li>Fails if path is not closed.</li> * <li>Fails if new size is greater than current size.</li> * </ul> * @param f The path to the file to be truncated * @param newLength The size the file is to be truncated to * * @return <code>true</code> if the file has been truncated to the desired * <code>newLength</code> and is immediately available to be reused for * write operations such as <code>append</code>, or * <code>false</code> if a background process of adjusting the length of * the last block has been started, and clients should wait for it to * complete before proceeding with further file updates. * @throws IOException IO failure * @throws UnsupportedOperationException if the operation is unsupported * (default). */ public boolean truncate(Path f, long newLength) throws IOException { throw new UnsupportedOperationException( "Not implemented by the " + getClass().getSimpleName() + " FileSystem implementation"); } /** * Delete a file/directory. * @deprecated Use {@link #delete(Path, boolean)} instead. */ @Deprecated public boolean delete(Path f) throws IOException { return delete(f, true); } /** Delete a file. * * @param f the path to delete. * @param recursive if path is a directory and set to * true, the directory is deleted else throws an exception. In * case of a file the recursive can be set to either true or false. * @return true if delete is successful else false. * @throws IOException IO failure */ public abstract boolean delete(Path f, boolean recursive) throws IOException; /** * Mark a path to be deleted when its FileSystem is closed. * When the JVM shuts down cleanly, all cached FileSystem objects will be * closed automatically. These the marked paths will be deleted as a result. * * If a FileSystem instance is not cached, i.e. has been created with * {@link #createFileSystem(URI, Configuration)}, then the paths will * be deleted in when {@link #close()} is called on that instance. * * The path must exist in the filesystem at the time of the method call; * it does not have to exist at the time of JVM shutdown. * * Notes * <ol> * <li>Clean shutdown of the JVM cannot be guaranteed.</li> * <li>The time to shut down a FileSystem will depends on the number of * files to delete. For filesystems where the cost of checking * for the existence of a file/directory and the actual delete operation * (for example: object stores) is high, the time to shutdown the JVM can be * significantly extended by over-use of this feature.</li> * <li>Connectivity problems with a remote filesystem may delay shutdown * further, and may cause the files to not be deleted.</li> * </ol> * @param f the path to delete. * @return true if deleteOnExit is successful, otherwise false. * @throws IOException IO failure */ public boolean deleteOnExit(Path f) throws IOException { if (!exists(f)) { return false; } synchronized (deleteOnExit) { deleteOnExit.add(f); } return true; } /** * Cancel the scheduled deletion of the path when the FileSystem is closed. * @param f the path to cancel deletion * @return true if the path was found in the delete-on-exit list. */ public boolean cancelDeleteOnExit(Path f) { synchronized (deleteOnExit) { return deleteOnExit.remove(f); } } /** * Delete all paths that were marked as delete-on-exit. This recursively * deletes all files and directories in the specified paths. * * The time to process this operation is {@code O(paths)}, with the actual * time dependent on the time for existence and deletion operations to * complete, successfully or not. */ protected void processDeleteOnExit() { synchronized (deleteOnExit) { for (Iterator<Path> iter = deleteOnExit.iterator(); iter.hasNext();) { Path path = iter.next(); try { if (exists(path)) { delete(path, true); } } catch (IOException e) { LOGGER.info("Ignoring failure to deleteOnExit for path {}", path); } iter.remove(); } } } /** Check if a path exists. * * It is highly discouraged to call this method back to back with other * {@link #getFileStatus(Path)} calls, as this will involve multiple redundant * RPC calls in HDFS. * * @param f source path * @return true if the path exists * @throws IOException IO failure */ public boolean exists(Path f) throws IOException { try { return getFileStatus(f) != null; } catch (FileNotFoundException e) { return false; } } /** True iff the named path is a directory. * Note: Avoid using this method. Instead reuse the FileStatus * returned by getFileStatus() or listStatus() methods. * * @param f path to check * @throws IOException IO failure * @deprecated Use {@link #getFileStatus(Path)} instead */ @Deprecated public boolean isDirectory(Path f) throws IOException { try { return getFileStatus(f).isDirectory(); } catch (FileNotFoundException e) { return false; // f does not exist } } /** True iff the named path is a regular file. * Note: Avoid using this method. Instead reuse the FileStatus * returned by {@link #getFileStatus(Path)} or listStatus() methods. * * @param f path to check * @throws IOException IO failure * @deprecated Use {@link #getFileStatus(Path)} instead */ @Deprecated public boolean isFile(Path f) throws IOException { try { return getFileStatus(f).isFile(); } catch (FileNotFoundException e) { return false; // f does not exist } } /** * The number of bytes in a file. * @return the number of bytes; 0 for a directory * @deprecated Use {@link #getFileStatus(Path)} instead. * @throws FileNotFoundException if the path does not resolve * @throws IOException IO failure */ @Deprecated public long getLength(Path f) throws IOException { return getFileStatus(f).getLen(); } /** Return the {@link ContentSummary} of a given {@link Path}. * @param f path to use * @throws FileNotFoundException if the path does not resolve * @throws IOException IO failure */ public ContentSummary getContentSummary(Path f) throws IOException { FileStatus status = getFileStatus(f); if (status.isFile()) { // f is a file long length = status.getLen(); return new ContentSummary.Builder().length(length).fileCount(1).directoryCount(0).spaceConsumed(length) .build(); } // f is a directory long[] summary = { 0, 0, 1 }; for (FileStatus s : listStatus(f)) { long length = s.getLen(); ContentSummary c = s.isDirectory() ? getContentSummary(s.getPath()) : new ContentSummary.Builder().length(length).fileCount(1).directoryCount(0) .spaceConsumed(length).build(); summary[0] += c.getLength(); summary[1] += c.getFileCount(); summary[2] += c.getDirectoryCount(); } return new ContentSummary.Builder().length(summary[0]).fileCount(summary[1]).directoryCount(summary[2]) .spaceConsumed(summary[0]).build(); } /** Return the {@link QuotaUsage} of a given {@link Path}. * @param f path to use * @return the quota usage * @throws IOException IO failure */ public QuotaUsage getQuotaUsage(Path f) throws IOException { return getContentSummary(f); } /** * The default filter accepts all paths. */ private static final PathFilter DEFAULT_FILTER = new PathFilter() { @Override public boolean accept(Path file) { return true; } }; /** * List the statuses of the files/directories in the given path if the path is * a directory. * <p> * Does not guarantee to return the List of files/directories status in a * sorted order. * <p> * Will not return null. Expect IOException upon access error. * @param f given path * @return the statuses of the files/directories in the given patch * @throws FileNotFoundException when the path does not exist * @throws IOException see specific implementation */ public abstract FileStatus[] listStatus(Path f) throws FileNotFoundException, IOException; /** * Represents a batch of directory entries when iteratively listing a * directory. This is a private API not meant for use by end users. * <p> * For internal use by FileSystem subclasses that override * {@link FileSystem#listStatusBatch(Path, byte[])} to implement iterative * listing. */ @InterfaceAudience.Private public static class DirectoryEntries { private final FileStatus[] entries; private final byte[] token; private final boolean hasMore; public DirectoryEntries(FileStatus[] entries, byte[] token, boolean hasMore) { this.entries = entries; if (token != null) { this.token = token.clone(); } else { this.token = null; } this.hasMore = hasMore; } public FileStatus[] getEntries() { return entries; } public byte[] getToken() { return token; } public boolean hasMore() { return hasMore; } } /** * Given an opaque iteration token, return the next batch of entries in a * directory. This is a private API not meant for use by end users. * <p> * This method should be overridden by FileSystem subclasses that want to * use the generic {@link FileSystem#listStatusIterator(Path)} implementation. * @param f Path to list * @param token opaque iteration token returned by previous call, or null * if this is the first call. * @return * @throws FileNotFoundException * @throws IOException */ @InterfaceAudience.Private protected DirectoryEntries listStatusBatch(Path f, byte[] token) throws FileNotFoundException, IOException { // The default implementation returns the entire listing as a single batch. // Thus, there is never a second batch, and no need to respect the passed // token or set a token in the returned DirectoryEntries. FileStatus[] listing = listStatus(f); return new DirectoryEntries(listing, null, false); } /** * Filter files/directories in the given path using the user-supplied path * filter. Results are added to the given array <code>results</code>. * @throws FileNotFoundException when the path does not exist * @throws IOException see specific implementation */ private void listStatus(ArrayList<FileStatus> results, Path f, PathFilter filter) throws FileNotFoundException, IOException { FileStatus listing[] = listStatus(f); Preconditions.checkNotNull(listing, "listStatus should not return NULL"); for (int i = 0; i < listing.length; i++) { if (filter.accept(listing[i].getPath())) { results.add(listing[i]); } } } /** * List corrupted file blocks. * @return an iterator over the corrupt files under the given path * (may contain duplicates if a file has more than one corrupt block) * @throws UnsupportedOperationException if the operation is unsupported * (default). * @throws IOException IO failure */ public RemoteIterator<Path> listCorruptFileBlocks(Path path) throws IOException { throw new UnsupportedOperationException( getClass().getCanonicalName() + " does not support listCorruptFileBlocks"); } /** * Filter files/directories in the given path using the user-supplied path * filter. * <p> * Does not guarantee to return the List of files/directories status in a * sorted order. * * @param f * a path name * @param filter * the user-supplied path filter * @return an array of FileStatus objects for the files under the given path * after applying the filter * @throws FileNotFoundException when the path does not exist * @throws IOException see specific implementation */ public FileStatus[] listStatus(Path f, PathFilter filter) throws FileNotFoundException, IOException { ArrayList<FileStatus> results = new ArrayList<>(); listStatus(results, f, filter); return results.toArray(new FileStatus[results.size()]); } /** * Filter files/directories in the given list of paths using default * path filter. * <p> * Does not guarantee to return the List of files/directories status in a * sorted order. * * @param files * a list of paths * @return a list of statuses for the files under the given paths after * applying the filter default Path filter * @throws FileNotFoundException when the path does not exist * @throws IOException see specific implementation */ public FileStatus[] listStatus(Path[] files) throws FileNotFoundException, IOException { return listStatus(files, DEFAULT_FILTER); } /** * Filter files/directories in the given list of paths using user-supplied * path filter. * <p> * Does not guarantee to return the List of files/directories status in a * sorted order. * * @param files * a list of paths * @param filter * the user-supplied path filter * @return a list of statuses for the files under the given paths after * applying the filter * @throws FileNotFoundException when the path does not exist * @throws IOException see specific implementation */ public FileStatus[] listStatus(Path[] files, PathFilter filter) throws FileNotFoundException, IOException { ArrayList<FileStatus> results = new ArrayList<FileStatus>(); for (int i = 0; i < files.length; i++) { listStatus(results, files[i], filter); } return results.toArray(new FileStatus[results.size()]); } /** * <p>Return all the files that match filePattern and are not checksum * files. Results are sorted by their names. * * <p> * A filename pattern is composed of <i>regular</i> characters and * <i>special pattern matching</i> characters, which are: * * <dl> * <dd> * <dl> * <p> * <dt> <tt> ? </tt> * <dd> Matches any single character. * * <p> * <dt> <tt> * </tt> * <dd> Matches zero or more characters. * * <p> * <dt> <tt> [<i>abc</i>] </tt> * <dd> Matches a single character from character set * <tt>{<i>a,b,c</i>}</tt>. * * <p> * <dt> <tt> [<i>a</i>-<i>b</i>] </tt> * <dd> Matches a single character from the character range * <tt>{<i>a...b</i>}</tt>. Note that character <tt><i>a</i></tt> must be * lexicographically less than or equal to character <tt><i>b</i></tt>. * * <p> * <dt> <tt> [^<i>a</i>] </tt> * <dd> Matches a single character that is not from character set or range * <tt>{<i>a</i>}</tt>. Note that the <tt>^</tt> character must occur * immediately to the right of the opening bracket. * * <p> * <dt> <tt> \<i>c</i> </tt> * <dd> Removes (escapes) any special meaning of character <i>c</i>. * * <p> * <dt> <tt> {ab,cd} </tt> * <dd> Matches a string from the string set <tt>{<i>ab, cd</i>} </tt> * * <p> * <dt> <tt> {ab,c{de,fh}} </tt> * <dd> Matches a string from the string set <tt>{<i>ab, cde, cfh</i>}</tt> * * </dl> * </dd> * </dl> * * @param pathPattern a glob specifying a path pattern * @return an array of paths that match the path pattern * @throws IOException IO failure */ public FileStatus[] globStatus(Path pathPattern) throws IOException { return new Globber(this, pathPattern, DEFAULT_FILTER).glob(); } /** * Return an array of {@link FileStatus} objects whose path names match * {@code pathPattern} and is accepted by the user-supplied path filter. * Results are sorted by their path names. * * @param pathPattern a glob specifying the path pattern * @param filter a user-supplied path filter * @return null if {@code pathPattern} has no glob and the path does not exist * an empty array if {@code pathPattern} has a glob and no path * matches it else an array of {@link FileStatus} objects matching the * pattern * @throws IOException if any I/O error occurs when fetching file status */ public FileStatus[] globStatus(Path pathPattern, PathFilter filter) throws IOException { return new Globber(this, pathPattern, filter).glob(); } /** * List the statuses of the files/directories in the given path if the path is * a directory. * Return the file's status and block locations If the path is a file. * * If a returned status is a file, it contains the file's block locations. * * @param f is the path * * @return an iterator that traverses statuses of the files/directories * in the given path * * @throws FileNotFoundException If <code>f</code> does not exist * @throws IOException If an I/O error occurred */ public RemoteIterator<LocatedFileStatus> listLocatedStatus(final Path f) throws FileNotFoundException, IOException { return listLocatedStatus(f, DEFAULT_FILTER); } /** * List a directory. * The returned results include its block location if it is a file * The results are filtered by the given path filter * @param f a path * @param filter a path filter * @return an iterator that traverses statuses of the files/directories * in the given path * @throws FileNotFoundException if <code>f</code> does not exist * @throws IOException if any I/O error occurred */ protected RemoteIterator<LocatedFileStatus> listLocatedStatus(final Path f, final PathFilter filter) throws FileNotFoundException, IOException { return new RemoteIterator<LocatedFileStatus>() { private final FileStatus[] stats = listStatus(f, filter); private int i = 0; @Override public boolean hasNext() { return i < stats.length; } @Override public LocatedFileStatus next() throws IOException { if (!hasNext()) { throw new NoSuchElementException("No more entries in " + f); } FileStatus result = stats[i++]; // for files, use getBlockLocations(FileStatus, int, int) to avoid // calling getFileStatus(Path) to load the FileStatus again BlockLocation[] locs = result.isFile() ? getFileBlockLocations(result, 0, result.getLen()) : null; return new LocatedFileStatus(result, locs); } }; } /** * Generic iterator for implementing {@link #listStatusIterator(Path)}. */ protected class DirListingIterator<T extends FileStatus> implements RemoteIterator<T> { private final Path path; private DirectoryEntries entries; private int i = 0; DirListingIterator(Path path) { this.path = path; } @Override public boolean hasNext() throws IOException { if (entries == null) { fetchMore(); } return i < entries.getEntries().length || entries.hasMore(); } private void fetchMore() throws IOException { byte[] token = null; if (entries != null) { token = entries.getToken(); } entries = listStatusBatch(path, token); i = 0; } @Override @SuppressWarnings("unchecked") public T next() throws IOException { Preconditions.checkState(hasNext(), "No more items in iterator"); if (i == entries.getEntries().length) { fetchMore(); } return (T) entries.getEntries()[i++]; } } /** * Returns a remote iterator so that followup calls are made on demand * while consuming the entries. Each FileSystem implementation should * override this method and provide a more efficient implementation, if * possible. * * Does not guarantee to return the iterator that traverses statuses * of the files in a sorted order. * * @param p target path * @return remote iterator * @throws FileNotFoundException if <code>p</code> does not exist * @throws IOException if any I/O error occurred */ public RemoteIterator<FileStatus> listStatusIterator(final Path p) throws FileNotFoundException, IOException { return new DirListingIterator<>(p); } /** * List the statuses and block locations of the files in the given path. * Does not guarantee to return the iterator that traverses statuses * of the files in a sorted order. * * <pre> * If the path is a directory, * if recursive is false, returns files in the directory; * if recursive is true, return files in the subtree rooted at the path. * If the path is a file, return the file's status and block locations. * </pre> * @param f is the path * @param recursive if the subdirectories need to be traversed recursively * * @return an iterator that traverses statuses of the files * * @throws FileNotFoundException when the path does not exist; * @throws IOException see specific implementation */ public RemoteIterator<LocatedFileStatus> listFiles(final Path f, final boolean recursive) throws FileNotFoundException, IOException { return new RemoteIterator<LocatedFileStatus>() { private Stack<RemoteIterator<LocatedFileStatus>> itors = new Stack<>(); private RemoteIterator<LocatedFileStatus> curItor = listLocatedStatus(f); private LocatedFileStatus curFile; @Override public boolean hasNext() throws IOException { while (curFile == null) { if (curItor.hasNext()) { handleFileStat(curItor.next()); } else if (!itors.empty()) { curItor = itors.pop(); } else { return false; } } return true; } /** * Process the input stat. * If it is a file, return the file stat. * If it is a directory, traverse the directory if recursive is true; * ignore it if recursive is false. * @param stat input status * @throws IOException if any IO error occurs */ private void handleFileStat(LocatedFileStatus stat) throws IOException { if (stat.isFile()) { // file curFile = stat; } else if (recursive) { // directory itors.push(curItor); curItor = listLocatedStatus(stat.getPath()); } } @Override public LocatedFileStatus next() throws IOException { if (hasNext()) { LocatedFileStatus result = curFile; curFile = null; return result; } throw new java.util.NoSuchElementException("No more entry in " + f); } }; } /** Return the current user's home directory in this FileSystem. * The default implementation returns {@code "/user/$USER/"}. */ public Path getHomeDirectory() { return this.makeQualified(new Path(USER_HOME_PREFIX + "/" + System.getProperty("user.name"))); } /** * Set the current working directory for the given FileSystem. All relative * paths will be resolved relative to it. * * @param new_dir Path of new working directory */ public abstract void setWorkingDirectory(Path new_dir); /** * Get the current working directory for the given FileSystem * @return the directory pathname */ public abstract Path getWorkingDirectory(); /** * Note: with the new FileContext class, getWorkingDirectory() * will be removed. * The working directory is implemented in FileContext. * * Some FileSystems like LocalFileSystem have an initial workingDir * that we use as the starting workingDir. For other file systems * like HDFS there is no built in notion of an initial workingDir. * * @return if there is built in notion of workingDir then it * is returned; else a null is returned. */ protected Path getInitialWorkingDirectory() { return null; } /** * Call {@link #mkdirs(Path, FsPermission)} with default permission. * @param f path * @return true if the directory was created * @throws IOException IO failure */ public boolean mkdirs(Path f) throws IOException { return mkdirs(f, FsPermission.getDirDefault()); } /** * Make the given file and all non-existent parents into * directories. Has roughly the semantics of Unix @{code mkdir -p}. * Existence of the directory hierarchy is not an error. * @param f path to create * @param permission to apply to f * @throws IOException IO failure */ public abstract boolean mkdirs(Path f, FsPermission permission) throws IOException; /** * The src file is on the local disk. Add it to filesystem at * the given dst name and the source is kept intact afterwards * @param src path * @param dst path * @throws IOException IO failure */ public void copyFromLocalFile(Path src, Path dst) throws IOException { copyFromLocalFile(false, src, dst); } /** * The src files is on the local disk. Add it to filesystem at * the given dst name, removing the source afterwards. * @param srcs source paths * @param dst path * @throws IOException IO failure */ public void moveFromLocalFile(Path[] srcs, Path dst) throws IOException { copyFromLocalFile(true, true, srcs, dst); } /** * The src file is on the local disk. Add it to the filesystem at * the given dst name, removing the source afterwards. * @param src local path * @param dst path * @throws IOException IO failure */ public void moveFromLocalFile(Path src, Path dst) throws IOException { copyFromLocalFile(true, src, dst); } /** * The src file is on the local disk. Add it to the filesystem at * the given dst name. * delSrc indicates if the source should be removed * @param delSrc whether to delete the src * @param src path * @param dst path */ public void copyFromLocalFile(boolean delSrc, Path src, Path dst) throws IOException { copyFromLocalFile(delSrc, true, src, dst); } /** * The src files are on the local disk. Add it to the filesystem at * the given dst name. * delSrc indicates if the source should be removed * @param delSrc whether to delete the src * @param overwrite whether to overwrite an existing file * @param srcs array of paths which are source * @param dst path * @throws IOException IO failure */ public void copyFromLocalFile(boolean delSrc, boolean overwrite, Path[] srcs, Path dst) throws IOException { Configuration conf = getConf(); FileUtil.copy(getLocal(conf), srcs, this, dst, delSrc, overwrite, conf); } /** * The src file is on the local disk. Add it to the filesystem at * the given dst name. * delSrc indicates if the source should be removed * @param delSrc whether to delete the src * @param overwrite whether to overwrite an existing file * @param src path * @param dst path * @throws IOException IO failure */ public void copyFromLocalFile(boolean delSrc, boolean overwrite, Path src, Path dst) throws IOException { Configuration conf = getConf(); FileUtil.copy(getLocal(conf), src, this, dst, delSrc, overwrite, conf); } /** * Copy it a file from the remote filesystem to the local one. * @param src path src file in the remote filesystem * @param dst path local destination * @throws IOException IO failure */ public void copyToLocalFile(Path src, Path dst) throws IOException { copyToLocalFile(false, src, dst); } /** * Copy a file to the local filesystem, then delete it from the * remote filesystem (if successfully copied). * @param src path src file in the remote filesystem * @param dst path local destination * @throws IOException IO failure */ public void moveToLocalFile(Path src, Path dst) throws IOException { copyToLocalFile(true, src, dst); } /** * Copy it a file from a remote filesystem to the local one. * delSrc indicates if the src will be removed or not. * @param delSrc whether to delete the src * @param src path src file in the remote filesystem * @param dst path local destination * @throws IOException IO failure */ public void copyToLocalFile(boolean delSrc, Path src, Path dst) throws IOException { copyToLocalFile(delSrc, src, dst, false); } /** * The src file is under this filesystem, and the dst is on the local disk. * Copy it from the remote filesystem to the local dst name. * delSrc indicates if the src will be removed * or not. useRawLocalFileSystem indicates whether to use RawLocalFileSystem * as the local file system or not. RawLocalFileSystem is non checksumming, * So, It will not create any crc files at local. * * @param delSrc * whether to delete the src * @param src * path * @param dst * path * @param useRawLocalFileSystem * whether to use RawLocalFileSystem as local file system or not. * * @throws IOException for any IO error */ public void copyToLocalFile(boolean delSrc, Path src, Path dst, boolean useRawLocalFileSystem) throws IOException { Configuration conf = getConf(); FileSystem local = null; if (useRawLocalFileSystem) { local = getLocal(conf).getRawFileSystem(); } else { local = getLocal(conf); } FileUtil.copy(this, src, local, dst, delSrc, conf); } /** * Returns a local file that the user can write output to. The caller * provides both the eventual target name in this FileSystem * and the local working file path. * If this FileSystem is local, we write directly into the target. If * the FileSystem is not local, we write into the tmp local area. * @param fsOutputFile path of output file * @param tmpLocalFile path of local tmp file * @throws IOException IO failure */ public Path startLocalOutput(Path fsOutputFile, Path tmpLocalFile) throws IOException { return tmpLocalFile; } /** * Called when we're all done writing to the target. * A local FS will do nothing, because we've written to exactly the * right place. * A remote FS will copy the contents of tmpLocalFile to the correct target at * fsOutputFile. * @param fsOutputFile path of output file * @param tmpLocalFile path to local tmp file * @throws IOException IO failure */ public void completeLocalOutput(Path fsOutputFile, Path tmpLocalFile) throws IOException { moveFromLocalFile(tmpLocalFile, fsOutputFile); } /** * Close this FileSystem instance. * Will release any held locks, delete all files queued for deletion * through calls to {@link #deleteOnExit(Path)}, and remove this FS instance * from the cache, if cached. * * After this operation, the outcome of any method call on this FileSystem * instance, or any input/output stream created by it is <i>undefined</i>. * @throws IOException IO failure */ @Override public void close() throws IOException { // delete all files that were marked as delete-on-exit. processDeleteOnExit(); CACHE.remove(this.key, this); } /** * Return the total size of all files in the filesystem. * @throws IOException IO failure */ public long getUsed() throws IOException { Path path = new Path("/"); return getUsed(path); } /** * Return the total size of all files from a specified path. * @throws IOException IO failure */ public long getUsed(Path path) throws IOException { return getContentSummary(path).getLength(); } /** * Get the block size for a particular file. * @param f the filename * @return the number of bytes in a block * @deprecated Use {@link #getFileStatus(Path)} instead * @throws FileNotFoundException if the path is not present * @throws IOException IO failure */ @Deprecated public long getBlockSize(Path f) throws IOException { return getFileStatus(f).getBlockSize(); } /** * Return the number of bytes that large input files should be optimally * be split into to minimize I/O time. * @deprecated use {@link #getDefaultBlockSize(Path)} instead */ @Deprecated public long getDefaultBlockSize() { // default to 32MB: large enough to minimize the impact of seeks return getConf().getLong("fs.local.block.size", 32 * 1024 * 1024); } /** * Return the number of bytes that large input files should be optimally * be split into to minimize I/O time. The given path will be used to * locate the actual filesystem. The full path does not have to exist. * @param f path of file * @return the default block size for the path's filesystem */ public long getDefaultBlockSize(Path f) { return getDefaultBlockSize(); } /** * Get the default replication. * @return the replication; the default value is "1". * @deprecated use {@link #getDefaultReplication(Path)} instead */ @Deprecated public short getDefaultReplication() { return 1; } /** * Get the default replication for a path. * The given path will be used to locate the actual FileSystem to query. * The full path does not have to exist. * @param path of the file * @return default replication for the path's filesystem */ public short getDefaultReplication(Path path) { return getDefaultReplication(); } /** * Return a file status object that represents the path. * @param f The path we want information from * @return a FileStatus object * @throws FileNotFoundException when the path does not exist * @throws IOException see specific implementation */ public abstract FileStatus getFileStatus(Path f) throws IOException; /** * Checks if the user can access a path. The mode specifies which access * checks to perform. If the requested permissions are granted, then the * method returns normally. If access is denied, then the method throws an * {@link AccessControlException}. * <p> * The default implementation calls {@link #getFileStatus(Path)} * and checks the returned permissions against the requested permissions. * * Note that the {@link #getFileStatus(Path)} call will be subject to * authorization checks. * Typically, this requires search (execute) permissions on each directory in * the path's prefix, but this is implementation-defined. Any file system * that provides a richer authorization model (such as ACLs) may override the * default implementation so that it checks against that model instead. * <p> * In general, applications should avoid using this method, due to the risk of * time-of-check/time-of-use race conditions. The permissions on a file may * change immediately after the access call returns. Most applications should * prefer running specific file system actions as the desired user represented * by a {@link UserGroupInformation}. * * @param path Path to check * @param mode type of access to check * @throws AccessControlException if access is denied * @throws FileNotFoundException if the path does not exist * @throws IOException see specific implementation */ @InterfaceAudience.LimitedPrivate({ "HDFS", "Hive" }) public void access(Path path, FsAction mode) throws AccessControlException, FileNotFoundException, IOException { checkAccessPermissions(this.getFileStatus(path), mode); } /** * This method provides the default implementation of * {@link #access(Path, FsAction)}. * * @param stat FileStatus to check * @param mode type of access to check * @throws AccessControlException if access is denied * @throws IOException for any error */ @InterfaceAudience.Private static void checkAccessPermissions(FileStatus stat, FsAction mode) throws AccessControlException, IOException { FsPermission perm = stat.getPermission(); UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); String user = ugi.getShortUserName(); if (user.equals(stat.getOwner())) { if (perm.getUserAction().implies(mode)) { return; } } else if (ugi.getGroups().contains(stat.getGroup())) { if (perm.getGroupAction().implies(mode)) { return; } } else { if (perm.getOtherAction().implies(mode)) { return; } } throw new AccessControlException(String.format("Permission denied: user=%s, path=\"%s\":%s:%s:%s%s", user, stat.getPath(), stat.getOwner(), stat.getGroup(), stat.isDirectory() ? "d" : "-", perm)); } /** * See {@link FileContext#fixRelativePart}. */ protected Path fixRelativePart(Path p) { if (p.isUriPathAbsolute()) { return p; } else { return new Path(getWorkingDirectory(), p); } } /** * See {@link FileContext#createSymlink(Path, Path, boolean)}. */ public void createSymlink(final Path target, final Path link, final boolean createParent) throws AccessControlException, FileAlreadyExistsException, FileNotFoundException, ParentNotDirectoryException, UnsupportedFileSystemException, IOException { // Supporting filesystems should override this method throw new UnsupportedOperationException("Filesystem does not support symlinks!"); } /** * See {@link FileContext#getFileLinkStatus(Path)}. * @throws FileNotFoundException when the path does not exist * @throws IOException see specific implementation */ public FileStatus getFileLinkStatus(final Path f) throws AccessControlException, FileNotFoundException, UnsupportedFileSystemException, IOException { // Supporting filesystems should override this method return getFileStatus(f); } /** * See {@link AbstractFileSystem#supportsSymlinks()}. */ public boolean supportsSymlinks() { return false; } /** * See {@link FileContext#getLinkTarget(Path)}. * @throws UnsupportedOperationException if the operation is unsupported * (default outcome). */ public Path getLinkTarget(Path f) throws IOException { // Supporting filesystems should override this method throw new UnsupportedOperationException("Filesystem does not support symlinks!"); } /** * See {@link AbstractFileSystem#getLinkTarget(Path)}. * @throws UnsupportedOperationException if the operation is unsupported * (default outcome). */ protected Path resolveLink(Path f) throws IOException { // Supporting filesystems should override this method throw new UnsupportedOperationException("Filesystem does not support symlinks!"); } /** * Get the checksum of a file, if the FS supports checksums. * * @param f The file path * @return The file checksum. The default return value is null, * which indicates that no checksum algorithm is implemented * in the corresponding FileSystem. * @throws IOException IO failure */ public FileChecksum getFileChecksum(Path f) throws IOException { return getFileChecksum(f, Long.MAX_VALUE); } /** * Get the checksum of a file, from the beginning of the file till the * specific length. * @param f The file path * @param length The length of the file range for checksum calculation * @return The file checksum or null if checksums are not supported. * @throws IOException IO failure */ public FileChecksum getFileChecksum(Path f, final long length) throws IOException { return null; } /** * Set the verify checksum flag. This is only applicable if the * corresponding filesystem supports checksums. * By default doesn't do anything. * @param verifyChecksum Verify checksum flag */ public void setVerifyChecksum(boolean verifyChecksum) { //doesn't do anything } /** * Set the write checksum flag. This is only applicable if the * corresponding filesystem supports checksums. * By default doesn't do anything. * @param writeChecksum Write checksum flag */ public void setWriteChecksum(boolean writeChecksum) { //doesn't do anything } /** * Returns a status object describing the use and capacity of the * filesystem. If the filesystem has multiple partitions, the * use and capacity of the root partition is reflected. * * @return a FsStatus object * @throws IOException * see specific implementation */ public FsStatus getStatus() throws IOException { return getStatus(null); } /** * Returns a status object describing the use and capacity of the * filesystem. If the filesystem has multiple partitions, the * use and capacity of the partition pointed to by the specified * path is reflected. * @param p Path for which status should be obtained. null means * the default partition. * @return a FsStatus object * @throws IOException * see specific implementation */ public FsStatus getStatus(Path p) throws IOException { return new FsStatus(Long.MAX_VALUE, 0, Long.MAX_VALUE); } /** * Set permission of a path. * @param p The path * @param permission permission * @throws IOException IO failure */ public void setPermission(Path p, FsPermission permission) throws IOException { } /** * Set owner of a path (i.e. a file or a directory). * The parameters username and groupname cannot both be null. * @param p The path * @param username If it is null, the original username remains unchanged. * @param groupname If it is null, the original groupname remains unchanged. * @throws IOException IO failure */ public void setOwner(Path p, String username, String groupname) throws IOException { } /** * Set access time of a file. * @param p The path * @param mtime Set the modification time of this file. * The number of milliseconds since Jan 1, 1970. * A value of -1 means that this call should not set modification time. * @param atime Set the access time of this file. * The number of milliseconds since Jan 1, 1970. * A value of -1 means that this call should not set access time. * @throws IOException IO failure */ public void setTimes(Path p, long mtime, long atime) throws IOException { } /** * Create a snapshot with a default name. * @param path The directory where snapshots will be taken. * @return the snapshot path. * @throws IOException IO failure * @throws UnsupportedOperationException if the operation is unsupported */ public final Path createSnapshot(Path path) throws IOException { return createSnapshot(path, null); } /** * Create a snapshot. * @param path The directory where snapshots will be taken. * @param snapshotName The name of the snapshot * @return the snapshot path. * @throws IOException IO failure * @throws UnsupportedOperationException if the operation is unsupported */ public Path createSnapshot(Path path, String snapshotName) throws IOException { throw new UnsupportedOperationException(getClass().getSimpleName() + " doesn't support createSnapshot"); } /** * Rename a snapshot. * @param path The directory path where the snapshot was taken * @param snapshotOldName Old name of the snapshot * @param snapshotNewName New name of the snapshot * @throws IOException IO failure * @throws UnsupportedOperationException if the operation is unsupported * (default outcome). */ public void renameSnapshot(Path path, String snapshotOldName, String snapshotNewName) throws IOException { throw new UnsupportedOperationException(getClass().getSimpleName() + " doesn't support renameSnapshot"); } /** * Delete a snapshot of a directory. * @param path The directory that the to-be-deleted snapshot belongs to * @param snapshotName The name of the snapshot * @throws IOException IO failure * @throws UnsupportedOperationException if the operation is unsupported * (default outcome). */ public void deleteSnapshot(Path path, String snapshotName) throws IOException { throw new UnsupportedOperationException(getClass().getSimpleName() + " doesn't support deleteSnapshot"); } /** * Modifies ACL entries of files and directories. This method can add new ACL * entries or modify the permissions on existing ACL entries. All existing * ACL entries that are not specified in this call are retained without * changes. (Modifications are merged into the current ACL.) * * @param path Path to modify * @param aclSpec List<AclEntry> describing modifications * @throws IOException if an ACL could not be modified * @throws UnsupportedOperationException if the operation is unsupported * (default outcome). */ public void modifyAclEntries(Path path, List<AclEntry> aclSpec) throws IOException { throw new UnsupportedOperationException(getClass().getSimpleName() + " doesn't support modifyAclEntries"); } /** * Removes ACL entries from files and directories. Other ACL entries are * retained. * * @param path Path to modify * @param aclSpec List describing entries to remove * @throws IOException if an ACL could not be modified * @throws UnsupportedOperationException if the operation is unsupported * (default outcome). */ public void removeAclEntries(Path path, List<AclEntry> aclSpec) throws IOException { throw new UnsupportedOperationException(getClass().getSimpleName() + " doesn't support removeAclEntries"); } /** * Removes all default ACL entries from files and directories. * * @param path Path to modify * @throws IOException if an ACL could not be modified * @throws UnsupportedOperationException if the operation is unsupported * (default outcome). */ public void removeDefaultAcl(Path path) throws IOException { throw new UnsupportedOperationException(getClass().getSimpleName() + " doesn't support removeDefaultAcl"); } /** * Removes all but the base ACL entries of files and directories. The entries * for user, group, and others are retained for compatibility with permission * bits. * * @param path Path to modify * @throws IOException if an ACL could not be removed * @throws UnsupportedOperationException if the operation is unsupported * (default outcome). */ public void removeAcl(Path path) throws IOException { throw new UnsupportedOperationException(getClass().getSimpleName() + " doesn't support removeAcl"); } /** * Fully replaces ACL of files and directories, discarding all existing * entries. * * @param path Path to modify * @param aclSpec List describing modifications, which must include entries * for user, group, and others for compatibility with permission bits. * @throws IOException if an ACL could not be modified * @throws UnsupportedOperationException if the operation is unsupported * (default outcome). */ public void setAcl(Path path, List<AclEntry> aclSpec) throws IOException { throw new UnsupportedOperationException(getClass().getSimpleName() + " doesn't support setAcl"); } /** * Gets the ACL of a file or directory. * * @param path Path to get * @return AclStatus describing the ACL of the file or directory * @throws IOException if an ACL could not be read * @throws UnsupportedOperationException if the operation is unsupported * (default outcome). */ public AclStatus getAclStatus(Path path) throws IOException { throw new UnsupportedOperationException(getClass().getSimpleName() + " doesn't support getAclStatus"); } /** * Set an xattr of a file or directory. * The name must be prefixed with the namespace followed by ".". For example, * "user.attr". * <p> * Refer to the HDFS extended attributes user documentation for details. * * @param path Path to modify * @param name xattr name. * @param value xattr value. * @throws IOException IO failure * @throws UnsupportedOperationException if the operation is unsupported * (default outcome). */ public void setXAttr(Path path, String name, byte[] value) throws IOException { setXAttr(path, name, value, EnumSet.of(XAttrSetFlag.CREATE, XAttrSetFlag.REPLACE)); } /** * Set an xattr of a file or directory. * The name must be prefixed with the namespace followed by ".". For example, * "user.attr". * <p> * Refer to the HDFS extended attributes user documentation for details. * * @param path Path to modify * @param name xattr name. * @param value xattr value. * @param flag xattr set flag * @throws IOException IO failure * @throws UnsupportedOperationException if the operation is unsupported * (default outcome). */ public void setXAttr(Path path, String name, byte[] value, EnumSet<XAttrSetFlag> flag) throws IOException { throw new UnsupportedOperationException(getClass().getSimpleName() + " doesn't support setXAttr"); } /** * Get an xattr name and value for a file or directory. * The name must be prefixed with the namespace followed by ".". For example, * "user.attr". * <p> * Refer to the HDFS extended attributes user documentation for details. * * @param path Path to get extended attribute * @param name xattr name. * @return byte[] xattr value. * @throws IOException IO failure * @throws UnsupportedOperationException if the operation is unsupported * (default outcome). */ public byte[] getXAttr(Path path, String name) throws IOException { throw new UnsupportedOperationException(getClass().getSimpleName() + " doesn't support getXAttr"); } /** * Get all of the xattr name/value pairs for a file or directory. * Only those xattrs which the logged-in user has permissions to view * are returned. * <p> * Refer to the HDFS extended attributes user documentation for details. * * @param path Path to get extended attributes * @return Map describing the XAttrs of the file or directory * @throws IOException IO failure * @throws UnsupportedOperationException if the operation is unsupported * (default outcome). */ public Map<String, byte[]> getXAttrs(Path path) throws IOException { throw new UnsupportedOperationException(getClass().getSimpleName() + " doesn't support getXAttrs"); } /** * Get all of the xattrs name/value pairs for a file or directory. * Only those xattrs which the logged-in user has permissions to view * are returned. * <p> * Refer to the HDFS extended attributes user documentation for details. * * @param path Path to get extended attributes * @param names XAttr names. * @return Map describing the XAttrs of the file or directory * @throws IOException IO failure * @throws UnsupportedOperationException if the operation is unsupported * (default outcome). */ public Map<String, byte[]> getXAttrs(Path path, List<String> names) throws IOException { throw new UnsupportedOperationException(getClass().getSimpleName() + " doesn't support getXAttrs"); } /** * Get all of the xattr names for a file or directory. * Only those xattr names which the logged-in user has permissions to view * are returned. * <p> * Refer to the HDFS extended attributes user documentation for details. * * @param path Path to get extended attributes * @return List<String> of the XAttr names of the file or directory * @throws IOException IO failure * @throws UnsupportedOperationException if the operation is unsupported * (default outcome). */ public List<String> listXAttrs(Path path) throws IOException { throw new UnsupportedOperationException(getClass().getSimpleName() + " doesn't support listXAttrs"); } /** * Remove an xattr of a file or directory. * The name must be prefixed with the namespace followed by ".". For example, * "user.attr". * <p> * Refer to the HDFS extended attributes user documentation for details. * * @param path Path to remove extended attribute * @param name xattr name * @throws IOException IO failure * @throws UnsupportedOperationException if the operation is unsupported * (default outcome). */ public void removeXAttr(Path path, String name) throws IOException { throw new UnsupportedOperationException(getClass().getSimpleName() + " doesn't support removeXAttr"); } /** * Set the storage policy for a given file or directory. * * @param src file or directory path. * @param policyName the name of the target storage policy. The list * of supported Storage policies can be retrieved * via {@link #getAllStoragePolicies}. * @throws IOException IO failure * @throws UnsupportedOperationException if the operation is unsupported * (default outcome). */ public void setStoragePolicy(final Path src, final String policyName) throws IOException { throw new UnsupportedOperationException(getClass().getSimpleName() + " doesn't support setStoragePolicy"); } /** * Unset the storage policy set for a given file or directory. * @param src file or directory path. * @throws IOException IO failure * @throws UnsupportedOperationException if the operation is unsupported * (default outcome). */ public void unsetStoragePolicy(final Path src) throws IOException { throw new UnsupportedOperationException(getClass().getSimpleName() + " doesn't support unsetStoragePolicy"); } /** * Query the effective storage policy ID for the given file or directory. * * @param src file or directory path. * @return storage policy for give file. * @throws IOException IO failure * @throws UnsupportedOperationException if the operation is unsupported * (default outcome). */ public BlockStoragePolicySpi getStoragePolicy(final Path src) throws IOException { throw new UnsupportedOperationException(getClass().getSimpleName() + " doesn't support getStoragePolicy"); } /** * Retrieve all the storage policies supported by this file system. * * @return all storage policies supported by this filesystem. * @throws IOException IO failure * @throws UnsupportedOperationException if the operation is unsupported * (default outcome). */ public Collection<? extends BlockStoragePolicySpi> getAllStoragePolicies() throws IOException { throw new UnsupportedOperationException( getClass().getSimpleName() + " doesn't support getAllStoragePolicies"); } /** * Get the root directory of Trash for current user when the path specified * is deleted. * * @param path the trash root of the path to be determined. * @return the default implementation returns {@code /user/$USER/.Trash} */ public Path getTrashRoot(Path path) { return this.makeQualified(new Path(getHomeDirectory().toUri().getPath(), TRASH_PREFIX)); } /** * Get all the trash roots for current user or all users. * * @param allUsers return trash roots for all users if true. * @return all the trash root directories. * Default FileSystem returns .Trash under users' home directories if * {@code /user/$USER/.Trash} exists. */ public Collection<FileStatus> getTrashRoots(boolean allUsers) { Path userHome = new Path(getHomeDirectory().toUri().getPath()); List<FileStatus> ret = new ArrayList<>(); try { if (!allUsers) { Path userTrash = new Path(userHome, TRASH_PREFIX); if (exists(userTrash)) { ret.add(getFileStatus(userTrash)); } } else { Path homeParent = userHome.getParent(); if (exists(homeParent)) { FileStatus[] candidates = listStatus(homeParent); for (FileStatus candidate : candidates) { Path userTrash = new Path(candidate.getPath(), TRASH_PREFIX); if (exists(userTrash)) { candidate.setPath(userTrash); ret.add(candidate); } } } } } catch (IOException e) { LOGGER.warn("Cannot get all trash roots", e); } return ret; } // making it volatile to be able to do a double checked locking private volatile static boolean FILE_SYSTEMS_LOADED = false; /** * Filesystems listed as services. */ private static final Map<String, Class<? extends FileSystem>> SERVICE_FILE_SYSTEMS = new HashMap<>(); /** * Load the filesystem declarations from service resources. * This is a synchronized operation. */ private static void loadFileSystems() { LOGGER.debug("Loading filesystems"); synchronized (FileSystem.class) { if (!FILE_SYSTEMS_LOADED) { ServiceLoader<FileSystem> serviceLoader = ServiceLoader.load(FileSystem.class); Iterator<FileSystem> it = serviceLoader.iterator(); while (it.hasNext()) { FileSystem fs; try { fs = it.next(); try { SERVICE_FILE_SYSTEMS.put(fs.getScheme(), fs.getClass()); if (LOGGER.isDebugEnabled()) { LOGGER.debug("{}:// = {} from {}", fs.getScheme(), fs.getClass(), ClassUtil.findContainingJar(fs.getClass())); } } catch (Exception e) { LOGGER.warn("Cannot load: {} from {}", fs, ClassUtil.findContainingJar(fs.getClass())); LOGGER.info("Full exception loading: {}", fs, e); } } catch (ServiceConfigurationError ee) { LOG.warn("Cannot load filesystem: " + ee); Throwable cause = ee.getCause(); // print all the nested exception messages while (cause != null) { LOG.warn(cause.toString()); cause = cause.getCause(); } // and at debug: the full stack LOG.debug("Stack Trace", ee); } } FILE_SYSTEMS_LOADED = true; } } } /** * Get the FileSystem implementation class of a filesystem. * This triggers a scan and load of all FileSystem implementations listed as * services and discovered via the {@link ServiceLoader} * @param scheme URL scheme of FS * @param conf configuration: can be null, in which case the check for * a filesystem binding declaration in the configuration is skipped. * @return the filesystem * @throws UnsupportedFileSystemException if there was no known implementation * for the scheme. * @throws IOException if the filesystem could not be loaded */ public static Class<? extends FileSystem> getFileSystemClass(String scheme, Configuration conf) throws IOException { if (!FILE_SYSTEMS_LOADED) { loadFileSystems(); } LOGGER.debug("Looking for FS supporting {}", scheme); Class<? extends FileSystem> clazz = null; if (conf != null) { String property = "fs." + scheme + ".impl"; LOGGER.debug("looking for configuration option {}", property); clazz = (Class<? extends FileSystem>) conf.getClass(property, null); } else { LOGGER.debug("No configuration: skipping check for fs.{}.impl", scheme); } if (clazz == null) { LOGGER.debug("Looking in service filesystems for implementation class"); clazz = SERVICE_FILE_SYSTEMS.get(scheme); } else { LOGGER.debug("Filesystem {} defined in configuration option", scheme); } if (clazz == null) { throw new UnsupportedFileSystemException("No FileSystem for scheme " + "\"" + scheme + "\""); } LOGGER.debug("FS for {} is {}", scheme, clazz); return clazz; } /** * Create and initialize a new instance of a FileSystem. * @param uri URI containing the FS schema and FS details * @param conf configuration to use to look for the FS instance declaration * and to pass to the {@link FileSystem#initialize(URI, Configuration)}. * @return the initialized filesystem. * @throws IOException problems loading or initializing the FileSystem */ private static FileSystem createFileSystem(URI uri, Configuration conf) throws IOException { Tracer tracer = FsTracer.get(conf); try (TraceScope scope = tracer.newScope("FileSystem#createFileSystem")) { scope.addKVAnnotation("scheme", uri.getScheme()); Class<?> clazz = getFileSystemClass(uri.getScheme(), conf); FileSystem fs = (FileSystem) ReflectionUtils.newInstance(clazz, conf); fs.initialize(uri, conf); return fs; } } /** Caching FileSystem objects. */ static class Cache { private final ClientFinalizer clientFinalizer = new ClientFinalizer(); private final Map<Key, FileSystem> map = new HashMap<>(); private final Set<Key> toAutoClose = new HashSet<>(); /** A variable that makes all objects in the cache unique. */ private static AtomicLong unique = new AtomicLong(1); FileSystem get(URI uri, Configuration conf) throws IOException { Key key = new Key(uri, conf); return getInternal(uri, conf, key); } /** The objects inserted into the cache using this method are all unique. */ FileSystem getUnique(URI uri, Configuration conf) throws IOException { Key key = new Key(uri, conf, unique.getAndIncrement()); return getInternal(uri, conf, key); } /** * Get the FS instance if the key maps to an instance, creating and * initializing the FS if it is not found. * If this is the first entry in the map and the JVM is not shutting down, * this registers a shutdown hook to close filesystems, and adds this * FS to the {@code toAutoClose} set if {@code "fs.automatic.close"} * is set in the configuration (default: true). * @param uri filesystem URI * @param conf configuration * @param key key to store/retrieve this FileSystem in the cache * @return a cached or newly instantiated FileSystem. * @throws IOException */ private FileSystem getInternal(URI uri, Configuration conf, Key key) throws IOException { FileSystem fs; synchronized (this) { fs = map.get(key); } if (fs != null) { return fs; } fs = createFileSystem(uri, conf); synchronized (this) { // refetch the lock again FileSystem oldfs = map.get(key); if (oldfs != null) { // a file system is created while lock is releasing fs.close(); // close the new file system return oldfs; // return the old file system } // now insert the new file system into the map if (map.isEmpty() && !ShutdownHookManager.get().isShutdownInProgress()) { ShutdownHookManager.get().addShutdownHook(clientFinalizer, SHUTDOWN_HOOK_PRIORITY); } fs.key = key; map.put(key, fs); if (conf.getBoolean(FS_AUTOMATIC_CLOSE_KEY, FS_AUTOMATIC_CLOSE_DEFAULT)) { toAutoClose.add(key); } return fs; } } synchronized void remove(Key key, FileSystem fs) { FileSystem cachedFs = map.remove(key); if (fs == cachedFs) { toAutoClose.remove(key); } else if (cachedFs != null) { map.put(key, cachedFs); } } /** * Close all FileSystems in the cache, whether they are marked for * automatic closing or not. * @throws IOException a problem arose closing one or more FileSystem. */ synchronized void closeAll() throws IOException { closeAll(false); } /** * Close all FileSystem instances in the Cache. * @param onlyAutomatic only close those that are marked for automatic closing * @throws IOException a problem arose closing one or more FileSystem. */ synchronized void closeAll(boolean onlyAutomatic) throws IOException { List<IOException> exceptions = new ArrayList<>(); // Make a copy of the keys in the map since we'll be modifying // the map while iterating over it, which isn't safe. List<Key> keys = new ArrayList<>(); keys.addAll(map.keySet()); for (Key key : keys) { final FileSystem fs = map.get(key); if (onlyAutomatic && !toAutoClose.contains(key)) { continue; } //remove from cache map.remove(key); toAutoClose.remove(key); if (fs != null) { try { fs.close(); } catch (IOException ioe) { exceptions.add(ioe); } } } if (!exceptions.isEmpty()) { throw MultipleIOException.createIOException(exceptions); } } private class ClientFinalizer implements Runnable { @Override public synchronized void run() { try { closeAll(true); } catch (IOException e) { LOGGER.info("FileSystem.Cache.closeAll() threw an exception:\n" + e); } } } synchronized void closeAll(UserGroupInformation ugi) throws IOException { List<FileSystem> targetFSList = new ArrayList<>(map.entrySet().size()); //Make a pass over the list and collect the FileSystems to close //we cannot close inline since close() removes the entry from the Map for (Map.Entry<Key, FileSystem> entry : map.entrySet()) { final Key key = entry.getKey(); final FileSystem fs = entry.getValue(); if (ugi.equals(key.ugi) && fs != null) { targetFSList.add(fs); } } List<IOException> exceptions = new ArrayList<>(); //now make a pass over the target list and close each for (FileSystem fs : targetFSList) { try { fs.close(); } catch (IOException ioe) { exceptions.add(ioe); } } if (!exceptions.isEmpty()) { throw MultipleIOException.createIOException(exceptions); } } /** FileSystem.Cache.Key */ static class Key { final String scheme; final String authority; final UserGroupInformation ugi; final long unique; // an artificial way to make a key unique Key(URI uri, Configuration conf) throws IOException { this(uri, conf, 0); } Key(URI uri, Configuration conf, long unique) throws IOException { scheme = uri.getScheme() == null ? "" : StringUtils.toLowerCase(uri.getScheme()); authority = uri.getAuthority() == null ? "" : StringUtils.toLowerCase(uri.getAuthority()); this.unique = unique; this.ugi = UserGroupInformation.getCurrentUser(); } @Override public int hashCode() { return (scheme + authority).hashCode() + ugi.hashCode() + (int) unique; } static boolean isEqual(Object a, Object b) { return a == b || (a != null && a.equals(b)); } @Override public boolean equals(Object obj) { if (obj == this) { return true; } if (obj instanceof Key) { Key that = (Key) obj; return isEqual(this.scheme, that.scheme) && isEqual(this.authority, that.authority) && isEqual(this.ugi, that.ugi) && (this.unique == that.unique); } return false; } @Override public String toString() { return "(" + ugi.toString() + ")@" + scheme + "://" + authority; } } } /** * Tracks statistics about how many reads, writes, and so forth have been * done in a FileSystem. * * Since there is only one of these objects per FileSystem, there will * typically be many threads writing to this object. Almost every operation * on an open file will involve a write to this object. In contrast, reading * statistics is done infrequently by most programs, and not at all by others. * Hence, this is optimized for writes. * * Each thread writes to its own thread-local area of memory. This removes * contention and allows us to scale up to many, many threads. To read * statistics, the reader thread totals up the contents of all of the * thread-local data areas. */ public static final class Statistics { /** * Statistics data. * * There is only a single writer to thread-local StatisticsData objects. * Hence, volatile is adequate here-- we do not need AtomicLong or similar * to prevent lost updates. * The Java specification guarantees that updates to volatile longs will * be perceived as atomic with respect to other threads, which is all we * need. */ public static class StatisticsData { private volatile long bytesRead; private volatile long bytesWritten; private volatile int readOps; private volatile int largeReadOps; private volatile int writeOps; private volatile long bytesReadLocalHost; private volatile long bytesReadDistanceOfOneOrTwo; private volatile long bytesReadDistanceOfThreeOrFour; private volatile long bytesReadDistanceOfFiveOrLarger; private volatile long bytesReadErasureCoded; /** * Add another StatisticsData object to this one. */ void add(StatisticsData other) { this.bytesRead += other.bytesRead; this.bytesWritten += other.bytesWritten; this.readOps += other.readOps; this.largeReadOps += other.largeReadOps; this.writeOps += other.writeOps; this.bytesReadLocalHost += other.bytesReadLocalHost; this.bytesReadDistanceOfOneOrTwo += other.bytesReadDistanceOfOneOrTwo; this.bytesReadDistanceOfThreeOrFour += other.bytesReadDistanceOfThreeOrFour; this.bytesReadDistanceOfFiveOrLarger += other.bytesReadDistanceOfFiveOrLarger; this.bytesReadErasureCoded += other.bytesReadErasureCoded; } /** * Negate the values of all statistics. */ void negate() { this.bytesRead = -this.bytesRead; this.bytesWritten = -this.bytesWritten; this.readOps = -this.readOps; this.largeReadOps = -this.largeReadOps; this.writeOps = -this.writeOps; this.bytesReadLocalHost = -this.bytesReadLocalHost; this.bytesReadDistanceOfOneOrTwo = -this.bytesReadDistanceOfOneOrTwo; this.bytesReadDistanceOfThreeOrFour = -this.bytesReadDistanceOfThreeOrFour; this.bytesReadDistanceOfFiveOrLarger = -this.bytesReadDistanceOfFiveOrLarger; this.bytesReadErasureCoded = -this.bytesReadErasureCoded; } @Override public String toString() { return bytesRead + " bytes read, " + bytesWritten + " bytes written, " + readOps + " read ops, " + largeReadOps + " large read ops, " + writeOps + " write ops"; } public long getBytesRead() { return bytesRead; } public long getBytesWritten() { return bytesWritten; } public int getReadOps() { return readOps; } public int getLargeReadOps() { return largeReadOps; } public int getWriteOps() { return writeOps; } public long getBytesReadLocalHost() { return bytesReadLocalHost; } public long getBytesReadDistanceOfOneOrTwo() { return bytesReadDistanceOfOneOrTwo; } public long getBytesReadDistanceOfThreeOrFour() { return bytesReadDistanceOfThreeOrFour; } public long getBytesReadDistanceOfFiveOrLarger() { return bytesReadDistanceOfFiveOrLarger; } public long getBytesReadErasureCoded() { return bytesReadErasureCoded; } } private interface StatisticsAggregator<T> { void accept(StatisticsData data); T aggregate(); } private final String scheme; /** * rootData is data that doesn't belong to any thread, but will be added * to the totals. This is useful for making copies of Statistics objects, * and for storing data that pertains to threads that have been garbage * collected. Protected by the Statistics lock. */ private final StatisticsData rootData; /** * Thread-local data. */ @SuppressWarnings("ThreadLocalNotStaticFinal") private final ThreadLocal<StatisticsData> threadData; /** * Set of all thread-local data areas. Protected by the Statistics lock. * The references to the statistics data are kept using weak references * to the associated threads. Proper clean-up is performed by the cleaner * thread when the threads are garbage collected. */ private final Set<StatisticsDataReference> allData; /** * Global reference queue and a cleaner thread that manage statistics data * references from all filesystem instances. */ private static final ReferenceQueue<Thread> STATS_DATA_REF_QUEUE; private static final Thread STATS_DATA_CLEANER; static { STATS_DATA_REF_QUEUE = new ReferenceQueue<>(); // start a single daemon cleaner thread STATS_DATA_CLEANER = new Thread(new StatisticsDataReferenceCleaner()); STATS_DATA_CLEANER.setName(StatisticsDataReferenceCleaner.class.getName()); STATS_DATA_CLEANER.setDaemon(true); STATS_DATA_CLEANER.start(); } public Statistics(String scheme) { this.scheme = scheme; this.rootData = new StatisticsData(); this.threadData = new ThreadLocal<>(); this.allData = new HashSet<>(); } /** * Copy constructor. * * @param other The input Statistics object which is cloned. */ public Statistics(Statistics other) { this.scheme = other.scheme; this.rootData = new StatisticsData(); other.visitAll(new StatisticsAggregator<Void>() { @Override public void accept(StatisticsData data) { rootData.add(data); } public Void aggregate() { return null; } }); this.threadData = new ThreadLocal<>(); this.allData = new HashSet<>(); } /** * A weak reference to a thread that also includes the data associated * with that thread. On the thread being garbage collected, it is enqueued * to the reference queue for clean-up. */ private final class StatisticsDataReference extends WeakReference<Thread> { private final StatisticsData data; private StatisticsDataReference(StatisticsData data, Thread thread) { super(thread, STATS_DATA_REF_QUEUE); this.data = data; } public StatisticsData getData() { return data; } /** * Performs clean-up action when the associated thread is garbage * collected. */ public void cleanUp() { // use the statistics lock for safety synchronized (Statistics.this) { /* * If the thread that created this thread-local data no longer exists, * remove the StatisticsData from our list and fold the values into * rootData. */ rootData.add(data); allData.remove(this); } } } /** * Background action to act on references being removed. */ private static class StatisticsDataReferenceCleaner implements Runnable { @Override public void run() { while (!Thread.interrupted()) { try { StatisticsDataReference ref = (StatisticsDataReference) STATS_DATA_REF_QUEUE.remove(); ref.cleanUp(); } catch (InterruptedException ie) { LOGGER.warn("Cleaner thread interrupted, will stop", ie); Thread.currentThread().interrupt(); } catch (Throwable th) { LOGGER.warn("Exception in the cleaner thread but it will" + " continue to run", th); } } } } /** * Get or create the thread-local data associated with the current thread. */ public StatisticsData getThreadStatistics() { StatisticsData data = threadData.get(); if (data == null) { data = new StatisticsData(); threadData.set(data); StatisticsDataReference ref = new StatisticsDataReference(data, Thread.currentThread()); synchronized (this) { allData.add(ref); } } return data; } /** * Increment the bytes read in the statistics. * @param newBytes the additional bytes read */ public void incrementBytesRead(long newBytes) { getThreadStatistics().bytesRead += newBytes; } /** * Increment the bytes written in the statistics. * @param newBytes the additional bytes written */ public void incrementBytesWritten(long newBytes) { getThreadStatistics().bytesWritten += newBytes; } /** * Increment the number of read operations. * @param count number of read operations */ public void incrementReadOps(int count) { getThreadStatistics().readOps += count; } /** * Increment the number of large read operations. * @param count number of large read operations */ public void incrementLargeReadOps(int count) { getThreadStatistics().largeReadOps += count; } /** * Increment the number of write operations. * @param count number of write operations */ public void incrementWriteOps(int count) { getThreadStatistics().writeOps += count; } /** * Increment the bytes read on erasure-coded files in the statistics. * @param newBytes the additional bytes read */ public void incrementBytesReadErasureCoded(long newBytes) { getThreadStatistics().bytesReadErasureCoded += newBytes; } /** * Increment the bytes read by the network distance in the statistics * In the common network topology setup, distance value should be an even * number such as 0, 2, 4, 6. To make it more general, we group distance * by {1, 2}, {3, 4} and {5 and beyond} for accounting. * @param distance the network distance * @param newBytes the additional bytes read */ public void incrementBytesReadByDistance(int distance, long newBytes) { switch (distance) { case 0: getThreadStatistics().bytesReadLocalHost += newBytes; break; case 1: case 2: getThreadStatistics().bytesReadDistanceOfOneOrTwo += newBytes; break; case 3: case 4: getThreadStatistics().bytesReadDistanceOfThreeOrFour += newBytes; break; default: getThreadStatistics().bytesReadDistanceOfFiveOrLarger += newBytes; break; } } /** * Apply the given aggregator to all StatisticsData objects associated with * this Statistics object. * * For each StatisticsData object, we will call accept on the visitor. * Finally, at the end, we will call aggregate to get the final total. * * @param visitor to use. * @return The total. */ private synchronized <T> T visitAll(StatisticsAggregator<T> visitor) { visitor.accept(rootData); for (StatisticsDataReference ref : allData) { StatisticsData data = ref.getData(); visitor.accept(data); } return visitor.aggregate(); } /** * Get the total number of bytes read. * @return the number of bytes */ public long getBytesRead() { return visitAll(new StatisticsAggregator<Long>() { private long bytesRead = 0; @Override public void accept(StatisticsData data) { bytesRead += data.bytesRead; } public Long aggregate() { return bytesRead; } }); } /** * Get the total number of bytes written. * @return the number of bytes */ public long getBytesWritten() { return visitAll(new StatisticsAggregator<Long>() { private long bytesWritten = 0; @Override public void accept(StatisticsData data) { bytesWritten += data.bytesWritten; } public Long aggregate() { return bytesWritten; } }); } /** * Get the number of file system read operations such as list files. * @return number of read operations */ public int getReadOps() { return visitAll(new StatisticsAggregator<Integer>() { private int readOps = 0; @Override public void accept(StatisticsData data) { readOps += data.readOps; readOps += data.largeReadOps; } public Integer aggregate() { return readOps; } }); } /** * Get the number of large file system read operations such as list files * under a large directory. * @return number of large read operations */ public int getLargeReadOps() { return visitAll(new StatisticsAggregator<Integer>() { private int largeReadOps = 0; @Override public void accept(StatisticsData data) { largeReadOps += data.largeReadOps; } public Integer aggregate() { return largeReadOps; } }); } /** * Get the number of file system write operations such as create, append * rename etc. * @return number of write operations */ public int getWriteOps() { return visitAll(new StatisticsAggregator<Integer>() { private int writeOps = 0; @Override public void accept(StatisticsData data) { writeOps += data.writeOps; } public Integer aggregate() { return writeOps; } }); } /** * In the common network topology setup, distance value should be an even * number such as 0, 2, 4, 6. To make it more general, we group distance * by {1, 2}, {3, 4} and {5 and beyond} for accounting. So if the caller * ask for bytes read for distance 2, the function will return the value * for group {1, 2}. * @param distance the network distance * @return the total number of bytes read by the network distance */ public long getBytesReadByDistance(int distance) { long bytesRead; switch (distance) { case 0: bytesRead = getData().getBytesReadLocalHost(); break; case 1: case 2: bytesRead = getData().getBytesReadDistanceOfOneOrTwo(); break; case 3: case 4: bytesRead = getData().getBytesReadDistanceOfThreeOrFour(); break; default: bytesRead = getData().getBytesReadDistanceOfFiveOrLarger(); break; } return bytesRead; } /** * Get all statistics data. * MR or other frameworks can use the method to get all statistics at once. * @return the StatisticsData */ public StatisticsData getData() { return visitAll(new StatisticsAggregator<StatisticsData>() { private StatisticsData all = new StatisticsData(); @Override public void accept(StatisticsData data) { all.add(data); } public StatisticsData aggregate() { return all; } }); } /** * Get the total number of bytes read on erasure-coded files. * @return the number of bytes */ public long getBytesReadErasureCoded() { return visitAll(new StatisticsAggregator<Long>() { private long bytesReadErasureCoded = 0; @Override public void accept(StatisticsData data) { bytesReadErasureCoded += data.bytesReadErasureCoded; } public Long aggregate() { return bytesReadErasureCoded; } }); } @Override public String toString() { return visitAll(new StatisticsAggregator<String>() { private StatisticsData total = new StatisticsData(); @Override public void accept(StatisticsData data) { total.add(data); } public String aggregate() { return total.toString(); } }); } /** * Resets all statistics to 0. * * In order to reset, we add up all the thread-local statistics data, and * set rootData to the negative of that. * * This may seem like a counterintuitive way to reset the statistics. Why * can't we just zero out all the thread-local data? Well, thread-local * data can only be modified by the thread that owns it. If we tried to * modify the thread-local data from this thread, our modification might get * interleaved with a read-modify-write operation done by the thread that * owns the data. That would result in our update getting lost. * * The approach used here avoids this problem because it only ever reads * (not writes) the thread-local data. Both reads and writes to rootData * are done under the lock, so we're free to modify rootData from any thread * that holds the lock. */ public void reset() { visitAll(new StatisticsAggregator<Void>() { private StatisticsData total = new StatisticsData(); @Override public void accept(StatisticsData data) { total.add(data); } public Void aggregate() { total.negate(); rootData.add(total); return null; } }); } /** * Get the uri scheme associated with this statistics object. * @return the schema associated with this set of statistics */ public String getScheme() { return scheme; } @VisibleForTesting synchronized int getAllThreadLocalDataSize() { return allData.size(); } } /** * Get the Map of Statistics object indexed by URI Scheme. * @return a Map having a key as URI scheme and value as Statistics object * @deprecated use {@link #getGlobalStorageStatistics()} */ @Deprecated public static synchronized Map<String, Statistics> getStatistics() { Map<String, Statistics> result = new HashMap<>(); for (Statistics stat : statisticsTable.values()) { result.put(stat.getScheme(), stat); } return result; } /** * Return the FileSystem classes that have Statistics. * @deprecated use {@link #getGlobalStorageStatistics()} */ @Deprecated public static synchronized List<Statistics> getAllStatistics() { return new ArrayList<>(statisticsTable.values()); } /** * Get the statistics for a particular file system. * @param cls the class to lookup * @return a statistics object * @deprecated use {@link #getGlobalStorageStatistics()} */ @Deprecated public static synchronized Statistics getStatistics(final String scheme, Class<? extends FileSystem> cls) { checkArgument(scheme != null, "No statistics is allowed for a file system with null scheme!"); Statistics result = statisticsTable.get(cls); if (result == null) { final Statistics newStats = new Statistics(scheme); statisticsTable.put(cls, newStats); result = newStats; GlobalStorageStatistics.INSTANCE.put(scheme, new StorageStatisticsProvider() { @Override public StorageStatistics provide() { return new FileSystemStorageStatistics(scheme, newStats); } }); } return result; } /** * Reset all statistics for all file systems. */ public static synchronized void clearStatistics() { GlobalStorageStatistics.INSTANCE.reset(); } /** * Print all statistics for all file systems to {@code System.out} */ public static synchronized void printStatistics() throws IOException { for (Map.Entry<Class<? extends FileSystem>, Statistics> pair : statisticsTable.entrySet()) { System.out.println(" FileSystem " + pair.getKey().getName() + ": " + pair.getValue()); } } // Symlinks are temporarily disabled - see HADOOP-10020 and HADOOP-10052 private static boolean symlinksEnabled = false; @VisibleForTesting public static boolean areSymlinksEnabled() { return symlinksEnabled; } @VisibleForTesting public static void enableSymlinks() { symlinksEnabled = true; } /** * Get the StorageStatistics for this FileSystem object. These statistics are * per-instance. They are not shared with any other FileSystem object. * * <p>This is a default method which is intended to be overridden by * subclasses. The default implementation returns an empty storage statistics * object.</p> * * @return The StorageStatistics for this FileSystem instance. * Will never be null. */ public StorageStatistics getStorageStatistics() { return new EmptyStorageStatistics(getUri().toString()); } /** * Get the global storage statistics. */ public static GlobalStorageStatistics getGlobalStorageStatistics() { return GlobalStorageStatistics.INSTANCE; } private static final class FileSystemDataOutputStreamBuilder extends FSDataOutputStreamBuilder<FSDataOutputStream, FileSystemDataOutputStreamBuilder> { /** * Constructor. */ protected FileSystemDataOutputStreamBuilder(FileSystem fileSystem, Path p) { super(fileSystem, p); } @Override public FSDataOutputStream build() throws IOException { if (getFlags().contains(CreateFlag.CREATE) || getFlags().contains(CreateFlag.OVERWRITE)) { if (isRecursive()) { return getFS().create(getPath(), getPermission(), getFlags(), getBufferSize(), getReplication(), getBlockSize(), getProgress(), getChecksumOpt()); } else { return getFS().createNonRecursive(getPath(), getPermission(), getFlags(), getBufferSize(), getReplication(), getBlockSize(), getProgress()); } } else if (getFlags().contains(CreateFlag.APPEND)) { return getFS().append(getPath(), getBufferSize(), getProgress()); } throw new IOException("Must specify either create, overwrite or append"); } @Override protected FileSystemDataOutputStreamBuilder getThisBuilder() { return this; } } /** * Create a new FSDataOutputStreamBuilder for the file with path. * Files are overwritten by default. * * @param path file path * @return a FSDataOutputStreamBuilder object to build the file * * HADOOP-14384. Temporarily reduce the visibility of method before the * builder interface becomes stable. */ public FSDataOutputStreamBuilder createFile(Path path) { return new FileSystemDataOutputStreamBuilder(this, path).create().overwrite(true); } /** * Create a Builder to append a file. * @param path file path. * @return a {@link FSDataOutputStreamBuilder} to build file append request. */ public FSDataOutputStreamBuilder appendFile(Path path) { return new FileSystemDataOutputStreamBuilder(this, path).append(); } }