Java tutorial
/** * Copyright 2014-2015 SHAF-WORK * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.shaf.core.io; import java.io.FileNotFoundException; import java.io.IOException; import java.text.SimpleDateFormat; import java.util.Set; import java.util.Stack; import java.util.TreeSet; import java.util.regex.Pattern; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.fs.permission.FsPermission; import org.shaf.core.util.IOUtils; import org.shaf.core.util.PatternUtils; import com.google.common.base.Objects; import com.google.common.base.Strings; import com.google.common.collect.ObjectArrays; /** * The base class for the I/O location. * * @author Mykola Galushka */ public class Location { /** * The location file system. */ private final FileSystem fs; /** * The location base path. */ private final Path base; /** * The location path. */ private final Path path; /** * The file name filter. */ private final FileNameFilter filter; /** * The location qualifier. */ private String qualifier; /** * Constructs a new I/O location. * * @param fs * the location file system. * @param base * the location base path. * @param path * the location path. * @param filter * the filter for selecting files. */ private Location(final FileSystem fs, final Path base, final Path path, final FileNameFilter filter) { if (fs == null) { throw new NullPointerException("file system"); } if (base == null) { throw new NullPointerException("base"); } else if (!base.isAbsolute()) { throw new IllegalArgumentException("The location base-path must be absolute, but received relative: " + base + "; Check the base-path format specified in by the process " + "configuration."); } if (path == null) { throw new NullPointerException("path"); } else if (!path.isAbsolute()) { throw new IllegalArgumentException("The location path must be absolute, but received relative: " + path + "; It is likely that backslash is missing at the beginning" + " of this path."); } if (filter == null) { throw new NullPointerException("filter"); } this.fs = fs; this.base = base; this.path = path; this.filter = filter; this.qualifier = "undefined"; } /** * Constructs a new I/O location. * * @param fs * the location file system. * @param base * the location base path. * @param path * the location path. * @param wildcard * the wildcard for filtering file. */ public Location(final FileSystem fs, final Path base, final Path path, final String wildcard) { this(fs, base, path, new FileNameFilter(wildcard)); } /** * Constructs a new I/O location. * * @param fs * the location file system. * @param base * the location base path. * @param path * the location path. * @param wildcard * the wildcard for filtering file. */ public Location(final FileSystem fs, final String base, final String path, final String wildcard) { this(fs, new Path(base), new Path(path), new FileNameFilter(wildcard)); } /** * Constructs a new I/O location. * * @param fs * the location file system. * @param base * the location base path. * @param path * the location path. */ public Location(final FileSystem fs, final Path base, final Path path) { this(fs, base, path, new FileNameFilter()); } /** * Constructs a new I/O location. * * @param fs * the location file system. * @param base * the location base path. * @param path * the location path. */ public Location(final FileSystem fs, final String base, final String path) { this(fs, new Path(base), new Path(path), new FileNameFilter()); } /** * Sets the location qualifier. * * @param descr * the location qualifier. * @return the reference to itself. */ public final Location setQualifier(final String qualifier) { this.qualifier = qualifier; return this; } /** * Checks and throws exception if the specified location is not found. * * @return the reference to itself. * @throws LocationAvailabilityException * if the specified location is not found. */ public final Location shouldBeFound() throws IOException { if (!this.exists()) { throw new LocationAvailabilityException(qualifier, this, true); } return this; } /** * Checks and throws exception if the specified location is found. * * @return the reference to itself. * @throws LocationAvailabilityException * if the specified location is found. */ public final Location shouldNotBeFound() throws IOException { if (this.exists()) { throw new LocationAvailabilityException(qualifier, this, false); } return this; } /** * Checks and throws exception if the specified location is file. * * @return the reference to itself. * @throws LocationAvailabilityException * if the specified location is not found. * @throws LocationTypeException * if the specified location is file. */ public final Location shouldBeDirectory() throws IOException { this.shouldBeFound(); if (this.isFile()) { throw new LocationTypeException(qualifier, this, true); } return this; } /** * Checks and throws exception if the specified location is directory. * * @return the reference to itself. * @throws LocationAvailabilityException * if the specified location is not found. * @throws LocationTypeException * if the specified location is directory. */ public final Location shouldBeFile() throws IOException { this.shouldBeFound(); if (this.isDirectory()) { throw new LocationTypeException(qualifier, this, false); } return this; } /** * Returns the absolute path to this location, which combines base and * relative paths. * * @return the absolute path to this location. */ private final Path getAbsolutePath() { return Path.mergePaths(base, path); } /** * Returns the location file system. * * @return the location file system. */ public final FileSystem getFileSystem() { return this.fs; } /** * Returns the location path. * * @return the location path. */ public final Path getPath() { return this.path; } /** * Returns the location path as string. * * @return the location path as string. */ public final String getPathAsString() { return this.path.toString(); } /** * Returns the location name. * * @return the location name. */ public final String getName() { return this.path == null ? null : this.path.getName(); } /** * Resolve the given path against this location. * * @param others * the paths to resolve against this location. * @return the location with resulting path. */ public final Location resolve(final Path... others) { Path rel = this.path; for (Path other : others) { if (other.isAbsolute()) { rel = Path.mergePaths(rel, other); } else { rel = Path.mergePaths(rel, new Path("/", other)); } } return new Location(this.fs, this.base, rel, this.filter); } /** * Resolve the given path against this location. * * @param others * the paths to resolve against this location. * @return the location with resulting path. */ public final Location resolve(final String... others) { Path[] paths = new Path[others.length]; for (int i = 0; i < others.length; i++) { paths[i] = new Path(others[i]); } return this.resolve(paths); } /** * Returns the path representing by the location to other location path. * * @param other * the other location. * @return the relative path. */ public final Path normalize(final Location other) throws IOException { if (this.base.compareTo(other.base) != 0) { throw new IOException("The location bases are not identical."); } if (this.getPathAsString().startsWith(other.getPathAsString())) { return new Path(this.getPathAsString().substring(other.getPathAsString().length() + 1)); } else { return null; } } /** * Returns the parent location. * * @return the parent location. */ public final Location getParent() { return new Location(this.fs, this.base, this.path == null ? null : this.path.getParent(), this.filter); } /** * Tests if this location exists. * * @return {@code true} if this location exists and {@code false} otherwise. * @throws IOException * if an I/O error occurs. */ public final boolean exists() throws IOException { return this.fs.exists(this.getAbsolutePath()); } /** * Tests if this location represents file. * * @return {@code true} if this location represents a file and {@code false} * otherwise. * @throws IOException * if an I/O error occurs. */ public final boolean isFile() throws IOException { return this.exists() && this.fs.isFile(this.getAbsolutePath()); } /** * Tests if this location represents directory. * * @return {@code true} if this location represents a directory and * {@code false} otherwise. * @throws IOException * if an I/O error occurs. */ public final boolean isDirectory() throws IOException { return this.exists() && this.fs.isDirectory(this.getAbsolutePath()); } /** * Makes directory represented by this location. * * @throws IOException * if failed to create directory. */ public final void mkdirs() throws IOException { if (!this.fs.mkdirs(getAbsolutePath())) { throw new IOException("Failed to create directory: " + this); } } /** * Deletes file or directory represented by this location. * * @param recursive * @throws IOException * if failed to remove directory. */ public final void remove(final boolean recursive) throws IOException { if (!this.fs.delete(this.getAbsolutePath(), recursive)) { throw new IOException("Failed to remove directory: " + this); } } /** * Renames file or directory represented by this location. * * @param path * a new file or directory name. * @return {@code true} if the renaming operation succeeds and {@code true} * otherwise. * @throws IOException * if an I/O error occurs. */ public final boolean rename(final Path path) throws IOException { return this.fs.rename(this.getAbsolutePath(), new Path(this.base, path)); } /** * Returns the data input stream for this location. * * @return the data input stream. * @throws IOException * if an I/O error occurs. */ public final FSDataInputStream getDataInputStream() throws IOException { return this.fs.open(this.getAbsolutePath()); } /** * Returns the data output stream for this location. * * @return the data output stream. * @throws IOException * if an I/O error occurs. */ public final FSDataOutputStream getDataOutputStream() throws IOException { return this.fs.create(this.getAbsolutePath(), true); } /** * Tests if the file name filter is defined. * * @return {@code true} if the file name filter is defined and {@code false} * otherwise. */ public final boolean isFilterDefined() { return this.filter.isPatternDefined(); } /** * Returns files selected in this I/O location. * * @return the selected files. * @throws IOException * if an I/O error occurs. * @throws FileNotFoundException * if a file is not found. */ public final Path[] getFiles() throws FileNotFoundException, IOException { Set<Path> content = new TreeSet<>(); for (FileStatus status : this.fs.listStatus(this.getAbsolutePath(), this.filter)) { if (status.isFile()) { content.add(new Path(IOUtils.normalizePath(status.getPath()).toString() .substring(this.getAbsolutePath().toString().length()))); } } return content.toArray(new Path[content.size()]); } /** * Returns directories selected in this I/O location. * * @param recursive * the flag, which indicates the recursive scan directories. * @return the selected directories. * @throws IOException * if an I/O error occurs. * @throws FileNotFoundException * if a file is not found. */ public final Path[] getDirectories(final boolean recursive) throws FileNotFoundException, IOException { Set<Path> content = new TreeSet<>(); Stack<Path> lookup = new Stack<>(); lookup.push(this.getAbsolutePath()); while (!lookup.empty()) { for (FileStatus status : this.fs.listStatus(lookup.pop())) { if (status.isDirectory()) { content.add(new Path(IOUtils.normalizePath(status.getPath()).toString() .substring(this.getAbsolutePath().toString().length()))); if (recursive) { lookup.push(status.getPath()); } } } } return content.toArray(new Path[content.size()]); } public final Location[] getContent() throws FileNotFoundException, IOException { Set<Path> paths = new TreeSet<>(); if (this.isDirectory()) { Stack<Path> lookup = new Stack<>(); lookup.push(this.getAbsolutePath()); boolean isRoot = true; while (!lookup.empty()) { FileStatus[] statuses = this.fs.listStatus(lookup.pop()); if (statuses != null) { for (FileStatus status : statuses) { if (isRoot && !this.filter.accept(status.getPath())) { continue; } paths.add(new Path(IOUtils.normalizePath(status.getPath()).toString() .substring(this.getAbsolutePath().toString().length() + 1))); if (status.isDirectory()) { lookup.push(status.getPath()); } } } isRoot = false; } } else { paths.add(this.path); } Location[] content = new Location[0]; for (Path path : paths) { content = ObjectArrays.concat(content, this.resolve(path)); } return content; } /** * Returns the location size. It includes the combined size of all files * represented by this location. * * @return the location size. * @throws IOException * if an I/O error occurs. * @throws FileNotFoundException * if a file is not found. */ public final long getSize() throws FileNotFoundException, IOException { long size = 0; if (this.isDirectory()) { if (this.isFilterDefined()) { for (Path file : this.getFiles()) { size += this.fs.getFileStatus(this.resolve(file).getAbsolutePath()).getLen(); } } else { for (Path dir : this.getDirectories(true)) { for (Path file : this.resolve(dir).getFiles()) { size += this.fs.getFileStatus(this.resolve(dir, file).getAbsolutePath()).getLen(); } } } } else if (this.isFile()) { size = this.fs.getFileStatus(this.getAbsolutePath()).getLen(); } else { throw new IOException("Unknown entity: " + this); } return size; } /** * Returns the location permissions. * * @return the location permissions. For the Windows OS this function * returns {@code null}. * @throws IOException * if failed to obtain permissions. */ public final FsPermission getPermission() throws IOException { if (System.getProperty("os.name").startsWith("Windows")) { return null; } else { return this.fs.getFileStatus(this.getAbsolutePath()).getPermission(); } } /** * Returns the location owner. * * @return the location owner. For the Windows OS this function returns * {@code null}. * @throws IOException * if failed to obtain owner. */ public final String getOwner() throws IOException { if (System.getProperty("os.name").startsWith("Windows")) { return null; } else { return this.fs.getFileStatus(this.getAbsolutePath()).getOwner(); } } /** * Returns the location group. * * @return the location group. For the Windows OS this function returns * {@code null}. * @throws IOException * if failed to obtain group. */ public final String getGroup() throws IOException { if (System.getProperty("os.name").startsWith("Windows")) { return null; } else { return this.fs.getFileStatus(this.getAbsolutePath()).getGroup(); } } /** * Returns the location modification time. * * @return the location modification time * @throws IOException * if failed to modification time. */ public final long getModificationTime() throws IOException { return this.fs.getFileStatus(this.getAbsolutePath()).getModificationTime(); } /** * Returns the location modification time as {@code String}. * * @return the location modification time as {@code String}. * @throws IOException * if failed to modification time. */ public final String getModificationTimeAsString() throws IOException { return new SimpleDateFormat("YYYY-MM-dd hh:mm").format(this.getModificationTime()); } /** * Returns {@code String} representation of {@code Entity} instance. */ @Override public String toString() { return Objects.toStringHelper(this).add("file-system", this.fs).add("base", this.base) .add("path", this.path).add("filter", this.filter).toString(); } /** * The file name filter. * * @author Mykola Galushka */ private static class FileNameFilter implements PathFilter { /** * The regular expression for filtering file names. */ private final Pattern pattern; /** * Constructs a new file name filter. * * @param pattern * the regular expression for filtering file names. */ public FileNameFilter(final String wildcard) { if (Strings.isNullOrEmpty(wildcard)) { this.pattern = null; } else { this.pattern = PatternUtils.getWildCardPattern(wildcard); } } /** * Constructs a new default file name filter. */ public FileNameFilter() { this(null); } /** * Tests if the regular expression for filtering file names is defined. * * @return {@code true} if the regular expression is defined and * {@code false} otherwise. */ public boolean isPatternDefined() { return this.pattern != null; } /** * Checks the specified path. */ @Override public boolean accept(Path path) { if (isPatternDefined()) { return this.pattern.matcher(path.getName()).find(); } else { return true; } } @Override public String toString() { return Objects.toStringHelper(this).add("pattern", this.pattern).toString(); } } }