Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package gobblin.util.filesystem; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; import java.util.Map; import org.apache.commons.io.IOCase; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; import com.google.common.collect.Maps; import gobblin.util.DecoratorUtils; import lombok.extern.slf4j.Slf4j; @Slf4j public class PathAlterationObserver { private final Map<PathAlterationListener, PathAlterationListener> listeners = Maps.newConcurrentMap(); private final FileStatusEntry rootEntry; private final PathFilter pathFilter; private final Comparator<Path> comparator; private final FileSystem fs; private final Path[] EMPTY_PATH_ARRAY = new Path[0]; /** * Final processing. */ public void destroy() { } /** * Construct an observer for the specified directory. * * @param directoryName the name of the directory to observe */ public PathAlterationObserver(final String directoryName) throws IOException { this(new Path(directoryName)); } /** * Construct an observer for the specified directory and file filter. * * @param directoryName the name of the directory to observe * @param pathFilter The file filter or null if none */ public PathAlterationObserver(final String directoryName, final PathFilter pathFilter) throws IOException { this(new Path(directoryName), pathFilter); } /** * Construct an observer for the specified directory. * * @param directory the directory to observe */ public PathAlterationObserver(final Path directory) throws IOException { this(directory, null); } /** * Construct an observer for the specified directory and file filter. * * @param directory the directory to observe * @param pathFilter The file filter or null if none */ public PathAlterationObserver(final Path directory, final PathFilter pathFilter) throws IOException { this(new FileStatusEntry(directory), pathFilter); } /** * The comparison between path is always case-sensitive in this general file system context. */ public PathAlterationObserver(final FileStatusEntry rootEntry, final PathFilter pathFilter) throws IOException { if (rootEntry == null) { throw new IllegalArgumentException("Root entry is missing"); } if (rootEntry.getPath() == null) { throw new IllegalArgumentException("Root directory is missing"); } this.rootEntry = rootEntry; this.pathFilter = pathFilter; this.fs = rootEntry.getPath().getFileSystem(new Configuration()); // By default, the comparsion is case sensitive. this.comparator = new Comparator<Path>() { @Override public int compare(Path o1, Path o2) { return IOCase.SENSITIVE.checkCompareTo(o1.toUri().toString(), o2.toUri().toString()); } }; } /** * Add a file system listener. * * @param listener The file system listener */ public void addListener(final PathAlterationListener listener) { if (listener != null) { this.listeners.put(listener, new ExceptionCatchingPathAlterationListenerDecorator(listener)); } } /** * Remove a file system listener. * * @param listener The file system listener */ public void removeListener(final PathAlterationListener listener) { if (listener != null) { this.listeners.remove(listener); } } /** * Returns the set of registered file system listeners. * * @return The file system listeners */ public Iterable<PathAlterationListener> getListeners() { return listeners.keySet(); } /** * Initialize the observer. * @throws IOException if an error occurs */ public void initialize() throws IOException { rootEntry.refresh(rootEntry.getPath()); final FileStatusEntry[] children = doListPathsEntry(rootEntry.getPath(), rootEntry); rootEntry.setChildren(children); } /** * Check whether the file and its chlidren have been created, modified or deleted. */ public void checkAndNotify() throws IOException { /* fire onStart() */ for (final PathAlterationListener listener : listeners.values()) { listener.onStart(this); } /* fire directory/file events */ final Path rootPath = rootEntry.getPath(); if (fs.exists(rootPath)) { // Current existed. checkAndNotify(rootEntry, rootEntry.getChildren(), listPaths(rootPath)); } else if (rootEntry.isExists()) { // Existed before and not existed now. checkAndNotify(rootEntry, rootEntry.getChildren(), EMPTY_PATH_ARRAY); } else { // Didn't exist and still doesn't } /* fire onStop() */ for (final PathAlterationListener listener : listeners.values()) { listener.onStop(this); } } /** * Compare two file lists for files which have been created, modified or deleted. * * @param parent The parent entry * @param previous The original list of paths * @param currentPaths The current list of paths */ private void checkAndNotify(final FileStatusEntry parent, final FileStatusEntry[] previous, final Path[] currentPaths) throws IOException { int c = 0; final FileStatusEntry[] current = currentPaths.length > 0 ? new FileStatusEntry[currentPaths.length] : FileStatusEntry.EMPTY_ENTRIES; for (final FileStatusEntry previousEntry : previous) { while (c < currentPaths.length && comparator.compare(previousEntry.getPath(), currentPaths[c]) > 0) { current[c] = createPathEntry(parent, currentPaths[c]); doCreate(current[c]); c++; } if (c < currentPaths.length && comparator.compare(previousEntry.getPath(), currentPaths[c]) == 0) { doMatch(previousEntry, currentPaths[c]); checkAndNotify(previousEntry, previousEntry.getChildren(), listPaths(currentPaths[c])); current[c] = previousEntry; c++; } else { checkAndNotify(previousEntry, previousEntry.getChildren(), EMPTY_PATH_ARRAY); doDelete(previousEntry); } } for (; c < currentPaths.length; c++) { current[c] = createPathEntry(parent, currentPaths[c]); doCreate(current[c]); } parent.setChildren(current); } /** * Create a new FileStatusEntry for the specified file. * * @param parent The parent file entry * @param childPath The file to create an entry for * @return A new file entry */ private FileStatusEntry createPathEntry(final FileStatusEntry parent, final Path childPath) throws IOException { final FileStatusEntry entry = parent.newChildInstance(childPath); entry.refresh(childPath); final FileStatusEntry[] children = doListPathsEntry(childPath, entry); entry.setChildren(children); return entry; } /** * List the path in the format of FileStatusEntry array * @param path The path to list files for * @param entry the parent entry * @return The child files */ private FileStatusEntry[] doListPathsEntry(Path path, FileStatusEntry entry) throws IOException { final Path[] paths = listPaths(path); final FileStatusEntry[] children = paths.length > 0 ? new FileStatusEntry[paths.length] : FileStatusEntry.EMPTY_ENTRIES; for (int i = 0; i < paths.length; i++) { children[i] = createPathEntry(entry, paths[i]); } return children; } /** * Fire directory/file created events to the registered listeners. * * @param entry The file entry */ private void doCreate(final FileStatusEntry entry) { for (final PathAlterationListener listener : listeners.values()) { if (entry.isDirectory()) { listener.onDirectoryCreate(entry.getPath()); } else { listener.onFileCreate(entry.getPath()); } } final FileStatusEntry[] children = entry.getChildren(); for (final FileStatusEntry aChildren : children) { doCreate(aChildren); } } /** * Fire directory/file change events to the registered listeners. * * @param entry The previous file system entry * @param path The current file */ private void doMatch(final FileStatusEntry entry, final Path path) throws IOException { if (entry.refresh(path)) { for (final PathAlterationListener listener : listeners.values()) { if (entry.isDirectory()) { listener.onDirectoryChange(path); } else { listener.onFileChange(path); } } } } /** * Fire directory/file delete events to the registered listeners. * * @param entry The file entry */ private void doDelete(final FileStatusEntry entry) { for (final PathAlterationListener listener : listeners.values()) { if (entry.isDirectory()) { listener.onDirectoryDelete(entry.getPath()); } else { listener.onFileDelete(entry.getPath()); } } } /** * List the contents of a directory denoted by Path * * @param path The path(File Object in general file system) to list the contents of * @return the directory contents or a zero length array if * the empty or the file is not a directory */ private Path[] listPaths(final Path path) throws IOException { Path[] children = null; ArrayList<Path> tmpChildrenPath = new ArrayList<>(); if (fs.isDirectory(path)) { // Get children's path list. FileStatus[] chiledrenFileStatus = pathFilter == null ? fs.listStatus(path) : fs.listStatus(path, pathFilter); for (FileStatus childFileStatus : chiledrenFileStatus) { tmpChildrenPath.add(childFileStatus.getPath()); } children = tmpChildrenPath.toArray(new Path[tmpChildrenPath.size()]); } if (children == null) { children = EMPTY_PATH_ARRAY; } if (comparator != null && children.length > 1) { Arrays.sort(children, comparator); } return children; } }