Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.ignite.internal.processors.hadoop.impl.delegate; import java.util.Arrays; import java.util.Collections; import java.util.List; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.ParentNotDirectoryException; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathExistsException; import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.ignite.IgniteException; import org.apache.ignite.hadoop.fs.CachingHadoopFileSystemFactory; import org.apache.ignite.hadoop.fs.HadoopFileSystemFactory; import org.apache.ignite.hadoop.fs.IgniteHadoopIgfsSecondaryFileSystem; import org.apache.ignite.igfs.IgfsBlockLocation; import org.apache.ignite.igfs.IgfsDirectoryNotEmptyException; import org.apache.ignite.igfs.IgfsException; import org.apache.ignite.igfs.IgfsFile; import org.apache.ignite.igfs.IgfsParentNotDirectoryException; import org.apache.ignite.igfs.IgfsPath; import org.apache.ignite.igfs.IgfsPathAlreadyExistsException; import org.apache.ignite.igfs.IgfsPathNotFoundException; import org.apache.ignite.igfs.IgfsUserContext; import org.apache.ignite.igfs.secondary.IgfsSecondaryFileSystemPositionedReadable; import org.apache.ignite.internal.processors.hadoop.delegate.HadoopDelegateUtils; import org.apache.ignite.internal.processors.hadoop.delegate.HadoopFileSystemFactoryDelegate; import org.apache.ignite.internal.processors.hadoop.delegate.HadoopIgfsSecondaryFileSystemDelegate; import org.apache.ignite.internal.processors.hadoop.impl.igfs.HadoopIgfsProperties; import org.apache.ignite.internal.processors.hadoop.impl.igfs.HadoopIgfsSecondaryFileSystemPositionedReadable; import org.apache.ignite.internal.processors.igfs.IgfsBlockLocationImpl; import org.apache.ignite.internal.processors.igfs.IgfsEntryInfo; import org.apache.ignite.internal.processors.igfs.IgfsFileImpl; import org.apache.ignite.internal.processors.igfs.IgfsUtils; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.lang.IgniteUuid; import org.jetbrains.annotations.Nullable; import java.io.FileNotFoundException; import java.io.IOException; import java.io.OutputStream; import java.net.URI; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.Map; /** * Secondary file system implementation. */ @SuppressWarnings("unused") public class HadoopIgfsSecondaryFileSystemDelegateImpl implements HadoopIgfsSecondaryFileSystemDelegate { /** The default user name. It is used if no user context is set. */ private final String dfltUsrName; /** Factory. */ private final HadoopFileSystemFactoryDelegate factory; /** * Constructor. * * @param proxy Proxy. */ public HadoopIgfsSecondaryFileSystemDelegateImpl(IgniteHadoopIgfsSecondaryFileSystem proxy) { assert proxy.getFileSystemFactory() != null; dfltUsrName = IgfsUtils.fixUserName(proxy.getDefaultUserName()); HadoopFileSystemFactory factory0 = proxy.getFileSystemFactory(); if (factory0 == null) factory0 = new CachingHadoopFileSystemFactory(); factory = HadoopDelegateUtils.fileSystemFactoryDelegate(getClass().getClassLoader(), factory0); } /** {@inheritDoc} */ @Override public boolean exists(IgfsPath path) { try { return fileSystemForUser().exists(convert(path)); } catch (IOException e) { throw handleSecondaryFsError(e, "Failed to check file existence [path=" + path + "]"); } } /** {@inheritDoc} */ @Nullable @Override public IgfsFile update(IgfsPath path, Map<String, String> props) { HadoopIgfsProperties props0 = new HadoopIgfsProperties(props); final FileSystem fileSys = fileSystemForUser(); Path hadoopPath = convert(path); try { if (!fileSys.exists(hadoopPath)) return null; if (props0.userName() != null || props0.groupName() != null) fileSys.setOwner(hadoopPath, props0.userName(), props0.groupName()); if (props0.permission() != null) fileSys.setPermission(hadoopPath, props0.permission()); } catch (IOException e) { throw handleSecondaryFsError(e, "Failed to update file properties [path=" + path + "]"); } return info(path); } /** {@inheritDoc} */ @Override public void rename(IgfsPath src, IgfsPath dest) { // Delegate to the secondary file system. try { if (!fileSystemForUser().rename(convert(src), convert(dest))) throw new IgfsException("Failed to rename (secondary file system returned false) " + "[src=" + src + ", dest=" + dest + ']'); } catch (IOException e) { throw handleSecondaryFsError(e, "Failed to rename file [src=" + src + ", dest=" + dest + ']'); } } /** {@inheritDoc} */ @Override public boolean delete(IgfsPath path, boolean recursive) { try { return fileSystemForUser().delete(convert(path), recursive); } catch (IOException e) { throw handleSecondaryFsError(e, "Failed to delete file [path=" + path + ", recursive=" + recursive + "]"); } } /** {@inheritDoc} */ @Override public void mkdirs(IgfsPath path) { try { if (!fileSystemForUser().mkdirs(convert(path))) throw new IgniteException("Failed to make directories [path=" + path + "]"); } catch (IOException e) { throw handleSecondaryFsError(e, "Failed to make directories [path=" + path + "]"); } } /** {@inheritDoc} */ @Override public void mkdirs(IgfsPath path, @Nullable Map<String, String> props) { try { if (!fileSystemForUser().mkdirs(convert(path), new HadoopIgfsProperties(props).permission())) throw new IgniteException("Failed to make directories [path=" + path + ", props=" + props + "]"); } catch (IOException e) { throw handleSecondaryFsError(e, "Failed to make directories [path=" + path + ", props=" + props + "]"); } } /** {@inheritDoc} */ @Override public Collection<IgfsPath> listPaths(IgfsPath path) { try { FileStatus[] statuses = fileSystemForUser().listStatus(convert(path)); if (statuses == null) throw new IgfsPathNotFoundException("Failed to list files (path not found): " + path); Collection<IgfsPath> res = new ArrayList<>(statuses.length); for (FileStatus status : statuses) res.add(new IgfsPath(path, status.getPath().getName())); return res; } catch (FileNotFoundException ignored) { throw new IgfsPathNotFoundException("Failed to list files (path not found): " + path); } catch (IOException e) { throw handleSecondaryFsError(e, "Failed to list statuses due to secondary file system exception: " + path); } } /** {@inheritDoc} */ @Override public Collection<IgfsFile> listFiles(IgfsPath path) { try { FileStatus[] statuses = fileSystemForUser().listStatus(convert(path)); if (statuses == null) throw new IgfsPathNotFoundException("Failed to list files (path not found): " + path); Collection<IgfsFile> res = new ArrayList<>(statuses.length); for (FileStatus s : statuses) { IgfsEntryInfo fsInfo = s.isDirectory() ? IgfsUtils.createDirectory(IgniteUuid.randomUuid(), null, properties(s), s.getAccessTime(), s.getModificationTime()) : IgfsUtils.createFile(IgniteUuid.randomUuid(), (int) s.getBlockSize(), s.getLen(), null, null, false, properties(s), s.getAccessTime(), s.getModificationTime()); res.add(new IgfsFileImpl(new IgfsPath(path, s.getPath().getName()), fsInfo, 1)); } return res; } catch (FileNotFoundException ignored) { throw new IgfsPathNotFoundException("Failed to list files (path not found): " + path); } catch (IOException e) { throw handleSecondaryFsError(e, "Failed to list statuses due to secondary file system exception: " + path); } } /** {@inheritDoc} */ @Override public IgfsSecondaryFileSystemPositionedReadable open(IgfsPath path, int bufSize) { return new HadoopIgfsSecondaryFileSystemPositionedReadable(fileSystemForUser(), convert(path), bufSize); } /** {@inheritDoc} */ @Override public OutputStream create(IgfsPath path, boolean overwrite) { try { return fileSystemForUser().create(convert(path), overwrite); } catch (IOException e) { throw handleSecondaryFsError(e, "Failed to create file [path=" + path + ", overwrite=" + overwrite + "]"); } } /** {@inheritDoc} */ @Override public OutputStream create(IgfsPath path, int bufSize, boolean overwrite, int replication, long blockSize, @Nullable Map<String, String> props) { HadoopIgfsProperties props0 = new HadoopIgfsProperties(props); try { return fileSystemForUser().create(convert(path), props0.permission(), overwrite, bufSize, (short) replication, blockSize, null); } catch (IOException e) { throw handleSecondaryFsError(e, "Failed to create file [path=" + path + ", props=" + props + ", overwrite=" + overwrite + ", bufSize=" + bufSize + ", replication=" + replication + ", blockSize=" + blockSize + "]"); } } /** {@inheritDoc} */ @Override public OutputStream append(IgfsPath path, int bufSize, boolean create, @Nullable Map<String, String> props) { try { Path hadoopPath = convert(path); FileSystem fs = fileSystemForUser(); if (create && !fs.exists(hadoopPath)) return fs.create(hadoopPath, false, bufSize); else return fs.append(convert(path), bufSize); } catch (IOException e) { throw handleSecondaryFsError(e, "Failed to append file [path=" + path + ", bufSize=" + bufSize + "]"); } } /** {@inheritDoc} */ @Override public IgfsFile info(final IgfsPath path) { try { final FileStatus status = fileSystemForUser().getFileStatus(convert(path)); if (status == null) return null; final Map<String, String> props = properties(status); return new IgfsFileImpl(new IgfsFile() { @Override public IgfsPath path() { return path; } @Override public boolean isFile() { return status.isFile(); } @Override public boolean isDirectory() { return status.isDirectory(); } @Override public int blockSize() { // By convention directory has blockSize == 0, while file has blockSize > 0: return isDirectory() ? 0 : (int) status.getBlockSize(); } @Override public long groupBlockSize() { return status.getBlockSize(); } @Override public long accessTime() { return status.getAccessTime(); } @Override public long modificationTime() { return status.getModificationTime(); } @Override public String property(String name) throws IllegalArgumentException { String val = props.get(name); if (val == null) throw new IllegalArgumentException( "File property not found [path=" + path + ", name=" + name + ']'); return val; } @Nullable @Override public String property(String name, @Nullable String dfltVal) { String val = props.get(name); return val == null ? dfltVal : val; } @Override public long length() { return status.getLen(); } /** {@inheritDoc} */ @Override public Map<String, String> properties() { return props; } }, 0); } catch (FileNotFoundException ignore) { return null; } catch (IOException e) { throw handleSecondaryFsError(e, "Failed to get file status [path=" + path + "]"); } } /** {@inheritDoc} */ @Override public long usedSpaceSize() { try { // We don't use FileSystem#getUsed() since it counts only the files // in the filesystem root, not all the files recursively. return fileSystemForUser().getContentSummary(new Path("/")).getSpaceConsumed(); } catch (IOException e) { throw handleSecondaryFsError(e, "Failed to get used space size of file system."); } } /** {@inheritDoc} */ @Override public void setTimes(IgfsPath path, long modificationTime, long accessTime) throws IgniteException { try { // We don't use FileSystem#getUsed() since it counts only the files // in the filesystem root, not all the files recursively. fileSystemForUser().setTimes(convert(path), modificationTime, accessTime); } catch (IOException e) { throw handleSecondaryFsError(e, "Failed set times for path: " + path); } } /** {@inheritDoc} */ @Override public Collection<IgfsBlockLocation> affinity(IgfsPath path, long start, long len, long maxLen) throws IgniteException { try { BlockLocation[] hadoopBlocks = fileSystemForUser().getFileBlockLocations(convert(path), start, len); List<IgfsBlockLocation> blks = new ArrayList<>(hadoopBlocks.length); for (int i = 0; i < hadoopBlocks.length; ++i) blks.add(convertBlockLocation(hadoopBlocks[i])); return blks; } catch (FileNotFoundException ignored) { return Collections.emptyList(); } catch (IOException e) { throw handleSecondaryFsError(e, "Failed affinity for path: " + path); } } /** {@inheritDoc} */ public void start() { factory.start(); } /** {@inheritDoc} */ public void stop() { factory.stop(); } /** * Convert IGFS path into Hadoop path. * * @param path IGFS path. * @return Hadoop path. */ private Path convert(IgfsPath path) { URI uri = fileSystemForUser().getUri(); return new Path(uri.getScheme(), uri.getAuthority(), path.toString()); } /** * Convert IGFS affinity block location into Hadoop affinity block location. * * @param block IGFS affinity block location. * @return Hadoop affinity block location. */ private IgfsBlockLocation convertBlockLocation(BlockLocation block) { try { String[] names = block.getNames(); String[] hosts = block.getHosts(); return new IgfsBlockLocationImpl(block.getOffset(), block.getLength(), Arrays.asList(names), Arrays.asList(hosts)); } catch (IOException e) { throw handleSecondaryFsError(e, "Failed convert block location: " + block); } } /** * Heuristically checks if exception was caused by invalid HDFS version and returns appropriate exception. * * @param e Exception to check. * @param detailMsg Detailed error message. * @return Appropriate exception. */ private IgfsException handleSecondaryFsError(IOException e, String detailMsg) { return cast(detailMsg, e); } /** * Cast IO exception to IGFS exception. * * @param msg Error message. * @param e IO exception. * @return IGFS exception. */ public static IgfsException cast(String msg, IOException e) { if (e instanceof FileNotFoundException) return new IgfsPathNotFoundException(e); else if (e instanceof ParentNotDirectoryException) return new IgfsParentNotDirectoryException(msg, e); else if (e instanceof PathIsNotEmptyDirectoryException) return new IgfsDirectoryNotEmptyException(e); else if (e instanceof PathExistsException) return new IgfsPathAlreadyExistsException(msg, e); else return new IgfsException(msg, e); } /** * Convert Hadoop FileStatus properties to map. * * @param status File status. * @return IGFS attributes. */ private static Map<String, String> properties(FileStatus status) { FsPermission perm = status.getPermission(); if (perm == null) perm = FsPermission.getDefault(); HashMap<String, String> res = new HashMap<>(3); res.put(IgfsUtils.PROP_PERMISSION, String.format("%04o", perm.toShort())); res.put(IgfsUtils.PROP_USER_NAME, status.getOwner()); res.put(IgfsUtils.PROP_GROUP_NAME, status.getGroup()); return res; } /** * Gets the FileSystem for the current context user. * @return the FileSystem instance, never null. */ private FileSystem fileSystemForUser() { String user = IgfsUserContext.currentUser(); if (F.isEmpty(user)) user = IgfsUtils.fixUserName(dfltUsrName); assert !F.isEmpty(user); try { return (FileSystem) factory.get(user); } catch (IOException ioe) { throw new IgniteException(ioe); } } /** {@inheritDoc} */ @Override public String toString() { return S.toString(HadoopIgfsSecondaryFileSystemDelegateImpl.class, this); } }