Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one or more contributor license * agreements. See the NOTICE file distributed with this work for additional information regarding * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. You may obtain a * copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under * the License. */ package com.aliyun.odps.fs; import java.io.FileNotFoundException; import java.io.IOException; import java.net.URI; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileAlreadyExistsException; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FsStatus; import org.apache.hadoop.fs.ParentNotDirectoryException; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.util.Progressable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.aliyun.odps.Odps; import com.aliyun.odps.VolumeException; import com.aliyun.odps.VolumeFSFile; import com.aliyun.odps.account.Account; import com.aliyun.odps.account.AliyunAccount; import com.aliyun.odps.tunnel.VolumeFSErrorCode; import com.aliyun.odps.volume.VolumeFSClient; import com.aliyun.odps.volume.VolumeFSInputStream; import com.aliyun.odps.volume.VolumeFSOutputStream; import com.aliyun.odps.volume.VolumeFSUtil; import com.aliyun.odps.volume.protocol.VolumeFSConstants; import com.aliyun.odps.volume.protocol.VolumeFSErrorMessageGenerator; /** * The ODPS Volume implementation of Hadoop {@link FileSystem} * * @author Emerson Zhao [mailto:zhenyi.zzy@alibaba-inc.com] * */ public class VolumeFileSystem extends FileSystem { private static final Logger LOG = LoggerFactory.getLogger(VolumeFileSystem.class); /* * Current project */ private String project; private String homeVolume; private short defaultReplication; private Path workingDir; private URI uri; private VolumeFSClient volumeClient; public VolumeFileSystem() { } /** * Return the protocol scheme for the FileSystem. * <p/> * * @return <code>odps</code> */ @Override public String getScheme() { return VolumeFileSystemConfigKeys.VOLUME_URI_SCHEME; } @Override public URI getUri() { return uri; } @Override public void initialize(URI uri, Configuration conf) throws IOException { conf.addResource(VolumeFSConstants.VOLUME_FS_CONFIG_FILE); super.initialize(uri, conf); setConf(conf); checkURI(uri); this.project = resolveProject(uri); this.volumeClient = createVolumeClient(conf); this.uri = URI.create(uri.getScheme() + VolumeFSConstants.SCHEME_SEPARATOR + uri.getAuthority()); this.homeVolume = getHomeVolume(conf); this.workingDir = getHomeDirectory(); this.defaultReplication = (short) conf.getInt(VolumeFileSystemConfigKeys.DFS_REPLICATION_KEY, VolumeFSConstants.DFS_REPLICATION_DEFAULT); } /* * You should create the homeVolume manually first */ private String getHomeVolume(Configuration conf) { String defaultVolume = conf.get(VolumeFileSystemConfigKeys.ODPS_HOME_VOLMUE); if (defaultVolume != null) { return defaultVolume; } else { return VolumeFSConstants.DEFAULT_HOME_VOLUME; } } private String resolveProject(URI uri) { return uri.getAuthority(); } private void checkURI(URI uri) throws IOException { String authority = uri.getAuthority(); if (authority == null) { throw new IOException("Incomplete ODPS URI, no project: " + uri); } } private VolumeFSClient createVolumeClient(Configuration conf) throws IOException { String accessId = conf.get(VolumeFileSystemConfigKeys.ODPS_ACCESS_ID); String accessKey = conf.get(VolumeFileSystemConfigKeys.ODPS_ACCESS_KEY); if (accessId == null || accessKey == null) { throw new IOException("Incomplete config, no accessId or accessKey"); } String serviceEndpoint = conf.get(VolumeFileSystemConfigKeys.ODPS_SERVICE_ENDPOINT); String tunnelEndpoint = conf.get(VolumeFileSystemConfigKeys.ODPS_TUNNEL_ENDPOINT); if (serviceEndpoint == null) { throw new IOException("Incomplete config, no " + VolumeFileSystemConfigKeys.ODPS_SERVICE_ENDPOINT); } Account account = new AliyunAccount(accessId, accessKey); Odps odps = new Odps(account); odps.setEndpoint(serviceEndpoint); return new VolumeFSClient(odps, project, serviceEndpoint, tunnelEndpoint, conf); } @Override public Path getHomeDirectory() { return new Path(VolumeFSConstants.ROOT_PATH + homeVolume, System.getProperty("user.name")); } @Override public FSDataInputStream open(Path f, int bufferSize) throws IOException { Path absF = fixRelativePart(f); String filePath = getPathName(absF); if (!exists(absF)) { throw new FileNotFoundException(filePath); } FileStatus fileStatus = getFileStatus(f); if (fileStatus.isDirectory()) { throw new FileNotFoundException(VolumeFSErrorMessageGenerator.isADirectory(filePath)); } return new FSDataInputStream( new VolumeFSInputStream(filePath, volumeClient, fileStatus.getLen(), getConf())); } @Override public FSDataOutputStream create(Path f, FsPermission permission, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { Path absF = fixRelativePart(f); String filePath = getPathName(absF); if (!VolumeFSUtil.isValidName(filePath)) { throw new IllegalArgumentException( VolumeFSErrorMessageGenerator.isNotAValidODPSVolumeFSFilename(filePath)); } if (VolumeFSUtil.checkPathIsJustVolume(filePath)) { throw new IOException( VolumeFSErrorMessageGenerator.theOpreationIsNotAllowed("Create file in the root path!")); } try { return new FSDataOutputStream(new VolumeFSOutputStream(filePath, volumeClient, permission, overwrite, replication, blockSize, progress), statistics); } catch (VolumeException e) { logException(e); throw wrapExceptions(filePath, e); } } @Override public FSDataOutputStream append(Path f, int bufferSize, Progressable progress) throws IOException { throw new IOException("Not supported"); } @Override public boolean rename(Path src, Path dst) throws IOException { statistics.incrementWriteOps(1); Path absSrc = fixRelativePart(src); Path absDst = fixRelativePart(dst); if (!exists(absSrc)) { throw new FileNotFoundException("Source path " + src + " does not exist"); } if (isDirectory(absDst)) { // destination is a directory: rename goes underneath it with the // source name absDst = new Path(absDst, absSrc.getName()); } if (exists(absDst)) { throw new FileAlreadyExistsException("Destination path " + dst + " already exists"); } if (absDst.getParent() != null && !exists(absDst.getParent())) { throw new FileNotFoundException( VolumeFSErrorMessageGenerator.noSuchFileOrDirectory(absDst.getParent().toString())); } if (VolumeFSUtil.isParentOf(absSrc, absDst)) { throw new IOException("Cannot rename " + absSrc + " under itself" + " : " + absDst); } String srcPath = getPathName(absSrc); String dstPath = getPathName(absDst); try { return volumeClient.rename(srcPath, dstPath); } catch (VolumeException e) { logException(e); throw wrapExceptions(srcPath, e); } } @Override public boolean delete(Path f, boolean recursive) throws IOException { statistics.incrementWriteOps(1); Path absF = fixRelativePart(f); String filePath = getPathName(absF); if (VolumeFSConstants.ROOT_PATH.equalsIgnoreCase(filePath.replaceAll("//", "/").trim()) && !recursive) { throw new IOException("Non recursive delete is not allowed on root path"); } try { getFileStatus(absF); } catch (IOException e) { if (e instanceof FileNotFoundException) { return false; } } try { return volumeClient.delete(filePath, recursive); } catch (VolumeException e) { IOException ioe = wrapExceptions(filePath, e); if (ioe instanceof FileNotFoundException) { return false; } else { logException(e); throw ioe; } } } @Override public FileStatus[] listStatus(Path f) throws FileNotFoundException, IOException { statistics.incrementReadOps(1); Path absF = fixRelativePart(f); String filePath = getPathName(absF); VolumeFSFile[] files; try { files = volumeClient.getFileInfosByPath(filePath); } catch (VolumeException e) { logException(e); throw wrapExceptions(filePath, e); } return VolumeFSUtil.transferFiles(files); } @Override public void setWorkingDirectory(Path new_dir) { Path dir = fixRelativePart(new_dir); String result = dir.toUri().getPath(); if (!VolumeFSUtil.isValidName(result)) { throw new IllegalArgumentException("Invalid Volume directory name " + result); } workingDir = dir; } @Override public Path getWorkingDirectory() { return workingDir; } @Override public boolean mkdirs(Path f, FsPermission permission) throws IOException { statistics.incrementWriteOps(1); Path absF = fixRelativePart(f); String filePath = getPathName(absF); try { return volumeClient.mkdirs(filePath); } catch (VolumeException e) { logException(e); throw wrapExceptions(filePath, e); } } @Override public boolean setReplication(Path src, short replication) throws IOException { statistics.incrementWriteOps(1); Path absF = fixRelativePart(src); String filePath = getPathName(absF); try { return volumeClient.setReplication(filePath, replication); } catch (VolumeException e) { logException(e); throw wrapExceptions(filePath, e); } } @Override public short getDefaultReplication() { return defaultReplication; } @Override public FileStatus getFileStatus(Path f) throws IOException { statistics.incrementReadOps(1); Path absF = fixRelativePart(f); String filePath = getPathName(absF); VolumeFSFile file; try { file = volumeClient.getFileInfo(filePath); } catch (VolumeException e) { logException(e); throw wrapExceptions(filePath, e); } return VolumeFSUtil.transferFile(file); } @Override public FsStatus getStatus(Path p) throws IOException { // TODO return new FsStatus(0, 0, 0); } private IOException wrapExceptions(String path, VolumeException e) { if (VolumeFSErrorCode.NoSuchPath.equalsIgnoreCase(e.getErrCode())) { return new FileNotFoundException(VolumeFSErrorMessageGenerator.noSuchFileOrDirectory(path)); } if (VolumeFSErrorCode.InvalidPath.equalsIgnoreCase(e.getErrCode())) { throw new IllegalArgumentException(VolumeFSErrorMessageGenerator.isNotAValidODPSVolumeFSFilename(path)); } if (VolumeFSErrorCode.NotAcceptableOperation.equalsIgnoreCase(e.getErrCode())) { throw new UnsupportedOperationException(e.getMessage()); } if (VolumeFSErrorCode.PathAlreadyExists.equalsIgnoreCase(e.getErrCode())) { return new FileAlreadyExistsException(e.getMessage()); } if (VolumeFSErrorCode.ParentNotDirectory.equalsIgnoreCase(e.getErrCode())) { return new ParentNotDirectoryException(e.getMessage()); } else { return new IOException(e.getMessage(), e); } } /** * Checks that the passed URI belongs to this filesystem and returns just the path component. * Expects a URI with an absolute path. * * @param file URI with absolute path * @return path component of {file} * @throws IllegalArgumentException if URI does not belong to this DFS */ private String getPathName(Path file) { checkPath(file); String result = file.toUri().getPath(); if (!VolumeFSUtil.isValidName(result)) { throw new IllegalArgumentException( "Pathname " + result + " from " + file + " is not a valid VolumeFS filename."); } return result; } private void logException(VolumeException e) { if (LOG.isDebugEnabled()) { LOG.debug(e.getMessage(), e); } } }