Java tutorial
/* * Copyright (c) 2011, Cloudera, Inc. All Rights Reserved. * * Cloudera, Inc. licenses this file to you under the Apache License, * Version 2.0 (the "License"). You may not use this file except in * compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * This software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR * CONDITIONS OF ANY KIND, either express or implied. See the License for * the specific language governing permissions and limitations under the * License. */ package com.cloudera.hoop.client.fs; import com.cloudera.alfredo.client.AuthenticatedURL; import com.cloudera.alfredo.client.Authenticator; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PositionedReadable; import org.apache.hadoop.fs.Seekable; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Progressable; import org.apache.hadoop.util.ReflectionUtils; import org.json.simple.JSONArray; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.FileNotFoundException; import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.OutputStream; import java.net.HttpURLConnection; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.net.URLEncoder; import java.text.MessageFormat; import java.util.HashMap; import java.util.Map; /** * Hoop implementation of the Hadoop FileSystem. * <p/> * This implementation allows a user to access HDFS over HTTP via a Hoop server. */ public class HoopFileSystem extends FileSystem { private AuthenticatedURL.Token authToken = new AuthenticatedURL.Token(); private URI uri; private Path workingDir; private String doAs; /** * Convenience method that creates a <code>HttpURLConnection</code> for the Hoop file system operations. * <p/> * This methods performs and injects any needed authentication credentials. * * @param method the HTTP method. * @param params the query string parameters. * @param path the file path * @return a <code>HttpURLConnection</code> for the Hoop server, authenticated and ready to use for * the specified path and file system operation. * @throws IOException */ private HttpURLConnection getConnection(String method, Map<String, String> params, Path path) throws IOException { params.put("doas", doAs); Class<? extends Authenticator> klass = getConf().getClass("hoop.authenticator.class", HoopKerberosAuthenticator.class, Authenticator.class); Authenticator authenticator = ReflectionUtils.newInstance(klass, getConf()); try { StringBuilder sb = new StringBuilder(); String separator = "?"; for (Map.Entry<String, String> entry : params.entrySet()) { sb.append(separator).append(entry.getKey()).append("=") .append(URLEncoder.encode(entry.getValue(), "UTF8")); separator = "&"; } path = makeQualified(path); URL url = new URL(path + sb.toString()); HttpURLConnection conn = new AuthenticatedURL(authenticator).openConnection(url, authToken); conn.setRequestMethod(method); if (method.equals("POST") || method.equals("PUT")) { conn.setDoOutput(true); } return conn; } catch (Exception ex) { throw new IOException(ex); } } /** * Convenience method that JSON Parses the <code>InputStream</code> of a <code>HttpURLConnection</code>. * * @param conn the <code>HttpURLConnection</code>. * @return the parsed JSON object. * @throws IOException thrown if the <code>InputStream</code> could not be JSON parsed. */ private static Object jsonParse(HttpURLConnection conn) throws IOException { try { JSONParser parser = new JSONParser(); return parser.parse(new InputStreamReader(conn.getInputStream())); } catch (ParseException ex) { throw new IOException("JSON parser error, " + ex.getMessage(), ex); } } /** * Validates the status of an <code>HttpURLConnection</code> against an expected HTTP * status code. If the current status code is not the expected one it throws an exception * with a detail message using Server side error messages if available. * * @param conn the <code>HttpURLConnection</code>. * @param expected the expected HTTP status code. * @throws IOException thrown if the current status code does not match the expected one. */ private static void validateResponse(HttpURLConnection conn, int expected) throws IOException { int status = conn.getResponseCode(); if (status != expected) { try { JSONObject json = (JSONObject) jsonParse(conn); throw new IOException(MessageFormat.format("HTTP status [{0}], {1} - {2}", json.get("status"), json.get("reason"), json.get("message"))); } catch (IOException ex) { if (ex.getCause() instanceof IOException) { throw (IOException) ex.getCause(); } throw new IOException( MessageFormat.format("HTTP status [{0}], {1}", status, conn.getResponseMessage())); } } } /** * Called after a new FileSystem instance is constructed. * * @param name a uri whose authority section names the host, port, etc. for this FileSystem * @param conf the configuration */ @Override public void initialize(URI name, Configuration conf) throws IOException { UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); doAs = ugi.getUserName(); super.initialize(name, conf); try { uri = new URI(name.getScheme() + "://" + name.getHost() + ":" + name.getPort()); } catch (URISyntaxException ex) { throw new IOException(ex); } } /** * Returns a URI whose scheme and authority identify this FileSystem. * * @return the URI whose scheme and authority identify this FileSystem. */ @Override public URI getUri() { return uri; } /** * Hoop subclass of the <code>FSDataInputStream</code>. * <p/> * This implementation does not support the * <code>PositionReadable</code> and <code>Seekable</code> methods. */ private static class HoopFSDataInputStream extends FilterInputStream implements Seekable, PositionedReadable { protected HoopFSDataInputStream(InputStream in, int bufferSize) { super(new BufferedInputStream(in, bufferSize)); } @Override public int read(long position, byte[] buffer, int offset, int length) throws IOException { throw new UnsupportedOperationException(); } @Override public void readFully(long position, byte[] buffer, int offset, int length) throws IOException { throw new UnsupportedOperationException(); } @Override public void readFully(long position, byte[] buffer) throws IOException { throw new UnsupportedOperationException(); } @Override public void seek(long pos) throws IOException { throw new UnsupportedOperationException(); } @Override public long getPos() throws IOException { throw new UnsupportedOperationException(); } @Override public boolean seekToNewSource(long targetPos) throws IOException { throw new UnsupportedOperationException(); } } /** * Opens an FSDataInputStream at the indicated Path. * </p> * IMPORTANT: the returned <code><FSDataInputStream/code> does not support the * <code>PositionReadable</code> and <code>Seekable</code> methods. * * @param f the file name to open * @param bufferSize the size of the buffer to be used. */ @Override public FSDataInputStream open(Path f, int bufferSize) throws IOException { Map<String, String> params = new HashMap<String, String>(); HttpURLConnection conn = getConnection("GET", params, f); validateResponse(conn, HttpURLConnection.HTTP_OK); return new FSDataInputStream(new HoopFSDataInputStream(conn.getInputStream(), bufferSize)); } /** * Hoop subclass of the <code>FSDataOutputStream</code>. * <p/> * This implementation closes the underlying HTTP connection validating the Http connection status * at closing time. */ private static class HoopFSDataOutputStream extends FSDataOutputStream { private HttpURLConnection conn; private int closeStatus; public HoopFSDataOutputStream(HttpURLConnection conn, OutputStream out, int closeStatus, Statistics stats) throws IOException { super(out, stats); this.conn = conn; this.closeStatus = closeStatus; } @Override public void close() throws IOException { try { super.close(); } finally { validateResponse(conn, closeStatus); } } } /** * Converts a <code>FsPermission</code> to a Unix string symbolic representation (ie: '-rwxr--r--') * @param p the permission. * @return the Unix string symbolic reprentation. */ private String permissionToString(FsPermission p) { return (p == null) ? "default" : "-" + p.getUserAction().SYMBOL + p.getGroupAction().SYMBOL + p.getOtherAction().SYMBOL; } /** * Opens an FSDataOutputStream at the indicated Path with write-progress * reporting. * <p/> * IMPORTANT: The <code>Progressable</code> parameter is not used. * * @param f the file name to open * @param permission * @param overwrite if a file with this name already exists, then if true, * the file will be overwritten, and if false an error will be thrown. * @param bufferSize the size of the buffer to be used. * @param replication required block replication for the file. * @param blockSize * @param progress * @throws IOException * @see #setPermission(Path, FsPermission) */ @Override public FSDataOutputStream create(Path f, FsPermission permission, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { Map<String, String> params = new HashMap<String, String>(); params.put("op", "create"); params.put("overwrite", Boolean.toString(overwrite)); params.put("replication", Short.toString(replication)); params.put("blocksize", Long.toString(blockSize)); params.put("permission", permissionToString(permission)); HttpURLConnection conn = getConnection("POST", params, f); try { OutputStream os = new BufferedOutputStream(conn.getOutputStream(), bufferSize); return new HoopFSDataOutputStream(conn, os, HttpURLConnection.HTTP_CREATED, statistics); } catch (IOException ex) { validateResponse(conn, HttpURLConnection.HTTP_CREATED); throw ex; } } /** * Append to an existing file (optional operation). * <p/> * IMPORTANT: The <code>Progressable</code> parameter is not used. * * @param f the existing file to be appended. * @param bufferSize the size of the buffer to be used. * @param progress for reporting progress if it is not null. * @throws IOException */ @Override public FSDataOutputStream append(Path f, int bufferSize, Progressable progress) throws IOException { Map<String, String> params = new HashMap<String, String>(); params.put("op", "append"); HttpURLConnection conn = getConnection("PUT", params, f); try { OutputStream os = new BufferedOutputStream(conn.getOutputStream(), bufferSize); return new HoopFSDataOutputStream(conn, os, HttpURLConnection.HTTP_OK, statistics); } catch (IOException ex) { validateResponse(conn, HttpURLConnection.HTTP_OK); throw ex; } } /** * Renames Path src to Path dst. Can take place on local fs * or remote DFS. */ @Override public boolean rename(Path src, Path dst) throws IOException { Map<String, String> params = new HashMap<String, String>(); params.put("op", "rename"); params.put("to", dst.toString()); HttpURLConnection conn = getConnection("PUT", params, src); validateResponse(conn, HttpURLConnection.HTTP_OK); JSONObject json = (JSONObject) jsonParse(conn); return (Boolean) json.get("rename"); } /** * Delete a file. * @deprecated Use delete(Path, boolean) instead */ @Deprecated @Override public boolean delete(Path f) throws IOException { return delete(f, false); } /** Delete a file. * * @param f the path to delete. * @param recursive if path is a directory and set to * true, the directory is deleted else throws an exception. In * case of a file the recursive can be set to either true or false. * @return true if delete is successful else false. * @throws IOException */ @Override public boolean delete(Path f, boolean recursive) throws IOException { Map<String, String> params = new HashMap<String, String>(); params.put("recursive", Boolean.toString(recursive)); HttpURLConnection conn = getConnection("DELETE", params, f); validateResponse(conn, HttpURLConnection.HTTP_OK); JSONObject json = (JSONObject) jsonParse(conn); return (Boolean) json.get("delete"); } /** * List the statuses of the files/directories in the given path if the path is * a directory. * * @param f * given path * @return the statuses of the files/directories in the given patch * @throws IOException */ @Override public FileStatus[] listStatus(Path f) throws IOException { Map<String, String> params = new HashMap<String, String>(); params.put("op", "list"); HttpURLConnection conn = getConnection("GET", params, f); validateResponse(conn, HttpURLConnection.HTTP_OK); JSONArray json = (JSONArray) jsonParse(conn); FileStatus[] array = new FileStatus[json.size()]; for (int i = 0; i < json.size(); i++) { array[i] = createFileStatus((JSONObject) json.get(i)); } return array; } /** * Set the current working directory for the given file system. All relative * paths will be resolved relative to it. * * @param new_dir */ @Override public void setWorkingDirectory(Path new_dir) { workingDir = new_dir; } /** * Get the current working directory for the given file system * @return the directory pathname */ @Override public Path getWorkingDirectory() { if (workingDir == null) { workingDir = getHomeDirectory(); } return workingDir; } /** * Make the given file and all non-existent parents into * directories. Has the semantics of Unix 'mkdir -p'. * Existence of the directory hierarchy is not an error. */ @Override public boolean mkdirs(Path f, FsPermission permission) throws IOException { Map<String, String> params = new HashMap<String, String>(); params.put("op", "mkdirs"); params.put("permission", permissionToString(permission)); HttpURLConnection conn = getConnection("POST", params, f); validateResponse(conn, HttpURLConnection.HTTP_OK); JSONObject json = (JSONObject) jsonParse(conn); return (Boolean) json.get("mkdirs"); } /** * Return a file status object that represents the path. * * @param f The path we want information from * @return a FileStatus object * @throws FileNotFoundException when the path does not exist; * IOException see specific implementation */ @Override public FileStatus getFileStatus(Path f) throws IOException { Map<String, String> params = new HashMap<String, String>(); params.put("op", "status"); HttpURLConnection conn = getConnection("GET", params, f); validateResponse(conn, HttpURLConnection.HTTP_OK); JSONObject json = (JSONObject) jsonParse(conn); return createFileStatus(json); } /** * Return the current user's home directory in this filesystem. * The default implementation returns "/user/$USER/". */ @Override public Path getHomeDirectory() { Map<String, String> params = new HashMap<String, String>(); params.put("op", "homedir"); try { HttpURLConnection conn = getConnection("GET", params, new Path("/")); validateResponse(conn, HttpURLConnection.HTTP_OK); JSONObject json = (JSONObject) jsonParse(conn); return new Path((String) json.get("homeDir")); } catch (IOException ex) { throw new RuntimeException(ex); } } /** * Set owner of a path (i.e. a file or a directory). * The parameters username and groupname cannot both be null. * * @param p The path * @param username If it is null, the original username remains unchanged. * @param groupname If it is null, the original groupname remains unchanged. */ @Override public void setOwner(Path p, String username, String groupname) throws IOException { Map<String, String> params = new HashMap<String, String>(); params.put("op", "setowner"); params.put("owner", username); params.put("group", groupname); HttpURLConnection conn = getConnection("PUT", params, p); validateResponse(conn, HttpURLConnection.HTTP_OK); } /** * Set permission of a path. * * @param p * @param permission */ @Override public void setPermission(Path p, FsPermission permission) throws IOException { Map<String, String> params = new HashMap<String, String>(); params.put("op", "setpermission"); params.put("permission", permissionToString(permission)); HttpURLConnection conn = getConnection("PUT", params, p); validateResponse(conn, HttpURLConnection.HTTP_OK); } /** * Set access time of a file * * @param p The path * @param mtime Set the modification time of this file. * The number of milliseconds since Jan 1, 1970. * A value of -1 means that this call should not set modification time. * @param atime Set the access time of this file. * The number of milliseconds since Jan 1, 1970. * A value of -1 means that this call should not set access time. */ @Override public void setTimes(Path p, long mtime, long atime) throws IOException { Map<String, String> params = new HashMap<String, String>(); params.put("op", "settimes"); params.put("mtime", Long.toString(mtime)); params.put("atime", Long.toString(atime)); HttpURLConnection conn = getConnection("PUT", params, p); validateResponse(conn, HttpURLConnection.HTTP_OK); } /** * Set replication for an existing file. * * @param src file name * @param replication new replication * @throws IOException * @return true if successful; * false if file does not exist or is a directory */ @Override public boolean setReplication(Path src, short replication) throws IOException { Map<String, String> params = new HashMap<String, String>(); params.put("op", "setreplication"); params.put("replication", Short.toString(replication)); HttpURLConnection conn = getConnection("PUT", params, src); validateResponse(conn, HttpURLConnection.HTTP_OK); JSONObject json = (JSONObject) jsonParse(conn); return (Boolean) json.get("setReplication"); } /** * Creates a <code>FileStatus</code> object using a JSON file-status payload * received from a Hoop server. * * @param json a JSON file-status payload received from a Hoop server * @return the corresponding <code>FileStatus</code> */ private FileStatus createFileStatus(JSONObject json) { Path path = new Path((String) json.get("path")); boolean isDir = (Boolean) json.get("isDir"); long len = (Long) json.get("len"); String owner = (String) json.get("owner"); String group = (String) json.get("group"); FsPermission permission = FsPermission.valueOf((String) json.get("permission")); long aTime = (Long) json.get("accessTime"); long mTime = (Long) json.get("modificationTime"); long blockSize = (Long) json.get("blockSize"); short replication = (short) (long) (Long) json.get("replication"); return new FileStatus(len, isDir, replication, blockSize, mTime, aTime, permission, owner, group, path); } }