Java tutorial
/* * Copyright 2011 Kazuyoshi Aizawa * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package iumfs.hdfs; import iumfs.FileExistException; import iumfs.IumfsFile; import iumfs.NotSupportedException; import iumfs.Request; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.Date; import java.util.List; import java.util.logging.Logger; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.*; import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException; /** * * Class which represents files on HDFS */ public class HdfsFile extends IumfsFile { private FileSystem fs; protected static final Logger logger = Logger.getLogger(Main.class.getName()); private String server; HdfsFile(String server, String pathname) { super(pathname); this.server = server; fs = getFileSystem(); Date now = new Date(); setAtime(now.getTime()); setCtime(now.getTime()); setMtime(now.getTime()); } @Override public long read(ByteBuffer buf, long size, long offset) throws FileNotFoundException, IOException, NotSupportedException { int ret; FSDataInputStream fsdis = fs.open(new Path(getPath())); ret = fsdis.read(offset, buf.array(), Request.RESPONSE_HEADER_SIZE, (int) size); fsdis.close(); logger.fine("read offset=" + offset + ",size=" + size); return ret; } @Override public long write(byte[] buf, long size, long offset) throws FileNotFoundException, IOException, NotSupportedException { // ? FileStatus fstat = fs.getFileStatus(new Path(getPath())); long filesize = fstat.getLen(); int written = 0; /* * ?? iumfscntl ???? write ? ?? PAGE * ???????? PAGE ?? ?) * * PAGESIZE PAGESIZE |---------------------|---------------------| * |<---------- filesize -------->| |<---- offset ------->|<-- size * --->| * * HDFS ? append ? filesize ????????? iumfs * ??????????? HDFS ??? */ if (offset + size < filesize) { // ????????? throw new NotSupportedException(); } FSDataOutputStream fsdos = fs.append(new Path(getPath())); /* * ??/??????? ???????? Append ?? */ try { if (offset > filesize) { // ??? // ??? null ?? fsdos.write(new byte[(int) (offset - filesize)]); fsdos.write(getDataByRange(buf, 0, size)); } else { // ??? fsdos.write(getDataByRange(buf, filesize - offset, size)); } } finally { written = fsdos.size(); fsdos.close(); } return written; } /* * <p>FileSystem.mkdir won't return with err even directory already exits. * So here we need to check the existance by ourself and return false. */ @Override public boolean mkdir() { try { /* * Create new directory on HDFS */ if (fs.exists(new Path(getPath())) == true) { logger.fine("cannot create directory"); return false; } if (fs.mkdirs(new Path(getPath())) == false) { logger.fine("cannot create directory"); return false; } return true; } catch (IOException ex) { /* * can't throw IOException here. So return false, and iumfs.mkdir * would back this 'false' to IOException... */ return false; } } @Override public boolean delete() { try { Path path = new Path(getPath()); if (fs.delete(path, true) == false) { logger.fine("cannot remove " + path.getName()); return false; } } catch (IOException ex) { return false; } return true; } /** * List files under directory which expres this object. This corresponds * readdir. * * @return */ @Override public File[] listFiles() { List<File> filelist = new ArrayList<File>(); FileStatus fstats[]; try { fstats = fs.listStatus(new Path(getPath())); } catch (IOException ex) { return null; } for (FileStatus fstat : fstats) { filelist.add(new File(fstat.getPath().getName())); } return filelist.toArray(new File[0]); } @Override public long getFileType() { try { if (fs.getFileStatus(new Path(getPath())).isDir()) { return IumfsFile.VDIR; } else { return IumfsFile.VREG; } } catch (IOException ex) { return IumfsFile.VREG; } } /* * TODO: Implement SetAttribute to change file permission */ @Override public long getPermission() { if (isDirectory()) { return (long) 0040777; // directory } else { return (long) 0100777; // regular file } } @Override public boolean isDirectory() { try { return fs.getFileStatus(new Path(getPath())).isDir(); } catch (IOException ex) { return false; } } /** * <p>FileSystem.create ?</p> <p>creat(2) ?????????????? * (O_TRUNC)????????? HDFS ?? ? * ??????????????</p> */ @Override public void create() throws IOException { /* * HDFS ???????? */ try { Path path = new Path(getPath()); //??? EEXIST ? if (fs.exists(path) == true) { logger.fine("cannot create file"); throw new FileExistException(); } FSDataOutputStream fsdos = fs.create(path); /* * ? */ fsdos.close(); } catch (AlreadyBeingCreatedException ex) { logger.fine("AlreadyBeingCreatedException when writing"); throw new FileExistException(); } catch (IOException ex) { logger.fine("IOException happend when writing"); throw ex; } } private FileSystem getFileSystem() { if (fs == null) { try { Configuration conf = new Configuration(); conf.set("fs.defaultFS", server); logger.finer("server=" + server); fs = FileSystem.get(conf); } catch (IOException ex) { ex.printStackTrace(); System.exit(1); } } return fs; } public static IumfsFile getFile(String server, String pathname) { return new HdfsFile(server, pathname); } /** * @return the server */ public String getServer() { return server; } /** * @param server the server to set */ public void setServer(String server) { this.server = server; } @Override public boolean exists() { try { return getFileSystem().exists(new Path(getPath())); } catch (IOException ex) { ex.printStackTrace(); return false; } } @Override public long length() { // ? FileStatus fstat; try { fstat = fs.getFileStatus(new Path(getPath())); } catch (IOException ex) { return 0; } return fstat.getLen(); } public byte[] getDataByRange(byte[] data, long from, long to) { logger.finer("from=" + from + "to=" + to); return Arrays.copyOfRange(data, (int) from, (int) to); } }