Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.alibaba.jstorm.hdfs; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; import java.nio.ByteBuffer; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.Map; import com.alibaba.jstorm.dfs.IDfs; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.alibaba.jstorm.client.ConfigExtension; import com.alibaba.jstorm.utils.JStormUtils; import backtype.storm.Config; import backtype.storm.utils.Utils; public class HdfsCache implements IDfs { public static final Logger LOG = LoggerFactory.getLogger(HdfsCache.class); private String baseDir = "/user/admin/cluster"; private Configuration hadoopConf; private FileSystem fs; public HdfsCache(Map conf) { this.hadoopConf = new Configuration(); String hdfsHostName = (String) conf.get(Config.BLOBSTORE_HDFS_HOSTNAME); Integer hdfsPort = JStormUtils.parseInt(conf.get(Config.BLOBSTORE_HDFS_PORT)); LOG.info("hdfs address hdfs://{}:{}", hdfsHostName, hdfsPort); hadoopConf.set("fs.defaultFS", String.format("hdfs://%s:%d", hdfsHostName, hdfsPort)); hadoopConf.setBoolean("fs.hdfs.impl.disable.cache", true); try { fs = FileSystem.get(hadoopConf); String configBaseDir = (String) conf.get("hdfs.base.dir"); if (configBaseDir != null) { this.baseDir = configBaseDir; } else { String clusterName = ConfigExtension.getClusterName(conf) != null ? ConfigExtension.getClusterName(conf) : "default"; baseDir = baseDir + "/" + clusterName; if (!exist(baseDir)) mkdir(baseDir); } } catch (IOException e) { LOG.error("Failed to instance hdfs cache", e); throw new RuntimeException(e.getMessage()); } } public String getBaseDir() { return baseDir; } public void create(String dstPath, byte[] data) throws IOException { Path path = new Path(dstPath); LOG.debug("Try to create file: {}", dstPath); FSDataOutputStream out = null; try { out = fs.create(path, true); out.write(data); } finally { if (out != null) out.close(); } } public byte[] read(String dstPath) throws IOException { Path path = new Path(dstPath); if (!fs.exists(path)) { throw new IOException(dstPath + " is not exist!"); } FSDataInputStream in = null; try { in = fs.open(path); LOG.debug("Try to read data from file-{}, dataLen={}", dstPath, in.available()); ByteBuffer buf = ByteBuffer.allocate(in.available()); in.read(buf); return buf.array(); } finally { if (in != null) in.close(); } } public Collection<String> readLines(String dstPath) throws IOException { Collection<String> ret = new HashSet<String>(); Path path = new Path(dstPath); if (!fs.exists(path)) { throw new IOException(dstPath + " is not exist!"); } FSDataInputStream in = null; try { in = fs.open(path); LOG.debug("Try to read string from file-{}", dstPath); BufferedReader reader = new BufferedReader(new InputStreamReader(in)); String line = reader.readLine(); while (line != null) { ret.add(line); line = reader.readLine(); } return ret; } finally { if (in != null) in.close(); } } public boolean exist(String path) throws IOException { return fs.exists(new Path(path)); } public void mkdir(String dstDir) throws IOException { Path path = new Path(dstDir); if (!fs.exists(path)) { LOG.debug("Try to makdir: {}", dstDir); fs.mkdirs(path); // mkdirs -p } } public boolean remove(String dstPath, boolean recursive) throws IOException { Path path = new Path(dstPath); if (fs.exists(path)) { LOG.debug("Try to remove file-{}", dstPath); return fs.delete(path, recursive); } else { return false; } } public void copyToDfs(String srcPath, String dstPath, boolean overwrite) throws IOException { Path path = new Path(dstPath); if (!fs.exists(path)) fs.mkdirs(path); fs.copyFromLocalFile(false, overwrite, new Path(srcPath), new Path(dstPath)); } public void copyToLocal(String srcPath, String dstPath) throws IOException { fs.copyToLocalFile(new Path(srcPath), new Path(dstPath)); } public Collection<String> listFile(String dstPath, boolean recursive) throws IOException { Collection<String> files = new HashSet<String>(); Path path = new Path(dstPath); if (fs.exists(path)) { RemoteIterator<LocatedFileStatus> itr = fs.listFiles(path, recursive); while (itr.hasNext()) { LocatedFileStatus status = itr.next(); files.add(status.getPath().getName()); } } return files; } public FileStatus[] listFileStatus(String dstPath) throws IOException { FileStatus[] ret = null; Path path = new Path(dstPath); if (fs.exists(path)) { ret = fs.listStatus(path); } return ret; } public FileStatus getFileStatus(String dstPath) throws IOException { FileStatus ret = null; Path path = new Path(dstPath); if (fs.exists(path)) { ret = fs.getFileStatus(path); } return ret; } public void close() { if (fs != null) { try { fs.close(); } catch (IOException e) { LOG.warn("Failed to close hadoop file system!", e); } } } public static void main(String[] args) { int argsStartIndex = 0; Map conf = new HashMap<Object, Object>(); if (args[0].equals("-conf")) { conf.putAll(Utils.loadConf(args[1])); argsStartIndex += 2; } else { System.out.println("-conf is missing!"); return; } conf.put("hdfs.base.dir", "/tmp"); conf.put("cluster.name", "test"); HdfsCache cache = new HdfsCache(conf); String topoPath = cache.getBaseDir() + "/topology1"; String cmd = args[argsStartIndex]; try { if (cmd.equals("create")) { String path = topoPath + "/" + args[++argsStartIndex]; cache.create(path, "Test".getBytes()); } else if (cmd.equals("read")) { String path = topoPath + "/" + args[++argsStartIndex]; byte[] data = cache.read(path); System.out.println("Read string from hdfs, string=" + new String(data)); } else if (cmd.equals("mkdir")) { String path = topoPath + "/" + args[++argsStartIndex]; cache.mkdir(path); } else if (cmd.equals("remove")) { String path = topoPath + "/" + args[++argsStartIndex]; cache.remove(path, true); } else if (cmd.equals("copy")) { boolean toHdfs = Boolean.valueOf(args[++argsStartIndex]); String srcPath = args[++argsStartIndex]; String dstPath = args[++argsStartIndex]; if (toHdfs) { cache.copyToDfs(srcPath, dstPath, false); } else { cache.copyToLocal(srcPath, dstPath); } } else { System.out.println("Unknown command: " + args[argsStartIndex + 0]); } } catch (IOException e) { System.out.println("Failed to execute cmd: " + cmd); e.printStackTrace(); } } }