Java tutorial
/* * Copyright (C) 2017 Baifendian Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.baifendian.swordfish.common.hadoop; import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.ByteArrayInputStream; import java.io.Closeable; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.FsStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class HdfsClient implements Closeable { /** * LOGGER */ private static final Logger LOGGER = LoggerFactory.getLogger(HdfsClient.class); /** * HdfsClient */ private static volatile HdfsClient instance; /** * {@link FileSystem} */ private FileSystem fileSystem; /** * @param conf * @throws HdfsException */ private HdfsClient(Configuration conf) throws HdfsException { try { fileSystem = FileSystem.get(conf); } catch (IOException e) { throw new HdfsException("Create HdfsClient failed.", e); } } /** * ? * * @param conf */ public static void init(Configuration conf) { if (instance == null) { synchronized (HdfsClient.class) { if (instance == null) { instance = new HdfsClient(conf); } } } } /** * ? HdfsClient (?) * * @return {@link HdfsClient} */ public static HdfsClient getInstance() throws HdfsException { if (instance == null) { LOGGER.error("Get HdfsClient instance failedplease call init(Configuration conf) first"); throw new HdfsException("Get HdfsClient instance failedplease call init(Configuration conf) first"); } return instance; } /** * HDFS * * @param fileName ?? * @param content * @param destPath * @param isOverwrite ??? */ public void addFile(String fileName, byte[] content, String destPath, boolean isOverwrite) throws HdfsException { LOGGER.debug("Begin addFile. fileName: {}, path: {}", fileName, destPath); // String destFile; if (destPath.charAt(destPath.length() - 1) != File.separatorChar) { destFile = destPath + File.separatorChar + fileName; } else { destFile = destPath + fileName; } // ? Path path = new Path(destFile); try { // ? if (fileSystem.exists(path)) { if (isOverwrite) { // fileSystem.delete(path, false);// } else { // ? LOGGER.error("File " + destFile + " already exists"); return; } } } catch (IOException e) { LOGGER.error("Operator Hdfs exception", e); throw new HdfsException("Operator Hdfs exception", e); } try (FSDataOutputStream out = fileSystem.create(path); InputStream in = new BufferedInputStream(new ByteArrayInputStream(content));) { byte[] b = new byte[1024]; int numBytes = 0; while ((numBytes = in.read(b)) > 0) { out.write(b, 0, numBytes); } out.hflush(); } catch (IOException e) { LOGGER.error("Operator Hdfs exception", e); throw new HdfsException("Operator Hdfs exception", e); } LOGGER.debug("End addFile. fileName:" + fileName + ", path:" + destPath); } /** * hdfs ? * * @param hdfsFile hdfs * @param localFile * @param overwrite ? */ public void readFile(String hdfsFile, String localFile, boolean overwrite) throws HdfsException { // Path pathObject = new Path(hdfsFile); File fileObject = new File(localFile); try { if (!isFile(pathObject)) { throw new HdfsException("File " + hdfsFile + " is not a valid file"); } // ???? if (!overwrite && fileObject.exists()) { LOGGER.info("{} has exist, do not overwrite", localFile); return; } // ?? File parentPath = fileObject.getParentFile(); if (parentPath != null && !parentPath.exists()) { FileUtils.forceMkdir(parentPath); } } catch (IOException e) { LOGGER.error("Operator Hdfs exception", e); throw new HdfsException("Operator Hdfs exception", e); } try (FSDataInputStream in = fileSystem.open(pathObject); OutputStream out = new BufferedOutputStream(new FileOutputStream(fileObject));) { byte[] b = new byte[1024]; int numBytes = 0; while ((numBytes = in.read(b)) > 0) { out.write(b, 0, numBytes); } out.flush(); } catch (IOException e) { LOGGER.error("Operator Hdfs exception", e); throw new HdfsException("Operator Hdfs exception", e); } } /** * hdfs ? * * @param hdfsFile hdfs * @return byte[] */ public byte[] readFile(String hdfsFile) throws HdfsException { // Path pathObject = new Path(hdfsFile); try { if (!isFile(pathObject)) { throw new HdfsException("File " + hdfsFile + " is not a valid file"); } } catch (IOException e) { LOGGER.error("Operator Hdfs exception", e); throw new HdfsException("Operator Hdfs exception", e); } try (FSDataInputStream in = fileSystem.open(pathObject);) { return IOUtils.toByteArray(in); } catch (IOException e) { LOGGER.error("Operator Hdfs exception", e); throw new HdfsException("Operator Hdfs exception", e); } } /** * ?? * * @param pathObject * @return * @throws IOException */ private boolean isFile(Path pathObject) throws IOException { if (!fileSystem.exists(pathObject) || fileSystem.isDirectory(pathObject)) { return false; } return true; } /** * * * @param path hdfs * @param recursive ?? * @return ?? */ public boolean delete(String path, boolean recursive) throws HdfsException { Path pathObject = new Path(path); try { return fileSystem.delete(pathObject, recursive); } catch (IOException e) { LOGGER.error("Delete path exception", e); throw new HdfsException("Delete path exception", e); } } /** * ??? * * @param path hdfs * @param destPath hdfs * @return ????? */ public boolean rename(String path, String destPath) throws HdfsException { Path pathObject = new Path(path); Path destPathObject = new Path(destPath); try { return fileSystem.rename(pathObject, destPathObject); } catch (IOException e) { LOGGER.error("Rename path exception", e); throw new HdfsException("Rename path exception", e); } } /** * * * @param dir * @throws HdfsException */ public void mkdir(String dir) throws HdfsException { mkdir(dir, FsPermission.getDefault()); } /** * ??? * * @param dir * @throws HdfsException */ public void mkdir(String dir, FsPermission perm) throws HdfsException { Path path = new Path(dir); try { if (fileSystem.exists(path)) { LOGGER.error("Dir {} already exists", dir); return; } fileSystem.mkdirs(path, perm); } catch (IOException e) { LOGGER.error("Create dir exception", e); throw new HdfsException("Create dir exception", e); } } /** * * * @param path * @param user * @param group * @throws HdfsException */ public void setOwner(Path path, String user, String group) throws HdfsException { try { fileSystem.setOwner(path, user, group); } catch (IOException e) { LOGGER.error("Create dir exception", e); throw new HdfsException("Create dir exception", e); } } /** * ??? */ public void setPermission(Path path, FsPermission perm) throws IOException { fileSystem.setPermission(path, perm); if (path.getParent() != null) { setPermission(path.getParent(), perm); } } /** * ???? * @param path * @param perm * @throws IOException */ public void setPermissionThis(Path path, FsPermission perm) throws IOException { fileSystem.setPermission(path, perm); } /** * copy ? * * @param srcPath hdfs ? * @param dstPath hdfs * @param deleteSource ?? * @param overwrite ? * @return ?? */ public boolean copy(String srcPath, String dstPath, boolean deleteSource, boolean overwrite) throws HdfsException { Path srcPathObj = new Path(srcPath); Path dstPathObj = new Path(dstPath); try { return FileUtil.copy(fileSystem, srcPathObj, fileSystem, dstPathObj, deleteSource, overwrite, fileSystem.getConf()); } catch (IOException e) { LOGGER.error("Copy exception", e); throw new HdfsException("Copy exception", e); } } /** * ? hdfs * * @param srcPath * @param dstPath * @param deleteSource * @param overwrite * @return * @throws HdfsException */ public boolean copyLocalToHdfs(String srcPath, String dstPath, boolean deleteSource, boolean overwrite) throws HdfsException { Path srcPathObj = new Path(srcPath); Path dstPathObj = new Path(dstPath); try { fileSystem.copyFromLocalFile(deleteSource, overwrite, srcPathObj, dstPathObj); } catch (IOException e) { LOGGER.error("Copy exception", e); throw new HdfsException("Copy exception", e); } return true; } /** * ? hdfs * * @param srcPath * @param dstPath * @param deleteSource * @param overwrite * @return * @throws HdfsException */ public boolean copyHdfsToLocal(String srcPath, String dstPath, boolean deleteSource, boolean overwrite) throws HdfsException { Path srcPathObj = new Path(srcPath); File dstFile = new File(dstPath); try { if (dstFile.exists()) { if (dstFile.isFile()) { if (overwrite) { dstFile.delete(); } } else { throw new HdfsException("Destination must not be a dir"); } } // ? dstFile.getParentFile().mkdirs(); // hdfs ? FileUtil.copy(fileSystem, srcPathObj, dstFile, deleteSource, fileSystem.getConf()); } catch (IOException e) { LOGGER.error("Copy exception", e); throw new HdfsException("Copy exception", e); } return true; } /** * ???? Byte) * * @param filePath * @return ? */ public long getFileLength(String filePath) { return getContentSummary(filePath).getLength(); } /** * ? * * @param filePath * @return * @throws IOException */ public boolean exists(String filePath) throws IOException { return fileSystem.exists(new Path(filePath)); } /** * ??? * * @param filePath * @return {@link FileStatus} */ public FileStatus getFileStatus(String filePath) { Path path = new Path(filePath); try { return fileSystem.getFileStatus(path); } catch (IOException e) { LOGGER.error("Get file status exception", e); throw new HdfsException("Get file status exception", e); } } /** * ? * * @param filePath * @return {@link FileStatus} */ public FileStatus[] listFileStatus(String filePath) { Path path = new Path(filePath); try { return fileSystem.listStatus(new Path(filePath)); } catch (IOException e) { LOGGER.error("Get file list exception", e); throw new HdfsException("Get file list exception", e); } } /** * ?????? * * @param filePath * @return {@link ContentSummary} */ public ContentSummary getContentSummary(String filePath) { Path path = new Path(filePath); try { return fileSystem.getContentSummary(path); } catch (IOException e) { LOGGER.error("Get file summary information exception", e); throw new HdfsException("Get file summary information exception", e); } } /** * ? hdfs url? * * @return url ? */ public String getUrl() { return fileSystem.getUri().toString(); } /** * hdfs client * * @throws HdfsException */ @Override public void close() throws HdfsException { if (fileSystem != null) { try { fileSystem.close(); } catch (IOException e) { LOGGER.error("Close HdfsClient instance failed", e); throw new HdfsException("Close HdfsClient instance failed", e); } } } /** * hdfs ???? * * @return * @throws IOException */ public FsStatus getCapacity() throws IOException { FsStatus ds = fileSystem.getStatus(); return ds; } }