org.openflamingo.uploader.util.FileSystemUtils.java Source code

Introduction

Here is the source code for org.openflamingo.uploader.util.FileSystemUtils.java

Source

/**
 * Flamingo HDFS File Uploader - a tool to upload from datasource to datasource and schedule jobs
 *
 * Copyright (C) 2011-2012 Cloudine.
 *
 * This file is part of Flamingo HDFS File Uploader.
 *
 * Flamingo HDFS File Uploader is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Flamingo HDFS File Uploader is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
package org.openflamingo.uploader.util;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.openflamingo.uploader.exception.FileSystemException;
import org.openflamingo.uploader.exception.SystemException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.net.InetAddress;
import java.net.URI;
import java.net.UnknownHostException;

/**
 * FileSystem Utility.
 *
 * @author Edward KIM
 * @since 0.1
 */
public class FileSystemUtils {

    /**
     * SLF4J Logging
     */
    private static Logger logger = LoggerFactory.getLogger(FileSystemUtils.class);

    /**
     *   . <tt>hdfs://</tt> ? <tt>file:</tt>    
     *   ?   <tt>file:</tt>? ?.
     *
     * @param path ? ? ? 
     * @return  ? 
     */
    public static String correctPath(String path) {
        if (path == null || path.length() < 1) {
            return null;
        }
        Path filePath = new Path(path);
        String scheme = filePath.toUri().getScheme();
        if (scheme == null) {
            return "file:" + path;
        }
        return path;
    }

    /**
     *   ? ? ? ?.
     *
     * @param path   
     * @param scheme ? ? scheme(; hdfs:// file:)
     */
    public static void checkScheme(String path, FileSystemScheme scheme) {
        if (path == null || path.length() < 1) {
            return;
        }
        if (!path.startsWith(scheme.getScheme())) {
            throw new SystemException(ExceptionUtils.getMessage(
                    "  '{}'?  ? ({})? .", path,
                    scheme.getScheme()));
        }
    }

    /**
     *      ?.
     *     ???  ?.
     *
     * @param path 
     * @throws FileSystemException  ?  ,   ? ?? 
     */
    public static void testCreateDir(Path path) {
        try {
            Configuration conf = new Configuration();
            FileSystem fs = path.getFileSystem(conf);
            if (fs.exists(path) && !fs.getFileStatus(path).isDir()) {
                throw new FileSystemException(ExceptionUtils
                        .getMessage("    ?: '{}'", path));
            }

            if (!fs.exists(path)) {
                if (!fs.mkdirs(path)) {
                    throw new FileSystemException(
                            ExceptionUtils.getMessage(" ?  : '{}'", path));
                }
                logger.info(" ?: {}", path);
            }
        } catch (Exception ex) {
            throw new FileSystemException("?     .", ex);
        }
    }

    /**
     *  ? ? ? .
     *
     * @param path ? ? ? 
     * @return ? 
     */
    public static FileSystem getFileSystem(Path path) {
        try {
            Configuration conf = new Configuration();
            return path.getFileSystem(conf);
        } catch (Exception ex) {
            throw new FileSystemException(ExceptionUtils
                    .getMessage("  '{}'? ? ? ?  .", path), ex);
        }
    }

    /**
     *  ? ? ? .
     *
     * @param path ? ? ? 
     * @return ? 
     */
    public static FileSystem getFileSystem(String path) {
        return getFileSystem(new Path(path));
    }

    /**
     * ?  ? ? ? ?? ?.
     *
     * @param path1 1
     * @param path2 2
     */
    public static void validateSameFileSystem(String path1, String path2) {
        Path p1 = new Path(correctPath(path1));
        Path p2 = new Path(correctPath(path2));
        FileSystem fs1 = null;
        FileSystem fs2 = null;
        try {
            fs1 = p1.getFileSystem(new Configuration());
            fs2 = p2.getFileSystem(new Configuration());
        } catch (Exception ex) {
            throw new SystemException(ExceptionUtils
                    .getMessage("'{}' ? '{}' ? ?   .", p1, p2), ex);
        }

        if (!compareFs(fs1, fs2)) {
            throw new SystemException(ExceptionUtils.getMessage(
                    " ?  ?? ? ? . ??  ?  ? ?? ? ?   : {}, {}",
                    p1, p2));
        }

        if (p1.equals(p2)) {
            throw new SystemException(
                    ExceptionUtils.getMessage(" ?  ?? : {}, {}", p1, p2));
        }
    }

    /**
     * ? ? ?  URI? scheme? ? ?? ?.
     *
     * @param fs1 ?1
     * @param fs2 ?2
     * @return ?? <tt>true</tt>
     */
    private static boolean compareFs(FileSystem fs1, FileSystem fs2) {
        URI uri1 = fs1.getUri();
        URI uri2 = fs2.getUri();
        if (uri1.getScheme() == null) {
            return false;
        }
        if (!uri1.getScheme().equals(uri2.getScheme())) {
            return false;
        }
        String srcHost = uri1.getHost();
        String dstHost = uri2.getHost();
        if ((srcHost != null) && (dstHost != null)) {
            try {
                srcHost = InetAddress.getByName(srcHost).getCanonicalHostName();
                dstHost = InetAddress.getByName(dstHost).getCanonicalHostName();
            } catch (UnknownHostException ue) {
                return false;
            }
            if (!srcHost.equals(dstHost)) {
                return false;
            }
        } else if (srcHost == null && dstHost != null) {
            return false;
        } else if (srcHost != null) {
            return false;
        }
        // ? ?
        return uri1.getPort() == uri2.getPort();
    }

}