ml.shifu.dtrain.util.HDFSUtils.java Source code

Introduction

Here is the source code for ml.shifu.dtrain.util.HDFSUtils.java

Source

/**
 * Copyright [2012-2014] eBay Software Foundation
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package ml.shifu.dtrain.util;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * {@link HDFSUtils} is a unified class to get HDFS FileSystem Object.
 */
public final class HDFSUtils {

    private final static Logger LOG = LoggerFactory.getLogger(HDFSUtils.class);

    /**
     * Conf object which is used to construct HDFS FileSystem.
     */
    private final static Configuration conf = new Configuration();

    /**
     * HDFS FileSystem
     */
    private static volatile FileSystem hdfs;

    /**
     * Local FileSystem
     */
    private static volatile FileSystem lfs;

    private HDFSUtils() {
        // prevent new HDFSUtils();
    }

    /**
     * Get HDFS FileSystem
     * 
     * @throws RuntimeException
     *             if any IOException to retrieve local file system.
     */
    public static FileSystem getFS() {
        if (hdfs == null) {
            synchronized (HDFSUtils.class) {
                if (hdfs == null) {
                    try {
                        // initialization
                        hdfs = FileSystem.get(conf);
                        hdfs.setVerifyChecksum(false);
                    } catch (IOException e) {
                        LOG.error("Error on creating hdfs FileSystem object.", e);
                        throw new RuntimeException(e);
                    }
                }
            }
        }
        return hdfs;
    }

    /**
     * Get local FileSystem
     * 
     * @throws RuntimeException
     *             if any IOException to retrieve local file system.
     */
    public static FileSystem getLocalFS() {
        if (lfs == null) {
            synchronized (HDFSUtils.class) {
                if (lfs == null) {
                    try {
                        // initialization
                        lfs = FileSystem.getLocal(conf).getRaw();
                    } catch (IOException e) {
                        LOG.error("Error on creating local FileSystem object.", e);
                        throw new RuntimeException(e);
                    }
                }
            }
        }
        return lfs;
    }
}