org.apache.hcatalog.har.HarOutputCommitterPostProcessor.java Source code

Introduction

Here is the source code for org.apache.hcatalog.har.HarOutputCommitterPostProcessor.java

Source

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.apache.hcatalog.har;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.tools.HadoopArchives;
import org.apache.hadoop.util.ToolRunner;
import org.apache.hcatalog.common.HCatConstants;
import org.apache.hcatalog.common.HCatException;

public class HarOutputCommitterPostProcessor {

    boolean isEnabled = false;

    public boolean isEnabled() {
        return isEnabled;
    }

    public void setEnabled(boolean enabled) {
        this.isEnabled = enabled;
    }

    public void exec(JobContext context, Partition partition, Path partPath) throws IOException {
        //    LOG.info("Archiving partition ["+partPath.toString()+"]");
        makeHar(context, partPath.toUri().toString(), harFile(partPath));
        partition.getParameters().put(hive_metastoreConstants.IS_ARCHIVED, "true");
    }

    public String harFile(Path ptnPath) throws IOException {
        String harFile = ptnPath.toString().replaceFirst("/+$", "") + ".har";
        //    LOG.info("har file : " + harFile);
        return harFile;
    }

    public String getParentFSPath(Path ptnPath) throws IOException {
        return ptnPath.toUri().getPath().replaceFirst("/+$", "");
    }

    public String getProcessedLocation(Path ptnPath) throws IOException {
        String harLocn = ("har://" + ptnPath.toUri().getPath()).replaceFirst("/+$", "") + ".har" + Path.SEPARATOR;
        //    LOG.info("har location : " + harLocn);
        return harLocn;
    }

    /**
     * Creates a har file from the contents of a given directory, using that as root.
     * @param dir Directory to archive
     * @param harFile The HAR file to create
     */
    public static void makeHar(JobContext context, String dir, String harFile) throws IOException {
        //    Configuration conf = context.getConfiguration();
        //    Credentials creds = context.getCredentials();

        //    HCatUtil.logAllTokens(LOG,context);

        int lastSep = harFile.lastIndexOf(Path.SEPARATOR_CHAR);
        Path archivePath = new Path(harFile.substring(0, lastSep));
        final String[] args = { "-archiveName", harFile.substring(lastSep + 1, harFile.length()), "-p", dir, "*",
                archivePath.toString() };
        //    for (String arg : args){
        //      LOG.info("Args to har : "+ arg);
        //    }
        try {
            Configuration newConf = new Configuration();
            FileSystem fs = archivePath.getFileSystem(newConf);

            String hadoopTokenFileLocationEnvSetting = System
                    .getenv(HCatConstants.SYSENV_HADOOP_TOKEN_FILE_LOCATION);
            if ((hadoopTokenFileLocationEnvSetting != null) && (!hadoopTokenFileLocationEnvSetting.isEmpty())) {
                newConf.set(HCatConstants.CONF_MAPREDUCE_JOB_CREDENTIALS_BINARY, hadoopTokenFileLocationEnvSetting);
                //      LOG.info("System.getenv(\"HADOOP_TOKEN_FILE_LOCATION\") =["+  System.getenv("HADOOP_TOKEN_FILE_LOCATION")+"]");
            }
            //      for (FileStatus ds : fs.globStatus(new Path(dir, "*"))){
            //        LOG.info("src : "+ds.getPath().toUri().toString());
            //      }

            final HadoopArchives har = new HadoopArchives(newConf);
            int rc = ToolRunner.run(har, args);
            if (rc != 0) {
                throw new Exception("Har returned error code " + rc);
            }

            //      for (FileStatus hs : fs.globStatus(new Path(harFile, "*"))){
            //        LOG.info("dest : "+hs.getPath().toUri().toString());
            //      }
            //      doHarCheck(fs,harFile);
            //      LOG.info("Nuking " + dir);
            fs.delete(new Path(dir), true);
        } catch (Exception e) {
            throw new HCatException("Error creating Har [" + harFile + "] from [" + dir + "]", e);
        }
    }

}