gobblin.compliance.utils.ProxyUtils.java Source code

Introduction

Here is the source code for gobblin.compliance.utils.ProxyUtils.java

Source

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package gobblin.compliance.utils;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.thrift.DelegationTokenIdentifier;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.thrift.TException;

import com.google.common.base.Optional;
import com.google.common.base.Preconditions;

import lombok.extern.slf4j.Slf4j;

import gobblin.compliance.ComplianceConfigurationKeys;
import gobblin.compliance.HiveProxyQueryExecutor;
import gobblin.configuration.ConfigurationKeys;
import gobblin.configuration.State;
import gobblin.util.HostUtils;
import gobblin.util.WriterUtils;

/**
 * A utility class for letting hadoop super user to proxy.
 *
 * @author adsharma
 */
@Slf4j
public class ProxyUtils {

    public static void setProxySettingsForFs(State state) {
        if (state.getPropAsBoolean(ComplianceConfigurationKeys.GOBBLIN_COMPLIANCE_SHOULD_PROXY,
                ComplianceConfigurationKeys.GOBBLIN_COMPLIANCE_DEFAULT_SHOULD_PROXY)) {
            String proxyUser = state.getProp(ComplianceConfigurationKeys.GOBBLIN_COMPLIANCE_PROXY_USER);
            String superUser = state.getProp(ComplianceConfigurationKeys.GOBBLIN_COMPLIANCE_SUPER_USER);
            String realm = state.getProp(ConfigurationKeys.KERBEROS_REALM);
            state.setProp(ConfigurationKeys.SHOULD_FS_PROXY_AS_USER, true);
            state.setProp(ConfigurationKeys.FS_PROXY_AS_USER_NAME, proxyUser);
            state.setProp(ConfigurationKeys.SUPER_USER_NAME_TO_PROXY_AS_OTHERS,
                    HostUtils.getPrincipalUsingHostname(superUser, realm));
            state.setProp(ConfigurationKeys.FS_PROXY_AUTH_METHOD, ConfigurationKeys.KERBEROS_AUTH);
        }
    }

    public static void cancelTokens(State state) throws IOException, InterruptedException, TException {
        Preconditions.checkArgument(state.contains(ConfigurationKeys.SUPER_USER_KEY_TAB_LOCATION),
                "Missing required property " + ConfigurationKeys.SUPER_USER_KEY_TAB_LOCATION);
        Preconditions.checkArgument(state.contains(ComplianceConfigurationKeys.GOBBLIN_COMPLIANCE_SUPER_USER),
                "Missing required property " + ComplianceConfigurationKeys.GOBBLIN_COMPLIANCE_SUPER_USER);
        Preconditions.checkArgument(state.contains(ConfigurationKeys.KERBEROS_REALM),
                "Missing required property " + ConfigurationKeys.KERBEROS_REALM);

        String superUser = state.getProp(ComplianceConfigurationKeys.GOBBLIN_COMPLIANCE_SUPER_USER);
        String keytabLocation = state.getProp(ConfigurationKeys.SUPER_USER_KEY_TAB_LOCATION);
        String realm = state.getProp(ConfigurationKeys.KERBEROS_REALM);

        UserGroupInformation.loginUserFromKeytab(HostUtils.getPrincipalUsingHostname(superUser, realm),
                keytabLocation);
        UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
        UserGroupInformation realUser = currentUser.getRealUser();
        Credentials credentials = realUser.getCredentials();
        for (Token<?> token : credentials.getAllTokens()) {
            if (token.getKind().equals(DelegationTokenIdentifier.HIVE_DELEGATION_KIND)) {
                log.info("Cancelling hive token");
                HiveMetaStoreClient hiveClient = new HiveMetaStoreClient(new HiveConf());
                hiveClient.cancelDelegationToken(token.encodeToUrlString());
            }
        }
    }

    public static FileSystem getOwnerFs(State state, Optional<String> owner) throws IOException {
        if (owner.isPresent()) {
            state.setProp(ComplianceConfigurationKeys.GOBBLIN_COMPLIANCE_PROXY_USER, owner.get());
        }
        ProxyUtils.setProxySettingsForFs(state);
        return WriterUtils.getWriterFs(state);
    }

    @SafeVarargs
    public static HiveProxyQueryExecutor getQueryExecutor(State state, Optional<String>... owners)
            throws IOException {
        List<String> proxies = new ArrayList<>();
        for (Optional<String> owner : owners) {
            if (owner.isPresent()) {
                proxies.add(owner.get());
            }
        }
        return new HiveProxyQueryExecutor(state, proxies);
    }
}