Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.twill.internal.yarn; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.HAUtil; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; import org.apache.twill.api.RunId; import org.apache.twill.filesystem.FileContextLocationFactory; import org.apache.twill.filesystem.ForwardingLocationFactory; import org.apache.twill.filesystem.HDFSLocationFactory; import org.apache.twill.filesystem.Location; import org.apache.twill.filesystem.LocationFactory; import org.apache.twill.internal.AbstractTwillService; import org.apache.twill.internal.Constants; import org.apache.twill.internal.state.Message; import org.apache.twill.internal.state.SystemMessages; import org.apache.twill.zookeeper.ZKClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.BufferedInputStream; import java.io.DataInputStream; import java.io.IOException; import java.net.InetSocketAddress; import java.net.URI; import java.net.URISyntaxException; import java.util.Map; /** * TODO: copied from Twill 0.7 AbstractYarnTwillService for CDAP-5844. * Remove after this fix is moved to Twill (TWILL-171). * * Abstract class for implementing {@link com.google.common.util.concurrent.Service} that runs in * YARN container which provides methods to handle secure token updates. */ public abstract class AbstractYarnTwillService extends AbstractTwillService { private static final Logger LOG = LoggerFactory.getLogger(AbstractYarnTwillService.class); protected final Location applicationLocation; protected volatile Credentials credentials; protected AbstractYarnTwillService(ZKClient zkClient, RunId runId, Location applicationLocation) { super(zkClient, runId); this.applicationLocation = applicationLocation; } /** * Returns the location of the secure store, or {@code null} if either not running in secure mode or an error * occur when trying to acquire the location. */ protected final Location getSecureStoreLocation() { if (!UserGroupInformation.isSecurityEnabled()) { return null; } try { return applicationLocation.append(Constants.Files.CREDENTIALS); } catch (IOException e) { LOG.error("Failed to create secure store location.", e); return null; } } /** * Attempts to handle secure store update. * * @param message The message received * @return {@code true} if the message requests for secure store update, {@code false} otherwise. */ protected final boolean handleSecureStoreUpdate(Message message) { if (!SystemMessages.SECURE_STORE_UPDATED.equals(message)) { return false; } // If not in secure mode, simply ignore the message. if (!UserGroupInformation.isSecurityEnabled()) { return true; } try { Credentials credentials = new Credentials(); Location location = getSecureStoreLocation(); try (DataInputStream input = new DataInputStream(new BufferedInputStream(location.getInputStream()))) { credentials.readTokenStorageStream(input); } UserGroupInformation.getCurrentUser().addCredentials(credentials); // CDAP-5844 Workaround for HDFS-9276, to update HDFS delegation token for long running application in HA mode cloneHaNnCredentials(location, UserGroupInformation.getCurrentUser()); this.credentials = credentials; LOG.info("Secure store updated from {}.", location); } catch (Throwable t) { LOG.error("Failed to update secure store.", t); } return true; } private static void cloneHaNnCredentials(Location location, UserGroupInformation ugi) throws IOException { Configuration hConf = getConfiguration(location.getLocationFactory()); String scheme = location.toURI().getScheme(); Map<String, Map<String, InetSocketAddress>> nsIdMap = DFSUtil.getHaNnRpcAddresses(hConf); for (Map.Entry<String, Map<String, InetSocketAddress>> entry : nsIdMap.entrySet()) { String nsId = entry.getKey(); Map<String, InetSocketAddress> addressesInNN = entry.getValue(); if (!HAUtil.isHAEnabled(hConf, nsId) || addressesInNN == null || addressesInNN.isEmpty()) { continue; } // The client may have a delegation token set for the logical // URI of the cluster. Clone this token to apply to each of the // underlying IPC addresses so that the IPC code can find it. URI uri = URI.create(scheme + "://" + nsId); LOG.info("Cloning delegation token for uri {}", uri); HAUtil.cloneDelegationTokenForLogicalUri(ugi, uri, addressesInNN.values()); } } /** * Gets the Hadoop Configuration from LocationFactory. */ private static Configuration getConfiguration(LocationFactory locationFactory) throws IOException { LOG.debug("getFileSystem(): locationFactory is a {}", locationFactory.getClass()); if (locationFactory instanceof HDFSLocationFactory) { return ((HDFSLocationFactory) locationFactory).getFileSystem().getConf(); } if (locationFactory instanceof ForwardingLocationFactory) { return getConfiguration(((ForwardingLocationFactory) locationFactory).getDelegate()); } if (locationFactory instanceof FileContextLocationFactory) { return ((FileContextLocationFactory) locationFactory).getConfiguration(); } throw new IllegalArgumentException( String.format("Unknown LocationFactory type: %s", locationFactory.getClass().getName())); } }