Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package gobblin.util; import java.io.IOException; import java.io.InputStream; import java.net.InetAddress; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.net.UnknownHostException; import java.util.Properties; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.base.Preconditions; import com.google.common.io.Closer; /** * Allows conversion of URLs identifying a Hadoop cluster (e.g. resource manager url or * a job tracker URL) to a human-readable name. * * <p>The class will automatically load a resource named {@link #URL_TO_NAME_MAP_RESOURCE_NAME} to * get a default mapping. It expects this resource to be in the Java Properties file format. The * name of the property is the cluster URL and the value is the human-readable name. * * <p><b>IMPORTANT:</b> Don't forget to escape colons ":" in the file as those may be interpreted * as name/value separators. */ public class ClustersNames { public static final String URL_TO_NAME_MAP_RESOURCE_NAME = "GobblinClustersNames.properties"; private static final Logger LOG = LoggerFactory.getLogger(ClustersNames.class); private static final Configuration HADOOP_CONFIGURATION = new Configuration(); private static ClustersNames THE_INSTANCE; private Properties urlToNameMap = new Properties(); protected ClustersNames() { try (Closer closer = Closer.create()) { InputStream propsInput = closer.register(getClass().getResourceAsStream(URL_TO_NAME_MAP_RESOURCE_NAME)); if (null == propsInput) { propsInput = closer.register(ClassLoader.getSystemResourceAsStream(URL_TO_NAME_MAP_RESOURCE_NAME)); } if (null != propsInput) { try { this.urlToNameMap.load(propsInput); LOG.info("Loaded cluster names map:" + this.urlToNameMap); } catch (IOException e) { LOG.warn("Unable to load cluster names map: " + e, e); } } else { LOG.info("no default cluster mapping found"); } } catch (IOException e) { LOG.warn("unable to close resource input stream for " + URL_TO_NAME_MAP_RESOURCE_NAME + ":" + e, e); } } public String getClusterName(String clusterUrl) { if (null == clusterUrl) return null; String res = this.urlToNameMap.getProperty(clusterUrl); return null != res ? res : normalizeClusterUrl(clusterUrl); } public void addClusterMapping(String clusterUrl, String clusterName) { Preconditions.checkNotNull(clusterUrl, "cluster URL expected"); Preconditions.checkNotNull(clusterName, "cluster name expected"); this.urlToNameMap.put(clusterUrl, clusterName); } public void addClusterMapping(URL clusterUrl, String clusterName) { Preconditions.checkNotNull(clusterUrl, "cluster URL expected"); Preconditions.checkNotNull(clusterName, "cluster name expected"); this.urlToNameMap.put(clusterUrl.toString(), clusterName); } // Strip out the port number if it is a valid URI private static String normalizeClusterUrl(String clusterIdentifier) { try { URI uri = new URI(clusterIdentifier.trim()); // URIs without protocol prefix if (!uri.isOpaque() && null != uri.getHost()) { clusterIdentifier = uri.getHost(); } else { clusterIdentifier = uri.toString().replaceAll("[/:]", " ").trim().replaceAll(" ", "_"); } } catch (URISyntaxException e) { //leave ID as is } return clusterIdentifier; } /** * * Returns the cluster name on which the application is running. Uses default hadoop {@link Configuration} to get the * url of the resourceManager or jobtracker. The URL is then translated into a human readable cluster name using * {@link #getClusterName(String)} * * @see #getClusterName(Configuration) * */ public String getClusterName() { return getClusterName(HADOOP_CONFIGURATION); } /** * Returns the cluster name on which the application is running. Uses Hadoop configuration passed in to get the * url of the resourceManager or jobtracker. The URL is then translated into a human readable cluster name using * {@link #getClusterName(String)} * * <p> * <b>MapReduce mode</b> Uses the value for "yarn.resourcemanager.address" from {@link Configuration} excluding the * port number. * </p> * * <p> * <b>Standalone mode (outside of hadoop)</b> Uses the Hostname of {@link InetAddress#getLocalHost()} * </p> * * <p> * Use {@link #getClusterName(String)} if you already have the cluster URL * </p> * * @see #getClusterName() * @param conf Hadoop configuration to use to get resourceManager or jobTracker URLs */ public String getClusterName(Configuration conf) { // ResourceManager address in Hadoop2 String clusterIdentifier = conf.get("yarn.resourcemanager.address"); clusterIdentifier = getClusterName(clusterIdentifier); // If job is running outside of Hadoop (Standalone) use hostname // If clusterIdentifier is localhost or 0.0.0.0 use hostname if (clusterIdentifier == null || StringUtils.startsWithIgnoreCase(clusterIdentifier, "localhost") || StringUtils.startsWithIgnoreCase(clusterIdentifier, "0.0.0.0")) { try { clusterIdentifier = InetAddress.getLocalHost().getHostName(); } catch (UnknownHostException e) { // Do nothing. Tag will not be generated } } return clusterIdentifier; } public static ClustersNames getInstance() { synchronized (ClustersNames.class) { if (null == THE_INSTANCE) { THE_INSTANCE = new ClustersNames(); } return THE_INSTANCE; } } }