Java tutorial
/*************************************************************************** * Copyright (c) 2012-2015 VMware, Inc. All Rights Reserved. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. ***************************************************************************/ package com.vmware.bdd.specpolicy; import java.io.File; import java.io.FileNotFoundException; import java.io.FileReader; import java.io.IOException; import java.io.Reader; import java.net.URL; import java.util.Collections; import org.apache.commons.collections.MapUtils; import org.apache.commons.configuration.ConfigurationUtils; import org.apache.commons.lang.ArrayUtils; import org.apache.log4j.Logger; import com.google.gson.Gson; import com.vmware.aurora.global.Configuration; import com.vmware.bdd.apitypes.ClusterCreate; import com.vmware.bdd.apitypes.ClusterType; import com.vmware.bdd.exception.BddException; import com.vmware.bdd.utils.AuAssert; import com.vmware.bdd.utils.CommonUtil; import com.vmware.bdd.utils.Constants; import com.vmware.bdd.utils.Version; public class ClusterSpecFactory { private enum MAPREDUCE_VERSION { V1, V2 } private enum HDP_VERSION { V1, V2_0, V2_1, V2_2, V2_3, UNKNOWN } private static final Logger logger = Logger.getLogger(ClusterSpecFactory.class); private static final String DEFAULT_TEMPLATE_SPEC_SUFFIX = "-default-template-spec.json"; private static final String HDFS_TEMPLATE_SPEC = "hdfs-template-spec.json"; private static final String HDFS_MAPRED_TEMPLATE_SPEC = "hdfs-mapred-template-spec.json"; private static final String HDFS_YARN_TEMPLATE_SPEC = "hdfs-yarn-template-spec.json"; private static final String HDFS_HBASE_TEMPLATE_SPEC = "hdfs-hbase-template-spec.json"; private static final String HDFS_MAPRED_MAPR_TEMPLATE_SPEC = "hdfs-mapred-mapr-template-spec.json"; private static final String HDFS_MAPRED_MAPR_V3_TEMPLATE_SPEC = "hdfs-mapred-mapr-v3-template-spec.json"; private static final String HDFS_HBASE_MAPR_TEMPLATE_SPEC = "hdfs-hbase-mapr-template-spec.json"; private static final String HDFS_HBASE_MAPR_V3_TEMPLATE_SPEC = "hdfs-hbase-mapr-v3-template-spec.json"; private static final String HDFS_GPHD_TEMPLATE_SPEC = "hdfs-gphd-template-spec.json"; private static final String HDFS_MAPRED_GPHD_TEMPLATE_SPEC = "hdfs-mapred-gphd-template-spec.json"; private static final String HDFS_HBASE_GPHD_TEMPLATE_SPEC = "hdfs-hbase-gphd-template-spec.json"; private static final String CM_HDFS_MAPRED_TEMPLATE_SPEC = "cm-hdfs-mapred-template-spec.json"; private static final String CM_HDFS_YARN_TEMPLATE_SPEC = "cm-hdfs-yarn-template-spec.json"; private static final String CM_HBASE_TEMPLATE_SPEC = "cm-hbase-template-spec.json"; private static final String AM_HDFS_V1_TEMPLATE_SPEC = "am-hdfs-v1-template-spec.json"; private static final String AM_HDFS_V2_TEMPLATE_SPEC = "am-hdfs-v2-template-spec.json"; private static final String AM_HDFS_MAPRED_TEMPLATE_SPEC = "am-hdfs-mapred-template-spec.json"; private static final String AM_HDP_2_0_HDFS_YARN_TEMPLATE_SPEC = "am-hdp-2-0-hdfs-yarn-template-spec.json"; private static final String AM_HDP_2_1_HDFS_YARN_TEMPLATE_SPEC = "am-hdp-2-1-hdfs-yarn-template-spec.json"; private static final String AM_HDP_2_2_HDFS_YARN_TEMPLATE_SPEC = "am-hdp-2-2-hdfs-yarn-template-spec.json"; private static final String AM_HDP_2_3_HDFS_YARN_TEMPLATE_SPEC = "am-hdp-2-3-hdfs-yarn-template-spec.json"; private static final String AM_HDP_CUSTOMIZED_HDFS_YARN_TEMPLATE_SPEC = "am-hdp-customized-hdfs-yarn-template-spec.json"; private static final String AM_HDP_V1_HBASE_TEMPLATE_SPEC = "am-hdp-1-hbase-template-spec.json"; private static final String AM_HDP_2_0_HBASE_TEMPLATE_SPEC = "am-hdp-2-0-hbase-template-spec.json"; private static final String AM_HDP_2_1_HBASE_TEMPLATE_SPEC = "am-hdp-2-1-hbase-template-spec.json"; private static final String AM_HDP_2_2_HBASE_TEMPLATE_SPEC = "am-hdp-2-2-hbase-template-spec.json"; private static final String AM_HDP_2_3_HBASE_TEMPLATE_SPEC = "am-hdp-2-3-hbase-template-spec.json"; private static final String AM_HDP_CUSTOMIZED_HBASE_TEMPLATE_SPEC = "am-hdp-customized-hbase-template-spec.json"; private static final String AM_HDFS_PURE_HBASE_TEMPLATE_SPEC = "am-hdfs-pure-hbase-template-spec.json"; private static File locateSpecFile(String filename, String appManagerType) { // try to locate file directly File specFile = new File(filename); if (specFile.exists()) { return specFile; } // search ${serengeti.home.dir}/conf directory String homeDir = System.getProperties().getProperty("serengeti.home.dir"); if (homeDir != null && !homeDir.trim().isEmpty()) { StringBuilder builder = new StringBuilder(); builder.append(homeDir).append(File.separator).append("conf").append(File.separator) .append(appManagerType).append(File.separator).append("spec-templates").append(File.separator) .append(filename); specFile = new File(builder.toString()); } if (!specFile.exists()) { logger.warn("template cluster file does not exist: " + specFile.getAbsolutePath()); // search in class paths URL filePath = ConfigurationUtils.locate(filename); if (filePath != null) { specFile = ConfigurationUtils.fileFromURL(filePath); } } if (!specFile.exists()) { throw BddException.INTERNAL(null, "Can not find template cluster spec file " + filename); } return specFile; } /** * Load and create cluster from spec file * * @return cluster spec */ public static ClusterCreate loadFromFile(File file) throws FileNotFoundException { Reader fileReader = null; try { fileReader = new FileReader(file); Gson gson = new Gson(); return gson.fromJson(fileReader, ClusterCreate.class); } finally { if (fileReader != null) { try { fileReader.close(); } catch (IOException e) { logger.error("Failed to release buffer: " + e.getMessage()); } } } } /** * Create default cluster spec. * * @param type * cluster type * @return default cluster spec * @throws FileNotFoundException */ public static ClusterCreate createDefaultSpec(ClusterType type, String vendor, String distroVersion, String appManagerType) throws FileNotFoundException { // loading from file each time is slow but fine if (vendor.trim().equalsIgnoreCase(Constants.MAPR_VENDOR)) { boolean isMapr3 = Version.compare(distroVersion, "4") < 0; switch (type) { case HDFS_MAPRED: return loadFromFile( locateSpecFile(isMapr3 ? HDFS_MAPRED_MAPR_V3_TEMPLATE_SPEC : HDFS_MAPRED_MAPR_TEMPLATE_SPEC, appManagerType)); case HDFS_HBASE: return loadFromFile( locateSpecFile(isMapr3 ? HDFS_HBASE_MAPR_V3_TEMPLATE_SPEC : HDFS_HBASE_MAPR_TEMPLATE_SPEC, appManagerType)); default: throw BddException.INVALID_PARAMETER("cluster type", type); } } else if (vendor.trim().equalsIgnoreCase(Constants.GPHD_VENDOR)) { switch (type) { case HDFS: return loadFromFile(locateSpecFile(HDFS_GPHD_TEMPLATE_SPEC, appManagerType)); case HDFS_MAPRED: return loadFromFile(locateSpecFile(HDFS_MAPRED_GPHD_TEMPLATE_SPEC, appManagerType)); case HDFS_HBASE: return loadFromFile(locateSpecFile(HDFS_HBASE_GPHD_TEMPLATE_SPEC, appManagerType)); default: throw BddException.INVALID_PARAMETER("cluster type", type); } } else if (Constants.AMBARI_PLUGIN_TYPE.equals(appManagerType) && vendor.trim().equalsIgnoreCase(Constants.HDP_VENDOR)) { MAPREDUCE_VERSION mr = getDefaultMapReduceVersion(vendor, distroVersion); HDP_VERSION hdpVersion = getDefaultHdfsVersion(vendor, distroVersion); if (type == null) { if (mr == MAPREDUCE_VERSION.V1) { return loadFromFile(locateSpecFile(AM_HDFS_MAPRED_TEMPLATE_SPEC, appManagerType)); } else { if (hdpVersion == HDP_VERSION.V2_0) { return loadFromFile(locateSpecFile(AM_HDP_2_0_HDFS_YARN_TEMPLATE_SPEC, appManagerType)); } else if (hdpVersion == HDP_VERSION.V2_1) { return loadFromFile(locateSpecFile(AM_HDP_2_1_HDFS_YARN_TEMPLATE_SPEC, appManagerType)); } else if (hdpVersion == HDP_VERSION.V2_2) { return loadFromFile(locateSpecFile(AM_HDP_2_2_HDFS_YARN_TEMPLATE_SPEC, appManagerType)); } else if (hdpVersion == HDP_VERSION.V2_3) { return loadFromFile(locateSpecFile(AM_HDP_2_3_HDFS_YARN_TEMPLATE_SPEC, appManagerType)); } else { return loadFromFile( locateSpecFile(AM_HDP_CUSTOMIZED_HDFS_YARN_TEMPLATE_SPEC, appManagerType)); } } } switch (type) { case HDFS: if (hdpVersion == HDP_VERSION.V1) { return loadFromFile(locateSpecFile(AM_HDFS_V1_TEMPLATE_SPEC, appManagerType)); } else { return loadFromFile(locateSpecFile(AM_HDFS_V2_TEMPLATE_SPEC, appManagerType)); } case HDFS_MAPRED: if (mr == MAPREDUCE_VERSION.V1) { return loadFromFile(locateSpecFile(AM_HDFS_MAPRED_TEMPLATE_SPEC, appManagerType)); } else { if (hdpVersion == HDP_VERSION.V2_0) { return loadFromFile(locateSpecFile(AM_HDP_2_0_HDFS_YARN_TEMPLATE_SPEC, appManagerType)); } else if (hdpVersion == HDP_VERSION.V2_1) { return loadFromFile(locateSpecFile(AM_HDP_2_1_HDFS_YARN_TEMPLATE_SPEC, appManagerType)); } else if (hdpVersion == HDP_VERSION.V2_2) { return loadFromFile(locateSpecFile(AM_HDP_2_2_HDFS_YARN_TEMPLATE_SPEC, appManagerType)); } else if (hdpVersion == HDP_VERSION.V2_3) { return loadFromFile(locateSpecFile(AM_HDP_2_3_HDFS_YARN_TEMPLATE_SPEC, appManagerType)); } else { return loadFromFile( locateSpecFile(AM_HDP_CUSTOMIZED_HDFS_YARN_TEMPLATE_SPEC, appManagerType)); } } case HDFS_HBASE: if (Configuration.getBoolean(Constants.AMBARI_HBASE_DEPEND_ON_MAPREDUCE)) { if (hdpVersion == HDP_VERSION.V1) { return loadFromFile(locateSpecFile(AM_HDP_V1_HBASE_TEMPLATE_SPEC, appManagerType)); } else { if (hdpVersion == HDP_VERSION.V2_0) { return loadFromFile(locateSpecFile(AM_HDP_2_0_HBASE_TEMPLATE_SPEC, appManagerType)); } else if (hdpVersion == HDP_VERSION.V2_1) { return loadFromFile(locateSpecFile(AM_HDP_2_1_HBASE_TEMPLATE_SPEC, appManagerType)); } else if (hdpVersion == HDP_VERSION.V2_2) { return loadFromFile(locateSpecFile(AM_HDP_2_2_HBASE_TEMPLATE_SPEC, appManagerType)); } else if (hdpVersion == HDP_VERSION.V2_3) { return loadFromFile(locateSpecFile(AM_HDP_2_3_HBASE_TEMPLATE_SPEC, appManagerType)); } else { return loadFromFile( locateSpecFile(AM_HDP_CUSTOMIZED_HBASE_TEMPLATE_SPEC, appManagerType)); } } } else { return loadFromFile(locateSpecFile(AM_HDFS_PURE_HBASE_TEMPLATE_SPEC, appManagerType)); } default: throw BddException.INVALID_PARAMETER("cluster type", type); } } else { MAPREDUCE_VERSION mr = getDefaultMapReduceVersion(vendor, distroVersion); if (Constants.CLOUDERA_MANAGER_PLUGIN_TYPE.equals(appManagerType)) { if (type.equals(ClusterType.HDFS_HBASE)) { return loadFromFile(locateSpecFile(CM_HBASE_TEMPLATE_SPEC, appManagerType)); } if (mr == MAPREDUCE_VERSION.V1) { return loadFromFile(locateSpecFile(CM_HDFS_MAPRED_TEMPLATE_SPEC, appManagerType)); } else { return loadFromFile(locateSpecFile(CM_HDFS_YARN_TEMPLATE_SPEC, appManagerType)); } } switch (type) { case HDFS: return loadFromFile(locateSpecFile(HDFS_TEMPLATE_SPEC, appManagerType)); case HDFS_MAPRED: if (mr == MAPREDUCE_VERSION.V1) { return loadFromFile(locateSpecFile(HDFS_MAPRED_TEMPLATE_SPEC, appManagerType)); } else { return loadFromFile(locateSpecFile(HDFS_YARN_TEMPLATE_SPEC, appManagerType)); } case HDFS_HBASE: return loadFromFile(locateSpecFile(HDFS_HBASE_TEMPLATE_SPEC, appManagerType)); default: return loadFromFile( locateSpecFile(vendor.toLowerCase() + DEFAULT_TEMPLATE_SPEC_SUFFIX, appManagerType)); } } } private static MAPREDUCE_VERSION getDefaultMapReduceVersion(String vendor, String distroVersion) { if (vendor.trim().equalsIgnoreCase(Constants.DEFAULT_VENDOR)) { return MAPREDUCE_VERSION.V2; } if (vendor.trim().equalsIgnoreCase(Constants.APACHE_VENDOR)) { return MAPREDUCE_VERSION.V1; } if (vendor.trim().equalsIgnoreCase(Constants.BIGTOP_VENDOR)) { return MAPREDUCE_VERSION.V2; } if (vendor.trim().equalsIgnoreCase(Constants.CDH_VENDOR)) { if (distroVersion.startsWith("5")) { return MAPREDUCE_VERSION.V2; } else { return MAPREDUCE_VERSION.V1; } } if (vendor.trim().equalsIgnoreCase(Constants.HDP_VENDOR)) { if (distroVersion.startsWith("2")) { return MAPREDUCE_VERSION.V2; } else { return MAPREDUCE_VERSION.V1; } } if (vendor.trim().equalsIgnoreCase(Constants.INTEL_VENDOR)) { if (distroVersion.startsWith("3")) { return MAPREDUCE_VERSION.V2; } else { return MAPREDUCE_VERSION.V1; } } if (vendor.trim().equalsIgnoreCase(Constants.PHD_VENDOR)) { return MAPREDUCE_VERSION.V2; } if (vendor.trim().equalsIgnoreCase(Constants.MAPR_VENDOR)) { return MAPREDUCE_VERSION.V1; } if (vendor.trim().equalsIgnoreCase(Constants.GPHD_VENDOR)) { return MAPREDUCE_VERSION.V1; } logger.error("Unknown distro vendor, return default mapreduce version 2"); return MAPREDUCE_VERSION.V2; } private static HDP_VERSION getDefaultHdfsVersion(String vendor, String distroVersion) { if (vendor.trim().equalsIgnoreCase(Constants.HDP_VENDOR)) { if (Version.compare(distroVersion, "1.0") >= 0 && Version.compare(distroVersion, "2.0") < 0) { return HDP_VERSION.V1; } else if (Version.compare(distroVersion, "2.0") >= 0 && Version.compare(distroVersion, "2.1") < 0) { return HDP_VERSION.V2_0; } else if (Version.compare(distroVersion, "2.1") >= 0 && Version.compare(distroVersion, "2.2") < 0) { return HDP_VERSION.V2_1; } else if (Version.compare(distroVersion, "2.2") >= 0 && Version.compare(distroVersion, "2.3") < 0) { return HDP_VERSION.V2_2; } else if (Version.compare(distroVersion, "2.3") >= 0 && Version.compare(distroVersion, "2.4") < 0) { return HDP_VERSION.V2_3; } else { return HDP_VERSION.UNKNOWN; } } logger.error("Unknown distro HDP version"); return HDP_VERSION.UNKNOWN; } /** * There are two approach to create a cluster: 1) specify a cluster type and * optionally overwriting the parameters 2) specify a customized spec with * cluster type not specified * * @param spec * spec with customized field * @return customized cluster spec * @throws FileNotFoundException */ public static ClusterCreate getCustomizedSpec(ClusterCreate spec, String appManagerType) throws FileNotFoundException { if (!ArrayUtils.isEmpty(spec.getNodeGroups())) { spec.setSpecFile(true); } if (spec.isSpecFile()) { return spec; } ClusterCreate newSpec = createDefaultSpec(spec.getType(), spec.getDistroVendor(), spec.getDistroVersion(), appManagerType); // --name if (spec.getName() != null) { newSpec.setName(spec.getName()); } //--password newSpec.setPassword(spec.getPassword()); // --appManager if (!CommonUtil.isBlank(spec.getAppManager())) { newSpec.setAppManager(spec.getAppManager()); } // --locaRepoURL if (!CommonUtil.isBlank(spec.getLocalRepoURL())) { newSpec.setLocalRepoURL(spec.getLocalRepoURL()); } // --distro if (spec.getDistro() != null) { newSpec.setDistro(spec.getDistro()); } //vendor if (spec.getDistroVendor() != null) { newSpec.setDistroVendor(spec.getDistroVendor()); } //version if (spec.getDistroVersion() != null) { newSpec.setDistroVersion(spec.getDistroVersion()); } // template name if (spec.getTemplateName() != null) { newSpec.setTemplateName(spec.getTemplateName()); } // --dsNames if (spec.getDsNames() != null) { newSpec.setDsNames(spec.getDsNames()); } // --rpNames if (spec.getRpNames() != null) { newSpec.setRpNames(spec.getRpNames()); } // --networkConfig if (spec.getNetworkConfig() != null) { newSpec.setNetworkConfig(spec.getNetworkConfig()); } // --topology if (spec.getTopologyPolicy() != null) { newSpec.setTopologyPolicy(spec.getTopologyPolicy()); } if (MapUtils.isNotEmpty(spec.getInfrastructure_config())) { newSpec.setInfrastructure_config(spec.getInfrastructure_config()); } return newSpec; } }