Java tutorial
/* * Copyright 2014-2015 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of * the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations under * the License. */ package co.cask.cdap.explore.guice; import co.cask.cdap.common.conf.CConfiguration; import co.cask.cdap.common.conf.Constants; import co.cask.cdap.common.runtime.RuntimeModule; import co.cask.cdap.data2.datafabric.dataset.RemoteDatasetFramework; import co.cask.cdap.data2.util.hbase.HBaseTableUtilFactory; import co.cask.cdap.explore.executor.ExploreExecutorHttpHandler; import co.cask.cdap.explore.executor.ExploreExecutorService; import co.cask.cdap.explore.executor.ExploreMetadataHttpHandler; import co.cask.cdap.explore.executor.ExploreStatusHandler; import co.cask.cdap.explore.executor.NamespacedExploreMetadataHttpHandler; import co.cask.cdap.explore.executor.NamespacedQueryExecutorHttpHandler; import co.cask.cdap.explore.executor.QueryExecutorHttpHandler; import co.cask.cdap.explore.service.ExploreService; import co.cask.cdap.explore.service.ExploreServiceUtils; import co.cask.cdap.explore.service.hive.BaseHiveExploreService; import co.cask.cdap.explore.service.hive.Hive14ExploreService; import co.cask.cdap.format.RecordFormats; import co.cask.cdap.gateway.handlers.CommonHandlers; import co.cask.cdap.hive.datasets.DatasetStorageHandler; import co.cask.http.HttpHandler; import com.google.common.base.Function; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.io.Files; import com.google.inject.Exposed; import com.google.inject.Inject; import com.google.inject.Injector; import com.google.inject.Module; import com.google.inject.PrivateModule; import com.google.inject.Provider; import com.google.inject.Provides; import com.google.inject.Scopes; import com.google.inject.Singleton; import com.google.inject.multibindings.Multibinder; import com.google.inject.name.Named; import com.google.inject.name.Names; import com.google.inject.util.Modules; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapreduce.MRConfig; import org.apache.twill.api.ClassAcceptor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; import java.net.URL; import java.util.LinkedHashSet; import java.util.List; import java.util.Set; /** * Guice runtime module for the explore functionality. */ public class ExploreRuntimeModule extends RuntimeModule { private static final Logger LOG = LoggerFactory.getLogger(ExploreRuntimeModule.class); @Override public Module getInMemoryModules() { // Turning off assertions for Hive packages, since some assertions in StandardStructObjectInspector do not work // when outer joins are run. It is okay to turn off Hive assertions since we assume Hive is a black-box that does // the right thing, and we only want to test our/our user's code. getClass().getClassLoader().setPackageAssertionStatus("org.apache.hadoop.hive", false); getClass().getClassLoader().setPackageAssertionStatus("org.apache.hive", false); return Modules.combine(new ExploreExecutorModule(), new ExploreLocalModule(true)); } @Override public Module getStandaloneModules() { return Modules.combine(new ExploreExecutorModule(), new ExploreLocalModule(false)); } @Override public Module getDistributedModules() { return Modules.combine(new ExploreExecutorModule(), new ExploreDistributedModule()); } private static final class ExploreExecutorModule extends PrivateModule { @Override protected void configure() { Named exploreSeriveName = Names.named(Constants.Service.EXPLORE_HTTP_USER_SERVICE); Multibinder<HttpHandler> handlerBinder = Multibinder.newSetBinder(binder(), HttpHandler.class, exploreSeriveName); handlerBinder.addBinding().to(NamespacedQueryExecutorHttpHandler.class); handlerBinder.addBinding().to(QueryExecutorHttpHandler.class); handlerBinder.addBinding().to(NamespacedExploreMetadataHttpHandler.class); handlerBinder.addBinding().to(ExploreMetadataHttpHandler.class); handlerBinder.addBinding().to(ExploreExecutorHttpHandler.class); handlerBinder.addBinding().to(ExploreStatusHandler.class); CommonHandlers.add(handlerBinder); bind(ExploreExecutorService.class).in(Scopes.SINGLETON); expose(ExploreExecutorService.class); } } private static final class ExploreLocalModule extends PrivateModule { private final boolean isInMemory; public ExploreLocalModule(boolean isInMemory) { this.isInMemory = isInMemory; } @Override protected void configure() { // Current version of hive used in standalone is Hive 14 bind(ExploreService.class).annotatedWith(Names.named("explore.service.impl")) .to(Hive14ExploreService.class); bind(ExploreService.class).toProvider(ExploreServiceProvider.class).in(Scopes.SINGLETON); expose(ExploreService.class); bind(boolean.class).annotatedWith(Names.named("explore.inmemory")).toInstance(isInMemory); bind(File.class).annotatedWith(Names.named(Constants.Explore.PREVIEWS_DIR_NAME)) .toProvider(PreviewsDirProvider.class); } private static final class PreviewsDirProvider implements Provider<File> { private final CConfiguration cConf; @Inject public PreviewsDirProvider(CConfiguration cConf) { this.cConf = cConf; } @Override public File get() { String localDirStr = cConf.get(Constants.Explore.LOCAL_DATA_DIR); File previewsDir = new File(localDirStr, "previewsDir"); previewsDir.mkdirs(); return previewsDir; } } @Singleton private static final class ExploreServiceProvider implements Provider<ExploreService> { private final CConfiguration cConf; private final Configuration hConf; private final ExploreService exploreService; private final boolean isInMemory; @Inject public ExploreServiceProvider(CConfiguration cConf, Configuration hConf, @Named("explore.service.impl") ExploreService exploreService, @Named("explore.inmemory") boolean isInMemory) { this.exploreService = exploreService; this.cConf = cConf; this.hConf = hConf; this.isInMemory = isInMemory; } private static final long seed = System.currentTimeMillis(); @Override public ExploreService get() { File hiveDataDir = new File(cConf.get(Constants.Explore.LOCAL_DATA_DIR)); // The properties set using setProperty will be included to any new HiveConf object created, // at the condition that the configuration is known by Hive, and so is one of the HiveConf.ConfVars // variables. System.setProperty(HiveConf.ConfVars.SCRATCHDIR.toString(), new File(hiveDataDir, cConf.get(Constants.AppFabric.TEMP_DIR)).getAbsolutePath()); // Reset hadoop tmp dir because Hive does not pick it up from hConf System.setProperty("hadoop.tmp.dir", hConf.get("hadoop.tmp.dir")); File warehouseDir = new File(cConf.get(Constants.Explore.LOCAL_DATA_DIR), "warehouse"); File databaseDir = new File(cConf.get(Constants.Explore.LOCAL_DATA_DIR), "database"); if (isInMemory) { // This seed is required to make all tests pass when launched together, and when several of them // start a hive metastore / hive server. warehouseDir = new File(warehouseDir, Long.toString(seed)); databaseDir = new File(databaseDir, Long.toString(seed)); } LOG.debug("Setting {} to {}", HiveConf.ConfVars.METASTOREWAREHOUSE.toString(), warehouseDir.getAbsoluteFile()); System.setProperty(HiveConf.ConfVars.METASTOREWAREHOUSE.toString(), warehouseDir.getAbsolutePath()); // Set derby log location System.setProperty("derby.stream.error.file", cConf.get(Constants.Explore.LOCAL_DATA_DIR) + File.separator + "derby.log"); String connectUrl = String.format("jdbc:derby:;databaseName=%s;create=true", databaseDir.getAbsoluteFile()); LOG.debug("Setting {} to {}", HiveConf.ConfVars.METASTORECONNECTURLKEY.toString(), connectUrl); System.setProperty(HiveConf.ConfVars.METASTORECONNECTURLKEY.toString(), connectUrl); // Some more local mode settings System.setProperty(HiveConf.ConfVars.LOCALMODEAUTO.toString(), "true"); System.setProperty(HiveConf.ConfVars.SUBMITVIACHILD.toString(), "false"); System.setProperty(MRConfig.FRAMEWORK_NAME, "local"); // Disable security // Also need to disable security by making HiveAuthFactory.loginFromKeytab a no-op, since Hive >=0.14 // ignores the HIVE_SERVER2_AUTHENTICATION property and instead uses UserGroupInformation.isSecurityEnabled() // (rewrite to HiveAuthFactory.loginFromKeytab bytecode is done in ExploreServiceUtils.traceDependencies) System.setProperty(HiveConf.ConfVars.HIVE_SERVER2_AUTHENTICATION.toString(), "NONE"); System.setProperty(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS.toString(), "false"); System.setProperty(HiveConf.ConfVars.METASTORE_USE_THRIFT_SASL.toString(), "false"); return exploreService; } } } private static final class ExploreDistributedModule extends PrivateModule { private static final Logger LOG = LoggerFactory.getLogger(ExploreDistributedModule.class); @Override protected void configure() { try { CConfiguration cConf = CConfiguration.create(); File tmpDir = new File(new File(cConf.get(Constants.CFG_LOCAL_DATA_DIR)), cConf.get(Constants.AppFabric.TEMP_DIR)).getAbsoluteFile(); tmpDir.mkdirs(); setupClasspath(tmpDir); // Set local tmp dir to an absolute location in the twill runnable otherwise Hive complains String localScratchPath = System.getProperty("java.io.tmpdir") + File.separator + "hive-" + System.getProperty("user.name"); System.setProperty(HiveConf.ConfVars.LOCALSCRATCHDIR.toString(), new File(localScratchPath).getAbsolutePath()); LOG.info("Setting {} to {}", HiveConf.ConfVars.LOCALSCRATCHDIR.toString(), System.getProperty(HiveConf.ConfVars.LOCALSCRATCHDIR.toString())); File previewDir = Files.createTempDir(); LOG.info("Storing preview files in {}", previewDir.getAbsolutePath()); bind(File.class).annotatedWith(Names.named(Constants.Explore.PREVIEWS_DIR_NAME)) .toInstance(previewDir); } catch (Throwable e) { throw Throwables.propagate(e); } } @Provides @Singleton @Exposed public ExploreService providesExploreService(Injector injector) { // Figure out which HiveExploreService class to load Class<? extends ExploreService> hiveExploreServiceCl = ExploreServiceUtils.getHiveService(); LOG.info("Using Explore service class {}", hiveExploreServiceCl.getName()); return injector.getInstance(hiveExploreServiceCl); } } private static void setupClasspath(File tmpDir) throws IOException { // Here we find the transitive dependencies and remove all paths that come from the boot class path - // those paths are not needed because the new JVM will have them in its boot class path. // It could even be wrong to keep them because in the target container, the boot class path may be different // (for example, if Hadoop uses a different Java version than CDAP). final Set<String> bootstrapClassPaths = ExploreServiceUtils.getBoostrapClasses(); ClassAcceptor classAcceptor = new ClassAcceptor() { /* Excluding any class contained in the bootstrapClassPaths and Kryo classes and hive-exec.jar * We need to remove Kryo dependency in the Explore container. Spark introduced version 2.21 version of Kryo, * which would be normally shipped to the Explore container. Yet, Hive requires Kryo 2.22, * and gets it from the Hive jars - hive-exec.jar to be precise. * we also exclude hive jars as hive dependencies are found in job.jar. * */ @Override public boolean accept(String className, URL classUrl, URL classPathUrl) { if (bootstrapClassPaths.contains(classPathUrl.getFile()) || className.startsWith("com.esotericsoftware.kryo") || classPathUrl.getFile().contains("hive")) { return false; } return true; } }; Set<File> hBaseTableDeps = ExploreServiceUtils.traceDependencies(null, classAcceptor, tmpDir, HBaseTableUtilFactory.getHBaseTableUtilClass().getName()); // Note the order of dependency jars is important so that HBase jars come first in the classpath order // LinkedHashSet maintains insertion order while removing duplicate entries. Set<File> orderedDependencies = new LinkedHashSet<>(); orderedDependencies.addAll(hBaseTableDeps); orderedDependencies.addAll(ExploreServiceUtils.traceDependencies(null, classAcceptor, tmpDir, RemoteDatasetFramework.class.getName(), DatasetStorageHandler.class.getName(), RecordFormats.class.getName())); // Note: the class path entries need to be prefixed with "file://" for the jars to work when // Hive starts local map-reduce job. ImmutableList.Builder<String> builder = ImmutableList.builder(); for (File dep : orderedDependencies) { builder.add("file://" + dep.getAbsolutePath()); } List<String> orderedDependenciesStr = builder.build(); // These dependency files need to be copied over to spark container System.setProperty(BaseHiveExploreService.SPARK_YARN_DIST_FILES, Joiner.on(',').join(Iterables.transform(orderedDependencies, new Function<File, String>() { @Override public String apply(File input) { return input.getAbsolutePath(); } }))); LOG.debug("Setting {} to {}", BaseHiveExploreService.SPARK_YARN_DIST_FILES, System.getProperty(BaseHiveExploreService.SPARK_YARN_DIST_FILES)); // These dependency files need to be copied over to hive job container System.setProperty(HiveConf.ConfVars.HIVEAUXJARS.toString(), Joiner.on(',').join(orderedDependenciesStr)); LOG.debug("Setting {} to {}", HiveConf.ConfVars.HIVEAUXJARS.toString(), System.getProperty(HiveConf.ConfVars.HIVEAUXJARS.toString())); // add hive-exec.jar to the HADOOP_CLASSPATH, which is used by the local mapreduce job launched by hive , // we need to add this, otherwise when hive runs a MapRedLocalTask it cannot find // "org.apache.hadoop.hive.serde2.SerDe" class in its classpath. List<String> orderedDependenciesWithHiveJar = Lists.newArrayList(orderedDependenciesStr); String hiveExecJar = new JobConf(org.apache.hadoop.hive.ql.exec.Task.class).getJar(); Preconditions.checkNotNull(hiveExecJar, "Couldn't locate hive-exec.jar to be included in HADOOP_CLASSPATH " + "for MapReduce jobs launched by Hive"); orderedDependenciesWithHiveJar.add(hiveExecJar); LOG.debug("Added hive-exec.jar {} to HADOOP_CLASSPATH to be included for MapReduce jobs launched by Hive", hiveExecJar); //TODO: Setup HADOOP_CLASSPATH hack, more info on why this is needed, see CDAP-9 LocalMapreduceClasspathSetter classpathSetter = new LocalMapreduceClasspathSetter(new HiveConf(), tmpDir.getAbsolutePath(), orderedDependenciesWithHiveJar); for (File jar : hBaseTableDeps) { classpathSetter.accept(jar.getAbsolutePath()); } classpathSetter.setupClasspathScript(); } }