Example usage for org.apache.hadoop.security UserGroupInformation setConfiguration

List of usage examples for org.apache.hadoop.security UserGroupInformation setConfiguration

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.security.*
  5. UserGroupInformation
  6. setConfiguration

Introduction

In this page you can find the example usage for org.apache.hadoop.security UserGroupInformation setConfiguration.

Prototype

@InterfaceAudience.Public
@InterfaceStability.Evolving
public static void setConfiguration(Configuration conf)

Source Link

Document

Set the static configuration for UGI.

Usage

From source file:azkaban.security.HadoopSecurityManager_H_2_0.java

License:Apache License

private HadoopSecurityManager_H_2_0(Props props) throws HadoopSecurityManagerException, IOException {

    // for now, assume the same/compatible native library, the same/compatible
    // hadoop-core jar
    String hadoopHome = props.getString("hadoop.home", null);
    String hadoopConfDir = props.getString("hadoop.conf.dir", null);

    if (hadoopHome == null) {
        hadoopHome = System.getenv("HADOOP_HOME");
    }/*from w w  w . java  2 s .  c  o  m*/
    if (hadoopConfDir == null) {
        hadoopConfDir = System.getenv("HADOOP_CONF_DIR");
    }

    List<URL> resources = new ArrayList<URL>();
    URL urlToHadoop = null;
    if (hadoopConfDir != null) {
        urlToHadoop = new File(hadoopConfDir).toURI().toURL();
        logger.info("Using hadoop config found in " + urlToHadoop);
        resources.add(urlToHadoop);
    } else if (hadoopHome != null) {
        urlToHadoop = new File(hadoopHome, "conf").toURI().toURL();
        logger.info("Using hadoop config found in " + urlToHadoop);
        resources.add(urlToHadoop);
    } else {
        logger.info("HADOOP_HOME not set, using default hadoop config.");
    }

    ucl = new URLClassLoader(resources.toArray(new URL[resources.size()]));

    conf = new Configuration();
    conf.setClassLoader(ucl);

    if (props.containsKey(FS_HDFS_IMPL_DISABLE_CACHE)) {
        logger.info("Setting " + FS_HDFS_IMPL_DISABLE_CACHE + " to " + props.get(FS_HDFS_IMPL_DISABLE_CACHE));
        conf.setBoolean(FS_HDFS_IMPL_DISABLE_CACHE, Boolean.valueOf(props.get(FS_HDFS_IMPL_DISABLE_CACHE)));
    }

    logger.info(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION + ": "
            + conf.get(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION));
    logger.info(CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION + ":  "
            + conf.get(CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION));
    logger.info(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY + ": "
            + conf.get(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY));

    UserGroupInformation.setConfiguration(conf);

    securityEnabled = UserGroupInformation.isSecurityEnabled();
    if (securityEnabled) {
        logger.info("The Hadoop cluster has enabled security");
        shouldProxy = true;
        try {

            keytabLocation = props.getString(AZKABAN_KEYTAB_LOCATION);
            keytabPrincipal = props.getString(AZKABAN_PRINCIPAL);
        } catch (UndefinedPropertyException e) {
            throw new HadoopSecurityManagerException(e.getMessage());
        }

        // try login
        try {
            if (loginUser == null) {
                logger.info("No login user. Creating login user");
                logger.info("Using principal from " + keytabPrincipal + " and " + keytabLocation);
                UserGroupInformation.loginUserFromKeytab(keytabPrincipal, keytabLocation);
                loginUser = UserGroupInformation.getLoginUser();
                logger.info("Logged in with user " + loginUser);
            } else {
                logger.info("loginUser (" + loginUser + ") already created, refreshing tgt.");
                loginUser.checkTGTAndReloginFromKeytab();
            }
        } catch (IOException e) {
            throw new HadoopSecurityManagerException("Failed to login with kerberos ", e);
        }

    }

    userUgiMap = new ConcurrentHashMap<String, UserGroupInformation>();

    logger.info("Hadoop Security Manager initialized");
}

From source file:azkaban.storage.HdfsAuth.java

License:Apache License

@Inject
public HdfsAuth(final Props props, final Configuration conf) {
    UserGroupInformation.setConfiguration(conf);
    this.isSecurityEnabled = UserGroupInformation.isSecurityEnabled();
    if (this.isSecurityEnabled) {
        log.info("The Hadoop cluster has enabled security");
        this.keytabPath = requireNonNull(props.getString(AZKABAN_KEYTAB_PATH));
        this.keytabPrincipal = requireNonNull(props.getString(AZKABAN_KERBEROS_PRINCIPAL));
    }//from   w w w. ja v a2 s .  co m
}

From source file:cascading.flow.tez.Hadoop2TezFlow.java

License:Open Source License

protected void initConfig(Map<Object, Object> properties, TezConfiguration parentConfig) {
    if (properties != null)
        parentConfig = createConfig(properties, parentConfig);

    if (parentConfig == null) // this is ok, getJobConf will pass a default parent in
        return;//from  w ww .  j av a2s.c om

    flowConf = new TezConfiguration(parentConfig); // prevent local values from being shared
    flowConf.set("fs.http.impl", HttpFileSystem.class.getName());
    flowConf.set("fs.https.impl", HttpFileSystem.class.getName());

    UserGroupInformation.setConfiguration(flowConf);

    flowStagingPath = createStagingRoot();
}

From source file:co.cask.cdap.common.twill.AbstractMasterTwillRunnable.java

License:Apache License

@Override
public final void initialize(TwillContext context) {
    super.initialize(context);

    name = context.getSpecification().getName();
    Map<String, String> configs = context.getSpecification().getConfigs();

    try {/*from  w w  w  .j  a  v a2s .c o  m*/
        // Load configuration
        hConf = new Configuration();
        hConf.clear();
        hConf.addResource(new File(configs.get("hConf")).toURI().toURL());

        UserGroupInformation.setConfiguration(hConf);

        cConf = CConfiguration.create(new File(configs.get("cConf")));

        LOG.debug("{} cConf {}", name, cConf);
        LOG.debug("{} HBase conf {}", name, hConf);

        doInit(context);

        services = Lists.newArrayList();
        getServices(services);
        Preconditions.checkArgument(!services.isEmpty(), "Should have at least one service");

    } catch (Throwable t) {
        throw Throwables.propagate(t);
    }
}

From source file:co.cask.cdap.internal.app.runtime.distributed.AbstractProgramTwillRunnable.java

License:Apache License

@Override
public void initialize(TwillContext context) {
    System.setSecurityManager(new RunnableSecurityManager(System.getSecurityManager()));

    runlatch = new CountDownLatch(1);
    name = context.getSpecification().getName();
    Map<String, String> configs = context.getSpecification().getConfigs();

    LOG.info("Initialize runnable: " + name);
    try {/*from  w  ww.j  a va  2s.  c om*/
        CommandLine cmdLine = parseArgs(context.getApplicationArguments());

        // Loads configurations
        hConf = new Configuration();
        hConf.clear();
        hConf.addResource(new File(configs.get("hConf")).toURI().toURL());

        UserGroupInformation.setConfiguration(hConf);

        cConf = CConfiguration.create(new File(configs.get("cConf")));

        Injector injector = Guice.createInjector(createModule(context));

        zkClientService = injector.getInstance(ZKClientService.class);
        kafkaClientService = injector.getInstance(KafkaClientService.class);
        metricsCollectionService = injector.getInstance(MetricsCollectionService.class);
        streamCoordinatorClient = injector.getInstance(StreamCoordinatorClient.class);

        // Initialize log appender
        logAppenderInitializer = injector.getInstance(LogAppenderInitializer.class);
        logAppenderInitializer.initialize();

        // Create the ProgramRunner
        programRunner = createProgramRunner(injector);

        try {
            Location programJarLocation = Locations
                    .toLocation(new File(cmdLine.getOptionValue(RunnableOptions.JAR)));
            program = Programs.create(cConf, programRunner, programJarLocation,
                    BundleJarUtil.unJar(programJarLocation, Files.createTempDir()));
        } catch (IOException e) {
            throw Throwables.propagate(e);
        }

        programOpts = createProgramOptions(cmdLine, context, configs);
        resourceReporter = new ProgramRunnableResourceReporter(program.getId().toEntityId(),
                metricsCollectionService, context);

        LOG.info("Runnable initialized: " + name);
    } catch (Throwable t) {
        LOG.error(t.getMessage(), t);
        throw Throwables.propagate(t);
    }
}

From source file:co.cask.cdap.internal.app.runtime.distributed.ServiceTwillRunnable.java

License:Apache License

@Override
public void initialize(TwillContext context) {
    name = context.getSpecification().getName();
    Map<String, String> configs = context.getSpecification().getConfigs();

    LOG.info("Initialize runnable: " + name);
    try {//w  w  w.ja  v a  2s. co m
        CommandLine cmdLine = parseArgs(context.getApplicationArguments());

        // Loads configurations
        hConf = new Configuration();
        hConf.clear();
        hConf.addResource(new File(configs.get("hConf")).toURI().toURL());

        UserGroupInformation.setConfiguration(hConf);

        cConf = CConfiguration.create();
        cConf.clear();
        cConf.addResource(new File(configs.get("cConf")).toURI().toURL());

        injector = Guice.createInjector(createModule(context));

        zkClientService = injector.getInstance(ZKClientService.class);
        kafkaClientService = injector.getInstance(KafkaClientService.class);
        metricsCollectionService = injector.getInstance(MetricsCollectionService.class);

        // Initialize log appender
        logAppenderInitializer = injector.getInstance(LogAppenderInitializer.class);
        logAppenderInitializer.initialize();

        transactionSystemClient = injector.getInstance(TransactionSystemClient.class);
        datasetFramework = injector.getInstance(DatasetFramework.class);
        discoveryServiceClient = injector.getInstance(DiscoveryServiceClient.class);

        try {
            program = injector.getInstance(ProgramFactory.class)
                    .create(cmdLine.getOptionValue(RunnableOptions.JAR));
        } catch (IOException e) {
            throw Throwables.propagate(e);
        }

        Arguments runtimeArguments = new Gson().fromJson(cmdLine.getOptionValue(RunnableOptions.RUNTIME_ARGS),
                BasicArguments.class);
        programOpts = new SimpleProgramOptions(name, createProgramArguments(context, configs),
                runtimeArguments);
        resourceReporter = new ProgramRunnableResourceReporter(program, metricsCollectionService, context);

        // These services need to be starting before initializing the delegate since they are used in
        // AbstractContext's constructor to create datasets.
        Futures.getUnchecked(Services.chainStart(zkClientService, kafkaClientService, metricsCollectionService,
                resourceReporter));

        ApplicationSpecification appSpec = program.getSpecification();
        String processorName = program.getName();
        runnableName = programOpts.getName();

        Arguments arguments = programOpts.getArguments();
        RunId runId = arguments.hasOption(ProgramOptionConstants.RUN_ID)
                ? RunIds.fromString(arguments.getOption(ProgramOptionConstants.RUN_ID))
                : RunIds.generate();

        ServiceSpecification serviceSpec = appSpec.getServices().get(processorName);
        final RuntimeSpecification runtimeSpec = serviceSpec.getRunnables().get(runnableName);
        String className = runtimeSpec.getRunnableSpecification().getClassName();
        LOG.info("Getting class : {}", program.getMainClass().getName());
        Class<?> clz = Class.forName(className, true, program.getClassLoader());
        Preconditions.checkArgument(TwillRunnable.class.isAssignableFrom(clz), "%s is not a TwillRunnable.",
                clz);

        if (clz.isAssignableFrom(HttpServiceTwillRunnable.class)) {
            // Special case for running http services since we need to instantiate the http service
            // using the program classloader.
            delegate = new HttpServiceTwillRunnable(program, runId, cConf, runnableName,
                    metricsCollectionService, discoveryServiceClient, datasetFramework,
                    transactionSystemClient);
        } else if (clz.isAssignableFrom(ServiceWorkerTwillRunnable.class)) {
            delegate = new ServiceWorkerTwillRunnable(program, runId, runnableName, program.getClassLoader(),
                    cConf, metricsCollectionService, datasetFramework, transactionSystemClient,
                    discoveryServiceClient);
        } else {
            delegate = (TwillRunnable) new InstantiatorFactory(false).get(TypeToken.of(clz)).create();
        }

        Reflections.visit(delegate, TypeToken.of(delegate.getClass()),
                new MetricsFieldSetter(new ServiceRunnableMetrics(metricsCollectionService,
                        program.getApplicationId(), program.getName(), runnableName, context.getInstanceId())),
                new PropertyFieldSetter(runtimeSpec.getRunnableSpecification().getConfigs()));

        final String[] argArray = RuntimeArguments.toPosixArray(programOpts.getUserArguments());
        LoggingContextAccessor.setLoggingContext(new UserServiceLoggingContext(program.getAccountId(),
                program.getApplicationId(), program.getName(), runnableName));
        delegate.initialize(new ForwardingTwillContext(context) {
            @Override
            public String[] getApplicationArguments() {
                return argArray;
            }

            @Override
            public TwillRunnableSpecification getSpecification() {
                return runtimeSpec.getRunnableSpecification();
            }

            @Override
            public Cancellable announce(String runnable, int port) {
                String serviceContextPath = String.format("%s.%s.%s.%s",
                        ProgramType.SERVICE.name().toLowerCase(), program.getAccountId(),
                        program.getApplicationId(), program.getName());
                // Currently ignoring the runnable's name (the param passed into announce), and simply announcing by the name
                // of the service it belongs to (reasoning: the primary runnable of a service will be identified by the
                // service's name, and the other runnables within the service are not discoverable externally).
                return super.announce(serviceContextPath, port);
            }
        });

        LOG.info("Runnable initialized: " + name);
    } catch (Throwable t) {
        LOG.error(t.getMessage(), t);
        throw Throwables.propagate(t);
    }
}

From source file:co.cask.cdap.logging.run.LogSaverTwillRunnable.java

License:Apache License

@Override
public void initialize(TwillContext context) {
    super.initialize(context);

    completion = SettableFuture.create();
    name = context.getSpecification().getName();
    Map<String, String> configs = context.getSpecification().getConfigs();

    LOG.info("Initialize runnable: " + name);
    try {// w w  w  .j  a v  a  2 s .  c  o  m
        // Load configuration
        Configuration hConf = new Configuration();
        hConf.clear();
        hConf.addResource(new File(configs.get("hConf")).toURI().toURL());

        UserGroupInformation.setConfiguration(hConf);

        CConfiguration cConf = CConfiguration.create(new File(configs.get("cConf")));

        cConf.set(Constants.LogSaver.ADDRESS, context.getHost().getCanonicalHostName());

        // Initialize ZK client
        String zookeeper = cConf.get(Constants.Zookeeper.QUORUM);
        if (zookeeper == null) {
            LOG.error("No ZooKeeper quorum provided.");
            throw new IllegalStateException("No ZooKeeper quorum provided.");
        }

        Injector injector = createGuiceInjector(cConf, hConf);
        zkClientService = injector.getInstance(ZKClientService.class);
        kafkaClientService = injector.getInstance(KafkaClientService.class);
        logSaverService = injector.getInstance(KafkaLogSaverService.class);

        int numPartitions = Integer.parseInt(
                cConf.get(LoggingConfiguration.NUM_PARTITIONS, LoggingConfiguration.DEFAULT_NUM_PARTITIONS));
        LOG.info("Num partitions = {}", numPartitions);

        logSaverStatusService = injector.getInstance(LogSaverStatusService.class);
        metricsCollectionService = injector.getInstance(MetricsCollectionService.class);
        LOG.info("Runnable initialized: " + name);
    } catch (Throwable t) {
        LOG.error(t.getMessage(), t);
        throw Throwables.propagate(t);
    }
}

From source file:co.cask.cdap.security.impersonation.UGIProviderTest.java

License:Apache License

@BeforeClass
public static void init() throws Exception {
    cConf = CConfiguration.create();//ww  w  .  j a va 2  s .c o m
    cConf.set(Constants.CFG_LOCAL_DATA_DIR, TEMP_FOLDER.newFolder().getAbsolutePath());

    // Start KDC
    miniKdc = new MiniKdc(MiniKdc.createConf(), TEMP_FOLDER.newFolder());
    miniKdc.start();
    System.setProperty("java.security.krb5.conf", miniKdc.getKrb5conf().getAbsolutePath());

    // Generate keytab
    keytabFile = TEMP_FOLDER.newFile();
    miniKdc.createPrincipal(keytabFile, "hdfs", "alice", "bob");

    // Start mini DFS cluster
    Configuration hConf = new Configuration();
    hConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TEMP_FOLDER.newFolder().getAbsolutePath());
    hConf.setBoolean("ipc.client.fallback-to-simple-auth-allowed", true);

    miniDFSCluster = new MiniDFSCluster.Builder(hConf).numDataNodes(1).build();
    miniDFSCluster.waitClusterUp();
    locationFactory = new FileContextLocationFactory(miniDFSCluster.getFileSystem().getConf());

    hConf = new Configuration();
    hConf.set("hadoop.security.authentication", "kerberos");
    UserGroupInformation.setConfiguration(hConf);
}

From source file:co.cask.tigon.internal.app.runtime.distributed.AbstractProgramTwillRunnable.java

License:Apache License

@Override
public void initialize(TwillContext context) {
    runlatch = new CountDownLatch(1);
    name = context.getSpecification().getName();
    Map<String, String> configs = context.getSpecification().getConfigs();

    LOG.info("Initialize runnable: " + name);
    try {/*w  w  w.  j av  a  2s  .  c  o  m*/
        CommandLine cmdLine = parseArgs(context.getApplicationArguments());

        // Loads configurations
        hConf = new Configuration();
        hConf.clear();
        hConf.addResource(new File(configs.get("hConf")).toURI().toURL());

        UserGroupInformation.setConfiguration(hConf);

        cConf = CConfiguration.create();
        cConf.clear();
        cConf.addResource(new File(configs.get("cConf")).toURI().toURL());

        injector = Guice.createInjector(createModule(context));

        zkClientService = injector.getInstance(ZKClientService.class);
        metricsCollectionService = injector.getInstance(MetricsCollectionService.class);

        try {
            program = injector.getInstance(ProgramFactory.class)
                    .create(cmdLine.getOptionValue(RunnableOptions.JAR));
        } catch (IOException e) {
            throw Throwables.propagate(e);
        }

        Arguments runtimeArguments = new Gson().fromJson(cmdLine.getOptionValue(RunnableOptions.RUNTIME_ARGS),
                BasicArguments.class);
        programOpts = new SimpleProgramOptions(name, createProgramArguments(context, configs),
                runtimeArguments);
        resourceReporter = new ProgramRunnableResourceReporter(program, metricsCollectionService, context);

        LOG.info("Runnable initialized: " + name);
    } catch (Throwable t) {
        LOG.error(t.getMessage(), t);
        throw Throwables.propagate(t);
    }
}

From source file:co.cask.tigon.internal.app.runtime.distributed.AbstractServiceTwillRunnable.java

License:Apache License

@Override
public final void initialize(TwillContext context) {
    super.initialize(context);

    name = context.getSpecification().getName();
    Map<String, String> configs = context.getSpecification().getConfigs();

    try {/*from www. j av a2  s  .  c  om*/
        // Load configuration
        hConf = new Configuration();
        hConf.clear();
        hConf.addResource(new File(configs.get("hConf")).toURI().toURL());

        UserGroupInformation.setConfiguration(hConf);

        cConf = CConfiguration.create();
        cConf.clear();
        cConf.addResource(new File(configs.get("cConf")).toURI().toURL());

        LOG.debug("{} cConf {}", name, cConf);
        LOG.debug("{} HBase conf {}", name, hConf);

        doInit(context);

        services = Lists.newArrayList();
        getServices(services);
        Preconditions.checkArgument(!services.isEmpty(), "Should have at least one service");

    } catch (Throwable t) {
        throw Throwables.propagate(t);
    }
}