Example usage for org.apache.hadoop.security UserGroupInformation getCurrentUser

List of usage examples for org.apache.hadoop.security UserGroupInformation getCurrentUser

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.security.*
  5. UserGroupInformation
  6. getCurrentUser

Introduction

In this page you can find the example usage for org.apache.hadoop.security UserGroupInformation getCurrentUser.

Prototype

@InterfaceAudience.Public
@InterfaceStability.Evolving
public static UserGroupInformation getCurrentUser() throws IOException

Source Link

Document

Return the current user, including any doAs in the current stack.

Usage

From source file:tachyon.hadoop.TFSTest.java

License:Apache License

private void mockUserGroupInformation() throws IOException {
    // need to mock out since FileSystem.get calls UGI, which some times has issues on some systems
    PowerMockito.mockStatic(UserGroupInformation.class);
    final UserGroupInformation ugi = Mockito.mock(UserGroupInformation.class);
    Mockito.when(ugi.getCurrentUser()).thenReturn(ugi);
}

From source file:uk.ac.gla.terrier.probos.cli.pbsdsh.java

License:Open Source License

@Override
public int run(String[] args) throws Exception {

    Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials();
    Iterator<Token<?>> iter = credentials.getAllTokens().iterator();
    LOG.debug("Executing with tokens:");
    while (iter.hasNext()) {
        Token<?> token = iter.next();
        LOG.debug(token.toString());//from   ww w.  j a v  a  2  s . c  o m
        if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) {
            iter.remove();
        }
    }

    Options options = new Options();
    options.addOption("h", true, "Specify hostname.");
    CommandLineParser parser = new GnuParser();
    CommandLine cmd = parser.parse(options, args);
    final String[] dibcommand = cmd.getArgs();

    int jobId = getJobId();
    if (jobId == -1) {
        System.err.println("PBS: PBS_JOBID not set");
        return 1;
    }
    String[] hosts;

    PBSJobStatusDistributed d = (PBSJobStatusDistributed) c.getJobStatus(jobId, 5);
    if (d.getState() == '?') {
        System.err.println("PBS: Job " + jobId + " was lost");
        return -1;
    }
    if (d.getState() != 'R') {
        System.err.println("PBS: Job " + jobId + " was not running");
        return -1;
    }

    int[] ports;
    if (cmd.hasOption('h')) {
        hosts = cmd.getOptionValues('h');
        ports = new int[hosts.length];
        String[] tmpH = d.getHostnames();
        int[] tmpP = d.getPorts();
        TObjectIntHashMap<String> host2port = new TObjectIntHashMap<String>(tmpH.length);
        for (int i = 0; i < tmpH.length; i++) {
            host2port.put(tmpH[i], tmpP[i]);
        }
        int i = 0;
        for (String h : hosts) {
            if (!host2port.contains(h)) {
                throw new IllegalArgumentException("Host " + h + " is not a member of this distributed job");
            }
            ports[i++] = host2port.get(h);
        }

    } else {
        hosts = d.getHostnames();
        ports = d.getPorts();
    }

    final String secret = d.getSecret();
    if (secret == null)
        throw new IllegalArgumentException(
                "No secret found - pbsdsh called too early? " + Arrays.toString(d.getHostnames()));

    LOG.debug("To run on " + Arrays.toString(hosts));
    final CyclicBarrier barrier = new CyclicBarrier(1 + hosts.length);
    int i = 0;
    for (final String h : hosts) {
        final int port = ports[i++];
        new Thread() {
            @Override
            public void run() {
                try {
                    if (connectToSister(h, port, secret, dibcommand) != 0)
                        LOG.error("Could not connect");
                } catch (Exception e) {
                    LOG.error("Could not connect", e);
                } finally {
                    try {
                        barrier.await();
                    } catch (Exception e) {
                        LOG.error("Barrier problem?");
                    }
                }
            }
        }.start();
    }
    barrier.await();
    return 0;
}

From source file:uk.ac.gla.terrier.probos.controller.ControllerServer.java

License:Open Source License

public ControllerServer(Configuration _hconf) throws IOException {
    this.yConf = new YarnConfiguration(_hconf);
    yConf.addResource("yarn-site.xml");
    UserGroupInformation.setConfiguration(yConf);

    this.pConf = new PConfiguration(_hconf);

    //do the Kerberos authentication
    if (UserGroupInformation.isSecurityEnabled()) {
        final String principal = pConf.get(PConfiguration.KEY_CONTROLLER_PRINCIPAL);
        String keytab = pConf.get(PConfiguration.KEY_CONTROLLER_KEYTAB);
        File fKeytab = new File(keytab);
        if (!fKeytab.exists()) {
            if (!fKeytab.isAbsolute()) {
                keytab = System.getProperty("probos.conf") + '/' + keytab;
                fKeytab = new File(keytab);
                pConf.set(PConfiguration.KEY_CONTROLLER_KEYTAB, keytab);
            }//from  ww w . j av  a  2  s. c om
            if (!fKeytab.exists())
                throw new FileNotFoundException("Could not find keytab file " + keytab);
        }

        LOG.debug("Starting login for " + principal + " using keytab " + keytab);
        SecurityUtil.login(pConf, PConfiguration.KEY_CONTROLLER_KEYTAB, PConfiguration.KEY_CONTROLLER_PRINCIPAL,
                Utils.getHostname());
        LOG.info("Switched principal to " + UserGroupInformation.getCurrentUser().getUserName());
    }

    this.mClient = MailClient.getMailClient(this.pConf);
    final String bindAddress = pConf.get(PConfiguration.KEY_CONTROLLER_BIND_ADDRESS);
    if (bindAddress == null)
        throw new IllegalArgumentException(PConfiguration.KEY_CONTROLLER_BIND_ADDRESS + " cannot be null");

    secretManager = new ControllerAPISecretManager(
            //delegationKeyUpdateInterval
            //renewal interval for delegation token
            7 * 24 * 3600 * 1000, //Yarn default is 7 day

            //delegationTokenMaxLifetime -- maximum lifetime for which a delegation token is valid
            //i.e. how long can we keep renewing the token for?
            14 * 24 * 3600 * 1000, //Yarn default is 14 days

            //delegationTokenRenewInterval -- how long should a token last?
            7 * 24 * 3600 * 1000, //Yarn default is 7 day

            //delegationTokenRemoverScanInterval -- how often are expired keys removed?
            3600 * 1000); //Yarn default is 1 hour

    //build the client rpc server: 8027
    int port = pConf.getInt(PConfiguration.KEY_CONTROLLER_PORT, 8027);
    LOG.info("Starting RPC server for " + PBSClient.class.getSimpleName() + " on port " + port);
    clientRpcserver = new RPC.Builder(yConf).setInstance(this).setBindAddress(bindAddress)
            .setProtocol(PBSClient.class).setPort(port).setSecretManager(secretManager).
            //setVerbose(true).
            build();
    System.setProperty("hadoop.policy.file", Constants.PRODUCT_NAME + "-policy.xml");
    clientRpcserver.refreshServiceAclWithLoadedConfiguration(yConf, new ControllerPolicyProvider());

    //build the master rpc server: 8028
    port = Constants.CONTROLLER_MASTER_PORT_OFFSET + pConf.getInt(PConfiguration.KEY_CONTROLLER_PORT, 8027);
    LOG.info("Starting RPC server for " + PBSMasterClient.class.getSimpleName() + " on port " + port);
    masterRpcserver = new RPC.Builder(yConf).setInstance(new ApplicationMasterAPI()).setBindAddress(bindAddress)
            .setProtocol(PBSMasterClient.class).setPort(port).setSecretManager(secretManager).
            //setVerbose(true).
            build();
    masterRpcserver.refreshServiceAclWithLoadedConfiguration(yConf, new ControllerPolicyProvider());

    port = Constants.CONTROLLER_INTERACTIVE_PORT_OFFSET
            + pConf.getInt(PConfiguration.KEY_CONTROLLER_PORT, 8027);
    LOG.info("Starting RPC server for " + PBSInteractiveClient.class.getSimpleName() + " on port " + port);
    //build the interactive rpc server: 8026
    interactiveRpcserver = new RPC.Builder(yConf).setInstance(new InteractiveTaskAPI())
            .setBindAddress(bindAddress).setProtocol(PBSInteractiveClient.class).setPort(port)
            .setSecretManager(secretManager).
            //setVerbose(true).
            build();
    interactiveRpcserver.refreshServiceAclWithLoadedConfiguration(yConf, new ControllerPolicyProvider());

    //build the webapp UI server
    final List<Entry<String, HttpServlet>> controllerServlets = new ArrayList<>();
    controllerServlets
            .add(new MapEntry<String, HttpServlet>("/", new QstatServlet("/", controllerServlets, this)));
    controllerServlets.add(
            new MapEntry<String, HttpServlet>("/pbsnodes", new PbsnodesServlet("/", controllerServlets, this)));
    //metrics is the Servlet from metrics.dropwizard for accessing metrics
    controllerServlets.add(new MapEntry<String, HttpServlet>("/metrics", new MetricsServlet(metrics)));
    //this is the hadoop servlet for accessing anything defined in JMX
    controllerServlets.add(new MapEntry<String, HttpServlet>("/jmx", new JMXJsonServlet()));
    final int httpport = pConf.getInt(PConfiguration.KEY_CONTROLLER_HTTP_PORT,
            Constants.DEFAULT_CONTROLLER_PORT + Constants.CONTROLLER_HTTP_PORT_OFFSET);
    LOG.info("Starting Jetty ProbosControllerHttp on port " + httpport);
    webServer = new WebServer("ProbosControllerHttp", controllerServlets, httpport);
    webServer.init(pConf);

    //this thread detects yarn jobs that have ended
    watcherThread = new Thread(new ControllerWatcher());
    watcherThread.setName(ControllerWatcher.class.getSimpleName());

    //ensure we have the directory
    Path _probosFolder = new Path(pConf.get(PConfiguration.KEY_CONTROLLER_JOBDIR));
    FileSystem controllerFS = FileSystem.get(yConf);
    if (!_probosFolder.isUriPathAbsolute()) {
        _probosFolder = _probosFolder.makeQualified(controllerFS.getUri(), controllerFS.getWorkingDirectory());
        assert _probosFolder.isUriPathAbsolute();
    }
    probosFolder = _probosFolder;
    if (!controllerFS.exists(probosFolder)) {
        throw new IllegalArgumentException(probosFolder.toString() + " does not exist");
    }

    //now initialise the metrics

    //jobs.queued.size
    metrics.register(MetricRegistry.name(ControllerServer.class, "jobs", "queued.size"), new Gauge<Integer>() {
        @Override
        public Integer getValue() {
            int sum = 0;
            for (int i : user2QueuedCount.values())
                sum += i;
            return sum;
        }
    });
    //jobs.size
    metrics.register(MetricRegistry.name(ControllerServer.class, "jobs", "size"), new Gauge<Integer>() {
        @Override
        public Integer getValue() {
            return jobArray.size();
        }
    });
    //jobs.held.size
    metrics.register(MetricRegistry.name(ControllerServer.class, "jobs", "held.size"), new Gauge<Integer>() {
        @Override
        public Integer getValue() {
            return jobHolds.size();
        }
    });

    //nodes.size
    metrics.register(MetricRegistry.name(ControllerServer.class, "nodes", "size"), new Gauge<Integer>() {
        @Override
        public Integer getValue() {
            try {
                return getNodesStatus().length;
            } catch (Exception e) {
                return 0;
            }
        }
    });

    //nodes.free.size
    metrics.register(MetricRegistry.name(ControllerServer.class, "nodes", "free.size"), new Gauge<Integer>() {
        @Override
        public Integer getValue() {
            try {
                PBSNodeStatus[] nodes = getNodesStatus();
                int count = 0;
                for (PBSNodeStatus n : nodes)
                    if ("free".equals(n.getState()))
                        count++;
                return count;
            } catch (Exception e) {
                return 0;
            }
        }
    });

    runningJobs = metrics.counter(MetricRegistry.name(ControllerServer.class, "jobs", "running.counter"));
    rejectedJobs = metrics.counter(MetricRegistry.name(ControllerServer.class, "jobs", "rejected.counter"));
    killedJobs = metrics.counter(MetricRegistry.name(ControllerServer.class, "jobs", "killed.counter"));
    mailEvents = metrics.counter(MetricRegistry.name(ControllerServer.class, "mails", "counter"));
    mailFailures = metrics.counter(MetricRegistry.name(ControllerServer.class, "mails", "failure.counter"));

}

From source file:uk.ac.gla.terrier.probos.job.ProbosJobService.java

License:Open Source License

@Override
public int run(String[] args) throws Exception {
    final String secret = System.getenv("PBS_SECRET");

    Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials();
    Iterator<Token<?>> iter = credentials.getAllTokens().iterator();
    LOG.info("Executing with tokens:");
    while (iter.hasNext()) {
        Token<?> token = iter.next();
        LOG.info(token.toString());// ww  w .ja  v a2s .  c  o m
        if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) {
            iter.remove();
        }
    }

    SshServer sshd = getSSHServer(secret);

    sshd.start();
    running.set(true);

    int jobId = Integer.parseInt(System.getenv("PBS_JOBID"));
    String hostPort = System.getenv("PBS_CONTROLLER");
    String[] hostPortSplit = hostPort.split(":");
    final String serverHostname = hostPortSplit[0];
    final int serverPort = Integer.parseInt(hostPortSplit[1]);
    final InetSocketAddress server = new InetSocketAddress(serverHostname, serverPort);

    Configuration conf = this.getConf();
    PBSInteractiveClient client = RPC.getProxy(PBSInteractiveClient.class,
            RPC.getProtocolVersion(PBSInteractiveClient.class), server, UserGroupInformation.getCurrentUser(),
            conf, NetUtils.getDefaultSocketFactory(conf));

    LOG.info("Sister for " + jobId + " started on " + sshd.getPort() + " with secret " + secret);
    informController(secret, sshd.getPort(), jobId, client);
    while (running.get()) {
        Thread.sleep(1000);
    }
    LOG.info("Ssh terminated by running variable");
    sshd.stop(true);
    RPC.stopProxy(client);
    return 0;
}

From source file:uk.ac.gla.terrier.probos.master.ProbosApplicationMasterServiceImpl.java

License:Open Source License

@SuppressWarnings("unchecked")
public ProbosApplicationMasterServiceImpl(ApplicationMasterParameters parameters, Configuration _conf)
        throws Exception {
    super(parameters, _conf);
    LOG.info("Starting " + this.getClass().getSimpleName() + " on " + Utils.getHostname());

    Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials();
    probosTokens = new ArrayList<Token<ProbosDelegationTokenIdentifier>>();
    Iterator<Token<?>> iter = credentials.getAllTokens().iterator();
    LOG.info("Executing on " + Utils.getHostname() + " with tokens:");
    while (iter.hasNext()) {
        Token<?> token = iter.next();
        LOG.info(token.toString());/*  ww w . jav a2 s .  c  om*/
        if (token.getKind().equals(ProbosDelegationTokenIdentifier.KIND_NAME)) {
            probosTokens.add((Token<ProbosDelegationTokenIdentifier>) token);
        }
    }
    renewer = new ProbosTokenRenewer();

    this.conf = _conf;
    StringWriter sw = new StringWriter();
    Configuration.dumpConfiguration(conf, sw);
    //LOG.info("Master conf is " + sw.toString());

    for (String k : REQUIRED_ENV) {
        if (System.getenv(k) == null)
            throw new IllegalArgumentException("Env " + k + " must be set");
    }

    String hostPort = System.getenv("PBS_CONTROLLER");
    String[] hostPortSplit = hostPort.split(":");
    final String serverHostname = hostPortSplit[0];
    final int port = Integer.parseInt(hostPortSplit[1]);

    final InetSocketAddress server = new InetSocketAddress(serverHostname, port);
    masterClient = RPC.getProxy(PBSMasterClient.class, RPC.getProtocolVersion(PBSMasterClient.class), server,
            UserGroupInformation.getCurrentUser(), conf, NetUtils.getDefaultSocketFactory(conf));
    controllerClient = PBSClientFactory.getPBSClient();
    LOG.info("Connected to controller " + hostPort);

    jobId = Integer.parseInt(System.getenv("PBS_JOBID"));
    container = System.getenv("CONTAINER_ID");
    masterClient.jobEvent(jobId, EventType.MASTER_START, container, null);

    final List<Entry<String, HttpServlet>> masterServlets = new ArrayList<>();
    masterServlets.add(new MapEntry<String, HttpServlet>("/",
            new JobProgressServlet("./", masterServlets, controllerClient, this)));
    masterServlets.add(new MapEntry<String, HttpServlet>("/qstatjob",
            new QstatJobServlet("./qstatjob", masterServlets, controllerClient, this)));
    masterServlets.add(new MapEntry<String, HttpServlet>("/conf",
            new ConfServlet("./conf", masterServlets, controllerClient, this)));
    masterServlets.add(new MapEntry<String, HttpServlet>("/kittenconf",
            new KittenConfServlet("./kittenconf", masterServlets, controllerClient, this)));
    masterServlets.add(new MapEntry<String, HttpServlet>("/logs",
            new LogsServlet("./logs", masterServlets, controllerClient, this)));

    //0 means any random free port
    webServer = new WebServer("ProbosControllerHttp", masterServlets, 0);
    webServer.init(_conf);
}

From source file:uk.ac.gla.terrier.probos.PBSClientFactory.java

License:Open Source License

public static PBSClient getPBSClient() throws IOException {
    if (forcedClient != null)
        return forcedClient;

    final Configuration c = new Configuration();
    final PConfiguration pConf = new PConfiguration(c);
    String _serverHostname = pConf.get(PConfiguration.KEY_CONTROLLER_HOSTNAME);
    if (System.getenv("PBS_DEFAULT") != null) {
        _serverHostname = System.getenv("PBS_DEFAULT");
    }//  ww w. jav  a2  s.c o m
    final String serverHostname = _serverHostname;
    LOG.debug("Connecting to server " + serverHostname);

    InetSocketAddress server = new InetSocketAddress(serverHostname,
            pConf.getInt(PConfiguration.KEY_CONTROLLER_PORT, 8027));
    LOG.debug("Connecting to server at address " + server.toString());
    PBSClient rtr = RPC.getProxy(PBSClient.class, RPC.getProtocolVersion(PBSClient.class), server,
            UserGroupInformation.getCurrentUser(), pConf, NetUtils.getDefaultSocketFactory(c));
    LOG.debug("Got RPC connection!");
    return rtr;
}

From source file:uk.gov.gchq.gaffer.slider.util.SliderUtils.java

License:Apache License

public static void replaceTokens(final ConfTree appConfig, final String clusterName) throws IOException {
    // Replace ${USER} and ${USER_NAME}
    SliderClient.replaceTokens(appConfig, UserGroupInformation.getCurrentUser().getShortUserName(),
            clusterName);//from  ww w .  ja v a2  s  .  c o  m
    // Replace ${CLUSTER_NAME}
    replaceClusterTokens(appConfig, clusterName);
}

From source file:voldemort.store.readonly.fetcher.HdfsFetcher.java

License:Apache License

public File fetch(String sourceFileUrl, String destinationFile, String hadoopConfigPath) throws IOException {
    if (this.globalThrottleLimit != null) {
        if (this.globalThrottleLimit.getSpeculativeRate() < this.minBytesPerSecond)
            throw new VoldemortException("Too many push jobs.");
        this.globalThrottleLimit.incrementNumJobs();
    }/*from   ww  w  . j ava2 s . co  m*/

    ObjectName jmxName = null;
    try {

        final Configuration config = new Configuration();
        FileSystem fs = null;
        config.setInt("io.socket.receive.buffer", bufferSize);
        config.set("hadoop.rpc.socket.factory.class.ClientProtocol", ConfigurableSocketFactory.class.getName());
        config.set("hadoop.security.group.mapping", "org.apache.hadoop.security.ShellBasedUnixGroupsMapping");

        final Path path = new Path(sourceFileUrl);

        boolean isHftpBasedFetch = sourceFileUrl.length() > 4 && sourceFileUrl.substring(0, 4).equals("hftp");
        logger.info("URL : " + sourceFileUrl + " and hftp protocol enabled = " + isHftpBasedFetch);
        logger.info("Hadoop path = " + hadoopConfigPath + " , keytab path = " + HdfsFetcher.keytabPath
                + " , kerberos principal = " + HdfsFetcher.kerberosPrincipal);

        if (hadoopConfigPath.length() > 0 && !isHftpBasedFetch) {

            config.addResource(new Path(hadoopConfigPath + "/core-site.xml"));
            config.addResource(new Path(hadoopConfigPath + "/hdfs-site.xml"));

            String security = config.get(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION);

            if (security == null || !security.equals("kerberos")) {
                logger.error("Security isn't turned on in the conf: "
                        + CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION + " = "
                        + config.get(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION));
                logger.error("Please make sure that the Hadoop config directory path is valid.");
                return null;
            } else {
                logger.info("Security is turned on in the conf. Trying to authenticate ...");

            }
        }

        try {

            if (HdfsFetcher.keytabPath.length() > 0 && !isHftpBasedFetch) {

                if (!new File(HdfsFetcher.keytabPath).exists()) {
                    logger.error("Invalid keytab file path. Please provide a valid keytab path");
                    return null;
                }

                // First login using the specified principal and keytab file
                UserGroupInformation.setConfiguration(config);
                UserGroupInformation.loginUserFromKeytab(HdfsFetcher.kerberosPrincipal, HdfsFetcher.keytabPath);

                /*
                 * If login is successful, get the filesystem object. NOTE:
                 * Ideally we do not need a doAs block for this. Consider
                 * removing it in the future once the Hadoop jars have the
                 * corresponding patch (tracked in the Hadoop Apache
                 * project: HDFS-3367)
                 */
                try {
                    logger.info("I've logged in and am now Doasing as "
                            + UserGroupInformation.getCurrentUser().getUserName());
                    fs = UserGroupInformation.getCurrentUser()
                            .doAs(new PrivilegedExceptionAction<FileSystem>() {

                                public FileSystem run() throws Exception {
                                    FileSystem fs = path.getFileSystem(config);
                                    return fs;
                                }
                            });
                } catch (InterruptedException e) {
                    logger.error(e.getMessage());
                } catch (Exception e) {
                    logger.error("Got an exception while getting the filesystem object: ");
                    logger.error("Exception class : " + e.getClass());
                    e.printStackTrace();
                    for (StackTraceElement et : e.getStackTrace()) {
                        logger.error(et.toString());
                    }
                }
            } else {
                fs = path.getFileSystem(config);
            }

        } catch (IOException e) {
            e.printStackTrace();
            logger.error("Error in authenticating or getting the Filesystem object !!!");
            return null;
        }

        CopyStats stats = new CopyStats(sourceFileUrl, sizeOfPath(fs, path));
        jmxName = JmxUtils.registerMbean("hdfs-copy-" + copyCount.getAndIncrement(), stats);
        File destination = new File(destinationFile);

        if (destination.exists()) {
            throw new VoldemortException(
                    "Version directory " + destination.getAbsolutePath() + " already exists");
        }

        logger.info("Starting fetch for : " + sourceFileUrl);
        boolean result = fetch(fs, path, destination, stats);
        logger.info("Completed fetch : " + sourceFileUrl);

        // Close the filesystem
        fs.close();

        if (result) {
            return destination;
        } else {
            return null;
        }
    } catch (IOException e) {
        logger.error("Error while getting Hadoop filesystem : " + e);
        return null;
    } finally {
        if (this.globalThrottleLimit != null) {
            this.globalThrottleLimit.decrementNumJobs();
        }
        if (jmxName != null)
            JmxUtils.unregisterMbean(jmxName);
    }
}

From source file:voldemort.store.readonly.fetcher.HdfsFetcher.java

License:Apache License

public static void main(String[] args) throws Exception {
    if (args.length < 1)
        Utils.croak("USAGE: java " + HdfsFetcher.class.getName()
                + " url [keytab location] [kerberos username] [hadoop-config-path]");
    String url = args[0];/*from   w  ww.ja  v a 2s . c  o  m*/

    String keytabLocation = "";
    String kerberosUser = "";
    String hadoopPath = "";
    if (args.length == 4) {
        keytabLocation = args[1];
        kerberosUser = args[2];
        hadoopPath = args[3];
    }

    long maxBytesPerSec = 1024 * 1024 * 1024;
    Path p = new Path(url);

    final Configuration config = new Configuration();
    final URI uri = new URI(url);
    config.setInt("io.file.buffer.size", VoldemortConfig.DEFAULT_BUFFER_SIZE);
    config.set("hadoop.rpc.socket.factory.class.ClientProtocol", ConfigurableSocketFactory.class.getName());
    config.setInt("io.socket.receive.buffer", 1 * 1024 * 1024 - 10000);

    FileSystem fs = null;
    p = new Path(url);
    HdfsFetcher.keytabPath = keytabLocation;
    HdfsFetcher.kerberosPrincipal = kerberosUser;

    boolean isHftpBasedFetch = url.length() > 4 && url.substring(0, 4).equals("hftp");
    logger.info("URL : " + url + " and hftp protocol enabled = " + isHftpBasedFetch);

    if (hadoopPath.length() > 0 && !isHftpBasedFetch) {
        config.set("hadoop.security.group.mapping", "org.apache.hadoop.security.ShellBasedUnixGroupsMapping");

        config.addResource(new Path(hadoopPath + "/core-site.xml"));
        config.addResource(new Path(hadoopPath + "/hdfs-site.xml"));

        String security = config.get(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION);

        if (security == null || !security.equals("kerberos")) {
            logger.info("Security isn't turned on in the conf: "
                    + CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION + " = "
                    + config.get(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION));
            logger.info("Fix that.  Exiting.");
            return;
        } else {
            logger.info("Security is turned on in the conf. Trying to authenticate ...");
        }
    }

    try {

        // Get the filesystem object
        if (keytabLocation.length() > 0 && !isHftpBasedFetch) {
            UserGroupInformation.setConfiguration(config);
            UserGroupInformation.loginUserFromKeytab(kerberosUser, keytabLocation);

            final Path path = p;
            try {
                logger.debug("I've logged in and am now Doasing as "
                        + UserGroupInformation.getCurrentUser().getUserName());
                fs = UserGroupInformation.getCurrentUser().doAs(new PrivilegedExceptionAction<FileSystem>() {

                    public FileSystem run() throws Exception {
                        FileSystem fs = path.getFileSystem(config);
                        return fs;
                    }
                });
            } catch (InterruptedException e) {
                logger.error(e.getMessage());
            } catch (Exception e) {
                logger.error("Got an exception while getting the filesystem object: ");
                logger.error("Exception class : " + e.getClass());
                e.printStackTrace();
                for (StackTraceElement et : e.getStackTrace()) {
                    logger.error(et.toString());
                }
            }
        } else {
            fs = p.getFileSystem(config);
        }

    } catch (IOException e) {
        e.printStackTrace();
        System.err.println("Error in getting Hadoop filesystem object !!! Exiting !!!");
        System.exit(-1);
    }

    FileStatus status = fs.listStatus(p)[0];
    long size = status.getLen();
    HdfsFetcher fetcher = new HdfsFetcher(null, maxBytesPerSec, VoldemortConfig.REPORTING_INTERVAL_BYTES,
            VoldemortConfig.DEFAULT_BUFFER_SIZE, 0, keytabLocation, kerberosUser);
    long start = System.currentTimeMillis();

    File location = fetcher.fetch(url, System.getProperty("java.io.tmpdir") + File.separator + start,
            hadoopPath);

    double rate = size * Time.MS_PER_SECOND / (double) (System.currentTimeMillis() - start);
    NumberFormat nf = NumberFormat.getInstance();
    nf.setMaximumFractionDigits(2);
    System.out.println(
            "Fetch to " + location + " completed: " + nf.format(rate / (1024.0 * 1024.0)) + " MB/sec.");
    fs.close();
}

From source file:wherehows.SchemaFetch.java

License:Open Source License

private static void scanPathHelper(Path path, FileSystem scanFs)
        throws IOException, InterruptedException, SQLException {
    String curPath = path.toUri().getPath();
    Path n = path;//from www. j a v  a 2  s .  co m
    if (path.getName().matches("^(\\.|_|tmp|temp|test|trash|backup|archive|ARCHIVE|storkinternal).*"))
        return;

    logger.info("  -- scanPath(" + curPath + ")\n");
    int x = isTable(path, scanFs);
    if (x > 0) {
        // System.err.println("  traceTable(" + path.toString() + ")");
        traceTableInfo(path, scanFs);
    } else if (x == 0) { // iterate over each table
        // FileStatus[] fslist = scanFs.listStatus(path);
        // System.err.println(" => " + fslist.length + " subdirs");
        for (FileStatus fstat : scanFs.listStatus(path)) {
            n = fstat.getPath();
            curPath = n.toUri().getPath();
            // System.err.println("  traceSubDir(" + curPath + ")");
            if (n == path) {
                continue;
            }
            try {
                if (isTable(n, scanFs) > 0) {
                    traceTableInfo(n, scanFs);
                } else if (scanFs.listStatus(n).length > 0 || scanFs.getContentSummary(n).getLength() > 0) {
                    scanPath(n, scanFs);
                } else {
                    logger.info("* scanPath() size = 0: " + curPath);
                }
            } catch (AccessControlException e) {
                logger.error("* scanPath(e) Permission denied. Cannot access: " + curPath + " owner:"
                        + fstat.getOwner() + " group: " + fstat.getGroup() + "with current user "
                        + UserGroupInformation.getCurrentUser());
                // System.err.println(e);
                continue;
            } // catch
        } // end of for
    } // end else
}