Example usage for org.apache.hadoop.ipc RPC getProxy

Introduction

In this page you can find the example usage for org.apache.hadoop.ipc RPC getProxy.

Prototype

public static <T> T getProxy(Class<T> protocol, long clientVersion, InetSocketAddress addr,
        UserGroupInformation ticket, Configuration conf, SocketFactory factory) throws IOException 

Source Link

Document

Construct a client-side proxy object that implements the named protocol, talking to a server at the named address.

Usage

From source file:com.intel.hadoopRPCBenchmark.protocol.BenchmarkEngineProtocolClientSideTranslatorPB.java

License:Apache License

public BenchmarkEngineProtocolClientSideTranslatorPB(InetSocketAddress nameNodeAddr, Configuration conf)
        throws IOException {
    RPC.setProtocolEngine(conf, BenchmarkEngineProtocolPB.class, ProtobufRpcEngine.class);
    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
    rpcProxy = RPC.getProxy(BenchmarkEngineProtocolPB.class,
            RPC.getProtocolVersion(BenchmarkEngineProtocolPB.class), nameNodeAddr, ugi, conf,
            NetUtils.getSocketFactory(conf, BenchmarkEngineProtocolPB.class));
}

---

From source file:com.splunk.shuttl.prototype.symlink.HadoopFileLocationPrototypeTest.java

License:Apache License

/** Create {@link ClientDatanodeProtocol} proxy using kerberos ticket */
public static ClientDatanodeProtocol createClientDatanodeProtocolProxy(DatanodeID datanodeid,
        Configuration conf, int socketTimeout) throws IOException {
    InetSocketAddress addr = NetUtils.createSocketAddr(datanodeid.getHost() + ":" + datanodeid.getIpcPort());
    if (ClientDatanodeProtocol.LOG.isDebugEnabled()) {
        ClientDatanodeProtocol.LOG.info("ClientDatanodeProtocol addr=" + addr);
    }/* ww  w.  j av a2 s  .  c o m*/
    return (ClientDatanodeProtocol) RPC.getProxy(ClientDatanodeProtocol.class, ClientDatanodeProtocol.versionID,
            addr, conf, NetUtils.getDefaultSocketFactory(conf), socketTimeout);
}

---

From source file:com.sun.grid.herd.HerdLoadSensor.java

License:Open Source License

/**
 * Create a connection to the namenode.//from w  w  w  . java 2s  .  co m
 * @param conf the Hadoop configuration
 * @return a handle to the namenode
 * @throws IOException Thrown if there is an error while communicating
 * with the namenode.
 */
private static NamenodeProtocol createNamenode(Configuration conf) throws IOException {
    InetSocketAddress address = NameNode.getAddress(conf);
    RetryPolicy timeoutPolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(3, 200,
            TimeUnit.MILLISECONDS);
    Map<String, RetryPolicy> policyMap = Collections.singletonMap("getBlocks", timeoutPolicy);

    UserGroupInformation info = null;

    try {
        info = UnixUserGroupInformation.login(conf);
    } catch (LoginException e) {
        throw new IOException(StringUtils.stringifyException(e));
    }

    VersionedProtocol proxy = RPC.getProxy(NamenodeProtocol.class, NamenodeProtocol.versionID, address, info,
            conf, NetUtils.getDefaultSocketFactory(conf));
    NamenodeProtocol ret = (NamenodeProtocol) RetryProxy.create(NamenodeProtocol.class, proxy, policyMap);

    return ret;
}

---

From source file:org.apache.ambari.servicemonitor.utils.MonitorUtils.java

License:Apache License

public static AdminOperationsProtocol createJTAdminProxy(InetSocketAddress addr, Configuration conf)
        throws IOException {
    return (AdminOperationsProtocol) RPC.getProxy(AdminOperationsProtocol.class,
            AdminOperationsProtocol.versionID, addr, UserGroupInformation.getCurrentUser(), conf,
            NetUtils.getSocketFactory(conf, AdminOperationsProtocol.class));
}

---

From source file:org.apache.ratis.hadooprpc.Proxy.java

License:Apache License

public static <PROTOCOL> PROTOCOL getProxy(Class<PROTOCOL> clazz, String addressStr, Configuration conf)
        throws IOException {
    RPC.setProtocolEngine(conf, clazz, ProtobufRpcEngineShaded.class);
    return RPC.getProxy(clazz, RPC.getProtocolVersion(clazz),
            org.apache.ratis.util.NetUtils.createSocketAddr(addressStr), UserGroupInformation.getCurrentUser(),
            conf, NetUtils.getSocketFactory(conf, clazz));
}

---

From source file:skewtune.mapreduce.STJobTracker.java

License:Apache License

@SuppressWarnings("unchecked")
STJobTracker(final JobConf conf, String jobtrackerIndentifier) throws IOException, InterruptedException {
    // find the owner of the process
    // get the desired principal to load
    String keytabFilename = conf.get(JTConfig.JT_KEYTAB_FILE);
    UserGroupInformation.setConfiguration(conf);
    if (keytabFilename != null) {
        String desiredUser = conf.get(JTConfig.JT_USER_NAME, System.getProperty("user.name"));
        UserGroupInformation.loginUserFromKeytab(desiredUser, keytabFilename);
        mrOwner = UserGroupInformation.getLoginUser();
    } else {// w w w .j a va 2 s.c o  m
        mrOwner = UserGroupInformation.getCurrentUser();
    }

    supergroup = conf.get(MR_SUPERGROUP, "supergroup");
    LOG.info("Starting jobtracker with owner as " + mrOwner.getShortUserName() + " and supergroup as "
            + supergroup);

    long secretKeyInterval = conf.getLong(MRConfig.DELEGATION_KEY_UPDATE_INTERVAL_KEY,
            MRConfig.DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT);
    long tokenMaxLifetime = conf.getLong(MRConfig.DELEGATION_TOKEN_MAX_LIFETIME_KEY,
            MRConfig.DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT);
    long tokenRenewInterval = conf.getLong(MRConfig.DELEGATION_TOKEN_RENEW_INTERVAL_KEY,
            MRConfig.DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT);
    secretManager = new DelegationTokenSecretManager(secretKeyInterval, tokenMaxLifetime, tokenRenewInterval,
            DELEGATION_TOKEN_GC_INTERVAL);
    secretManager.startThreads();

    //
    // Grab some static constants
    //

    NUM_HEARTBEATS_IN_SECOND = conf.getInt(JT_HEARTBEATS_IN_SECOND, DEFAULT_NUM_HEARTBEATS_IN_SECOND);
    if (NUM_HEARTBEATS_IN_SECOND < MIN_NUM_HEARTBEATS_IN_SECOND) {
        NUM_HEARTBEATS_IN_SECOND = DEFAULT_NUM_HEARTBEATS_IN_SECOND;
    }

    HEARTBEATS_SCALING_FACTOR = conf.getFloat(JT_HEARTBEATS_SCALING_FACTOR, DEFAULT_HEARTBEATS_SCALING_FACTOR);
    if (HEARTBEATS_SCALING_FACTOR < MIN_HEARTBEATS_SCALING_FACTOR) {
        HEARTBEATS_SCALING_FACTOR = DEFAULT_HEARTBEATS_SCALING_FACTOR;
    }

    // whether to dump or not every heartbeat message even when DEBUG is enabled
    dumpHeartbeat = conf.getBoolean(JT_HEARTBEATS_DUMP, false);

    // This is a directory of temporary submission files. We delete it
    // on startup, and can delete any files that we're done with
    this.conf = conf;
    JobConf jobConf = new JobConf(conf);

    // Set ports, start RPC servers, setup security policy etc.
    InetSocketAddress addr = getAddress(conf);
    this.localMachine = addr.getHostName();
    this.port = addr.getPort();

    int handlerCount = conf.getInt(JT_IPC_HANDLER_COUNT, 10);
    this.interTrackerServer = RPC.getServer(SkewTuneClientProtocol.class, this, addr.getHostName(),
            addr.getPort(), handlerCount, false, conf, secretManager);
    if (LOG.isDebugEnabled()) {
        Properties p = System.getProperties();
        for (Iterator it = p.keySet().iterator(); it.hasNext();) {
            String key = (String) it.next();
            String val = p.getProperty(key);
            LOG.debug("Property '" + key + "' is " + val);
        }
    }

    InetSocketAddress infoSocAddr = NetUtils
            .createSocketAddr(conf.get(JT_HTTP_ADDRESS, String.format("%s:0", this.localMachine)));
    String infoBindAddress = infoSocAddr.getHostName();
    int tmpInfoPort = infoSocAddr.getPort();
    this.startTime = System.currentTimeMillis();
    infoServer = new HttpServer("job", infoBindAddress, tmpInfoPort, tmpInfoPort == 0, conf);
    infoServer.setAttribute("job.tracker", this);
    infoServer.addServlet("jobcompletion", "/completion", JobCompletionServlet.class);
    infoServer.addServlet("taskspeculation", "/speculation", SpeculationEventServlet.class);
    infoServer.addServlet("skewreport", "/skew", SkewReportServlet.class);
    infoServer.addServlet("tasksplit", "/split/*", SplitTaskServlet.class);
    infoServer.addServlet("tasksplitV2", "/splitV2/*", SplitTaskV2Servlet.class);
    infoServer.start();

    this.trackerIdentifier = jobtrackerIndentifier;

    // The rpc/web-server ports can be ephemeral ports...
    // ... ensure we have the correct info
    this.port = interTrackerServer.getListenerAddress().getPort();
    this.conf.set(JT_IPC_ADDRESS, (this.localMachine + ":" + this.port));
    LOG.info("JobTracker up at: " + this.port);
    this.infoPort = this.infoServer.getPort();
    this.conf.set(JT_HTTP_ADDRESS, infoBindAddress + ":" + this.infoPort);
    LOG.info("JobTracker webserver: " + this.infoServer.getPort());
    this.defaultNotificationUrl = String.format("http://%s:%d/completion?jobid=$jobId&status=$jobStatus",
            infoBindAddress, this.infoPort);
    LOG.info("JobTracker completion URI: " + defaultNotificationUrl);
    //        this.defaultSpeculationEventUrl = String.format("http://%s:%d/speculation?taskid=$taskId&remainTime=$taskRemainTime",infoBindAddress,this.infoPort);
    this.defaultSpeculationEventUrl = String.format("http://%s:%d/speculation?jobid=$jobId", infoBindAddress,
            this.infoPort);
    LOG.info("JobTracker speculation event URI: " + defaultSpeculationEventUrl);
    this.defaultSkewReportUrl = String.format("http://%s:%d/skew", infoBindAddress, this.infoPort);
    LOG.info("JobTracker skew report event URI: " + defaultSkewReportUrl);
    this.trackerHttp = String.format("http://%s:%d", infoBindAddress, this.infoPort);

    while (!Thread.currentThread().isInterrupted()) {
        try {
            // if we haven't contacted the namenode go ahead and do it
            if (fs == null) {
                fs = mrOwner.doAs(new PrivilegedExceptionAction<FileSystem>() {
                    @Override
                    public FileSystem run() throws IOException {
                        return FileSystem.get(conf);
                    }
                });
            }

            // clean up the system dir, which will only work if hdfs is out
            // of safe mode
            if (systemDir == null) {
                systemDir = new Path(getSystemDir());
            }
            try {
                FileStatus systemDirStatus = fs.getFileStatus(systemDir);
                if (!systemDirStatus.getOwner().equals(mrOwner.getShortUserName())) {
                    throw new AccessControlException(
                            "The systemdir " + systemDir + " is not owned by " + mrOwner.getShortUserName());
                }
                if (!systemDirStatus.getPermission().equals(SYSTEM_DIR_PERMISSION)) {
                    LOG.warn("Incorrect permissions on " + systemDir + ". Setting it to "
                            + SYSTEM_DIR_PERMISSION);
                    fs.setPermission(systemDir, new FsPermission(SYSTEM_DIR_PERMISSION));
                } else {
                    break;
                }
            } catch (FileNotFoundException fnf) {
            } // ignore
        } catch (AccessControlException ace) {
            LOG.warn("Failed to operate on " + JTConfig.JT_SYSTEM_DIR + "(" + systemDir
                    + ") because of permissions.");
            LOG.warn("Manually delete the " + JTConfig.JT_SYSTEM_DIR + "(" + systemDir
                    + ") and then start the JobTracker.");
            LOG.warn("Bailing out ... ");
            throw ace;
        } catch (IOException ie) {
            LOG.info("problem cleaning system directory: " + systemDir, ie);
        }
        Thread.sleep(FS_ACCESS_RETRY_PERIOD);
    }

    if (Thread.currentThread().isInterrupted()) {
        throw new InterruptedException();
    }

    // initialize cluster variable
    cluster = new Cluster(this.conf);

    // now create a job client proxy
    jtClient = (ClientProtocol) RPC.getProxy(ClientProtocol.class, ClientProtocol.versionID,
            JobTracker.getAddress(conf), mrOwner, this.conf,
            NetUtils.getSocketFactory(conf, ClientProtocol.class));

    new SpeculativeScheduler().start();

    // initialize task event fetcher
    new TaskCompletionEventFetcher().start();

    // Same with 'localDir' except it's always on the local disk.
    asyncDiskService = new MRAsyncDiskService(FileSystem.getLocal(conf), conf.getLocalDirs());
    asyncDiskService.moveAndDeleteFromEachVolume(SUBDIR);

    // keep at least one asynchronous worker per CPU core
    int numProcs = Runtime.getRuntime().availableProcessors();
    LOG.info("# of available processors = " + numProcs);
    int maxFactor = conf.getInt(JT_MAX_ASYNC_WORKER_FACTOR, 2);
    asyncWorkers = new ThreadPoolExecutor(numProcs, numProcs * maxFactor, 30, TimeUnit.SECONDS,
            new SynchronousQueue<Runnable>(true), new ThreadPoolExecutor.CallerRunsPolicy());

    speculativeSplit = conf.getBoolean(JT_SPECULATIVE_SPLIT, false);
}

---

From source file:uk.ac.gla.terrier.probos.job.ProbosJobService.java

License:Open Source License

@Override
public int run(String[] args) throws Exception {
    final String secret = System.getenv("PBS_SECRET");

    Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials();
    Iterator<Token<?>> iter = credentials.getAllTokens().iterator();
    LOG.info("Executing with tokens:");
    while (iter.hasNext()) {
        Token<?> token = iter.next();
        LOG.info(token.toString());//from   w ww . j ava 2  s  .  c  om
        if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) {
            iter.remove();
        }
    }

    SshServer sshd = getSSHServer(secret);

    sshd.start();
    running.set(true);

    int jobId = Integer.parseInt(System.getenv("PBS_JOBID"));
    String hostPort = System.getenv("PBS_CONTROLLER");
    String[] hostPortSplit = hostPort.split(":");
    final String serverHostname = hostPortSplit[0];
    final int serverPort = Integer.parseInt(hostPortSplit[1]);
    final InetSocketAddress server = new InetSocketAddress(serverHostname, serverPort);

    Configuration conf = this.getConf();
    PBSInteractiveClient client = RPC.getProxy(PBSInteractiveClient.class,
            RPC.getProtocolVersion(PBSInteractiveClient.class), server, UserGroupInformation.getCurrentUser(),
            conf, NetUtils.getDefaultSocketFactory(conf));

    LOG.info("Sister for " + jobId + " started on " + sshd.getPort() + " with secret " + secret);
    informController(secret, sshd.getPort(), jobId, client);
    while (running.get()) {
        Thread.sleep(1000);
    }
    LOG.info("Ssh terminated by running variable");
    sshd.stop(true);
    RPC.stopProxy(client);
    return 0;
}

---

From source file:uk.ac.gla.terrier.probos.master.ProbosApplicationMasterServiceImpl.java

License:Open Source License

@SuppressWarnings("unchecked")
public ProbosApplicationMasterServiceImpl(ApplicationMasterParameters parameters, Configuration _conf)
        throws Exception {
    super(parameters, _conf);
    LOG.info("Starting " + this.getClass().getSimpleName() + " on " + Utils.getHostname());

    Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials();
    probosTokens = new ArrayList<Token<ProbosDelegationTokenIdentifier>>();
    Iterator<Token<?>> iter = credentials.getAllTokens().iterator();
    LOG.info("Executing on " + Utils.getHostname() + " with tokens:");
    while (iter.hasNext()) {
        Token<?> token = iter.next();
        LOG.info(token.toString());/* w w  w  .  jav  a 2 s  . com*/
        if (token.getKind().equals(ProbosDelegationTokenIdentifier.KIND_NAME)) {
            probosTokens.add((Token<ProbosDelegationTokenIdentifier>) token);
        }
    }
    renewer = new ProbosTokenRenewer();

    this.conf = _conf;
    StringWriter sw = new StringWriter();
    Configuration.dumpConfiguration(conf, sw);
    //LOG.info("Master conf is " + sw.toString());

    for (String k : REQUIRED_ENV) {
        if (System.getenv(k) == null)
            throw new IllegalArgumentException("Env " + k + " must be set");
    }

    String hostPort = System.getenv("PBS_CONTROLLER");
    String[] hostPortSplit = hostPort.split(":");
    final String serverHostname = hostPortSplit[0];
    final int port = Integer.parseInt(hostPortSplit[1]);

    final InetSocketAddress server = new InetSocketAddress(serverHostname, port);
    masterClient = RPC.getProxy(PBSMasterClient.class, RPC.getProtocolVersion(PBSMasterClient.class), server,
            UserGroupInformation.getCurrentUser(), conf, NetUtils.getDefaultSocketFactory(conf));
    controllerClient = PBSClientFactory.getPBSClient();
    LOG.info("Connected to controller " + hostPort);

    jobId = Integer.parseInt(System.getenv("PBS_JOBID"));
    container = System.getenv("CONTAINER_ID");
    masterClient.jobEvent(jobId, EventType.MASTER_START, container, null);

    final List<Entry<String, HttpServlet>> masterServlets = new ArrayList<>();
    masterServlets.add(new MapEntry<String, HttpServlet>("/",
            new JobProgressServlet("./", masterServlets, controllerClient, this)));
    masterServlets.add(new MapEntry<String, HttpServlet>("/qstatjob",
            new QstatJobServlet("./qstatjob", masterServlets, controllerClient, this)));
    masterServlets.add(new MapEntry<String, HttpServlet>("/conf",
            new ConfServlet("./conf", masterServlets, controllerClient, this)));
    masterServlets.add(new MapEntry<String, HttpServlet>("/kittenconf",
            new KittenConfServlet("./kittenconf", masterServlets, controllerClient, this)));
    masterServlets.add(new MapEntry<String, HttpServlet>("/logs",
            new LogsServlet("./logs", masterServlets, controllerClient, this)));

    //0 means any random free port
    webServer = new WebServer("ProbosControllerHttp", masterServlets, 0);
    webServer.init(_conf);
}

---

From source file:uk.ac.gla.terrier.probos.PBSClientFactory.java

License:Open Source License

public static PBSClient getPBSClient() throws IOException {
    if (forcedClient != null)
        return forcedClient;

    final Configuration c = new Configuration();
    final PConfiguration pConf = new PConfiguration(c);
    String _serverHostname = pConf.get(PConfiguration.KEY_CONTROLLER_HOSTNAME);
    if (System.getenv("PBS_DEFAULT") != null) {
        _serverHostname = System.getenv("PBS_DEFAULT");
    }/*  www. j a  va  2  s  .c o m*/
    final String serverHostname = _serverHostname;
    LOG.debug("Connecting to server " + serverHostname);

    InetSocketAddress server = new InetSocketAddress(serverHostname,
            pConf.getInt(PConfiguration.KEY_CONTROLLER_PORT, 8027));
    LOG.debug("Connecting to server at address " + server.toString());
    PBSClient rtr = RPC.getProxy(PBSClient.class, RPC.getProtocolVersion(PBSClient.class), server,
            UserGroupInformation.getCurrentUser(), pConf, NetUtils.getDefaultSocketFactory(c));
    LOG.debug("Got RPC connection!");
    return rtr;
}

---