Example usage for org.apache.hadoop.security UserGroupInformation getCurrentUser

List of usage examples for org.apache.hadoop.security UserGroupInformation getCurrentUser

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.security.*
  5. UserGroupInformation
  6. getCurrentUser

Introduction

In this page you can find the example usage for org.apache.hadoop.security UserGroupInformation getCurrentUser.

Prototype

@InterfaceAudience.Public
@InterfaceStability.Evolving
public static UserGroupInformation getCurrentUser() throws IOException

Source Link

Document

Return the current user, including any doAs in the current stack.

Usage

From source file:common.NameNode.java

License:Apache License

/** {@inheritDoc} */
public boolean mkdirs(String src, FsPermission masked, boolean createParent) throws IOException {
    stateChangeLog.debug("*DIR* NameNode.mkdirs: " + src);
    if (!checkPathLength(src)) {
        throw new IOException("mkdirs: Pathname too long.  Limit " + MAX_PATH_LENGTH + " characters, "
                + MAX_PATH_DEPTH + " levels.");
    }/*from   w w w  . j a v  a2s. c o m*/
    return namesystem.mkdirs(src,
            new PermissionStatus(UserGroupInformation.getCurrentUser().getShortUserName(), null, masked),
            createParent);
}

From source file:common.NameNode.java

License:Apache License

/** @inheritDoc */
public void createSymlink(String target, String link, FsPermission dirPerms, boolean createParent)
        throws IOException {
    myMetrics.numcreateSymlinkOps.inc();
    /* We enforce the MAX_PATH_LENGTH limit even though a symlink target 
     * URI may refer to a non-HDFS file system. 
     *///from   ww  w  .  j a  v a 2s . c  o m
    if (!checkPathLength(link)) {
        throw new IOException("Symlink path exceeds " + MAX_PATH_LENGTH + " character limit");

    }
    if ("".equals(target)) {
        throw new IOException("Invalid symlink target");
    }
    final UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
    namesystem.createSymlink(target, link, new PermissionStatus(ugi.getUserName(), null, dirPerms),
            createParent);
}

From source file:common.NameNode.java

License:Apache License

@Override
public void refreshUserToGroupsMappings(Configuration conf) throws IOException {
    LOG.info("Refreshing all user-to-groups mappings. Requested by user: "
            + UserGroupInformation.getCurrentUser().getShortUserName());
    Groups.getUserToGroupsMappingService(conf).refresh();
}

From source file:de.huberlin.wbi.hiway.am.HiWay.java

License:Apache License

/**
 * Main run function for the application master
 * //from ww w .  ja v  a 2s.com
 * @return True if there were no errors
 * @throws YarnException
 *             YarnException
 * @throws IOException
 *             IOException
 */
@SuppressWarnings("unchecked")
public boolean run() throws YarnException, IOException {
    System.out.println("Starting ApplicationMaster");

    Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials();
    try (DataOutputBuffer dob = new DataOutputBuffer()) {
        credentials.writeTokenStorageToStream(dob);
        // Now remove the AM->RM token so that containers cannot access it.
        Iterator<Token<?>> iter = credentials.getAllTokens().iterator();
        while (iter.hasNext()) {
            Token<?> token = iter.next();
            if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) {
                iter.remove();
            }
        }
        allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());

        allocListener = new RMCallbackHandler(this);
        amRMClient = AMRMClientAsync.createAMRMClientAsync(1000, allocListener);
        amRMClient.init(conf);
        amRMClient.start();

        containerListener = new NMCallbackHandler(this);
        nmClientAsync = new NMClientAsyncImpl(containerListener);
        nmClientAsync.init(conf);
        nmClientAsync.start();

        Data workflowData = new Data(workflowPath);
        workflowData.stageIn();

        // Register self with ResourceManager. This will start heartbeating to the RM.
        appMasterHostname = NetUtils.getHostname();
        RegisterApplicationMasterResponse response = amRMClient.registerApplicationMaster(appMasterHostname,
                appMasterRpcPort, appMasterTrackingUrl);

        switch (schedulerName) {
        case staticRoundRobin:
        case heft:
            scheduler = schedulerName.equals(HiWayConfiguration.HIWAY_SCHEDULER_OPTS.staticRoundRobin)
                    ? new RoundRobin(getWorkflowName(), hdfs, conf)
                    : new HEFT(getWorkflowName(), hdfs, conf);
            break;
        case greedyQueue:
            scheduler = new GreedyQueue(getWorkflowName(), conf, hdfs);
            break;
        default:
            C3PO c3po = new C3PO(getWorkflowName(), hdfs, conf);
            switch (schedulerName) {
            case conservative:
                c3po.setConservatismWeight(12d);
                c3po.setnClones(0);
                c3po.setPlacementAwarenessWeight(0.01d);
                c3po.setOutlookWeight(0.01d);
                break;
            case cloning:
                c3po.setConservatismWeight(0.01d);
                c3po.setnClones(1);
                c3po.setPlacementAwarenessWeight(0.01d);
                c3po.setOutlookWeight(0.01d);
                break;
            case placementAware:
                c3po.setConservatismWeight(0.01d);
                c3po.setnClones(0);
                c3po.setPlacementAwarenessWeight(12d);
                c3po.setOutlookWeight(0.01d);
                break;
            case outlooking:
                c3po.setConservatismWeight(0.01d);
                c3po.setnClones(0);
                c3po.setPlacementAwarenessWeight(0.01d);
                c3po.setOutlookWeight(12d);
                break;
            default:
                c3po.setConservatismWeight(3d);
                c3po.setnClones(2);
                c3po.setPlacementAwarenessWeight(1d);
                c3po.setOutlookWeight(2d);
            }
            scheduler = c3po;
        }

        scheduler.initialize();
        writeEntryToLog(new JsonReportEntry(getRunId(), null, null, null, null, null, HiwayDBI.KEY_WF_NAME,
                getWorkflowName()));
        parseWorkflow();
        scheduler.updateRuntimeEstimates(getRunId().toString());
        federatedReport = new Data(appId + ".log");

        // Dump out information about cluster capability as seen by the resource manager
        int maxMem = response.getMaximumResourceCapability().getMemory();
        int maxCores = response.getMaximumResourceCapability().getVirtualCores();
        System.out.println("Max mem capabililty of resources in this cluster " + maxMem);

        // A resource ask cannot exceed the max.
        if (containerMemory > maxMem) {
            System.out.println("Container memory specified above max threshold of cluster."
                    + " Using max value." + ", specified=" + containerMemory + ", max=" + maxMem);
            containerMemory = maxMem;
        }
        if (containerCores > maxCores) {
            System.out.println("Container vcores specified above max threshold of cluster."
                    + " Using max value." + ", specified=" + containerCores + ", max=" + maxCores);
            containerCores = maxCores;
        }

        while (!done) {
            try {
                while (scheduler.hasNextNodeRequest()) {
                    ContainerRequest containerAsk = setupContainerAskForRM(scheduler.getNextNodeRequest());
                    amRMClient.addContainerRequest(containerAsk);
                }
                Thread.sleep(1000);
                System.out.println("Current application state: requested=" + numRequestedContainers
                        + ", completed=" + numCompletedContainers + ", failed=" + numFailedContainers
                        + ", killed=" + numKilledContainers + ", allocated=" + numAllocatedContainers);
            } catch (InterruptedException e) {
                e.printStackTrace();
                System.exit(-1);
            }
        }
        finish();
    } catch (Exception e) {
        e.printStackTrace();
        System.exit(-1);
    }
    return success;
}

From source file:de.huberlin.wbi.hiway.am.WorkflowDriver.java

License:Apache License

/**
 * Main run function for the application master. Does more initialization (sic!).
 * Calls the abstract {@link #parseWorkflow()}, then {@link #executeWorkflow()} and finally {@link #finish()}.
 * @return True if there were no errors/*www .  j a va2s.com*/
 */
protected boolean run() throws IOException {
    /* log */ Logger.writeToStdout("Starting ApplicationMaster");

    Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials();

    try (DataOutputBuffer dob = new DataOutputBuffer()) {

        credentials.writeTokenStorageToStream(dob);
        // remove the AM->RM token so that containers cannot access it.
        Iterator<Token<?>> iter = credentials.getAllTokens().iterator();
        while (iter.hasNext()) {
            Token<?> token = iter.next();
            if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) {
                iter.remove();
            }
        }
        allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());

        // Resource Manager communications setup
        RMCallbackHandler allocListener = new RMCallbackHandler(this);
        amRMClient = AMRMClientAsync.createAMRMClientAsync(1000, allocListener);
        amRMClient.init(conf);
        amRMClient.start();

        // Node Managers communications setup
        containerListener = new NMCallbackHandler(this);
        nmClientAsync = new NMClientAsyncImpl(containerListener);
        nmClientAsync.init(conf);
        nmClientAsync.start();

        // get workflow file
        if (hdfs.exists(workflowPath)) {
            Path localPath = new Path(workflowPath.getName());
            hdfs.copyToLocalFile(false, workflowPath, localPath);
            workflowPath = localPath;
            workflowFile = new Data(workflowPath);
            workflowFile.stageOut();
        } else {
            // TODO this doesn't work; the path is triggered when running the application e.g., as hiway workflows/test.dax
            // but stageIn then fails, because in the HDFS, there is only test.dax and not workflows/test.dax
            workflowFile = new Data(workflowPath);
            workflowFile.stageIn();
        }

        // Register self with ResourceManager. This will start heartbeating to the RM.
        /* the hostname of the container running the Hi-WAY ApplicationMaster */
        String appMasterHostname = NetUtils.getHostname();
        /* the port on which the ApplicationMaster listens for status updates from clients */
        int appMasterRpcPort = -1;
        /* the tracking URL to which the ApplicationMaster publishes info for clients to monitor */
        String appMasterTrackingUrl = "";
        RegisterApplicationMasterResponse response = amRMClient.registerApplicationMaster(appMasterHostname,
                appMasterRpcPort, appMasterTrackingUrl);

        // initialize scheduler
        switch (schedulerEnumValue) {
        case roundRobin:
        case heft:
            int workerMemory = conf.getInt(YarnConfiguration.NM_PMEM_MB, YarnConfiguration.DEFAULT_NM_PMEM_MB);
            scheduler = schedulerEnumValue.equals(HiWayConfiguration.HIWAY_SCHEDULERS.roundRobin)
                    ? new RoundRobin(getWorkflowName())
                    : new HEFT(getWorkflowName(), workerMemory / containerMemory);
            break;
        case greedy:
            scheduler = new GreedyQueue(getWorkflowName());
            break;
        case memoryAware:
            scheduler = new MemoryAware(getWorkflowName(), amRMClient);
            break;
        case perfectDaxGQ:
            scheduler = new PerfectDaxGreedyQueue(getWorkflowName());
            break;
        default:
            C3PO c3po = new C3PO(getWorkflowName());
            switch (schedulerEnumValue) {
            case dataAware:
                c3po.setConservatismWeight(0.01d);
                c3po.setnClones(0);
                c3po.setPlacementAwarenessWeight(12d);
                c3po.setOutlookWeight(0.01d);
                break;
            default:
                c3po.setConservatismWeight(3d);
                c3po.setnClones(2);
                c3po.setPlacementAwarenessWeight(1d);
                c3po.setOutlookWeight(2d);
            }
            scheduler = c3po;
        }
        scheduler.init(conf, hdfs, containerMemory, customMemoryMap, containerCores, requestPriority);
        scheduler.initializeProvenanceManager();

        /* log */ logger.writeEntryToLog(new JsonReportEntry(getRunId(), null, null, null, null, null,
                HiwayDBI.KEY_WF_NAME, getWorkflowName()));
        logger.federatedReport = new Data(appId + ".log");

        // parse workflow, obtain ready tasks
        Collection<TaskInstance> readyTasks = parseWorkflow();

        // scheduler updates runtime estimates for all tasks comprising the workflow
        scheduler.updateRuntimeEstimates(getRunId().toString());

        scheduler.addTasks(readyTasks);

        // Dump out information about cluster capability as seen by the resource manager
        maxMem = response.getMaximumResourceCapability().getMemory();
        maxCores = response.getMaximumResourceCapability().getVirtualCores();
        /* log */ Logger.writeToStdout("Max mem capabililty of resources in this cluster " + maxMem);

        // A resource ask cannot exceed the max.
        if (containerMemory > maxMem) {
            /* log */ Logger.writeToStdout("Container memory specified above max threshold of cluster."
                    + " Using max value." + ", specified=" + containerMemory + ", max=" + maxMem);
            containerMemory = maxMem;
        }
        if (containerCores > maxCores) {
            /* log */ Logger.writeToStdout("Container vcores specified above max threshold of cluster."
                    + " Using max value." + ", specified=" + containerCores + ", max=" + maxCores);
            containerCores = maxCores;
        }

        // this is the actual work loop:
        // ask for resources until the workflow is done.
        executeWorkflow();

        finish();

    } catch (Exception e) {
        e.printStackTrace(System.out);
        System.exit(-1);
    }
    return success;
}

From source file:disAMS.AMRMClient.Impl.AMRMClientImpl.java

License:Apache License

private void updateAMRMToken(Token token) throws IOException {
    org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> amrmToken = new org.apache.hadoop.security.token.Token<AMRMTokenIdentifier>(
            token.getIdentifier().array(), token.getPassword().array(), new Text(token.getKind()),
            new Text(token.getService()));
    // Preserve the token service sent by the RM when adding the token
    // to ensure we replace the previous token setup by the RM.
    // Afterwards we can update the service address for the RPC layer.
    UserGroupInformation currentUGI = UserGroupInformation.getCurrentUser();
    currentUGI.addToken(amrmToken);/*from  ww  w.j a  v a 2  s .c om*/
    amrmToken.setService(ClientRMProxy.getAMRMTokenService(getConfig()));
}

From source file:edu.cmu.graphchi.toolkits.collaborative_filtering.yarn.ApplicationMaster.java

License:Apache License

/**
 * Main run function for the application master
 *
 * @throws YarnException/*from  w  w  w  . j  ava  2 s .c  o m*/
 * @throws IOException
 */
@SuppressWarnings({ "unchecked" })
public boolean run() throws YarnException, IOException {
    yarnClient.start();
    LOG.info("Starting ApplicationMaster");

    Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials();
    DataOutputBuffer dob = new DataOutputBuffer();
    credentials.writeTokenStorageToStream(dob);
    // Now remove the AM->RM token so that containers cannot access it.
    Iterator<Token<?>> iter = credentials.getAllTokens().iterator();
    while (iter.hasNext()) {
        Token<?> token = iter.next();
        if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) {
            iter.remove();
        }
    }
    allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());

    amRMClient = AMRMClient.createAMRMClient();
    amRMClient.init(conf);
    amRMClient.start();

    containerListener = createNMCallbackHandler();
    nmClientAsync = new NMClientAsyncImpl(containerListener);
    nmClientAsync.init(conf);
    nmClientAsync.start();

    // Register self with ResourceManager
    // This will start heartbeating to the RM
    appMasterHostname = NetUtils.getHostname();
    RegisterApplicationMasterResponse response = amRMClient.registerApplicationMaster(appMasterHostname,
            appMasterRpcPort, appMasterTrackingUrl);

    //TODO: Figure out how to do this.
    List<NodeReport> reports = this.yarnClient.getNodeReports();
    LOG.info("Cluster Status");
    List<Resource> availableResources = new ArrayList<Resource>();
    for (NodeReport nr : reports) {
        LOG.info("    NodeId: " + nr.getNodeId() + " Capabilities " + nr.getCapability() + " Used Resources "
                + nr.getUsed());
        int availableMem = nr.getCapability().getMemory() - nr.getUsed().getMemory();
        int availableVCores = nr.getCapability().getVirtualCores() - nr.getUsed().getVirtualCores();
        Resource resource = Resource.newInstance(availableMem, availableVCores);

        availableResources.add(resource);
    }

    /*Based on available resources scheduler should decide the best allocation of recommenders to resources
    and return a list of resources that should be requested from the ResourceManager*/
    DataSetDescription datasetDesc = new DataSetDescription(this.setup.dataMetadataFile);
    List<RecommenderPool> recommenderPools = RecommenderScheduler.splitRecommenderPool(availableResources,
            recommenders, datasetDesc, this.setup.nShards);

    for (RecommenderPool res : recommenderPools) {
        ContainerRequest containerAsk = setupContainerAskForRM(res.getTotalMemory(), requestPriority);
        LOG.info("CONTAINER ASK: " + containerAsk);
        amRMClient.addContainerRequest(containerAsk);
    }

    float progress = 0;

    List<RecommenderPool> pendingPools = new ArrayList<RecommenderPool>();
    for (RecommenderPool p : recommenderPools)
        pendingPools.add(p);

    this.numTotalContainers = recommenderPools.size();
    this.numCompletedContainers.set(0);
    this.numAllocatedContainers.set(0);

    while (numCompletedContainers.get() != numTotalContainers) {
        try {
            Thread.sleep(200);

            AllocateResponse allocResp = amRMClient.allocate(progress);
            List<Container> newContainers = allocResp.getAllocatedContainers();
            List<ContainerStatus> completedContainers = allocResp.getCompletedContainersStatuses();

            if (this.numAllocatedContainers.get() >= this.numTotalContainers && pendingPools.size() != 0) {
                //Ask for new containers for pending pools
                LOG.warn("The number of allocated containers has exceeded number of total containers, but "
                        + "the pending pool size is still not 0. Asking for new containers from RM");
                for (RecommenderPool res : pendingPools) {
                    ContainerRequest containerAsk = setupContainerAskForRM(res.getTotalMemory(),
                            requestPriority);
                    LOG.info("NEW CONTAINER ASK: " + containerAsk);
                    amRMClient.addContainerRequest(containerAsk);
                }

            }

            if (newContainers.size() > 0) {
                LOG.info("Allocated " + newContainers.size() + " new containers");
                numAllocatedContainers.addAndGet(newContainers.size());
                for (Container container : newContainers) {
                    //Find matching recommender pool from pendingRecommender pools.
                    RecommenderPool pool = null;
                    for (RecommenderPool p : pendingPools) {
                        if (p.getTotalMemory() == container.getResource().getMemory()) {
                            pool = p;
                            break;
                        }
                    }
                    if (pool == null) {
                        LOG.warn("No Takers for Container " + container + " Releasing container");
                        amRMClient.releaseAssignedContainer(container.getId());
                    } else {
                        startContainer(container, pool);
                        //This pool has now got a container. Remove it from pending pools
                        pendingPools.remove(pool);
                    }
                }

            }

            onContainersCompleted(completedContainers);

        } catch (InterruptedException ex) {
        }
    }
    finish();

    return success;
}

From source file:eu.stratosphere.nephele.taskmanager.TaskManager.java

License:Apache License

/**
 * Constructs a new task manager, starts its IPC service and attempts to discover the job manager to
 * receive an initial configuration. All parameters are obtained from the 
 * {@link GlobalConfiguration}, which must be loaded prior to instantiating the task manager.
 *///  w w  w  .  j  a v  a  2 s.  c o m
public TaskManager() throws Exception {

    LOG.info("TaskManager started as user " + UserGroupInformation.getCurrentUser().getShortUserName());
    LOG.info("User system property: " + System.getProperty("user.name"));

    // IMPORTANT! At this point, the GlobalConfiguration must have been read!

    final InetSocketAddress jobManagerAddress;
    {
        LOG.info("Reading location of job manager from configuration");

        final String address = GlobalConfiguration.getString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, null);
        final int port = GlobalConfiguration.getInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY,
                ConfigConstants.DEFAULT_JOB_MANAGER_IPC_PORT);

        if (address == null) {
            throw new Exception("Job manager address not configured in the GlobalConfiguration.");
        }

        // Try to convert configured address to {@link InetAddress}
        try {
            final InetAddress tmpAddress = InetAddress.getByName(address);
            jobManagerAddress = new InetSocketAddress(tmpAddress, port);
        } catch (UnknownHostException e) {
            LOG.fatal("Could not resolve JobManager host name.");
            throw new Exception("Could not resolve JobManager host name: " + e.getMessage(), e);
        }

        LOG.info("Connecting to JobManager at: " + jobManagerAddress);
    }

    // Create RPC connection to the JobManager
    try {
        this.jobManager = RPC.getProxy(JobManagerProtocol.class, jobManagerAddress,
                NetUtils.getSocketFactory());
    } catch (IOException e) {
        LOG.fatal("Could not connect to the JobManager: " + e.getMessage(), e);
        throw new Exception("Failed to initialize connection to JobManager: " + e.getMessage(), e);
    }

    int ipcPort = GlobalConfiguration.getInteger(ConfigConstants.TASK_MANAGER_IPC_PORT_KEY, -1);
    int dataPort = GlobalConfiguration.getInteger(ConfigConstants.TASK_MANAGER_DATA_PORT_KEY, -1);
    if (ipcPort == -1) {
        ipcPort = getAvailablePort();
    }
    if (dataPort == -1) {
        dataPort = getAvailablePort();
    }

    // Determine our own public facing address and start the server
    {
        final InetAddress taskManagerAddress;
        try {
            taskManagerAddress = getTaskManagerAddress(jobManagerAddress);
        } catch (Exception e) {
            throw new RuntimeException("The TaskManager failed to determine its own network address.", e);
        }

        this.localInstanceConnectionInfo = new InstanceConnectionInfo(taskManagerAddress, ipcPort, dataPort);
        LOG.info("TaskManager connection information:" + this.localInstanceConnectionInfo);

        // Start local RPC server
        try {
            this.taskManagerServer = RPC.getServer(this, taskManagerAddress.getHostAddress(), ipcPort,
                    IPC_HANDLER_COUNT);
            this.taskManagerServer.start();
        } catch (IOException e) {
            LOG.fatal("Failed to start TaskManager server. " + e.getMessage(), e);
            throw new Exception("Failed to start taskmanager server. " + e.getMessage(), e);
        }
    }

    // Try to create local stub of the global input split provider
    try {
        this.globalInputSplitProvider = RPC.getProxy(InputSplitProviderProtocol.class, jobManagerAddress,
                NetUtils.getSocketFactory());
    } catch (IOException e) {
        LOG.fatal(e.getMessage(), e);
        throw new Exception("Failed to initialize connection to global input split provider: " + e.getMessage(),
                e);
    }

    // Try to create local stub for the lookup service
    try {
        this.lookupService = RPC.getProxy(ChannelLookupProtocol.class, jobManagerAddress,
                NetUtils.getSocketFactory());
    } catch (IOException e) {
        LOG.fatal(e.getMessage(), e);
        throw new Exception("Failed to initialize channel lookup protocol. " + e.getMessage(), e);
    }

    // Try to create local stub for the accumulators
    try {
        this.accumulatorProtocolProxy = RPC.getProxy(AccumulatorProtocol.class, jobManagerAddress,
                NetUtils.getSocketFactory());
    } catch (IOException e) {
        LOG.fatal("Failed to initialize accumulator protocol: " + e.getMessage(), e);
        throw new Exception("Failed to initialize accumulator protocol: " + e.getMessage(), e);
    }

    // Load profiler if it should be used
    if (GlobalConfiguration.getBoolean(ProfilingUtils.ENABLE_PROFILING_KEY, false)) {

        final String profilerClassName = GlobalConfiguration.getString(ProfilingUtils.TASKMANAGER_CLASSNAME_KEY,
                "eu.stratosphere.nephele.profiling.impl.TaskManagerProfilerImpl");

        this.profiler = ProfilingUtils.loadTaskManagerProfiler(profilerClassName,
                jobManagerAddress.getAddress(), this.localInstanceConnectionInfo);

        if (this.profiler == null) {
            LOG.error("Cannot find class name for the profiler.");
        } else {
            LOG.info("Profiling of jobs is enabled.");
        }
    } else {
        this.profiler = null;
        LOG.info("Profiling of jobs is disabled.");
    }

    // Get the directory for storing temporary files
    final String[] tmpDirPaths = GlobalConfiguration
            .getString(ConfigConstants.TASK_MANAGER_TMP_DIR_KEY, ConfigConstants.DEFAULT_TASK_MANAGER_TMP_PATH)
            .split(",|" + File.pathSeparator);

    checkTempDirs(tmpDirPaths);

    final int pageSize = GlobalConfiguration.getInteger(ConfigConstants.TASK_MANAGER_NETWORK_BUFFER_SIZE_KEY,
            ConfigConstants.DEFAULT_TASK_MANAGER_NETWORK_BUFFER_SIZE);

    int numBuffers = GlobalConfiguration.getInteger(ConfigConstants.TASK_MANAGER_NETWORK_NUM_BUFFERS_KEY,
            ConfigConstants.DEFAULT_TASK_MANAGER_NETWORK_NUM_BUFFERS);

    int bufferSize = GlobalConfiguration.getInteger(ConfigConstants.TASK_MANAGER_NETWORK_BUFFER_SIZE_KEY,
            ConfigConstants.DEFAULT_TASK_MANAGER_NETWORK_BUFFER_SIZE);

    int numInThreads = GlobalConfiguration.getInteger(ConfigConstants.TASK_MANAGER_NET_NUM_IN_THREADS_KEY,
            ConfigConstants.DEFAULT_TASK_MANAGER_NET_NUM_IN_THREADS);

    int numOutThreads = GlobalConfiguration.getInteger(ConfigConstants.TASK_MANAGER_NET_NUM_OUT_THREADS_KEY,
            ConfigConstants.DEFAULT_TASK_MANAGER_NET_NUM_OUT_THREADS);

    int lowWaterMark = GlobalConfiguration.getInteger(ConfigConstants.TASK_MANAGER_NET_NETTY_LOW_WATER_MARK,
            ConfigConstants.DEFAULT_TASK_MANAGER_NET_NETTY_LOW_WATER_MARK);

    int highWaterMark = GlobalConfiguration.getInteger(ConfigConstants.TASK_MANAGER_NET_NETTY_HIGH_WATER_MARK,
            ConfigConstants.DEFAULT_TASK_MANAGER_NET_NETTY_HIGH_WATER_MARK);

    // Initialize the channel manager
    try {
        this.channelManager = new ChannelManager(this.lookupService, this.localInstanceConnectionInfo,
                numBuffers, bufferSize, numInThreads, numOutThreads, lowWaterMark, highWaterMark);
    } catch (IOException ioe) {
        LOG.error(StringUtils.stringifyException(ioe));
        throw new Exception("Failed to instantiate channel manager. " + ioe.getMessage(), ioe);
    }

    {
        HardwareDescription resources = HardwareDescriptionFactory.extractFromSystem();

        // Check whether the memory size has been explicitly configured. if so that overrides the default mechanism
        // of taking as much as is mentioned in the hardware description
        long memorySize = GlobalConfiguration.getInteger(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, -1);

        if (memorySize > 0) {
            // manually configured memory size. override the value in the hardware config
            resources = HardwareDescriptionFactory.construct(resources.getNumberOfCPUCores(),
                    resources.getSizeOfPhysicalMemory(), memorySize * 1024L * 1024L);
        }
        this.hardwareDescription = resources;

        // Initialize the memory manager
        LOG.info("Initializing memory manager with " + (resources.getSizeOfFreeMemory() >>> 20)
                + " megabytes of memory. " + "Page size is " + pageSize + " bytes.");

        try {
            @SuppressWarnings("unused")
            final boolean lazyAllocation = GlobalConfiguration.getBoolean(
                    ConfigConstants.TASK_MANAGER_MEMORY_LAZY_ALLOCATION_KEY,
                    ConfigConstants.DEFAULT_TASK_MANAGER_MEMORY_LAZY_ALLOCATION);

            this.memoryManager = new DefaultMemoryManager(resources.getSizeOfFreeMemory(), pageSize);
        } catch (Throwable t) {
            LOG.fatal("Unable to initialize memory manager with " + (resources.getSizeOfFreeMemory() >>> 20)
                    + " megabytes of memory.", t);
            throw new Exception("Unable to initialize memory manager.", t);
        }
    }

    this.ioManager = new IOManager(tmpDirPaths);

    this.heartbeatThread = new Thread() {
        @Override
        public void run() {
            runHeartbeatLoop();
        }
    };

    this.heartbeatThread.setName("Heartbeat Thread");
    this.heartbeatThread.start();
}

From source file:eu.stratosphere.nephele.taskmanager.TaskManager.java

License:Apache License

/**
 * Entry point for the program./*from   w ww .ja  v  a  2s. c om*/
 * 
 * @param args
 *        arguments from the command line
 * @throws IOException 
 */
@SuppressWarnings("static-access")
public static void main(String[] args) throws IOException {
    Option configDirOpt = OptionBuilder.withArgName("config directory").hasArg()
            .withDescription("Specify configuration directory.").create("configDir");
    // tempDir option is used by the YARN client.
    Option tempDir = OptionBuilder.withArgName("temporary directory (overwrites configured option)").hasArg()
            .withDescription("Specify temporary directory.").create(ARG_CONF_DIR);
    configDirOpt.setRequired(true);
    tempDir.setRequired(false);
    Options options = new Options();
    options.addOption(configDirOpt);
    options.addOption(tempDir);

    CommandLineParser parser = new GnuParser();
    CommandLine line = null;
    try {
        line = parser.parse(options, args);
    } catch (ParseException e) {
        System.err.println("CLI Parsing failed. Reason: " + e.getMessage());
        System.exit(FAILURE_RETURN_CODE);
    }

    String configDir = line.getOptionValue(configDirOpt.getOpt(), null);
    String tempDirVal = line.getOptionValue(tempDir.getOpt(), null);

    // First, try to load global configuration
    GlobalConfiguration.loadConfiguration(configDir);
    if (tempDirVal != null // the YARN TM runner has set a value for the temp dir
            // the configuration does not contain a temp direcory
            && GlobalConfiguration.getString(ConfigConstants.TASK_MANAGER_TMP_DIR_KEY, null) == null) {
        Configuration c = GlobalConfiguration.getConfiguration();
        c.setString(ConfigConstants.TASK_MANAGER_TMP_DIR_KEY, tempDirVal);
        LOG.info("Setting temporary directory to " + tempDirVal);
        GlobalConfiguration.includeConfiguration(c);
    }
    System.err.println("Configuration " + GlobalConfiguration.getConfiguration());
    LOG.info("Current user " + UserGroupInformation.getCurrentUser().getShortUserName());

    {
        // log the available JVM memory
        long maxMemoryMiBytes = Runtime.getRuntime().maxMemory() >>> 20;
        LOG.info("Starting TaskManager in a JVM with " + maxMemoryMiBytes + " MiBytes maximum heap size.");
    }

    // Create a new task manager object
    try {
        new TaskManager();
    } catch (Exception e) {
        LOG.fatal("Taskmanager startup failed: " + e.getMessage(), e);
        System.exit(FAILURE_RETURN_CODE);
    }

    // park the main thread to keep the JVM alive (all other threads may be daemon threads)
    Object mon = new Object();
    synchronized (mon) {
        try {
            mon.wait();
        } catch (InterruptedException ex) {
        }
    }
}

From source file:eu.stratosphere.runtime.util.EnvironmentInformation.java

License:Apache License

public static String getUserRunning() {
    try {/*from   w w w .  j ava2 s.c  om*/
        return UserGroupInformation.getCurrentUser().getShortUserName();
    } catch (Throwable t) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Cannot determine user/group information for the current user.", t);
        } else {
            LOG.info("Cannot determine user/group information for the current user.");
        }
        return UNKNOWN;
    }
}