Example usage for org.apache.hadoop.yarn.api.records ContainerId toString

List of usage examples for org.apache.hadoop.yarn.api.records ContainerId toString

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.yarn.api.records.*
  5. ContainerId
  6. toString

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records ContainerId toString.

Prototype

@Override
public String toString()

Source Link

Usage

From source file:com.datatorrent.stram.StreamingAppMasterService.java

License:Apache License

/**
 * Check for containers that were allocated in a previous attempt.
 * If the containers are still alive, wait for them to check in via heartbeat.
 *///w w  w  .j a v a2s.c  o m
private void checkContainerStatus() {
    Collection<StreamingContainerAgent> containers = this.dnmgr.getContainerAgents();
    for (StreamingContainerAgent ca : containers) {
        ContainerId containerId = ConverterUtils.toContainerId(ca.container.getExternalId());
        NodeId nodeId = ConverterUtils.toNodeId(ca.container.host);

        // put container back into the allocated list
        org.apache.hadoop.yarn.api.records.Token containerToken = null;
        Resource resource = Resource.newInstance(ca.container.getAllocatedMemoryMB(),
                ca.container.getAllocatedVCores());
        Priority priority = Priority.newInstance(ca.container.getResourceRequestPriority());
        Container yarnContainer = Container.newInstance(containerId, nodeId, ca.container.nodeHttpAddress,
                resource, priority, containerToken);
        this.allocatedContainers.put(containerId.toString(), new AllocatedContainer(yarnContainer));

        // check the status
        nmClient.getContainerStatusAsync(containerId, nodeId);
    }
}

From source file:com.datatorrent.stram.util.LoggerUtil.java

License:Apache License

/**
 * Makes MDC properties/* ww  w .  ja  v  a 2s . co m*/
 */
public static void setupMDC(String service) {
    MDC.put("apex.service", service);

    String value = StramClientUtils.getHostName();
    MDC.put("apex.node", value == null ? "unknown" : value);

    value = System.getenv(Environment.USER.key());
    if (value != null) {
        MDC.put("apex.user", value);
    }

    value = System.getenv(Environment.CONTAINER_ID.name());
    if (value != null) {
        ContainerId containerId = ConverterUtils.toContainerId(value);
        ApplicationId applicationId = containerId.getApplicationAttemptId().getApplicationId();
        MDC.put("apex.containerId", containerId.toString());
        MDC.put("apex.applicationId", applicationId.toString());
    }

    value = System.getProperty(APPLICATION_NAME.getLongName());
    if (value != null) {
        MDC.put("apex.application", value);
    }
}

From source file:de.huberlin.wbi.hiway.am.cuneiform.CuneiformApplicationMaster.java

License:Apache License

@Override
public void taskSuccess(TaskInstance task, ContainerId containerId) {
    try {//from  w w w  .  j a  va  2  s  .  c  om
        Invocation invocation = ((CuneiformTaskInstance) task).getInvocation();
        invocation.evalReport(task.getReport());
        ticketSrc.sendMsg(new TicketFinishedMsg(creActor, invocation.getTicket(), task.getReport()));
        System.out.println("Message sent.");

        // set output files
        for (String outputName : invocation.getStageOutList()) {
            if (!getFiles().containsKey(outputName)) {
                Data output = new Data(outputName);
                getFiles().put(outputName, output);
            }
            Data output = getFiles().get(outputName);
            output.setContainerId(containerId.toString());

            task.addOutputData(output);
        }

    } catch (JSONException | NotDerivableException e) {
        System.out.println(
                "Error when attempting to evaluate report of invocation " + task.toString() + ". exiting");
        e.printStackTrace();
        System.exit(-1);
    }
}

From source file:de.huberlin.wbi.hiway.am.cuneiforme.CuneiformEApplicationMaster.java

License:Apache License

@Override
public void taskSuccess(TaskInstance task, ContainerId containerId) {
    try {/*from   w  ww  .  ja v  a2s  . c o m*/
        (new Data(task.getId() + "_reply", containerId.toString())).stageIn();
        JSONObject reply = parseEffiFile(task.getId() + "_reply");
        workflow.addReply(reply);

        logger.writeEntryToLog(new JsonReportEntry(task.getWorkflowId(), task.getTaskId(), task.getTaskName(),
                task.getLanguageLabel(), task.getId(), null, JsonReportEntry.KEY_INVOC_OUTPUT,
                reply.toString()));
        for (String fileName : RemoteWorkflow.getOutputSet(requests.get(task), reply)) {
            files.put(fileName, new Data(fileName, containerId.toString()));
        }
    } catch (IOException | JSONException e) {
        e.printStackTrace(System.out);
        System.exit(-1);
    }
}

From source file:de.huberlin.wbi.hiway.am.cuneiformj.CuneiformJApplicationMaster.java

License:Apache License

@Override
public void taskSuccess(TaskInstance task, ContainerId containerId) {
    try {//from   w w w. j  a  v  a 2 s . c  om
        Invocation invocation = ((CuneiformJTaskInstance) task).getInvocation();
        invocation.evalReport(task.getReport());
        ticketSrc.sendMsg(new TicketFinishedMsg(creActor, invocation.getTicket(), task.getReport()));

        // set output files
        for (String outputName : invocation.getStageOutList()) {
            if (!getFiles().containsKey(outputName)) {
                Data output = new Data(outputName);
                getFiles().put(outputName, output);
            }
            Data output = getFiles().get(outputName);
            output.setContainerId(containerId.toString());

            task.addOutputData(output);
        }

    } catch (JSONException | NotDerivableException e) {
        Logger.writeToStdout(
                "Error when attempting to evaluate report of invocation " + task.toString() + ". exiting");
        e.printStackTrace(System.out);
        System.exit(-1);
    }
}

From source file:de.huberlin.wbi.hiway.am.HiWay.java

License:Apache License

@SuppressWarnings("static-method")
public void evaluateReport(TaskInstance task, ContainerId containerId) {
    try {/*  ww w .j a v  a 2s . co  m*/
        Data reportFile = new Data(Invocation.REPORT_FILENAME, containerId.toString());
        reportFile.stageIn();
        Data stdoutFile = new Data(Invocation.STDOUT_FILENAME, containerId.toString());
        stdoutFile.stageIn();
        Data stderrFile = new Data(Invocation.STDERR_FILENAME, containerId.toString());
        stderrFile.stageIn();

        // (a) evaluate report
        Set<JsonReportEntry> report = task.getReport();
        try (BufferedReader reader = new BufferedReader(new FileReader(Invocation.REPORT_FILENAME))) {
            String line;
            while ((line = reader.readLine()) != null) {
                line = line.trim();
                if (line.isEmpty())
                    continue;
                report.add(new JsonReportEntry(line));
            }
        }
        try (BufferedReader reader = new BufferedReader(new FileReader(Invocation.STDOUT_FILENAME))) {
            String line;
            StringBuffer sb = new StringBuffer();
            while ((line = reader.readLine()) != null) {
                sb.append(line.replaceAll("\\\\", "\\\\\\\\").replaceAll("\"", "\\\"")).append('\n');
            }
            String s = sb.toString();
            if (s.length() > 0) {
                JsonReportEntry re = new JsonReportEntry(task.getWorkflowId(), task.getTaskId(),
                        task.getTaskName(), task.getLanguageLabel(), task.getId(), null,
                        JsonReportEntry.KEY_INVOC_STDOUT, sb.toString());
                report.add(re);
            }
        }
        try (BufferedReader reader = new BufferedReader(new FileReader(Invocation.STDERR_FILENAME))) {
            String line;
            StringBuffer sb = new StringBuffer();
            while ((line = reader.readLine()) != null) {
                sb.append(line.replaceAll("\\\\", "\\\\\\\\").replaceAll("\"", "\\\"")).append('\n');
            }
            String s = sb.toString();
            if (s.length() > 0) {
                JsonReportEntry re = new JsonReportEntry(task.getWorkflowId(), task.getTaskId(),
                        task.getTaskName(), task.getLanguageLabel(), task.getId(), null,
                        JsonReportEntry.KEY_INVOC_STDERR, sb.toString());
                report.add(re);
            }
        }

    } catch (Exception e) {
        System.out.println(
                "Error when attempting to evaluate report of invocation " + task.toString() + ". exiting");
        e.printStackTrace();
        System.exit(-1);
    }
}

From source file:de.huberlin.wbi.hiway.am.HiWay.java

License:Apache License

@SuppressWarnings("static-method")
public void taskFailure(TaskInstance task, ContainerId containerId) {
    String line;// w  ww. j av  a 2s . com

    try {
        System.err.println("[script]");
        try (BufferedReader reader = new BufferedReader(new StringReader(task.getCommand()))) {
            int i = 0;
            while ((line = reader.readLine()) != null)
                System.err.println(String.format("%02d  %s", Integer.valueOf(++i), line));
        }

        Data stdoutFile = new Data(Invocation.STDOUT_FILENAME, containerId.toString());
        stdoutFile.stageIn();

        System.err.println("[out]");
        try (BufferedReader reader = new BufferedReader(new FileReader(stdoutFile.getLocalPath().toString()))) {
            while ((line = reader.readLine()) != null)
                System.err.println(line);
        }

        Data stderrFile = new Data(Invocation.STDERR_FILENAME, containerId.toString());
        stderrFile.stageIn();

        System.err.println("[err]");
        try (BufferedReader reader = new BufferedReader(new FileReader(stderrFile.getLocalPath().toString()))) {
            while ((line = reader.readLine()) != null)
                System.err.println(line);
        }
    } catch (IOException e) {
        e.printStackTrace();
        System.exit(-1);
    }

    System.err.println("[end]");
}

From source file:de.huberlin.wbi.hiway.am.HiWay.java

License:Apache License

public void taskSuccess(TaskInstance task, ContainerId containerId) {
    try {// w w  w. ja v a 2 s  .  c o m
        for (TaskInstance childTask : task.getChildTasks()) {
            if (childTask.readyToExecute())
                scheduler.addTaskToQueue(childTask);
        }
    } catch (WorkflowStructureUnknownException e) {
        e.printStackTrace();
        System.exit(-1);
    }
    for (Data data : task.getOutputData()) {
        data.setContainerId(containerId.toString());
    }
    if (scheduler.getNumberOfReadyTasks() == 0 && scheduler.getNumberOfRunningTasks() == 0) {
        done = true;
    }
}

From source file:de.huberlin.wbi.hiway.am.WorkflowDriver.java

License:Apache License

/**
 * Adds the succeeded task's children to the to-be-scheduled queue (if they're ready). <br/>
 * Set's the container ID on all of the task's output {@link Data} objects.  <br/>
 * Checks if the workflow execution is done (scheduler has neither ready nor running tasks).  <br/>
 * Is called by the {@link RMCallbackHandler#onContainersCompleted(List)} after receiving (and checking for diagnostics) the container completed message from the Resource Manager.
 *///w w  w.j a  va 2 s . c o m
public void taskSuccess(TaskInstance task, ContainerId containerId) {
    try {
        for (TaskInstance childTask : task.getChildTasks()) {
            if (childTask.readyToExecute())
                scheduler.enqueueResourceRequest(childTask);
        }
    } catch (WorkflowStructureUnknownException e) {
        e.printStackTrace(System.out);
        System.exit(-1);
    }
    for (Data data : task.getOutputData()) {
        data.setContainerId(containerId.toString());
    }
    if (scheduler.getNumberOfReadyTasks() == 0 && scheduler.getNumberOfRunningTasks() == 0) {
        setDone();
    }
}

From source file:de.huberlin.wbi.hiway.am.WorkflowDriver.java

License:Apache License

@SuppressWarnings("static-method")
public void taskFailure(TaskInstance task, ContainerId containerId) {
    String line;//from w w w . j av  a  2s .c o m

    try {
        Logger.writeToStdout("[script]");
        try (BufferedReader reader = new BufferedReader(new StringReader(task.getCommand()))) {
            int i = 0;
            while ((line = reader.readLine()) != null)
                /* log */ Logger.writeToStdout(String.format("%02d  %s", ++i, line));
        }

        Data stdoutFile = new Data(task.getId() + "_" + Invocation.STDOUT_FILENAME, containerId.toString());
        stdoutFile.stageIn();

        /* log */ Logger.writeToStdout("[out]");
        /* log */ try (BufferedReader reader = new BufferedReader(
                new FileReader(stdoutFile.getLocalPath().toString()))) {
            while ((line = reader.readLine()) != null)
                Logger.writeToStdout(line);
        }

        Data stderrFile = new Data(task.getId() + "_" + Invocation.STDERR_FILENAME, containerId.toString());
        stderrFile.stageIn();

        /* log */ Logger.writeToStdout("[err]");
        /* log */ try (BufferedReader reader = new BufferedReader(
                new FileReader(stderrFile.getLocalPath().toString()))) {
            while ((line = reader.readLine()) != null)
                Logger.writeToStdout(line);
        }
    } catch (IOException e) {
        e.printStackTrace(System.out);
    }

    /*log */ Logger.writeToStdout("[end]");
}