List of usage examples for org.apache.hadoop.yarn.util SystemClock SystemClock
@Deprecated
public SystemClock()
From source file:com.datatorrent.stram.CheckpointTest.java
License:Apache License
@Test public void testUpdateRecoveryCheckpoint() throws Exception { Clock clock = new SystemClock(); LogicalPlan dag = new LogicalPlan(); dag.setAttribute(LogicalPlan.APPLICATION_PATH, testMeta.dir); dag.setAttribute(com.datatorrent.api.Context.OperatorContext.STORAGE_AGENT, new MemoryStorageAgent()); GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class); GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class); GenericTestOperator o3SL = dag.addOperator("o3SL", StatelessOperator.class); dag.addStream("o1.output1", o1.outport1, o2.inport1); dag.addStream("o2.output1", o2.outport1, o3SL.inport1); StreamingContainerManager dnm = new StreamingContainerManager(dag); PhysicalPlan plan = dnm.getPhysicalPlan(); for (PTOperator oper : plan.getAllOperators().values()) { Assert.assertEquals("activation windowId " + oper, Checkpoint.INITIAL_CHECKPOINT, oper.getRecoveryCheckpoint()); Assert.assertEquals("checkpoints " + oper, Collections.emptyList(), oper.checkpoints); }/*from ww w . j av a2s.c om*/ List<PTOperator> nodes1 = plan.getOperators(dag.getMeta(o1)); Assert.assertNotNull(nodes1); Assert.assertEquals(1, nodes1.size()); PTOperator o1p1 = nodes1.get(0); PTOperator o2p1 = plan.getOperators(dag.getMeta(o2)).get(0); PTOperator o3SLp1 = plan.getOperators(dag.getMeta(o3SL)).get(0); // recovery checkpoint won't update in deploy state for (PTOperator oper : plan.getAllOperators().values()) { Assert.assertEquals("", PTOperator.State.PENDING_DEPLOY, oper.getState()); } dnm.updateRecoveryCheckpoints(o2p1, new UpdateCheckpointsContext(clock)); Assert.assertEquals("no checkpoints " + o2p1, Checkpoint.INITIAL_CHECKPOINT, o2p1.getRecoveryCheckpoint()); UpdateCheckpointsContext ctx = new UpdateCheckpointsContext(clock); dnm.updateRecoveryCheckpoints(o1p1, ctx); Assert.assertEquals("no checkpoints " + o1p1, Checkpoint.INITIAL_CHECKPOINT, o1p1.getRecoveryCheckpoint()); Assert.assertEquals("number dependencies " + ctx.visited, 3, ctx.visited.size()); // adding checkpoints to upstream only does not move recovery checkpoint Checkpoint cp3 = new Checkpoint(3L, 0, 0); Checkpoint cp5 = new Checkpoint(5L, 0, 0); Checkpoint cp4 = new Checkpoint(4L, 0, 0); o1p1.checkpoints.add(cp3); o1p1.checkpoints.add(cp5); dnm.updateRecoveryCheckpoints(o1p1, new UpdateCheckpointsContext(clock)); Assert.assertEquals("checkpoint " + o1p1, Checkpoint.INITIAL_CHECKPOINT, o1p1.getRecoveryCheckpoint()); o2p1.checkpoints.add(new Checkpoint(3L, 0, 0)); dnm.updateRecoveryCheckpoints(o1p1, new UpdateCheckpointsContext(clock)); Assert.assertEquals("checkpoint " + o1p1, Checkpoint.INITIAL_CHECKPOINT, o1p1.getRecoveryCheckpoint()); Assert.assertEquals("checkpoint " + o2p1, Checkpoint.INITIAL_CHECKPOINT, o2p1.getRecoveryCheckpoint()); // set leaf operator checkpoint dnm.addCheckpoint(o3SLp1, cp5); dnm.updateRecoveryCheckpoints(o1p1, new UpdateCheckpointsContext(clock)); Assert.assertEquals("checkpoint " + o1p1, Checkpoint.INITIAL_CHECKPOINT, o1p1.getRecoveryCheckpoint()); Assert.assertEquals("checkpoint " + o2p1, Checkpoint.INITIAL_CHECKPOINT, o2p1.getRecoveryCheckpoint()); // set all operators as active to enable recovery window id update for (PTOperator oper : plan.getAllOperators().values()) { oper.setState(PTOperator.State.ACTIVE); } dnm.updateRecoveryCheckpoints(o1p1, new UpdateCheckpointsContext(clock)); Assert.assertEquals("checkpoint " + o1p1, cp3, o1p1.getRecoveryCheckpoint()); Assert.assertEquals("checkpoint " + o2p1, cp3, o1p1.getRecoveryCheckpoint()); Assert.assertEquals("checkpoint " + o3SLp1, cp5, o3SLp1.getRecoveryCheckpoint()); o2p1.checkpoints.add(cp4); dnm.updateRecoveryCheckpoints(o1p1, new UpdateCheckpointsContext(clock)); Assert.assertEquals("checkpoint " + o1p1, cp3, o1p1.getRecoveryCheckpoint()); Assert.assertEquals("checkpoint " + o2p1, cp4, o2p1.getRecoveryCheckpoint()); o1p1.checkpoints.add(1, cp4); Assert.assertEquals(o1p1.checkpoints, getCheckpoints(3L, 4L, 5L)); dnm.updateRecoveryCheckpoints(o1p1, new UpdateCheckpointsContext(clock)); Assert.assertEquals("checkpoint " + o1p1, cp4, o1p1.getRecoveryCheckpoint()); Assert.assertEquals(o1p1.checkpoints, getCheckpoints(4L, 5L)); // out of sequence windowIds should be sorted dnm.addCheckpoint(o2p1, new Checkpoint(2L, 0, 0)); Assert.assertEquals("add first", getCheckpoints(2L, 4L), o2p1.checkpoints); dnm.addCheckpoint(o2p1, new Checkpoint(3L, 0, 0)); Assert.assertEquals("add middle", getCheckpoints(2L, 3L, 4L), o2p1.checkpoints); dnm.addCheckpoint(o2p1, new Checkpoint(4L, 0, 0)); Assert.assertEquals("ignore duplicate", getCheckpoints(2L, 3L, 4L), o2p1.checkpoints); dnm.addCheckpoint(o2p1, new Checkpoint(5L, 0, 0)); Assert.assertEquals("add latest", getCheckpoints(2L, 3L, 4L, 5L), o2p1.checkpoints); }
From source file:com.datatorrent.stram.plan.logical.DelayOperatorTest.java
License:Apache License
@Test public void testCheckpointUpdate() { LogicalPlan dag = StramTestSupport.createDAG(testMeta); TestGeneratorInputOperator opA = dag.addOperator("A", TestGeneratorInputOperator.class); GenericTestOperator opB = dag.addOperator("B", GenericTestOperator.class); GenericTestOperator opC = dag.addOperator("C", GenericTestOperator.class); GenericTestOperator opD = dag.addOperator("D", GenericTestOperator.class); DefaultDelayOperator<Object> opDelay = dag.addOperator("opDelay", new DefaultDelayOperator<>()); dag.addStream("AtoB", opA.outport, opB.inport1); dag.addStream("BtoC", opB.outport1, opC.inport1); dag.addStream("CtoD", opC.outport1, opD.inport1); dag.addStream("CtoDelay", opC.outport2, opDelay.input); dag.addStream("DelayToB", opDelay.output, opB.inport2); dag.validate();//from www . j av a2 s .c om dag.setAttribute(com.datatorrent.api.Context.OperatorContext.STORAGE_AGENT, new MemoryStorageAgent()); StreamingContainerManager scm = new StreamingContainerManager(dag); PhysicalPlan plan = scm.getPhysicalPlan(); // set all operators as active to enable recovery window id update for (PTOperator oper : plan.getAllOperators().values()) { oper.setState(PTOperator.State.ACTIVE); } Clock clock = new SystemClock(); PTOperator opA1 = plan.getOperators(dag.getMeta(opA)).get(0); PTOperator opB1 = plan.getOperators(dag.getMeta(opB)).get(0); PTOperator opC1 = plan.getOperators(dag.getMeta(opC)).get(0); PTOperator opDelay1 = plan.getOperators(dag.getMeta(opDelay)).get(0); PTOperator opD1 = plan.getOperators(dag.getMeta(opD)).get(0); Checkpoint cp3 = new Checkpoint(3L, 0, 0); Checkpoint cp5 = new Checkpoint(5L, 0, 0); Checkpoint cp4 = new Checkpoint(4L, 0, 0); opB1.checkpoints.add(cp3); opC1.checkpoints.add(cp3); opC1.checkpoints.add(cp4); opDelay1.checkpoints.add(cp3); opDelay1.checkpoints.add(cp5); opD1.checkpoints.add(cp5); // construct grouping that would be supplied through LogicalPlan Set<OperatorMeta> stronglyConnected = Sets.newHashSet(dag.getMeta(opB), dag.getMeta(opC), dag.getMeta(opDelay)); Map<OperatorMeta, Set<OperatorMeta>> groups = new HashMap<>(); for (OperatorMeta om : stronglyConnected) { groups.put(om, stronglyConnected); } UpdateCheckpointsContext ctx = new UpdateCheckpointsContext(clock, false, groups); scm.updateRecoveryCheckpoints(opB1, ctx); Assert.assertEquals("checkpoint " + opA1, Checkpoint.INITIAL_CHECKPOINT, opA1.getRecoveryCheckpoint()); Assert.assertEquals("checkpoint " + opB1, cp3, opC1.getRecoveryCheckpoint()); Assert.assertEquals("checkpoint " + opC1, cp3, opC1.getRecoveryCheckpoint()); Assert.assertEquals("checkpoint " + opD1, cp5, opD1.getRecoveryCheckpoint()); }
From source file:com.datatorrent.stram.StreamingContainerManager.java
License:Apache License
public StreamingContainerManager(LogicalPlan dag) { this(dag, false, new SystemClock()); }
From source file:com.datatorrent.stram.StreamingContainerManager.java
License:Apache License
private StreamingContainerManager(CheckpointState checkpointedState, boolean enableEventRecording) { this.vars = checkpointedState.finals; this.clock = new SystemClock(); this.plan = checkpointedState.physicalPlan; this.eventBus = new MBassador<StramEvent>(BusConfiguration.Default(1, 1, 1)); setupWsClient();//from w w w. j a v a 2 s . c o m setupRecording(enableEventRecording); setupStringCodecs(); this.journal = new Journal(this); try { saveMetaInfo(); } catch (IOException ex) { LOG.error("Error saving meta info to DFS", ex); } try { this.containerFile = new FSJsonLineFile(new Path(this.vars.appPath + "/containers"), new FsPermission((short) 0644)); this.containerFile.append(getAppMasterContainerInfo()); } catch (IOException ex) { LOG.error("Caught exception when instantiating for container info file", ex); } }
From source file:com.datatorrent.stram.StreamingContainerManager.java
License:Apache License
/** * Get the instance for the given application. If the application directory contains a checkpoint, the state will be restored. * * @param rh/* ww w.j a v a2 s. c o m*/ * @param dag * @param enableEventRecording * @return instance of {@link StreamingContainerManager} * @throws IOException */ public static StreamingContainerManager getInstance(RecoveryHandler rh, LogicalPlan dag, boolean enableEventRecording) throws IOException { try { CheckpointState checkpointedState = (CheckpointState) rh.restore(); StreamingContainerManager scm; if (checkpointedState == null) { scm = new StreamingContainerManager(dag, enableEventRecording, new SystemClock()); } else { scm = new StreamingContainerManager(checkpointedState, enableEventRecording); // find better way to support final transient members PhysicalPlan plan = checkpointedState.physicalPlan; for (Field f : plan.getClass().getDeclaredFields()) { if (f.getType() == PlanContext.class) { f.setAccessible(true); try { f.set(plan, scm); } catch (Exception e) { throw new RuntimeException("Failed to set " + f, e); } f.setAccessible(false); } } DataInputStream logStream = rh.getLog(); scm.journal.replay(logStream); logStream.close(); // restore checkpoint info plan.syncCheckpoints(scm.vars.windowStartMillis, scm.clock.getTime()); scm.committedWindowId = scm.updateCheckpoints(true); // at this point the physical plan has been fully restored // populate container agents for existing containers for (PTContainer c : plan.getContainers()) { if (c.getExternalId() != null) { LOG.debug("Restore container agent {} for {}", c.getExternalId(), c); StreamingContainerAgent sca = new StreamingContainerAgent(c, scm.newStreamingContainerContext(c), scm); scm.containers.put(c.getExternalId(), sca); } else { LOG.debug("Requesting new resource for {}", c.toIdStateString()); scm.requestContainer(c); } } } scm.recoveryHandler = rh; scm.checkpoint(); return scm; } catch (IOException e) { throw new IllegalStateException("Failed to read checkpointed state", e); } }
From source file:org.apache.hama.bsp.BSPApplicationMaster.java
License:Apache License
private BSPApplicationMaster(String[] args) throws Exception { if (args.length != 1) { throw new IllegalArgumentException(); }//from ww w. j a v a 2 s . c o m this.jobFile = args[0]; this.localConf = new YarnConfiguration(); this.jobConf = getSubmitConfiguration(jobFile); fs = FileSystem.get(jobConf); this.applicationName = jobConf.get("bsp.job.name", "<no bsp job name defined>"); if (applicationName.isEmpty()) { this.applicationName = "<no bsp job name defined>"; } this.appAttemptId = getApplicationAttemptId(); this.yarnRPC = YarnRPC.create(localConf); this.clock = new SystemClock(); this.startTime = clock.getTime(); this.jobId = new BSPJobID(appAttemptId.toString(), 0); this.hostname = BSPNetUtils.getCanonicalHostname(); this.clientPort = BSPNetUtils.getFreePort(12000); // start our synchronization service startSyncServer(); startRPCServers(); /* * Make sure that this executes after the start the RPC servers, because we * are readjusting the configuration. */ rewriteSubmitConfiguration(jobFile, jobConf); String jobSplit = jobConf.get("bsp.job.split.file"); splits = null; if (jobSplit != null) { DataInputStream splitFile = fs.open(new Path(jobSplit)); try { splits = BSPJobClient.readSplitFile(splitFile); } finally { splitFile.close(); } } this.amrmRPC = getYarnRPCConnection(localConf); registerApplicationMaster(amrmRPC, hostname, clientPort, "http://localhost:8080"); }
From source file:org.apache.impala.yarn.server.resourcemanager.scheduler.fair.AllocationFileLoaderService.java
License:Apache License
public AllocationFileLoaderService() { this(new SystemClock()); }
From source file:org.apache.tajo.master.querymaster.QueryMaster.java
License:Apache License
public void init(Configuration conf) { LOG.info("QueryMaster init"); try {/* w w w .j a v a 2s .c om*/ this.systemConf = (TajoConf) conf; this.connPool = RpcConnectionPool.getPool(systemConf); querySessionTimeout = systemConf.getIntVar(TajoConf.ConfVars.QUERY_SESSION_TIMEOUT); queryMasterContext = new QueryMasterContext(systemConf); clock = new SystemClock(); this.dispatcher = new TajoAsyncDispatcher("querymaster_" + System.currentTimeMillis()); addIfService(dispatcher); globalPlanner = new GlobalPlanner(systemConf, workerContext); dispatcher.register(QueryStartEvent.EventType.class, new QueryStartEventHandler()); } catch (Throwable t) { LOG.error(t.getMessage(), t); throw new RuntimeException(t); } super.init(conf); }
From source file:org.apache.tajo.master.rm.NodeLivelinessMonitor.java
License:Apache License
public NodeLivelinessMonitor(Dispatcher d) { super(NodeLivelinessMonitor.class.getSimpleName(), new SystemClock()); this.dispatcher = d.getEventHandler(); }
From source file:org.apache.tajo.master.rm.WorkerLivelinessMonitor.java
License:Apache License
public WorkerLivelinessMonitor(Dispatcher d) { super(WorkerLivelinessMonitor.class.getSimpleName(), new SystemClock()); this.dispatcher = d.getEventHandler(); }