List of usage examples for org.apache.hadoop.yarn.util SystemClock SystemClock
@Deprecated
public SystemClock()
From source file:org.apache.tajo.master.session.SessionLivelinessMonitor.java
License:Apache License
public SessionLivelinessMonitor(Dispatcher d) { super(SessionLivelinessMonitor.class.getSimpleName(), new SystemClock()); this.dispatcher = d.getEventHandler(); }
From source file:org.apache.tajo.master.TajoMaster.java
License:Apache License
@Override public void serviceInit(Configuration _conf) throws Exception { if (!(_conf instanceof TajoConf)) { throw new IllegalArgumentException("_conf should be a TajoConf type."); }// w ww .j a va 2s . co m this.systemConf = (TajoConf) _conf; Runtime.getRuntime().addShutdownHook(new Thread(new ShutdownHook())); context = new MasterContext(systemConf); clock = new SystemClock(); try { RackResolver.init(systemConf); initResourceManager(); initWebServer(); this.dispatcher = new AsyncDispatcher(); addIfService(dispatcher); // check the system directory and create if they are not created. checkAndInitializeSystemDirectories(); diagnoseTajoMaster(); this.storeManager = StorageManager.getFileStorageManager(systemConf); catalogServer = new CatalogServer(FunctionLoader.load()); addIfService(catalogServer); catalog = new LocalCatalogWrapper(catalogServer, systemConf); sessionManager = new SessionManager(dispatcher); addIfService(sessionManager); globalEngine = new GlobalEngine(context); addIfService(globalEngine); queryManager = new QueryManager(context); addIfService(queryManager); tajoMasterClientService = new TajoMasterClientService(context); addIfService(tajoMasterClientService); tajoMasterService = new QueryCoordinatorService(context); addIfService(tajoMasterService); } catch (Exception e) { LOG.error(e.getMessage(), e); throw e; } super.serviceInit(systemConf); LOG.info("Tajo Master is initialized."); }
From source file:org.apache.tajo.querymaster.QueryMaster.java
License:Apache License
public void init(Configuration conf) { LOG.info("QueryMaster init"); if (!(conf instanceof TajoConf)) { throw new IllegalArgumentException("conf should be a TajoConf type"); }/* w w w . j av a2 s . co m*/ try { this.systemConf = (TajoConf) conf; this.connPool = RpcConnectionPool.getPool(); querySessionTimeout = systemConf.getIntVar(TajoConf.ConfVars.QUERY_SESSION_TIMEOUT); queryMasterContext = new QueryMasterContext(systemConf); clock = new SystemClock(); this.dispatcher = new AsyncDispatcher(); addIfService(dispatcher); globalPlanner = new GlobalPlanner(systemConf, workerContext); dispatcher.register(QueryStartEvent.EventType.class, new QueryStartEventHandler()); dispatcher.register(QueryStopEvent.EventType.class, new QueryStopEventHandler()); } catch (Throwable t) { LOG.error(t.getMessage(), t); throw new RuntimeException(t); } super.init(conf); }
From source file:org.apache.tez.client.LocalClient.java
License:Apache License
protected Thread createDAGAppMaster(final ApplicationSubmissionContext appContext) { Thread thread = new Thread(new Runnable() { @Override/*w w w .ja va 2s .c om*/ public void run() { try { ApplicationId appId = appContext.getApplicationId(); // Set up working directory for DAGAppMaster Path staging = TezCommonUtils.getTezSystemStagingPath(conf, appId.toString()); Path userDir = TezCommonUtils.getTezSystemStagingPath(conf, appId.toString() + "_wd"); LOG.info("Using working directory: " + userDir.toUri().getPath()); FileSystem fs = FileSystem.get(conf); // copy data from staging directory to working directory to simulate the resource localizing FileUtil.copy(fs, staging, fs, userDir, false, conf); // Prepare Environment Path logDir = new Path(userDir, "localmode-log-dir"); Path localDir = new Path(userDir, "localmode-local-dir"); fs.mkdirs(logDir); fs.mkdirs(localDir); UserGroupInformation.setConfiguration(conf); // Add session specific credentials to the AM credentials. ByteBuffer tokens = appContext.getAMContainerSpec().getTokens(); Credentials amCredentials; if (tokens != null) { amCredentials = TezCommonUtils.parseCredentialsBytes(tokens.array()); } else { amCredentials = new Credentials(); } // Construct, initialize, and start the DAGAppMaster ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(appId, 0); ContainerId cId = ContainerId.newInstance(applicationAttemptId, 1); String currentHost = InetAddress.getLocalHost().getHostName(); int nmPort = YarnConfiguration.DEFAULT_NM_PORT; int nmHttpPort = YarnConfiguration.DEFAULT_NM_WEBAPP_PORT; long appSubmitTime = System.currentTimeMillis(); dagAppMaster = createDAGAppMaster(applicationAttemptId, cId, currentHost, nmPort, nmHttpPort, new SystemClock(), appSubmitTime, isSession, userDir.toUri().getPath(), new String[] { localDir.toUri().getPath() }, new String[] { logDir.toUri().getPath() }, amCredentials, UserGroupInformation.getCurrentUser().getShortUserName()); clientHandler = new DAGClientHandler(dagAppMaster); DAGAppMaster.initAndStartAppMaster(dagAppMaster, conf); } catch (Throwable t) { LOG.fatal("Error starting DAGAppMaster", t); if (dagAppMaster != null) { dagAppMaster.stop(); } amFailException = t; } } }); thread.setName("DAGAppMaster Thread"); LOG.info("DAGAppMaster thread has been created"); return thread; }
From source file:org.apache.tez.client.LocalClient.java
License:Apache License
@VisibleForTesting protected DAGAppMaster createDAGAppMaster(ApplicationAttemptId applicationAttemptId, ContainerId cId, String currentHost, int nmPort, int nmHttpPort, Clock clock, long appSubmitTime, boolean isSession, String userDir, String[] localDirs, String[] logDirs, Credentials credentials, String jobUserName) { return new DAGAppMaster(applicationAttemptId, cId, currentHost, nmPort, nmHttpPort, new SystemClock(), appSubmitTime, isSession, userDir, localDirs, logDirs, versionInfo.getVersion(), 1, credentials, jobUserName);/* w w w . j av a2 s. c om*/ }
From source file:org.apache.tez.dag.app.dag.impl.TestDAGRecovery.java
License:Apache License
@Before public void setUp() { mockAppContext = mock(AppContext.class, RETURNS_DEEP_STUBS); when(mockAppContext.getCurrentDAG().getDagUGI()).thenReturn(null); mockEventHandler = mock(EventHandler.class); tezCounters.findCounter("grp_1", "counter_1").increment(1); DAGPlan dagPlan = TestDAGImpl.createTestDAGPlan(); dag = new DAGImpl(dagId, new Configuration(), dagPlan, mockEventHandler, mock(TaskAttemptListener.class), new Credentials(), new SystemClock(), user, mock(TaskHeartbeatHandler.class), mockAppContext); }
From source file:org.apache.tez.dag.app.dag.impl.TestTaskAttempt.java
License:Apache License
@Test(timeout = 5000) public void testLocalityRequest() { TaskAttemptImpl.ScheduleTaskattemptTransition sta = new TaskAttemptImpl.ScheduleTaskattemptTransition(); EventHandler eventHandler = mock(EventHandler.class); Set<String> hosts = new TreeSet<String>(); hosts.add("host1"); hosts.add("host2"); hosts.add("host3"); TaskLocationHint locationHint = TaskLocationHint.createTaskLocationHint(hosts, null); TezTaskID taskID = TezTaskID.getInstance(TezVertexID.getInstance(TezDAGID.getInstance("1", 1, 1), 1), 1); TaskAttemptImpl taImpl = new MockTaskAttemptImpl(taskID, 1, eventHandler, mock(TaskAttemptListener.class), new Configuration(), new SystemClock(), mock(TaskHeartbeatHandler.class), mock(AppContext.class), locationHint, false, Resource.newInstance(1024, 1), createFakeContainerContext(), false); TaskAttemptEventSchedule sEvent = mock(TaskAttemptEventSchedule.class); sta.transition(taImpl, sEvent);//www. ja va 2s. c o m ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class); verify(eventHandler, times(1)).handle(arg.capture()); if (!(arg.getAllValues().get(0) instanceof AMSchedulerEventTALaunchRequest)) { fail("Second event not of type " + AMSchedulerEventTALaunchRequest.class.getName()); } // TODO Move the Rack request check to the client after TEZ-125 is fixed. Set<String> requestedRacks = taImpl.taskRacks; assertEquals(1, requestedRacks.size()); assertEquals(3, taImpl.taskHosts.size()); for (int i = 0; i < 3; i++) { String host = ("host" + (i + 1)); assertEquals(host, true, taImpl.taskHosts.contains(host)); } }
From source file:org.apache.tez.dag.app.dag.impl.TestTaskAttempt.java
License:Apache License
@Test(timeout = 5000) public void testPriority() { TaskAttemptImpl.ScheduleTaskattemptTransition sta = new TaskAttemptImpl.ScheduleTaskattemptTransition(); EventHandler eventHandler = mock(EventHandler.class); TezTaskID taskID = TezTaskID.getInstance(TezVertexID.getInstance(TezDAGID.getInstance("1", 1, 1), 1), 1); TaskAttemptImpl taImpl = new MockTaskAttemptImpl(taskID, 1, eventHandler, mock(TaskAttemptListener.class), new Configuration(), new SystemClock(), mock(TaskHeartbeatHandler.class), mock(AppContext.class), null, false, Resource.newInstance(1024, 1), createFakeContainerContext(), false); TaskAttemptImpl taImplReScheduled = new MockTaskAttemptImpl(taskID, 1, eventHandler, mock(TaskAttemptListener.class), new Configuration(), new SystemClock(), mock(TaskHeartbeatHandler.class), mock(AppContext.class), null, true, Resource.newInstance(1024, 1), createFakeContainerContext(), false); ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class); TaskAttemptEventSchedule sEvent = mock(TaskAttemptEventSchedule.class); when(sEvent.getPriorityLowLimit()).thenReturn(3); when(sEvent.getPriorityHighLimit()).thenReturn(1); sta.transition(taImpl, sEvent);//from w w w . jav a2s . co m verify(eventHandler, times(1)).handle(arg.capture()); AMSchedulerEventTALaunchRequest launchEvent = (AMSchedulerEventTALaunchRequest) arg.getValue(); Assert.assertEquals(2, launchEvent.getPriority()); sta.transition(taImplReScheduled, sEvent); verify(eventHandler, times(2)).handle(arg.capture()); launchEvent = (AMSchedulerEventTALaunchRequest) arg.getValue(); Assert.assertEquals(1, launchEvent.getPriority()); when(sEvent.getPriorityLowLimit()).thenReturn(6); when(sEvent.getPriorityHighLimit()).thenReturn(4); sta.transition(taImpl, sEvent); verify(eventHandler, times(3)).handle(arg.capture()); launchEvent = (AMSchedulerEventTALaunchRequest) arg.getValue(); Assert.assertEquals(5, launchEvent.getPriority()); sta.transition(taImplReScheduled, sEvent); verify(eventHandler, times(4)).handle(arg.capture()); launchEvent = (AMSchedulerEventTALaunchRequest) arg.getValue(); Assert.assertEquals(4, launchEvent.getPriority()); when(sEvent.getPriorityLowLimit()).thenReturn(5); when(sEvent.getPriorityHighLimit()).thenReturn(5); sta.transition(taImpl, sEvent); verify(eventHandler, times(5)).handle(arg.capture()); launchEvent = (AMSchedulerEventTALaunchRequest) arg.getValue(); Assert.assertEquals(5, launchEvent.getPriority()); sta.transition(taImplReScheduled, sEvent); verify(eventHandler, times(6)).handle(arg.capture()); launchEvent = (AMSchedulerEventTALaunchRequest) arg.getValue(); Assert.assertEquals(5, launchEvent.getPriority()); }
From source file:org.apache.tez.dag.app.dag.impl.TestTaskAttempt.java
License:Apache License
@Test(timeout = 5000) // Tests that an attempt is made to resolve the localized hosts to racks. // TODO Move to the client post TEZ-125. public void testHostResolveAttempt() throws Exception { TaskAttemptImpl.ScheduleTaskattemptTransition sta = new TaskAttemptImpl.ScheduleTaskattemptTransition(); EventHandler eventHandler = mock(EventHandler.class); String hosts[] = new String[] { "127.0.0.1", "host2", "host3" }; Set<String> resolved = new TreeSet<String>(Arrays.asList(new String[] { "host1", "host2", "host3" })); TaskLocationHint locationHint = TaskLocationHint .createTaskLocationHint(new TreeSet<String>(Arrays.asList(hosts)), null); TezTaskID taskID = TezTaskID.getInstance(TezVertexID.getInstance(TezDAGID.getInstance("1", 1, 1), 1), 1); TaskAttemptImpl taImpl = new MockTaskAttemptImpl(taskID, 1, eventHandler, mock(TaskAttemptListener.class), new Configuration(), new SystemClock(), mock(TaskHeartbeatHandler.class), mock(AppContext.class), locationHint, false, Resource.newInstance(1024, 1), createFakeContainerContext(), false); TaskAttemptImpl spyTa = spy(taImpl); when(spyTa.resolveHosts(hosts)).thenReturn(resolved.toArray(new String[3])); TaskAttemptEventSchedule mockTAEvent = mock(TaskAttemptEventSchedule.class); sta.transition(spyTa, mockTAEvent);//from ww w.j a va2 s. co m verify(spyTa).resolveHosts(hosts); ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class); verify(eventHandler, times(1)).handle(arg.capture()); if (!(arg.getAllValues().get(0) instanceof AMSchedulerEventTALaunchRequest)) { fail("Second Event not of type ContainerRequestEvent"); } Map<String, Boolean> expected = new HashMap<String, Boolean>(); expected.put("host1", true); expected.put("host2", true); expected.put("host3", true); Set<String> requestedHosts = spyTa.taskHosts; for (String h : requestedHosts) { expected.remove(h); } assertEquals(0, expected.size()); }
From source file:org.apache.tez.dag.app.dag.impl.TestTaskAttempt.java
License:Apache License
@Test(timeout = 5000) // Ensure the dag does not go into an error state if a attempt kill is // received while STARTING public void testLaunchFailedWhileKilling() throws Exception { ApplicationId appId = ApplicationId.newInstance(1, 2); ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 0); TezDAGID dagID = TezDAGID.getInstance(appId, 1); TezVertexID vertexID = TezVertexID.getInstance(dagID, 1); TezTaskID taskID = TezTaskID.getInstance(vertexID, 1); TezTaskAttemptID taskAttemptID = TezTaskAttemptID.getInstance(taskID, 0); MockEventHandler eventHandler = new MockEventHandler(); TaskAttemptListener taListener = mock(TaskAttemptListener.class); when(taListener.getAddress()).thenReturn(new InetSocketAddress("localhost", 0)); Configuration taskConf = new Configuration(); taskConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class); taskConf.setBoolean("fs.file.impl.disable.cache", true); TaskLocationHint locationHint = TaskLocationHint .createTaskLocationHint(new HashSet<String>(Arrays.asList(new String[] { "127.0.0.1" })), null); Resource resource = Resource.newInstance(1024, 1); AppContext mockAppContext = mock(AppContext.class); doReturn(new ClusterInfo()).when(mockAppContext).getClusterInfo(); TaskAttemptImpl taImpl = new MockTaskAttemptImpl(taskID, 1, eventHandler, taListener, taskConf, new SystemClock(), mock(TaskHeartbeatHandler.class), mockAppContext, locationHint, false, resource, createFakeContainerContext(), false); NodeId nid = NodeId.newInstance("127.0.0.1", 0); ContainerId contId = ContainerId.newInstance(appAttemptId, 3); Container container = mock(Container.class); when(container.getId()).thenReturn(contId); when(container.getNodeId()).thenReturn(nid); taImpl.handle(new TaskAttemptEventSchedule(taskAttemptID, 0, 0)); // At state STARTING. taImpl.handle(new TaskAttemptEventKillRequest(taskAttemptID, null, TaskAttemptTerminationCause.TERMINATED_BY_CLIENT)); // At some KILLING state. taImpl.handle(new TaskAttemptEventKillRequest(taskAttemptID, null, TaskAttemptTerminationCause.TERMINATED_BY_CLIENT)); // taImpl.handle(new TaskAttemptEventContainerTerminating(taskAttemptID, // null));/*from w w w . ja v a 2 s . c o m*/ assertFalse(eventHandler.internalError); }