org.apache.hadoop.yarn.api.records.NodeId Java Examples
The following examples show how to use
org.apache.hadoop.yarn.api.records.NodeId.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestContainerManagerSecurity.java From big-c with Apache License 2.0 | 6 votes |
private void startContainer(final YarnRPC rpc, org.apache.hadoop.yarn.api.records.Token nmToken, org.apache.hadoop.yarn.api.records.Token containerToken, NodeId nodeId, String user) throws Exception { ContainerLaunchContext context = Records.newRecord(ContainerLaunchContext.class); StartContainerRequest scRequest = StartContainerRequest.newInstance(context,containerToken); List<StartContainerRequest> list = new ArrayList<StartContainerRequest>(); list.add(scRequest); StartContainersRequest allRequests = StartContainersRequest.newInstance(list); ContainerManagementProtocol proxy = null; try { proxy = getContainerManagementProtocolProxy(rpc, nmToken, nodeId, user); StartContainersResponse response = proxy.startContainers(allRequests); for(SerializedException ex : response.getFailedRequests().values()){ parseAndThrowException(ex.deSerialize()); } } finally { if (proxy != null) { rpc.stopProxy(proxy, conf); } } }
Example #2
Source File: TestAMNodeMap.java From incubator-tez with Apache License 2.0 | 6 votes |
@Test(timeout=5000) public void testHealthUpdateUnknownNode() { AppContext appContext = mock(AppContext.class); AMNodeMap amNodeMap = new AMNodeMap(eventHandler, appContext); amNodeMap.init(new Configuration(false)); amNodeMap.start(); NodeId nodeId = NodeId.newInstance("unknownhost", 2342); NodeReport nodeReport = generateNodeReport(nodeId, NodeState.UNHEALTHY); amNodeMap.handle(new AMNodeEventStateChanged(nodeReport)); dispatcher.await(); amNodeMap.stop(); // No exceptions - the status update was ignored. Not bothering to capture // the log message for verification. }
Example #3
Source File: TestRMAppAttemptTransitions.java From hadoop with Apache License 2.0 | 6 votes |
@Test public void testFinishingToFinishing() { Container amContainer = allocateApplicationAttempt(); launchApplicationAttempt(amContainer); runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false); FinalApplicationStatus finalStatus = FinalApplicationStatus.SUCCEEDED; String trackingUrl = "mytrackingurl"; String diagnostics = "Successful"; unregisterApplicationAttempt(amContainer, finalStatus, trackingUrl, diagnostics); // container must be AM container to move from FINISHING to FINISHED NodeId anyNodeId = NodeId.newInstance("host", 1234); applicationAttempt.handle( new RMAppAttemptContainerFinishedEvent( applicationAttempt.getAppAttemptId(), BuilderUtils.newContainerStatus( BuilderUtils.newContainerId( applicationAttempt.getAppAttemptId(), 42), ContainerState.COMPLETE, "", 0), anyNodeId)); testAppAttemptFinishingState(amContainer, finalStatus, trackingUrl, diagnostics); }
Example #4
Source File: TestRMAppAttemptTransitions.java From hadoop with Apache License 2.0 | 6 votes |
@Test public void testSuccessfulFinishingToFinished() { Container amContainer = allocateApplicationAttempt(); launchApplicationAttempt(amContainer); runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false); FinalApplicationStatus finalStatus = FinalApplicationStatus.SUCCEEDED; String trackingUrl = "mytrackingurl"; String diagnostics = "Successful"; unregisterApplicationAttempt(amContainer, finalStatus, trackingUrl, diagnostics); NodeId anyNodeId = NodeId.newInstance("host", 1234); applicationAttempt.handle( new RMAppAttemptContainerFinishedEvent( applicationAttempt.getAppAttemptId(), BuilderUtils.newContainerStatus(amContainer.getId(), ContainerState.COMPLETE, "", 0), anyNodeId)); testAppAttemptFinishedState(amContainer, finalStatus, trackingUrl, diagnostics, 0, false); }
Example #5
Source File: AggregatedLogsBlock.java From hadoop with Apache License 2.0 | 6 votes |
private NodeId verifyAndGetNodeId(Block html) { String nodeIdStr = $(NM_NODENAME); if (nodeIdStr == null || nodeIdStr.isEmpty()) { html.h1()._("Cannot get container logs without a NodeId")._(); return null; } NodeId nodeId = null; try { nodeId = ConverterUtils.toNodeId(nodeIdStr); } catch (IllegalArgumentException e) { html.h1()._("Cannot get container logs. Invalid nodeId: " + nodeIdStr) ._(); return null; } return nodeId; }
Example #6
Source File: TestContainerManagerSecurity.java From hadoop with Apache License 2.0 | 6 votes |
private String testStopContainer(YarnRPC rpc, ApplicationAttemptId appAttemptId, NodeId nodeId, ContainerId containerId, Token nmToken, boolean isExceptionExpected) { try { stopContainer(rpc, nmToken, Arrays.asList(new ContainerId[] { containerId }), appAttemptId, nodeId); if (isExceptionExpected) { fail("Exception was expected!!"); } return ""; } catch (Exception e) { e.printStackTrace(); return e.getMessage(); } }
Example #7
Source File: TestRMNodeLabelsManager.java From hadoop with Apache License 2.0 | 6 votes |
@Test(timeout=5000) public void testGetLabelResourceWhenMultipleNMsExistingInSameHost() throws IOException { // active two NM to n1, one large and one small mgr.activateNode(NodeId.newInstance("n1", 1), SMALL_RESOURCE); mgr.activateNode(NodeId.newInstance("n1", 2), SMALL_RESOURCE); mgr.activateNode(NodeId.newInstance("n1", 3), SMALL_RESOURCE); mgr.activateNode(NodeId.newInstance("n1", 4), SMALL_RESOURCE); // check resource of no label, it should be small * 4 Assert.assertEquals( mgr.getResourceByLabel(CommonNodeLabelsManager.NO_LABEL, null), Resources.multiply(SMALL_RESOURCE, 4)); // change two of these nodes to p1, check resource of no_label and P1 mgr.addToCluserNodeLabels(toSet("p1")); mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1:1"), toSet("p1"), toNodeId("n1:2"), toSet("p1"))); // check resource Assert.assertEquals( mgr.getResourceByLabel(CommonNodeLabelsManager.NO_LABEL, null), Resources.multiply(SMALL_RESOURCE, 2)); Assert.assertEquals( mgr.getResourceByLabel("p1", null), Resources.multiply(SMALL_RESOURCE, 2)); }
Example #8
Source File: RMAppAttemptImpl.java From hadoop with Apache License 2.0 | 6 votes |
private void sendFinishedContainersToNM() { for (NodeId nodeId : finishedContainersSentToAM.keySet()) { // Clear and get current values List<ContainerStatus> currentSentContainers = finishedContainersSentToAM.put(nodeId, new ArrayList<ContainerStatus>()); List<ContainerId> containerIdList = new ArrayList<ContainerId>(currentSentContainers.size()); for (ContainerStatus containerStatus : currentSentContainers) { containerIdList.add(containerStatus.getContainerId()); } eventHandler.handle(new RMNodeFinishedContainersPulledByAMEvent(nodeId, containerIdList)); } }
Example #9
Source File: TestResourceTrackerService.java From hadoop with Apache License 2.0 | 6 votes |
@Test public void testNodeRegistrationFailure() throws Exception { writeToHostsFile("host1"); Configuration conf = new Configuration(); conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, hostFile .getAbsolutePath()); rm = new MockRM(conf); rm.start(); ResourceTrackerService resourceTrackerService = rm.getResourceTrackerService(); RegisterNodeManagerRequest req = Records.newRecord( RegisterNodeManagerRequest.class); NodeId nodeId = NodeId.newInstance("host2", 1234); req.setNodeId(nodeId); req.setHttpPort(1234); // trying to register a invalid node. RegisterNodeManagerResponse response = resourceTrackerService.registerNodeManager(req); Assert.assertEquals(NodeAction.SHUTDOWN,response.getNodeAction()); Assert .assertEquals( "Disallowed NodeManager from host2, Sending SHUTDOWN signal to the NodeManager.", response.getDiagnosticsMessage()); }
Example #10
Source File: TestAMNodeTracker.java From tez with Apache License 2.0 | 5 votes |
private void _testSingleNodeNotBlacklisted(AMNodeTracker amNodeTracker, TestEventHandler handler, int schedulerId) { amNodeTracker.handle(new AMNodeEventNodeCountUpdated(1, schedulerId)); NodeId nodeId = NodeId.newInstance("host1", 1234); amNodeTracker.nodeSeen(nodeId, schedulerId); AMNodeImpl node = (AMNodeImpl) amNodeTracker.get(nodeId, schedulerId); ContainerId cId1 = mock(ContainerId.class); ContainerId cId2 = mock(ContainerId.class); amNodeTracker.handle(new AMNodeEventContainerAllocated(nodeId, schedulerId, cId1)); amNodeTracker.handle(new AMNodeEventContainerAllocated(nodeId, schedulerId, cId2)); TezTaskAttemptID ta1 = mock(TezTaskAttemptID.class); TezTaskAttemptID ta2 = mock(TezTaskAttemptID.class); amNodeTracker.handle(new AMNodeEventTaskAttemptEnded(nodeId, schedulerId, cId1, ta1, true)); dispatcher.await(); assertEquals(1, node.numFailedTAs); assertEquals(AMNodeState.ACTIVE, node.getState()); amNodeTracker.handle(new AMNodeEventTaskAttemptEnded(nodeId, schedulerId, cId2, ta2, true)); dispatcher.await(); assertEquals(2, node.numFailedTAs); assertEquals(1, handler.events.size()); assertEquals(AMNodeEventType.N_IGNORE_BLACKLISTING_ENABLED, handler.events.get(0).getType()); assertEquals(AMNodeState.FORCED_ACTIVE, node.getState()); // Blacklisting should be ignored since the node should have been blacklisted, but has not been // as a result of being a single node for the source assertTrue(amNodeTracker.isBlacklistingIgnored(schedulerId)); }
Example #11
Source File: NodeInfo.java From big-c with Apache License 2.0 | 5 votes |
public FakeRMNodeImpl(NodeId nodeId, String nodeAddr, String httpAddress, Resource perNode, String rackName, String healthReport, int cmdPort, String hostName, NodeState state) { this.nodeId = nodeId; this.nodeAddr = nodeAddr; this.httpAddress = httpAddress; this.perNode = perNode; this.rackName = rackName; this.healthReport = healthReport; this.cmdPort = cmdPort; this.hostName = hostName; this.state = state; toCleanUpApplications = new ArrayList<ApplicationId>(); toCleanUpContainers = new ArrayList<ContainerId>(); }
Example #12
Source File: TestRMAppAttemptTransitions.java From hadoop with Apache License 2.0 | 5 votes |
@SuppressWarnings("deprecation") @Test public void testContainersCleanupForLastAttempt() { // create a failed attempt. applicationAttempt = new RMAppAttemptImpl(applicationAttempt.getAppAttemptId(), spyRMContext, scheduler, masterService, submissionContext, new Configuration(), true, BuilderUtils.newResourceRequest( RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY, submissionContext.getResource(), 1)); when(submissionContext.getKeepContainersAcrossApplicationAttempts()) .thenReturn(true); when(submissionContext.getMaxAppAttempts()).thenReturn(1); Container amContainer = allocateApplicationAttempt(); launchApplicationAttempt(amContainer); runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false); ContainerStatus cs1 = ContainerStatus.newInstance(amContainer.getId(), ContainerState.COMPLETE, "some error", 123); ApplicationAttemptId appAttemptId = applicationAttempt.getAppAttemptId(); NodeId anyNodeId = NodeId.newInstance("host", 1234); applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent( appAttemptId, cs1, anyNodeId)); assertEquals(YarnApplicationAttemptState.RUNNING, applicationAttempt.createApplicationAttemptState()); sendAttemptUpdateSavedEvent(applicationAttempt); assertEquals(RMAppAttemptState.FAILED, applicationAttempt.getAppAttemptState()); assertFalse(transferStateFromPreviousAttempt); verifyApplicationAttemptFinished(RMAppAttemptState.FAILED); }
Example #13
Source File: CommonNodeLabelsManager.java From hadoop with Apache License 2.0 | 5 votes |
protected Map<NodeId, Set<String>> normalizeNodeIdToLabels( Map<NodeId, Set<String>> nodeIdToLabels) { Map<NodeId, Set<String>> newMap = new HashMap<NodeId, Set<String>>(); for (Entry<NodeId, Set<String>> entry : nodeIdToLabels.entrySet()) { NodeId id = entry.getKey(); Set<String> labels = entry.getValue(); newMap.put(id, normalizeLabels(labels)); } return newMap; }
Example #14
Source File: MRApp.java From big-c with Apache License 2.0 | 5 votes |
@Override public void handle(ContainerAllocatorEvent event) { ContainerId cId = ContainerId.newContainerId(getContext().getApplicationAttemptId(), containerCount++); NodeId nodeId = NodeId.newInstance(NM_HOST, NM_PORT); Resource resource = Resource.newInstance(1234, 2); ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier(cId, nodeId.toString(), "user", resource, System.currentTimeMillis() + 10000, 42, 42, Priority.newInstance(0), 0); Token containerToken = newContainerToken(nodeId, "password".getBytes(), containerTokenIdentifier); Container container = Container.newInstance(cId, nodeId, NM_HOST + ":" + NM_HTTP_PORT, resource, null, containerToken); JobID id = TypeConverter.fromYarn(applicationId); JobId jobId = TypeConverter.toYarn(id); getContext().getEventHandler().handle(new JobHistoryEvent(jobId, new NormalizedResourceEvent( org.apache.hadoop.mapreduce.TaskType.REDUCE, 100))); getContext().getEventHandler().handle(new JobHistoryEvent(jobId, new NormalizedResourceEvent( org.apache.hadoop.mapreduce.TaskType.MAP, 100))); getContext().getEventHandler().handle( new TaskAttemptContainerAssignedEvent(event.getAttemptID(), container, null)); }
Example #15
Source File: CommonNodeLabelsManager.java From big-c with Apache License 2.0 | 5 votes |
protected Map<NodeId, Set<String>> normalizeNodeIdToLabels( Map<NodeId, Set<String>> nodeIdToLabels) { Map<NodeId, Set<String>> newMap = new HashMap<NodeId, Set<String>>(); for (Entry<NodeId, Set<String>> entry : nodeIdToLabels.entrySet()) { NodeId id = entry.getKey(); Set<String> labels = entry.getValue(); newMap.put(id, normalizeLabels(labels)); } return newMap; }
Example #16
Source File: TestContainerManager.java From hadoop with Apache License 2.0 | 5 votes |
public static Token createContainerToken(ContainerId cId, long rmIdentifier, NodeId nodeId, String user, NMContainerTokenSecretManager containerTokenSecretManager) throws IOException { return createContainerToken(cId, rmIdentifier, nodeId, user, containerTokenSecretManager, null); }
Example #17
Source File: ContainerStartDataPBImpl.java From big-c with Apache License 2.0 | 5 votes |
@Override public void setAssignedNode(NodeId nodeId) { maybeInitBuilder(); if (nodeId == null) { builder.clearAssignedNodeId(); } this.nodeId = nodeId; }
Example #18
Source File: NodeStatus.java From big-c with Apache License 2.0 | 5 votes |
public static NodeStatus newInstance(NodeId nodeId, int responseId, List<ContainerStatus> containerStatuses, List<ApplicationId> keepAliveApplications, NodeHealthStatus nodeHealthStatus) { NodeStatus nodeStatus = Records.newRecord(NodeStatus.class); nodeStatus.setResponseId(responseId); nodeStatus.setNodeId(nodeId); nodeStatus.setContainersStatuses(containerStatuses); nodeStatus.setKeepAliveApplications(keepAliveApplications); nodeStatus.setNodeHealthStatus(nodeHealthStatus); return nodeStatus; }
Example #19
Source File: NodeLabelTestBase.java From big-c with Apache License 2.0 | 5 votes |
public NodeId toNodeId(String str) { if (str.contains(":")) { int idx = str.indexOf(':'); NodeId id = NodeId.newInstance(str.substring(0, idx), Integer.valueOf(str.substring(idx + 1))); return id; } else { return NodeId.newInstance(str, CommonNodeLabelsManager.WILDCARD_PORT); } }
Example #20
Source File: TestAHSWebServices.java From big-c with Apache License 2.0 | 5 votes |
@Test public void testSingleContainer() throws Exception { ApplicationId appId = ApplicationId.newInstance(0, 1); ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1); ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1); WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("applicationhistory").path("apps") .path(appId.toString()).path("appattempts") .path(appAttemptId.toString()).path("containers") .path(containerId.toString()) .queryParam("user.name", USERS[round]) .accept(MediaType.APPLICATION_JSON) .get(ClientResponse.class); if (round == 1) { assertEquals( Status.FORBIDDEN, response.getClientResponseStatus()); return; } assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); JSONObject json = response.getEntity(JSONObject.class); assertEquals("incorrect number of elements", 1, json.length()); JSONObject container = json.getJSONObject("container"); assertEquals(containerId.toString(), container.getString("containerId")); assertEquals("test diagnostics info", container.getString("diagnosticsInfo")); assertEquals("-1", container.getString("allocatedMB")); assertEquals("-1", container.getString("allocatedVCores")); assertEquals(NodeId.newInstance("test host", 100).toString(), container.getString("assignedNodeId")); assertEquals("-1", container.getString("priority")); Configuration conf = new YarnConfiguration(); assertEquals(WebAppUtils.getHttpSchemePrefix(conf) + WebAppUtils.getAHSWebAppURLWithoutScheme(conf) + "/applicationhistory/logs/test host:100/container_0_0001_01_000001/" + "container_0_0001_01_000001/user1", container.getString("logUrl")); assertEquals(ContainerState.COMPLETE.toString(), container.getString("containerState")); }
Example #21
Source File: PerSourceNodeTracker.java From tez with Apache License 2.0 | 5 votes |
private void sendIngoreBlacklistingStateToNodes() { AMNodeEventType eventType = ignoreBlacklisting ? AMNodeEventType.N_IGNORE_BLACKLISTING_ENABLED : AMNodeEventType.N_IGNORE_BLACKLISTING_DISABLED; for (NodeId nodeId : nodeMap.keySet()) { sendEvent(new AMNodeEvent(nodeId, sourceId, eventType)); } }
Example #22
Source File: TestRMAdminCLI.java From hadoop with Apache License 2.0 | 5 votes |
@Test public void testReplaceLabelsOnNode() throws Exception { // Successfully replace labels dummyNodeLabelsManager .addToCluserNodeLabels(ImmutableSet.of("x", "y", "Y")); String[] args = { "-replaceLabelsOnNode", "node1:8000,x node2:8000=y node3,x node4=Y", "-directlyAccessNodeLabelStore" }; assertEquals(0, rmAdminCLI.run(args)); assertTrue(dummyNodeLabelsManager.getNodeLabels().containsKey( NodeId.newInstance("node1", 8000))); assertTrue(dummyNodeLabelsManager.getNodeLabels().containsKey( NodeId.newInstance("node2", 8000))); assertTrue(dummyNodeLabelsManager.getNodeLabels().containsKey( NodeId.newInstance("node3", 0))); assertTrue(dummyNodeLabelsManager.getNodeLabels().containsKey( NodeId.newInstance("node4", 0))); // no labels, should fail args = new String[] { "-replaceLabelsOnNode" }; assertTrue(0 != rmAdminCLI.run(args)); // no labels, should fail args = new String[] { "-replaceLabelsOnNode", "-directlyAccessNodeLabelStore" }; assertTrue(0 != rmAdminCLI.run(args)); // no labels, should fail args = new String[] { "-replaceLabelsOnNode", " " }; assertTrue(0 != rmAdminCLI.run(args)); args = new String[] { "-replaceLabelsOnNode", ", " }; assertTrue(0 != rmAdminCLI.run(args)); }
Example #23
Source File: IgniteApplicationMasterSelfTest.java From ignite with Apache License 2.0 | 5 votes |
/** * @param host Host. * @param cpu Cpu count. * @param mem Memory. * @return Container. */ private Container createContainer(String host, int cpu, int mem) { return Container.newInstance( ContainerId.newContainerId(ApplicationAttemptId.newInstance(ApplicationId.newInstance(0l, 0), 0), ThreadLocalRandom.current().nextLong()), NodeId.newInstance(host, 0), "example.com", new MockResource(mem, cpu), Priority.newInstance(0), null ); }
Example #24
Source File: CapacityScheduler.java From big-c with Apache License 2.0 | 5 votes |
private synchronized void initScheduler(Configuration configuration) throws IOException { this.conf = loadCapacitySchedulerConfiguration(configuration); validateConf(this.conf); this.minimumAllocation = this.conf.getMinimumAllocation(); initMaximumResourceCapability(this.conf.getMaximumAllocation()); this.calculator = this.conf.getResourceCalculator(); this.usePortForNodeName = this.conf.getUsePortForNodeName(); this.nodeContainerUpdateMap = new HashMap<NodeId, ConcurrentLinkedQueue<NodeContainerUpdate>>(); this.applications = new ConcurrentHashMap<ApplicationId, SchedulerApplication<FiCaSchedulerApp>>(); this.labelManager = rmContext.getNodeLabelManager(); authorizer = YarnAuthorizationProvider.getInstance(yarnConf); initializeQueues(this.conf); scheduleAsynchronously = this.conf.getScheduleAynschronously(); asyncScheduleInterval = this.conf.getLong(ASYNC_SCHEDULER_INTERVAL, DEFAULT_ASYNC_SCHEDULER_INTERVAL); if (scheduleAsynchronously) { asyncSchedulerThread = new AsyncScheduleThread(this); } LOG.info("Initialized CapacityScheduler with " + "calculator=" + getResourceCalculator().getClass() + ", " + "minimumAllocation=<" + getMinimumResourceCapability() + ">, " + "maximumAllocation=<" + getMaximumResourceCapability() + ">, " + "asynchronousScheduling=" + scheduleAsynchronously + ", " + "asyncScheduleInterval=" + asyncScheduleInterval + "ms"); }
Example #25
Source File: TestContainerLauncherImpl.java From hadoop with Apache License 2.0 | 5 votes |
private Token createNewContainerToken(ContainerId contId, String containerManagerAddr) { long currentTime = System.currentTimeMillis(); return MRApp.newContainerToken(NodeId.newInstance("127.0.0.1", 1234), "password".getBytes(), new ContainerTokenIdentifier( contId, containerManagerAddr, "user", Resource.newInstance(1024, 1, 1), currentTime + 10000L, 123, currentTime, Priority.newInstance(0), 0)); }
Example #26
Source File: RMAdminCLI.java From hadoop with Apache License 2.0 | 5 votes |
private int replaceLabelsOnNodes(Map<NodeId, Set<String>> map) throws IOException, YarnException { if (directlyAccessNodeLabelStore) { getNodeLabelManagerInstance(getConf()).replaceLabelsOnNode(map); } else { ResourceManagerAdministrationProtocol adminProtocol = createAdminProtocol(); ReplaceLabelsOnNodeRequest request = ReplaceLabelsOnNodeRequest.newInstance(map); adminProtocol.replaceLabelsOnNode(request); } return 0; }
Example #27
Source File: NodeLabelTestBase.java From hadoop with Apache License 2.0 | 5 votes |
public static void assertMapEquals(Map<NodeId, Set<String>> m1, ImmutableMap<NodeId, Set<String>> m2) { Assert.assertEquals(m1.size(), m2.size()); for (NodeId k : m1.keySet()) { Assert.assertTrue(m2.containsKey(k)); assertCollectionEquals(m1.get(k), m2.get(k)); } }
Example #28
Source File: TestAMNodeTracker.java From tez with Apache License 2.0 | 5 votes |
@Test(timeout=10000) public void testNodeSelfBlacklistAlternateScheduler2() { AppContext appContext = mock(AppContext.class); Configuration conf = new Configuration(false); conf.setInt(TezConfiguration.TEZ_AM_MAX_TASK_FAILURES_PER_NODE, 2); TestEventHandler handler = new TestEventHandler(); AMNodeTracker amNodeTracker = new AMNodeTracker(handler, appContext); doReturn(amNodeTracker).when(appContext).getNodeTracker(); AMContainerMap amContainerMap = mock(AMContainerMap.class); TaskSchedulerManager taskSchedulerManager = mock(TaskSchedulerManager.class); dispatcher.register(AMNodeEventType.class, amNodeTracker); dispatcher.register(AMContainerEventType.class, amContainerMap); dispatcher.register(AMSchedulerEventType.class, taskSchedulerManager); amNodeTracker.init(conf); amNodeTracker.start(); try { // Register multiple nodes from a scheduler which isn't being tested. // This should not affect the blacklisting behaviour for (int i = 0 ; i < 100 ; i++) { amNodeTracker.nodeSeen(NodeId.newInstance("fakenode" + i, 3333), 0); } _testNodeSelfBlacklist(amNodeTracker, handler, 1); assertFalse(amNodeTracker.isBlacklistingIgnored(0)); } finally { amNodeTracker.stop(); } }
Example #29
Source File: NMClientAsyncImpl.java From hadoop with Apache License 2.0 | 5 votes |
public ContainerEvent(ContainerId containerId, NodeId nodeId, Token containerToken, ContainerEventType type) { super(type); this.containerId = containerId; this.nodeId = nodeId; this.containerToken = containerToken; }
Example #30
Source File: AMNodeMap.java From incubator-tez with Apache License 2.0 | 5 votes |
public void handle(AMNodeEvent rEvent) { // No synchronization required until there's multiple dispatchers. NodeId nodeId = rEvent.getNodeId(); switch (rEvent.getType()) { case N_NODE_WAS_BLACKLISTED: // When moving away from IGNORE_BLACKLISTING state, nodes will send out // blacklisted events. These need to be ignored. addToBlackList(nodeId); computeIgnoreBlacklisting(); break; case N_NODE_COUNT_UPDATED: AMNodeEventNodeCountUpdated event = (AMNodeEventNodeCountUpdated) rEvent; numClusterNodes = event.getNodeCount(); LOG.info("Num cluster nodes = " + numClusterNodes); computeIgnoreBlacklisting(); break; case N_TURNED_UNHEALTHY: case N_TURNED_HEALTHY: AMNode amNode = nodeMap.get(nodeId); if (amNode == null) { LOG.info("Ignoring RM Health Update for unknwon node: " + nodeId); } else { amNode.handle(rEvent); } break; default: nodeMap.get(nodeId).handle(rEvent); } }