org.apache.hadoop.yarn.api.records.NodeId Java Examples

The following examples show how to use org.apache.hadoop.yarn.api.records.NodeId. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestContainerManagerSecurity.java    From big-c with Apache License 2.0 6 votes vote down vote up
private void startContainer(final YarnRPC rpc,
    org.apache.hadoop.yarn.api.records.Token nmToken,
    org.apache.hadoop.yarn.api.records.Token containerToken,
    NodeId nodeId, String user) throws Exception {

  ContainerLaunchContext context =
      Records.newRecord(ContainerLaunchContext.class);
  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(context,containerToken);
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  ContainerManagementProtocol proxy = null;
  try {
    proxy = getContainerManagementProtocolProxy(rpc, nmToken, nodeId, user);
    StartContainersResponse response = proxy.startContainers(allRequests);
    for(SerializedException ex : response.getFailedRequests().values()){
      parseAndThrowException(ex.deSerialize());
    }
  } finally {
    if (proxy != null) {
      rpc.stopProxy(proxy, conf);
    }
  }
}
 
Example #2
Source File: TestAMNodeMap.java    From incubator-tez with Apache License 2.0 6 votes vote down vote up
@Test(timeout=5000)
public void testHealthUpdateUnknownNode() {
  AppContext appContext = mock(AppContext.class);

  AMNodeMap amNodeMap = new AMNodeMap(eventHandler, appContext);
  amNodeMap.init(new Configuration(false));
  amNodeMap.start();

  NodeId nodeId = NodeId.newInstance("unknownhost", 2342);

  NodeReport nodeReport = generateNodeReport(nodeId, NodeState.UNHEALTHY);
  amNodeMap.handle(new AMNodeEventStateChanged(nodeReport));
  dispatcher.await();

  amNodeMap.stop();
  // No exceptions - the status update was ignored. Not bothering to capture
  // the log message for verification.
}
 
Example #3
Source File: TestRMAppAttemptTransitions.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testFinishingToFinishing() {
  Container amContainer = allocateApplicationAttempt();
  launchApplicationAttempt(amContainer);
  runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
  FinalApplicationStatus finalStatus = FinalApplicationStatus.SUCCEEDED;
  String trackingUrl = "mytrackingurl";
  String diagnostics = "Successful";
  unregisterApplicationAttempt(amContainer, finalStatus, trackingUrl,
      diagnostics);
  // container must be AM container to move from FINISHING to FINISHED
  NodeId anyNodeId = NodeId.newInstance("host", 1234);
  applicationAttempt.handle(
      new RMAppAttemptContainerFinishedEvent(
          applicationAttempt.getAppAttemptId(),
          BuilderUtils.newContainerStatus(
              BuilderUtils.newContainerId(
                  applicationAttempt.getAppAttemptId(), 42),
              ContainerState.COMPLETE, "", 0), anyNodeId));
  testAppAttemptFinishingState(amContainer, finalStatus, trackingUrl,
      diagnostics);
}
 
Example #4
Source File: TestRMAppAttemptTransitions.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testSuccessfulFinishingToFinished() {
  Container amContainer = allocateApplicationAttempt();
  launchApplicationAttempt(amContainer);
  runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
  FinalApplicationStatus finalStatus = FinalApplicationStatus.SUCCEEDED;
  String trackingUrl = "mytrackingurl";
  String diagnostics = "Successful";
  unregisterApplicationAttempt(amContainer, finalStatus, trackingUrl,
      diagnostics);
  NodeId anyNodeId = NodeId.newInstance("host", 1234);
  applicationAttempt.handle(
      new RMAppAttemptContainerFinishedEvent(
          applicationAttempt.getAppAttemptId(),
          BuilderUtils.newContainerStatus(amContainer.getId(),
              ContainerState.COMPLETE, "", 0), anyNodeId));
  testAppAttemptFinishedState(amContainer, finalStatus, trackingUrl,
      diagnostics, 0, false);
}
 
Example #5
Source File: AggregatedLogsBlock.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private NodeId verifyAndGetNodeId(Block html) {
  String nodeIdStr = $(NM_NODENAME);
  if (nodeIdStr == null || nodeIdStr.isEmpty()) {
    html.h1()._("Cannot get container logs without a NodeId")._();
    return null;
  }
  NodeId nodeId = null;
  try {
    nodeId = ConverterUtils.toNodeId(nodeIdStr);
  } catch (IllegalArgumentException e) {
    html.h1()._("Cannot get container logs. Invalid nodeId: " + nodeIdStr)
        ._();
    return null;
  }
  return nodeId;
}
 
Example #6
Source File: TestContainerManagerSecurity.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private String testStopContainer(YarnRPC rpc,
    ApplicationAttemptId appAttemptId, NodeId nodeId,
    ContainerId containerId, Token nmToken, boolean isExceptionExpected) {
  try {
    stopContainer(rpc, nmToken,
        Arrays.asList(new ContainerId[] { containerId }), appAttemptId,
        nodeId);
    if (isExceptionExpected) {
      fail("Exception was expected!!");
    }
    return "";
  } catch (Exception e) {
    e.printStackTrace();
    return e.getMessage();
  }
}
 
Example #7
Source File: TestRMNodeLabelsManager.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test(timeout=5000)
public void testGetLabelResourceWhenMultipleNMsExistingInSameHost() throws IOException {
  // active two NM to n1, one large and one small
  mgr.activateNode(NodeId.newInstance("n1", 1), SMALL_RESOURCE);
  mgr.activateNode(NodeId.newInstance("n1", 2), SMALL_RESOURCE);
  mgr.activateNode(NodeId.newInstance("n1", 3), SMALL_RESOURCE);
  mgr.activateNode(NodeId.newInstance("n1", 4), SMALL_RESOURCE);
  
  // check resource of no label, it should be small * 4
  Assert.assertEquals(
      mgr.getResourceByLabel(CommonNodeLabelsManager.NO_LABEL, null),
      Resources.multiply(SMALL_RESOURCE, 4));
  
  // change two of these nodes to p1, check resource of no_label and P1
  mgr.addToCluserNodeLabels(toSet("p1"));
  mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1:1"), toSet("p1"),
      toNodeId("n1:2"), toSet("p1")));
  
  // check resource
  Assert.assertEquals(
      mgr.getResourceByLabel(CommonNodeLabelsManager.NO_LABEL, null),
      Resources.multiply(SMALL_RESOURCE, 2));    
  Assert.assertEquals(
          mgr.getResourceByLabel("p1", null),
          Resources.multiply(SMALL_RESOURCE, 2));
}
 
Example #8
Source File: RMAppAttemptImpl.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private void sendFinishedContainersToNM() {
  for (NodeId nodeId : finishedContainersSentToAM.keySet()) {

    // Clear and get current values
    List<ContainerStatus> currentSentContainers =
        finishedContainersSentToAM.put(nodeId,
          new ArrayList<ContainerStatus>());
    List<ContainerId> containerIdList =
        new ArrayList<ContainerId>(currentSentContainers.size());
    for (ContainerStatus containerStatus : currentSentContainers) {
      containerIdList.add(containerStatus.getContainerId());
    }
    eventHandler.handle(new RMNodeFinishedContainersPulledByAMEvent(nodeId,
      containerIdList));
  }
}
 
Example #9
Source File: TestResourceTrackerService.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testNodeRegistrationFailure() throws Exception {
  writeToHostsFile("host1");
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, hostFile
      .getAbsolutePath());
  rm = new MockRM(conf);
  rm.start();
  
  ResourceTrackerService resourceTrackerService = rm.getResourceTrackerService();
  RegisterNodeManagerRequest req = Records.newRecord(
      RegisterNodeManagerRequest.class);
  NodeId nodeId = NodeId.newInstance("host2", 1234);
  req.setNodeId(nodeId);
  req.setHttpPort(1234);
  // trying to register a invalid node.
  RegisterNodeManagerResponse response = resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.SHUTDOWN,response.getNodeAction());
  Assert
    .assertEquals(
      "Disallowed NodeManager from  host2, Sending SHUTDOWN signal to the NodeManager.",
      response.getDiagnosticsMessage());
}
 
Example #10
Source File: TestAMNodeTracker.java    From tez with Apache License 2.0 5 votes vote down vote up
private void _testSingleNodeNotBlacklisted(AMNodeTracker amNodeTracker,
                                           TestEventHandler handler, int schedulerId) {
  amNodeTracker.handle(new AMNodeEventNodeCountUpdated(1, schedulerId));
  NodeId nodeId = NodeId.newInstance("host1", 1234);
  amNodeTracker.nodeSeen(nodeId, schedulerId);

  AMNodeImpl node = (AMNodeImpl) amNodeTracker.get(nodeId, schedulerId);

  ContainerId cId1 = mock(ContainerId.class);
  ContainerId cId2 = mock(ContainerId.class);

  amNodeTracker.handle(new AMNodeEventContainerAllocated(nodeId, schedulerId, cId1));
  amNodeTracker.handle(new AMNodeEventContainerAllocated(nodeId, schedulerId, cId2));

  TezTaskAttemptID ta1 = mock(TezTaskAttemptID.class);
  TezTaskAttemptID ta2 = mock(TezTaskAttemptID.class);

  amNodeTracker.handle(new AMNodeEventTaskAttemptEnded(nodeId, schedulerId, cId1, ta1, true));
  dispatcher.await();
  assertEquals(1, node.numFailedTAs);
  assertEquals(AMNodeState.ACTIVE, node.getState());

  amNodeTracker.handle(new AMNodeEventTaskAttemptEnded(nodeId, schedulerId, cId2, ta2, true));
  dispatcher.await();
  assertEquals(2, node.numFailedTAs);
  assertEquals(1, handler.events.size());
  assertEquals(AMNodeEventType.N_IGNORE_BLACKLISTING_ENABLED, handler.events.get(0).getType());
  assertEquals(AMNodeState.FORCED_ACTIVE, node.getState());
  // Blacklisting should be ignored since the node should have been blacklisted, but has not been
  // as a result of being a single node for the source
  assertTrue(amNodeTracker.isBlacklistingIgnored(schedulerId));
}
 
Example #11
Source File: NodeInfo.java    From big-c with Apache License 2.0 5 votes vote down vote up
public FakeRMNodeImpl(NodeId nodeId, String nodeAddr, String httpAddress,
    Resource perNode, String rackName, String healthReport,
    int cmdPort, String hostName, NodeState state) {
  this.nodeId = nodeId;
  this.nodeAddr = nodeAddr;
  this.httpAddress = httpAddress;
  this.perNode = perNode;
  this.rackName = rackName;
  this.healthReport = healthReport;
  this.cmdPort = cmdPort;
  this.hostName = hostName;
  this.state = state;
  toCleanUpApplications = new ArrayList<ApplicationId>();
  toCleanUpContainers = new ArrayList<ContainerId>();
}
 
Example #12
Source File: TestRMAppAttemptTransitions.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("deprecation")
@Test
public void testContainersCleanupForLastAttempt() {
  // create a failed attempt.
  applicationAttempt =
      new RMAppAttemptImpl(applicationAttempt.getAppAttemptId(), spyRMContext,
        scheduler, masterService, submissionContext, new Configuration(),
        true, BuilderUtils.newResourceRequest(
            RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY,
            submissionContext.getResource(), 1));
  when(submissionContext.getKeepContainersAcrossApplicationAttempts())
    .thenReturn(true);
  when(submissionContext.getMaxAppAttempts()).thenReturn(1);
  Container amContainer = allocateApplicationAttempt();
  launchApplicationAttempt(amContainer);
  runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
  ContainerStatus cs1 =
      ContainerStatus.newInstance(amContainer.getId(),
        ContainerState.COMPLETE, "some error", 123);
  ApplicationAttemptId appAttemptId = applicationAttempt.getAppAttemptId();
  NodeId anyNodeId = NodeId.newInstance("host", 1234);
  applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
    appAttemptId, cs1, anyNodeId));
  assertEquals(YarnApplicationAttemptState.RUNNING,
      applicationAttempt.createApplicationAttemptState());
  sendAttemptUpdateSavedEvent(applicationAttempt);
  assertEquals(RMAppAttemptState.FAILED,
    applicationAttempt.getAppAttemptState());
  assertFalse(transferStateFromPreviousAttempt);
  verifyApplicationAttemptFinished(RMAppAttemptState.FAILED);
}
 
Example #13
Source File: CommonNodeLabelsManager.java    From hadoop with Apache License 2.0 5 votes vote down vote up
protected Map<NodeId, Set<String>> normalizeNodeIdToLabels(
    Map<NodeId, Set<String>> nodeIdToLabels) {
  Map<NodeId, Set<String>> newMap = new HashMap<NodeId, Set<String>>();
  for (Entry<NodeId, Set<String>> entry : nodeIdToLabels.entrySet()) {
    NodeId id = entry.getKey();
    Set<String> labels = entry.getValue();
    newMap.put(id, normalizeLabels(labels)); 
  }
  return newMap;
}
 
Example #14
Source File: MRApp.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public void handle(ContainerAllocatorEvent event) {
  ContainerId cId =
      ContainerId.newContainerId(getContext().getApplicationAttemptId(),
        containerCount++);
  NodeId nodeId = NodeId.newInstance(NM_HOST, NM_PORT);
  Resource resource = Resource.newInstance(1234, 2);
  ContainerTokenIdentifier containerTokenIdentifier =
      new ContainerTokenIdentifier(cId, nodeId.toString(), "user",
      resource, System.currentTimeMillis() + 10000, 42, 42,
      Priority.newInstance(0), 0);
  Token containerToken = newContainerToken(nodeId, "password".getBytes(),
        containerTokenIdentifier);
  Container container = Container.newInstance(cId, nodeId,
      NM_HOST + ":" + NM_HTTP_PORT, resource, null, containerToken);
  JobID id = TypeConverter.fromYarn(applicationId);
  JobId jobId = TypeConverter.toYarn(id);
  getContext().getEventHandler().handle(new JobHistoryEvent(jobId, 
      new NormalizedResourceEvent(
          org.apache.hadoop.mapreduce.TaskType.REDUCE,
      100)));
  getContext().getEventHandler().handle(new JobHistoryEvent(jobId, 
      new NormalizedResourceEvent(
          org.apache.hadoop.mapreduce.TaskType.MAP,
      100)));
  getContext().getEventHandler().handle(
      new TaskAttemptContainerAssignedEvent(event.getAttemptID(),
          container, null));
}
 
Example #15
Source File: CommonNodeLabelsManager.java    From big-c with Apache License 2.0 5 votes vote down vote up
protected Map<NodeId, Set<String>> normalizeNodeIdToLabels(
    Map<NodeId, Set<String>> nodeIdToLabels) {
  Map<NodeId, Set<String>> newMap = new HashMap<NodeId, Set<String>>();
  for (Entry<NodeId, Set<String>> entry : nodeIdToLabels.entrySet()) {
    NodeId id = entry.getKey();
    Set<String> labels = entry.getValue();
    newMap.put(id, normalizeLabels(labels)); 
  }
  return newMap;
}
 
Example #16
Source File: TestContainerManager.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public static Token createContainerToken(ContainerId cId, long rmIdentifier,
    NodeId nodeId, String user,
    NMContainerTokenSecretManager containerTokenSecretManager)
    throws IOException {
  return createContainerToken(cId, rmIdentifier, nodeId, user,
    containerTokenSecretManager, null);
}
 
Example #17
Source File: ContainerStartDataPBImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public void setAssignedNode(NodeId nodeId) {
  maybeInitBuilder();
  if (nodeId == null) {
    builder.clearAssignedNodeId();
  }
  this.nodeId = nodeId;
}
 
Example #18
Source File: NodeStatus.java    From big-c with Apache License 2.0 5 votes vote down vote up
public static NodeStatus newInstance(NodeId nodeId, int responseId,
    List<ContainerStatus> containerStatuses,
    List<ApplicationId> keepAliveApplications,
    NodeHealthStatus nodeHealthStatus) {
  NodeStatus nodeStatus = Records.newRecord(NodeStatus.class);
  nodeStatus.setResponseId(responseId);
  nodeStatus.setNodeId(nodeId);
  nodeStatus.setContainersStatuses(containerStatuses);
  nodeStatus.setKeepAliveApplications(keepAliveApplications);
  nodeStatus.setNodeHealthStatus(nodeHealthStatus);
  return nodeStatus;
}
 
Example #19
Source File: NodeLabelTestBase.java    From big-c with Apache License 2.0 5 votes vote down vote up
public NodeId toNodeId(String str) {
  if (str.contains(":")) {
    int idx = str.indexOf(':');
    NodeId id =
        NodeId.newInstance(str.substring(0, idx),
            Integer.valueOf(str.substring(idx + 1)));
    return id;
  } else {
    return NodeId.newInstance(str, CommonNodeLabelsManager.WILDCARD_PORT);
  }
}
 
Example #20
Source File: TestAHSWebServices.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testSingleContainer() throws Exception {
  ApplicationId appId = ApplicationId.newInstance(0, 1);
  ApplicationAttemptId appAttemptId =
      ApplicationAttemptId.newInstance(appId, 1);
  ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1);
  WebResource r = resource();
  ClientResponse response =
      r.path("ws").path("v1").path("applicationhistory").path("apps")
        .path(appId.toString()).path("appattempts")
        .path(appAttemptId.toString()).path("containers")
        .path(containerId.toString())
        .queryParam("user.name", USERS[round])
        .accept(MediaType.APPLICATION_JSON)
        .get(ClientResponse.class);
  if (round == 1) {
    assertEquals(
        Status.FORBIDDEN, response.getClientResponseStatus());
    return;
  }
  assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
  JSONObject json = response.getEntity(JSONObject.class);
  assertEquals("incorrect number of elements", 1, json.length());
  JSONObject container = json.getJSONObject("container");
  assertEquals(containerId.toString(), container.getString("containerId"));
  assertEquals("test diagnostics info", container.getString("diagnosticsInfo"));
  assertEquals("-1", container.getString("allocatedMB"));
  assertEquals("-1", container.getString("allocatedVCores"));
  assertEquals(NodeId.newInstance("test host", 100).toString(),
    container.getString("assignedNodeId"));
  assertEquals("-1", container.getString("priority"));
  Configuration conf = new YarnConfiguration();
  assertEquals(WebAppUtils.getHttpSchemePrefix(conf) +
      WebAppUtils.getAHSWebAppURLWithoutScheme(conf) +
      "/applicationhistory/logs/test host:100/container_0_0001_01_000001/" +
      "container_0_0001_01_000001/user1", container.getString("logUrl"));
  assertEquals(ContainerState.COMPLETE.toString(),
    container.getString("containerState"));
}
 
Example #21
Source File: PerSourceNodeTracker.java    From tez with Apache License 2.0 5 votes vote down vote up
private void sendIngoreBlacklistingStateToNodes() {
  AMNodeEventType eventType =
      ignoreBlacklisting ? AMNodeEventType.N_IGNORE_BLACKLISTING_ENABLED
          : AMNodeEventType.N_IGNORE_BLACKLISTING_DISABLED;
  for (NodeId nodeId : nodeMap.keySet()) {
    sendEvent(new AMNodeEvent(nodeId, sourceId, eventType));
  }
}
 
Example #22
Source File: TestRMAdminCLI.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testReplaceLabelsOnNode() throws Exception {
  // Successfully replace labels
  dummyNodeLabelsManager
      .addToCluserNodeLabels(ImmutableSet.of("x", "y", "Y"));
  String[] args =
      { "-replaceLabelsOnNode",
          "node1:8000,x node2:8000=y node3,x node4=Y",
          "-directlyAccessNodeLabelStore" };
  assertEquals(0, rmAdminCLI.run(args));
  assertTrue(dummyNodeLabelsManager.getNodeLabels().containsKey(
      NodeId.newInstance("node1", 8000)));
  assertTrue(dummyNodeLabelsManager.getNodeLabels().containsKey(
      NodeId.newInstance("node2", 8000)));
  assertTrue(dummyNodeLabelsManager.getNodeLabels().containsKey(
      NodeId.newInstance("node3", 0)));
  assertTrue(dummyNodeLabelsManager.getNodeLabels().containsKey(
      NodeId.newInstance("node4", 0)));

  // no labels, should fail
  args = new String[] { "-replaceLabelsOnNode" };
  assertTrue(0 != rmAdminCLI.run(args));

  // no labels, should fail
  args =
      new String[] { "-replaceLabelsOnNode", "-directlyAccessNodeLabelStore" };
  assertTrue(0 != rmAdminCLI.run(args));

  // no labels, should fail
  args = new String[] { "-replaceLabelsOnNode", " " };
  assertTrue(0 != rmAdminCLI.run(args));

  args = new String[] { "-replaceLabelsOnNode", ", " };
  assertTrue(0 != rmAdminCLI.run(args));
}
 
Example #23
Source File: IgniteApplicationMasterSelfTest.java    From ignite with Apache License 2.0 5 votes vote down vote up
/**
 * @param host Host.
 * @param cpu Cpu count.
 * @param mem Memory.
 * @return Container.
 */
private Container createContainer(String host, int cpu, int mem) {
    return Container.newInstance(
        ContainerId.newContainerId(ApplicationAttemptId.newInstance(ApplicationId.newInstance(0l, 0), 0),
            ThreadLocalRandom.current().nextLong()),
        NodeId.newInstance(host, 0),
        "example.com",
        new MockResource(mem, cpu),
        Priority.newInstance(0),
        null
    );
}
 
Example #24
Source File: CapacityScheduler.java    From big-c with Apache License 2.0 5 votes vote down vote up
private synchronized void initScheduler(Configuration configuration) throws
    IOException {
  this.conf = loadCapacitySchedulerConfiguration(configuration);
  validateConf(this.conf);
  this.minimumAllocation = this.conf.getMinimumAllocation();
  initMaximumResourceCapability(this.conf.getMaximumAllocation());
  this.calculator = this.conf.getResourceCalculator();
  this.usePortForNodeName = this.conf.getUsePortForNodeName();
  
  this.nodeContainerUpdateMap = new HashMap<NodeId, ConcurrentLinkedQueue<NodeContainerUpdate>>();
 
  this.applications =
      new ConcurrentHashMap<ApplicationId,
          SchedulerApplication<FiCaSchedulerApp>>();
  this.labelManager = rmContext.getNodeLabelManager();
  authorizer = YarnAuthorizationProvider.getInstance(yarnConf);
  initializeQueues(this.conf);

  scheduleAsynchronously = this.conf.getScheduleAynschronously();
  asyncScheduleInterval =
      this.conf.getLong(ASYNC_SCHEDULER_INTERVAL,
          DEFAULT_ASYNC_SCHEDULER_INTERVAL);
  if (scheduleAsynchronously) {
    asyncSchedulerThread = new AsyncScheduleThread(this);
  }

  LOG.info("Initialized CapacityScheduler with " +
      "calculator=" + getResourceCalculator().getClass() + ", " +
      "minimumAllocation=<" + getMinimumResourceCapability() + ">, " +
      "maximumAllocation=<" + getMaximumResourceCapability() + ">, " +
      "asynchronousScheduling=" + scheduleAsynchronously + ", " +
      "asyncScheduleInterval=" + asyncScheduleInterval + "ms");
}
 
Example #25
Source File: TestContainerLauncherImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private Token createNewContainerToken(ContainerId contId,
    String containerManagerAddr) {
  long currentTime = System.currentTimeMillis();
  return MRApp.newContainerToken(NodeId.newInstance("127.0.0.1",
      1234), "password".getBytes(), new ContainerTokenIdentifier(
      contId, containerManagerAddr, "user",
      Resource.newInstance(1024, 1, 1),
      currentTime + 10000L, 123, currentTime, Priority.newInstance(0), 0));
}
 
Example #26
Source File: RMAdminCLI.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private int replaceLabelsOnNodes(Map<NodeId, Set<String>> map)
    throws IOException, YarnException {
  if (directlyAccessNodeLabelStore) {
    getNodeLabelManagerInstance(getConf()).replaceLabelsOnNode(map);
  } else {
    ResourceManagerAdministrationProtocol adminProtocol =
        createAdminProtocol();
    ReplaceLabelsOnNodeRequest request =
        ReplaceLabelsOnNodeRequest.newInstance(map);
    adminProtocol.replaceLabelsOnNode(request);
  }
  return 0;
}
 
Example #27
Source File: NodeLabelTestBase.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public static void assertMapEquals(Map<NodeId, Set<String>> m1,
    ImmutableMap<NodeId, Set<String>> m2) {
  Assert.assertEquals(m1.size(), m2.size());
  for (NodeId k : m1.keySet()) {
    Assert.assertTrue(m2.containsKey(k));
    assertCollectionEquals(m1.get(k), m2.get(k));
  }
}
 
Example #28
Source File: TestAMNodeTracker.java    From tez with Apache License 2.0 5 votes vote down vote up
@Test(timeout=10000)
public void testNodeSelfBlacklistAlternateScheduler2() {
  AppContext appContext = mock(AppContext.class);
  Configuration conf = new Configuration(false);
  conf.setInt(TezConfiguration.TEZ_AM_MAX_TASK_FAILURES_PER_NODE, 2);
  TestEventHandler handler = new TestEventHandler();
  AMNodeTracker amNodeTracker = new AMNodeTracker(handler, appContext);
  doReturn(amNodeTracker).when(appContext).getNodeTracker();
  AMContainerMap amContainerMap = mock(AMContainerMap.class);
  TaskSchedulerManager taskSchedulerManager =
      mock(TaskSchedulerManager.class);
  dispatcher.register(AMNodeEventType.class, amNodeTracker);
  dispatcher.register(AMContainerEventType.class, amContainerMap);
  dispatcher.register(AMSchedulerEventType.class, taskSchedulerManager);
  amNodeTracker.init(conf);
  amNodeTracker.start();
  try {
    // Register multiple nodes from a scheduler which isn't being tested.
    // This should not affect the blacklisting behaviour
    for (int i = 0 ; i < 100 ; i++) {
      amNodeTracker.nodeSeen(NodeId.newInstance("fakenode" + i, 3333), 0);
    }
    _testNodeSelfBlacklist(amNodeTracker, handler, 1);
    assertFalse(amNodeTracker.isBlacklistingIgnored(0));
  } finally {
    amNodeTracker.stop();
  }
}
 
Example #29
Source File: NMClientAsyncImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public ContainerEvent(ContainerId containerId, NodeId nodeId,
    Token containerToken, ContainerEventType type) {
  super(type);
  this.containerId = containerId;
  this.nodeId = nodeId;
  this.containerToken = containerToken;
}
 
Example #30
Source File: AMNodeMap.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
public void handle(AMNodeEvent rEvent) {
  // No synchronization required until there's multiple dispatchers.
  NodeId nodeId = rEvent.getNodeId();
  switch (rEvent.getType()) {
  case N_NODE_WAS_BLACKLISTED:
    // When moving away from IGNORE_BLACKLISTING state, nodes will send out
    // blacklisted events. These need to be ignored.
    addToBlackList(nodeId);
    computeIgnoreBlacklisting();
    break;
  case N_NODE_COUNT_UPDATED:
    AMNodeEventNodeCountUpdated event = (AMNodeEventNodeCountUpdated) rEvent;
    numClusterNodes = event.getNodeCount();
    LOG.info("Num cluster nodes = " + numClusterNodes);
    computeIgnoreBlacklisting();
    break;
  case N_TURNED_UNHEALTHY:
  case N_TURNED_HEALTHY:
    AMNode amNode = nodeMap.get(nodeId);
    if (amNode == null) {
      LOG.info("Ignoring RM Health Update for unknwon node: " + nodeId);
    } else {
      amNode.handle(rEvent);
    }
    break;
  default:
    nodeMap.get(nodeId).handle(rEvent);
  }
}