Java Code Examples for org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse#getNMTokensFromPreviousAttempts()

The following examples show how to use org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse#getNMTokensFromPreviousAttempts() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: TestAMRestart.java From hadoop with Apache License 2.0

4 votes

@Test(timeout = 30000)
public void testNMTokensRebindOnAMRestart() throws Exception {
  YarnConfiguration conf = new YarnConfiguration();
  conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 3);

  MockRM rm1 = new MockRM(conf);
  rm1.start();
  RMApp app1 =
      rm1.submitApp(200, "myname", "myuser",
        new HashMap<ApplicationAccessType, String>(), false, "default", -1,
        null, "MAPREDUCE", false, true);
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", 8000, rm1.getResourceTrackerService());
  nm1.registerNode();
  MockNM nm2 =
      new MockNM("127.1.1.1:4321", 8000, rm1.getResourceTrackerService());
  nm2.registerNode();
  MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);

  List<Container> containers = new ArrayList<Container>();
  // nmTokens keeps track of all the nmTokens issued in the allocate call.
  List<NMToken> expectedNMTokens = new ArrayList<NMToken>();

  // am1 allocate 2 container on nm1.
  // first container
  while (true) {
    AllocateResponse response =
        am1.allocate("127.0.0.1", 2000, 2,
          new ArrayList<ContainerId>());
    nm1.nodeHeartbeat(true);
    containers.addAll(response.getAllocatedContainers());
    expectedNMTokens.addAll(response.getNMTokens());
    if (containers.size() == 2) {
      break;
    }
    Thread.sleep(200);
    System.out.println("Waiting for container to be allocated.");
  }
  // launch the container-2
  nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 2, ContainerState.RUNNING);
  ContainerId containerId2 =
      ContainerId.newContainerId(am1.getApplicationAttemptId(), 2);
  rm1.waitForState(nm1, containerId2, RMContainerState.RUNNING);
  // launch the container-3
  nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 3, ContainerState.RUNNING);
  ContainerId containerId3 =
      ContainerId.newContainerId(am1.getApplicationAttemptId(), 3);
  rm1.waitForState(nm1, containerId3, RMContainerState.RUNNING);
  
  // fail am1
  nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
  am1.waitForState(RMAppAttemptState.FAILED);
  rm1.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED);

  // restart the am
  MockAM am2 = MockRM.launchAM(app1, rm1, nm1);
  RegisterApplicationMasterResponse registerResponse =
      am2.registerAppAttempt();
  rm1.waitForState(app1.getApplicationId(), RMAppState.RUNNING);

  // check am2 get the nm token from am1.
  Assert.assertEquals(expectedNMTokens,
    registerResponse.getNMTokensFromPreviousAttempts());

  // am2 allocate 1 container on nm2
  containers = new ArrayList<Container>();
  while (true) {
    AllocateResponse allocateResponse =
        am2.allocate("127.1.1.1", 4000, 1,
          new ArrayList<ContainerId>());
    nm2.nodeHeartbeat(true);
    containers.addAll(allocateResponse.getAllocatedContainers());
    expectedNMTokens.addAll(allocateResponse.getNMTokens());
    if (containers.size() == 1) {
      break;
    }
    Thread.sleep(200);
    System.out.println("Waiting for container to be allocated.");
  }
  nm1.nodeHeartbeat(am2.getApplicationAttemptId(), 2, ContainerState.RUNNING);
  ContainerId am2ContainerId2 =
      ContainerId.newContainerId(am2.getApplicationAttemptId(), 2);
  rm1.waitForState(nm1, am2ContainerId2, RMContainerState.RUNNING);

  // fail am2.
  nm1.nodeHeartbeat(am2.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
  am2.waitForState(RMAppAttemptState.FAILED);
  rm1.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED);

  // restart am
  MockAM am3 = MockRM.launchAM(app1, rm1, nm1);
  registerResponse = am3.registerAppAttempt();
  rm1.waitForState(app1.getApplicationId(), RMAppState.RUNNING);

  // check am3 get the NM token from both am1 and am2;
  List<NMToken> transferredTokens = registerResponse.getNMTokensFromPreviousAttempts();
  Assert.assertEquals(2, transferredTokens.size());
  Assert.assertTrue(transferredTokens.containsAll(expectedNMTokens));
  rm1.stop();
}

Example 2

Source File: TestAMRestart.java From big-c with Apache License 2.0

4 votes

@Test(timeout = 30000)
public void testNMTokensRebindOnAMRestart() throws Exception {
  YarnConfiguration conf = new YarnConfiguration();
  conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 3);

  MockRM rm1 = new MockRM(conf);
  rm1.start();
  RMApp app1 =
      rm1.submitApp(200, "myname", "myuser",
        new HashMap<ApplicationAccessType, String>(), false, "default", -1,
        null, "MAPREDUCE", false, true);
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", 8000, rm1.getResourceTrackerService());
  nm1.registerNode();
  MockNM nm2 =
      new MockNM("127.1.1.1:4321", 8000, rm1.getResourceTrackerService());
  nm2.registerNode();
  MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);

  List<Container> containers = new ArrayList<Container>();
  // nmTokens keeps track of all the nmTokens issued in the allocate call.
  List<NMToken> expectedNMTokens = new ArrayList<NMToken>();

  // am1 allocate 2 container on nm1.
  // first container
  while (true) {
    AllocateResponse response =
        am1.allocate("127.0.0.1", 2000, 2,
          new ArrayList<ContainerId>());
    nm1.nodeHeartbeat(true);
    containers.addAll(response.getAllocatedContainers());
    expectedNMTokens.addAll(response.getNMTokens());
    if (containers.size() == 2) {
      break;
    }
    Thread.sleep(200);
    System.out.println("Waiting for container to be allocated.");
  }
  // launch the container-2
  nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 2, ContainerState.RUNNING);
  ContainerId containerId2 =
      ContainerId.newContainerId(am1.getApplicationAttemptId(), 2);
  rm1.waitForState(nm1, containerId2, RMContainerState.RUNNING);
  // launch the container-3
  nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 3, ContainerState.RUNNING);
  ContainerId containerId3 =
      ContainerId.newContainerId(am1.getApplicationAttemptId(), 3);
  rm1.waitForState(nm1, containerId3, RMContainerState.RUNNING);
  
  // fail am1
  nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
  am1.waitForState(RMAppAttemptState.FAILED);
  rm1.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED);

  // restart the am
  MockAM am2 = MockRM.launchAM(app1, rm1, nm1);
  RegisterApplicationMasterResponse registerResponse =
      am2.registerAppAttempt();
  rm1.waitForState(app1.getApplicationId(), RMAppState.RUNNING);

  // check am2 get the nm token from am1.
  Assert.assertEquals(expectedNMTokens,
    registerResponse.getNMTokensFromPreviousAttempts());

  // am2 allocate 1 container on nm2
  containers = new ArrayList<Container>();
  while (true) {
    AllocateResponse allocateResponse =
        am2.allocate("127.1.1.1", 4000, 1,
          new ArrayList<ContainerId>());
    nm2.nodeHeartbeat(true);
    containers.addAll(allocateResponse.getAllocatedContainers());
    expectedNMTokens.addAll(allocateResponse.getNMTokens());
    if (containers.size() == 1) {
      break;
    }
    Thread.sleep(200);
    System.out.println("Waiting for container to be allocated.");
  }
  nm1.nodeHeartbeat(am2.getApplicationAttemptId(), 2, ContainerState.RUNNING);
  ContainerId am2ContainerId2 =
      ContainerId.newContainerId(am2.getApplicationAttemptId(), 2);
  rm1.waitForState(nm1, am2ContainerId2, RMContainerState.RUNNING);

  // fail am2.
  nm1.nodeHeartbeat(am2.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
  am2.waitForState(RMAppAttemptState.FAILED);
  rm1.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED);

  // restart am
  MockAM am3 = MockRM.launchAM(app1, rm1, nm1);
  registerResponse = am3.registerAppAttempt();
  rm1.waitForState(app1.getApplicationId(), RMAppState.RUNNING);

  // check am3 get the NM token from both am1 and am2;
  List<NMToken> transferredTokens = registerResponse.getNMTokensFromPreviousAttempts();
  Assert.assertEquals(2, transferredTokens.size());
  Assert.assertTrue(transferredTokens.containsAll(expectedNMTokens));
  rm1.stop();
}