Java Code Examples for org.apache.hadoop.service.Service.STATE#STARTED

The following examples show how to use org.apache.hadoop.service.Service.STATE#STARTED . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: TestNodeStatusUpdater.java From hadoop with Apache License 2.0

5 votes

@Test
public void testApplicationKeepAlive() throws Exception {
  MyNodeManager nm = new MyNodeManager();
  try {
    YarnConfiguration conf = createNMConfig();
    conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
    conf.setLong(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS,
        4000l);
    nm.init(conf);
    nm.start();
    // HB 2 -> app cancelled by RM.
    while (heartBeatID < 12) {
      Thread.sleep(1000l);
    }
    MyResourceTracker3 rt =
        (MyResourceTracker3) nm.getNodeStatusUpdater().getRMClient();
    rt.context.getApplications().remove(rt.appId);
    Assert.assertEquals(1, rt.keepAliveRequests.size());
    int numKeepAliveRequests = rt.keepAliveRequests.get(rt.appId).size();
    LOG.info("Number of Keep Alive Requests: [" + numKeepAliveRequests + "]");
    Assert.assertTrue(numKeepAliveRequests == 2 || numKeepAliveRequests == 3);
    while (heartBeatID < 20) {
      Thread.sleep(1000l);
    }
    int numKeepAliveRequests2 = rt.keepAliveRequests.get(rt.appId).size();
    Assert.assertEquals(numKeepAliveRequests, numKeepAliveRequests2);
  } finally {
    if (nm.getServiceState() == STATE.STARTED)
      nm.stop();
  }
}

Example 2

Source File: TestResourceManagerAdministrationProtocolPBClientImpl.java From big-c with Apache License 2.0

5 votes

/**
 * Start resource manager server
 */

@BeforeClass
public static void setUpResourceManager() throws IOException,
        InterruptedException {
  Configuration.addDefaultResource("config-with-security.xml");
  Configuration configuration = new YarnConfiguration();
  resourceManager = new ResourceManager() {
    @Override
    protected void doSecureLogin() throws IOException {
    }
  };
  resourceManager.init(configuration);
  new Thread() {
    public void run() {
      resourceManager.start();
    }
  }.start();
  int waitCount = 0;
  while (resourceManager.getServiceState() == STATE.INITED
          && waitCount++ < 10) {
    LOG.info("Waiting for RM to start...");
    Thread.sleep(1000);
  }
  if (resourceManager.getServiceState() != STATE.STARTED) {
    throw new IOException("ResourceManager failed to start. Final state is "
            + resourceManager.getServiceState());
  }
  LOG.info("ResourceManager RMAdmin address: "
          + configuration.get(YarnConfiguration.RM_ADMIN_ADDRESS));

  client = new ResourceManagerAdministrationProtocolPBClientImpl(1L,
          getProtocolAddress(configuration), configuration);

}

Example 3

Source File: TestAMRMClient.java From big-c with Apache License 2.0

5 votes

@AfterClass
public static void tearDown() {
  if (yarnClient != null && yarnClient.getServiceState() == STATE.STARTED) {
    yarnClient.stop();
  }
  if (yarnCluster != null && yarnCluster.getServiceState() == STATE.STARTED) {
    yarnCluster.stop();
  }
}

Example 4

Source File: QueueACLsTestBase.java From big-c with Apache License 2.0

5 votes

@Before
public void setup() throws InterruptedException, IOException {
  conf = createConfiguration();
  rpc = YarnRPC.create(conf);
  rmAddress = conf.getSocketAddr(
    YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS,
    YarnConfiguration.DEFAULT_RM_PORT);
  
  AccessControlList adminACL = new AccessControlList("");
  conf.set(YarnConfiguration.YARN_ADMIN_ACL, adminACL.getAclString());

  resourceManager = new MockRM(conf) {
    protected ClientRMService createClientRMService() {
      return new ClientRMService(getRMContext(), this.scheduler,
        this.rmAppManager, this.applicationACLsManager,
        this.queueACLsManager, getRMContext().getRMDelegationTokenSecretManager());
    };

    @Override
    protected void doSecureLogin() throws IOException {
    }
  };
  new Thread() {
    public void run() {
      resourceManager.start();
    };
  }.start();
  int waitCount = 0;
  while (resourceManager.getServiceState() == STATE.INITED
      && waitCount++ < 60) {
    LOG.info("Waiting for RM to start...");
    Thread.sleep(1500);
  }
  if (resourceManager.getServiceState() != STATE.STARTED) {
    // RM could have failed.
    throw new IOException("ResourceManager failed to start. Final state is "
        + resourceManager.getServiceState());
  }
}

Example 5

Source File: TestGetGroups.java From big-c with Apache License 2.0

5 votes

@BeforeClass
public static void setUpResourceManager() throws IOException, InterruptedException {
  conf = new YarnConfiguration();
  resourceManager = new ResourceManager() {
    @Override
    protected void doSecureLogin() throws IOException {
    };
  };
  resourceManager.init(conf);
  new Thread() {
    public void run() {
      resourceManager.start();
    };
  }.start();
  int waitCount = 0;
  while (resourceManager.getServiceState() == STATE.INITED
      && waitCount++ < 10) {
    LOG.info("Waiting for RM to start...");
    Thread.sleep(1000);
  }
  if (resourceManager.getServiceState() != STATE.STARTED) {
    throw new IOException(
        "ResourceManager failed to start. Final state is "
            + resourceManager.getServiceState());
  }
  LOG.info("ResourceManager RMAdmin address: " +
      conf.get(YarnConfiguration.RM_ADMIN_ADDRESS));
}

Example 6

Source File: TestResourceManagerAdministrationProtocolPBClientImpl.java From hadoop with Apache License 2.0

5 votes

/**
 * Start resource manager server
 */

@BeforeClass
public static void setUpResourceManager() throws IOException,
        InterruptedException {
  Configuration.addDefaultResource("config-with-security.xml");
  Configuration configuration = new YarnConfiguration();
  resourceManager = new ResourceManager() {
    @Override
    protected void doSecureLogin() throws IOException {
    }
  };
  resourceManager.init(configuration);
  new Thread() {
    public void run() {
      resourceManager.start();
    }
  }.start();
  int waitCount = 0;
  while (resourceManager.getServiceState() == STATE.INITED
          && waitCount++ < 10) {
    LOG.info("Waiting for RM to start...");
    Thread.sleep(1000);
  }
  if (resourceManager.getServiceState() != STATE.STARTED) {
    throw new IOException("ResourceManager failed to start. Final state is "
            + resourceManager.getServiceState());
  }
  LOG.info("ResourceManager RMAdmin address: "
          + configuration.get(YarnConfiguration.RM_ADMIN_ADDRESS));

  client = new ResourceManagerAdministrationProtocolPBClientImpl(1L,
          getProtocolAddress(configuration), configuration);

}

Example 7

Source File: TestGetGroups.java From hadoop with Apache License 2.0

5 votes

@BeforeClass
public static void setUpResourceManager() throws IOException, InterruptedException {
  conf = new YarnConfiguration();
  resourceManager = new ResourceManager() {
    @Override
    protected void doSecureLogin() throws IOException {
    };
  };
  resourceManager.init(conf);
  new Thread() {
    public void run() {
      resourceManager.start();
    };
  }.start();
  int waitCount = 0;
  while (resourceManager.getServiceState() == STATE.INITED
      && waitCount++ < 10) {
    LOG.info("Waiting for RM to start...");
    Thread.sleep(1000);
  }
  if (resourceManager.getServiceState() != STATE.STARTED) {
    throw new IOException(
        "ResourceManager failed to start. Final state is "
            + resourceManager.getServiceState());
  }
  LOG.info("ResourceManager RMAdmin address: " +
      conf.get(YarnConfiguration.RM_ADMIN_ADDRESS));
}

Example 8

Source File: TestAMRMClient.java From hadoop with Apache License 2.0

5 votes

@Test (timeout=60000)
public void testAMRMClient() throws YarnException, IOException {
  AMRMClient<ContainerRequest> amClient = null;
  try {
    // start am rm client
    amClient = AMRMClient.<ContainerRequest>createAMRMClient();

    //setting an instance NMTokenCache
    amClient.setNMTokenCache(new NMTokenCache());
    //asserting we are not using the singleton instance cache
    Assert.assertNotSame(NMTokenCache.getSingleton(), 
        amClient.getNMTokenCache());

    amClient.init(conf);
    amClient.start();

    amClient.registerApplicationMaster("Host", 10000, "");

    testAllocation((AMRMClientImpl<ContainerRequest>)amClient);

    amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED,
        null, null);

  } finally {
    if (amClient != null && amClient.getServiceState() == STATE.STARTED) {
      amClient.stop();
    }
  }
}

Example 9

Source File: TestAMRMClient.java From hadoop with Apache License 2.0

5 votes

@AfterClass
public static void tearDown() {
  if (yarnClient != null && yarnClient.getServiceState() == STATE.STARTED) {
    yarnClient.stop();
  }
  if (yarnCluster != null && yarnCluster.getServiceState() == STATE.STARTED) {
    yarnCluster.stop();
  }
}

Example 10

Source File: TestAMRMClient.java From big-c with Apache License 2.0

5 votes

@Test (timeout=60000)
public void testAMRMClient() throws YarnException, IOException {
  AMRMClient<ContainerRequest> amClient = null;
  try {
    // start am rm client
    amClient = AMRMClient.<ContainerRequest>createAMRMClient();

    //setting an instance NMTokenCache
    amClient.setNMTokenCache(new NMTokenCache());
    //asserting we are not using the singleton instance cache
    Assert.assertNotSame(NMTokenCache.getSingleton(), 
        amClient.getNMTokenCache());

    amClient.init(conf);
    amClient.start();

    amClient.registerApplicationMaster("Host", 10000, "");

    testAllocation((AMRMClientImpl<ContainerRequest>)amClient);

    amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED,
        null, null);

  } finally {
    if (amClient != null && amClient.getServiceState() == STATE.STARTED) {
      amClient.stop();
    }
  }
}

Example 11

Source File: TestAMRMClient.java From hadoop with Apache License 2.0

4 votes

@Test (timeout=60000)
public void testAMRMClientWithBlacklist() throws YarnException, IOException {
  AMRMClientImpl<ContainerRequest> amClient = null;
  try {
    // start am rm client
    amClient =
        (AMRMClientImpl<ContainerRequest>) AMRMClient
          .<ContainerRequest> createAMRMClient();
    amClient.init(conf);
    amClient.start();
    amClient.registerApplicationMaster("Host", 10000, "");
    String[] nodes = {"node1", "node2", "node3"};
    
    // Add nodes[0] and nodes[1]
    List<String> nodeList01 = new ArrayList<String>();
    nodeList01.add(nodes[0]);
    nodeList01.add(nodes[1]);
    amClient.updateBlacklist(nodeList01, null);
    assertEquals(2, amClient.blacklistAdditions.size());
    assertEquals(0, amClient.blacklistRemovals.size());
    
    // Add nodes[0] again, verify it is not added duplicated.
    List<String> nodeList02 = new ArrayList<String>();
    nodeList02.add(nodes[0]);
    nodeList02.add(nodes[2]);
    amClient.updateBlacklist(nodeList02, null);
    assertEquals(3, amClient.blacklistAdditions.size());
    assertEquals(0, amClient.blacklistRemovals.size());
    
    // Add nodes[1] and nodes[2] to removal list, 
    // Verify addition list remove these two nodes.
    List<String> nodeList12 = new ArrayList<String>();
    nodeList12.add(nodes[1]);
    nodeList12.add(nodes[2]);
    amClient.updateBlacklist(null, nodeList12);
    assertEquals(1, amClient.blacklistAdditions.size());
    assertEquals(2, amClient.blacklistRemovals.size());
    
    // Add nodes[1] again to addition list, 
    // Verify removal list will remove this node.
    List<String> nodeList1 = new ArrayList<String>();
    nodeList1.add(nodes[1]);
    amClient.updateBlacklist(nodeList1, null);
    assertEquals(2, amClient.blacklistAdditions.size());
    assertEquals(1, amClient.blacklistRemovals.size());
  } finally {
    if (amClient != null && amClient.getServiceState() == STATE.STARTED) {
      amClient.stop();
    }
  }
}

Example 12

Source File: TestAMRMClient.java From hadoop with Apache License 2.0

4 votes

@Test (timeout=60000)
public void testAllocationWithBlacklist() throws YarnException, IOException {
  AMRMClientImpl<ContainerRequest> amClient = null;
  try {
    // start am rm client
    amClient =
        (AMRMClientImpl<ContainerRequest>) AMRMClient
          .<ContainerRequest> createAMRMClient();
    amClient.init(conf);
    amClient.start();
    amClient.registerApplicationMaster("Host", 10000, "");
    
    assertEquals(0, amClient.ask.size());
    assertEquals(0, amClient.release.size());
    
    ContainerRequest storedContainer1 = 
        new ContainerRequest(capability, nodes, racks, priority);
    amClient.addContainerRequest(storedContainer1);
    assertEquals(3, amClient.ask.size());
    assertEquals(0, amClient.release.size());
    
    List<String> localNodeBlacklist = new ArrayList<String>();
    localNodeBlacklist.add(node);
    
    // put node in black list, so no container assignment
    amClient.updateBlacklist(localNodeBlacklist, null);

    int allocatedContainerCount = getAllocatedContainersNumber(amClient,
      DEFAULT_ITERATION);
    // the only node is in blacklist, so no allocation
    assertEquals(0, allocatedContainerCount);

    // Remove node from blacklist, so get assigned with 2
    amClient.updateBlacklist(null, localNodeBlacklist);
    ContainerRequest storedContainer2 = 
            new ContainerRequest(capability, nodes, racks, priority);
    amClient.addContainerRequest(storedContainer2);
    allocatedContainerCount = getAllocatedContainersNumber(amClient,
        DEFAULT_ITERATION);
    assertEquals(2, allocatedContainerCount);
    
    // Test in case exception in allocate(), blacklist is kept
    assertTrue(amClient.blacklistAdditions.isEmpty());
    assertTrue(amClient.blacklistRemovals.isEmpty());
    
    // create a invalid ContainerRequest - memory value is minus
    ContainerRequest invalidContainerRequest = 
        new ContainerRequest(Resource.newInstance(-1024, 1, 1),
            nodes, racks, priority);
    amClient.addContainerRequest(invalidContainerRequest);
    amClient.updateBlacklist(localNodeBlacklist, null);
    try {
      // allocate() should complain as ContainerRequest is invalid.
      amClient.allocate(0.1f);
      fail("there should be an exception here.");
    } catch (Exception e) {
      assertEquals(1, amClient.blacklistAdditions.size());
    }
  } finally {
    if (amClient != null && amClient.getServiceState() == STATE.STARTED) {
      amClient.stop();
    }
  }
}

Example 13

Source File: TestAMRMClient.java From hadoop with Apache License 2.0

4 votes

@Test (timeout=60000)
public void testAMRMClientMatchingFitInferredRack() throws YarnException, IOException {
  AMRMClientImpl<ContainerRequest> amClient = null;
  try {
    // start am rm client
    amClient = new AMRMClientImpl<ContainerRequest>();
    amClient.init(conf);
    amClient.start();
    amClient.registerApplicationMaster("Host", 10000, "");
    
    Resource capability = Resource.newInstance(1024, 2, 2);

    ContainerRequest storedContainer1 = 
        new ContainerRequest(capability, nodes, null, priority);
    amClient.addContainerRequest(storedContainer1);

    // verify matching with original node and inferred rack
    List<? extends Collection<ContainerRequest>> matches;
    ContainerRequest storedRequest;
    // exact match node
    matches = amClient.getMatchingRequests(priority, node, capability);
    verifyMatches(matches, 1);
    storedRequest = matches.get(0).iterator().next();
    assertEquals(storedContainer1, storedRequest);
    // inferred match rack
    matches = amClient.getMatchingRequests(priority, rack, capability);
    verifyMatches(matches, 1);
    storedRequest = matches.get(0).iterator().next();
    assertEquals(storedContainer1, storedRequest);
    
    // inferred rack match no longer valid after request is removed
    amClient.removeContainerRequest(storedContainer1);
    matches = amClient.getMatchingRequests(priority, rack, capability);
    assertTrue(matches.isEmpty());
    
    amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED,
        null, null);

  } finally {
    if (amClient != null && amClient.getServiceState() == STATE.STARTED) {
      amClient.stop();
    }
  }
}

Example 14

Source File: TestNodeStatusUpdater.java From big-c with Apache License 2.0

4 votes

@Test
public void testNMRegistration() throws InterruptedException {
  nm = new NodeManager() {
    @Override
    protected NodeStatusUpdater createNodeStatusUpdater(Context context,
        Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
      return new MyNodeStatusUpdater(context, dispatcher, healthChecker,
                                     metrics);
    }
  };

  YarnConfiguration conf = createNMConfig();
  nm.init(conf);

  // verify that the last service is the nodeStatusUpdater (ie registration
  // with RM)
  Object[] services  = nm.getServices().toArray();
  Object lastService = services[services.length-1];
  Assert.assertTrue("last service is NOT the node status updater",
      lastService instanceof NodeStatusUpdater);

  new Thread() {
    public void run() {
      try {
        nm.start();
      } catch (Throwable e) {
        TestNodeStatusUpdater.this.nmStartError = e;
        throw new YarnRuntimeException(e);
      }
    }
  }.start();

  System.out.println(" ----- thread already started.."
      + nm.getServiceState());

  int waitCount = 0;
  while (nm.getServiceState() == STATE.INITED && waitCount++ != 50) {
    LOG.info("Waiting for NM to start..");
    if (nmStartError != null) {
      LOG.error("Error during startup. ", nmStartError);
      Assert.fail(nmStartError.getCause().getMessage());
    }
    Thread.sleep(2000);
  }
  if (nm.getServiceState() != STATE.STARTED) {
    // NM could have failed.
    Assert.fail("NodeManager failed to start");
  }

  waitCount = 0;
  while (heartBeatID <= 3 && waitCount++ != 200) {
    Thread.sleep(1000);
  }
  Assert.assertFalse(heartBeatID <= 3);
  Assert.assertEquals("Number of registered NMs is wrong!!", 1,
      this.registeredNodes.size());

  nm.stop();
}

Example 15

Source File: TestApplicationACLs.java From hadoop with Apache License 2.0

4 votes

@BeforeClass
public static void setup() throws InterruptedException, IOException {
  RMStateStore store = RMStateStoreFactory.getStore(conf);
  conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
  AccessControlList adminACL = new AccessControlList("");
  adminACL.addGroup(SUPER_GROUP);
  conf.set(YarnConfiguration.YARN_ADMIN_ACL, adminACL.getAclString());
  resourceManager = new MockRM(conf) {

    @Override
    protected QueueACLsManager createQueueACLsManager(
        ResourceScheduler scheduler,
        Configuration conf) {
      QueueACLsManager mockQueueACLsManager = mock(QueueACLsManager.class);
      when(mockQueueACLsManager.checkAccess(any(UserGroupInformation.class),
          any(QueueACL.class), anyString())).thenAnswer(new Answer() {
        public Object answer(InvocationOnMock invocation) {
          return isQueueUser;
        }
      });
      return mockQueueACLsManager;
    }

    protected ClientRMService createClientRMService() {
      return new ClientRMService(getRMContext(), this.scheduler,
          this.rmAppManager, this.applicationACLsManager,
          this.queueACLsManager, null);
    };
  };
  new Thread() {
    public void run() {
      UserGroupInformation.createUserForTesting(ENEMY, new String[] {});
      UserGroupInformation.createUserForTesting(FRIEND,
          new String[] { FRIENDLY_GROUP });
      UserGroupInformation.createUserForTesting(SUPER_USER,
          new String[] { SUPER_GROUP });
      resourceManager.start();
    };
  }.start();
  int waitCount = 0;
  while (resourceManager.getServiceState() == STATE.INITED
      && waitCount++ < 60) {
    LOG.info("Waiting for RM to start...");
    Thread.sleep(1500);
  }
  if (resourceManager.getServiceState() != STATE.STARTED) {
    // RM could have failed.
    throw new IOException(
        "ResourceManager failed to start. Final state is "
            + resourceManager.getServiceState());
  }

  UserGroupInformation owner = UserGroupInformation
      .createRemoteUser(APP_OWNER);
  rmClient = owner.doAs(new PrivilegedExceptionAction<ApplicationClientProtocol>() {
    @Override
    public ApplicationClientProtocol run() throws Exception {
      return (ApplicationClientProtocol) rpc.getProxy(ApplicationClientProtocol.class,
          rmAddress, conf);
    }
  });
}

Example 16

Source File: TestNodeStatusUpdater.java From big-c with Apache License 2.0

4 votes

@Test
public void testRMVersionLessThanMinimum() throws InterruptedException {
  final AtomicInteger numCleanups = new AtomicInteger(0);
  YarnConfiguration conf = createNMConfig();
  conf.set(YarnConfiguration.NM_RESOURCEMANAGER_MINIMUM_VERSION, "3.0.0");
  nm = new NodeManager() {
    @Override
    protected NodeStatusUpdater createNodeStatusUpdater(Context context,
                                                        Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
      MyNodeStatusUpdater myNodeStatusUpdater = new MyNodeStatusUpdater(
          context, dispatcher, healthChecker, metrics);
      MyResourceTracker2 myResourceTracker2 = new MyResourceTracker2();
      myResourceTracker2.heartBeatNodeAction = NodeAction.NORMAL;
      myResourceTracker2.rmVersion = "3.0.0";
      myNodeStatusUpdater.resourceTracker = myResourceTracker2;
      return myNodeStatusUpdater;
    }

    @Override
    protected ContainerManagerImpl createContainerManager(Context context,
        ContainerExecutor exec, DeletionService del,
        NodeStatusUpdater nodeStatusUpdater,
        ApplicationACLsManager aclsManager,
        LocalDirsHandlerService dirsHandler) {
      return new ContainerManagerImpl(context, exec, del, nodeStatusUpdater,
          metrics, aclsManager, dirsHandler) {

        @Override
        public void cleanUpApplicationsOnNMShutDown() {
          super.cleanUpApplicationsOnNMShutDown();
          numCleanups.incrementAndGet();
        }
      };
    }
  };

  nm.init(conf);
  nm.start();

  // NM takes a while to reach the STARTED state.
  int waitCount = 0;
  while (nm.getServiceState() != STATE.STARTED && waitCount++ != 20) {
    LOG.info("Waiting for NM to stop..");
    Thread.sleep(1000);
  }
  Assert.assertTrue(nm.getServiceState() == STATE.STARTED);
  nm.stop();
}

Example 17

Source File: TestAMRMClient.java From big-c with Apache License 2.0

4 votes

@Test (timeout=60000)
public void testAMRMClientWithBlacklist() throws YarnException, IOException {
  AMRMClientImpl<ContainerRequest> amClient = null;
  try {
    // start am rm client
    amClient =
        (AMRMClientImpl<ContainerRequest>) AMRMClient
          .<ContainerRequest> createAMRMClient();
    amClient.init(conf);
    amClient.start();
    amClient.registerApplicationMaster("Host", 10000, "");
    String[] nodes = {"node1", "node2", "node3"};
    
    // Add nodes[0] and nodes[1]
    List<String> nodeList01 = new ArrayList<String>();
    nodeList01.add(nodes[0]);
    nodeList01.add(nodes[1]);
    amClient.updateBlacklist(nodeList01, null);
    assertEquals(2, amClient.blacklistAdditions.size());
    assertEquals(0, amClient.blacklistRemovals.size());
    
    // Add nodes[0] again, verify it is not added duplicated.
    List<String> nodeList02 = new ArrayList<String>();
    nodeList02.add(nodes[0]);
    nodeList02.add(nodes[2]);
    amClient.updateBlacklist(nodeList02, null);
    assertEquals(3, amClient.blacklistAdditions.size());
    assertEquals(0, amClient.blacklistRemovals.size());
    
    // Add nodes[1] and nodes[2] to removal list, 
    // Verify addition list remove these two nodes.
    List<String> nodeList12 = new ArrayList<String>();
    nodeList12.add(nodes[1]);
    nodeList12.add(nodes[2]);
    amClient.updateBlacklist(null, nodeList12);
    assertEquals(1, amClient.blacklistAdditions.size());
    assertEquals(2, amClient.blacklistRemovals.size());
    
    // Add nodes[1] again to addition list, 
    // Verify removal list will remove this node.
    List<String> nodeList1 = new ArrayList<String>();
    nodeList1.add(nodes[1]);
    amClient.updateBlacklist(nodeList1, null);
    assertEquals(2, amClient.blacklistAdditions.size());
    assertEquals(1, amClient.blacklistRemovals.size());
  } finally {
    if (amClient != null && amClient.getServiceState() == STATE.STARTED) {
      amClient.stop();
    }
  }
}

Example 18

Source File: TestApplicationACLs.java From big-c with Apache License 2.0

4 votes

@BeforeClass
public static void setup() throws InterruptedException, IOException {
  RMStateStore store = RMStateStoreFactory.getStore(conf);
  conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
  AccessControlList adminACL = new AccessControlList("");
  adminACL.addGroup(SUPER_GROUP);
  conf.set(YarnConfiguration.YARN_ADMIN_ACL, adminACL.getAclString());
  resourceManager = new MockRM(conf) {

    @Override
    protected QueueACLsManager createQueueACLsManager(
        ResourceScheduler scheduler,
        Configuration conf) {
      QueueACLsManager mockQueueACLsManager = mock(QueueACLsManager.class);
      when(mockQueueACLsManager.checkAccess(any(UserGroupInformation.class),
          any(QueueACL.class), anyString())).thenAnswer(new Answer() {
        public Object answer(InvocationOnMock invocation) {
          return isQueueUser;
        }
      });
      return mockQueueACLsManager;
    }

    protected ClientRMService createClientRMService() {
      return new ClientRMService(getRMContext(), this.scheduler,
          this.rmAppManager, this.applicationACLsManager,
          this.queueACLsManager, null);
    };
  };
  new Thread() {
    public void run() {
      UserGroupInformation.createUserForTesting(ENEMY, new String[] {});
      UserGroupInformation.createUserForTesting(FRIEND,
          new String[] { FRIENDLY_GROUP });
      UserGroupInformation.createUserForTesting(SUPER_USER,
          new String[] { SUPER_GROUP });
      resourceManager.start();
    };
  }.start();
  int waitCount = 0;
  while (resourceManager.getServiceState() == STATE.INITED
      && waitCount++ < 60) {
    LOG.info("Waiting for RM to start...");
    Thread.sleep(1500);
  }
  if (resourceManager.getServiceState() != STATE.STARTED) {
    // RM could have failed.
    throw new IOException(
        "ResourceManager failed to start. Final state is "
            + resourceManager.getServiceState());
  }

  UserGroupInformation owner = UserGroupInformation
      .createRemoteUser(APP_OWNER);
  rmClient = owner.doAs(new PrivilegedExceptionAction<ApplicationClientProtocol>() {
    @Override
    public ApplicationClientProtocol run() throws Exception {
      return (ApplicationClientProtocol) rpc.getProxy(ApplicationClientProtocol.class,
          rmAddress, conf);
    }
  });
}

Example 19

Source File: TestNodeStatusUpdater.java From hadoop with Apache License 2.0

4 votes

@Test
public void testRMVersionLessThanMinimum() throws InterruptedException {
  final AtomicInteger numCleanups = new AtomicInteger(0);
  YarnConfiguration conf = createNMConfig();
  conf.set(YarnConfiguration.NM_RESOURCEMANAGER_MINIMUM_VERSION, "3.0.0");
  nm = new NodeManager() {
    @Override
    protected NodeStatusUpdater createNodeStatusUpdater(Context context,
                                                        Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
      MyNodeStatusUpdater myNodeStatusUpdater = new MyNodeStatusUpdater(
          context, dispatcher, healthChecker, metrics);
      MyResourceTracker2 myResourceTracker2 = new MyResourceTracker2();
      myResourceTracker2.heartBeatNodeAction = NodeAction.NORMAL;
      myResourceTracker2.rmVersion = "3.0.0";
      myNodeStatusUpdater.resourceTracker = myResourceTracker2;
      return myNodeStatusUpdater;
    }

    @Override
    protected ContainerManagerImpl createContainerManager(Context context,
        ContainerExecutor exec, DeletionService del,
        NodeStatusUpdater nodeStatusUpdater,
        ApplicationACLsManager aclsManager,
        LocalDirsHandlerService dirsHandler) {
      return new ContainerManagerImpl(context, exec, del, nodeStatusUpdater,
          metrics, aclsManager, dirsHandler) {

        @Override
        public void cleanUpApplicationsOnNMShutDown() {
          super.cleanUpApplicationsOnNMShutDown();
          numCleanups.incrementAndGet();
        }
      };
    }
  };

  nm.init(conf);
  nm.start();

  // NM takes a while to reach the STARTED state.
  int waitCount = 0;
  while (nm.getServiceState() != STATE.STARTED && waitCount++ != 20) {
    LOG.info("Waiting for NM to stop..");
    Thread.sleep(1000);
  }
  Assert.assertTrue(nm.getServiceState() == STATE.STARTED);
  nm.stop();
}

Example 20

Source File: TestAMRMClient.java From big-c with Apache License 2.0

4 votes

@Test (timeout=60000)
public void testAllocationWithBlacklist() throws YarnException, IOException {
  AMRMClientImpl<ContainerRequest> amClient = null;
  try {
    // start am rm client
    amClient =
        (AMRMClientImpl<ContainerRequest>) AMRMClient
          .<ContainerRequest> createAMRMClient();
    amClient.init(conf);
    amClient.start();
    amClient.registerApplicationMaster("Host", 10000, "");
    
    assertEquals(0, amClient.ask.size());
    assertEquals(0, amClient.release.size());
    
    ContainerRequest storedContainer1 = 
        new ContainerRequest(capability, nodes, racks, priority);
    amClient.addContainerRequest(storedContainer1);
    assertEquals(3, amClient.ask.size());
    assertEquals(0, amClient.release.size());
    
    List<String> localNodeBlacklist = new ArrayList<String>();
    localNodeBlacklist.add(node);
    
    // put node in black list, so no container assignment
    amClient.updateBlacklist(localNodeBlacklist, null);

    int allocatedContainerCount = getAllocatedContainersNumber(amClient,
      DEFAULT_ITERATION);
    // the only node is in blacklist, so no allocation
    assertEquals(0, allocatedContainerCount);

    // Remove node from blacklist, so get assigned with 2
    amClient.updateBlacklist(null, localNodeBlacklist);
    ContainerRequest storedContainer2 = 
            new ContainerRequest(capability, nodes, racks, priority);
    amClient.addContainerRequest(storedContainer2);
    allocatedContainerCount = getAllocatedContainersNumber(amClient,
        DEFAULT_ITERATION);
    assertEquals(2, allocatedContainerCount);
    
    // Test in case exception in allocate(), blacklist is kept
    assertTrue(amClient.blacklistAdditions.isEmpty());
    assertTrue(amClient.blacklistRemovals.isEmpty());
    
    // create a invalid ContainerRequest - memory value is minus
    ContainerRequest invalidContainerRequest = 
        new ContainerRequest(Resource.newInstance(-1024, 1),
            nodes, racks, priority);
    amClient.addContainerRequest(invalidContainerRequest);
    amClient.updateBlacklist(localNodeBlacklist, null);
    try {
      // allocate() should complain as ContainerRequest is invalid.
      amClient.allocate(0.1f);
      fail("there should be an exception here.");
    } catch (Exception e) {
      assertEquals(1, amClient.blacklistAdditions.size());
    }
  } finally {
    if (amClient != null && amClient.getServiceState() == STATE.STARTED) {
      amClient.stop();
    }
  }
}