Java Code Examples for org.apache.hadoop.service.Service.STATE#STARTED

The following examples show how to use org.apache.hadoop.service.Service.STATE#STARTED . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: big-c   File: TestAMRMClient.java    License: Apache License 2.0 5 votes vote down vote up
@Test (timeout=60000)
public void testAMRMClient() throws YarnException, IOException {
  AMRMClient<ContainerRequest> amClient = null;
  try {
    // start am rm client
    amClient = AMRMClient.<ContainerRequest>createAMRMClient();

    //setting an instance NMTokenCache
    amClient.setNMTokenCache(new NMTokenCache());
    //asserting we are not using the singleton instance cache
    Assert.assertNotSame(NMTokenCache.getSingleton(), 
        amClient.getNMTokenCache());

    amClient.init(conf);
    amClient.start();

    amClient.registerApplicationMaster("Host", 10000, "");

    testAllocation((AMRMClientImpl<ContainerRequest>)amClient);

    amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED,
        null, null);

  } finally {
    if (amClient != null && amClient.getServiceState() == STATE.STARTED) {
      amClient.stop();
    }
  }
}
 
Example 2
Source Project: hadoop   File: TestNodeStatusUpdater.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testApplicationKeepAlive() throws Exception {
  MyNodeManager nm = new MyNodeManager();
  try {
    YarnConfiguration conf = createNMConfig();
    conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
    conf.setLong(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS,
        4000l);
    nm.init(conf);
    nm.start();
    // HB 2 -> app cancelled by RM.
    while (heartBeatID < 12) {
      Thread.sleep(1000l);
    }
    MyResourceTracker3 rt =
        (MyResourceTracker3) nm.getNodeStatusUpdater().getRMClient();
    rt.context.getApplications().remove(rt.appId);
    Assert.assertEquals(1, rt.keepAliveRequests.size());
    int numKeepAliveRequests = rt.keepAliveRequests.get(rt.appId).size();
    LOG.info("Number of Keep Alive Requests: [" + numKeepAliveRequests + "]");
    Assert.assertTrue(numKeepAliveRequests == 2 || numKeepAliveRequests == 3);
    while (heartBeatID < 20) {
      Thread.sleep(1000l);
    }
    int numKeepAliveRequests2 = rt.keepAliveRequests.get(rt.appId).size();
    Assert.assertEquals(numKeepAliveRequests, numKeepAliveRequests2);
  } finally {
    if (nm.getServiceState() == STATE.STARTED)
      nm.stop();
  }
}
 
Example 3
Source Project: hadoop   File: TestAMRMClient.java    License: Apache License 2.0 5 votes vote down vote up
@AfterClass
public static void tearDown() {
  if (yarnClient != null && yarnClient.getServiceState() == STATE.STARTED) {
    yarnClient.stop();
  }
  if (yarnCluster != null && yarnCluster.getServiceState() == STATE.STARTED) {
    yarnCluster.stop();
  }
}
 
Example 4
Source Project: hadoop   File: TestAMRMClient.java    License: Apache License 2.0 5 votes vote down vote up
@Test (timeout=60000)
public void testAMRMClient() throws YarnException, IOException {
  AMRMClient<ContainerRequest> amClient = null;
  try {
    // start am rm client
    amClient = AMRMClient.<ContainerRequest>createAMRMClient();

    //setting an instance NMTokenCache
    amClient.setNMTokenCache(new NMTokenCache());
    //asserting we are not using the singleton instance cache
    Assert.assertNotSame(NMTokenCache.getSingleton(), 
        amClient.getNMTokenCache());

    amClient.init(conf);
    amClient.start();

    amClient.registerApplicationMaster("Host", 10000, "");

    testAllocation((AMRMClientImpl<ContainerRequest>)amClient);

    amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED,
        null, null);

  } finally {
    if (amClient != null && amClient.getServiceState() == STATE.STARTED) {
      amClient.stop();
    }
  }
}
 
Example 5
Source Project: hadoop   File: TestGetGroups.java    License: Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void setUpResourceManager() throws IOException, InterruptedException {
  conf = new YarnConfiguration();
  resourceManager = new ResourceManager() {
    @Override
    protected void doSecureLogin() throws IOException {
    };
  };
  resourceManager.init(conf);
  new Thread() {
    public void run() {
      resourceManager.start();
    };
  }.start();
  int waitCount = 0;
  while (resourceManager.getServiceState() == STATE.INITED
      && waitCount++ < 10) {
    LOG.info("Waiting for RM to start...");
    Thread.sleep(1000);
  }
  if (resourceManager.getServiceState() != STATE.STARTED) {
    throw new IOException(
        "ResourceManager failed to start. Final state is "
            + resourceManager.getServiceState());
  }
  LOG.info("ResourceManager RMAdmin address: " +
      conf.get(YarnConfiguration.RM_ADMIN_ADDRESS));
}
 
Example 6
/**
 * Start resource manager server
 */

@BeforeClass
public static void setUpResourceManager() throws IOException,
        InterruptedException {
  Configuration.addDefaultResource("config-with-security.xml");
  Configuration configuration = new YarnConfiguration();
  resourceManager = new ResourceManager() {
    @Override
    protected void doSecureLogin() throws IOException {
    }
  };
  resourceManager.init(configuration);
  new Thread() {
    public void run() {
      resourceManager.start();
    }
  }.start();
  int waitCount = 0;
  while (resourceManager.getServiceState() == STATE.INITED
          && waitCount++ < 10) {
    LOG.info("Waiting for RM to start...");
    Thread.sleep(1000);
  }
  if (resourceManager.getServiceState() != STATE.STARTED) {
    throw new IOException("ResourceManager failed to start. Final state is "
            + resourceManager.getServiceState());
  }
  LOG.info("ResourceManager RMAdmin address: "
          + configuration.get(YarnConfiguration.RM_ADMIN_ADDRESS));

  client = new ResourceManagerAdministrationProtocolPBClientImpl(1L,
          getProtocolAddress(configuration), configuration);

}
 
Example 7
Source Project: big-c   File: TestGetGroups.java    License: Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void setUpResourceManager() throws IOException, InterruptedException {
  conf = new YarnConfiguration();
  resourceManager = new ResourceManager() {
    @Override
    protected void doSecureLogin() throws IOException {
    };
  };
  resourceManager.init(conf);
  new Thread() {
    public void run() {
      resourceManager.start();
    };
  }.start();
  int waitCount = 0;
  while (resourceManager.getServiceState() == STATE.INITED
      && waitCount++ < 10) {
    LOG.info("Waiting for RM to start...");
    Thread.sleep(1000);
  }
  if (resourceManager.getServiceState() != STATE.STARTED) {
    throw new IOException(
        "ResourceManager failed to start. Final state is "
            + resourceManager.getServiceState());
  }
  LOG.info("ResourceManager RMAdmin address: " +
      conf.get(YarnConfiguration.RM_ADMIN_ADDRESS));
}
 
Example 8
Source Project: big-c   File: QueueACLsTestBase.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void setup() throws InterruptedException, IOException {
  conf = createConfiguration();
  rpc = YarnRPC.create(conf);
  rmAddress = conf.getSocketAddr(
    YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS,
    YarnConfiguration.DEFAULT_RM_PORT);
  
  AccessControlList adminACL = new AccessControlList("");
  conf.set(YarnConfiguration.YARN_ADMIN_ACL, adminACL.getAclString());

  resourceManager = new MockRM(conf) {
    protected ClientRMService createClientRMService() {
      return new ClientRMService(getRMContext(), this.scheduler,
        this.rmAppManager, this.applicationACLsManager,
        this.queueACLsManager, getRMContext().getRMDelegationTokenSecretManager());
    };

    @Override
    protected void doSecureLogin() throws IOException {
    }
  };
  new Thread() {
    public void run() {
      resourceManager.start();
    };
  }.start();
  int waitCount = 0;
  while (resourceManager.getServiceState() == STATE.INITED
      && waitCount++ < 60) {
    LOG.info("Waiting for RM to start...");
    Thread.sleep(1500);
  }
  if (resourceManager.getServiceState() != STATE.STARTED) {
    // RM could have failed.
    throw new IOException("ResourceManager failed to start. Final state is "
        + resourceManager.getServiceState());
  }
}
 
Example 9
Source Project: big-c   File: TestAMRMClient.java    License: Apache License 2.0 5 votes vote down vote up
@AfterClass
public static void tearDown() {
  if (yarnClient != null && yarnClient.getServiceState() == STATE.STARTED) {
    yarnClient.stop();
  }
  if (yarnCluster != null && yarnCluster.getServiceState() == STATE.STARTED) {
    yarnCluster.stop();
  }
}
 
Example 10
/**
 * Start resource manager server
 */

@BeforeClass
public static void setUpResourceManager() throws IOException,
        InterruptedException {
  Configuration.addDefaultResource("config-with-security.xml");
  Configuration configuration = new YarnConfiguration();
  resourceManager = new ResourceManager() {
    @Override
    protected void doSecureLogin() throws IOException {
    }
  };
  resourceManager.init(configuration);
  new Thread() {
    public void run() {
      resourceManager.start();
    }
  }.start();
  int waitCount = 0;
  while (resourceManager.getServiceState() == STATE.INITED
          && waitCount++ < 10) {
    LOG.info("Waiting for RM to start...");
    Thread.sleep(1000);
  }
  if (resourceManager.getServiceState() != STATE.STARTED) {
    throw new IOException("ResourceManager failed to start. Final state is "
            + resourceManager.getServiceState());
  }
  LOG.info("ResourceManager RMAdmin address: "
          + configuration.get(YarnConfiguration.RM_ADMIN_ADDRESS));

  client = new ResourceManagerAdministrationProtocolPBClientImpl(1L,
          getProtocolAddress(configuration), configuration);

}
 
Example 11
Source Project: hadoop   File: TestNodeStatusUpdater.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testRMVersionLessThanMinimum() throws InterruptedException {
  final AtomicInteger numCleanups = new AtomicInteger(0);
  YarnConfiguration conf = createNMConfig();
  conf.set(YarnConfiguration.NM_RESOURCEMANAGER_MINIMUM_VERSION, "3.0.0");
  nm = new NodeManager() {
    @Override
    protected NodeStatusUpdater createNodeStatusUpdater(Context context,
                                                        Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
      MyNodeStatusUpdater myNodeStatusUpdater = new MyNodeStatusUpdater(
          context, dispatcher, healthChecker, metrics);
      MyResourceTracker2 myResourceTracker2 = new MyResourceTracker2();
      myResourceTracker2.heartBeatNodeAction = NodeAction.NORMAL;
      myResourceTracker2.rmVersion = "3.0.0";
      myNodeStatusUpdater.resourceTracker = myResourceTracker2;
      return myNodeStatusUpdater;
    }

    @Override
    protected ContainerManagerImpl createContainerManager(Context context,
        ContainerExecutor exec, DeletionService del,
        NodeStatusUpdater nodeStatusUpdater,
        ApplicationACLsManager aclsManager,
        LocalDirsHandlerService dirsHandler) {
      return new ContainerManagerImpl(context, exec, del, nodeStatusUpdater,
          metrics, aclsManager, dirsHandler) {

        @Override
        public void cleanUpApplicationsOnNMShutDown() {
          super.cleanUpApplicationsOnNMShutDown();
          numCleanups.incrementAndGet();
        }
      };
    }
  };

  nm.init(conf);
  nm.start();

  // NM takes a while to reach the STARTED state.
  int waitCount = 0;
  while (nm.getServiceState() != STATE.STARTED && waitCount++ != 20) {
    LOG.info("Waiting for NM to stop..");
    Thread.sleep(1000);
  }
  Assert.assertTrue(nm.getServiceState() == STATE.STARTED);
  nm.stop();
}
 
Example 12
Source Project: big-c   File: TestAMRMClient.java    License: Apache License 2.0 4 votes vote down vote up
@Test (timeout=60000)
public void testAMRMClientWithBlacklist() throws YarnException, IOException {
  AMRMClientImpl<ContainerRequest> amClient = null;
  try {
    // start am rm client
    amClient =
        (AMRMClientImpl<ContainerRequest>) AMRMClient
          .<ContainerRequest> createAMRMClient();
    amClient.init(conf);
    amClient.start();
    amClient.registerApplicationMaster("Host", 10000, "");
    String[] nodes = {"node1", "node2", "node3"};
    
    // Add nodes[0] and nodes[1]
    List<String> nodeList01 = new ArrayList<String>();
    nodeList01.add(nodes[0]);
    nodeList01.add(nodes[1]);
    amClient.updateBlacklist(nodeList01, null);
    assertEquals(2, amClient.blacklistAdditions.size());
    assertEquals(0, amClient.blacklistRemovals.size());
    
    // Add nodes[0] again, verify it is not added duplicated.
    List<String> nodeList02 = new ArrayList<String>();
    nodeList02.add(nodes[0]);
    nodeList02.add(nodes[2]);
    amClient.updateBlacklist(nodeList02, null);
    assertEquals(3, amClient.blacklistAdditions.size());
    assertEquals(0, amClient.blacklistRemovals.size());
    
    // Add nodes[1] and nodes[2] to removal list, 
    // Verify addition list remove these two nodes.
    List<String> nodeList12 = new ArrayList<String>();
    nodeList12.add(nodes[1]);
    nodeList12.add(nodes[2]);
    amClient.updateBlacklist(null, nodeList12);
    assertEquals(1, amClient.blacklistAdditions.size());
    assertEquals(2, amClient.blacklistRemovals.size());
    
    // Add nodes[1] again to addition list, 
    // Verify removal list will remove this node.
    List<String> nodeList1 = new ArrayList<String>();
    nodeList1.add(nodes[1]);
    amClient.updateBlacklist(nodeList1, null);
    assertEquals(2, amClient.blacklistAdditions.size());
    assertEquals(1, amClient.blacklistRemovals.size());
  } finally {
    if (amClient != null && amClient.getServiceState() == STATE.STARTED) {
      amClient.stop();
    }
  }
}
 
Example 13
Source Project: hadoop   File: TestApplicationACLs.java    License: Apache License 2.0 4 votes vote down vote up
@BeforeClass
public static void setup() throws InterruptedException, IOException {
  RMStateStore store = RMStateStoreFactory.getStore(conf);
  conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
  AccessControlList adminACL = new AccessControlList("");
  adminACL.addGroup(SUPER_GROUP);
  conf.set(YarnConfiguration.YARN_ADMIN_ACL, adminACL.getAclString());
  resourceManager = new MockRM(conf) {

    @Override
    protected QueueACLsManager createQueueACLsManager(
        ResourceScheduler scheduler,
        Configuration conf) {
      QueueACLsManager mockQueueACLsManager = mock(QueueACLsManager.class);
      when(mockQueueACLsManager.checkAccess(any(UserGroupInformation.class),
          any(QueueACL.class), anyString())).thenAnswer(new Answer() {
        public Object answer(InvocationOnMock invocation) {
          return isQueueUser;
        }
      });
      return mockQueueACLsManager;
    }

    protected ClientRMService createClientRMService() {
      return new ClientRMService(getRMContext(), this.scheduler,
          this.rmAppManager, this.applicationACLsManager,
          this.queueACLsManager, null);
    };
  };
  new Thread() {
    public void run() {
      UserGroupInformation.createUserForTesting(ENEMY, new String[] {});
      UserGroupInformation.createUserForTesting(FRIEND,
          new String[] { FRIENDLY_GROUP });
      UserGroupInformation.createUserForTesting(SUPER_USER,
          new String[] { SUPER_GROUP });
      resourceManager.start();
    };
  }.start();
  int waitCount = 0;
  while (resourceManager.getServiceState() == STATE.INITED
      && waitCount++ < 60) {
    LOG.info("Waiting for RM to start...");
    Thread.sleep(1500);
  }
  if (resourceManager.getServiceState() != STATE.STARTED) {
    // RM could have failed.
    throw new IOException(
        "ResourceManager failed to start. Final state is "
            + resourceManager.getServiceState());
  }

  UserGroupInformation owner = UserGroupInformation
      .createRemoteUser(APP_OWNER);
  rmClient = owner.doAs(new PrivilegedExceptionAction<ApplicationClientProtocol>() {
    @Override
    public ApplicationClientProtocol run() throws Exception {
      return (ApplicationClientProtocol) rpc.getProxy(ApplicationClientProtocol.class,
          rmAddress, conf);
    }
  });
}
 
Example 14
Source Project: hadoop   File: TestAMRMClient.java    License: Apache License 2.0 4 votes vote down vote up
@Test (timeout=60000)
public void testAMRMClientMatchingFitInferredRack() throws YarnException, IOException {
  AMRMClientImpl<ContainerRequest> amClient = null;
  try {
    // start am rm client
    amClient = new AMRMClientImpl<ContainerRequest>();
    amClient.init(conf);
    amClient.start();
    amClient.registerApplicationMaster("Host", 10000, "");
    
    Resource capability = Resource.newInstance(1024, 2, 2);

    ContainerRequest storedContainer1 = 
        new ContainerRequest(capability, nodes, null, priority);
    amClient.addContainerRequest(storedContainer1);

    // verify matching with original node and inferred rack
    List<? extends Collection<ContainerRequest>> matches;
    ContainerRequest storedRequest;
    // exact match node
    matches = amClient.getMatchingRequests(priority, node, capability);
    verifyMatches(matches, 1);
    storedRequest = matches.get(0).iterator().next();
    assertEquals(storedContainer1, storedRequest);
    // inferred match rack
    matches = amClient.getMatchingRequests(priority, rack, capability);
    verifyMatches(matches, 1);
    storedRequest = matches.get(0).iterator().next();
    assertEquals(storedContainer1, storedRequest);
    
    // inferred rack match no longer valid after request is removed
    amClient.removeContainerRequest(storedContainer1);
    matches = amClient.getMatchingRequests(priority, rack, capability);
    assertTrue(matches.isEmpty());
    
    amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED,
        null, null);

  } finally {
    if (amClient != null && amClient.getServiceState() == STATE.STARTED) {
      amClient.stop();
    }
  }
}
 
Example 15
Source Project: hadoop   File: TestAMRMClient.java    License: Apache License 2.0 4 votes vote down vote up
@Test (timeout=60000)
public void testAllocationWithBlacklist() throws YarnException, IOException {
  AMRMClientImpl<ContainerRequest> amClient = null;
  try {
    // start am rm client
    amClient =
        (AMRMClientImpl<ContainerRequest>) AMRMClient
          .<ContainerRequest> createAMRMClient();
    amClient.init(conf);
    amClient.start();
    amClient.registerApplicationMaster("Host", 10000, "");
    
    assertEquals(0, amClient.ask.size());
    assertEquals(0, amClient.release.size());
    
    ContainerRequest storedContainer1 = 
        new ContainerRequest(capability, nodes, racks, priority);
    amClient.addContainerRequest(storedContainer1);
    assertEquals(3, amClient.ask.size());
    assertEquals(0, amClient.release.size());
    
    List<String> localNodeBlacklist = new ArrayList<String>();
    localNodeBlacklist.add(node);
    
    // put node in black list, so no container assignment
    amClient.updateBlacklist(localNodeBlacklist, null);

    int allocatedContainerCount = getAllocatedContainersNumber(amClient,
      DEFAULT_ITERATION);
    // the only node is in blacklist, so no allocation
    assertEquals(0, allocatedContainerCount);

    // Remove node from blacklist, so get assigned with 2
    amClient.updateBlacklist(null, localNodeBlacklist);
    ContainerRequest storedContainer2 = 
            new ContainerRequest(capability, nodes, racks, priority);
    amClient.addContainerRequest(storedContainer2);
    allocatedContainerCount = getAllocatedContainersNumber(amClient,
        DEFAULT_ITERATION);
    assertEquals(2, allocatedContainerCount);
    
    // Test in case exception in allocate(), blacklist is kept
    assertTrue(amClient.blacklistAdditions.isEmpty());
    assertTrue(amClient.blacklistRemovals.isEmpty());
    
    // create a invalid ContainerRequest - memory value is minus
    ContainerRequest invalidContainerRequest = 
        new ContainerRequest(Resource.newInstance(-1024, 1, 1),
            nodes, racks, priority);
    amClient.addContainerRequest(invalidContainerRequest);
    amClient.updateBlacklist(localNodeBlacklist, null);
    try {
      // allocate() should complain as ContainerRequest is invalid.
      amClient.allocate(0.1f);
      fail("there should be an exception here.");
    } catch (Exception e) {
      assertEquals(1, amClient.blacklistAdditions.size());
    }
  } finally {
    if (amClient != null && amClient.getServiceState() == STATE.STARTED) {
      amClient.stop();
    }
  }
}
 
Example 16
Source Project: hadoop   File: TestAMRMClient.java    License: Apache License 2.0 4 votes vote down vote up
@Test (timeout=60000)
public void testAMRMClientWithBlacklist() throws YarnException, IOException {
  AMRMClientImpl<ContainerRequest> amClient = null;
  try {
    // start am rm client
    amClient =
        (AMRMClientImpl<ContainerRequest>) AMRMClient
          .<ContainerRequest> createAMRMClient();
    amClient.init(conf);
    amClient.start();
    amClient.registerApplicationMaster("Host", 10000, "");
    String[] nodes = {"node1", "node2", "node3"};
    
    // Add nodes[0] and nodes[1]
    List<String> nodeList01 = new ArrayList<String>();
    nodeList01.add(nodes[0]);
    nodeList01.add(nodes[1]);
    amClient.updateBlacklist(nodeList01, null);
    assertEquals(2, amClient.blacklistAdditions.size());
    assertEquals(0, amClient.blacklistRemovals.size());
    
    // Add nodes[0] again, verify it is not added duplicated.
    List<String> nodeList02 = new ArrayList<String>();
    nodeList02.add(nodes[0]);
    nodeList02.add(nodes[2]);
    amClient.updateBlacklist(nodeList02, null);
    assertEquals(3, amClient.blacklistAdditions.size());
    assertEquals(0, amClient.blacklistRemovals.size());
    
    // Add nodes[1] and nodes[2] to removal list, 
    // Verify addition list remove these two nodes.
    List<String> nodeList12 = new ArrayList<String>();
    nodeList12.add(nodes[1]);
    nodeList12.add(nodes[2]);
    amClient.updateBlacklist(null, nodeList12);
    assertEquals(1, amClient.blacklistAdditions.size());
    assertEquals(2, amClient.blacklistRemovals.size());
    
    // Add nodes[1] again to addition list, 
    // Verify removal list will remove this node.
    List<String> nodeList1 = new ArrayList<String>();
    nodeList1.add(nodes[1]);
    amClient.updateBlacklist(nodeList1, null);
    assertEquals(2, amClient.blacklistAdditions.size());
    assertEquals(1, amClient.blacklistRemovals.size());
  } finally {
    if (amClient != null && amClient.getServiceState() == STATE.STARTED) {
      amClient.stop();
    }
  }
}
 
Example 17
Source Project: big-c   File: TestNodeStatusUpdater.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testNMRegistration() throws InterruptedException {
  nm = new NodeManager() {
    @Override
    protected NodeStatusUpdater createNodeStatusUpdater(Context context,
        Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
      return new MyNodeStatusUpdater(context, dispatcher, healthChecker,
                                     metrics);
    }
  };

  YarnConfiguration conf = createNMConfig();
  nm.init(conf);

  // verify that the last service is the nodeStatusUpdater (ie registration
  // with RM)
  Object[] services  = nm.getServices().toArray();
  Object lastService = services[services.length-1];
  Assert.assertTrue("last service is NOT the node status updater",
      lastService instanceof NodeStatusUpdater);

  new Thread() {
    public void run() {
      try {
        nm.start();
      } catch (Throwable e) {
        TestNodeStatusUpdater.this.nmStartError = e;
        throw new YarnRuntimeException(e);
      }
    }
  }.start();

  System.out.println(" ----- thread already started.."
      + nm.getServiceState());

  int waitCount = 0;
  while (nm.getServiceState() == STATE.INITED && waitCount++ != 50) {
    LOG.info("Waiting for NM to start..");
    if (nmStartError != null) {
      LOG.error("Error during startup. ", nmStartError);
      Assert.fail(nmStartError.getCause().getMessage());
    }
    Thread.sleep(2000);
  }
  if (nm.getServiceState() != STATE.STARTED) {
    // NM could have failed.
    Assert.fail("NodeManager failed to start");
  }

  waitCount = 0;
  while (heartBeatID <= 3 && waitCount++ != 200) {
    Thread.sleep(1000);
  }
  Assert.assertFalse(heartBeatID <= 3);
  Assert.assertEquals("Number of registered NMs is wrong!!", 1,
      this.registeredNodes.size());

  nm.stop();
}
 
Example 18
Source Project: big-c   File: TestNodeStatusUpdater.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testRMVersionLessThanMinimum() throws InterruptedException {
  final AtomicInteger numCleanups = new AtomicInteger(0);
  YarnConfiguration conf = createNMConfig();
  conf.set(YarnConfiguration.NM_RESOURCEMANAGER_MINIMUM_VERSION, "3.0.0");
  nm = new NodeManager() {
    @Override
    protected NodeStatusUpdater createNodeStatusUpdater(Context context,
                                                        Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
      MyNodeStatusUpdater myNodeStatusUpdater = new MyNodeStatusUpdater(
          context, dispatcher, healthChecker, metrics);
      MyResourceTracker2 myResourceTracker2 = new MyResourceTracker2();
      myResourceTracker2.heartBeatNodeAction = NodeAction.NORMAL;
      myResourceTracker2.rmVersion = "3.0.0";
      myNodeStatusUpdater.resourceTracker = myResourceTracker2;
      return myNodeStatusUpdater;
    }

    @Override
    protected ContainerManagerImpl createContainerManager(Context context,
        ContainerExecutor exec, DeletionService del,
        NodeStatusUpdater nodeStatusUpdater,
        ApplicationACLsManager aclsManager,
        LocalDirsHandlerService dirsHandler) {
      return new ContainerManagerImpl(context, exec, del, nodeStatusUpdater,
          metrics, aclsManager, dirsHandler) {

        @Override
        public void cleanUpApplicationsOnNMShutDown() {
          super.cleanUpApplicationsOnNMShutDown();
          numCleanups.incrementAndGet();
        }
      };
    }
  };

  nm.init(conf);
  nm.start();

  // NM takes a while to reach the STARTED state.
  int waitCount = 0;
  while (nm.getServiceState() != STATE.STARTED && waitCount++ != 20) {
    LOG.info("Waiting for NM to stop..");
    Thread.sleep(1000);
  }
  Assert.assertTrue(nm.getServiceState() == STATE.STARTED);
  nm.stop();
}
 
Example 19
Source Project: big-c   File: TestApplicationACLs.java    License: Apache License 2.0 4 votes vote down vote up
@BeforeClass
public static void setup() throws InterruptedException, IOException {
  RMStateStore store = RMStateStoreFactory.getStore(conf);
  conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
  AccessControlList adminACL = new AccessControlList("");
  adminACL.addGroup(SUPER_GROUP);
  conf.set(YarnConfiguration.YARN_ADMIN_ACL, adminACL.getAclString());
  resourceManager = new MockRM(conf) {

    @Override
    protected QueueACLsManager createQueueACLsManager(
        ResourceScheduler scheduler,
        Configuration conf) {
      QueueACLsManager mockQueueACLsManager = mock(QueueACLsManager.class);
      when(mockQueueACLsManager.checkAccess(any(UserGroupInformation.class),
          any(QueueACL.class), anyString())).thenAnswer(new Answer() {
        public Object answer(InvocationOnMock invocation) {
          return isQueueUser;
        }
      });
      return mockQueueACLsManager;
    }

    protected ClientRMService createClientRMService() {
      return new ClientRMService(getRMContext(), this.scheduler,
          this.rmAppManager, this.applicationACLsManager,
          this.queueACLsManager, null);
    };
  };
  new Thread() {
    public void run() {
      UserGroupInformation.createUserForTesting(ENEMY, new String[] {});
      UserGroupInformation.createUserForTesting(FRIEND,
          new String[] { FRIENDLY_GROUP });
      UserGroupInformation.createUserForTesting(SUPER_USER,
          new String[] { SUPER_GROUP });
      resourceManager.start();
    };
  }.start();
  int waitCount = 0;
  while (resourceManager.getServiceState() == STATE.INITED
      && waitCount++ < 60) {
    LOG.info("Waiting for RM to start...");
    Thread.sleep(1500);
  }
  if (resourceManager.getServiceState() != STATE.STARTED) {
    // RM could have failed.
    throw new IOException(
        "ResourceManager failed to start. Final state is "
            + resourceManager.getServiceState());
  }

  UserGroupInformation owner = UserGroupInformation
      .createRemoteUser(APP_OWNER);
  rmClient = owner.doAs(new PrivilegedExceptionAction<ApplicationClientProtocol>() {
    @Override
    public ApplicationClientProtocol run() throws Exception {
      return (ApplicationClientProtocol) rpc.getProxy(ApplicationClientProtocol.class,
          rmAddress, conf);
    }
  });
}
 
Example 20
Source Project: big-c   File: TestAMRMClient.java    License: Apache License 2.0 4 votes vote down vote up
@Test (timeout=60000)
public void testAllocationWithBlacklist() throws YarnException, IOException {
  AMRMClientImpl<ContainerRequest> amClient = null;
  try {
    // start am rm client
    amClient =
        (AMRMClientImpl<ContainerRequest>) AMRMClient
          .<ContainerRequest> createAMRMClient();
    amClient.init(conf);
    amClient.start();
    amClient.registerApplicationMaster("Host", 10000, "");
    
    assertEquals(0, amClient.ask.size());
    assertEquals(0, amClient.release.size());
    
    ContainerRequest storedContainer1 = 
        new ContainerRequest(capability, nodes, racks, priority);
    amClient.addContainerRequest(storedContainer1);
    assertEquals(3, amClient.ask.size());
    assertEquals(0, amClient.release.size());
    
    List<String> localNodeBlacklist = new ArrayList<String>();
    localNodeBlacklist.add(node);
    
    // put node in black list, so no container assignment
    amClient.updateBlacklist(localNodeBlacklist, null);

    int allocatedContainerCount = getAllocatedContainersNumber(amClient,
      DEFAULT_ITERATION);
    // the only node is in blacklist, so no allocation
    assertEquals(0, allocatedContainerCount);

    // Remove node from blacklist, so get assigned with 2
    amClient.updateBlacklist(null, localNodeBlacklist);
    ContainerRequest storedContainer2 = 
            new ContainerRequest(capability, nodes, racks, priority);
    amClient.addContainerRequest(storedContainer2);
    allocatedContainerCount = getAllocatedContainersNumber(amClient,
        DEFAULT_ITERATION);
    assertEquals(2, allocatedContainerCount);
    
    // Test in case exception in allocate(), blacklist is kept
    assertTrue(amClient.blacklistAdditions.isEmpty());
    assertTrue(amClient.blacklistRemovals.isEmpty());
    
    // create a invalid ContainerRequest - memory value is minus
    ContainerRequest invalidContainerRequest = 
        new ContainerRequest(Resource.newInstance(-1024, 1),
            nodes, racks, priority);
    amClient.addContainerRequest(invalidContainerRequest);
    amClient.updateBlacklist(localNodeBlacklist, null);
    try {
      // allocate() should complain as ContainerRequest is invalid.
      amClient.allocate(0.1f);
      fail("there should be an exception here.");
    } catch (Exception e) {
      assertEquals(1, amClient.blacklistAdditions.size());
    }
  } finally {
    if (amClient != null && amClient.getServiceState() == STATE.STARTED) {
      amClient.stop();
    }
  }
}