Java Code Examples for org.apache.hadoop.yarn.conf.YarnConfiguration#setLong()

The following examples show how to use org.apache.hadoop.yarn.conf.YarnConfiguration#setLong() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestRMEmbeddedElector.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Before
public void setup() throws IOException {
  conf = new YarnConfiguration();
  conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
  conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, true);
  conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_EMBEDDED, true);
  conf.set(YarnConfiguration.RM_CLUSTER_ID, "yarn-test-cluster");
  conf.set(YarnConfiguration.RM_ZK_ADDRESS, hostPort);
  conf.setInt(YarnConfiguration.RM_ZK_TIMEOUT_MS, 2000);

  conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID);
  conf.set(YarnConfiguration.RM_HA_ID, RM1_NODE_ID);
  setRpcAddressForRM(RM1_NODE_ID, RM1_PORT_BASE);
  setRpcAddressForRM(RM2_NODE_ID, RM2_PORT_BASE);

  conf.setLong(YarnConfiguration.CLIENT_FAILOVER_SLEEPTIME_BASE_MS, 100L);

  callbackCalled = new AtomicBoolean(false);
}
 
Example 2
Source File: TestFSRMStateStore.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
public RMStateStore getRMStateStore() throws Exception {
  YarnConfiguration conf = new YarnConfiguration();
  conf.set(YarnConfiguration.FS_RM_STATE_STORE_URI,
      workingDirPathURI.toString());
  conf.set(YarnConfiguration.FS_RM_STATE_STORE_RETRY_POLICY_SPEC,
          "100,6000");
  conf.setInt(YarnConfiguration.FS_RM_STATE_STORE_NUM_RETRIES, 8);
  conf.setLong(YarnConfiguration.FS_RM_STATE_STORE_RETRY_INTERVAL_MS,
          900L);
  if (adminCheckEnable) {
    conf.setBoolean(
      YarnConfiguration.YARN_INTERMEDIATE_DATA_ENCRYPTION, true);
  }
  this.store = new TestFileSystemRMStore(conf);
  Assert.assertEquals(store.getNumRetries(), 8);
  Assert.assertEquals(store.getRetryInterval(), 900L);
  return store;
}
 
Example 3
Source File: TestNodeStatusUpdater.java    From big-c with Apache License 2.0 6 votes vote down vote up
private YarnConfiguration createNMConfig() {
  YarnConfiguration conf = new YarnConfiguration();
  String localhostAddress = null;
  try {
    localhostAddress = InetAddress.getByName("localhost").getCanonicalHostName();
  } catch (UnknownHostException e) {
    Assert.fail("Unable to get localhost address: " + e.getMessage());
  }
  conf.setInt(YarnConfiguration.NM_PMEM_MB, 5 * 1024); // 5GB
  conf.set(YarnConfiguration.NM_ADDRESS, localhostAddress + ":12345");
  conf.set(YarnConfiguration.NM_LOCALIZER_ADDRESS, localhostAddress + ":12346");
  conf.set(YarnConfiguration.NM_LOG_DIRS, logsDir.getAbsolutePath());
  conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
    remoteLogsDir.getAbsolutePath());
  conf.set(YarnConfiguration.NM_LOCAL_DIRS, nmLocalDir.getAbsolutePath());
  conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 1);
  return conf;
}
 
Example 4
Source File: ProtocolHATestBase.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Before
public void setup() throws IOException {
  failoverThread = null;
  keepRunning = true;
  conf = new YarnConfiguration();
  conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
  conf.setInt(YarnConfiguration.CLIENT_FAILOVER_MAX_ATTEMPTS, 5);
  conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID);
  setRpcAddressForRM(RM1_NODE_ID, RM1_PORT_BASE);
  setRpcAddressForRM(RM2_NODE_ID, RM2_PORT_BASE);

  conf.setLong(YarnConfiguration.CLIENT_FAILOVER_SLEEPTIME_BASE_MS, 100L);

  conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true);
  conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_USE_RPC, true);
}
 
Example 5
Source File: TestFSRMStateStore.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
public RMStateStore getRMStateStore() throws Exception {
  YarnConfiguration conf = new YarnConfiguration();
  conf.set(YarnConfiguration.FS_RM_STATE_STORE_URI,
      workingDirPathURI.toString());
  conf.set(YarnConfiguration.FS_RM_STATE_STORE_RETRY_POLICY_SPEC,
          "100,6000");
  conf.setInt(YarnConfiguration.FS_RM_STATE_STORE_NUM_RETRIES, 8);
  conf.setLong(YarnConfiguration.FS_RM_STATE_STORE_RETRY_INTERVAL_MS,
          900L);
  if (adminCheckEnable) {
    conf.setBoolean(
      YarnConfiguration.YARN_INTERMEDIATE_DATA_ENCRYPTION, true);
  }
  this.store = new TestFileSystemRMStore(conf);
  Assert.assertEquals(store.getNumRetries(), 8);
  Assert.assertEquals(store.getRetryInterval(), 900L);
  return store;
}
 
Example 6
Source File: ProtocolHATestBase.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Before
public void setup() throws IOException {
  failoverThread = null;
  keepRunning = true;
  conf = new YarnConfiguration();
  conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
  conf.setInt(YarnConfiguration.CLIENT_FAILOVER_MAX_ATTEMPTS, 5);
  conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID);
  setRpcAddressForRM(RM1_NODE_ID, RM1_PORT_BASE);
  setRpcAddressForRM(RM2_NODE_ID, RM2_PORT_BASE);

  conf.setLong(YarnConfiguration.CLIENT_FAILOVER_SLEEPTIME_BASE_MS, 100L);

  conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true);
  conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_USE_RPC, true);
}
 
Example 7
Source File: TestRMFailover.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Before
public void setup() throws IOException {
  fakeAppId = ApplicationId.newInstance(System.currentTimeMillis(), 0);
  conf = new YarnConfiguration();
  conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
  conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID);
  setRpcAddressForRM(RM1_NODE_ID, RM1_PORT_BASE);
  setRpcAddressForRM(RM2_NODE_ID, RM2_PORT_BASE);

  conf.setLong(YarnConfiguration.CLIENT_FAILOVER_SLEEPTIME_BASE_MS, 100L);

  conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true);
  conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_USE_RPC, true);

  cluster = new MiniYARNCluster(TestRMFailover.class.getName(), 2, 1, 1, 1);
}
 
Example 8
Source File: TestNodeStatusUpdater.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private YarnConfiguration createNMConfig() {
  YarnConfiguration conf = new YarnConfiguration();
  String localhostAddress = null;
  try {
    localhostAddress = InetAddress.getByName("localhost").getCanonicalHostName();
  } catch (UnknownHostException e) {
    Assert.fail("Unable to get localhost address: " + e.getMessage());
  }
  conf.setInt(YarnConfiguration.NM_PMEM_MB, 5 * 1024); // 5GB
  conf.set(YarnConfiguration.NM_ADDRESS, localhostAddress + ":12345");
  conf.set(YarnConfiguration.NM_LOCALIZER_ADDRESS, localhostAddress + ":12346");
  conf.set(YarnConfiguration.NM_LOG_DIRS, logsDir.getAbsolutePath());
  conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
    remoteLogsDir.getAbsolutePath());
  conf.set(YarnConfiguration.NM_LOCAL_DIRS, nmLocalDir.getAbsolutePath());
  conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 1);
  return conf;
}
 
Example 9
Source File: TestNodeManagerReboot.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private YarnConfiguration createNMConfig() {
  YarnConfiguration conf = new YarnConfiguration();
  conf.setInt(YarnConfiguration.NM_PMEM_MB, 5 * 1024); // 5GB
  conf.set(YarnConfiguration.NM_ADDRESS, "127.0.0.1:12345");
  conf.set(YarnConfiguration.NM_LOCALIZER_ADDRESS, "127.0.0.1:12346");
  conf.set(YarnConfiguration.NM_LOG_DIRS, logsDir.getAbsolutePath());
  conf.set(YarnConfiguration.NM_LOCAL_DIRS, nmLocalDir.getAbsolutePath());
  conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 1);
  return conf;
}
 
Example 10
Source File: TestAbstractYarnScheduler.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testMaximimumAllocationVCores() throws Exception {
  final int node1MaxVCores = 15;
  final int node2MaxVCores = 5;
  final int node3MaxVCores = 6;
  final int configuredMaxVCores = 10;
  configureScheduler();
  YarnConfiguration conf = getConf();
  conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
      configuredMaxVCores);
  conf.setLong(
      YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS,
      1000 * 1000);
  MockRM rm = new MockRM(conf);
  try {
    rm.start();
    testMaximumAllocationVCoresHelper(
        (AbstractYarnScheduler) rm.getResourceScheduler(),
        node1MaxVCores, node2MaxVCores, node3MaxVCores,
        configuredMaxVCores, configuredMaxVCores, configuredMaxVCores,
        configuredMaxVCores, configuredMaxVCores, configuredMaxVCores);
  } finally {
    rm.stop();
  }

  conf.setLong(
      YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS,
      0);
  rm = new MockRM(conf);
  try {
    rm.start();
    testMaximumAllocationVCoresHelper(
        (AbstractYarnScheduler) rm.getResourceScheduler(),
        node1MaxVCores, node2MaxVCores, node3MaxVCores,
        configuredMaxVCores, configuredMaxVCores, configuredMaxVCores,
        node2MaxVCores, node3MaxVCores, node2MaxVCores);
  } finally {
    rm.stop();
  }
}
 
Example 11
Source File: TestAbstractYarnScheduler.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testMaximimumAllocationMemory() throws Exception {
  final int node1MaxMemory = 15 * 1024;
  final int node2MaxMemory = 5 * 1024;
  final int node3MaxMemory = 6 * 1024;
  final int configuredMaxMemory = 10 * 1024;
  configureScheduler();
  YarnConfiguration conf = getConf();
  conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
      configuredMaxMemory);
  conf.setLong(
      YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS,
      1000 * 1000);
  MockRM rm = new MockRM(conf);
  try {
    rm.start();
    testMaximumAllocationMemoryHelper(
        (AbstractYarnScheduler) rm.getResourceScheduler(),
        node1MaxMemory, node2MaxMemory, node3MaxMemory,
        configuredMaxMemory, configuredMaxMemory, configuredMaxMemory,
        configuredMaxMemory, configuredMaxMemory, configuredMaxMemory);
  } finally {
    rm.stop();
  }

  conf.setLong(
      YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS,
      0);
  rm = new MockRM(conf);
  try {
    rm.start();
    testMaximumAllocationMemoryHelper(
        (AbstractYarnScheduler) rm.getResourceScheduler(),
        node1MaxMemory, node2MaxMemory, node3MaxMemory,
        configuredMaxMemory, configuredMaxMemory, configuredMaxMemory,
        node2MaxMemory, node3MaxMemory, node2MaxMemory);
  } finally {
    rm.stop();
  }
}
 
Example 12
Source File: TestZKRMStateStoreZKClientConnections.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test (timeout = 20000)
public void testZKClientRetry() throws Exception {
  TestZKClient zkClientTester = new TestZKClient();
  final String path = "/test";
  YarnConfiguration conf = new YarnConfiguration();
  conf.setInt(YarnConfiguration.RM_ZK_TIMEOUT_MS, ZK_TIMEOUT_MS);
  conf.setLong(YarnConfiguration.RM_ZK_RETRY_INTERVAL_MS, 100);
  final ZKRMStateStore store =
      (ZKRMStateStore) zkClientTester.getRMStateStore(conf);
  TestDispatcher dispatcher = new TestDispatcher();
  store.setRMDispatcher(dispatcher);
  final AtomicBoolean assertionFailedInThread = new AtomicBoolean(false);

  stopServer();
  Thread clientThread = new Thread() {
    @Override
    public void run() {
      try {
        store.getDataWithRetries(path, true);
      } catch (Exception e) {
        e.printStackTrace();
        assertionFailedInThread.set(true);
      }
    }
  };
  Thread.sleep(2000);
  startServer();
  clientThread.join();
  Assert.assertFalse(assertionFailedInThread.get());
}
 
Example 13
Source File: TestNodeManagerResync.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private YarnConfiguration createNMConfig() {
  YarnConfiguration conf = new YarnConfiguration();
  conf.setInt(YarnConfiguration.NM_PMEM_MB, 5*1024); // 5GB
  conf.set(YarnConfiguration.NM_ADDRESS, "127.0.0.1:12345");
  conf.set(YarnConfiguration.NM_LOCALIZER_ADDRESS, "127.0.0.1:12346");
  conf.set(YarnConfiguration.NM_LOG_DIRS, logsDir.getAbsolutePath());
  conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
    remoteLogsDir.getAbsolutePath());
  conf.set(YarnConfiguration.NM_LOCAL_DIRS, nmLocalDir.getAbsolutePath());
  conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 1);
  return conf;
}
 
Example 14
Source File: TestNodeManagerResync.java    From big-c with Apache License 2.0 5 votes vote down vote up
private YarnConfiguration createNMConfig() {
  YarnConfiguration conf = new YarnConfiguration();
  conf.setInt(YarnConfiguration.NM_PMEM_MB, 5*1024); // 5GB
  conf.set(YarnConfiguration.NM_ADDRESS, "127.0.0.1:12345");
  conf.set(YarnConfiguration.NM_LOCALIZER_ADDRESS, "127.0.0.1:12346");
  conf.set(YarnConfiguration.NM_LOG_DIRS, logsDir.getAbsolutePath());
  conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
    remoteLogsDir.getAbsolutePath());
  conf.set(YarnConfiguration.NM_LOCAL_DIRS, nmLocalDir.getAbsolutePath());
  conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 1);
  return conf;
}
 
Example 15
Source File: TestAMRMClient.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void setup() throws Exception {
  // start minicluster
  conf = new YarnConfiguration();
  conf.setLong(
    YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS,
    rolling_interval_sec);
  conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, am_expire_ms);
  conf.setInt(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, 100);
  conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 1);
  yarnCluster = new MiniYARNCluster(TestAMRMClient.class.getName(), nodeCount, 1, 1);
  yarnCluster.init(conf);
  yarnCluster.start();

  // start rm client
  yarnClient = YarnClient.createYarnClient();
  yarnClient.init(conf);
  yarnClient.start();

  // get node info
  nodeReports = yarnClient.getNodeReports(NodeState.RUNNING);
  
  priority = Priority.newInstance(1);
  priority2 = Priority.newInstance(2);
  capability = Resource.newInstance(1024, 1, 1);

  node = nodeReports.get(0).getNodeId().getHost();
  rack = nodeReports.get(0).getRackName();
  nodes = new String[]{ node };
  racks = new String[]{ rack };
}
 
Example 16
Source File: TestNodeStatusUpdater.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testApplicationKeepAlive() throws Exception {
  MyNodeManager nm = new MyNodeManager();
  try {
    YarnConfiguration conf = createNMConfig();
    conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
    conf.setLong(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS,
        4000l);
    nm.init(conf);
    nm.start();
    // HB 2 -> app cancelled by RM.
    while (heartBeatID < 12) {
      Thread.sleep(1000l);
    }
    MyResourceTracker3 rt =
        (MyResourceTracker3) nm.getNodeStatusUpdater().getRMClient();
    rt.context.getApplications().remove(rt.appId);
    Assert.assertEquals(1, rt.keepAliveRequests.size());
    int numKeepAliveRequests = rt.keepAliveRequests.get(rt.appId).size();
    LOG.info("Number of Keep Alive Requests: [" + numKeepAliveRequests + "]");
    Assert.assertTrue(numKeepAliveRequests == 2 || numKeepAliveRequests == 3);
    while (heartBeatID < 20) {
      Thread.sleep(1000l);
    }
    int numKeepAliveRequests2 = rt.keepAliveRequests.get(rt.appId).size();
    Assert.assertEquals(numKeepAliveRequests, numKeepAliveRequests2);
  } finally {
    if (nm.getServiceState() == STATE.STARTED)
      nm.stop();
  }
}
 
Example 17
Source File: TestNodeStatusUpdater.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 200000)
public void testNodeStatusUpdaterRetryAndNMShutdown()
    throws Exception {
  final long connectionWaitSecs = 1000;
  final long connectionRetryIntervalMs = 1000;
  YarnConfiguration conf = createNMConfig();
  conf.setLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS,
      connectionWaitSecs);
  conf.setLong(YarnConfiguration
          .RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS,
      connectionRetryIntervalMs);
  conf.setLong(YarnConfiguration.NM_SLEEP_DELAY_BEFORE_SIGKILL_MS, 5000);
  conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 1);
  CyclicBarrier syncBarrier = new CyclicBarrier(2);
  nm = new MyNodeManager2(syncBarrier, conf);
  nm.init(conf);
  nm.start();
  // start a container
  ContainerId cId = TestNodeManagerShutdown.createContainerId();
  FileContext localFS = FileContext.getLocalFSFileContext();
  TestNodeManagerShutdown.startContainer(nm, cId, localFS, nmLocalDir,
    new File("start_file.txt"));

  try {
    syncBarrier.await(10000, TimeUnit.MILLISECONDS);
  } catch (Exception e) {
  }
  Assert.assertFalse("Containers not cleaned up when NM stopped",
    assertionFailedInThread.get());
  Assert.assertTrue(((MyNodeManager2) nm).isStopped);
  Assert.assertTrue("calculate heartBeatCount based on" +
      " connectionWaitSecs and RetryIntervalSecs", heartBeatID == 2);
}
 
Example 18
Source File: TestNodeStatusUpdater.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testApplicationKeepAlive() throws Exception {
  MyNodeManager nm = new MyNodeManager();
  try {
    YarnConfiguration conf = createNMConfig();
    conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
    conf.setLong(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS,
        4000l);
    nm.init(conf);
    nm.start();
    // HB 2 -> app cancelled by RM.
    while (heartBeatID < 12) {
      Thread.sleep(1000l);
    }
    MyResourceTracker3 rt =
        (MyResourceTracker3) nm.getNodeStatusUpdater().getRMClient();
    rt.context.getApplications().remove(rt.appId);
    Assert.assertEquals(1, rt.keepAliveRequests.size());
    int numKeepAliveRequests = rt.keepAliveRequests.get(rt.appId).size();
    LOG.info("Number of Keep Alive Requests: [" + numKeepAliveRequests + "]");
    Assert.assertTrue(numKeepAliveRequests == 2 || numKeepAliveRequests == 3);
    while (heartBeatID < 20) {
      Thread.sleep(1000l);
    }
    int numKeepAliveRequests2 = rt.keepAliveRequests.get(rt.appId).size();
    Assert.assertEquals(numKeepAliveRequests, numKeepAliveRequests2);
  } finally {
    if (nm.getServiceState() == STATE.STARTED)
      nm.stop();
  }
}
 
Example 19
Source File: TestAbstractYarnScheduler.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Test
public void testUpdateMaxAllocationUsesTotal() throws IOException {
  final int configuredMaxVCores = 20;
  final int configuredMaxMemory = 10 * 1024;
  Resource configuredMaximumResource = Resource.newInstance
      (configuredMaxMemory, configuredMaxVCores, configuredMaxVCores);

  configureScheduler();
  YarnConfiguration conf = getConf();
  conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
      configuredMaxVCores);
  conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
      configuredMaxMemory);
  conf.setLong(
      YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS,
      0);

  MockRM rm = new MockRM(conf);
  try {
    rm.start();
    AbstractYarnScheduler scheduler = (AbstractYarnScheduler) rm
        .getResourceScheduler();

    Resource emptyResource = Resource.newInstance(0, 0, 0);
    Resource fullResource1 = Resource.newInstance(1024, 5, 5);
    Resource fullResource2 = Resource.newInstance(2048, 10, 10);

    SchedulerNode mockNode1 = mock(SchedulerNode.class);
    when(mockNode1.getNodeID()).thenReturn(NodeId.newInstance("foo", 8080));
    when(mockNode1.getAvailableResource()).thenReturn(emptyResource);
    when(mockNode1.getTotalResource()).thenReturn(fullResource1);

    SchedulerNode mockNode2 = mock(SchedulerNode.class);
    when(mockNode1.getNodeID()).thenReturn(NodeId.newInstance("bar", 8081));
    when(mockNode2.getAvailableResource()).thenReturn(emptyResource);
    when(mockNode2.getTotalResource()).thenReturn(fullResource2);

    verifyMaximumResourceCapability(configuredMaximumResource, scheduler);

    scheduler.nodes = new HashMap<NodeId, SchedulerNode>();

    scheduler.nodes.put(mockNode1.getNodeID(), mockNode1);
    scheduler.updateMaximumAllocation(mockNode1, true);
    verifyMaximumResourceCapability(fullResource1, scheduler);

    scheduler.nodes.put(mockNode2.getNodeID(), mockNode2);
    scheduler.updateMaximumAllocation(mockNode2, true);
    verifyMaximumResourceCapability(fullResource2, scheduler);

    scheduler.nodes.remove(mockNode2.getNodeID());
    scheduler.updateMaximumAllocation(mockNode2, false);
    verifyMaximumResourceCapability(fullResource1, scheduler);

    scheduler.nodes.remove(mockNode1.getNodeID());
    scheduler.updateMaximumAllocation(mockNode1, false);
    verifyMaximumResourceCapability(configuredMaximumResource, scheduler);
  } finally {
    rm.stop();
  }
}
 
Example 20
Source File: TestAbstractYarnScheduler.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Test
public void testMaxAllocationAfterUpdateNodeResource() throws IOException {
  final int configuredMaxVCores = 20;
  final int configuredMaxMemory = 10 * 1024;
  Resource configuredMaximumResource = Resource.newInstance
      (configuredMaxMemory, configuredMaxVCores, configuredMaxVCores);

  configureScheduler();
  YarnConfiguration conf = getConf();
  conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
      configuredMaxVCores);
  conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
      configuredMaxMemory);
  conf.setLong(
      YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS,
      0);

  MockRM rm = new MockRM(conf);
  try {
    rm.start();
    AbstractYarnScheduler scheduler = (AbstractYarnScheduler) rm
        .getResourceScheduler();
    verifyMaximumResourceCapability(configuredMaximumResource, scheduler);

    Resource resource1 = Resource.newInstance(2048, 5, 5);
    Resource resource2 = Resource.newInstance(4096, 10, 10);
    Resource resource3 = Resource.newInstance(512, 1, 1);
    Resource resource4 = Resource.newInstance(1024, 2, 2);

    RMNode node1 = MockNodes.newNodeInfo(
        0, resource1, 1, "127.0.0.2");
    scheduler.handle(new NodeAddedSchedulerEvent(node1));
    RMNode node2 = MockNodes.newNodeInfo(
        0, resource3, 2, "127.0.0.3");
    scheduler.handle(new NodeAddedSchedulerEvent(node2));
    verifyMaximumResourceCapability(resource1, scheduler);

    // increase node1 resource
    scheduler.updateNodeResource(node1, ResourceOption.newInstance(
        resource2, 0));
    verifyMaximumResourceCapability(resource2, scheduler);

    // decrease node1 resource
    scheduler.updateNodeResource(node1, ResourceOption.newInstance(
        resource1, 0));
    verifyMaximumResourceCapability(resource1, scheduler);

    // increase node2 resource
    scheduler.updateNodeResource(node2, ResourceOption.newInstance(
        resource4, 0));
    verifyMaximumResourceCapability(resource1, scheduler);

    // decrease node2 resource
    scheduler.updateNodeResource(node2, ResourceOption.newInstance(
        resource3, 0));
    verifyMaximumResourceCapability(resource1, scheduler);
  } finally {
    rm.stop();
  }
}