Java Code Examples for org.apache.hadoop.yarn.conf.YarnConfiguration#setLong()

The following examples show how to use org.apache.hadoop.yarn.conf.YarnConfiguration#setLong() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: TestRMEmbeddedElector.java From big-c with Apache License 2.0

6 votes

@Before
public void setup() throws IOException {
  conf = new YarnConfiguration();
  conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
  conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, true);
  conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_EMBEDDED, true);
  conf.set(YarnConfiguration.RM_CLUSTER_ID, "yarn-test-cluster");
  conf.set(YarnConfiguration.RM_ZK_ADDRESS, hostPort);
  conf.setInt(YarnConfiguration.RM_ZK_TIMEOUT_MS, 2000);

  conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID);
  conf.set(YarnConfiguration.RM_HA_ID, RM1_NODE_ID);
  setRpcAddressForRM(RM1_NODE_ID, RM1_PORT_BASE);
  setRpcAddressForRM(RM2_NODE_ID, RM2_PORT_BASE);

  conf.setLong(YarnConfiguration.CLIENT_FAILOVER_SLEEPTIME_BASE_MS, 100L);

  callbackCalled = new AtomicBoolean(false);
}

Example 2

Source File: TestFSRMStateStore.java From big-c with Apache License 2.0

6 votes

@Override
public RMStateStore getRMStateStore() throws Exception {
  YarnConfiguration conf = new YarnConfiguration();
  conf.set(YarnConfiguration.FS_RM_STATE_STORE_URI,
      workingDirPathURI.toString());
  conf.set(YarnConfiguration.FS_RM_STATE_STORE_RETRY_POLICY_SPEC,
          "100,6000");
  conf.setInt(YarnConfiguration.FS_RM_STATE_STORE_NUM_RETRIES, 8);
  conf.setLong(YarnConfiguration.FS_RM_STATE_STORE_RETRY_INTERVAL_MS,
          900L);
  if (adminCheckEnable) {
    conf.setBoolean(
      YarnConfiguration.YARN_INTERMEDIATE_DATA_ENCRYPTION, true);
  }
  this.store = new TestFileSystemRMStore(conf);
  Assert.assertEquals(store.getNumRetries(), 8);
  Assert.assertEquals(store.getRetryInterval(), 900L);
  return store;
}

Example 3

Source File: TestNodeStatusUpdater.java From big-c with Apache License 2.0

6 votes

private YarnConfiguration createNMConfig() {
  YarnConfiguration conf = new YarnConfiguration();
  String localhostAddress = null;
  try {
    localhostAddress = InetAddress.getByName("localhost").getCanonicalHostName();
  } catch (UnknownHostException e) {
    Assert.fail("Unable to get localhost address: " + e.getMessage());
  }
  conf.setInt(YarnConfiguration.NM_PMEM_MB, 5 * 1024); // 5GB
  conf.set(YarnConfiguration.NM_ADDRESS, localhostAddress + ":12345");
  conf.set(YarnConfiguration.NM_LOCALIZER_ADDRESS, localhostAddress + ":12346");
  conf.set(YarnConfiguration.NM_LOG_DIRS, logsDir.getAbsolutePath());
  conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
    remoteLogsDir.getAbsolutePath());
  conf.set(YarnConfiguration.NM_LOCAL_DIRS, nmLocalDir.getAbsolutePath());
  conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 1);
  return conf;
}

Example 4

Source File: ProtocolHATestBase.java From hadoop with Apache License 2.0

6 votes

@Before
public void setup() throws IOException {
  failoverThread = null;
  keepRunning = true;
  conf = new YarnConfiguration();
  conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
  conf.setInt(YarnConfiguration.CLIENT_FAILOVER_MAX_ATTEMPTS, 5);
  conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID);
  setRpcAddressForRM(RM1_NODE_ID, RM1_PORT_BASE);
  setRpcAddressForRM(RM2_NODE_ID, RM2_PORT_BASE);

  conf.setLong(YarnConfiguration.CLIENT_FAILOVER_SLEEPTIME_BASE_MS, 100L);

  conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true);
  conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_USE_RPC, true);
}

Example 5

Source File: TestFSRMStateStore.java From hadoop with Apache License 2.0

6 votes

@Override
public RMStateStore getRMStateStore() throws Exception {
  YarnConfiguration conf = new YarnConfiguration();
  conf.set(YarnConfiguration.FS_RM_STATE_STORE_URI,
      workingDirPathURI.toString());
  conf.set(YarnConfiguration.FS_RM_STATE_STORE_RETRY_POLICY_SPEC,
          "100,6000");
  conf.setInt(YarnConfiguration.FS_RM_STATE_STORE_NUM_RETRIES, 8);
  conf.setLong(YarnConfiguration.FS_RM_STATE_STORE_RETRY_INTERVAL_MS,
          900L);
  if (adminCheckEnable) {
    conf.setBoolean(
      YarnConfiguration.YARN_INTERMEDIATE_DATA_ENCRYPTION, true);
  }
  this.store = new TestFileSystemRMStore(conf);
  Assert.assertEquals(store.getNumRetries(), 8);
  Assert.assertEquals(store.getRetryInterval(), 900L);
  return store;
}

Example 6

Source File: ProtocolHATestBase.java From big-c with Apache License 2.0

6 votes

@Before
public void setup() throws IOException {
  failoverThread = null;
  keepRunning = true;
  conf = new YarnConfiguration();
  conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
  conf.setInt(YarnConfiguration.CLIENT_FAILOVER_MAX_ATTEMPTS, 5);
  conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID);
  setRpcAddressForRM(RM1_NODE_ID, RM1_PORT_BASE);
  setRpcAddressForRM(RM2_NODE_ID, RM2_PORT_BASE);

  conf.setLong(YarnConfiguration.CLIENT_FAILOVER_SLEEPTIME_BASE_MS, 100L);

  conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true);
  conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_USE_RPC, true);
}

Example 7

Source File: TestRMFailover.java From big-c with Apache License 2.0

6 votes

@Before
public void setup() throws IOException {
  fakeAppId = ApplicationId.newInstance(System.currentTimeMillis(), 0);
  conf = new YarnConfiguration();
  conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
  conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID);
  setRpcAddressForRM(RM1_NODE_ID, RM1_PORT_BASE);
  setRpcAddressForRM(RM2_NODE_ID, RM2_PORT_BASE);

  conf.setLong(YarnConfiguration.CLIENT_FAILOVER_SLEEPTIME_BASE_MS, 100L);

  conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true);
  conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_USE_RPC, true);

  cluster = new MiniYARNCluster(TestRMFailover.class.getName(), 2, 1, 1, 1);
}

Example 8

Source File: TestNodeStatusUpdater.java From hadoop with Apache License 2.0

6 votes

private YarnConfiguration createNMConfig() {
  YarnConfiguration conf = new YarnConfiguration();
  String localhostAddress = null;
  try {
    localhostAddress = InetAddress.getByName("localhost").getCanonicalHostName();
  } catch (UnknownHostException e) {
    Assert.fail("Unable to get localhost address: " + e.getMessage());
  }
  conf.setInt(YarnConfiguration.NM_PMEM_MB, 5 * 1024); // 5GB
  conf.set(YarnConfiguration.NM_ADDRESS, localhostAddress + ":12345");
  conf.set(YarnConfiguration.NM_LOCALIZER_ADDRESS, localhostAddress + ":12346");
  conf.set(YarnConfiguration.NM_LOG_DIRS, logsDir.getAbsolutePath());
  conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
    remoteLogsDir.getAbsolutePath());
  conf.set(YarnConfiguration.NM_LOCAL_DIRS, nmLocalDir.getAbsolutePath());
  conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 1);
  return conf;
}

Example 9

Source File: TestNodeManagerReboot.java From hadoop with Apache License 2.0

5 votes

private YarnConfiguration createNMConfig() {
  YarnConfiguration conf = new YarnConfiguration();
  conf.setInt(YarnConfiguration.NM_PMEM_MB, 5 * 1024); // 5GB
  conf.set(YarnConfiguration.NM_ADDRESS, "127.0.0.1:12345");
  conf.set(YarnConfiguration.NM_LOCALIZER_ADDRESS, "127.0.0.1:12346");
  conf.set(YarnConfiguration.NM_LOG_DIRS, logsDir.getAbsolutePath());
  conf.set(YarnConfiguration.NM_LOCAL_DIRS, nmLocalDir.getAbsolutePath());
  conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 1);
  return conf;
}

Example 10

Source File: TestAbstractYarnScheduler.java From hadoop with Apache License 2.0

5 votes

@Test
public void testMaximimumAllocationVCores() throws Exception {
  final int node1MaxVCores = 15;
  final int node2MaxVCores = 5;
  final int node3MaxVCores = 6;
  final int configuredMaxVCores = 10;
  configureScheduler();
  YarnConfiguration conf = getConf();
  conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
      configuredMaxVCores);
  conf.setLong(
      YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS,
      1000 * 1000);
  MockRM rm = new MockRM(conf);
  try {
    rm.start();
    testMaximumAllocationVCoresHelper(
        (AbstractYarnScheduler) rm.getResourceScheduler(),
        node1MaxVCores, node2MaxVCores, node3MaxVCores,
        configuredMaxVCores, configuredMaxVCores, configuredMaxVCores,
        configuredMaxVCores, configuredMaxVCores, configuredMaxVCores);
  } finally {
    rm.stop();
  }

  conf.setLong(
      YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS,
      0);
  rm = new MockRM(conf);
  try {
    rm.start();
    testMaximumAllocationVCoresHelper(
        (AbstractYarnScheduler) rm.getResourceScheduler(),
        node1MaxVCores, node2MaxVCores, node3MaxVCores,
        configuredMaxVCores, configuredMaxVCores, configuredMaxVCores,
        node2MaxVCores, node3MaxVCores, node2MaxVCores);
  } finally {
    rm.stop();
  }
}

Example 11

Source File: TestAbstractYarnScheduler.java From big-c with Apache License 2.0

5 votes

@Test
public void testMaximimumAllocationMemory() throws Exception {
  final int node1MaxMemory = 15 * 1024;
  final int node2MaxMemory = 5 * 1024;
  final int node3MaxMemory = 6 * 1024;
  final int configuredMaxMemory = 10 * 1024;
  configureScheduler();
  YarnConfiguration conf = getConf();
  conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
      configuredMaxMemory);
  conf.setLong(
      YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS,
      1000 * 1000);
  MockRM rm = new MockRM(conf);
  try {
    rm.start();
    testMaximumAllocationMemoryHelper(
        (AbstractYarnScheduler) rm.getResourceScheduler(),
        node1MaxMemory, node2MaxMemory, node3MaxMemory,
        configuredMaxMemory, configuredMaxMemory, configuredMaxMemory,
        configuredMaxMemory, configuredMaxMemory, configuredMaxMemory);
  } finally {
    rm.stop();
  }

  conf.setLong(
      YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS,
      0);
  rm = new MockRM(conf);
  try {
    rm.start();
    testMaximumAllocationMemoryHelper(
        (AbstractYarnScheduler) rm.getResourceScheduler(),
        node1MaxMemory, node2MaxMemory, node3MaxMemory,
        configuredMaxMemory, configuredMaxMemory, configuredMaxMemory,
        node2MaxMemory, node3MaxMemory, node2MaxMemory);
  } finally {
    rm.stop();
  }
}

Example 12

Source File: TestZKRMStateStoreZKClientConnections.java From hadoop with Apache License 2.0

5 votes

@Test (timeout = 20000)
public void testZKClientRetry() throws Exception {
  TestZKClient zkClientTester = new TestZKClient();
  final String path = "/test";
  YarnConfiguration conf = new YarnConfiguration();
  conf.setInt(YarnConfiguration.RM_ZK_TIMEOUT_MS, ZK_TIMEOUT_MS);
  conf.setLong(YarnConfiguration.RM_ZK_RETRY_INTERVAL_MS, 100);
  final ZKRMStateStore store =
      (ZKRMStateStore) zkClientTester.getRMStateStore(conf);
  TestDispatcher dispatcher = new TestDispatcher();
  store.setRMDispatcher(dispatcher);
  final AtomicBoolean assertionFailedInThread = new AtomicBoolean(false);

  stopServer();
  Thread clientThread = new Thread() {
    @Override
    public void run() {
      try {
        store.getDataWithRetries(path, true);
      } catch (Exception e) {
        e.printStackTrace();
        assertionFailedInThread.set(true);
      }
    }
  };
  Thread.sleep(2000);
  startServer();
  clientThread.join();
  Assert.assertFalse(assertionFailedInThread.get());
}

Example 13

Source File: TestNodeManagerResync.java From hadoop with Apache License 2.0

5 votes

private YarnConfiguration createNMConfig() {
  YarnConfiguration conf = new YarnConfiguration();
  conf.setInt(YarnConfiguration.NM_PMEM_MB, 5*1024); // 5GB
  conf.set(YarnConfiguration.NM_ADDRESS, "127.0.0.1:12345");
  conf.set(YarnConfiguration.NM_LOCALIZER_ADDRESS, "127.0.0.1:12346");
  conf.set(YarnConfiguration.NM_LOG_DIRS, logsDir.getAbsolutePath());
  conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
    remoteLogsDir.getAbsolutePath());
  conf.set(YarnConfiguration.NM_LOCAL_DIRS, nmLocalDir.getAbsolutePath());
  conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 1);
  return conf;
}

Example 14

Source File: TestNodeManagerResync.java From big-c with Apache License 2.0

5 votes

private YarnConfiguration createNMConfig() {
  YarnConfiguration conf = new YarnConfiguration();
  conf.setInt(YarnConfiguration.NM_PMEM_MB, 5*1024); // 5GB
  conf.set(YarnConfiguration.NM_ADDRESS, "127.0.0.1:12345");
  conf.set(YarnConfiguration.NM_LOCALIZER_ADDRESS, "127.0.0.1:12346");
  conf.set(YarnConfiguration.NM_LOG_DIRS, logsDir.getAbsolutePath());
  conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
    remoteLogsDir.getAbsolutePath());
  conf.set(YarnConfiguration.NM_LOCAL_DIRS, nmLocalDir.getAbsolutePath());
  conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 1);
  return conf;
}

Example 15

Source File: TestAMRMClient.java From hadoop with Apache License 2.0

5 votes

@BeforeClass
public static void setup() throws Exception {
  // start minicluster
  conf = new YarnConfiguration();
  conf.setLong(
    YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS,
    rolling_interval_sec);
  conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, am_expire_ms);
  conf.setInt(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, 100);
  conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 1);
  yarnCluster = new MiniYARNCluster(TestAMRMClient.class.getName(), nodeCount, 1, 1);
  yarnCluster.init(conf);
  yarnCluster.start();

  // start rm client
  yarnClient = YarnClient.createYarnClient();
  yarnClient.init(conf);
  yarnClient.start();

  // get node info
  nodeReports = yarnClient.getNodeReports(NodeState.RUNNING);
  
  priority = Priority.newInstance(1);
  priority2 = Priority.newInstance(2);
  capability = Resource.newInstance(1024, 1, 1);

  node = nodeReports.get(0).getNodeId().getHost();
  rack = nodeReports.get(0).getRackName();
  nodes = new String[]{ node };
  racks = new String[]{ rack };
}

Example 16

Source File: TestNodeStatusUpdater.java From big-c with Apache License 2.0

5 votes

@Test
public void testApplicationKeepAlive() throws Exception {
  MyNodeManager nm = new MyNodeManager();
  try {
    YarnConfiguration conf = createNMConfig();
    conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
    conf.setLong(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS,
        4000l);
    nm.init(conf);
    nm.start();
    // HB 2 -> app cancelled by RM.
    while (heartBeatID < 12) {
      Thread.sleep(1000l);
    }
    MyResourceTracker3 rt =
        (MyResourceTracker3) nm.getNodeStatusUpdater().getRMClient();
    rt.context.getApplications().remove(rt.appId);
    Assert.assertEquals(1, rt.keepAliveRequests.size());
    int numKeepAliveRequests = rt.keepAliveRequests.get(rt.appId).size();
    LOG.info("Number of Keep Alive Requests: [" + numKeepAliveRequests + "]");
    Assert.assertTrue(numKeepAliveRequests == 2 || numKeepAliveRequests == 3);
    while (heartBeatID < 20) {
      Thread.sleep(1000l);
    }
    int numKeepAliveRequests2 = rt.keepAliveRequests.get(rt.appId).size();
    Assert.assertEquals(numKeepAliveRequests, numKeepAliveRequests2);
  } finally {
    if (nm.getServiceState() == STATE.STARTED)
      nm.stop();
  }
}

Example 17

Source File: TestNodeStatusUpdater.java From big-c with Apache License 2.0

5 votes

@Test(timeout = 200000)
public void testNodeStatusUpdaterRetryAndNMShutdown()
    throws Exception {
  final long connectionWaitSecs = 1000;
  final long connectionRetryIntervalMs = 1000;
  YarnConfiguration conf = createNMConfig();
  conf.setLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS,
      connectionWaitSecs);
  conf.setLong(YarnConfiguration
          .RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS,
      connectionRetryIntervalMs);
  conf.setLong(YarnConfiguration.NM_SLEEP_DELAY_BEFORE_SIGKILL_MS, 5000);
  conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 1);
  CyclicBarrier syncBarrier = new CyclicBarrier(2);
  nm = new MyNodeManager2(syncBarrier, conf);
  nm.init(conf);
  nm.start();
  // start a container
  ContainerId cId = TestNodeManagerShutdown.createContainerId();
  FileContext localFS = FileContext.getLocalFSFileContext();
  TestNodeManagerShutdown.startContainer(nm, cId, localFS, nmLocalDir,
    new File("start_file.txt"));

  try {
    syncBarrier.await(10000, TimeUnit.MILLISECONDS);
  } catch (Exception e) {
  }
  Assert.assertFalse("Containers not cleaned up when NM stopped",
    assertionFailedInThread.get());
  Assert.assertTrue(((MyNodeManager2) nm).isStopped);
  Assert.assertTrue("calculate heartBeatCount based on" +
      " connectionWaitSecs and RetryIntervalSecs", heartBeatID == 2);
}

Example 18

Source File: TestNodeStatusUpdater.java From hadoop with Apache License 2.0

5 votes

@Test
public void testApplicationKeepAlive() throws Exception {
  MyNodeManager nm = new MyNodeManager();
  try {
    YarnConfiguration conf = createNMConfig();
    conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
    conf.setLong(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS,
        4000l);
    nm.init(conf);
    nm.start();
    // HB 2 -> app cancelled by RM.
    while (heartBeatID < 12) {
      Thread.sleep(1000l);
    }
    MyResourceTracker3 rt =
        (MyResourceTracker3) nm.getNodeStatusUpdater().getRMClient();
    rt.context.getApplications().remove(rt.appId);
    Assert.assertEquals(1, rt.keepAliveRequests.size());
    int numKeepAliveRequests = rt.keepAliveRequests.get(rt.appId).size();
    LOG.info("Number of Keep Alive Requests: [" + numKeepAliveRequests + "]");
    Assert.assertTrue(numKeepAliveRequests == 2 || numKeepAliveRequests == 3);
    while (heartBeatID < 20) {
      Thread.sleep(1000l);
    }
    int numKeepAliveRequests2 = rt.keepAliveRequests.get(rt.appId).size();
    Assert.assertEquals(numKeepAliveRequests, numKeepAliveRequests2);
  } finally {
    if (nm.getServiceState() == STATE.STARTED)
      nm.stop();
  }
}

Example 19

Source File: TestAbstractYarnScheduler.java From hadoop with Apache License 2.0

4 votes

@Test
public void testUpdateMaxAllocationUsesTotal() throws IOException {
  final int configuredMaxVCores = 20;
  final int configuredMaxMemory = 10 * 1024;
  Resource configuredMaximumResource = Resource.newInstance
      (configuredMaxMemory, configuredMaxVCores, configuredMaxVCores);

  configureScheduler();
  YarnConfiguration conf = getConf();
  conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
      configuredMaxVCores);
  conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
      configuredMaxMemory);
  conf.setLong(
      YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS,
      0);

  MockRM rm = new MockRM(conf);
  try {
    rm.start();
    AbstractYarnScheduler scheduler = (AbstractYarnScheduler) rm
        .getResourceScheduler();

    Resource emptyResource = Resource.newInstance(0, 0, 0);
    Resource fullResource1 = Resource.newInstance(1024, 5, 5);
    Resource fullResource2 = Resource.newInstance(2048, 10, 10);

    SchedulerNode mockNode1 = mock(SchedulerNode.class);
    when(mockNode1.getNodeID()).thenReturn(NodeId.newInstance("foo", 8080));
    when(mockNode1.getAvailableResource()).thenReturn(emptyResource);
    when(mockNode1.getTotalResource()).thenReturn(fullResource1);

    SchedulerNode mockNode2 = mock(SchedulerNode.class);
    when(mockNode1.getNodeID()).thenReturn(NodeId.newInstance("bar", 8081));
    when(mockNode2.getAvailableResource()).thenReturn(emptyResource);
    when(mockNode2.getTotalResource()).thenReturn(fullResource2);

    verifyMaximumResourceCapability(configuredMaximumResource, scheduler);

    scheduler.nodes = new HashMap<NodeId, SchedulerNode>();

    scheduler.nodes.put(mockNode1.getNodeID(), mockNode1);
    scheduler.updateMaximumAllocation(mockNode1, true);
    verifyMaximumResourceCapability(fullResource1, scheduler);

    scheduler.nodes.put(mockNode2.getNodeID(), mockNode2);
    scheduler.updateMaximumAllocation(mockNode2, true);
    verifyMaximumResourceCapability(fullResource2, scheduler);

    scheduler.nodes.remove(mockNode2.getNodeID());
    scheduler.updateMaximumAllocation(mockNode2, false);
    verifyMaximumResourceCapability(fullResource1, scheduler);

    scheduler.nodes.remove(mockNode1.getNodeID());
    scheduler.updateMaximumAllocation(mockNode1, false);
    verifyMaximumResourceCapability(configuredMaximumResource, scheduler);
  } finally {
    rm.stop();
  }
}

Example 20

Source File: TestAbstractYarnScheduler.java From hadoop with Apache License 2.0

4 votes

@Test
public void testMaxAllocationAfterUpdateNodeResource() throws IOException {
  final int configuredMaxVCores = 20;
  final int configuredMaxMemory = 10 * 1024;
  Resource configuredMaximumResource = Resource.newInstance
      (configuredMaxMemory, configuredMaxVCores, configuredMaxVCores);

  configureScheduler();
  YarnConfiguration conf = getConf();
  conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
      configuredMaxVCores);
  conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
      configuredMaxMemory);
  conf.setLong(
      YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS,
      0);

  MockRM rm = new MockRM(conf);
  try {
    rm.start();
    AbstractYarnScheduler scheduler = (AbstractYarnScheduler) rm
        .getResourceScheduler();
    verifyMaximumResourceCapability(configuredMaximumResource, scheduler);

    Resource resource1 = Resource.newInstance(2048, 5, 5);
    Resource resource2 = Resource.newInstance(4096, 10, 10);
    Resource resource3 = Resource.newInstance(512, 1, 1);
    Resource resource4 = Resource.newInstance(1024, 2, 2);

    RMNode node1 = MockNodes.newNodeInfo(
        0, resource1, 1, "127.0.0.2");
    scheduler.handle(new NodeAddedSchedulerEvent(node1));
    RMNode node2 = MockNodes.newNodeInfo(
        0, resource3, 2, "127.0.0.3");
    scheduler.handle(new NodeAddedSchedulerEvent(node2));
    verifyMaximumResourceCapability(resource1, scheduler);

    // increase node1 resource
    scheduler.updateNodeResource(node1, ResourceOption.newInstance(
        resource2, 0));
    verifyMaximumResourceCapability(resource2, scheduler);

    // decrease node1 resource
    scheduler.updateNodeResource(node1, ResourceOption.newInstance(
        resource1, 0));
    verifyMaximumResourceCapability(resource1, scheduler);

    // increase node2 resource
    scheduler.updateNodeResource(node2, ResourceOption.newInstance(
        resource4, 0));
    verifyMaximumResourceCapability(resource1, scheduler);

    // decrease node2 resource
    scheduler.updateNodeResource(node2, ResourceOption.newInstance(
        resource3, 0));
    verifyMaximumResourceCapability(resource1, scheduler);
  } finally {
    rm.stop();
  }
}