org.apache.hadoop.yarn.server.resourcemanager.RMContext Java Examples

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.RMContext. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestFSAppAttempt.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
/**
 * Ensure that when negative paramaters are given (signaling delay scheduling
 * no tin use), the least restrictive locality level is returned.
 */
public void testLocalityLevelWithoutDelays() {
  FSLeafQueue queue = Mockito.mock(FSLeafQueue.class);
  Priority prio = Mockito.mock(Priority.class);
  Mockito.when(prio.getPriority()).thenReturn(1);

  RMContext rmContext = resourceManager.getRMContext();
  ApplicationAttemptId applicationAttemptId = createAppAttemptId(1, 1);
  FSAppAttempt schedulerApp =
      new FSAppAttempt(scheduler, applicationAttemptId, "user1", queue ,
          null, rmContext);
  assertEquals(NodeType.OFF_SWITCH, schedulerApp.getAllowedLocalityLevel(
      prio, 10, -1.0, -1.0));
}
 
Example #2
Source File: AMRMTokenSecretManager.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Create an {@link AMRMTokenSecretManager}
 */
public AMRMTokenSecretManager(Configuration conf, RMContext rmContext) {
  this.rmContext = rmContext;
  this.timer = new Timer();
  this.rollingInterval =
      conf
        .getLong(
          YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS,
          YarnConfiguration.DEFAULT_RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS) * 1000;
  // Adding delay = 1.5 * expiry interval makes sure that all active AMs get
  // the updated shared-key.
  this.activationDelay =
      (long) (conf.getLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS,
          YarnConfiguration.DEFAULT_RM_AM_EXPIRY_INTERVAL_MS) * 1.5);
  LOG.info("AMRMTokenKeyRollingInterval: " + this.rollingInterval
      + "ms and AMRMTokenKeyActivationDelay: " + this.activationDelay + " ms");
  if (rollingInterval <= activationDelay * 2) {
    throw new IllegalArgumentException(
        YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS
            + " should be more than 3 X "
            + YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS);
  }
}
 
Example #3
Source File: RMNodeImpl.java    From big-c with Apache License 2.0 6 votes vote down vote up
public RMNodeImpl(NodeId nodeId, RMContext context, String hostName,
    int cmPort, int httpPort, Node node, Resource capability, String nodeManagerVersion) {
  this.nodeId = nodeId;
  this.context = context;
  this.hostName = hostName;
  this.commandPort = cmPort;
  this.httpPort = httpPort;
  this.totalCapability = capability; 
  this.nodeAddress = hostName + ":" + cmPort;
  this.httpAddress = hostName + ":" + httpPort;
  this.node = node;
  this.healthReport = "Healthy";
  this.lastHealthReportTime = System.currentTimeMillis();
  this.nodeManagerVersion = nodeManagerVersion;

  this.latestNodeHeartBeatResponse.setResponseId(0);

  ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
  this.readLock = lock.readLock();
  this.writeLock = lock.writeLock();

  this.stateMachine = stateMachineFactory.make(this);
  
  this.nodeUpdateQueue = new ConcurrentLinkedQueue<UpdatedContainerInfo>();  
}
 
Example #4
Source File: TestCapacityScheduler.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
  resourceManager = new ResourceManager() {
    @Override
    protected RMNodeLabelsManager createNodeLabelManager() {
      RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
      mgr.init(getConfig());
      return mgr;
    }
  };
  CapacitySchedulerConfiguration csConf 
     = new CapacitySchedulerConfiguration();
  setupQueueConfiguration(csConf);
  YarnConfiguration conf = new YarnConfiguration(csConf);
  conf.setClass(YarnConfiguration.RM_SCHEDULER, 
      CapacityScheduler.class, ResourceScheduler.class);
  resourceManager.init(conf);
  resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey();
  resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey();
  ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start();
  mockContext = mock(RMContext.class);
  when(mockContext.getConfigurationProvider()).thenReturn(
      new LocalConfigurationProvider());
}
 
Example #5
Source File: SchedulerUtils.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public static void normalizeAndValidateRequest(ResourceRequest resReq,
    Resource maximumResource, String queueName, YarnScheduler scheduler,
    boolean isRecovery, RMContext rmContext, QueueInfo queueInfo)
    throws InvalidResourceRequestException {

  if (queueInfo == null) {
    try {
      queueInfo = scheduler.getQueueInfo(queueName, false, false);
    } catch (IOException e) {
      // it is possible queue cannot get when queue mapping is set, just ignore
      // the queueInfo here, and move forward
    }
  }
  SchedulerUtils.normalizeNodeLabelExpressionInRequest(resReq, queueInfo);
  if (!isRecovery) {
    validateResourceRequest(resReq, maximumResource, queueInfo, rmContext);
  }
}
 
Example #6
Source File: RMContainerImpl.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public RMContainerImpl(Container container,
    ApplicationAttemptId appAttemptId, NodeId nodeId,
    String user, RMContext rmContext, long creationTime) {
  this.stateMachine = stateMachineFactory.make(this);
  this.containerId = container.getId();
  this.nodeId = nodeId;
  this.container = container;
  this.appAttemptId = appAttemptId;
  this.user = user;
  this.creationTime = creationTime;
  this.rmContext = rmContext;
  this.eventHandler = rmContext.getDispatcher().getEventHandler();
  this.containerAllocationExpirer = rmContext.getContainerAllocationExpirer();
  this.isAMContainer = false;
  this.resourceRequests = null;

  ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
  this.readLock = lock.readLock();
  this.writeLock = lock.writeLock();

  rmContext.getRMApplicationHistoryWriter().containerStarted(this);
  rmContext.getSystemMetricsPublisher().containerCreated(
      this, this.creationTime);
}
 
Example #7
Source File: TestMaxRunningAppsEnforcer.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Before
public void setup() throws Exception {
  Configuration conf = new Configuration();
  clock = new TestFairScheduler.MockClock();
  scheduler = mock(FairScheduler.class);
  when(scheduler.getConf()).thenReturn(
      new FairSchedulerConfiguration(conf));
  when(scheduler.getClock()).thenReturn(clock);
  AllocationConfiguration allocConf = new AllocationConfiguration(
      conf);
  when(scheduler.getAllocationConfiguration()).thenReturn(allocConf);
  
  queueManager = new QueueManager(scheduler);
  queueManager.initialize(conf);
  queueMaxApps = allocConf.queueMaxApps;
  userMaxApps = allocConf.userMaxApps;
  maxAppsEnforcer = new MaxRunningAppsEnforcer(scheduler);
  appNum = 0;
  rmContext = mock(RMContext.class);
  when(rmContext.getEpoch()).thenReturn(0L);
}
 
Example #8
Source File: CapacityScheduler.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
public synchronized void
reinitialize(Configuration conf, RMContext rmContext) throws IOException {
  Configuration configuration = new Configuration(conf);
  CapacitySchedulerConfiguration oldConf = this.conf;
  this.conf = loadCapacitySchedulerConfiguration(configuration);
  validateConf(this.conf);
  try {
    LOG.info("Re-initializing queues...");
    refreshMaximumAllocation(this.conf.getMaximumAllocation());
    reinitializeQueues(this.conf);
  } catch (Throwable t) {
    this.conf = oldConf;
    refreshMaximumAllocation(this.conf.getMaximumAllocation());
    throw new IOException("Failed to re-init queues", t);
  }
}
 
Example #9
Source File: NMHeartBeatHandler.java    From incubator-myriad with Apache License 2.0 6 votes vote down vote up
@Override
public void beforeRMNodeEventHandled(RMNodeEvent event, RMContext context) {
  switch (event.getType()) {
    case STARTED:
      // Since the RMNode was just started, it should not have a non-zero capacity
      RMNode rmNode = context.getRMNodes().get(event.getNodeId());
      
      if (isNonZeroCapacityNode(rmNode)) {
        Resource totalCapability = rmNode.getTotalCapability();
        logger.warn(
            "FineGrainedScaling feature got invoked for a NM with non-zero capacity. Host: {}, Mem: {}, CPU: {}. Setting the " +
            "NM's capacity to (0G,0CPU)", rmNode.getHostName(), totalCapability.getMemory(), totalCapability.getVirtualCores());
        totalCapability.setMemory(0);
        totalCapability.setVirtualCores(0);
      }
      break;

    case STATUS_UPDATE:
      handleStatusUpdate(event, context);
      break;

    default:
      break;
  }
}
 
Example #10
Source File: TestCapacityScheduler.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
  resourceManager = new ResourceManager() {
    @Override
    protected RMNodeLabelsManager createNodeLabelManager() {
      RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
      mgr.init(getConfig());
      return mgr;
    }
  };
  CapacitySchedulerConfiguration csConf 
     = new CapacitySchedulerConfiguration();
  setupQueueConfiguration(csConf);
  YarnConfiguration conf = new YarnConfiguration(csConf);
  conf.setClass(YarnConfiguration.RM_SCHEDULER, 
      CapacityScheduler.class, ResourceScheduler.class);
  resourceManager.init(conf);
  resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey();
  resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey();
  ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start();
  mockContext = mock(RMContext.class);
  when(mockContext.getConfigurationProvider()).thenReturn(
      new LocalConfigurationProvider());
}
 
Example #11
Source File: SchedulerUtils.java    From big-c with Apache License 2.0 6 votes vote down vote up
public static void normalizeAndValidateRequest(ResourceRequest resReq,
    Resource maximumResource, String queueName, YarnScheduler scheduler,
    boolean isRecovery, RMContext rmContext, QueueInfo queueInfo)
    throws InvalidResourceRequestException {

  if (queueInfo == null) {
    try {
      queueInfo = scheduler.getQueueInfo(queueName, false, false);
    } catch (IOException e) {
      // it is possible queue cannot get when queue mapping is set, just ignore
      // the queueInfo here, and move forward
    }
  }
  SchedulerUtils.normalizeNodeLabelExpressionInRequest(resReq, queueInfo);
  if (!isRecovery) {
    validateResourceRequest(resReq, maximumResource, queueInfo, rmContext);
  }
}
 
Example #12
Source File: RMNodeImpl.java    From big-c with Apache License 2.0 6 votes vote down vote up
private static void handleRunningAppOnNode(RMNodeImpl rmNode,
    RMContext context, ApplicationId appId, NodeId nodeId) {
  RMApp app = context.getRMApps().get(appId);

  // if we failed getting app by appId, maybe something wrong happened, just
  // add the app to the finishedApplications list so that the app can be
  // cleaned up on the NM
  if (null == app) {
    LOG.warn("Cannot get RMApp by appId=" + appId
        + ", just added it to finishedApplications list for cleanup");
    rmNode.finishedApplications.add(appId);
    return;
  }

  context.getDispatcher().getEventHandler()
      .handle(new RMAppRunningOnNodeEvent(appId, nodeId));
}
 
Example #13
Source File: TestNMExpiry.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() {
  Configuration conf = new Configuration();
  // Dispatcher that processes events inline
  Dispatcher dispatcher = new InlineDispatcher();
  RMContext context = new RMContextImpl(dispatcher, null,
      null, null, null, null, null, null, null, null);
  dispatcher.register(SchedulerEventType.class,
      new InlineDispatcher.EmptyEventHandler());
  dispatcher.register(RMNodeEventType.class,
      new NodeEventDispatcher(context));
  NMLivelinessMonitor nmLivelinessMonitor = new TestNmLivelinessMonitor(
      dispatcher);
  nmLivelinessMonitor.init(conf);
  nmLivelinessMonitor.start();
  NodesListManager nodesListManager = new NodesListManager(context);
  nodesListManager.init(conf);
  RMContainerTokenSecretManager containerTokenSecretManager =
      new RMContainerTokenSecretManager(conf);
  containerTokenSecretManager.start();
  NMTokenSecretManagerInRM nmTokenSecretManager =
      new NMTokenSecretManagerInRM(conf);
  nmTokenSecretManager.start();
  resourceTrackerService = new ResourceTrackerService(context,
      nodesListManager, nmLivelinessMonitor, containerTokenSecretManager,
      nmTokenSecretManager);
  
  resourceTrackerService.init(conf);
  resourceTrackerService.start();
}
 
Example #14
Source File: TestDelegationTokenRenewerLifecycle.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testStartupFailure() throws Exception {
  Configuration conf = new Configuration();
  DelegationTokenRenewer delegationTokenRenewer =
      new DelegationTokenRenewer();
  RMContext mockContext = mock(RMContext.class);
  ClientRMService mockClientRMService = mock(ClientRMService.class);
  when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
  delegationTokenRenewer.setRMContext(mockContext);
  delegationTokenRenewer.init(conf);
  delegationTokenRenewer.stop();
}
 
Example #15
Source File: ReservationSystemTestUtil.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public static void initializeRMContext(int numContainers,
    AbstractYarnScheduler scheduler, RMContext mockRMContext) {

  when(mockRMContext.getScheduler()).thenReturn(scheduler);
  Resource r = calculateClusterResource(numContainers);
  doReturn(r).when(scheduler).getClusterResource();
}
 
Example #16
Source File: CompositeInterceptor.java    From incubator-myriad with Apache License 2.0 5 votes vote down vote up
@Override
public void beforeRMNodeEventHandled(RMNodeEvent event, RMContext context) {
  for (YarnSchedulerInterceptor interceptor : interceptors.values()) {
    if (interceptor.getCallBackFilter().allowCallBacksForNode(event.getNodeId())) {
      interceptor.beforeRMNodeEventHandled(event, context);
    }
  }
}
 
Example #17
Source File: SchedulerUtils.java    From big-c with Apache License 2.0 5 votes vote down vote up
public static void normalizeAndValidateRequest(ResourceRequest resReq,
    Resource maximumResource, String queueName, YarnScheduler scheduler,
    boolean isRecovery, RMContext rmContext)
    throws InvalidResourceRequestException {
  normalizeAndValidateRequest(resReq, maximumResource, queueName, scheduler,
      isRecovery, rmContext, null);
}
 
Example #18
Source File: TestRMWebApp.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public static ClientRMService mockClientRMService(RMContext rmContext) {
  ClientRMService clientRMService = mock(ClientRMService.class);
  List<ApplicationReport> appReports = new ArrayList<ApplicationReport>();
  for (RMApp app : rmContext.getRMApps().values()) {
    ApplicationReport appReport =
        ApplicationReport.newInstance(
            app.getApplicationId(), (ApplicationAttemptId) null,
            app.getUser(), app.getQueue(),
            app.getName(), (String) null, 0, (Token) null,
            app.createApplicationState(),
            app.getDiagnostics().toString(), (String) null,
            app.getStartTime(), app.getFinishTime(),
            app.getFinalApplicationStatus(),
            (ApplicationResourceUsageReport) null, app.getTrackingUrl(),
            app.getProgress(), app.getApplicationType(), (Token) null);
    appReports.add(appReport);
  }
  GetApplicationsResponse response = mock(GetApplicationsResponse.class);
  when(response.getApplicationList()).thenReturn(appReports);
  try {
    when(clientRMService.getApplications(any(GetApplicationsRequest.class)))
        .thenReturn(response);
  } catch (YarnException e) {
    Assert.fail("Exception is not expteced.");
  }
  return clientRMService;
}
 
Example #19
Source File: TestRMWebApp.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test public void testNodesPage() {
  // 10 nodes. Two of each type.
  final RMContext rmContext = mockRMContext(3, 2, 12, 8*GiB);
  Injector injector = WebAppTests.createMockInjector(RMContext.class,
      rmContext,
      new Module() {
    @Override
    public void configure(Binder binder) {
      try {
        binder.bind(ResourceManager.class).toInstance(mockRm(rmContext));
      } catch (IOException e) {
        throw new IllegalStateException(e);
      }
    }
  });

  // All nodes
  NodesPage instance = injector.getInstance(NodesPage.class);
  instance.render();
  WebAppTests.flushOutput(injector);

  // Unhealthy nodes
  instance.moreParams().put(YarnWebParams.NODE_STATE,
    NodeState.UNHEALTHY.toString());
  instance.render();
  WebAppTests.flushOutput(injector);

  // Lost nodes
  instance.moreParams().put(YarnWebParams.NODE_STATE,
    NodeState.LOST.toString());
  instance.render();
  WebAppTests.flushOutput(injector);

}
 
Example #20
Source File: TestRMWebApp.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public static ResourceManager mockRm(RMContext rmContext) throws IOException {
  ResourceManager rm = mock(ResourceManager.class);
  ResourceScheduler rs = mockCapacityScheduler();
  ApplicationACLsManager aclMgr = mockAppACLsManager();
  ClientRMService clientRMService = mockClientRMService(rmContext);
  when(rm.getResourceScheduler()).thenReturn(rs);
  when(rm.getRMContext()).thenReturn(rmContext);
  when(rm.getApplicationACLsManager()).thenReturn(aclMgr);
  when(rm.getClientRMService()).thenReturn(clientRMService);
  return rm;
}
 
Example #21
Source File: AMLauncher.java    From big-c with Apache License 2.0 5 votes vote down vote up
public AMLauncher(RMContext rmContext, RMAppAttempt application,
    AMLauncherEventType eventType, Configuration conf) {
  this.application = application;
  this.conf = conf;
  this.eventType = eventType;
  this.rmContext = rmContext;
  this.handler = rmContext.getDispatcher().getEventHandler();
  this.masterContainer = application.getMasterContainer();
}
 
Example #22
Source File: TestCapacityScheduler.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testNumClusterNodes() throws Exception {
  YarnConfiguration conf = new YarnConfiguration();
  CapacityScheduler cs = new CapacityScheduler();
  cs.setConf(conf);
  RMContext rmContext = TestUtils.getMockRMContext();
  cs.setRMContext(rmContext);
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration();
  setupQueueConfiguration(csConf);
  cs.init(csConf);
  cs.start();
  assertEquals(0, cs.getNumClusterNodes());

  RMNode n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1);
  RMNode n2 = MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 2);
  cs.handle(new NodeAddedSchedulerEvent(n1));
  cs.handle(new NodeAddedSchedulerEvent(n2));
  assertEquals(2, cs.getNumClusterNodes());

  cs.handle(new NodeRemovedSchedulerEvent(n1));
  assertEquals(1, cs.getNumClusterNodes());
  cs.handle(new NodeAddedSchedulerEvent(n1));
  assertEquals(2, cs.getNumClusterNodes());
  cs.handle(new NodeRemovedSchedulerEvent(n2));
  cs.handle(new NodeRemovedSchedulerEvent(n1));
  assertEquals(0, cs.getNumClusterNodes());

  cs.stop();
}
 
Example #23
Source File: TestRMWebApp.java    From big-c with Apache License 2.0 5 votes vote down vote up
public static ResourceManager mockFifoRm(int apps, int racks, int nodes,
                                     int mbsPerNode)
throws Exception {
  ResourceManager rm = mock(ResourceManager.class);
  RMContext rmContext = mockRMContext(apps, racks, nodes,
      mbsPerNode);
  ResourceScheduler rs = mockFifoScheduler(rmContext);
  when(rm.getResourceScheduler()).thenReturn(rs);
  when(rm.getRMContext()).thenReturn(rmContext);
  return rm;
}
 
Example #24
Source File: TestRMDelegationTokens.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public TestRMDelegationTokenSecretManager(long delegationKeyUpdateInterval,
    long delegationTokenMaxLifetime, long delegationTokenRenewInterval,
    long delegationTokenRemoverScanInterval, RMContext rmContext) {
  super(delegationKeyUpdateInterval, delegationTokenMaxLifetime,
    delegationTokenRenewInterval, delegationTokenRemoverScanInterval,
    rmContext);
}
 
Example #25
Source File: TestProportionalCapacityPreemptionPolicy.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Before
@SuppressWarnings("unchecked")
public void setup() {
  conf = new Configuration(false);
  conf.setLong(WAIT_TIME_BEFORE_KILL, 10000);
  conf.setLong(MONITORING_INTERVAL, 3000);
  // report "ideal" preempt
  conf.setFloat(TOTAL_PREEMPTION_PER_ROUND, (float) 1.0);
  conf.setFloat(NATURAL_TERMINATION_FACTOR, (float) 1.0);
  conf.set(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES,
      ProportionalCapacityPreemptionPolicy.class.getCanonicalName());
  conf.setBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true);
  // FairScheduler doesn't support this test,
  // Set CapacityScheduler as the scheduler for this test.
  conf.set("yarn.resourcemanager.scheduler.class",
      CapacityScheduler.class.getName());

  mClock = mock(Clock.class);
  mCS = mock(CapacityScheduler.class);
  when(mCS.getResourceCalculator()).thenReturn(rc);
  lm = mock(RMNodeLabelsManager.class);
  schedConf = new CapacitySchedulerConfiguration();
  when(mCS.getConfiguration()).thenReturn(schedConf);
  rmContext = mock(RMContext.class);
  when(mCS.getRMContext()).thenReturn(rmContext);
  when(rmContext.getNodeLabelManager()).thenReturn(lm);
  mDisp = mock(EventHandler.class);
  rand = new Random();
  long seed = rand.nextLong();
  System.out.println(name.getMethodName() + " SEED: " + seed);
  rand.setSeed(seed);
  appAlloc = 0;
}
 
Example #26
Source File: TestRMWebApp.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test public void testNodesPage() {
  // 10 nodes. Two of each type.
  final RMContext rmContext = mockRMContext(3, 2, 12, 8*GiB);
  Injector injector = WebAppTests.createMockInjector(RMContext.class,
      rmContext,
      new Module() {
    @Override
    public void configure(Binder binder) {
      try {
        binder.bind(ResourceManager.class).toInstance(mockRm(rmContext));
      } catch (IOException e) {
        throw new IllegalStateException(e);
      }
    }
  });

  // All nodes
  NodesPage instance = injector.getInstance(NodesPage.class);
  instance.render();
  WebAppTests.flushOutput(injector);

  // Unhealthy nodes
  instance.moreParams().put(YarnWebParams.NODE_STATE,
    NodeState.UNHEALTHY.toString());
  instance.render();
  WebAppTests.flushOutput(injector);

  // Lost nodes
  instance.moreParams().put(YarnWebParams.NODE_STATE,
    NodeState.LOST.toString());
  instance.render();
  WebAppTests.flushOutput(injector);

}
 
Example #27
Source File: NMHeartBeatHandler.java    From incubator-myriad with Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
protected void handleStatusUpdate(RMNodeEvent event, RMContext context) {
  if (!(event instanceof RMNodeStatusEvent)) {
    logger.error("{} not an instance of {}", event.getClass().getName(), RMNodeStatusEvent.class.getName());
    return;
  }

  RMNodeStatusEvent statusEvent = (RMNodeStatusEvent) event;
  RMNode rmNode = context.getRMNodes().get(event.getNodeId());
  String hostName = rmNode.getNodeID().getHost();

  Node host = nodeStore.getNode(hostName);
  if (host != null) {
    host.snapshotRunningContainers();
  }

  /*
   * Set the new node capacity which is the sum of the current node resources plus those offered by Mesos. 
   * If the sum is greater than the max capacity of the node, reject the offer.
   */
  Resource offeredResources = getNewResourcesOfferedByMesos(hostName);
  Resource currentResources = getResourcesUnderUse(statusEvent);
  
  if (offerWithinResourceLimits(currentResources, offeredResources)) {
    yarnNodeCapacityMgr.setNodeCapacity(rmNode, Resources.add(currentResources, offeredResources));
    logger.info("Updated resources for {} with {} cores and {} memory", rmNode.getNode().getName(), 
            offeredResources.getVirtualCores(), offeredResources.getMemory());
  } else {
    logger.info("Did not update {} with {} cores and {} memory, over max cpu cores and/or max memory", 
            rmNode.getNode().getName(), offeredResources.getVirtualCores(), offeredResources.getMemory());
  }
}
 
Example #28
Source File: SchedulerUtils.java    From big-c with Apache License 2.0 5 votes vote down vote up
public static void normalizeAndvalidateRequest(ResourceRequest resReq,
    Resource maximumResource, String queueName, YarnScheduler scheduler,
    RMContext rmContext, QueueInfo queueInfo)
    throws InvalidResourceRequestException {
  normalizeAndValidateRequest(resReq, maximumResource, queueName, scheduler,
      false, rmContext, queueInfo);
}
 
Example #29
Source File: FiCaSchedulerApp.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public FiCaSchedulerApp(ApplicationAttemptId applicationAttemptId, 
    String user, Queue queue, ActiveUsersManager activeUsersManager,
    RMContext rmContext) {
  super(applicationAttemptId, user, queue, activeUsersManager, rmContext);
  
  RMApp rmApp = rmContext.getRMApps().get(getApplicationId());
  
  Resource amResource;
  String partition;

  if (rmApp == null || rmApp.getAMResourceRequest() == null) {
    // the rmApp may be undefined (the resource manager checks for this too)
    // and unmanaged applications do not provide an amResource request
    // in these cases, provide a default using the scheduler
    amResource = rmContext.getScheduler().getMinimumResourceCapability();
    partition = CommonNodeLabelsManager.NO_LABEL;
  } else {
    amResource = rmApp.getAMResourceRequest().getCapability();
    partition =
        (rmApp.getAMResourceRequest().getNodeLabelExpression() == null)
        ? CommonNodeLabelsManager.NO_LABEL
        : rmApp.getAMResourceRequest().getNodeLabelExpression();
  }
  
  setAppAMNodePartitionName(partition);
  setAMResource(partition, amResource);
}
 
Example #30
Source File: TestFifoScheduler.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private RMAppImpl createMockRMApp(ApplicationAttemptId attemptId,
    RMContext context) {
  RMAppImpl app = mock(RMAppImpl.class);
  when(app.getApplicationId()).thenReturn(attemptId.getApplicationId());
  RMAppAttemptImpl attempt = mock(RMAppAttemptImpl.class);
  when(attempt.getAppAttemptId()).thenReturn(attemptId);
  RMAppAttemptMetrics attemptMetric = mock(RMAppAttemptMetrics.class);
  when(attempt.getRMAppAttemptMetrics()).thenReturn(attemptMetric);
  when(app.getCurrentAppAttempt()).thenReturn(attempt);
  context.getRMApps().putIfAbsent(attemptId.getApplicationId(), app);
  return app;
}