Java Code Examples for org.apache.hadoop.yarn.server.resourcemanager.RMContext

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.RMContext. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: TestFSAppAttempt.java    License: Apache License 2.0 6 votes vote down vote up
@Test
/**
 * Ensure that when negative paramaters are given (signaling delay scheduling
 * no tin use), the least restrictive locality level is returned.
 */
public void testLocalityLevelWithoutDelays() {
  FSLeafQueue queue = Mockito.mock(FSLeafQueue.class);
  Priority prio = Mockito.mock(Priority.class);
  Mockito.when(prio.getPriority()).thenReturn(1);

  RMContext rmContext = resourceManager.getRMContext();
  ApplicationAttemptId applicationAttemptId = createAppAttemptId(1, 1);
  FSAppAttempt schedulerApp =
      new FSAppAttempt(scheduler, applicationAttemptId, "user1", queue ,
          null, rmContext);
  assertEquals(NodeType.OFF_SWITCH, schedulerApp.getAllowedLocalityLevel(
      prio, 10, -1.0, -1.0));
}
 
Example 2
Source Project: hadoop   Source File: RMContainerImpl.java    License: Apache License 2.0 6 votes vote down vote up
public RMContainerImpl(Container container,
    ApplicationAttemptId appAttemptId, NodeId nodeId,
    String user, RMContext rmContext, long creationTime) {
  this.stateMachine = stateMachineFactory.make(this);
  this.containerId = container.getId();
  this.nodeId = nodeId;
  this.container = container;
  this.appAttemptId = appAttemptId;
  this.user = user;
  this.creationTime = creationTime;
  this.rmContext = rmContext;
  this.eventHandler = rmContext.getDispatcher().getEventHandler();
  this.containerAllocationExpirer = rmContext.getContainerAllocationExpirer();
  this.isAMContainer = false;
  this.resourceRequests = null;

  ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
  this.readLock = lock.readLock();
  this.writeLock = lock.writeLock();

  rmContext.getRMApplicationHistoryWriter().containerStarted(this);
  rmContext.getSystemMetricsPublisher().containerCreated(
      this, this.creationTime);
}
 
Example 3
Source Project: big-c   Source File: CapacityScheduler.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public synchronized void
reinitialize(Configuration conf, RMContext rmContext) throws IOException {
  Configuration configuration = new Configuration(conf);
  CapacitySchedulerConfiguration oldConf = this.conf;
  this.conf = loadCapacitySchedulerConfiguration(configuration);
  validateConf(this.conf);
  try {
    LOG.info("Re-initializing queues...");
    refreshMaximumAllocation(this.conf.getMaximumAllocation());
    reinitializeQueues(this.conf);
  } catch (Throwable t) {
    this.conf = oldConf;
    refreshMaximumAllocation(this.conf.getMaximumAllocation());
    throw new IOException("Failed to re-init queues", t);
  }
}
 
Example 4
Source Project: big-c   Source File: RMNodeImpl.java    License: Apache License 2.0 6 votes vote down vote up
public RMNodeImpl(NodeId nodeId, RMContext context, String hostName,
    int cmPort, int httpPort, Node node, Resource capability, String nodeManagerVersion) {
  this.nodeId = nodeId;
  this.context = context;
  this.hostName = hostName;
  this.commandPort = cmPort;
  this.httpPort = httpPort;
  this.totalCapability = capability; 
  this.nodeAddress = hostName + ":" + cmPort;
  this.httpAddress = hostName + ":" + httpPort;
  this.node = node;
  this.healthReport = "Healthy";
  this.lastHealthReportTime = System.currentTimeMillis();
  this.nodeManagerVersion = nodeManagerVersion;

  this.latestNodeHeartBeatResponse.setResponseId(0);

  ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
  this.readLock = lock.readLock();
  this.writeLock = lock.writeLock();

  this.stateMachine = stateMachineFactory.make(this);
  
  this.nodeUpdateQueue = new ConcurrentLinkedQueue<UpdatedContainerInfo>();  
}
 
Example 5
Source Project: hadoop   Source File: TestCapacityScheduler.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
  resourceManager = new ResourceManager() {
    @Override
    protected RMNodeLabelsManager createNodeLabelManager() {
      RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
      mgr.init(getConfig());
      return mgr;
    }
  };
  CapacitySchedulerConfiguration csConf 
     = new CapacitySchedulerConfiguration();
  setupQueueConfiguration(csConf);
  YarnConfiguration conf = new YarnConfiguration(csConf);
  conf.setClass(YarnConfiguration.RM_SCHEDULER, 
      CapacityScheduler.class, ResourceScheduler.class);
  resourceManager.init(conf);
  resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey();
  resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey();
  ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start();
  mockContext = mock(RMContext.class);
  when(mockContext.getConfigurationProvider()).thenReturn(
      new LocalConfigurationProvider());
}
 
Example 6
Source Project: big-c   Source File: RMNodeImpl.java    License: Apache License 2.0 6 votes vote down vote up
private static void handleRunningAppOnNode(RMNodeImpl rmNode,
    RMContext context, ApplicationId appId, NodeId nodeId) {
  RMApp app = context.getRMApps().get(appId);

  // if we failed getting app by appId, maybe something wrong happened, just
  // add the app to the finishedApplications list so that the app can be
  // cleaned up on the NM
  if (null == app) {
    LOG.warn("Cannot get RMApp by appId=" + appId
        + ", just added it to finishedApplications list for cleanup");
    rmNode.finishedApplications.add(appId);
    return;
  }

  context.getDispatcher().getEventHandler()
      .handle(new RMAppRunningOnNodeEvent(appId, nodeId));
}
 
Example 7
Source Project: hadoop   Source File: SchedulerUtils.java    License: Apache License 2.0 6 votes vote down vote up
public static void normalizeAndValidateRequest(ResourceRequest resReq,
    Resource maximumResource, String queueName, YarnScheduler scheduler,
    boolean isRecovery, RMContext rmContext, QueueInfo queueInfo)
    throws InvalidResourceRequestException {

  if (queueInfo == null) {
    try {
      queueInfo = scheduler.getQueueInfo(queueName, false, false);
    } catch (IOException e) {
      // it is possible queue cannot get when queue mapping is set, just ignore
      // the queueInfo here, and move forward
    }
  }
  SchedulerUtils.normalizeNodeLabelExpressionInRequest(resReq, queueInfo);
  if (!isRecovery) {
    validateResourceRequest(resReq, maximumResource, queueInfo, rmContext);
  }
}
 
Example 8
Source Project: hadoop   Source File: TestMaxRunningAppsEnforcer.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setup() throws Exception {
  Configuration conf = new Configuration();
  clock = new TestFairScheduler.MockClock();
  scheduler = mock(FairScheduler.class);
  when(scheduler.getConf()).thenReturn(
      new FairSchedulerConfiguration(conf));
  when(scheduler.getClock()).thenReturn(clock);
  AllocationConfiguration allocConf = new AllocationConfiguration(
      conf);
  when(scheduler.getAllocationConfiguration()).thenReturn(allocConf);
  
  queueManager = new QueueManager(scheduler);
  queueManager.initialize(conf);
  queueMaxApps = allocConf.queueMaxApps;
  userMaxApps = allocConf.userMaxApps;
  maxAppsEnforcer = new MaxRunningAppsEnforcer(scheduler);
  appNum = 0;
  rmContext = mock(RMContext.class);
  when(rmContext.getEpoch()).thenReturn(0L);
}
 
Example 9
Source Project: incubator-myriad   Source File: NMHeartBeatHandler.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void beforeRMNodeEventHandled(RMNodeEvent event, RMContext context) {
  switch (event.getType()) {
    case STARTED:
      // Since the RMNode was just started, it should not have a non-zero capacity
      RMNode rmNode = context.getRMNodes().get(event.getNodeId());
      
      if (isNonZeroCapacityNode(rmNode)) {
        Resource totalCapability = rmNode.getTotalCapability();
        logger.warn(
            "FineGrainedScaling feature got invoked for a NM with non-zero capacity. Host: {}, Mem: {}, CPU: {}. Setting the " +
            "NM's capacity to (0G,0CPU)", rmNode.getHostName(), totalCapability.getMemory(), totalCapability.getVirtualCores());
        totalCapability.setMemory(0);
        totalCapability.setVirtualCores(0);
      }
      break;

    case STATUS_UPDATE:
      handleStatusUpdate(event, context);
      break;

    default:
      break;
  }
}
 
Example 10
Source Project: big-c   Source File: SchedulerUtils.java    License: Apache License 2.0 6 votes vote down vote up
public static void normalizeAndValidateRequest(ResourceRequest resReq,
    Resource maximumResource, String queueName, YarnScheduler scheduler,
    boolean isRecovery, RMContext rmContext, QueueInfo queueInfo)
    throws InvalidResourceRequestException {

  if (queueInfo == null) {
    try {
      queueInfo = scheduler.getQueueInfo(queueName, false, false);
    } catch (IOException e) {
      // it is possible queue cannot get when queue mapping is set, just ignore
      // the queueInfo here, and move forward
    }
  }
  SchedulerUtils.normalizeNodeLabelExpressionInRequest(resReq, queueInfo);
  if (!isRecovery) {
    validateResourceRequest(resReq, maximumResource, queueInfo, rmContext);
  }
}
 
Example 11
Source Project: big-c   Source File: TestCapacityScheduler.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
  resourceManager = new ResourceManager() {
    @Override
    protected RMNodeLabelsManager createNodeLabelManager() {
      RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
      mgr.init(getConfig());
      return mgr;
    }
  };
  CapacitySchedulerConfiguration csConf 
     = new CapacitySchedulerConfiguration();
  setupQueueConfiguration(csConf);
  YarnConfiguration conf = new YarnConfiguration(csConf);
  conf.setClass(YarnConfiguration.RM_SCHEDULER, 
      CapacityScheduler.class, ResourceScheduler.class);
  resourceManager.init(conf);
  resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey();
  resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey();
  ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start();
  mockContext = mock(RMContext.class);
  when(mockContext.getConfigurationProvider()).thenReturn(
      new LocalConfigurationProvider());
}
 
Example 12
Source Project: big-c   Source File: AMRMTokenSecretManager.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Create an {@link AMRMTokenSecretManager}
 */
public AMRMTokenSecretManager(Configuration conf, RMContext rmContext) {
  this.rmContext = rmContext;
  this.timer = new Timer();
  this.rollingInterval =
      conf
        .getLong(
          YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS,
          YarnConfiguration.DEFAULT_RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS) * 1000;
  // Adding delay = 1.5 * expiry interval makes sure that all active AMs get
  // the updated shared-key.
  this.activationDelay =
      (long) (conf.getLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS,
          YarnConfiguration.DEFAULT_RM_AM_EXPIRY_INTERVAL_MS) * 1.5);
  LOG.info("AMRMTokenKeyRollingInterval: " + this.rollingInterval
      + "ms and AMRMTokenKeyActivationDelay: " + this.activationDelay + " ms");
  if (rollingInterval <= activationDelay * 2) {
    throw new IllegalArgumentException(
        YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS
            + " should be more than 3 X "
            + YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS);
  }
}
 
Example 13
Source Project: hadoop   Source File: FifoSchedulerInfo.java    License: Apache License 2.0 5 votes vote down vote up
public FifoSchedulerInfo(final ResourceManager rm) {

    RMContext rmContext = rm.getRMContext();

    FifoScheduler fs = (FifoScheduler) rm.getResourceScheduler();
    qName = fs.getQueueInfo("", false, false).getQueueName();
    QueueInfo qInfo = fs.getQueueInfo(qName, true, true);

    this.usedCapacity = qInfo.getCurrentCapacity();
    this.capacity = qInfo.getCapacity();
    this.minQueueMemoryCapacity = fs.getMinimumResourceCapability().getMemory();
    this.maxQueueMemoryCapacity = fs.getMaximumResourceCapability().getMemory();
    this.qstate = qInfo.getQueueState();

    this.numNodes = rmContext.getRMNodes().size();
    this.usedNodeCapacity = 0;
    this.availNodeCapacity = 0;
    this.totalNodeCapacity = 0;
    this.numContainers = 0;

    for (RMNode ni : rmContext.getRMNodes().values()) {
      SchedulerNodeReport report = fs.getNodeReport(ni.getNodeID());
      this.usedNodeCapacity += report.getUsedResource().getMemory();
      this.availNodeCapacity += report.getAvailableResource().getMemory();
      this.totalNodeCapacity += ni.getTotalCapability().getMemory();
      this.numContainers += fs.getNodeReport(ni.getNodeID()).getNumContainers();
    }
  }
 
Example 14
Source Project: big-c   Source File: TestRMWebApp.java    License: Apache License 2.0 5 votes vote down vote up
public static FifoScheduler mockFifoScheduler(RMContext rmContext)
    throws Exception {
  CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
  setupFifoQueueConfiguration(conf);

  FifoScheduler fs = new FifoScheduler();
  fs.setConf(new YarnConfiguration());
  fs.setRMContext(rmContext);
  fs.init(conf);
  return fs;
}
 
Example 15
Source Project: big-c   Source File: SchedulerUtils.java    License: Apache License 2.0 5 votes vote down vote up
public static void normalizeAndvalidateRequest(ResourceRequest resReq,
    Resource maximumResource, String queueName, YarnScheduler scheduler,
    RMContext rmContext)
    throws InvalidResourceRequestException {
  normalizeAndValidateRequest(resReq, maximumResource, queueName, scheduler,
      false, rmContext, null);
}
 
Example 16
Source Project: hadoop   Source File: AbstractReservationSystem.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void reinitialize(Configuration conf, RMContext rmContext)
    throws YarnException {
  writeLock.lock();
  try {
    if (!initialized) {
      initialize(conf);
      initialized = true;
    } else {
      initializeNewPlans(conf);
    }
  } finally {
    writeLock.unlock();
  }
}
 
Example 17
Source Project: incubator-myriad   Source File: MyriadModule.java    License: Apache License 2.0 5 votes vote down vote up
public MyriadModule(String configFile, Configuration hadoopConf, AbstractYarnScheduler yarnScheduler, RMContext rmContext,
                    InterceptorRegistry interceptorRegistry) {
  this.cfg = this.generateMyriadConfiguration(configFile);
  this.hadoopConf = hadoopConf;
  this.yarnScheduler = yarnScheduler;
  this.rmContext = rmContext;
  this.interceptorRegistry = interceptorRegistry;
}
 
Example 18
Source Project: big-c   Source File: RMContainerImpl.java    License: Apache License 2.0 5 votes vote down vote up
public RMContainerImpl(Container container,
    ApplicationAttemptId appAttemptId, NodeId nodeId,
    String user, RMContext rmContext, long creationTime) {
  this.stateMachine = stateMachineFactory.make(this);
  this.containerId = container.getId();
  this.nodeId = nodeId;
  this.container = container;
  this.appAttemptId = appAttemptId;
  this.user = user;
  this.creationTime = creationTime;
  this.rmContext = rmContext;
  this.eventHandler = rmContext.getDispatcher().getEventHandler();
  this.containerAllocationExpirer = rmContext.getContainerAllocationExpirer();
  this.isAMContainer = false;
  this.resourceRequests  = null;
  this.resumeOpportunity = 0;
  
  this.utilization = 1;
  this.suspendTime = new LinkedList<Long>();
  this.resumeTime  = new LinkedList<Long>();

  ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
  this.readLock = lock.readLock();
  this.writeLock = lock.writeLock();
  
  this.PR_NUMBER=rmContext.getYarnConfiguration().getInt(
  		"yarn.resourcemanager.monitor.capacity.preemption.pr_number", 2);
  		

  rmContext.getRMApplicationHistoryWriter().containerStarted(this);
  rmContext.getSystemMetricsPublisher().containerCreated(
      this, this.creationTime);
}
 
Example 19
@Before
@SuppressWarnings("unchecked")
public void setup() {
  conf = new Configuration(false);
  conf.setLong(WAIT_TIME_BEFORE_KILL, 10000);
  conf.setLong(MONITORING_INTERVAL, 3000);
  // report "ideal" preempt
  conf.setFloat(TOTAL_PREEMPTION_PER_ROUND, (float) 1.0);
  conf.setFloat(NATURAL_TERMINATION_FACTOR, (float) 1.0);
  conf.set(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES,
      ProportionalCapacityPreemptionPolicy.class.getCanonicalName());
  conf.setBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true);
  // FairScheduler doesn't support this test,
  // Set CapacityScheduler as the scheduler for this test.
  conf.set("yarn.resourcemanager.scheduler.class",
      CapacityScheduler.class.getName());

  mClock = mock(Clock.class);
  mCS = mock(CapacityScheduler.class);
  when(mCS.getResourceCalculator()).thenReturn(rc);
  lm = mock(RMNodeLabelsManager.class);
  schedConf = new CapacitySchedulerConfiguration();
  when(mCS.getConfiguration()).thenReturn(schedConf);
  rmContext = mock(RMContext.class);
  when(mCS.getRMContext()).thenReturn(rmContext);
  when(rmContext.getNodeLabelManager()).thenReturn(lm);
  mDisp = mock(EventHandler.class);
  rand = new Random();
  long seed = rand.nextLong();
  System.out.println(name.getMethodName() + " SEED: " + seed);
  rand.setSeed(seed);
  appAlloc = 0;
}
 
Example 20
Source Project: big-c   Source File: FSAppAttempt.java    License: Apache License 2.0 5 votes vote down vote up
public FSAppAttempt(FairScheduler scheduler,
    ApplicationAttemptId applicationAttemptId, String user, FSLeafQueue queue,
    ActiveUsersManager activeUsersManager, RMContext rmContext) {
  super(applicationAttemptId, user, queue, activeUsersManager, rmContext);

  this.scheduler = scheduler;
  this.startTime = scheduler.getClock().getTime();
  this.priority = Priority.newInstance(1);
  this.resourceWeights = new ResourceWeights();
}
 
Example 21
Source Project: hadoop   Source File: TestAMRMClientOnRMRestart.java    License: Apache License 2.0 5 votes vote down vote up
public MyFifoScheduler(RMContext rmContext) {
  super();
  try {
    Configuration conf = new Configuration();
    reinitialize(conf, rmContext);
  } catch (IOException ie) {
    assert (false);
  }
}
 
Example 22
Source Project: hadoop   Source File: FairScheduler.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void reinitialize(Configuration conf, RMContext rmContext)
    throws IOException {
  try {
    allocsLoader.reloadAllocations();
  } catch (Exception e) {
    LOG.error("Failed to reload allocations file", e);
  }
}
 
Example 23
Source Project: big-c   Source File: TestRMDelegationTokens.java    License: Apache License 2.0 5 votes vote down vote up
public TestRMDelegationTokenSecretManager(long delegationKeyUpdateInterval,
    long delegationTokenMaxLifetime, long delegationTokenRenewInterval,
    long delegationTokenRemoverScanInterval, RMContext rmContext) {
  super(delegationKeyUpdateInterval, delegationTokenMaxLifetime,
    delegationTokenRenewInterval, delegationTokenRemoverScanInterval,
    rmContext);
}
 
Example 24
Source Project: big-c   Source File: TestAMRMClientOnRMRestart.java    License: Apache License 2.0 5 votes vote down vote up
public MyFifoScheduler(RMContext rmContext) {
  super();
  try {
    Configuration conf = new Configuration();
    reinitialize(conf, rmContext);
  } catch (IOException ie) {
    assert (false);
  }
}
 
Example 25
Source Project: big-c   Source File: FairSchedulerTestBase.java    License: Apache License 2.0 5 votes vote down vote up
protected void createApplicationWithAMResource(ApplicationAttemptId attId,
    String queue, String user, Resource amResource) {
  RMContext rmContext = resourceManager.getRMContext();
  RMApp rmApp = new RMAppImpl(attId.getApplicationId(), rmContext, conf,
      null, null, null, ApplicationSubmissionContext.newInstance(null, null,
      null, null, null, false, false, 0, amResource, null), null, null,
      0, null, null, null);
  rmContext.getRMApps().put(attId.getApplicationId(), rmApp);
  AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent(
      attId.getApplicationId(), queue, user);
  scheduler.handle(appAddedEvent);
  AppAttemptAddedSchedulerEvent attempAddedEvent =
      new AppAttemptAddedSchedulerEvent(attId, false);
  scheduler.handle(attempAddedEvent);
}
 
Example 26
Source Project: big-c   Source File: AMLauncher.java    License: Apache License 2.0 5 votes vote down vote up
public AMLauncher(RMContext rmContext, RMAppAttempt application,
    AMLauncherEventType eventType, Configuration conf) {
  this.application = application;
  this.conf = conf;
  this.eventType = eventType;
  this.rmContext = rmContext;
  this.handler = rmContext.getDispatcher().getEventHandler();
  this.masterContainer = application.getMasterContainer();
}
 
Example 27
Source Project: hadoop   Source File: FiCaSchedulerApp.java    License: Apache License 2.0 5 votes vote down vote up
public FiCaSchedulerApp(ApplicationAttemptId applicationAttemptId, 
    String user, Queue queue, ActiveUsersManager activeUsersManager,
    RMContext rmContext) {
  super(applicationAttemptId, user, queue, activeUsersManager, rmContext);
  
  RMApp rmApp = rmContext.getRMApps().get(getApplicationId());
  
  Resource amResource;
  String partition;

  if (rmApp == null || rmApp.getAMResourceRequest() == null) {
    // the rmApp may be undefined (the resource manager checks for this too)
    // and unmanaged applications do not provide an amResource request
    // in these cases, provide a default using the scheduler
    amResource = rmContext.getScheduler().getMinimumResourceCapability();
    partition = CommonNodeLabelsManager.NO_LABEL;
  } else {
    amResource = rmApp.getAMResourceRequest().getCapability();
    partition =
        (rmApp.getAMResourceRequest().getNodeLabelExpression() == null)
        ? CommonNodeLabelsManager.NO_LABEL
        : rmApp.getAMResourceRequest().getNodeLabelExpression();
  }
  
  setAppAMNodePartitionName(partition);
  setAMResource(partition, amResource);
}
 
Example 28
Source Project: big-c   Source File: SchedulerUtils.java    License: Apache License 2.0 5 votes vote down vote up
public static void normalizeAndvalidateRequest(ResourceRequest resReq,
    Resource maximumResource, String queueName, YarnScheduler scheduler,
    RMContext rmContext, QueueInfo queueInfo)
    throws InvalidResourceRequestException {
  normalizeAndValidateRequest(resReq, maximumResource, queueName, scheduler,
      false, rmContext, queueInfo);
}
 
Example 29
Source Project: incubator-myriad   Source File: NMHeartBeatHandler.java    License: Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
protected void handleStatusUpdate(RMNodeEvent event, RMContext context) {
  if (!(event instanceof RMNodeStatusEvent)) {
    logger.error("{} not an instance of {}", event.getClass().getName(), RMNodeStatusEvent.class.getName());
    return;
  }

  RMNodeStatusEvent statusEvent = (RMNodeStatusEvent) event;
  RMNode rmNode = context.getRMNodes().get(event.getNodeId());
  String hostName = rmNode.getNodeID().getHost();

  Node host = nodeStore.getNode(hostName);
  if (host != null) {
    host.snapshotRunningContainers();
  }

  /*
   * Set the new node capacity which is the sum of the current node resources plus those offered by Mesos. 
   * If the sum is greater than the max capacity of the node, reject the offer.
   */
  Resource offeredResources = getNewResourcesOfferedByMesos(hostName);
  Resource currentResources = getResourcesUnderUse(statusEvent);
  
  if (offerWithinResourceLimits(currentResources, offeredResources)) {
    yarnNodeCapacityMgr.setNodeCapacity(rmNode, Resources.add(currentResources, offeredResources));
    logger.info("Updated resources for {} with {} cores and {} memory", rmNode.getNode().getName(), 
            offeredResources.getVirtualCores(), offeredResources.getMemory());
  } else {
    logger.info("Did not update {} with {} cores and {} memory, over max cpu cores and/or max memory", 
            rmNode.getNode().getName(), offeredResources.getVirtualCores(), offeredResources.getMemory());
  }
}
 
Example 30
Source Project: big-c   Source File: TestRMWebApp.java    License: Apache License 2.0 5 votes vote down vote up
@Test public void testNodesPage() {
  // 10 nodes. Two of each type.
  final RMContext rmContext = mockRMContext(3, 2, 12, 8*GiB);
  Injector injector = WebAppTests.createMockInjector(RMContext.class,
      rmContext,
      new Module() {
    @Override
    public void configure(Binder binder) {
      try {
        binder.bind(ResourceManager.class).toInstance(mockRm(rmContext));
      } catch (IOException e) {
        throw new IllegalStateException(e);
      }
    }
  });

  // All nodes
  NodesPage instance = injector.getInstance(NodesPage.class);
  instance.render();
  WebAppTests.flushOutput(injector);

  // Unhealthy nodes
  instance.moreParams().put(YarnWebParams.NODE_STATE,
    NodeState.UNHEALTHY.toString());
  instance.render();
  WebAppTests.flushOutput(injector);

  // Lost nodes
  instance.moreParams().put(YarnWebParams.NODE_STATE,
    NodeState.LOST.toString());
  instance.render();
  WebAppTests.flushOutput(injector);

}