org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler Java Examples

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: RMServerUtils.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Utility method to validate a list resource requests, by insuring that the
 * requested memory/vcore is non-negative and not greater than max
 */
public static void normalizeAndValidateRequests(List<ResourceRequest> ask,
    Resource maximumResource, String queueName, YarnScheduler scheduler,
    RMContext rmContext)
    throws InvalidResourceRequestException {

  QueueInfo queueInfo = null;
  try {
    queueInfo = scheduler.getQueueInfo(queueName, false, false);
  } catch (IOException e) {
  }

  for (ResourceRequest resReq : ask) {
    SchedulerUtils.normalizeAndvalidateRequest(resReq, maximumResource,
        queueName, scheduler, rmContext, queueInfo);
  }
}
 
Example #2
Source File: TestClientRMService.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private static YarnScheduler mockYarnScheduler() {
  YarnScheduler yarnScheduler = mock(YarnScheduler.class);
  when(yarnScheduler.getMinimumResourceCapability()).thenReturn(
      Resources.createResource(
          YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB));
  when(yarnScheduler.getMaximumResourceCapability()).thenReturn(
      Resources.createResource(
          YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB));
  when(yarnScheduler.getAppsInQueue(QUEUE_1)).thenReturn(
      Arrays.asList(getApplicationAttemptId(101), getApplicationAttemptId(102)));
  when(yarnScheduler.getAppsInQueue(QUEUE_2)).thenReturn(
      Arrays.asList(getApplicationAttemptId(103)));
  ApplicationAttemptId attemptId = getApplicationAttemptId(1);
  when(yarnScheduler.getAppResourceUsageReport(attemptId)).thenReturn(null);
  ResourceCalculator rc = new DefaultResourceCalculator();
  when(yarnScheduler.getResourceCalculator()).thenReturn(rc);
  return yarnScheduler;
}
 
Example #3
Source File: TestClientRMService.java    From big-c with Apache License 2.0 6 votes vote down vote up
public ClientRMService createRMService() throws IOException {
  YarnScheduler yarnScheduler = mockYarnScheduler();
  RMContext rmContext = mock(RMContext.class);
  mockRMContext(yarnScheduler, rmContext);
  ConcurrentHashMap<ApplicationId, RMApp> apps = getRMApps(rmContext,
      yarnScheduler);
  when(rmContext.getRMApps()).thenReturn(apps);
  when(rmContext.getYarnConfiguration()).thenReturn(new Configuration());
  RMAppManager appManager = new RMAppManager(rmContext, yarnScheduler, null,
      mock(ApplicationACLsManager.class), new Configuration());
  when(rmContext.getDispatcher().getEventHandler()).thenReturn(
      new EventHandler<Event>() {
        public void handle(Event event) {
        }
      });

  ApplicationACLsManager mockAclsManager = mock(ApplicationACLsManager.class);
  QueueACLsManager mockQueueACLsManager = mock(QueueACLsManager.class);
  when(
      mockQueueACLsManager.checkAccess(any(UserGroupInformation.class),
          any(QueueACL.class), anyString())).thenReturn(true);
  return new ClientRMService(rmContext, yarnScheduler, appManager,
      mockAclsManager, mockQueueACLsManager, null);
}
 
Example #4
Source File: TestClientRMService.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private ConcurrentHashMap<ApplicationId, RMApp> getRMApps(
    RMContext rmContext, YarnScheduler yarnScheduler) {
  ConcurrentHashMap<ApplicationId, RMApp> apps = 
    new ConcurrentHashMap<ApplicationId, RMApp>();
  ApplicationId applicationId1 = getApplicationId(1);
  ApplicationId applicationId2 = getApplicationId(2);
  ApplicationId applicationId3 = getApplicationId(3);
  YarnConfiguration config = new YarnConfiguration();
  apps.put(applicationId1, getRMApp(rmContext, yarnScheduler, applicationId1,
      config, "testqueue", 10, 3, 3));
  apps.put(applicationId2, getRMApp(rmContext, yarnScheduler, applicationId2,
      config, "a", 20, 2, 2));
  apps.put(applicationId3, getRMApp(rmContext, yarnScheduler, applicationId3,
      config, "testqueue", 40, 5, 5));
  return apps;
}
 
Example #5
Source File: TestClientRMService.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private void mockRMContext(YarnScheduler yarnScheduler, RMContext rmContext)
    throws IOException {
  Dispatcher dispatcher = mock(Dispatcher.class);
  when(rmContext.getDispatcher()).thenReturn(dispatcher);
  EventHandler eventHandler = mock(EventHandler.class);
  when(dispatcher.getEventHandler()).thenReturn(eventHandler);
  QueueInfo queInfo = recordFactory.newRecordInstance(QueueInfo.class);
  queInfo.setQueueName("testqueue");
  when(yarnScheduler.getQueueInfo(eq("testqueue"), anyBoolean(), anyBoolean()))
      .thenReturn(queInfo);
  when(yarnScheduler.getQueueInfo(eq("nonexistentqueue"), anyBoolean(), anyBoolean()))
      .thenThrow(new IOException("queue does not exist"));
  RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
  when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer);
  SystemMetricsPublisher publisher = mock(SystemMetricsPublisher.class);
  when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher);
  ConcurrentHashMap<ApplicationId, RMApp> apps = getRMApps(rmContext,
      yarnScheduler);
  when(rmContext.getRMApps()).thenReturn(apps);
  when(yarnScheduler.getAppsInQueue(eq("testqueue"))).thenReturn(
      getSchedulerApps(apps));
   ResourceScheduler rs = mock(ResourceScheduler.class);
   when(rmContext.getScheduler()).thenReturn(rs);
}
 
Example #6
Source File: TestClientRMService.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testGetApplicationResourceUsageReportDummy() throws YarnException,
    IOException {
  ApplicationAttemptId attemptId = getApplicationAttemptId(1);
  YarnScheduler yarnScheduler = mockYarnScheduler();
  RMContext rmContext = mock(RMContext.class);
  mockRMContext(yarnScheduler, rmContext);
  when(rmContext.getDispatcher().getEventHandler()).thenReturn(
      new EventHandler<Event>() {
        public void handle(Event event) {
        }
      });
  ApplicationSubmissionContext asContext = 
      mock(ApplicationSubmissionContext.class);
  YarnConfiguration config = new YarnConfiguration();
  RMAppAttemptImpl rmAppAttemptImpl = new RMAppAttemptImpl(attemptId,
      rmContext, yarnScheduler, null, asContext, config, false, null);
  ApplicationResourceUsageReport report = rmAppAttemptImpl
      .getApplicationResourceUsageReport();
  assertEquals(report, RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT);
}
 
Example #7
Source File: TestClientRMService.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public ClientRMService createRMService() throws IOException {
  YarnScheduler yarnScheduler = mockYarnScheduler();
  RMContext rmContext = mock(RMContext.class);
  mockRMContext(yarnScheduler, rmContext);
  ConcurrentHashMap<ApplicationId, RMApp> apps = getRMApps(rmContext,
      yarnScheduler);
  when(rmContext.getRMApps()).thenReturn(apps);
  when(rmContext.getYarnConfiguration()).thenReturn(new Configuration());
  RMAppManager appManager = new RMAppManager(rmContext, yarnScheduler, null,
      mock(ApplicationACLsManager.class), new Configuration());
  when(rmContext.getDispatcher().getEventHandler()).thenReturn(
      new EventHandler<Event>() {
        public void handle(Event event) {
        }
      });

  ApplicationACLsManager mockAclsManager = mock(ApplicationACLsManager.class);
  QueueACLsManager mockQueueACLsManager = mock(QueueACLsManager.class);
  when(
      mockQueueACLsManager.checkAccess(any(UserGroupInformation.class),
          any(QueueACL.class), anyString())).thenReturn(true);
  return new ClientRMService(rmContext, yarnScheduler, appManager,
      mockAclsManager, mockQueueACLsManager, null);
}
 
Example #8
Source File: TestClientRMService.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testGetApplicationResourceUsageReportDummy() throws YarnException,
    IOException {
  ApplicationAttemptId attemptId = getApplicationAttemptId(1);
  YarnScheduler yarnScheduler = mockYarnScheduler();
  RMContext rmContext = mock(RMContext.class);
  mockRMContext(yarnScheduler, rmContext);
  when(rmContext.getDispatcher().getEventHandler()).thenReturn(
      new EventHandler<Event>() {
        public void handle(Event event) {
        }
      });
  ApplicationSubmissionContext asContext = 
      mock(ApplicationSubmissionContext.class);
  YarnConfiguration config = new YarnConfiguration();
  RMAppAttemptImpl rmAppAttemptImpl = new RMAppAttemptImpl(attemptId,
      rmContext, yarnScheduler, null, asContext, config, false, null);
  ApplicationResourceUsageReport report = rmAppAttemptImpl
      .getApplicationResourceUsageReport();
  assertEquals(report, RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT);
}
 
Example #9
Source File: RMServerUtils.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Utility method to validate a list resource requests, by insuring that the
 * requested memory/vcore is non-negative and not greater than max
 */
public static void normalizeAndValidateRequests(List<ResourceRequest> ask,
    Resource maximumResource, String queueName, YarnScheduler scheduler,
    RMContext rmContext)
    throws InvalidResourceRequestException {

  QueueInfo queueInfo = null;
  try {
    queueInfo = scheduler.getQueueInfo(queueName, false, false);
  } catch (IOException e) {
  }

  for (ResourceRequest resReq : ask) {
    SchedulerUtils.normalizeAndvalidateRequest(resReq, maximumResource,
        queueName, scheduler, rmContext, queueInfo);
  }
}
 
Example #10
Source File: TestClientRMService.java    From big-c with Apache License 2.0 6 votes vote down vote up
private void mockRMContext(YarnScheduler yarnScheduler, RMContext rmContext)
    throws IOException {
  Dispatcher dispatcher = mock(Dispatcher.class);
  when(rmContext.getDispatcher()).thenReturn(dispatcher);
  EventHandler eventHandler = mock(EventHandler.class);
  when(dispatcher.getEventHandler()).thenReturn(eventHandler);
  QueueInfo queInfo = recordFactory.newRecordInstance(QueueInfo.class);
  queInfo.setQueueName("testqueue");
  when(yarnScheduler.getQueueInfo(eq("testqueue"), anyBoolean(), anyBoolean()))
      .thenReturn(queInfo);
  when(yarnScheduler.getQueueInfo(eq("nonexistentqueue"), anyBoolean(), anyBoolean()))
      .thenThrow(new IOException("queue does not exist"));
  RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
  when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer);
  SystemMetricsPublisher publisher = mock(SystemMetricsPublisher.class);
  when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher);
  ConcurrentHashMap<ApplicationId, RMApp> apps = getRMApps(rmContext,
      yarnScheduler);
  when(rmContext.getRMApps()).thenReturn(apps);
  when(yarnScheduler.getAppsInQueue(eq("testqueue"))).thenReturn(
      getSchedulerApps(apps));
   ResourceScheduler rs = mock(ResourceScheduler.class);
   when(rmContext.getScheduler()).thenReturn(rs);
}
 
Example #11
Source File: RMAppAttemptImpl.java    From big-c with Apache License 2.0 6 votes vote down vote up
public RMAppAttemptImpl(ApplicationAttemptId appAttemptId,
    RMContext rmContext, YarnScheduler scheduler,
    ApplicationMasterService masterService,
    ApplicationSubmissionContext submissionContext,
    Configuration conf, boolean maybeLastAttempt, ResourceRequest amReq) {
  this.conf = conf;
  this.applicationAttemptId = appAttemptId;
  this.rmContext = rmContext;
  this.eventHandler = rmContext.getDispatcher().getEventHandler();
  this.submissionContext = submissionContext;
  this.scheduler = scheduler;
  this.masterService = masterService;

  ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
  this.readLock = lock.readLock();
  this.writeLock = lock.writeLock();

  this.proxiedTrackingUrl = generateProxyUriWithScheme();
  this.maybeLastAttempt = maybeLastAttempt;
  this.stateMachine = stateMachineFactory.make(this);

  this.attemptMetrics =
      new RMAppAttemptMetrics(applicationAttemptId, rmContext);
  
  this.amReq = amReq;
}
 
Example #12
Source File: TestClientRMService.java    From big-c with Apache License 2.0 6 votes vote down vote up
private ConcurrentHashMap<ApplicationId, RMApp> getRMApps(
    RMContext rmContext, YarnScheduler yarnScheduler) {
  ConcurrentHashMap<ApplicationId, RMApp> apps = 
    new ConcurrentHashMap<ApplicationId, RMApp>();
  ApplicationId applicationId1 = getApplicationId(1);
  ApplicationId applicationId2 = getApplicationId(2);
  ApplicationId applicationId3 = getApplicationId(3);
  YarnConfiguration config = new YarnConfiguration();
  apps.put(applicationId1, getRMApp(rmContext, yarnScheduler, applicationId1,
      config, "testqueue", 10, 3));
  apps.put(applicationId2, getRMApp(rmContext, yarnScheduler, applicationId2,
      config, "a", 20, 2));
  apps.put(applicationId3, getRMApp(rmContext, yarnScheduler, applicationId3,
      config, "testqueue", 40, 5));
  return apps;
}
 
Example #13
Source File: RMAppManager.java    From big-c with Apache License 2.0 6 votes vote down vote up
public RMAppManager(RMContext context,
    YarnScheduler scheduler, ApplicationMasterService masterService,
    ApplicationACLsManager applicationACLsManager, Configuration conf) {
  this.rmContext = context;
  this.scheduler = scheduler;
  this.masterService = masterService;
  this.applicationACLsManager = applicationACLsManager;
  this.conf = conf;
  this.maxCompletedAppsInMemory = conf.getInt(
      YarnConfiguration.RM_MAX_COMPLETED_APPLICATIONS,
      YarnConfiguration.DEFAULT_RM_MAX_COMPLETED_APPLICATIONS);
  this.maxCompletedAppsInStateStore =
      conf.getInt(
        YarnConfiguration.RM_STATE_STORE_MAX_COMPLETED_APPLICATIONS,
        YarnConfiguration.DEFAULT_RM_STATE_STORE_MAX_COMPLETED_APPLICATIONS);
  if (this.maxCompletedAppsInStateStore > this.maxCompletedAppsInMemory) {
    this.maxCompletedAppsInStateStore = this.maxCompletedAppsInMemory;
  }
}
 
Example #14
Source File: TestClientRMService.java    From big-c with Apache License 2.0 6 votes vote down vote up
private static YarnScheduler mockYarnScheduler() {
  YarnScheduler yarnScheduler = mock(YarnScheduler.class);
  when(yarnScheduler.getMinimumResourceCapability()).thenReturn(
      Resources.createResource(
          YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB));
  when(yarnScheduler.getMaximumResourceCapability()).thenReturn(
      Resources.createResource(
          YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB));
  when(yarnScheduler.getAppsInQueue(QUEUE_1)).thenReturn(
      Arrays.asList(getApplicationAttemptId(101), getApplicationAttemptId(102)));
  when(yarnScheduler.getAppsInQueue(QUEUE_2)).thenReturn(
      Arrays.asList(getApplicationAttemptId(103)));
  ApplicationAttemptId attemptId = getApplicationAttemptId(1);
  when(yarnScheduler.getAppResourceUsageReport(attemptId)).thenReturn(null);
  ResourceCalculator rc = new DefaultResourceCalculator();
  when(yarnScheduler.getResourceCalculator()).thenReturn(rc);
  return yarnScheduler;
}
 
Example #15
Source File: RMAppManager.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public RMAppManager(RMContext context,
    YarnScheduler scheduler, ApplicationMasterService masterService,
    ApplicationACLsManager applicationACLsManager, Configuration conf) {
  this.rmContext = context;
  this.scheduler = scheduler;
  this.masterService = masterService;
  this.applicationACLsManager = applicationACLsManager;
  this.conf = conf;
  this.maxCompletedAppsInMemory = conf.getInt(
      YarnConfiguration.RM_MAX_COMPLETED_APPLICATIONS,
      YarnConfiguration.DEFAULT_RM_MAX_COMPLETED_APPLICATIONS);
  this.maxCompletedAppsInStateStore =
      conf.getInt(
        YarnConfiguration.RM_STATE_STORE_MAX_COMPLETED_APPLICATIONS,
        YarnConfiguration.DEFAULT_RM_STATE_STORE_MAX_COMPLETED_APPLICATIONS);
  if (this.maxCompletedAppsInStateStore > this.maxCompletedAppsInMemory) {
    this.maxCompletedAppsInStateStore = this.maxCompletedAppsInMemory;
  }
}
 
Example #16
Source File: RMAppAttemptImpl.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public RMAppAttemptImpl(ApplicationAttemptId appAttemptId,
    RMContext rmContext, YarnScheduler scheduler,
    ApplicationMasterService masterService,
    ApplicationSubmissionContext submissionContext,
    Configuration conf, boolean maybeLastAttempt, ResourceRequest amReq) {
  this.conf = conf;
  this.applicationAttemptId = appAttemptId;
  this.rmContext = rmContext;
  this.eventHandler = rmContext.getDispatcher().getEventHandler();
  this.submissionContext = submissionContext;
  this.scheduler = scheduler;
  this.masterService = masterService;

  ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
  this.readLock = lock.readLock();
  this.writeLock = lock.writeLock();

  this.proxiedTrackingUrl = generateProxyUriWithScheme();
  this.maybeLastAttempt = maybeLastAttempt;
  this.stateMachine = stateMachineFactory.make(this);

  this.attemptMetrics =
      new RMAppAttemptMetrics(applicationAttemptId, rmContext);
  
  this.amReq = amReq;
}
 
Example #17
Source File: ClientRMService.java    From big-c with Apache License 2.0 5 votes vote down vote up
public ClientRMService(RMContext rmContext, YarnScheduler scheduler,
    RMAppManager rmAppManager, ApplicationACLsManager applicationACLsManager,
    QueueACLsManager queueACLsManager,
    RMDelegationTokenSecretManager rmDTSecretManager) {
  this(rmContext, scheduler, rmAppManager, applicationACLsManager,
      queueACLsManager, rmDTSecretManager, new UTCClock());
}
 
Example #18
Source File: TestClientRMService.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testGetApplicationReport() throws Exception {
  YarnScheduler yarnScheduler = mock(YarnScheduler.class);
  RMContext rmContext = mock(RMContext.class);
  mockRMContext(yarnScheduler, rmContext);

  ApplicationId appId1 = getApplicationId(1);

  ApplicationACLsManager mockAclsManager = mock(ApplicationACLsManager.class);
  when(
      mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(),
          ApplicationAccessType.VIEW_APP, null, appId1)).thenReturn(true);

  ClientRMService rmService = new ClientRMService(rmContext, yarnScheduler,
      null, mockAclsManager, null, null);
  try {
    RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
    GetApplicationReportRequest request = recordFactory
        .newRecordInstance(GetApplicationReportRequest.class);
    request.setApplicationId(appId1);
    GetApplicationReportResponse response = 
        rmService.getApplicationReport(request);
    ApplicationReport report = response.getApplicationReport();
    ApplicationResourceUsageReport usageReport = 
        report.getApplicationResourceUsageReport();
    Assert.assertEquals(10, usageReport.getMemorySeconds());
    Assert.assertEquals(3, usageReport.getVcoreSeconds());
  } finally {
    rmService.close();
  }
}
 
Example #19
Source File: ClientRMService.java    From big-c with Apache License 2.0 5 votes vote down vote up
public ClientRMService(RMContext rmContext, YarnScheduler scheduler,
    RMAppManager rmAppManager, ApplicationACLsManager applicationACLsManager,
    QueueACLsManager queueACLsManager,
    RMDelegationTokenSecretManager rmDTSecretManager, Clock clock) {
  super(ClientRMService.class.getName());
  this.scheduler = scheduler;
  this.rmContext = rmContext;
  this.rmAppManager = rmAppManager;
  this.applicationsACLsManager = applicationACLsManager;
  this.queueACLsManager = queueACLsManager;
  this.rmDTSecretManager = rmDTSecretManager;
  this.reservationSystem = rmContext.getReservationSystem();
  this.clock = clock;
  this.rValidator = new ReservationInputValidator(clock);
}
 
Example #20
Source File: TestKillApplicationWithRMHA.java    From big-c with Apache License 2.0 5 votes vote down vote up
public MyClientRMService(RMContext rmContext, YarnScheduler scheduler,
    RMAppManager rmAppManager,
    ApplicationACLsManager applicationACLsManager,
    QueueACLsManager queueACLsManager,
    RMDelegationTokenSecretManager rmDTSecretManager) {
  super(rmContext, scheduler, rmAppManager, applicationACLsManager,
      queueACLsManager, rmDTSecretManager);
  this.rmContext = rmContext;
}
 
Example #21
Source File: TestRMAppTransitions.java    From big-c with Apache License 2.0 5 votes vote down vote up
protected RMApp createNewTestApp(ApplicationSubmissionContext submissionContext) {
  ApplicationId applicationId = MockApps.newAppID(appId++);
  String user = MockApps.newUserName();
  String name = MockApps.newAppName();
  String queue = MockApps.newQueue();
  // ensure max application attempts set to known value
  conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, maxAppAttempts);
  scheduler = mock(YarnScheduler.class);

  ApplicationMasterService masterService =
      new ApplicationMasterService(rmContext, scheduler);
  
  if(submissionContext == null) {
    submissionContext = new ApplicationSubmissionContextPBImpl();
  }
  // applicationId will not be used because RMStateStore is mocked,
  // but applicationId is still set for safety
  submissionContext.setApplicationId(applicationId);

  RMApp application =
      new RMAppImpl(applicationId, rmContext, conf, name, user, queue,
        submissionContext, scheduler, masterService,
        System.currentTimeMillis(), "YARN", null, null);

  testAppStartState(applicationId, user, name, queue, application);
  this.rmContext.getRMApps().putIfAbsent(application.getApplicationId(),
      application);
  return application;
}
 
Example #22
Source File: RMHATestBase.java    From big-c with Apache License 2.0 5 votes vote down vote up
public MyRMAppManager(RMContext context, YarnScheduler scheduler,
    ApplicationMasterService masterService,
    ApplicationACLsManager applicationACLsManager, Configuration conf) {
  super(context, scheduler, masterService, applicationACLsManager, conf);
  this.conf = conf;
  this.rmContext = context;
}
 
Example #23
Source File: TestContainerAllocation.java    From big-c with Apache License 2.0 5 votes vote down vote up
private void checkTaskContainersHost(ApplicationAttemptId attemptId,
    ContainerId containerId, ResourceManager rm, String host) {
  YarnScheduler scheduler = rm.getRMContext().getScheduler();
  SchedulerAppReport appReport = scheduler.getSchedulerAppInfo(attemptId);

  Assert.assertTrue(appReport.getLiveContainers().size() > 0);
  for (RMContainer c : appReport.getLiveContainers()) {
    if (c.getContainerId().equals(containerId)) {
      Assert.assertEquals(host, c.getAllocatedNode().getHost());
    }
  }
}
 
Example #24
Source File: ProtocolHATestBase.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public CustomedClientRMService(RMContext rmContext,
    YarnScheduler scheduler, RMAppManager rmAppManager,
    ApplicationACLsManager applicationACLsManager,
    QueueACLsManager queueACLsManager,
    RMDelegationTokenSecretManager rmDTSecretManager) {
  super(rmContext, scheduler, rmAppManager, applicationACLsManager,
      queueACLsManager, rmDTSecretManager);
}
 
Example #25
Source File: TestClientRMService.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testGetApplicationReport() throws Exception {
  YarnScheduler yarnScheduler = mock(YarnScheduler.class);
  RMContext rmContext = mock(RMContext.class);
  mockRMContext(yarnScheduler, rmContext);

  ApplicationId appId1 = getApplicationId(1);

  ApplicationACLsManager mockAclsManager = mock(ApplicationACLsManager.class);
  when(
      mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(),
          ApplicationAccessType.VIEW_APP, null, appId1)).thenReturn(true);

  ClientRMService rmService = new ClientRMService(rmContext, yarnScheduler,
      null, mockAclsManager, null, null);
  try {
    RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
    GetApplicationReportRequest request = recordFactory
        .newRecordInstance(GetApplicationReportRequest.class);
    request.setApplicationId(appId1);
    GetApplicationReportResponse response = 
        rmService.getApplicationReport(request);
    ApplicationReport report = response.getApplicationReport();
    ApplicationResourceUsageReport usageReport = 
        report.getApplicationResourceUsageReport();
    Assert.assertEquals(10, usageReport.getMemorySeconds());
    Assert.assertEquals(3, usageReport.getVcoreSeconds());
    Assert.assertEquals(3, usageReport.getGcoreSeconds());
  } finally {
    rmService.close();
  }
}
 
Example #26
Source File: TestKillApplicationWithRMHA.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public MyClientRMService(RMContext rmContext, YarnScheduler scheduler,
    RMAppManager rmAppManager,
    ApplicationACLsManager applicationACLsManager,
    QueueACLsManager queueACLsManager,
    RMDelegationTokenSecretManager rmDTSecretManager) {
  super(rmContext, scheduler, rmAppManager, applicationACLsManager,
      queueACLsManager, rmDTSecretManager);
  this.rmContext = rmContext;
}
 
Example #27
Source File: RMHATestBase.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public MyRMAppManager(RMContext context, YarnScheduler scheduler,
    ApplicationMasterService masterService,
    ApplicationACLsManager applicationACLsManager, Configuration conf) {
  super(context, scheduler, masterService, applicationACLsManager, conf);
  this.conf = conf;
  this.rmContext = context;
}
 
Example #28
Source File: TestContainerAllocation.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private void checkTaskContainersHost(ApplicationAttemptId attemptId,
    ContainerId containerId, ResourceManager rm, String host) {
  YarnScheduler scheduler = rm.getRMContext().getScheduler();
  SchedulerAppReport appReport = scheduler.getSchedulerAppInfo(attemptId);

  Assert.assertTrue(appReport.getLiveContainers().size() > 0);
  for (RMContainer c : appReport.getLiveContainers()) {
    if (c.getContainerId().equals(containerId)) {
      Assert.assertEquals(host, c.getAllocatedNode().getHost());
    }
  }
}
 
Example #29
Source File: TestRMAppTransitions.java    From hadoop with Apache License 2.0 5 votes vote down vote up
protected RMApp createNewTestApp(ApplicationSubmissionContext submissionContext) {
  ApplicationId applicationId = MockApps.newAppID(appId++);
  String user = MockApps.newUserName();
  String name = MockApps.newAppName();
  String queue = MockApps.newQueue();
  // ensure max application attempts set to known value
  conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, maxAppAttempts);
  scheduler = mock(YarnScheduler.class);

  ApplicationMasterService masterService =
      new ApplicationMasterService(rmContext, scheduler);
  
  if(submissionContext == null) {
    submissionContext = new ApplicationSubmissionContextPBImpl();
  }
  // applicationId will not be used because RMStateStore is mocked,
  // but applicationId is still set for safety
  submissionContext.setApplicationId(applicationId);

  RMApp application =
      new RMAppImpl(applicationId, rmContext, conf, name, user, queue,
        submissionContext, scheduler, masterService,
        System.currentTimeMillis(), "YARN", null, null);

  testAppStartState(applicationId, user, name, queue, application);
  this.rmContext.getRMApps().putIfAbsent(application.getApplicationId(),
      application);
  return application;
}
 
Example #30
Source File: ProtocolHATestBase.java    From big-c with Apache License 2.0 5 votes vote down vote up
public CustomedClientRMService(RMContext rmContext,
    YarnScheduler scheduler, RMAppManager rmAppManager,
    ApplicationACLsManager applicationACLsManager,
    QueueACLsManager queueACLsManager,
    RMDelegationTokenSecretManager rmDTSecretManager) {
  super(rmContext, scheduler, rmAppManager, applicationACLsManager,
      queueACLsManager, rmDTSecretManager);
}