Java Code Examples for org.apache.hadoop.yarn.api.records.ContainerId#newContainerId()

The following examples show how to use org.apache.hadoop.yarn.api.records.ContainerId#newContainerId() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestClientRMService.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testGetContainerReport() throws YarnException, IOException {
  ClientRMService rmService = createRMService();
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  GetContainerReportRequest request = recordFactory
      .newRecordInstance(GetContainerReportRequest.class);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
      ApplicationId.newInstance(123456, 1), 1);
  ContainerId containerId = ContainerId.newContainerId(attemptId, 1);
  request.setContainerId(containerId);

  try {
    GetContainerReportResponse response = rmService
        .getContainerReport(request);
    Assert.assertEquals(containerId, response.getContainerReport()
        .getContainerId());
  } catch (ApplicationNotFoundException ex) {
    Assert.fail(ex.getMessage());
  }
}
 
Example 2
Source File: TestProportionalCapacityPreemptionPolicy.java    From big-c with Apache License 2.0 6 votes vote down vote up
RMContainer mockContainer(ApplicationAttemptId appAttId, int id,
    Resource r, int cpriority) {
  ContainerId cId = ContainerId.newContainerId(appAttId, id);
  Container c = mock(Container.class);
  when(c.getResource()).thenReturn(r);
  when(c.getPriority()).thenReturn(Priority.create(cpriority));
  RMContainer mC = mock(RMContainer.class);
  when(mC.getContainerId()).thenReturn(cId);
  when(mC.getContainer()).thenReturn(c);
  when(mC.getApplicationAttemptId()).thenReturn(appAttId);
  if (priority.AMCONTAINER.getValue() == cpriority) {
    when(mC.isAMContainer()).thenReturn(true);
  }
  if (priority.LABELEDCONTAINER.getValue() == cpriority) {
    when(mC.getAllocatedNode()).thenReturn(NodeId.newInstance("node1", 0));
  }
  return mC;
}
 
Example 3
Source File: TestMRJobsWithHistoryService.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private void verifyJobReport(JobReport jobReport, JobId jobId) {
  List<AMInfo> amInfos = jobReport.getAMInfos();
  Assert.assertEquals(1, amInfos.size());
  AMInfo amInfo = amInfos.get(0);
  ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(jobId.getAppId(), 1);
  ContainerId amContainerId = ContainerId.newContainerId(appAttemptId, 1);
  Assert.assertEquals(appAttemptId, amInfo.getAppAttemptId());
  Assert.assertEquals(amContainerId, amInfo.getContainerId());
  Assert.assertTrue(jobReport.getSubmitTime() > 0);
  Assert.assertTrue(jobReport.getStartTime() > 0
      && jobReport.getStartTime() >= jobReport.getSubmitTime());
  Assert.assertTrue(jobReport.getFinishTime() > 0
      && jobReport.getFinishTime() >= jobReport.getStartTime());
}
 
Example 4
Source File: TestFileSystemApplicationHistoryStore.java    From big-c with Apache License 2.0 5 votes vote down vote up
private void testWriteHistoryData(
    int num, boolean missingContainer, boolean missingApplicationAttempt)
        throws IOException {
  // write application history data
  for (int i = 1; i <= num; ++i) {
    ApplicationId appId = ApplicationId.newInstance(0, i);
    writeApplicationStartData(appId);

    // write application attempt history data
    for (int j = 1; j <= num; ++j) {
      ApplicationAttemptId appAttemptId =
          ApplicationAttemptId.newInstance(appId, j);
      writeApplicationAttemptStartData(appAttemptId);

      if (missingApplicationAttempt && j == num) {
        continue;
      }
      // write container history data
      for (int k = 1; k <= num; ++k) {
        ContainerId containerId = ContainerId.newContainerId(appAttemptId, k);
        writeContainerStartData(containerId);
        if (missingContainer && k == num) {
          continue;
        }
        writeContainerFinishData(containerId);
      }
      writeApplicationAttemptFinishData(appAttemptId);
    }
    writeApplicationFinishData(appId);
  }
}
 
Example 5
Source File: TestNodeManagerShutdown.java    From big-c with Apache License 2.0 5 votes vote down vote up
public static ContainerId createContainerId() {
  ApplicationId appId = ApplicationId.newInstance(0, 0);
  ApplicationAttemptId appAttemptId =
      ApplicationAttemptId.newInstance(appId, 1);
  ContainerId containerId = ContainerId.newContainerId(appAttemptId, 0);
  return containerId;
}
 
Example 6
Source File: TestContainerId.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public static ContainerId newContainerId(int appId, int appAttemptId,
    long timestamp, long containerId) {
  ApplicationId applicationId = ApplicationId.newInstance(timestamp, appId);
  ApplicationAttemptId applicationAttemptId =
      ApplicationAttemptId.newInstance(applicationId, appAttemptId);
  return ContainerId.newContainerId(applicationAttemptId, containerId);
}
 
Example 7
Source File: MRApp.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public static ContainerId newContainerId(int appId, int appAttemptId,
    long timestamp, int containerId) {
  ApplicationId applicationId = ApplicationId.newInstance(timestamp, appId);
  ApplicationAttemptId applicationAttemptId =
      ApplicationAttemptId.newInstance(applicationId, appAttemptId);
  return ContainerId.newContainerId(applicationAttemptId, containerId);
}
 
Example 8
Source File: TestFileSystemApplicationHistoryStore.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private void testWriteHistoryData(
    int num, boolean missingContainer, boolean missingApplicationAttempt)
        throws IOException {
  // write application history data
  for (int i = 1; i <= num; ++i) {
    ApplicationId appId = ApplicationId.newInstance(0, i);
    writeApplicationStartData(appId);

    // write application attempt history data
    for (int j = 1; j <= num; ++j) {
      ApplicationAttemptId appAttemptId =
          ApplicationAttemptId.newInstance(appId, j);
      writeApplicationAttemptStartData(appAttemptId);

      if (missingApplicationAttempt && j == num) {
        continue;
      }
      // write container history data
      for (int k = 1; k <= num; ++k) {
        ContainerId containerId = ContainerId.newContainerId(appAttemptId, k);
        writeContainerStartData(containerId);
        if (missingContainer && k == num) {
          continue;
        }
        writeContainerFinishData(containerId);
      }
      writeApplicationAttemptFinishData(appAttemptId);
    }
    writeApplicationFinishData(appId);
  }
}
 
Example 9
Source File: TestProtocolRecords.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testRegisterNodeManagerRequest() {
  ApplicationId appId = ApplicationId.newInstance(123456789, 1);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);
  ContainerId containerId = ContainerId.newContainerId(attemptId, 1);

  NMContainerStatus containerReport =
      NMContainerStatus.newInstance(containerId,
        ContainerState.RUNNING, Resource.newInstance(1024, 1, 2), "diagnostics",
        0, Priority.newInstance(10), 1234);
  List<NMContainerStatus> reports = Arrays.asList(containerReport);
  RegisterNodeManagerRequest request =
      RegisterNodeManagerRequest.newInstance(
        NodeId.newInstance("1.1.1.1", 1000), 8080,
          Resource.newInstance(1024, 1, 2), "NM-version-id", reports,
          Arrays.asList(appId));
  RegisterNodeManagerRequest requestProto =
      new RegisterNodeManagerRequestPBImpl(
        ((RegisterNodeManagerRequestPBImpl) request).getProto());
  Assert.assertEquals(containerReport, requestProto
    .getNMContainerStatuses().get(0));
  Assert.assertEquals(8080, requestProto.getHttpPort());
  Assert.assertEquals("NM-version-id", requestProto.getNMVersion());
  Assert.assertEquals(NodeId.newInstance("1.1.1.1", 1000),
    requestProto.getNodeId());
  Assert.assertEquals(Resource.newInstance(1024, 1, 2),
    requestProto.getResource());
  Assert.assertEquals(1, requestProto.getRunningApplications().size());
  Assert.assertEquals(appId, requestProto.getRunningApplications().get(0)); 
}
 
Example 10
Source File: TestContainerManager.java    From big-c with Apache License 2.0 5 votes vote down vote up
private ContainerId createContainerId(int id) {
  ApplicationId appId = ApplicationId.newInstance(0, 0);
  ApplicationAttemptId appAttemptId =
      ApplicationAttemptId.newInstance(appId, 1);
  ContainerId containerId = ContainerId.newContainerId(appAttemptId, id);
  return containerId;
}
 
Example 11
Source File: TestYarnCLI.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test
public void testGetContainerReportException() throws Exception {
  ApplicationCLI cli = createAndGetAppCLI();
  ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
      applicationId, 1);
  long cntId = 1;
  ContainerId containerId1 = ContainerId.newContainerId(attemptId, cntId++);
  when(client.getContainerReport(containerId1)).thenThrow(
      new ApplicationNotFoundException("History file for application"
          + applicationId + " is not found"));

  int exitCode = cli.run(new String[] { "container", "-status",
      containerId1.toString() });
  verify(sysOut).println(
      "Application for Container with id '" + containerId1
          + "' doesn't exist in RM or Timeline Server.");
  Assert.assertNotSame("should return non-zero exit code.", 0, exitCode);
  ContainerId containerId2 = ContainerId.newContainerId(attemptId, cntId++);
  when(client.getContainerReport(containerId2)).thenThrow(
      new ApplicationAttemptNotFoundException(
          "History file for application attempt" + attemptId
              + " is not found"));

  exitCode = cli.run(new String[] { "container", "-status",
      containerId2.toString() });
  verify(sysOut).println(
      "Application Attempt for Container with id '" + containerId2
          + "' doesn't exist in RM or Timeline Server.");
  Assert.assertNotSame("should return non-zero exit code.", 0, exitCode);

  ContainerId containerId3 = ContainerId.newContainerId(attemptId, cntId++);
  when(client.getContainerReport(containerId3)).thenThrow(
      new ContainerNotFoundException("History file for container"
          + containerId3 + " is not found"));
  exitCode = cli.run(new String[] { "container", "-status",
      containerId3.toString() });
  verify(sysOut).println(
      "Container with id '" + containerId3
          + "' doesn't exist in RM or Timeline Server.");
  Assert.assertNotSame("should return non-zero exit code.", 0, exitCode);
}
 
Example 12
Source File: TestTaskAttempt.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Test
public void testLaunchFailedWhileKilling() throws Exception {
  ApplicationId appId = ApplicationId.newInstance(1, 2);
  ApplicationAttemptId appAttemptId =
    ApplicationAttemptId.newInstance(appId, 0);
  JobId jobId = MRBuilderUtils.newJobId(appId, 1);
  TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP);
  TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId, 0);
  Path jobFile = mock(Path.class);

  MockEventHandler eventHandler = new MockEventHandler();
  TaskAttemptListener taListener = mock(TaskAttemptListener.class);
  when(taListener.getAddress()).thenReturn(new InetSocketAddress("localhost", 0));

  JobConf jobConf = new JobConf();
  jobConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
  jobConf.setBoolean("fs.file.impl.disable.cache", true);
  jobConf.set(JobConf.MAPRED_MAP_TASK_ENV, "");
  jobConf.set(MRJobConfig.APPLICATION_ATTEMPT_ID, "10");

  TaskSplitMetaInfo splits = mock(TaskSplitMetaInfo.class);
  when(splits.getLocations()).thenReturn(new String[] {"127.0.0.1"});

  TaskAttemptImpl taImpl =
    new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile, 1,
        splits, jobConf, taListener,
        new Token(), new Credentials(),
        new SystemClock(), null);

  NodeId nid = NodeId.newInstance("127.0.0.1", 0);
  ContainerId contId = ContainerId.newContainerId(appAttemptId, 3);
  Container container = mock(Container.class);
  when(container.getId()).thenReturn(contId);
  when(container.getNodeId()).thenReturn(nid);

  taImpl.handle(new TaskAttemptEvent(attemptId,
      TaskAttemptEventType.TA_SCHEDULE));
  taImpl.handle(new TaskAttemptContainerAssignedEvent(attemptId,
      container, mock(Map.class)));
  taImpl.handle(new TaskAttemptEvent(attemptId,
      TaskAttemptEventType.TA_KILL));
  taImpl.handle(new TaskAttemptEvent(attemptId,
      TaskAttemptEventType.TA_CONTAINER_CLEANED));
  taImpl.handle(new TaskAttemptEvent(attemptId,
      TaskAttemptEventType.TA_CONTAINER_LAUNCH_FAILED));
  assertFalse(eventHandler.internalError);
  assertEquals("Task attempt is not assigned on the local node", 
      Locality.NODE_LOCAL, taImpl.getLocality());
}
 
Example 13
Source File: TestAMRestart.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test(timeout = 20000)
public void testPreemptedAMRestartOnRMRestart() throws Exception {
  YarnConfiguration conf = new YarnConfiguration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
    ResourceScheduler.class);
  conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true);
  conf.setBoolean(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED, false);

  conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName());
  // explicitly set max-am-retry count as 1.
  conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1);
  MemoryRMStateStore memStore = new MemoryRMStateStore();
  memStore.init(conf);

  MockRM rm1 = new MockRM(conf, memStore);
  rm1.start();
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", 8000, rm1.getResourceTrackerService());
  nm1.registerNode();
  RMApp app1 = rm1.submitApp(200);
  RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
  MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
  CapacityScheduler scheduler =
      (CapacityScheduler) rm1.getResourceScheduler();
  ContainerId amContainer =
      ContainerId.newContainerId(am1.getApplicationAttemptId(), 1);

  // Forcibly preempt the am container;
  scheduler.killContainer(scheduler.getRMContainer(amContainer));

  am1.waitForState(RMAppAttemptState.FAILED);
  Assert.assertTrue(! attempt1.shouldCountTowardsMaxAttemptRetry());
  rm1.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED);

  // state store has 1 attempt stored.
  ApplicationStateData appState =
      memStore.getState().getApplicationState().get(app1.getApplicationId());
  Assert.assertEquals(1, appState.getAttemptCount());
  // attempt stored has the preempted container exit status.
  Assert.assertEquals(ContainerExitStatus.PREEMPTED,
    appState.getAttempt(am1.getApplicationAttemptId())
      .getAMContainerExitStatus());
  // Restart rm.
  MockRM rm2 = new MockRM(conf, memStore);
  nm1.setResourceTrackerService(rm2.getResourceTrackerService());
  nm1.registerNode();
  rm2.start();

  // Restarted RM should re-launch the am.
  MockAM am2 =
      rm2.waitForNewAMToLaunchAndRegister(app1.getApplicationId(), 2, nm1);
  MockRM.finishAMAndVerifyAppState(app1, rm2, nm1, am2);
  RMAppAttempt attempt2 =
      rm2.getRMContext().getRMApps().get(app1.getApplicationId())
        .getCurrentAppAttempt();
  Assert.assertTrue(attempt2.shouldCountTowardsMaxAttemptRetry());
  Assert.assertEquals(ContainerExitStatus.INVALID,
    appState.getAttempt(am2.getApplicationAttemptId())
      .getAMContainerExitStatus());
  rm1.stop();
  rm2.stop();
}
 
Example 14
Source File: TestDagAwareYarnTaskScheduler.java    From tez with Apache License 2.0 4 votes vote down vote up
@Test(timeout=50000)
public void testIdleContainerAssignmentReuseNewContainers() throws Exception {
  AMRMClientAsyncWrapperForTest mockRMClient = spy(new AMRMClientAsyncWrapperForTest());

  String appHost = "host";
  int appPort = 0;
  String appUrl = "url";

  Configuration conf = new Configuration();
  conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, true);
  conf.setInt(TezConfiguration.TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS, 100);
  conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_RACK_FALLBACK_ENABLED, false);
  conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_NON_LOCAL_FALLBACK_ENABLED, false);
  conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_NEW_CONTAINERS_ENABLED, true);
  conf.setInt(TezConfiguration.TEZ_AM_RM_HEARTBEAT_INTERVAL_MS_MAX, 100);
  conf.setInt(TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MIN_MILLIS, 4000);
  conf.setInt(TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MAX_MILLIS, 5000);
  conf.setInt(TezConfiguration.TEZ_AM_SESSION_MIN_HELD_CONTAINERS, 5);

  DagInfo mockDagInfo = mock(DagInfo.class);
  when(mockDagInfo.getTotalVertices()).thenReturn(10);
  when(mockDagInfo.getVertexDescendants(anyInt())).thenReturn(new BitSet());
  TaskSchedulerContext mockApp = setupMockTaskSchedulerContext(appHost, appPort, appUrl, conf);
  when(mockApp.getCurrentDagInfo()).thenReturn(mockDagInfo);
  when(mockApp.isSession()).thenReturn(true);
  TaskSchedulerContextDrainable drainableAppCallback = createDrainableContext(mockApp);

  MockClock clock = new MockClock(1000);
  NewTaskSchedulerForTest scheduler = new NewTaskSchedulerForTest(drainableAppCallback,
      mockRMClient, clock);

  scheduler.initialize();
  drainableAppCallback.drain();

  scheduler.start();
  drainableAppCallback.drain();
  verify(mockRMClient).start();
  verify(mockRMClient).registerApplicationMaster(appHost, appPort, appUrl);
  RegisterApplicationMasterResponse regResponse = mockRMClient.getRegistrationResponse();
  verify(mockApp).setApplicationRegistrationData(regResponse.getMaximumResourceCapability(),
      regResponse.getApplicationACLs(), regResponse.getClientToAMTokenMasterKey(),
      regResponse.getQueue());

  assertEquals(scheduler.getClusterNodeCount(), mockRMClient.getClusterNodeCount());

  final String rack1 = "/r1";
  final String rack2 = "/r2";
  final String node1Rack1 = "n1r1";
  final String node2Rack1 = "n2r1";
  final String node1Rack2 = "n1r2";
  MockDNSToSwitchMapping.addRackMapping(node1Rack1, rack1);
  MockDNSToSwitchMapping.addRackMapping(node2Rack1, rack1);
  MockDNSToSwitchMapping.addRackMapping(node1Rack2, rack2);

  Priority priorityv0 = Priority.newInstance(1);
  MockTaskInfo taskv0t0 = new MockTaskInfo("taskv0t0", priorityv0, node1Rack1, rack1);

  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(1, 1), 1);
  ContainerId cid1 = ContainerId.newContainerId(attemptId, 1);
  NodeId n2r1 = NodeId.newInstance(node2Rack1, 1);
  Container container1 = Container.newInstance(cid1, n2r1, null, taskv0t0.capability, priorityv0, null);

  // verify idle container is kept for now
  scheduler.onContainersAllocated(Collections.singletonList(container1));
  clock.incrementTime(2000);
  drainableAppCallback.drain();
  verify(mockApp, never()).containerBeingReleased(cid1);
  verify(mockRMClient, never()).releaseAssignedContainer(cid1);

  // verify idle container is released without being assigned to a task because rack-local reuse is
  // disabled
  TaskRequestCaptor taskRequestCaptor = new TaskRequestCaptor(mockRMClient,
      scheduler, drainableAppCallback);
  TaskRequest reqv0t0 = taskRequestCaptor.scheduleTask(taskv0t0);
  clock.incrementTime(10000);
  drainableAppCallback.drain();
  verify(mockApp, never()).taskAllocated(taskv0t0.task, taskv0t0.cookie, container1);
  verify(mockRMClient, never()).removeContainerRequest(reqv0t0);
  verify(mockApp, never()).containerBeingReleased(cid1);
  verify(mockRMClient).releaseAssignedContainer(cid1);

  // cancel the task request
  assertFalse(scheduler.deallocateTask(taskv0t0.task, false, null, null));

  // allocate another container that's node-local
  ContainerId cid2 = ContainerId.newContainerId(attemptId, 2);
  NodeId n1r1 = NodeId.newInstance(node1Rack1, 1);
  Container container2 = Container.newInstance(cid2, n1r1, null, taskv0t0.capability, priorityv0, null);
  scheduler.onContainersAllocated(Collections.singletonList(container2));
  clock.incrementTime(2000);
  drainableAppCallback.drain();
  verify(mockApp, never()).containerBeingReleased(cid2);
  verify(mockRMClient, never()).releaseAssignedContainer(cid2);

  // reschedule the task, verify it's now scheduled without a container request
  // since node-local idle container is available
  reqv0t0 = taskRequestCaptor.scheduleTask(taskv0t0, false);
  verify(mockApp).taskAllocated(taskv0t0.task, taskv0t0.cookie, container2);
  verify(mockRMClient).removeContainerRequest(reqv0t0);
}
 
Example 15
Source File: TestApplicationHistoryManagerOnTimelineStore.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test
public void testGetContainerReport() throws Exception {
  final ContainerId containerId =
      ContainerId.newContainerId(ApplicationAttemptId.newInstance(
          ApplicationId.newInstance(0, 1), 1), 1);
  ContainerReport container;
  if (callerUGI == null) {
    container = historyManager.getContainer(containerId);
  } else {
    try {
      container =
          callerUGI.doAs(new PrivilegedExceptionAction<ContainerReport> () {
        @Override
        public ContainerReport run() throws Exception {
          return historyManager.getContainer(containerId);
        }
      });
      if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) {
        // The exception is expected
        Assert.fail();
      }
    } catch (AuthorizationException e) {
      if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) {
        // The exception is expected
        return;
      }
      throw e;
    }
  }
  Assert.assertNotNull(container);
  Assert.assertEquals(Integer.MAX_VALUE + 1L, container.getCreationTime());
  Assert.assertEquals(Integer.MAX_VALUE + 2L, container.getFinishTime());
  Assert.assertEquals(Resource.newInstance(-1, -1),
      container.getAllocatedResource());
  Assert.assertEquals(NodeId.newInstance("test host", 100),
      container.getAssignedNode());
  Assert.assertEquals(Priority.UNDEFINED, container.getPriority());
  Assert
      .assertEquals("test diagnostics info", container.getDiagnosticsInfo());
  Assert.assertEquals(ContainerState.COMPLETE, container.getContainerState());
  Assert.assertEquals(-1, container.getContainerExitStatus());
  Assert.assertEquals("http://0.0.0.0:8188/applicationhistory/logs/" +
      "test host:100/container_0_0001_01_000001/"
      + "container_0_0001_01_000001/user1", container.getLogUrl());
}
 
Example 16
Source File: BuilderUtils.java    From big-c with Apache License 2.0 4 votes vote down vote up
public static ContainerId newContainerId(RecordFactory recordFactory,
    ApplicationId appId, ApplicationAttemptId appAttemptId,
    int containerId) {
  return ContainerId.newContainerId(appAttemptId, containerId);
}
 
Example 17
Source File: TestAggregatedLogFormat.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Test(timeout=10000)
public void testContainerLogsFileAccess() throws IOException {
  // This test will run only if NativeIO is enabled as SecureIOUtils 
  // require it to be enabled.
  Assume.assumeTrue(NativeIO.isAvailable());
  Configuration conf = new Configuration();
  conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
      "kerberos");
  UserGroupInformation.setConfiguration(conf);
  File workDir = new File(testWorkDir, "testContainerLogsFileAccess1");
  Path remoteAppLogFile =
      new Path(workDir.getAbsolutePath(), "aggregatedLogFile");
  Path srcFileRoot = new Path(workDir.getAbsolutePath(), "srcFiles");

  String data = "Log File content for container : ";
  // Creating files for container1. Log aggregator will try to read log files
  // with illegal user.
  ApplicationId applicationId = ApplicationId.newInstance(1, 1);
  ApplicationAttemptId applicationAttemptId =
      ApplicationAttemptId.newInstance(applicationId, 1);
  ContainerId testContainerId1 =
      ContainerId.newContainerId(applicationAttemptId, 1);
  Path appDir =
      new Path(srcFileRoot, testContainerId1.getApplicationAttemptId()
          .getApplicationId().toString());
  Path srcFilePath1 = new Path(appDir, testContainerId1.toString());
  String stdout = "stdout";
  String stderr = "stderr";
  writeSrcFile(srcFilePath1, stdout, data + testContainerId1.toString()
      + stdout);
  writeSrcFile(srcFilePath1, stderr, data + testContainerId1.toString()
      + stderr);

  UserGroupInformation ugi =
      UserGroupInformation.getCurrentUser();
  LogWriter logWriter = new LogWriter(conf, remoteAppLogFile, ugi);

  LogKey logKey = new LogKey(testContainerId1);
  String randomUser = "randomUser";
  LogValue logValue =
      spy(new LogValue(Collections.singletonList(srcFileRoot.toString()),
          testContainerId1, randomUser));
  
  // It is trying simulate a situation where first log file is owned by
  // different user (probably symlink) and second one by the user itself.
  // The first file should not be aggregated. Because this log file has the invalid
  // user name.
  when(logValue.getUser()).thenReturn(randomUser).thenReturn(
      ugi.getShortUserName());
  logWriter.append(logKey, logValue);

  logWriter.close();
  
  BufferedReader in =
      new BufferedReader(new FileReader(new File(remoteAppLogFile
          .toUri().getRawPath())));
  String line;
  StringBuffer sb = new StringBuffer("");
  while ((line = in.readLine()) != null) {
    LOG.info(line);
    sb.append(line);
  }
  line = sb.toString();

  String expectedOwner = ugi.getShortUserName();
  if (Path.WINDOWS) {
    final String adminsGroupString = "Administrators";
    if (Arrays.asList(ugi.getGroupNames()).contains(adminsGroupString)) {
      expectedOwner = adminsGroupString;
    }
  }

  // This file: stderr should not be aggregated.
  // And we will not aggregate the log message.
  String stdoutFile1 =
      StringUtils.join(
          File.separator,
          Arrays.asList(new String[] {
              workDir.getAbsolutePath(), "srcFiles",
              testContainerId1.getApplicationAttemptId().getApplicationId()
                  .toString(), testContainerId1.toString(), stderr }));

  // The file: stdout is expected to be aggregated.
  String stdoutFile2 =
      StringUtils.join(
          File.separator,
          Arrays.asList(new String[] {
              workDir.getAbsolutePath(), "srcFiles",
              testContainerId1.getApplicationAttemptId().getApplicationId()
                  .toString(), testContainerId1.toString(), stdout }));
  String message2 =
      "Owner '" + expectedOwner + "' for path "
          + stdoutFile2 + " did not match expected owner '"
          + ugi.getShortUserName() + "'";
  
  Assert.assertFalse(line.contains(message2));
  Assert.assertFalse(line.contains(data + testContainerId1.toString()
      + stderr));
  Assert.assertTrue(line.contains(data + testContainerId1.toString()
      + stdout));
}
 
Example 18
Source File: TestFileSystemApplicationHistoryStore.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("deprecation")
private void testReadHistoryData(
    int num, boolean missingContainer, boolean missingApplicationAttempt)
        throws IOException {
  // read application history data
  Assert.assertEquals(num, store.getAllApplications().size());
  for (int i = 1; i <= num; ++i) {
    ApplicationId appId = ApplicationId.newInstance(0, i);
    ApplicationHistoryData appData = store.getApplication(appId);
    Assert.assertNotNull(appData);
    Assert.assertEquals(appId.toString(), appData.getApplicationName());
    Assert.assertEquals(appId.toString(), appData.getDiagnosticsInfo());

    // read application attempt history data
    Assert.assertEquals(num, store.getApplicationAttempts(appId).size());
    for (int j = 1; j <= num; ++j) {
      ApplicationAttemptId appAttemptId =
          ApplicationAttemptId.newInstance(appId, j);
      ApplicationAttemptHistoryData attemptData =
          store.getApplicationAttempt(appAttemptId);
      Assert.assertNotNull(attemptData);
      Assert.assertEquals(appAttemptId.toString(), attemptData.getHost());
      
      if (missingApplicationAttempt && j == num) {
        Assert.assertNull(attemptData.getDiagnosticsInfo());
        continue;
      } else {
        Assert.assertEquals(appAttemptId.toString(),
            attemptData.getDiagnosticsInfo());
      }

      // read container history data
      Assert.assertEquals(num, store.getContainers(appAttemptId).size());
      for (int k = 1; k <= num; ++k) {
        ContainerId containerId = ContainerId.newContainerId(appAttemptId, k);
        ContainerHistoryData containerData = store.getContainer(containerId);
        Assert.assertNotNull(containerData);
        Assert.assertEquals(Priority.newInstance(containerId.getId()),
          containerData.getPriority());
        if (missingContainer && k == num) {
          Assert.assertNull(containerData.getDiagnosticsInfo());
        } else {
          Assert.assertEquals(containerId.toString(),
              containerData.getDiagnosticsInfo());
        }
      }
      ContainerHistoryData masterContainer =
          store.getAMContainer(appAttemptId);
      Assert.assertNotNull(masterContainer);
      Assert.assertEquals(ContainerId.newContainerId(appAttemptId, 1),
        masterContainer.getContainerId());
    }
  }
}
 
Example 19
Source File: TestAMRestart.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Test(timeout = 30000)
public void testNMTokensRebindOnAMRestart() throws Exception {
  YarnConfiguration conf = new YarnConfiguration();
  conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 3);

  MockRM rm1 = new MockRM(conf);
  rm1.start();
  RMApp app1 =
      rm1.submitApp(200, "myname", "myuser",
        new HashMap<ApplicationAccessType, String>(), false, "default", -1,
        null, "MAPREDUCE", false, true);
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", 8000, rm1.getResourceTrackerService());
  nm1.registerNode();
  MockNM nm2 =
      new MockNM("127.1.1.1:4321", 8000, rm1.getResourceTrackerService());
  nm2.registerNode();
  MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);

  List<Container> containers = new ArrayList<Container>();
  // nmTokens keeps track of all the nmTokens issued in the allocate call.
  List<NMToken> expectedNMTokens = new ArrayList<NMToken>();

  // am1 allocate 2 container on nm1.
  // first container
  while (true) {
    AllocateResponse response =
        am1.allocate("127.0.0.1", 2000, 2,
          new ArrayList<ContainerId>());
    nm1.nodeHeartbeat(true);
    containers.addAll(response.getAllocatedContainers());
    expectedNMTokens.addAll(response.getNMTokens());
    if (containers.size() == 2) {
      break;
    }
    Thread.sleep(200);
    System.out.println("Waiting for container to be allocated.");
  }
  // launch the container-2
  nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 2, ContainerState.RUNNING);
  ContainerId containerId2 =
      ContainerId.newContainerId(am1.getApplicationAttemptId(), 2);
  rm1.waitForState(nm1, containerId2, RMContainerState.RUNNING);
  // launch the container-3
  nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 3, ContainerState.RUNNING);
  ContainerId containerId3 =
      ContainerId.newContainerId(am1.getApplicationAttemptId(), 3);
  rm1.waitForState(nm1, containerId3, RMContainerState.RUNNING);
  
  // fail am1
  nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
  am1.waitForState(RMAppAttemptState.FAILED);
  rm1.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED);

  // restart the am
  MockAM am2 = MockRM.launchAM(app1, rm1, nm1);
  RegisterApplicationMasterResponse registerResponse =
      am2.registerAppAttempt();
  rm1.waitForState(app1.getApplicationId(), RMAppState.RUNNING);

  // check am2 get the nm token from am1.
  Assert.assertEquals(expectedNMTokens,
    registerResponse.getNMTokensFromPreviousAttempts());

  // am2 allocate 1 container on nm2
  containers = new ArrayList<Container>();
  while (true) {
    AllocateResponse allocateResponse =
        am2.allocate("127.1.1.1", 4000, 1,
          new ArrayList<ContainerId>());
    nm2.nodeHeartbeat(true);
    containers.addAll(allocateResponse.getAllocatedContainers());
    expectedNMTokens.addAll(allocateResponse.getNMTokens());
    if (containers.size() == 1) {
      break;
    }
    Thread.sleep(200);
    System.out.println("Waiting for container to be allocated.");
  }
  nm1.nodeHeartbeat(am2.getApplicationAttemptId(), 2, ContainerState.RUNNING);
  ContainerId am2ContainerId2 =
      ContainerId.newContainerId(am2.getApplicationAttemptId(), 2);
  rm1.waitForState(nm1, am2ContainerId2, RMContainerState.RUNNING);

  // fail am2.
  nm1.nodeHeartbeat(am2.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
  am2.waitForState(RMAppAttemptState.FAILED);
  rm1.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED);

  // restart am
  MockAM am3 = MockRM.launchAM(app1, rm1, nm1);
  registerResponse = am3.registerAppAttempt();
  rm1.waitForState(app1.getApplicationId(), RMAppState.RUNNING);

  // check am3 get the NM token from both am1 and am2;
  List<NMToken> transferredTokens = registerResponse.getNMTokensFromPreviousAttempts();
  Assert.assertEquals(2, transferredTokens.size());
  Assert.assertTrue(transferredTokens.containsAll(expectedNMTokens));
  rm1.stop();
}
 
Example 20
Source File: TestRPC.java    From hadoop with Apache License 2.0 4 votes vote down vote up
private void test(String rpcClass) throws Exception {
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.IPC_RPC_IMPL, rpcClass);
  YarnRPC rpc = YarnRPC.create(conf);
  String bindAddr = "localhost:0";
  InetSocketAddress addr = NetUtils.createSocketAddr(bindAddr);
  Server server = rpc.getServer(ContainerManagementProtocol.class, 
          new DummyContainerManager(), addr, conf, null, 1);
  server.start();
  RPC.setProtocolEngine(conf, ContainerManagementProtocolPB.class, ProtobufRpcEngine.class);
  ContainerManagementProtocol proxy = (ContainerManagementProtocol) 
      rpc.getProxy(ContainerManagementProtocol.class, 
          NetUtils.getConnectAddress(server), conf);
  ContainerLaunchContext containerLaunchContext = 
      recordFactory.newRecordInstance(ContainerLaunchContext.class);

  ApplicationId applicationId = ApplicationId.newInstance(0, 0);
  ApplicationAttemptId applicationAttemptId =
      ApplicationAttemptId.newInstance(applicationId, 0);
  ContainerId containerId =
      ContainerId.newContainerId(applicationAttemptId, 100);
  NodeId nodeId = NodeId.newInstance("localhost", 1234);
  Resource resource = Resource.newInstance(1234, 2);
  ContainerTokenIdentifier containerTokenIdentifier =
      new ContainerTokenIdentifier(containerId, "localhost", "user",
        resource, System.currentTimeMillis() + 10000, 42, 42,
        Priority.newInstance(0), 0);
  Token containerToken = newContainerToken(nodeId, "password".getBytes(),
        containerTokenIdentifier);

  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(containerLaunchContext,
        containerToken);
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  proxy.startContainers(allRequests);

  List<ContainerId> containerIds = new ArrayList<ContainerId>();
  containerIds.add(containerId);
  GetContainerStatusesRequest gcsRequest =
      GetContainerStatusesRequest.newInstance(containerIds);
  GetContainerStatusesResponse response =
      proxy.getContainerStatuses(gcsRequest);
  List<ContainerStatus> statuses = response.getContainerStatuses();

  //test remote exception
  boolean exception = false;
  try {
    StopContainersRequest stopRequest =
        recordFactory.newRecordInstance(StopContainersRequest.class);
    stopRequest.setContainerIds(containerIds);
    proxy.stopContainers(stopRequest);
    } catch (YarnException e) {
    exception = true;
    Assert.assertTrue(e.getMessage().contains(EXCEPTION_MSG));
    Assert.assertTrue(e.getMessage().contains(EXCEPTION_CAUSE));
    System.out.println("Test Exception is " + e.getMessage());
  } catch (Exception ex) {
    ex.printStackTrace();
  }
  Assert.assertTrue(exception);
  
  server.stop();
  Assert.assertNotNull(statuses.get(0));
  Assert.assertEquals(ContainerState.RUNNING, statuses.get(0).getState());
}