Java Code Examples for org.apache.hadoop.yarn.api.records.ApplicationId#newInstance()

The following examples show how to use org.apache.hadoop.yarn.api.records.ApplicationId#newInstance() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: big-c   File: TestYarnCLI.java    License: Apache License 2.0 6 votes vote down vote up
@Test (timeout = 10000)
public void testAppsHelpCommand() throws Exception {
  ApplicationCLI cli = createAndGetAppCLI();
  ApplicationCLI spyCli = spy(cli);
  int result = spyCli.run(new String[] { "application", "-help" });
  Assert.assertTrue(result == 0);
  verify(spyCli).printUsage(any(String.class), any(Options.class));
  Assert.assertEquals(createApplicationCLIHelpMessage(),
      sysOutStream.toString());

  sysOutStream.reset();
  ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
  result = cli.run(
      new String[] {"application", "-kill", applicationId.toString(), "args" });
  verify(spyCli).printUsage(any(String.class), any(Options.class));
  Assert.assertEquals(createApplicationCLIHelpMessage(),
      sysOutStream.toString());

  sysOutStream.reset();
  NodeId nodeId = NodeId.newInstance("host0", 0);
  result = cli.run(
      new String[] { "application", "-status", nodeId.toString(), "args" });
  verify(spyCli).printUsage(any(String.class), any(Options.class));
  Assert.assertEquals(createApplicationCLIHelpMessage(),
      sysOutStream.toString());
}
 
Example 2
Source Project: tez   File: TestTaskAttempt.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout = 5000)
public void testEventSerializingHash() throws Exception {
  ApplicationId appId = ApplicationId.newInstance(1, 2);
  TezDAGID dagID = TezDAGID.getInstance(appId, 1);
  TezVertexID vertexID = TezVertexID.getInstance(dagID, 1);
  TezTaskID taskID1 = TezTaskID.getInstance(vertexID, 1);
  TezTaskID taskID2 = TezTaskID.getInstance(vertexID, 2);
  TezTaskAttemptID taID11 = TezTaskAttemptID.getInstance(taskID1, 0);
  TezTaskAttemptID taID12 = TezTaskAttemptID.getInstance(taskID1, 1);
  TezTaskAttemptID taID21 = TezTaskAttemptID.getInstance(taskID2, 1);
  
  TaskAttemptEvent taEventFail11 = new TaskAttemptEvent(taID11, TaskAttemptEventType.TA_FAILED);
  TaskAttemptEvent taEventKill11 = new TaskAttemptEvent(taID11, TaskAttemptEventType.TA_KILL_REQUEST);
  TaskAttemptEvent taEventKill12 = new TaskAttemptEvent(taID12, TaskAttemptEventType.TA_KILL_REQUEST);
  TaskAttemptEvent taEventKill21 = new TaskAttemptEvent(taID21, TaskAttemptEventType.TA_KILL_REQUEST);
  TaskEvent tEventKill1 = new TaskEvent(taskID1, TaskEventType.T_ATTEMPT_KILLED);
  TaskEvent tEventFail1 = new TaskEvent(taskID1, TaskEventType.T_ATTEMPT_FAILED);
  TaskEvent tEventFail2 = new TaskEvent(taskID2, TaskEventType.T_ATTEMPT_FAILED);
  
  // all of them should have the same value
  assertEquals(taEventFail11.getSerializingHash(), taEventKill11.getSerializingHash());
  assertEquals(taEventKill11.getSerializingHash(), taEventKill12.getSerializingHash());
  assertEquals(tEventFail1.getSerializingHash(), tEventKill1.getSerializingHash());
  assertEquals(taEventFail11.getSerializingHash(), tEventKill1.getSerializingHash());
  assertEquals(taEventKill21.getSerializingHash(), tEventFail2.getSerializingHash());
  // events from different tasks may not have the same value
  assertFalse(tEventFail1.getSerializingHash() == tEventFail2.getSerializingHash());
}
 
Example 3
Source Project: tez   File: TestTaskExecution2.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout = 5000)
public void testHeartbeatShouldDie() throws IOException, InterruptedException, TezException,
    ExecutionException {

  ListeningExecutorService executor = null;
  try {
    ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
    executor = MoreExecutors.listeningDecorator(rawExecutor);
    ApplicationId appId = ApplicationId.newInstance(10000, 1);
    TaskExecutionTestHelpers.TezTaskUmbilicalForTest
        umbilical = new TaskExecutionTestHelpers.TezTaskUmbilicalForTest();
    TaskReporter taskReporter = createTaskReporter(appId, umbilical);

    TezTaskRunner2 taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor,
        TestProcessor.CONF_EMPTY);
    // Setup the executor
    Future<TaskRunner2Result> taskRunnerFuture =
        taskExecutor.submit(new TaskRunnerCallable2ForTest(taskRunner));
    // Signal the processor to go through
    TestProcessor.awaitStart();
    umbilical.signalSendShouldDie();
    umbilical.awaitRegisteredEvent();
    // Not signaling an actual start to verify task interruption

    TaskRunner2Result result = taskRunnerFuture.get();
    verifyTaskRunnerResult(result, EndReason.CONTAINER_STOP_REQUESTED, null, true, null);


    TestProcessor.awaitCompletion();
    assertTrue(TestProcessor.wasInterrupted());
    assertNull(taskReporter.currentCallable);
    // TODO Is this statement correct ?
    // No completion events since shouldDie was requested by the AM, which should have killed the
    // task.
    umbilical.verifyNoCompletionEvents();
    assertTrue(TestProcessor.wasAborted());
  } finally {
    executor.shutdownNow();
  }
}
 
Example 4
Source Project: big-c   File: TestAHSWebServices.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testInvalidApp() {
  ApplicationId appId = ApplicationId.newInstance(0, MAX_APPS + 1);
  WebResource r = resource();
  ClientResponse response =
      r.path("ws").path("v1").path("applicationhistory").path("apps")
        .path(appId.toString())
        .queryParam("user.name", USERS[round])
        .accept(MediaType.APPLICATION_JSON)
        .get(ClientResponse.class);
  assertEquals("404 not found expected", Status.NOT_FOUND,
      response.getClientResponseStatus());
}
 
Example 5
Source Project: tez   File: TestRecoveryParser.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout=5000)
public void testRecoverableSummary_VertexInCommitting() throws IOException {
  ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
  TezDAGID dagID = TezDAGID.getInstance(appId, 1);
  AppContext appContext = mock(AppContext.class);
  when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath+"/1"));
  when(appContext.getClock()).thenReturn(new SystemClock());
  when(mockDAGImpl.getID()).thenReturn(dagID);

  RecoveryService rService = new RecoveryService(appContext);
  Configuration conf = new Configuration();
  conf.setBoolean(RecoveryService.TEZ_TEST_RECOVERY_DRAIN_EVENTS_WHEN_STOPPED, true);
  rService.init(conf);
  rService.start();

  DAGPlan dagPlan = TestDAGImpl.createTestDAGPlan();
  // write a DAGSubmittedEvent first to initialize summaryStream
  rService.handle(new DAGHistoryEvent(dagID,
      new DAGSubmittedEvent(dagID, 1L, dagPlan, ApplicationAttemptId.newInstance(appId, 1),
          null, "user", new Configuration(), null, null)));
  // It should be fine to skip other events, just for testing.
  rService.handle(new DAGHistoryEvent(dagID,
      new VertexCommitStartedEvent(TezVertexID.getInstance(dagID, 0), 0L)));
  rService.stop();

  DAGRecoveryData dagData = parser.parseRecoveryData();
  assertEquals(dagID, dagData.recoveredDagID);
  assertTrue(dagData.nonRecoverable);
  assertTrue(dagData.reason.contains("Vertex Commit was in progress"));
}
 
Example 6
Source Project: tez   File: TestTaskExecution2.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout = 5000)
public void testIgnoreErrorsDuringFailure() throws IOException, InterruptedException, TezException,
    ExecutionException {

  ListeningExecutorService executor = null;
  try {
    ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
    executor = MoreExecutors.listeningDecorator(rawExecutor);
    ApplicationId appId = ApplicationId.newInstance(10000, 1);
    TaskExecutionTestHelpers.TezTaskUmbilicalForTest
        umbilical = new TaskExecutionTestHelpers.TezTaskUmbilicalForTest();

    TaskReporter taskReporter = new TaskReporter(umbilical, 100, 1000, 100, new AtomicLong(0),
      createContainerId(appId).toString()) {
      @Override
      protected boolean isShuttingDown() {
        return true;
      }
    };

    TezTaskRunner2 taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor,
        TestProcessor.CONF_THROW_IO_EXCEPTION);
    // Setup the executor

    taskExecutor.submit(new TaskRunnerCallable2ForTest(taskRunner));

    // Signal the processor to go through
    TestProcessor.awaitStart();
    TestProcessor.signal();

    umbilical.verifyNoCompletionEvents();
  } finally {
    executor.shutdownNow();
  }
}
 
Example 7
Source Project: hadoop   File: TestYarnClient.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout = 10000)
public void testGetContainerReport() throws YarnException, IOException {
  Configuration conf = new Configuration();
  conf.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED,
      true);
  final YarnClient client = new MockYarnClient();
  client.init(conf);
  client.start();

  List<ApplicationReport> expectedReports = ((MockYarnClient) client)
      .getReports();

  ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
  ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
      applicationId, 1);
  ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1);
  ContainerReport report = client.getContainerReport(containerId);
  Assert.assertNotNull(report);
  Assert.assertEquals(report.getContainerId().toString(),
      (ContainerId.newContainerId(expectedReports.get(0)
          .getCurrentApplicationAttemptId(), 1)).toString());
  containerId = ContainerId.newContainerId(appAttemptId, 3);
  report = client.getContainerReport(containerId);
  Assert.assertNotNull(report);
  Assert.assertEquals(report.getContainerId().toString(),
      (ContainerId.newContainerId(expectedReports.get(0)
          .getCurrentApplicationAttemptId(), 3)).toString());
  client.stop();
}
 
Example 8
Source Project: hadoop   File: TestYarnCLI.java    License: Apache License 2.0 5 votes vote down vote up
@Test (timeout = 10000)
public void testContainersHelpCommand() throws Exception {
  ApplicationCLI cli = createAndGetAppCLI();
  ApplicationCLI spyCli = spy(cli);
  int result = spyCli.run(new String[] { "container", "-help" });
  Assert.assertTrue(result == 0);
  verify(spyCli).printUsage(any(String.class), any(Options.class));
  Assert.assertEquals(createContainerCLIHelpMessage(),
      sysOutStream.toString());

  sysOutStream.reset();
  ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
  ApplicationAttemptId appAttemptId =
      ApplicationAttemptId.newInstance(applicationId, 6);
  result = cli.run(
      new String[] {"container", "-list", appAttemptId.toString(), "args" });
  verify(spyCli).printUsage(any(String.class), any(Options.class));
  Assert.assertEquals(createContainerCLIHelpMessage(),
      sysOutStream.toString());

  sysOutStream.reset();
  ContainerId containerId = ContainerId.newContainerId(appAttemptId, 7);
  result = cli.run(
      new String[] { "container", "-status", containerId.toString(), "args" });
  verify(spyCli).printUsage(any(String.class), any(Options.class));
  Assert.assertEquals(createContainerCLIHelpMessage(),
      sysOutStream.toString());
}
 
Example 9
Source Project: big-c   File: TestYarnCLI.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testGetContainerReport() throws Exception {
  ApplicationCLI cli = createAndGetAppCLI();
  ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
      applicationId, 1);
  ContainerId containerId = ContainerId.newContainerId(attemptId, 1);
  ContainerReport container = ContainerReport.newInstance(containerId, null,
      NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234, 5678,
      "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE,
      "http://" + NodeId.newInstance("host", 2345).toString());
  when(client.getContainerReport(any(ContainerId.class))).thenReturn(
      container);
  int result = cli.run(new String[] { "container", "-status",
      containerId.toString() });
  assertEquals(0, result);
  verify(client).getContainerReport(containerId);
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  PrintWriter pw = new PrintWriter(baos);
  pw.println("Container Report : ");
  pw.println("\tContainer-Id : container_1234_0005_01_000001");
  pw.println("\tStart-Time : 1234");
  pw.println("\tFinish-Time : 5678");
  pw.println("\tState : COMPLETE");
  pw.println("\tLOG-URL : logURL");
  pw.println("\tHost : host:1234");
  pw.println("\tNodeHttpAddress : http://host:2345");
  pw.println("\tDiagnostics : diagnosticInfo");
  pw.close();
  String appReportStr = baos.toString("UTF-8");
  Assert.assertEquals(appReportStr, sysOutStream.toString());
  verify(sysOut, times(1)).println(isA(String.class));
}
 
Example 10
private ContainerId mockContainerId(int cId) {
  ApplicationId appId = ApplicationId.newInstance(1000, 1);
  ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
  ContainerId containerId = ContainerId.newInstance(appAttemptId, cId);
  return containerId;
}
 
Example 11
Source Project: hadoop   File: ConverterUtils.java    License: Apache License 2.0 4 votes vote down vote up
private static ApplicationId toApplicationId(RecordFactory recordFactory,
    Iterator<String> it) {
  ApplicationId appId = ApplicationId.newInstance(Long.parseLong(it.next()),
      Integer.parseInt(it.next()));
  return appId;
}
 
Example 12
Source Project: tez   File: TestRecovery.java    License: Apache License 2.0 4 votes vote down vote up
@Test(timeout = 1800000)
public void testTwoRoundsRecoverying() throws Exception {
  ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(),
          1);
  TezDAGID dagId = TezDAGID.getInstance(appId, 1);
  TezVertexID vertexId0 = TezVertexID.getInstance(dagId, 0);
  TezVertexID vertexId1 = TezVertexID.getInstance(dagId, 1);
  TezVertexID vertexId2 = TezVertexID.getInstance(dagId, 2);
  ContainerId containerId = ContainerId.newInstance(
          ApplicationAttemptId.newInstance(appId, 1), 1);
  NodeId nodeId = NodeId.newInstance("localhost", 10);
  List<TezEvent> initGeneratedEvents = Lists.newArrayList(
          new TezEvent(InputDataInformationEvent.createWithObjectPayload(0, new Object()), null));


  List<SimpleShutdownCondition> shutdownConditions = Lists.newArrayList(

          new SimpleShutdownCondition(TIMING.POST, new DAGInitializedEvent(
                  dagId, 0L, "username", "dagName", null)),
          new SimpleShutdownCondition(TIMING.POST, new DAGStartedEvent(dagId,
                  0L, "username", "dagName")),
          new SimpleShutdownCondition(TIMING.POST,
                  new VertexInitializedEvent(vertexId0, "Tokenizer", 0L, 0L, 0,
                          "", null, initGeneratedEvents, null)),
          new SimpleShutdownCondition(TIMING.POST, new VertexStartedEvent(
                  vertexId0, 0L, 0L)),
          new SimpleShutdownCondition(TIMING.POST,
                  new VertexConfigurationDoneEvent(vertexId0, 0L, 2, null, null,
                          null, true)),
          new SimpleShutdownCondition(TIMING.POST, new TaskStartedEvent(
                  TezTaskID.getInstance(vertexId0, 0), "vertexName", 0L, 0L)),
          new SimpleShutdownCondition(TIMING.POST,
                  new TaskAttemptStartedEvent(TezTaskAttemptID.getInstance(
                          TezTaskID.getInstance(vertexId0, 0), 0), "vertexName", 0L,
                          containerId, nodeId, "", "", "")),
          new SimpleShutdownCondition(TIMING.POST, new TaskFinishedEvent(
                  TezTaskID.getInstance(vertexId0, 0), "vertexName", 0L, 0L,
                  null, TaskState.SUCCEEDED, "", new TezCounters(), 0)),
          new SimpleShutdownCondition(TIMING.POST, new VertexFinishedEvent(
                  vertexId0, "vertexName", 1, 0L, 0L, 0L, 0L, 0L,
                  VertexState.SUCCEEDED, "", new TezCounters(),
                  new VertexStats(), new HashMap<String, Integer>(), null)),
          new SimpleShutdownCondition(TIMING.POST, new VertexFinishedEvent(
                  vertexId1, "vertexName", 1, 0L, 0L, 0L, 0L, 0L,
                  VertexState.SUCCEEDED, "", new TezCounters(),
                  new VertexStats(), new HashMap<String, Integer>(), null)),
          new SimpleShutdownCondition(TIMING.POST, new VertexFinishedEvent(
                  vertexId2, "vertexName", 1, 0L, 0L, 0L, 0L, 0L,
                  VertexState.SUCCEEDED, "", new TezCounters(),
                  new VertexStats(), new HashMap<String, Integer>(), null)),
          new SimpleShutdownCondition(TIMING.POST, new DAGFinishedEvent(
                  dagId, 0L, 0L, DAGState.SUCCEEDED, "", new TezCounters(),
                  "username", "dagName", new HashMap<String, Integer>(),
                  ApplicationAttemptId.newInstance(appId, 1), null))

  );

  Random rand = new Random();
  for (int i = 0; i < shutdownConditions.size() - 1; i++) {
    // randomly choose half of the test scenario to avoid
    // timeout.
    if (rand.nextDouble()<0.5) {
      int nextSimpleConditionIndex = i + 1 + rand.nextInt(shutdownConditions.size() - i - 1);
      if (nextSimpleConditionIndex == shutdownConditions.size() - 1) {
        testOrderedWordCountMultipleRoundRecoverying(
                new RecoveryServiceWithEventHandlingHook.MultipleRoundShutdownCondition(
                        Lists.newArrayList(shutdownConditions.get(i), shutdownConditions.get(nextSimpleConditionIndex)))
                , true,
                shutdownConditions.get(i).getHistoryEvent().getEventType() == HistoryEventType.VERTEX_STARTED);
      }
    }
  }
}
 
Example 13
Source Project: big-c   File: TestTaskAttempt.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testContainerCleanedWhileCommitting() throws Exception {
  ApplicationId appId = ApplicationId.newInstance(1, 2);
  ApplicationAttemptId appAttemptId =
    ApplicationAttemptId.newInstance(appId, 0);
  JobId jobId = MRBuilderUtils.newJobId(appId, 1);
  TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP);
  TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId, 0);
  Path jobFile = mock(Path.class);

  MockEventHandler eventHandler = new MockEventHandler();
  TaskAttemptListener taListener = mock(TaskAttemptListener.class);
  when(taListener.getAddress()).thenReturn(new InetSocketAddress("localhost", 0));

  JobConf jobConf = new JobConf();
  jobConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
  jobConf.setBoolean("fs.file.impl.disable.cache", true);
  jobConf.set(JobConf.MAPRED_MAP_TASK_ENV, "");
  jobConf.set(MRJobConfig.APPLICATION_ATTEMPT_ID, "10");

  TaskSplitMetaInfo splits = mock(TaskSplitMetaInfo.class);
  when(splits.getLocations()).thenReturn(new String[] {});

  AppContext appCtx = mock(AppContext.class);
  ClusterInfo clusterInfo = mock(ClusterInfo.class);
  Resource resource = mock(Resource.class);
  when(appCtx.getClusterInfo()).thenReturn(clusterInfo);
  when(resource.getMemory()).thenReturn(1024);

  TaskAttemptImpl taImpl =
    new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile, 1,
        splits, jobConf, taListener,
        new Token(), new Credentials(),
        new SystemClock(), appCtx);

  NodeId nid = NodeId.newInstance("127.0.0.1", 0);
  ContainerId contId = ContainerId.newContainerId(appAttemptId, 3);
  Container container = mock(Container.class);
  when(container.getId()).thenReturn(contId);
  when(container.getNodeId()).thenReturn(nid);
  when(container.getNodeHttpAddress()).thenReturn("localhost:0");

  taImpl.handle(new TaskAttemptEvent(attemptId,
      TaskAttemptEventType.TA_SCHEDULE));
  taImpl.handle(new TaskAttemptContainerAssignedEvent(attemptId,
      container, mock(Map.class)));
  taImpl.handle(new TaskAttemptContainerLaunchedEvent(attemptId, 0));
  taImpl.handle(new TaskAttemptEvent(attemptId,
      TaskAttemptEventType.TA_COMMIT_PENDING));

  assertEquals("Task attempt is not in commit pending state", taImpl.getState(),
      TaskAttemptState.COMMIT_PENDING);
  taImpl.handle(new TaskAttemptEvent(attemptId,
      TaskAttemptEventType.TA_CONTAINER_CLEANED));
  assertFalse("InternalError occurred trying to handle TA_CONTAINER_CLEANED",
      eventHandler.internalError);
  assertEquals("Task attempt is assigned locally", Locality.OFF_SWITCH,
      taImpl.getLocality());
}
 
Example 14
Source Project: hadoop   File: TestTaskAttempt.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testAppDiognosticEventOnNewTask() throws Exception {
  ApplicationId appId = ApplicationId.newInstance(1, 2);
  ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
      appId, 0);
  JobId jobId = MRBuilderUtils.newJobId(appId, 1);
  TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP);
  TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId, 0);
  Path jobFile = mock(Path.class);

  MockEventHandler eventHandler = new MockEventHandler();
  TaskAttemptListener taListener = mock(TaskAttemptListener.class);
  when(taListener.getAddress()).thenReturn(
      new InetSocketAddress("localhost", 0));

  JobConf jobConf = new JobConf();
  jobConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
  jobConf.setBoolean("fs.file.impl.disable.cache", true);
  jobConf.set(JobConf.MAPRED_MAP_TASK_ENV, "");
  jobConf.set(MRJobConfig.APPLICATION_ATTEMPT_ID, "10");

  TaskSplitMetaInfo splits = mock(TaskSplitMetaInfo.class);
  when(splits.getLocations()).thenReturn(new String[] { "127.0.0.1" });

  AppContext appCtx = mock(AppContext.class);
  ClusterInfo clusterInfo = mock(ClusterInfo.class);
  Resource resource = mock(Resource.class);
  when(appCtx.getClusterInfo()).thenReturn(clusterInfo);
  when(resource.getMemory()).thenReturn(1024);

  TaskAttemptImpl taImpl = new MapTaskAttemptImpl(taskId, 1, eventHandler,
      jobFile, 1, splits, jobConf, taListener,
      new Token(), new Credentials(), new SystemClock(), appCtx);

  NodeId nid = NodeId.newInstance("127.0.0.1", 0);
  ContainerId contId = ContainerId.newContainerId(appAttemptId, 3);
  Container container = mock(Container.class);
  when(container.getId()).thenReturn(contId);
  when(container.getNodeId()).thenReturn(nid);
  when(container.getNodeHttpAddress()).thenReturn("localhost:0");
  taImpl.handle(new TaskAttemptDiagnosticsUpdateEvent(attemptId,
      "Task got killed"));
  assertFalse(
      "InternalError occurred trying to handle TA_DIAGNOSTICS_UPDATE on assigned task",
      eventHandler.internalError);
}
 
Example 15
Source Project: tez   File: TestTaskAttempt.java    License: Apache License 2.0 4 votes vote down vote up
@Test(timeout = 5000)
public void testLastDataEventRecording() throws Exception {
  ApplicationId appId = ApplicationId.newInstance(1, 2);
  ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
      appId, 0);
  TezDAGID dagID = TezDAGID.getInstance(appId, 1);
  TezVertexID vertexID = TezVertexID.getInstance(dagID, 1);
  TezTaskID taskID = TezTaskID.getInstance(vertexID, 1);

  MockEventHandler eventHandler = spy(new MockEventHandler());
  TaskCommunicatorManagerInterface taListener = createMockTaskAttemptListener();

  Configuration taskConf = new Configuration();
  taskConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
  taskConf.setBoolean("fs.file.impl.disable.cache", true);
  taskConf.setBoolean(TezConfiguration.TEZ_AM_SPECULATION_ENABLED, true);

  locationHint = TaskLocationHint.createTaskLocationHint(
      new HashSet<String>(Arrays.asList(new String[]{"127.0.0.1"})), null);
  Resource resource = Resource.newInstance(1024, 1);

  NodeId nid = NodeId.newInstance("127.0.0.1", 0);
  @SuppressWarnings("deprecation")
  ContainerId contId = ContainerId.newInstance(appAttemptId, 3);
  Container container = mock(Container.class);
  when(container.getId()).thenReturn(contId);
  when(container.getNodeId()).thenReturn(nid);
  when(container.getNodeHttpAddress()).thenReturn("localhost:0");

  AMContainerMap containers = new AMContainerMap(
      mock(ContainerHeartbeatHandler.class), mock(TaskCommunicatorManagerInterface.class),
      new ContainerContextMatcher(), appCtx);
  containers.addContainerIfNew(container, 0, 0, 0);

  doReturn(new ClusterInfo()).when(appCtx).getClusterInfo();
  doReturn(containers).when(appCtx).getAllContainers();

  TaskHeartbeatHandler mockHeartbeatHandler = mock(TaskHeartbeatHandler.class);
  TaskAttemptImpl taImpl = new MockTaskAttemptImpl(taskID, 1, eventHandler,
      taListener, taskConf, new SystemClock(),
      mockHeartbeatHandler, appCtx, false,
      resource, createFakeContainerContext(), false);
  TezTaskAttemptID taskAttemptID = taImpl.getID();

  taImpl.handle(new TaskAttemptEventSchedule(taskAttemptID, 0, 0));
  taImpl.handle(new TaskAttemptEventSubmitted(taskAttemptID, contId));
  taImpl.handle(new TaskAttemptEventStartedRemotely(taskAttemptID));
  assertEquals("Task attempt is not in the RUNNING state", taImpl.getState(),
      TaskAttemptState.RUNNING);
  
  long ts1 = 1024;
  long ts2 = 2048;
  TezTaskAttemptID mockId1 = mock(TezTaskAttemptID.class);
  TezTaskAttemptID mockId2 = mock(TezTaskAttemptID.class);
  TezEvent mockTezEvent1 = mock(TezEvent.class, RETURNS_DEEP_STUBS);
  when(mockTezEvent1.getEventReceivedTime()).thenReturn(ts1);
  when(mockTezEvent1.getSourceInfo().getTaskAttemptID()).thenReturn(mockId1);
  TezEvent mockTezEvent2 = mock(TezEvent.class, RETURNS_DEEP_STUBS);
  when(mockTezEvent2.getEventReceivedTime()).thenReturn(ts2);
  when(mockTezEvent2.getSourceInfo().getTaskAttemptID()).thenReturn(mockId2);
  TaskAttemptEventStatusUpdate statusEvent =
      new TaskAttemptEventStatusUpdate(taskAttemptID, new TaskStatusUpdateEvent(null, 0.1f, null, false));

  assertEquals(0, taImpl.lastDataEvents.size());
  taImpl.setLastEventSent(mockTezEvent1);
  assertEquals(1, taImpl.lastDataEvents.size());
  assertEquals(ts1, taImpl.lastDataEvents.get(0).getTimestamp());
  assertEquals(mockId1, taImpl.lastDataEvents.get(0).getTaskAttemptId());
  taImpl.handle(statusEvent);
  taImpl.setLastEventSent(mockTezEvent2);
  assertEquals(1, taImpl.lastDataEvents.size());
  assertEquals(ts2, taImpl.lastDataEvents.get(0).getTimestamp());
  assertEquals(mockId2, taImpl.lastDataEvents.get(0).getTaskAttemptId()); // over-write earlier value
  statusEvent.setReadErrorReported(true);
  taImpl.handle(statusEvent);
  taImpl.setLastEventSent(mockTezEvent1);
  assertEquals(2, taImpl.lastDataEvents.size());
  assertEquals(ts1, taImpl.lastDataEvents.get(1).getTimestamp());
  assertEquals(mockId1, taImpl.lastDataEvents.get(1).getTaskAttemptId()); // add new event
  taImpl.setLastEventSent(mockTezEvent2);
  assertEquals(2, taImpl.lastDataEvents.size());
  assertEquals(ts2, taImpl.lastDataEvents.get(1).getTimestamp());
  assertEquals(mockId2, taImpl.lastDataEvents.get(1).getTaskAttemptId()); // over-write earlier value
}
 
Example 16
Source Project: tez   File: TestTezClientUtils.java    License: Apache License 2.0 4 votes vote down vote up
@Test(timeout = 5000)
public void testAMLoggingOptsPerLogger() throws IOException, YarnException {

  TezConfiguration tezConf = new TezConfiguration();
  tezConf.set(TezConfiguration.TEZ_AM_LOG_LEVEL,
      "WARN;org.apache.hadoop.ipc=DEBUG;org.apache.hadoop.security=DEBUG");
  tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, STAGING_DIR.getAbsolutePath());

  ApplicationId appId = ApplicationId.newInstance(1000, 1);
  Credentials credentials = new Credentials();
  JobTokenSecretManager jobTokenSecretManager = new JobTokenSecretManager();
  TezClientUtils.createSessionToken(appId.toString(), jobTokenSecretManager, credentials);
  DAG dag = DAG.create("testdag");
  dag.addVertex(Vertex.create("testVertex", ProcessorDescriptor.create("processorClassname"), 1)
      .setTaskLaunchCmdOpts("initialLaunchOpts"));
  AMConfiguration amConf =
      new AMConfiguration(tezConf, new HashMap<String, LocalResource>(), credentials);
  ApplicationSubmissionContext appSubmissionContext =
      TezClientUtils.createApplicationSubmissionContext(appId, dag, "amName", amConf,
          new HashMap<String, LocalResource>(), credentials, false, new TezApiVersionInfo(),
          null, null);

  List<String> expectedCommands = new LinkedList<String>();
  expectedCommands.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator");
  expectedCommands.add(
      "-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE);
  expectedCommands.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "=" +
      ApplicationConstants.LOG_DIR_EXPANSION_VAR);
  expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "WARN" + "," +
      TezConstants.TEZ_CONTAINER_LOGGER_NAME);

  List<String> commands = appSubmissionContext.getAMContainerSpec().getCommands();
  assertEquals(1, commands.size());
  for (String expectedCmd : expectedCommands) {
    assertTrue(commands.get(0).contains(expectedCmd));
  }

  Map<String, String> environment = appSubmissionContext.getAMContainerSpec().getEnvironment();
  String logEnv = environment.get(TezConstants.TEZ_CONTAINER_LOG_PARAMS);
  assertEquals("org.apache.hadoop.ipc=DEBUG;org.apache.hadoop.security=DEBUG", logEnv);
}
 
Example 17
Source Project: big-c   File: BuilderUtils.java    License: Apache License 2.0 4 votes vote down vote up
public static ApplicationId newApplicationId(RecordFactory recordFactory,
    long clusterTimeStamp, int id) {
  return ApplicationId.newInstance(clusterTimeStamp, id);
}
 
Example 18
Source Project: big-c   File: TestYarnCLI.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testGetApplicationReport() throws Exception {
  for (int i = 0; i < 2; ++i) {
    ApplicationCLI cli = createAndGetAppCLI();
    ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
    ApplicationResourceUsageReport usageReport = i == 0 ? null :
        ApplicationResourceUsageReport.newInstance(
            2, 0, null, null, null, 123456, 4567);
    ApplicationReport newApplicationReport = ApplicationReport.newInstance(
        applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
        "user", "queue", "appname", "host", 124, null,
        YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0,
        FinalApplicationStatus.SUCCEEDED, usageReport, "N/A", 0.53789f, "YARN",
        null);
    when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(
        newApplicationReport);
    int result = cli.run(new String[] { "application", "-status", applicationId.toString() });
    assertEquals(0, result);
    verify(client, times(1 + i)).getApplicationReport(applicationId);
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    PrintWriter pw = new PrintWriter(baos);
    pw.println("Application Report : ");
    pw.println("\tApplication-Id : application_1234_0005");
    pw.println("\tApplication-Name : appname");
    pw.println("\tApplication-Type : YARN");
    pw.println("\tUser : user");
    pw.println("\tQueue : queue");
    pw.println("\tStart-Time : 0");
    pw.println("\tFinish-Time : 0");
    pw.println("\tProgress : 53.79%");
    pw.println("\tState : FINISHED");
    pw.println("\tFinal-State : SUCCEEDED");
    pw.println("\tTracking-URL : N/A");
    pw.println("\tRPC Port : 124");
    pw.println("\tAM Host : host");
    pw.println("\tAggregate Resource Allocation : " +
        (i == 0 ? "N/A" : "123456 MB-seconds, 4567 vcore-seconds"));
    pw.println("\tDiagnostics : diagnostics");
    pw.close();
    String appReportStr = baos.toString("UTF-8");
    Assert.assertEquals(appReportStr, sysOutStream.toString());
    sysOutStream.reset();
    verify(sysOut, times(1 + i)).println(isA(String.class));
  }
}
 
Example 19
Source Project: hadoop   File: TestNMProxy.java    License: Apache License 2.0 4 votes vote down vote up
@Test(timeout = 20000)
public void testNMProxyRetry() throws Exception {
  containerManager.start();
  containerManager.setBlockNewContainerRequests(false);
  StartContainersRequest allRequests =
      Records.newRecord(StartContainersRequest.class);
  ApplicationId appId = ApplicationId.newInstance(1, 1);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);

  org.apache.hadoop.yarn.api.records.Token nmToken =
      context.getNMTokenSecretManager().createNMToken(attemptId,
        context.getNodeId(), user);
  final InetSocketAddress address =
      conf.getSocketAddr(YarnConfiguration.NM_BIND_HOST,
        YarnConfiguration.NM_ADDRESS, YarnConfiguration.DEFAULT_NM_ADDRESS,
        YarnConfiguration.DEFAULT_NM_PORT);
  Token<NMTokenIdentifier> token =
      ConverterUtils.convertFromYarn(nmToken,
        SecurityUtil.buildTokenService(address));
  UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
  ugi.addToken(token);

  ContainerManagementProtocol proxy =
      NMProxy.createNMProxy(conf, ContainerManagementProtocol.class, ugi,
        YarnRPC.create(conf), address);

  retryCount = 0;
  shouldThrowNMNotYetReadyException = false;
  proxy.startContainers(allRequests);
  Assert.assertEquals(5, retryCount);

  retryCount = 0;
  shouldThrowNMNotYetReadyException = false;
  proxy.stopContainers(Records.newRecord(StopContainersRequest.class));
  Assert.assertEquals(5, retryCount);

  retryCount = 0;
  shouldThrowNMNotYetReadyException = false;
  proxy.getContainerStatuses(Records
    .newRecord(GetContainerStatusesRequest.class));
  Assert.assertEquals(5, retryCount);

  retryCount = 0;
  shouldThrowNMNotYetReadyException = true;
  proxy.startContainers(allRequests);
  Assert.assertEquals(5, retryCount);
}
 
Example 20
Source Project: incubator-tez   File: TestTaskAttempt.java    License: Apache License 2.0 4 votes vote down vote up
@Test(timeout = 5000)
// Ensure ContainerTerminating and ContainerTerminated is handled correctly by
// the TaskAttempt
public void testContainerTerminatedAfterSuccess() throws Exception {
  ApplicationId appId = ApplicationId.newInstance(1, 2);
  ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
      appId, 0);
  TezDAGID dagID = TezDAGID.getInstance(appId, 1);
  TezVertexID vertexID = TezVertexID.getInstance(dagID, 1);
  TezTaskID taskID = TezTaskID.getInstance(vertexID, 1);
  TezTaskAttemptID taskAttemptID = TezTaskAttemptID.getInstance(taskID, 0);

  MockEventHandler eventHandler = spy(new MockEventHandler());
  TaskAttemptListener taListener = mock(TaskAttemptListener.class);
  when(taListener.getAddress()).thenReturn(
      new InetSocketAddress("localhost", 0));

  Configuration taskConf = new Configuration();
  taskConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
  taskConf.setBoolean("fs.file.impl.disable.cache", true);

  TaskLocationHint locationHint = new TaskLocationHint(
      new HashSet<String>(Arrays.asList(new String[] {"127.0.0.1"})), null);
  Resource resource = Resource.newInstance(1024, 1);

  NodeId nid = NodeId.newInstance("127.0.0.1", 0);
  ContainerId contId = ContainerId.newInstance(appAttemptId, 3);
  Container container = mock(Container.class);
  when(container.getId()).thenReturn(contId);
  when(container.getNodeId()).thenReturn(nid);
  when(container.getNodeHttpAddress()).thenReturn("localhost:0");

  AppContext appCtx = mock(AppContext.class);
  AMContainerMap containers = new AMContainerMap(
      mock(ContainerHeartbeatHandler.class), mock(TaskAttemptListener.class),
      new ContainerContextMatcher(), appCtx);
  containers.addContainerIfNew(container);

  doReturn(new ClusterInfo()).when(appCtx).getClusterInfo();
  doReturn(containers).when(appCtx).getAllContainers();

  TaskAttemptImpl taImpl = new MockTaskAttemptImpl(taskID, 1, eventHandler,
      taListener, taskConf, new SystemClock(),
      mock(TaskHeartbeatHandler.class), appCtx, locationHint, false,
      resource, createFakeContainerContext(), false);

  ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class);

  taImpl.handle(new TaskAttemptEventSchedule(taskAttemptID, null));
  // At state STARTING.
  taImpl.handle(new TaskAttemptEventStartedRemotely(taskAttemptID, contId,
      null));
  assertEquals("Task attempt is not in the RUNNING state", taImpl.getState(),
      TaskAttemptState.RUNNING);

  int expectedEventsAtRunning = 3;
  verify(eventHandler, times(expectedEventsAtRunning)).handle(arg.capture());

  taImpl.handle(new TaskAttemptEvent(taskAttemptID, TaskAttemptEventType.TA_DONE));

  assertEquals("Task attempt is not in the  SUCCEEDED state", taImpl.getState(),
      TaskAttemptState.SUCCEEDED);

  assertEquals(0, taImpl.getDiagnostics().size());

  int expectedEvenstAfterTerminating = expectedEventsAtRunning + 3;
  arg = ArgumentCaptor.forClass(Event.class);
  verify(eventHandler, times(expectedEvenstAfterTerminating)).handle(arg.capture());

  verifyEventType(
      arg.getAllValues().subList(expectedEventsAtRunning,
          expectedEvenstAfterTerminating), TaskEventTAUpdate.class, 1);
  verifyEventType(
      arg.getAllValues().subList(expectedEventsAtRunning,
          expectedEvenstAfterTerminating), AMSchedulerEventTAEnded.class, 1);

  taImpl.handle(new TaskAttemptEventContainerTerminated(taskAttemptID,
      "Terminated"));
  int expectedEventAfterTerminated = expectedEvenstAfterTerminating + 0;
  arg = ArgumentCaptor.forClass(Event.class);
  verify(eventHandler, times(expectedEventAfterTerminated)).handle(arg.capture());

  // Verify that the diagnostic message included in the Terminated event is not
  // captured - TA already succeeded.
  assertEquals(0, taImpl.getDiagnostics().size());
}