org.apache.tez.dag.api.TezException Java Examples

private List<URL> processAdditionalResources(TezDAGID dagId, Map<String, LocalResource> lrDiff)
    throws TezException {
  if (lrDiff == null || lrDiff.isEmpty()) {
    return Collections.emptyList();
  } else {
    LOG.info("Localizing additional local resources for AM : " + lrDiff);
    List<URL> downloadedURLs;
    try {
      TezUtilsInternal.setHadoopCallerContext(hadoopShim, dagId);
      downloadedURLs = RelocalizationUtils.processAdditionalResources(
          Maps.transformValues(lrDiff, new Function<LocalResource, URI>() {

            @Override
            public URI apply(LocalResource input) {
              return getLocalResourceUri(input);
            }
          }), getConfig(), workingDirectory);
    } catch (IOException e) {
      throw new TezException(e);
    } finally {
      hadoopShim.clearHadoopCallerContext();
    }
    LOG.info("Done downloading additional AM resources");
    return downloadedURLs;
  }
}
 

public void setEdgeProperty(EdgeProperty newEdgeProperty) throws AMUserCodeException {
  boolean wasUnInitialized;
  synchronized (this) {
    this.edgeProperty = newEdgeProperty;
    wasUnInitialized = (edgeManager == null);
    try {
      createEdgeManager();
    } catch (TezException e) {
      throw new AMUserCodeException(Source.EdgeManager, e);
    }
  }
  initialize();
  if (wasUnInitialized) {
    sendEvent(new VertexEventNullEdgeInitialized(sourceVertex.getVertexId(), this,
        destinationVertex));
    sendEvent(new VertexEventNullEdgeInitialized(destinationVertex.getVertexId(), this,
        sourceVertex));
  }
}
 

private long checkAndHandleDAGClientTimeout() throws TezException {
  if (EnumSet.of(DAGAppMasterState.NEW, DAGAppMasterState.RECOVERING).contains(this.state)
      || sessionStopped.get()) {
    // AM new or recovering so do not kill session at this time
    // if session already completed or shutting down, this should be a a no-op
    return -1;
  }

  long currentTime = clock.getTime();
  long nextExpiry = clientHandler.getLastHeartbeatTime()
      + clientAMHeartbeatTimeoutIntervalMillis;
  if (currentTime < nextExpiry) {
    // reschedule timer to 1 sec after the next expiry window
    // to ensure that we time out as intended if there are no heartbeats
    return ((nextExpiry+1000) - currentTime);
  }

  String message = "Client-to-AM Heartbeat timeout interval expired, shutting down AM as client"
      + " stopped heartbeating to it"
      + ", lastClientAMHeartbeatTime=" + clientHandler.getLastHeartbeatTime()
      + ", clientAMHeartbeatTimeoutIntervalMillis="
      + clientAMHeartbeatTimeoutIntervalMillis + " ms";
  addDiagnostic(message);
  shutdownTezAM(message);
  return -1;
}
 

@Override
public DAGStatus getDAGStatus(@Nullable Set<StatusGetOpts> statusOptions)
    throws IOException, TezException {
  final String url = String.format("%s/%s/%s?fields=%s", baseUri, ATSConstants.TEZ_DAG_ID, dagId,
      FILTER_BY_FIELDS);
  try {
    DAGStatusProto.Builder statusBuilder;
    final JSONObject jsonRoot = getJsonRootEntity(url);

    statusBuilder = parseDagStatus(jsonRoot, statusOptions);
    if (statusBuilder == null) {
      throw new TezException("Failed to get DagStatus from ATS");
    }

    return new DAGStatus(statusBuilder, DagStatusSource.TIMELINE);
  } catch (JSONException je) {
    throw new TezException("Failed to parse DagStatus json from YARN Timeline", je);
  }
}
 

@VisibleForTesting
ContainerLauncher createContainerLauncher(
    NamedEntityDescriptor containerLauncherDescriptor,
    AppContext context,
    ContainerLauncherContext containerLauncherContext,
    TaskCommunicatorManagerInterface taskCommunicatorManagerInterface,
    String workingDirectory,
    int containerLauncherIndex,
    boolean isPureLocalMode) throws TezException {
  if (containerLauncherDescriptor.getEntityName().equals(
      TezConstants.getTezYarnServicePluginName())) {
    return createYarnContainerLauncher(containerLauncherContext);
  } else if (containerLauncherDescriptor.getEntityName()
      .equals(TezConstants.getTezUberServicePluginName())) {
    return createUberContainerLauncher(containerLauncherContext, context,
        taskCommunicatorManagerInterface,
        workingDirectory, isPureLocalMode);
  } else {
    return createCustomContainerLauncher(containerLauncherContext, containerLauncherDescriptor);
  }
}
 

DAGStatus getDAGStatusViaAM(Set<StatusGetOpts> statusOptions, long timeout)
    throws IOException, TezException {
  if(LOG.isDebugEnabled()) {
    LOG.debug("GetDAGStatus via AM for app: " + appId + " dag:" + dagId);
  }
  GetDAGStatusRequestProto.Builder requestProtoBuilder =
      GetDAGStatusRequestProto.newBuilder()
        .setDagId(dagId).setTimeout(timeout);

  if (statusOptions != null) {
    requestProtoBuilder.addAllStatusOptions(
      DagTypeConverters.convertStatusGetOptsToProto(statusOptions));
  }

  try {
    return new DAGStatus(
      proxy.getDAGStatus(null,
        requestProtoBuilder.build()).getDagStatus(), DagStatusSource.AM);
  } catch (ServiceException e) {
    RPCUtil.unwrapAndThrowException(e);
    // Should not reach here
    throw new TezException(e);
  }
}
 

@VisibleForTesting
TaskScheduler createTaskScheduler(String host, int port, String trackingUrl,
                                  AppContext appContext,
                                  NamedEntityDescriptor taskSchedulerDescriptor,
                                  long customAppIdIdentifier,
                                  int schedulerId) throws TezException {
  TaskSchedulerContext rawContext =
      new TaskSchedulerContextImpl(this, appContext, schedulerId, trackingUrl,
          customAppIdIdentifier, host, port, taskSchedulerDescriptor.getUserPayload());
  TaskSchedulerContext wrappedContext = wrapTaskSchedulerContext(rawContext);
  String schedulerName = taskSchedulerDescriptor.getEntityName();
  if (schedulerName.equals(TezConstants.getTezYarnServicePluginName())) {
    return createYarnTaskScheduler(wrappedContext, schedulerId);
  } else if (schedulerName.equals(TezConstants.getTezUberServicePluginName())) {
    return createUberTaskScheduler(wrappedContext, schedulerId);
  } else {
    return createCustomTaskScheduler(wrappedContext, taskSchedulerDescriptor, schedulerId);
  }
}
 

@Test(timeout = 20000)
public void testFatalErrorReported() throws IOException, TezException, InterruptedException {

  TezClient tezClient = getTezClient("testFatalErrorReported");
  DAGClient dagClient = null;

  try {
    FailingProcessor.configureForFatalFail();
    DAG dag = DAG.create("testFatalErrorReportedDag").addVertex(
        Vertex
            .create(VERTEX_NAME, ProcessorDescriptor.create(FailingProcessor.class.getName()), 1));

    dagClient = tezClient.submitDAG(dag);
    dagClient.waitForCompletion();
    assertEquals(DAGStatus.State.FAILED, dagClient.getDAGStatus(null).getState());
    assertEquals(1, dagClient.getVertexStatus(VERTEX_NAME, null).getProgress().getFailedTaskAttemptCount());
  } finally {
    if (dagClient != null) {
      dagClient.close();
    }
    tezClient.stop();
  }
}
 

private Map<String, LocalResource> getAdditionalLocalResourceDiff(
    DAG dag, Map<String, LocalResource> additionalResources) throws TezException {
  if (additionalResources == null) {
    return Collections.emptyMap();
  }
  // Check for existing resources.
  Iterator<Entry<String, LocalResource>> lrIter = additionalResources.entrySet().iterator();
  while (lrIter.hasNext()) {
    Entry<String, LocalResource> lrEntry = lrIter.next();
    LocalResource existing = amResources.get(lrEntry.getKey());
    if (existing != null) {
      if (!isSameFile(dag, lrEntry.getKey(), existing, lrEntry.getValue())) {
        throw new TezUncheckedException(
            "Cannot add different additional resources with the same name : "
                + lrEntry.getKey() + ", Existing: [" + existing + "], New: ["
                + lrEntry.getValue() + "]");
      } else {
        lrIter.remove();
      }
    }
  }
  return containerSignatureMatcher.getAdditionalResources(amResources, additionalResources);
}
 

@Override
public TryKillDAGResponseProto tryKillDAG(RpcController controller,
    TryKillDAGRequestProto request) throws ServiceException {
  UserGroupInformation user = getRPCUser();
  try {
    String dagId = request.getDagId();
    if (!real.getACLManager(dagId).checkDAGModifyAccess(user)) {
      throw new AccessControlException("User " + user + " cannot perform DAG modify operation");
    }
    real.updateLastHeartbeatTime();
    real.tryKillDAG(dagId);
    return TryKillDAGResponseProto.newBuilder().build();
  } catch (TezException e) {
    throw wrapException(e);
  }
}
 

private HistoryEvent testProtoConversion(HistoryEvent event) throws IOException, TezException {
  ByteArrayOutputStream os = new ByteArrayOutputStream();
  HistoryEvent deserializedEvent = null;
  CodedOutputStream codedOutputStream = CodedOutputStream.newInstance(os);
  event.toProtoStream(codedOutputStream);
  codedOutputStream.flush();
  os.flush();
  os.close();
  deserializedEvent = ReflectionUtils.createClazzInstance(
      event.getClass().getName());
  LOG.info("Serialized event to byte array"
      + ", eventType=" + event.getEventType()
      + ", bufLen=" + os.toByteArray().length);
  deserializedEvent.fromProtoStream(
      CodedInputStream.newInstance(os.toByteArray()));
  return deserializedEvent;
}
 

public static TimelineReaderStrategy getTimelineReaderStrategy(Configuration conf,
                                                               boolean useHttps,
                                                               int connTimeout) throws TezException {

  TimelineReaderStrategy timelineReaderStrategy;

  if (!isTimelineClientSupported()) {
    throw new TezException("Reading from timeline is not supported." +
        " token delegation support: " + tokenDelegationSupported() +
        ", is secure timeline: " + UserGroupInformation.isSecurityEnabled());
  }

  timelineReaderStrategy = getTimelineReaderStrategy(tokenDelegationSupported(), conf, useHttps,
      connTimeout);

  if (LOG.isDebugEnabled()) {
    LOG.debug("Using " + timelineReaderStrategy.getClass().getName() + " to read timeline data");
  }

  return timelineReaderStrategy;
}
 

private DagInfo getDagInfoFromSimpleHistory(String dagId) throws TezException, IOException {
  TezDAGID tezDAGID = TezDAGID.fromString(dagId);
  ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(tezDAGID
      .getApplicationId(), 1);
  Path historyPath = new Path(conf.get("fs.defaultFS")
      + SIMPLE_HISTORY_DIR + HISTORY_TXT + "."
      + applicationAttemptId);
  FileSystem fs = historyPath.getFileSystem(conf);

  Path localPath = new Path(DOWNLOAD_DIR, HISTORY_TXT);
  fs.copyToLocalFile(historyPath, localPath);
  File localFile = new File(DOWNLOAD_DIR, HISTORY_TXT);

  //Now parse via SimpleHistory
  SimpleHistoryParser parser = new SimpleHistoryParser(Arrays.asList(localFile));
  DagInfo dagInfo = parser.getDAGData(dagId);
  assertTrue(dagInfo.getDagId().equals(dagId));
  return dagInfo;
}
 

public static TezChild newTezChild(Configuration conf, String host, int port, String containerIdentifier,
    String tokenIdentifier, int attemptNumber, String[] localDirs, String workingDirectory,
    Map<String, String> serviceProviderEnvMap, @Nullable String pid,
    ExecutionContext executionContext, Credentials credentials, long memAvailable, String user,
    TezTaskUmbilicalProtocol tezUmbilical, boolean updateSysCounters, HadoopShim hadoopShim)
    throws IOException, InterruptedException, TezException {

  // Pull in configuration specified for the session.
  // TODO TEZ-1233. This needs to be moved over the wire rather than localizing the file
  // for each and every task, and reading it back from disk. Also needs to be per vertex.
  Limits.setConfiguration(conf);

  TezUtilsInternal.setSecurityUtilConfigration(LOG, conf);

  // singleton of ObjectRegistry for this JVM
  ObjectRegistryImpl objectRegistry = new ObjectRegistryImpl();

  return new TezChild(conf, host, port, containerIdentifier, tokenIdentifier,
      attemptNumber, workingDirectory, localDirs, serviceProviderEnvMap, objectRegistry, pid,
      executionContext, credentials, memAvailable, user, tezUmbilical, updateSysCounters,
      hadoopShim);
}
 

@Override
public RecordReader<K, V> getRecordReader(InputSplit split, JobConf job,
    Reporter reporter) throws IOException {
  TezGroupedSplit groupedSplit = (TezGroupedSplit) split;
  try {
    initInputFormatFromSplit(groupedSplit);
  } catch (TezException e) {
    throw new IOException(e);
  }
  return new TezGroupedSplitsRecordReader(groupedSplit, job, reporter);
}
 

@Test(timeout = 60000)
public void testInvalidQueueSubmission() throws Exception {

  TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
  YarnClient yarnClient = YarnClient.createYarnClient();
  try {

    yarnClient.init(mrrTezCluster.getConfig());
    yarnClient.start();

    SimpleSessionExample job = new SimpleSessionExample();
    tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, false);
    tezConf.set(TezConfiguration.TEZ_QUEUE_NAME, "nonexistent");

    String[] inputPaths = new String[1];
    String[] outputPaths = new String[1];
    String inputDirStr = "/tmp/owc-input";
    inputPaths[0] = inputDirStr;
    Path inputDir = new Path(inputDirStr);
    remoteFs.mkdirs(inputDir);
    String outputDirStr = "/tmp/owc-output";
    outputPaths[0] = outputDirStr;
    int result = job.run(tezConf, new String[] { StringUtils.join(",", inputPaths),
        StringUtils.join(",", outputPaths), "2" }, null);
    Assert.assertTrue("Job should have failed", result != 0);
  } catch (TezException e) {
    Assert.assertTrue(e.getMessage().contains("Failed to submit application"));
  } finally {
    if (yarnClient != null) {
      yarnClient.stop();
    }
  }
}
 

@Test(timeout = 5000)
public void testKilledAfterComplete() throws IOException, InterruptedException, TezException,
    ExecutionException {

  ListeningExecutorService executor = null;
  try {
    ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
    executor = MoreExecutors.listeningDecorator(rawExecutor);
    ApplicationId appId = ApplicationId.newInstance(10000, 1);
    TaskExecutionTestHelpers.TezTaskUmbilicalForTest
        umbilical = new TaskExecutionTestHelpers.TezTaskUmbilicalForTest();
    TaskReporter taskReporter = createTaskReporter(appId, umbilical);

    TezTaskRunner2ForTest taskRunner =
        createTaskRunnerForTest(appId, umbilical, taskReporter, executor,
            TestProcessor.CONF_EMPTY);
    // Setup the executor
    Future<TaskRunner2Result> taskRunnerFuture =
        taskExecutor.submit(new TaskRunnerCallable2ForTest(taskRunner));
    // Signal the processor to go through
    TestProcessor.awaitStart();
    TestProcessor.signal();
    TestProcessor.awaitCompletion();

    taskRunner.awaitCallableCompletion();

    taskRunner.killTask();
    TaskRunner2Result result = taskRunnerFuture.get();
    verifyTaskRunnerResult(result, EndReason.SUCCESS, null, false, null);

    assertFalse(TestProcessor.wasInterrupted());
    assertNull(taskReporter.currentCallable);
    umbilical.verifyTaskSuccessEvent();
  } finally {
    executor.shutdownNow();
  }
}
 

@Override
TaskScheduler createCustomTaskScheduler(TaskSchedulerContext taskSchedulerContext,
                                        NamedEntityDescriptor taskSchedulerDescriptor, int schedulerId)
                                            throws TezException {
  taskSchedulerContexts.add(taskSchedulerContext);
  TaskScheduler taskScheduler = spy(super.createCustomTaskScheduler(taskSchedulerContext, taskSchedulerDescriptor, schedulerId));
  testTaskSchedulers.add(taskScheduler);
  return taskScheduler;
}
 

@Test
public void testHeartbeatShouldDie() throws IOException, InterruptedException, TezException,
    ExecutionException {

  ListeningExecutorService executor = null;
  try {
    ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
    executor = MoreExecutors.listeningDecorator(rawExecutor);
    ApplicationId appId = ApplicationId.newInstance(10000, 1);
    TezTaskUmbilicalForTest umbilical = new TezTaskUmbilicalForTest();
    TaskReporter taskReporter = createTaskReporter(appId, umbilical);

    TezTaskRunner taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor,
        TestProcessor.CONF_EMPTY);
    // Setup the executor
    Future<Boolean> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable(taskRunner));
    // Signal the processor to go through
    TestProcessor.awaitStart();
    umbilical.signalSendShouldDie();
    umbilical.awaitRegisteredEvent();
    // Not signaling an actual start to verify task interruption

    boolean result = taskRunnerFuture.get();
    assertFalse(result);

    TestProcessor.awaitCompletion();
    assertTrue(TestProcessor.wasInterrupted());
    assertNull(taskReporter.currentCallable);
    // TODO Is this statement correct ?
    // No completion events since shouldDie was requested by the AM, which should have killed the
    // task.
    umbilical.verifyNoCompletionEvents();
  } finally {
    executor.shutdownNow();
  }
}
 

@Private
static DAGClient getDAGClient(ApplicationId appId, TezConfiguration tezConf, YarnConfiguration
    yarnConf, FrameworkClient frameworkClient, UserGroupInformation ugi)
    throws IOException, TezException {
  return new DAGClientImpl(appId, getDefaultTezDAGID(appId), tezConf,
      yarnConf, frameworkClient, ugi);
}
 

@Override
public boolean taskKilled(TezTaskAttemptID taskAttemptID, Throwable t, String diagnostics,
                          EventMetaData srcMeta) throws IOException, TezException {
  if(!isShuttingDown()) {
    return currentCallable.taskTerminated(taskAttemptID, true, null, t, diagnostics, srcMeta);
  }
  return false;
}
 

@Test(timeout = 120000)
public void testMultipleMRRSleepJobViaSession() throws IOException,
InterruptedException, TezException, ClassNotFoundException, YarnException {
  Path remoteStagingDir = remoteFs.makeQualified(new Path("/tmp", String
      .valueOf(new Random().nextInt(100000))));
  remoteFs.mkdirs(remoteStagingDir);
  TezConfiguration tezConf = new TezConfiguration(
      mrrTezCluster.getConfig());
  tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR,
      remoteStagingDir.toString());

  TezClient tezSession = TezClient.create("testsession", tezConf, true);
  tezSession.start();
  Assert.assertEquals(TezAppMasterStatus.INITIALIZING,
      tezSession.getAppMasterStatus());

  State finalState = testMRRSleepJobDagSubmitCore(true, false, false,
      tezSession, false, null, null);
  Assert.assertEquals(DAGStatus.State.SUCCEEDED, finalState);
  Assert.assertEquals(TezAppMasterStatus.READY,
      tezSession.getAppMasterStatus());
  finalState = testMRRSleepJobDagSubmitCore(true, false, false,
      tezSession, false, null, null);
  Assert.assertEquals(DAGStatus.State.SUCCEEDED, finalState);
  Assert.assertEquals(TezAppMasterStatus.READY,
      tezSession.getAppMasterStatus());

  stopAndVerifyYarnApp(tezSession);
}
 

public ContainerLauncherManager(AppContext context,
                                TaskCommunicatorManagerInterface taskCommunicatorManagerInterface,
                                String workingDirectory,
                                List<NamedEntityDescriptor> containerLauncherDescriptors,
                                boolean isLocalMode) throws TezException {
  super(ContainerLauncherManager.class.getName());

  this.isIncompleteCtor = false;
  this.appContext = context;
  if (containerLauncherDescriptors == null || containerLauncherDescriptors.isEmpty()) {
    throw new IllegalArgumentException("ContainerLauncherDescriptors must be specified");
  }
  containerLauncherContexts = new ContainerLauncherContext[containerLauncherDescriptors.size()];
  containerLaunchers = new ContainerLauncherWrapper[containerLauncherDescriptors.size()];
  containerLauncherServiceWrappers = new ServicePluginLifecycleAbstractService[containerLauncherDescriptors.size()];


  for (int i = 0; i < containerLauncherDescriptors.size(); i++) {
    UserPayload userPayload = containerLauncherDescriptors.get(i).getUserPayload();
    ContainerLauncherContext containerLauncherContext =
        new ContainerLauncherContextImpl(context, this, taskCommunicatorManagerInterface, userPayload, i);
    containerLauncherContexts[i] = containerLauncherContext;
    containerLaunchers[i] = new ContainerLauncherWrapper(createContainerLauncher(containerLauncherDescriptors.get(i), context,
        containerLauncherContext, taskCommunicatorManagerInterface, workingDirectory, i, isLocalMode));
    containerLauncherServiceWrappers[i] = new ServicePluginLifecycleAbstractService<>(containerLaunchers[i].getContainerLauncher());
  }
}
 

@Override
public TezHeartbeatResponse heartbeat(TezHeartbeatRequest request) throws IOException,
    TezException {
  umbilicalLock.lock();
  if (request.getEvents() != null) {
    requestEvents.addAll(request.getEvents());
  }
  try {
    if (shouldThrowException) {
      LOG.info("TestUmbilical throwing Exception");
      throw new IOException(HEARTBEAT_EXCEPTION_STRING);
    }
    TezHeartbeatResponse response = new TezHeartbeatResponse();
    response.setLastRequestId(request.getRequestId());
    if (shouldSendDieSignal) {
      LOG.info("TestUmbilical returning shouldDie=true");
      response.setShouldDie();
    }
    return response;
  } finally {
    if (pendingEvent) {
      eventEnacted = true;
      LOG.info("Signalling Event");
      eventCondition.signal();
    }
    umbilicalLock.unlock();
  }
}
 

@Override
public DAGClientAMProtocolRPC.PreWarmResponseProto preWarm(
  RpcController controller,
  PreWarmRequestProto request) throws ServiceException {
  try {
    real.preWarmContainers(
      DagTypeConverters.convertPreWarmContextFromProto(
        request.getPreWarmContext()));
    return PreWarmResponseProto.newBuilder().build();
  } catch (TezException e) {
    throw wrapException(e);
  }
}
 

@Test(timeout = 5000)
public void testSignalDeprecatedFatalErrorAndLoop() throws IOException, InterruptedException, TezException,
    ExecutionException {

  ListeningExecutorService executor = null;
  try {
    ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
    executor = MoreExecutors.listeningDecorator(rawExecutor);
    ApplicationId appId = ApplicationId.newInstance(10000, 1);
    TaskExecutionTestHelpers.TezTaskUmbilicalForTest
        umbilical = new TaskExecutionTestHelpers.TezTaskUmbilicalForTest();
    TaskReporter taskReporter = createTaskReporter(appId, umbilical);

    TezTaskRunner2 taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor,
        TestProcessor.CONF_SIGNAL_DEPRECATEDFATAL_AND_LOOP);
    // Setup the executor
    Future<TaskRunner2Result> taskRunnerFuture =
        taskExecutor.submit(new TaskRunnerCallable2ForTest(taskRunner));
    // Signal the processor to go through
    TestProcessor.awaitStart();
    TestProcessor.signal();

    TestProcessor.awaitLoop();
    // The fatal error should have caused an interrupt.

    TaskRunner2Result result = taskRunnerFuture.get();
    verifyTaskRunnerResult(result, EndReason.TASK_ERROR, createProcessorIOException(), false, TaskFailureType.NON_FATAL);

    TestProcessor.awaitCompletion();
    assertTrue(TestProcessor.wasInterrupted());
    assertNull(taskReporter.currentCallable);
    umbilical.verifyTaskFailedEvent(
        FAILURE_START_STRING,
        IOException.class.getName() + ": " + IOException.class.getSimpleName());
    // Signal fatal error should cause the processor to fail.
    assertTrue(TestProcessor.wasAborted());
  } finally {
    executor.shutdownNow();
  }
}
 

@Test(timeout = 5000)
public void testCustomTaskCommSpecified() throws IOException, TezException {

  AppContext appContext = mock(AppContext.class);
  TaskHeartbeatHandler thh = mock(TaskHeartbeatHandler.class);
  ContainerHeartbeatHandler chh = mock(ContainerHeartbeatHandler.class);

  String customTaskCommName = "customTaskComm";
  List<NamedEntityDescriptor> taskCommDescriptors = new LinkedList<>();
  ByteBuffer bb = ByteBuffer.allocate(4);
  bb.putInt(0, 3);
  UserPayload customPayload = UserPayload.create(bb);
  taskCommDescriptors.add(
      new NamedEntityDescriptor(customTaskCommName, FakeTaskComm.class.getName())
          .setUserPayload(customPayload));

  TaskCommManagerForMultipleCommTest tcm =
      new TaskCommManagerForMultipleCommTest(appContext, thh, chh, taskCommDescriptors);

  try {
    tcm.init(new Configuration(false));
    tcm.start();

    assertEquals(1, tcm.getNumTaskComms());
    assertFalse(tcm.getYarnTaskCommCreated());
    assertFalse(tcm.getUberTaskCommCreated());

    assertEquals(customTaskCommName, tcm.getTaskCommName(0));
    assertEquals(bb, tcm.getTaskCommContext(0).getInitialUserPayload().getPayload());

  } finally {
    tcm.stop();
  }
}
 

/**
 * Utility method that unwraps and returns appropriate exceptions.
 *
 * @param se
 *          ServiceException
 * @return An instance of the actual exception, which will be a subclass of
 *         {@link TezException} or {@link IOException}
 */
public static Void unwrapAndThrowNonIOException(ServiceException se)
    throws TezException {
  try {
    return unwrapAndThrowException(se);
  } catch (IOException ioe) {
    throw new TezException(ioe);
  }
}
 

@Test(timeout = 5000)
public void testSignalFatalAndThrow() throws IOException, InterruptedException, TezException,
    ExecutionException {

  ListeningExecutorService executor = null;
  try {
    ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
    executor = MoreExecutors.listeningDecorator(rawExecutor);
    ApplicationId appId = ApplicationId.newInstance(10000, 1);
    TaskExecutionTestHelpers.TezTaskUmbilicalForTest
        umbilical = new TaskExecutionTestHelpers.TezTaskUmbilicalForTest();
    TaskReporter taskReporter = createTaskReporter(appId, umbilical);

    TezTaskRunner2 taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor,
        TestProcessor.CONF_SIGNAL_FATAL_AND_THROW);
    // Setup the executor
    Future<TaskRunner2Result> taskRunnerFuture =
        taskExecutor.submit(new TaskRunnerCallable2ForTest(taskRunner));
    // Signal the processor to go through
    TestProcessor.awaitStart();
    TestProcessor.signal();

    TaskRunner2Result result = taskRunnerFuture.get();
    verifyTaskRunnerResult(result, EndReason.TASK_ERROR, createProcessorIOException(), false, TaskFailureType.FATAL);

    TestProcessor.awaitCompletion();
    assertNull(taskReporter.currentCallable);
    umbilical.verifyTaskFailedEvent(
        FAILURE_START_STRING,
        IOException.class.getName() + ": " + IOException.class.getSimpleName(), TaskFailureType.FATAL);
    assertTrue(TestProcessor.wasAborted());
  } finally {
    executor.shutdownNow();
  }
}
 

protected void parse(String dagId, JSONObjectSource source)
    throws JSONException, TezException, IOException {
  Map<String, JSONObject> vertexJsonMap = Maps.newHashMap();
  Map<String, JSONObject> taskJsonMap = Maps.newHashMap();
  Map<String, JSONObject> attemptJsonMap = Maps.newHashMap();

  readEventsFromSource(dagId, source, vertexJsonMap, taskJsonMap, attemptJsonMap);
  postProcessMaps(vertexJsonMap, taskJsonMap, attemptJsonMap);
}