org.apache.tez.dag.api.TezException Java Examples

The following examples show how to use org.apache.tez.dag.api.TezException. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: DAGAppMaster.java    From tez with Apache License 2.0 6 votes vote down vote up
private List<URL> processAdditionalResources(TezDAGID dagId, Map<String, LocalResource> lrDiff)
    throws TezException {
  if (lrDiff == null || lrDiff.isEmpty()) {
    return Collections.emptyList();
  } else {
    LOG.info("Localizing additional local resources for AM : " + lrDiff);
    List<URL> downloadedURLs;
    try {
      TezUtilsInternal.setHadoopCallerContext(hadoopShim, dagId);
      downloadedURLs = RelocalizationUtils.processAdditionalResources(
          Maps.transformValues(lrDiff, new Function<LocalResource, URI>() {

            @Override
            public URI apply(LocalResource input) {
              return getLocalResourceUri(input);
            }
          }), getConfig(), workingDirectory);
    } catch (IOException e) {
      throw new TezException(e);
    } finally {
      hadoopShim.clearHadoopCallerContext();
    }
    LOG.info("Done downloading additional AM resources");
    return downloadedURLs;
  }
}
 
Example #2
Source File: Edge.java    From tez with Apache License 2.0 6 votes vote down vote up
public void setEdgeProperty(EdgeProperty newEdgeProperty) throws AMUserCodeException {
  boolean wasUnInitialized;
  synchronized (this) {
    this.edgeProperty = newEdgeProperty;
    wasUnInitialized = (edgeManager == null);
    try {
      createEdgeManager();
    } catch (TezException e) {
      throw new AMUserCodeException(Source.EdgeManager, e);
    }
  }
  initialize();
  if (wasUnInitialized) {
    sendEvent(new VertexEventNullEdgeInitialized(sourceVertex.getVertexId(), this,
        destinationVertex));
    sendEvent(new VertexEventNullEdgeInitialized(destinationVertex.getVertexId(), this,
        sourceVertex));
  }
}
 
Example #3
Source File: DAGAppMaster.java    From tez with Apache License 2.0 6 votes vote down vote up
private long checkAndHandleDAGClientTimeout() throws TezException {
  if (EnumSet.of(DAGAppMasterState.NEW, DAGAppMasterState.RECOVERING).contains(this.state)
      || sessionStopped.get()) {
    // AM new or recovering so do not kill session at this time
    // if session already completed or shutting down, this should be a a no-op
    return -1;
  }

  long currentTime = clock.getTime();
  long nextExpiry = clientHandler.getLastHeartbeatTime()
      + clientAMHeartbeatTimeoutIntervalMillis;
  if (currentTime < nextExpiry) {
    // reschedule timer to 1 sec after the next expiry window
    // to ensure that we time out as intended if there are no heartbeats
    return ((nextExpiry+1000) - currentTime);
  }

  String message = "Client-to-AM Heartbeat timeout interval expired, shutting down AM as client"
      + " stopped heartbeating to it"
      + ", lastClientAMHeartbeatTime=" + clientHandler.getLastHeartbeatTime()
      + ", clientAMHeartbeatTimeoutIntervalMillis="
      + clientAMHeartbeatTimeoutIntervalMillis + " ms";
  addDiagnostic(message);
  shutdownTezAM(message);
  return -1;
}
 
Example #4
Source File: DAGClientTimelineImpl.java    From tez with Apache License 2.0 6 votes vote down vote up
@Override
public DAGStatus getDAGStatus(@Nullable Set<StatusGetOpts> statusOptions)
    throws IOException, TezException {
  final String url = String.format("%s/%s/%s?fields=%s", baseUri, ATSConstants.TEZ_DAG_ID, dagId,
      FILTER_BY_FIELDS);
  try {
    DAGStatusProto.Builder statusBuilder;
    final JSONObject jsonRoot = getJsonRootEntity(url);

    statusBuilder = parseDagStatus(jsonRoot, statusOptions);
    if (statusBuilder == null) {
      throw new TezException("Failed to get DagStatus from ATS");
    }

    return new DAGStatus(statusBuilder, DagStatusSource.TIMELINE);
  } catch (JSONException je) {
    throw new TezException("Failed to parse DagStatus json from YARN Timeline", je);
  }
}
 
Example #5
Source File: ContainerLauncherManager.java    From tez with Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
ContainerLauncher createContainerLauncher(
    NamedEntityDescriptor containerLauncherDescriptor,
    AppContext context,
    ContainerLauncherContext containerLauncherContext,
    TaskCommunicatorManagerInterface taskCommunicatorManagerInterface,
    String workingDirectory,
    int containerLauncherIndex,
    boolean isPureLocalMode) throws TezException {
  if (containerLauncherDescriptor.getEntityName().equals(
      TezConstants.getTezYarnServicePluginName())) {
    return createYarnContainerLauncher(containerLauncherContext);
  } else if (containerLauncherDescriptor.getEntityName()
      .equals(TezConstants.getTezUberServicePluginName())) {
    return createUberContainerLauncher(containerLauncherContext, context,
        taskCommunicatorManagerInterface,
        workingDirectory, isPureLocalMode);
  } else {
    return createCustomContainerLauncher(containerLauncherContext, containerLauncherDescriptor);
  }
}
 
Example #6
Source File: DAGClientRPCImpl.java    From tez with Apache License 2.0 6 votes vote down vote up
DAGStatus getDAGStatusViaAM(Set<StatusGetOpts> statusOptions, long timeout)
    throws IOException, TezException {
  if(LOG.isDebugEnabled()) {
    LOG.debug("GetDAGStatus via AM for app: " + appId + " dag:" + dagId);
  }
  GetDAGStatusRequestProto.Builder requestProtoBuilder =
      GetDAGStatusRequestProto.newBuilder()
        .setDagId(dagId).setTimeout(timeout);

  if (statusOptions != null) {
    requestProtoBuilder.addAllStatusOptions(
      DagTypeConverters.convertStatusGetOptsToProto(statusOptions));
  }

  try {
    return new DAGStatus(
      proxy.getDAGStatus(null,
        requestProtoBuilder.build()).getDagStatus(), DagStatusSource.AM);
  } catch (ServiceException e) {
    RPCUtil.unwrapAndThrowException(e);
    // Should not reach here
    throw new TezException(e);
  }
}
 
Example #7
Source File: TaskSchedulerManager.java    From tez with Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
TaskScheduler createTaskScheduler(String host, int port, String trackingUrl,
                                  AppContext appContext,
                                  NamedEntityDescriptor taskSchedulerDescriptor,
                                  long customAppIdIdentifier,
                                  int schedulerId) throws TezException {
  TaskSchedulerContext rawContext =
      new TaskSchedulerContextImpl(this, appContext, schedulerId, trackingUrl,
          customAppIdIdentifier, host, port, taskSchedulerDescriptor.getUserPayload());
  TaskSchedulerContext wrappedContext = wrapTaskSchedulerContext(rawContext);
  String schedulerName = taskSchedulerDescriptor.getEntityName();
  if (schedulerName.equals(TezConstants.getTezYarnServicePluginName())) {
    return createYarnTaskScheduler(wrappedContext, schedulerId);
  } else if (schedulerName.equals(TezConstants.getTezUberServicePluginName())) {
    return createUberTaskScheduler(wrappedContext, schedulerId);
  } else {
    return createCustomTaskScheduler(wrappedContext, taskSchedulerDescriptor, schedulerId);
  }
}
 
Example #8
Source File: TestTaskErrorsUsingLocalMode.java    From tez with Apache License 2.0 6 votes vote down vote up
@Test(timeout = 20000)
public void testFatalErrorReported() throws IOException, TezException, InterruptedException {

  TezClient tezClient = getTezClient("testFatalErrorReported");
  DAGClient dagClient = null;

  try {
    FailingProcessor.configureForFatalFail();
    DAG dag = DAG.create("testFatalErrorReportedDag").addVertex(
        Vertex
            .create(VERTEX_NAME, ProcessorDescriptor.create(FailingProcessor.class.getName()), 1));

    dagClient = tezClient.submitDAG(dag);
    dagClient.waitForCompletion();
    assertEquals(DAGStatus.State.FAILED, dagClient.getDAGStatus(null).getState());
    assertEquals(1, dagClient.getVertexStatus(VERTEX_NAME, null).getProgress().getFailedTaskAttemptCount());
  } finally {
    if (dagClient != null) {
      dagClient.close();
    }
    tezClient.stop();
  }
}
 
Example #9
Source File: DAGAppMaster.java    From incubator-tez with Apache License 2.0 6 votes vote down vote up
private Map<String, LocalResource> getAdditionalLocalResourceDiff(
    DAG dag, Map<String, LocalResource> additionalResources) throws TezException {
  if (additionalResources == null) {
    return Collections.emptyMap();
  }
  // Check for existing resources.
  Iterator<Entry<String, LocalResource>> lrIter = additionalResources.entrySet().iterator();
  while (lrIter.hasNext()) {
    Entry<String, LocalResource> lrEntry = lrIter.next();
    LocalResource existing = amResources.get(lrEntry.getKey());
    if (existing != null) {
      if (!isSameFile(dag, lrEntry.getKey(), existing, lrEntry.getValue())) {
        throw new TezUncheckedException(
            "Cannot add different additional resources with the same name : "
                + lrEntry.getKey() + ", Existing: [" + existing + "], New: ["
                + lrEntry.getValue() + "]");
      } else {
        lrIter.remove();
      }
    }
  }
  return containerSignatureMatcher.getAdditionalResources(amResources, additionalResources);
}
 
Example #10
Source File: DAGClientAMProtocolBlockingPBServerImpl.java    From tez with Apache License 2.0 6 votes vote down vote up
@Override
public TryKillDAGResponseProto tryKillDAG(RpcController controller,
    TryKillDAGRequestProto request) throws ServiceException {
  UserGroupInformation user = getRPCUser();
  try {
    String dagId = request.getDagId();
    if (!real.getACLManager(dagId).checkDAGModifyAccess(user)) {
      throw new AccessControlException("User " + user + " cannot perform DAG modify operation");
    }
    real.updateLastHeartbeatTime();
    real.tryKillDAG(dagId);
    return TryKillDAGResponseProto.newBuilder().build();
  } catch (TezException e) {
    throw wrapException(e);
  }
}
 
Example #11
Source File: TestHistoryEventsProtoConversion.java    From tez with Apache License 2.0 6 votes vote down vote up
private HistoryEvent testProtoConversion(HistoryEvent event) throws IOException, TezException {
  ByteArrayOutputStream os = new ByteArrayOutputStream();
  HistoryEvent deserializedEvent = null;
  CodedOutputStream codedOutputStream = CodedOutputStream.newInstance(os);
  event.toProtoStream(codedOutputStream);
  codedOutputStream.flush();
  os.flush();
  os.close();
  deserializedEvent = ReflectionUtils.createClazzInstance(
      event.getClass().getName());
  LOG.info("Serialized event to byte array"
      + ", eventType=" + event.getEventType()
      + ", bufLen=" + os.toByteArray().length);
  deserializedEvent.fromProtoStream(
      CodedInputStream.newInstance(os.toByteArray()));
  return deserializedEvent;
}
 
Example #12
Source File: TimelineReaderFactory.java    From tez with Apache License 2.0 6 votes vote down vote up
public static TimelineReaderStrategy getTimelineReaderStrategy(Configuration conf,
                                                               boolean useHttps,
                                                               int connTimeout) throws TezException {

  TimelineReaderStrategy timelineReaderStrategy;

  if (!isTimelineClientSupported()) {
    throw new TezException("Reading from timeline is not supported." +
        " token delegation support: " + tokenDelegationSupported() +
        ", is secure timeline: " + UserGroupInformation.isSecurityEnabled());
  }

  timelineReaderStrategy = getTimelineReaderStrategy(tokenDelegationSupported(), conf, useHttps,
      connTimeout);

  if (LOG.isDebugEnabled()) {
    LOG.debug("Using " + timelineReaderStrategy.getClass().getName() + " to read timeline data");
  }

  return timelineReaderStrategy;
}
 
Example #13
Source File: TestHistoryParser.java    From tez with Apache License 2.0 6 votes vote down vote up
private DagInfo getDagInfoFromSimpleHistory(String dagId) throws TezException, IOException {
  TezDAGID tezDAGID = TezDAGID.fromString(dagId);
  ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(tezDAGID
      .getApplicationId(), 1);
  Path historyPath = new Path(conf.get("fs.defaultFS")
      + SIMPLE_HISTORY_DIR + HISTORY_TXT + "."
      + applicationAttemptId);
  FileSystem fs = historyPath.getFileSystem(conf);

  Path localPath = new Path(DOWNLOAD_DIR, HISTORY_TXT);
  fs.copyToLocalFile(historyPath, localPath);
  File localFile = new File(DOWNLOAD_DIR, HISTORY_TXT);

  //Now parse via SimpleHistory
  SimpleHistoryParser parser = new SimpleHistoryParser(Arrays.asList(localFile));
  DagInfo dagInfo = parser.getDAGData(dagId);
  assertTrue(dagInfo.getDagId().equals(dagId));
  return dagInfo;
}
 
Example #14
Source File: TezChild.java    From tez with Apache License 2.0 6 votes vote down vote up
public static TezChild newTezChild(Configuration conf, String host, int port, String containerIdentifier,
    String tokenIdentifier, int attemptNumber, String[] localDirs, String workingDirectory,
    Map<String, String> serviceProviderEnvMap, @Nullable String pid,
    ExecutionContext executionContext, Credentials credentials, long memAvailable, String user,
    TezTaskUmbilicalProtocol tezUmbilical, boolean updateSysCounters, HadoopShim hadoopShim)
    throws IOException, InterruptedException, TezException {

  // Pull in configuration specified for the session.
  // TODO TEZ-1233. This needs to be moved over the wire rather than localizing the file
  // for each and every task, and reading it back from disk. Also needs to be per vertex.
  Limits.setConfiguration(conf);

  TezUtilsInternal.setSecurityUtilConfigration(LOG, conf);

  // singleton of ObjectRegistry for this JVM
  ObjectRegistryImpl objectRegistry = new ObjectRegistryImpl();

  return new TezChild(conf, host, port, containerIdentifier, tokenIdentifier,
      attemptNumber, workingDirectory, localDirs, serviceProviderEnvMap, objectRegistry, pid,
      executionContext, credentials, memAvailable, user, tezUmbilical, updateSysCounters,
      hadoopShim);
}
 
Example #15
Source File: TezGroupedSplitsInputFormat.java    From tez with Apache License 2.0 5 votes vote down vote up
@Override
public RecordReader<K, V> getRecordReader(InputSplit split, JobConf job,
    Reporter reporter) throws IOException {
  TezGroupedSplit groupedSplit = (TezGroupedSplit) split;
  try {
    initInputFormatFromSplit(groupedSplit);
  } catch (TezException e) {
    throw new IOException(e);
  }
  return new TezGroupedSplitsRecordReader(groupedSplit, job, reporter);
}
 
Example #16
Source File: TestTezJobs.java    From tez with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 60000)
public void testInvalidQueueSubmission() throws Exception {

  TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
  YarnClient yarnClient = YarnClient.createYarnClient();
  try {

    yarnClient.init(mrrTezCluster.getConfig());
    yarnClient.start();

    SimpleSessionExample job = new SimpleSessionExample();
    tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, false);
    tezConf.set(TezConfiguration.TEZ_QUEUE_NAME, "nonexistent");

    String[] inputPaths = new String[1];
    String[] outputPaths = new String[1];
    String inputDirStr = "/tmp/owc-input";
    inputPaths[0] = inputDirStr;
    Path inputDir = new Path(inputDirStr);
    remoteFs.mkdirs(inputDir);
    String outputDirStr = "/tmp/owc-output";
    outputPaths[0] = outputDirStr;
    int result = job.run(tezConf, new String[] { StringUtils.join(",", inputPaths),
        StringUtils.join(",", outputPaths), "2" }, null);
    Assert.assertTrue("Job should have failed", result != 0);
  } catch (TezException e) {
    Assert.assertTrue(e.getMessage().contains("Failed to submit application"));
  } finally {
    if (yarnClient != null) {
      yarnClient.stop();
    }
  }
}
 
Example #17
Source File: TestTaskExecution2.java    From tez with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 5000)
public void testKilledAfterComplete() throws IOException, InterruptedException, TezException,
    ExecutionException {

  ListeningExecutorService executor = null;
  try {
    ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
    executor = MoreExecutors.listeningDecorator(rawExecutor);
    ApplicationId appId = ApplicationId.newInstance(10000, 1);
    TaskExecutionTestHelpers.TezTaskUmbilicalForTest
        umbilical = new TaskExecutionTestHelpers.TezTaskUmbilicalForTest();
    TaskReporter taskReporter = createTaskReporter(appId, umbilical);

    TezTaskRunner2ForTest taskRunner =
        createTaskRunnerForTest(appId, umbilical, taskReporter, executor,
            TestProcessor.CONF_EMPTY);
    // Setup the executor
    Future<TaskRunner2Result> taskRunnerFuture =
        taskExecutor.submit(new TaskRunnerCallable2ForTest(taskRunner));
    // Signal the processor to go through
    TestProcessor.awaitStart();
    TestProcessor.signal();
    TestProcessor.awaitCompletion();

    taskRunner.awaitCallableCompletion();

    taskRunner.killTask();
    TaskRunner2Result result = taskRunnerFuture.get();
    verifyTaskRunnerResult(result, EndReason.SUCCESS, null, false, null);

    assertFalse(TestProcessor.wasInterrupted());
    assertNull(taskReporter.currentCallable);
    umbilical.verifyTaskSuccessEvent();
  } finally {
    executor.shutdownNow();
  }
}
 
Example #18
Source File: TestTaskSchedulerManager.java    From tez with Apache License 2.0 5 votes vote down vote up
@Override
TaskScheduler createCustomTaskScheduler(TaskSchedulerContext taskSchedulerContext,
                                        NamedEntityDescriptor taskSchedulerDescriptor, int schedulerId)
                                            throws TezException {
  taskSchedulerContexts.add(taskSchedulerContext);
  TaskScheduler taskScheduler = spy(super.createCustomTaskScheduler(taskSchedulerContext, taskSchedulerDescriptor, schedulerId));
  testTaskSchedulers.add(taskScheduler);
  return taskScheduler;
}
 
Example #19
Source File: TestTaskExecution.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
@Test
public void testHeartbeatShouldDie() throws IOException, InterruptedException, TezException,
    ExecutionException {

  ListeningExecutorService executor = null;
  try {
    ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
    executor = MoreExecutors.listeningDecorator(rawExecutor);
    ApplicationId appId = ApplicationId.newInstance(10000, 1);
    TezTaskUmbilicalForTest umbilical = new TezTaskUmbilicalForTest();
    TaskReporter taskReporter = createTaskReporter(appId, umbilical);

    TezTaskRunner taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor,
        TestProcessor.CONF_EMPTY);
    // Setup the executor
    Future<Boolean> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable(taskRunner));
    // Signal the processor to go through
    TestProcessor.awaitStart();
    umbilical.signalSendShouldDie();
    umbilical.awaitRegisteredEvent();
    // Not signaling an actual start to verify task interruption

    boolean result = taskRunnerFuture.get();
    assertFalse(result);

    TestProcessor.awaitCompletion();
    assertTrue(TestProcessor.wasInterrupted());
    assertNull(taskReporter.currentCallable);
    // TODO Is this statement correct ?
    // No completion events since shouldDie was requested by the AM, which should have killed the
    // task.
    umbilical.verifyNoCompletionEvents();
  } finally {
    executor.shutdownNow();
  }
}
 
Example #20
Source File: TezClient.java    From tez with Apache License 2.0 5 votes vote down vote up
@Private
static DAGClient getDAGClient(ApplicationId appId, TezConfiguration tezConf, YarnConfiguration
    yarnConf, FrameworkClient frameworkClient, UserGroupInformation ugi)
    throws IOException, TezException {
  return new DAGClientImpl(appId, getDefaultTezDAGID(appId), tezConf,
      yarnConf, frameworkClient, ugi);
}
 
Example #21
Source File: TaskReporter.java    From tez with Apache License 2.0 5 votes vote down vote up
@Override
public boolean taskKilled(TezTaskAttemptID taskAttemptID, Throwable t, String diagnostics,
                          EventMetaData srcMeta) throws IOException, TezException {
  if(!isShuttingDown()) {
    return currentCallable.taskTerminated(taskAttemptID, true, null, t, diagnostics, srcMeta);
  }
  return false;
}
 
Example #22
Source File: TestMRRJobsDAGApi.java    From tez with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 120000)
public void testMultipleMRRSleepJobViaSession() throws IOException,
InterruptedException, TezException, ClassNotFoundException, YarnException {
  Path remoteStagingDir = remoteFs.makeQualified(new Path("/tmp", String
      .valueOf(new Random().nextInt(100000))));
  remoteFs.mkdirs(remoteStagingDir);
  TezConfiguration tezConf = new TezConfiguration(
      mrrTezCluster.getConfig());
  tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR,
      remoteStagingDir.toString());

  TezClient tezSession = TezClient.create("testsession", tezConf, true);
  tezSession.start();
  Assert.assertEquals(TezAppMasterStatus.INITIALIZING,
      tezSession.getAppMasterStatus());

  State finalState = testMRRSleepJobDagSubmitCore(true, false, false,
      tezSession, false, null, null);
  Assert.assertEquals(DAGStatus.State.SUCCEEDED, finalState);
  Assert.assertEquals(TezAppMasterStatus.READY,
      tezSession.getAppMasterStatus());
  finalState = testMRRSleepJobDagSubmitCore(true, false, false,
      tezSession, false, null, null);
  Assert.assertEquals(DAGStatus.State.SUCCEEDED, finalState);
  Assert.assertEquals(TezAppMasterStatus.READY,
      tezSession.getAppMasterStatus());

  stopAndVerifyYarnApp(tezSession);
}
 
Example #23
Source File: ContainerLauncherManager.java    From tez with Apache License 2.0 5 votes vote down vote up
public ContainerLauncherManager(AppContext context,
                                TaskCommunicatorManagerInterface taskCommunicatorManagerInterface,
                                String workingDirectory,
                                List<NamedEntityDescriptor> containerLauncherDescriptors,
                                boolean isLocalMode) throws TezException {
  super(ContainerLauncherManager.class.getName());

  this.isIncompleteCtor = false;
  this.appContext = context;
  if (containerLauncherDescriptors == null || containerLauncherDescriptors.isEmpty()) {
    throw new IllegalArgumentException("ContainerLauncherDescriptors must be specified");
  }
  containerLauncherContexts = new ContainerLauncherContext[containerLauncherDescriptors.size()];
  containerLaunchers = new ContainerLauncherWrapper[containerLauncherDescriptors.size()];
  containerLauncherServiceWrappers = new ServicePluginLifecycleAbstractService[containerLauncherDescriptors.size()];


  for (int i = 0; i < containerLauncherDescriptors.size(); i++) {
    UserPayload userPayload = containerLauncherDescriptors.get(i).getUserPayload();
    ContainerLauncherContext containerLauncherContext =
        new ContainerLauncherContextImpl(context, this, taskCommunicatorManagerInterface, userPayload, i);
    containerLauncherContexts[i] = containerLauncherContext;
    containerLaunchers[i] = new ContainerLauncherWrapper(createContainerLauncher(containerLauncherDescriptors.get(i), context,
        containerLauncherContext, taskCommunicatorManagerInterface, workingDirectory, i, isLocalMode));
    containerLauncherServiceWrappers[i] = new ServicePluginLifecycleAbstractService<>(containerLaunchers[i].getContainerLauncher());
  }
}
 
Example #24
Source File: TaskExecutionTestHelpers.java    From tez with Apache License 2.0 5 votes vote down vote up
@Override
public TezHeartbeatResponse heartbeat(TezHeartbeatRequest request) throws IOException,
    TezException {
  umbilicalLock.lock();
  if (request.getEvents() != null) {
    requestEvents.addAll(request.getEvents());
  }
  try {
    if (shouldThrowException) {
      LOG.info("TestUmbilical throwing Exception");
      throw new IOException(HEARTBEAT_EXCEPTION_STRING);
    }
    TezHeartbeatResponse response = new TezHeartbeatResponse();
    response.setLastRequestId(request.getRequestId());
    if (shouldSendDieSignal) {
      LOG.info("TestUmbilical returning shouldDie=true");
      response.setShouldDie();
    }
    return response;
  } finally {
    if (pendingEvent) {
      eventEnacted = true;
      LOG.info("Signalling Event");
      eventCondition.signal();
    }
    umbilicalLock.unlock();
  }
}
 
Example #25
Source File: DAGClientAMProtocolBlockingPBServerImpl.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
@Override
public DAGClientAMProtocolRPC.PreWarmResponseProto preWarm(
  RpcController controller,
  PreWarmRequestProto request) throws ServiceException {
  try {
    real.preWarmContainers(
      DagTypeConverters.convertPreWarmContextFromProto(
        request.getPreWarmContext()));
    return PreWarmResponseProto.newBuilder().build();
  } catch (TezException e) {
    throw wrapException(e);
  }
}
 
Example #26
Source File: TestTaskExecution2.java    From tez with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 5000)
public void testSignalDeprecatedFatalErrorAndLoop() throws IOException, InterruptedException, TezException,
    ExecutionException {

  ListeningExecutorService executor = null;
  try {
    ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
    executor = MoreExecutors.listeningDecorator(rawExecutor);
    ApplicationId appId = ApplicationId.newInstance(10000, 1);
    TaskExecutionTestHelpers.TezTaskUmbilicalForTest
        umbilical = new TaskExecutionTestHelpers.TezTaskUmbilicalForTest();
    TaskReporter taskReporter = createTaskReporter(appId, umbilical);

    TezTaskRunner2 taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor,
        TestProcessor.CONF_SIGNAL_DEPRECATEDFATAL_AND_LOOP);
    // Setup the executor
    Future<TaskRunner2Result> taskRunnerFuture =
        taskExecutor.submit(new TaskRunnerCallable2ForTest(taskRunner));
    // Signal the processor to go through
    TestProcessor.awaitStart();
    TestProcessor.signal();

    TestProcessor.awaitLoop();
    // The fatal error should have caused an interrupt.

    TaskRunner2Result result = taskRunnerFuture.get();
    verifyTaskRunnerResult(result, EndReason.TASK_ERROR, createProcessorIOException(), false, TaskFailureType.NON_FATAL);

    TestProcessor.awaitCompletion();
    assertTrue(TestProcessor.wasInterrupted());
    assertNull(taskReporter.currentCallable);
    umbilical.verifyTaskFailedEvent(
        FAILURE_START_STRING,
        IOException.class.getName() + ": " + IOException.class.getSimpleName());
    // Signal fatal error should cause the processor to fail.
    assertTrue(TestProcessor.wasAborted());
  } finally {
    executor.shutdownNow();
  }
}
 
Example #27
Source File: TestTaskCommunicatorManager.java    From tez with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 5000)
public void testCustomTaskCommSpecified() throws IOException, TezException {

  AppContext appContext = mock(AppContext.class);
  TaskHeartbeatHandler thh = mock(TaskHeartbeatHandler.class);
  ContainerHeartbeatHandler chh = mock(ContainerHeartbeatHandler.class);

  String customTaskCommName = "customTaskComm";
  List<NamedEntityDescriptor> taskCommDescriptors = new LinkedList<>();
  ByteBuffer bb = ByteBuffer.allocate(4);
  bb.putInt(0, 3);
  UserPayload customPayload = UserPayload.create(bb);
  taskCommDescriptors.add(
      new NamedEntityDescriptor(customTaskCommName, FakeTaskComm.class.getName())
          .setUserPayload(customPayload));

  TaskCommManagerForMultipleCommTest tcm =
      new TaskCommManagerForMultipleCommTest(appContext, thh, chh, taskCommDescriptors);

  try {
    tcm.init(new Configuration(false));
    tcm.start();

    assertEquals(1, tcm.getNumTaskComms());
    assertFalse(tcm.getYarnTaskCommCreated());
    assertFalse(tcm.getUberTaskCommCreated());

    assertEquals(customTaskCommName, tcm.getTaskCommName(0));
    assertEquals(bb, tcm.getTaskCommContext(0).getInitialUserPayload().getPayload());

  } finally {
    tcm.stop();
  }
}
 
Example #28
Source File: RPCUtil.java    From tez with Apache License 2.0 5 votes vote down vote up
/**
 * Utility method that unwraps and returns appropriate exceptions.
 *
 * @param se
 *          ServiceException
 * @return An instance of the actual exception, which will be a subclass of
 *         {@link TezException} or {@link IOException}
 */
public static Void unwrapAndThrowNonIOException(ServiceException se)
    throws TezException {
  try {
    return unwrapAndThrowException(se);
  } catch (IOException ioe) {
    throw new TezException(ioe);
  }
}
 
Example #29
Source File: TestTaskExecution2.java    From tez with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 5000)
public void testSignalFatalAndThrow() throws IOException, InterruptedException, TezException,
    ExecutionException {

  ListeningExecutorService executor = null;
  try {
    ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
    executor = MoreExecutors.listeningDecorator(rawExecutor);
    ApplicationId appId = ApplicationId.newInstance(10000, 1);
    TaskExecutionTestHelpers.TezTaskUmbilicalForTest
        umbilical = new TaskExecutionTestHelpers.TezTaskUmbilicalForTest();
    TaskReporter taskReporter = createTaskReporter(appId, umbilical);

    TezTaskRunner2 taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor,
        TestProcessor.CONF_SIGNAL_FATAL_AND_THROW);
    // Setup the executor
    Future<TaskRunner2Result> taskRunnerFuture =
        taskExecutor.submit(new TaskRunnerCallable2ForTest(taskRunner));
    // Signal the processor to go through
    TestProcessor.awaitStart();
    TestProcessor.signal();

    TaskRunner2Result result = taskRunnerFuture.get();
    verifyTaskRunnerResult(result, EndReason.TASK_ERROR, createProcessorIOException(), false, TaskFailureType.FATAL);

    TestProcessor.awaitCompletion();
    assertNull(taskReporter.currentCallable);
    umbilical.verifyTaskFailedEvent(
        FAILURE_START_STRING,
        IOException.class.getName() + ": " + IOException.class.getSimpleName(), TaskFailureType.FATAL);
    assertTrue(TestProcessor.wasAborted());
  } finally {
    executor.shutdownNow();
  }
}
 
Example #30
Source File: SimpleHistoryParser.java    From tez with Apache License 2.0 5 votes vote down vote up
protected void parse(String dagId, JSONObjectSource source)
    throws JSONException, TezException, IOException {
  Map<String, JSONObject> vertexJsonMap = Maps.newHashMap();
  Map<String, JSONObject> taskJsonMap = Maps.newHashMap();
  Map<String, JSONObject> attemptJsonMap = Maps.newHashMap();

  readEventsFromSource(dagId, source, vertexJsonMap, taskJsonMap, attemptJsonMap);
  postProcessMaps(vertexJsonMap, taskJsonMap, attemptJsonMap);
}