org.apache.tez.dag.api.TezException Java Examples
The following examples show how to use
org.apache.tez.dag.api.TezException.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: DAGAppMaster.java From tez with Apache License 2.0 | 6 votes |
private List<URL> processAdditionalResources(TezDAGID dagId, Map<String, LocalResource> lrDiff) throws TezException { if (lrDiff == null || lrDiff.isEmpty()) { return Collections.emptyList(); } else { LOG.info("Localizing additional local resources for AM : " + lrDiff); List<URL> downloadedURLs; try { TezUtilsInternal.setHadoopCallerContext(hadoopShim, dagId); downloadedURLs = RelocalizationUtils.processAdditionalResources( Maps.transformValues(lrDiff, new Function<LocalResource, URI>() { @Override public URI apply(LocalResource input) { return getLocalResourceUri(input); } }), getConfig(), workingDirectory); } catch (IOException e) { throw new TezException(e); } finally { hadoopShim.clearHadoopCallerContext(); } LOG.info("Done downloading additional AM resources"); return downloadedURLs; } }
Example #2
Source File: Edge.java From tez with Apache License 2.0 | 6 votes |
public void setEdgeProperty(EdgeProperty newEdgeProperty) throws AMUserCodeException { boolean wasUnInitialized; synchronized (this) { this.edgeProperty = newEdgeProperty; wasUnInitialized = (edgeManager == null); try { createEdgeManager(); } catch (TezException e) { throw new AMUserCodeException(Source.EdgeManager, e); } } initialize(); if (wasUnInitialized) { sendEvent(new VertexEventNullEdgeInitialized(sourceVertex.getVertexId(), this, destinationVertex)); sendEvent(new VertexEventNullEdgeInitialized(destinationVertex.getVertexId(), this, sourceVertex)); } }
Example #3
Source File: DAGAppMaster.java From tez with Apache License 2.0 | 6 votes |
private long checkAndHandleDAGClientTimeout() throws TezException { if (EnumSet.of(DAGAppMasterState.NEW, DAGAppMasterState.RECOVERING).contains(this.state) || sessionStopped.get()) { // AM new or recovering so do not kill session at this time // if session already completed or shutting down, this should be a a no-op return -1; } long currentTime = clock.getTime(); long nextExpiry = clientHandler.getLastHeartbeatTime() + clientAMHeartbeatTimeoutIntervalMillis; if (currentTime < nextExpiry) { // reschedule timer to 1 sec after the next expiry window // to ensure that we time out as intended if there are no heartbeats return ((nextExpiry+1000) - currentTime); } String message = "Client-to-AM Heartbeat timeout interval expired, shutting down AM as client" + " stopped heartbeating to it" + ", lastClientAMHeartbeatTime=" + clientHandler.getLastHeartbeatTime() + ", clientAMHeartbeatTimeoutIntervalMillis=" + clientAMHeartbeatTimeoutIntervalMillis + " ms"; addDiagnostic(message); shutdownTezAM(message); return -1; }
Example #4
Source File: DAGClientTimelineImpl.java From tez with Apache License 2.0 | 6 votes |
@Override public DAGStatus getDAGStatus(@Nullable Set<StatusGetOpts> statusOptions) throws IOException, TezException { final String url = String.format("%s/%s/%s?fields=%s", baseUri, ATSConstants.TEZ_DAG_ID, dagId, FILTER_BY_FIELDS); try { DAGStatusProto.Builder statusBuilder; final JSONObject jsonRoot = getJsonRootEntity(url); statusBuilder = parseDagStatus(jsonRoot, statusOptions); if (statusBuilder == null) { throw new TezException("Failed to get DagStatus from ATS"); } return new DAGStatus(statusBuilder, DagStatusSource.TIMELINE); } catch (JSONException je) { throw new TezException("Failed to parse DagStatus json from YARN Timeline", je); } }
Example #5
Source File: ContainerLauncherManager.java From tez with Apache License 2.0 | 6 votes |
@VisibleForTesting ContainerLauncher createContainerLauncher( NamedEntityDescriptor containerLauncherDescriptor, AppContext context, ContainerLauncherContext containerLauncherContext, TaskCommunicatorManagerInterface taskCommunicatorManagerInterface, String workingDirectory, int containerLauncherIndex, boolean isPureLocalMode) throws TezException { if (containerLauncherDescriptor.getEntityName().equals( TezConstants.getTezYarnServicePluginName())) { return createYarnContainerLauncher(containerLauncherContext); } else if (containerLauncherDescriptor.getEntityName() .equals(TezConstants.getTezUberServicePluginName())) { return createUberContainerLauncher(containerLauncherContext, context, taskCommunicatorManagerInterface, workingDirectory, isPureLocalMode); } else { return createCustomContainerLauncher(containerLauncherContext, containerLauncherDescriptor); } }
Example #6
Source File: DAGClientRPCImpl.java From tez with Apache License 2.0 | 6 votes |
DAGStatus getDAGStatusViaAM(Set<StatusGetOpts> statusOptions, long timeout) throws IOException, TezException { if(LOG.isDebugEnabled()) { LOG.debug("GetDAGStatus via AM for app: " + appId + " dag:" + dagId); } GetDAGStatusRequestProto.Builder requestProtoBuilder = GetDAGStatusRequestProto.newBuilder() .setDagId(dagId).setTimeout(timeout); if (statusOptions != null) { requestProtoBuilder.addAllStatusOptions( DagTypeConverters.convertStatusGetOptsToProto(statusOptions)); } try { return new DAGStatus( proxy.getDAGStatus(null, requestProtoBuilder.build()).getDagStatus(), DagStatusSource.AM); } catch (ServiceException e) { RPCUtil.unwrapAndThrowException(e); // Should not reach here throw new TezException(e); } }
Example #7
Source File: TaskSchedulerManager.java From tez with Apache License 2.0 | 6 votes |
@VisibleForTesting TaskScheduler createTaskScheduler(String host, int port, String trackingUrl, AppContext appContext, NamedEntityDescriptor taskSchedulerDescriptor, long customAppIdIdentifier, int schedulerId) throws TezException { TaskSchedulerContext rawContext = new TaskSchedulerContextImpl(this, appContext, schedulerId, trackingUrl, customAppIdIdentifier, host, port, taskSchedulerDescriptor.getUserPayload()); TaskSchedulerContext wrappedContext = wrapTaskSchedulerContext(rawContext); String schedulerName = taskSchedulerDescriptor.getEntityName(); if (schedulerName.equals(TezConstants.getTezYarnServicePluginName())) { return createYarnTaskScheduler(wrappedContext, schedulerId); } else if (schedulerName.equals(TezConstants.getTezUberServicePluginName())) { return createUberTaskScheduler(wrappedContext, schedulerId); } else { return createCustomTaskScheduler(wrappedContext, taskSchedulerDescriptor, schedulerId); } }
Example #8
Source File: TestTaskErrorsUsingLocalMode.java From tez with Apache License 2.0 | 6 votes |
@Test(timeout = 20000) public void testFatalErrorReported() throws IOException, TezException, InterruptedException { TezClient tezClient = getTezClient("testFatalErrorReported"); DAGClient dagClient = null; try { FailingProcessor.configureForFatalFail(); DAG dag = DAG.create("testFatalErrorReportedDag").addVertex( Vertex .create(VERTEX_NAME, ProcessorDescriptor.create(FailingProcessor.class.getName()), 1)); dagClient = tezClient.submitDAG(dag); dagClient.waitForCompletion(); assertEquals(DAGStatus.State.FAILED, dagClient.getDAGStatus(null).getState()); assertEquals(1, dagClient.getVertexStatus(VERTEX_NAME, null).getProgress().getFailedTaskAttemptCount()); } finally { if (dagClient != null) { dagClient.close(); } tezClient.stop(); } }
Example #9
Source File: DAGAppMaster.java From incubator-tez with Apache License 2.0 | 6 votes |
private Map<String, LocalResource> getAdditionalLocalResourceDiff( DAG dag, Map<String, LocalResource> additionalResources) throws TezException { if (additionalResources == null) { return Collections.emptyMap(); } // Check for existing resources. Iterator<Entry<String, LocalResource>> lrIter = additionalResources.entrySet().iterator(); while (lrIter.hasNext()) { Entry<String, LocalResource> lrEntry = lrIter.next(); LocalResource existing = amResources.get(lrEntry.getKey()); if (existing != null) { if (!isSameFile(dag, lrEntry.getKey(), existing, lrEntry.getValue())) { throw new TezUncheckedException( "Cannot add different additional resources with the same name : " + lrEntry.getKey() + ", Existing: [" + existing + "], New: [" + lrEntry.getValue() + "]"); } else { lrIter.remove(); } } } return containerSignatureMatcher.getAdditionalResources(amResources, additionalResources); }
Example #10
Source File: DAGClientAMProtocolBlockingPBServerImpl.java From tez with Apache License 2.0 | 6 votes |
@Override public TryKillDAGResponseProto tryKillDAG(RpcController controller, TryKillDAGRequestProto request) throws ServiceException { UserGroupInformation user = getRPCUser(); try { String dagId = request.getDagId(); if (!real.getACLManager(dagId).checkDAGModifyAccess(user)) { throw new AccessControlException("User " + user + " cannot perform DAG modify operation"); } real.updateLastHeartbeatTime(); real.tryKillDAG(dagId); return TryKillDAGResponseProto.newBuilder().build(); } catch (TezException e) { throw wrapException(e); } }
Example #11
Source File: TestHistoryEventsProtoConversion.java From tez with Apache License 2.0 | 6 votes |
private HistoryEvent testProtoConversion(HistoryEvent event) throws IOException, TezException { ByteArrayOutputStream os = new ByteArrayOutputStream(); HistoryEvent deserializedEvent = null; CodedOutputStream codedOutputStream = CodedOutputStream.newInstance(os); event.toProtoStream(codedOutputStream); codedOutputStream.flush(); os.flush(); os.close(); deserializedEvent = ReflectionUtils.createClazzInstance( event.getClass().getName()); LOG.info("Serialized event to byte array" + ", eventType=" + event.getEventType() + ", bufLen=" + os.toByteArray().length); deserializedEvent.fromProtoStream( CodedInputStream.newInstance(os.toByteArray())); return deserializedEvent; }
Example #12
Source File: TimelineReaderFactory.java From tez with Apache License 2.0 | 6 votes |
public static TimelineReaderStrategy getTimelineReaderStrategy(Configuration conf, boolean useHttps, int connTimeout) throws TezException { TimelineReaderStrategy timelineReaderStrategy; if (!isTimelineClientSupported()) { throw new TezException("Reading from timeline is not supported." + " token delegation support: " + tokenDelegationSupported() + ", is secure timeline: " + UserGroupInformation.isSecurityEnabled()); } timelineReaderStrategy = getTimelineReaderStrategy(tokenDelegationSupported(), conf, useHttps, connTimeout); if (LOG.isDebugEnabled()) { LOG.debug("Using " + timelineReaderStrategy.getClass().getName() + " to read timeline data"); } return timelineReaderStrategy; }
Example #13
Source File: TestHistoryParser.java From tez with Apache License 2.0 | 6 votes |
private DagInfo getDagInfoFromSimpleHistory(String dagId) throws TezException, IOException { TezDAGID tezDAGID = TezDAGID.fromString(dagId); ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(tezDAGID .getApplicationId(), 1); Path historyPath = new Path(conf.get("fs.defaultFS") + SIMPLE_HISTORY_DIR + HISTORY_TXT + "." + applicationAttemptId); FileSystem fs = historyPath.getFileSystem(conf); Path localPath = new Path(DOWNLOAD_DIR, HISTORY_TXT); fs.copyToLocalFile(historyPath, localPath); File localFile = new File(DOWNLOAD_DIR, HISTORY_TXT); //Now parse via SimpleHistory SimpleHistoryParser parser = new SimpleHistoryParser(Arrays.asList(localFile)); DagInfo dagInfo = parser.getDAGData(dagId); assertTrue(dagInfo.getDagId().equals(dagId)); return dagInfo; }
Example #14
Source File: TezChild.java From tez with Apache License 2.0 | 6 votes |
public static TezChild newTezChild(Configuration conf, String host, int port, String containerIdentifier, String tokenIdentifier, int attemptNumber, String[] localDirs, String workingDirectory, Map<String, String> serviceProviderEnvMap, @Nullable String pid, ExecutionContext executionContext, Credentials credentials, long memAvailable, String user, TezTaskUmbilicalProtocol tezUmbilical, boolean updateSysCounters, HadoopShim hadoopShim) throws IOException, InterruptedException, TezException { // Pull in configuration specified for the session. // TODO TEZ-1233. This needs to be moved over the wire rather than localizing the file // for each and every task, and reading it back from disk. Also needs to be per vertex. Limits.setConfiguration(conf); TezUtilsInternal.setSecurityUtilConfigration(LOG, conf); // singleton of ObjectRegistry for this JVM ObjectRegistryImpl objectRegistry = new ObjectRegistryImpl(); return new TezChild(conf, host, port, containerIdentifier, tokenIdentifier, attemptNumber, workingDirectory, localDirs, serviceProviderEnvMap, objectRegistry, pid, executionContext, credentials, memAvailable, user, tezUmbilical, updateSysCounters, hadoopShim); }
Example #15
Source File: TezGroupedSplitsInputFormat.java From tez with Apache License 2.0 | 5 votes |
@Override public RecordReader<K, V> getRecordReader(InputSplit split, JobConf job, Reporter reporter) throws IOException { TezGroupedSplit groupedSplit = (TezGroupedSplit) split; try { initInputFormatFromSplit(groupedSplit); } catch (TezException e) { throw new IOException(e); } return new TezGroupedSplitsRecordReader(groupedSplit, job, reporter); }
Example #16
Source File: TestTezJobs.java From tez with Apache License 2.0 | 5 votes |
@Test(timeout = 60000) public void testInvalidQueueSubmission() throws Exception { TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig()); YarnClient yarnClient = YarnClient.createYarnClient(); try { yarnClient.init(mrrTezCluster.getConfig()); yarnClient.start(); SimpleSessionExample job = new SimpleSessionExample(); tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, false); tezConf.set(TezConfiguration.TEZ_QUEUE_NAME, "nonexistent"); String[] inputPaths = new String[1]; String[] outputPaths = new String[1]; String inputDirStr = "/tmp/owc-input"; inputPaths[0] = inputDirStr; Path inputDir = new Path(inputDirStr); remoteFs.mkdirs(inputDir); String outputDirStr = "/tmp/owc-output"; outputPaths[0] = outputDirStr; int result = job.run(tezConf, new String[] { StringUtils.join(",", inputPaths), StringUtils.join(",", outputPaths), "2" }, null); Assert.assertTrue("Job should have failed", result != 0); } catch (TezException e) { Assert.assertTrue(e.getMessage().contains("Failed to submit application")); } finally { if (yarnClient != null) { yarnClient.stop(); } } }
Example #17
Source File: TestTaskExecution2.java From tez with Apache License 2.0 | 5 votes |
@Test(timeout = 5000) public void testKilledAfterComplete() throws IOException, InterruptedException, TezException, ExecutionException { ListeningExecutorService executor = null; try { ExecutorService rawExecutor = Executors.newFixedThreadPool(1); executor = MoreExecutors.listeningDecorator(rawExecutor); ApplicationId appId = ApplicationId.newInstance(10000, 1); TaskExecutionTestHelpers.TezTaskUmbilicalForTest umbilical = new TaskExecutionTestHelpers.TezTaskUmbilicalForTest(); TaskReporter taskReporter = createTaskReporter(appId, umbilical); TezTaskRunner2ForTest taskRunner = createTaskRunnerForTest(appId, umbilical, taskReporter, executor, TestProcessor.CONF_EMPTY); // Setup the executor Future<TaskRunner2Result> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable2ForTest(taskRunner)); // Signal the processor to go through TestProcessor.awaitStart(); TestProcessor.signal(); TestProcessor.awaitCompletion(); taskRunner.awaitCallableCompletion(); taskRunner.killTask(); TaskRunner2Result result = taskRunnerFuture.get(); verifyTaskRunnerResult(result, EndReason.SUCCESS, null, false, null); assertFalse(TestProcessor.wasInterrupted()); assertNull(taskReporter.currentCallable); umbilical.verifyTaskSuccessEvent(); } finally { executor.shutdownNow(); } }
Example #18
Source File: TestTaskSchedulerManager.java From tez with Apache License 2.0 | 5 votes |
@Override TaskScheduler createCustomTaskScheduler(TaskSchedulerContext taskSchedulerContext, NamedEntityDescriptor taskSchedulerDescriptor, int schedulerId) throws TezException { taskSchedulerContexts.add(taskSchedulerContext); TaskScheduler taskScheduler = spy(super.createCustomTaskScheduler(taskSchedulerContext, taskSchedulerDescriptor, schedulerId)); testTaskSchedulers.add(taskScheduler); return taskScheduler; }
Example #19
Source File: TestTaskExecution.java From incubator-tez with Apache License 2.0 | 5 votes |
@Test public void testHeartbeatShouldDie() throws IOException, InterruptedException, TezException, ExecutionException { ListeningExecutorService executor = null; try { ExecutorService rawExecutor = Executors.newFixedThreadPool(1); executor = MoreExecutors.listeningDecorator(rawExecutor); ApplicationId appId = ApplicationId.newInstance(10000, 1); TezTaskUmbilicalForTest umbilical = new TezTaskUmbilicalForTest(); TaskReporter taskReporter = createTaskReporter(appId, umbilical); TezTaskRunner taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor, TestProcessor.CONF_EMPTY); // Setup the executor Future<Boolean> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable(taskRunner)); // Signal the processor to go through TestProcessor.awaitStart(); umbilical.signalSendShouldDie(); umbilical.awaitRegisteredEvent(); // Not signaling an actual start to verify task interruption boolean result = taskRunnerFuture.get(); assertFalse(result); TestProcessor.awaitCompletion(); assertTrue(TestProcessor.wasInterrupted()); assertNull(taskReporter.currentCallable); // TODO Is this statement correct ? // No completion events since shouldDie was requested by the AM, which should have killed the // task. umbilical.verifyNoCompletionEvents(); } finally { executor.shutdownNow(); } }
Example #20
Source File: TezClient.java From tez with Apache License 2.0 | 5 votes |
@Private static DAGClient getDAGClient(ApplicationId appId, TezConfiguration tezConf, YarnConfiguration yarnConf, FrameworkClient frameworkClient, UserGroupInformation ugi) throws IOException, TezException { return new DAGClientImpl(appId, getDefaultTezDAGID(appId), tezConf, yarnConf, frameworkClient, ugi); }
Example #21
Source File: TaskReporter.java From tez with Apache License 2.0 | 5 votes |
@Override public boolean taskKilled(TezTaskAttemptID taskAttemptID, Throwable t, String diagnostics, EventMetaData srcMeta) throws IOException, TezException { if(!isShuttingDown()) { return currentCallable.taskTerminated(taskAttemptID, true, null, t, diagnostics, srcMeta); } return false; }
Example #22
Source File: TestMRRJobsDAGApi.java From tez with Apache License 2.0 | 5 votes |
@Test(timeout = 120000) public void testMultipleMRRSleepJobViaSession() throws IOException, InterruptedException, TezException, ClassNotFoundException, YarnException { Path remoteStagingDir = remoteFs.makeQualified(new Path("/tmp", String .valueOf(new Random().nextInt(100000)))); remoteFs.mkdirs(remoteStagingDir); TezConfiguration tezConf = new TezConfiguration( mrrTezCluster.getConfig()); tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString()); TezClient tezSession = TezClient.create("testsession", tezConf, true); tezSession.start(); Assert.assertEquals(TezAppMasterStatus.INITIALIZING, tezSession.getAppMasterStatus()); State finalState = testMRRSleepJobDagSubmitCore(true, false, false, tezSession, false, null, null); Assert.assertEquals(DAGStatus.State.SUCCEEDED, finalState); Assert.assertEquals(TezAppMasterStatus.READY, tezSession.getAppMasterStatus()); finalState = testMRRSleepJobDagSubmitCore(true, false, false, tezSession, false, null, null); Assert.assertEquals(DAGStatus.State.SUCCEEDED, finalState); Assert.assertEquals(TezAppMasterStatus.READY, tezSession.getAppMasterStatus()); stopAndVerifyYarnApp(tezSession); }
Example #23
Source File: ContainerLauncherManager.java From tez with Apache License 2.0 | 5 votes |
public ContainerLauncherManager(AppContext context, TaskCommunicatorManagerInterface taskCommunicatorManagerInterface, String workingDirectory, List<NamedEntityDescriptor> containerLauncherDescriptors, boolean isLocalMode) throws TezException { super(ContainerLauncherManager.class.getName()); this.isIncompleteCtor = false; this.appContext = context; if (containerLauncherDescriptors == null || containerLauncherDescriptors.isEmpty()) { throw new IllegalArgumentException("ContainerLauncherDescriptors must be specified"); } containerLauncherContexts = new ContainerLauncherContext[containerLauncherDescriptors.size()]; containerLaunchers = new ContainerLauncherWrapper[containerLauncherDescriptors.size()]; containerLauncherServiceWrappers = new ServicePluginLifecycleAbstractService[containerLauncherDescriptors.size()]; for (int i = 0; i < containerLauncherDescriptors.size(); i++) { UserPayload userPayload = containerLauncherDescriptors.get(i).getUserPayload(); ContainerLauncherContext containerLauncherContext = new ContainerLauncherContextImpl(context, this, taskCommunicatorManagerInterface, userPayload, i); containerLauncherContexts[i] = containerLauncherContext; containerLaunchers[i] = new ContainerLauncherWrapper(createContainerLauncher(containerLauncherDescriptors.get(i), context, containerLauncherContext, taskCommunicatorManagerInterface, workingDirectory, i, isLocalMode)); containerLauncherServiceWrappers[i] = new ServicePluginLifecycleAbstractService<>(containerLaunchers[i].getContainerLauncher()); } }
Example #24
Source File: TaskExecutionTestHelpers.java From tez with Apache License 2.0 | 5 votes |
@Override public TezHeartbeatResponse heartbeat(TezHeartbeatRequest request) throws IOException, TezException { umbilicalLock.lock(); if (request.getEvents() != null) { requestEvents.addAll(request.getEvents()); } try { if (shouldThrowException) { LOG.info("TestUmbilical throwing Exception"); throw new IOException(HEARTBEAT_EXCEPTION_STRING); } TezHeartbeatResponse response = new TezHeartbeatResponse(); response.setLastRequestId(request.getRequestId()); if (shouldSendDieSignal) { LOG.info("TestUmbilical returning shouldDie=true"); response.setShouldDie(); } return response; } finally { if (pendingEvent) { eventEnacted = true; LOG.info("Signalling Event"); eventCondition.signal(); } umbilicalLock.unlock(); } }
Example #25
Source File: DAGClientAMProtocolBlockingPBServerImpl.java From incubator-tez with Apache License 2.0 | 5 votes |
@Override public DAGClientAMProtocolRPC.PreWarmResponseProto preWarm( RpcController controller, PreWarmRequestProto request) throws ServiceException { try { real.preWarmContainers( DagTypeConverters.convertPreWarmContextFromProto( request.getPreWarmContext())); return PreWarmResponseProto.newBuilder().build(); } catch (TezException e) { throw wrapException(e); } }
Example #26
Source File: TestTaskExecution2.java From tez with Apache License 2.0 | 5 votes |
@Test(timeout = 5000) public void testSignalDeprecatedFatalErrorAndLoop() throws IOException, InterruptedException, TezException, ExecutionException { ListeningExecutorService executor = null; try { ExecutorService rawExecutor = Executors.newFixedThreadPool(1); executor = MoreExecutors.listeningDecorator(rawExecutor); ApplicationId appId = ApplicationId.newInstance(10000, 1); TaskExecutionTestHelpers.TezTaskUmbilicalForTest umbilical = new TaskExecutionTestHelpers.TezTaskUmbilicalForTest(); TaskReporter taskReporter = createTaskReporter(appId, umbilical); TezTaskRunner2 taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor, TestProcessor.CONF_SIGNAL_DEPRECATEDFATAL_AND_LOOP); // Setup the executor Future<TaskRunner2Result> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable2ForTest(taskRunner)); // Signal the processor to go through TestProcessor.awaitStart(); TestProcessor.signal(); TestProcessor.awaitLoop(); // The fatal error should have caused an interrupt. TaskRunner2Result result = taskRunnerFuture.get(); verifyTaskRunnerResult(result, EndReason.TASK_ERROR, createProcessorIOException(), false, TaskFailureType.NON_FATAL); TestProcessor.awaitCompletion(); assertTrue(TestProcessor.wasInterrupted()); assertNull(taskReporter.currentCallable); umbilical.verifyTaskFailedEvent( FAILURE_START_STRING, IOException.class.getName() + ": " + IOException.class.getSimpleName()); // Signal fatal error should cause the processor to fail. assertTrue(TestProcessor.wasAborted()); } finally { executor.shutdownNow(); } }
Example #27
Source File: TestTaskCommunicatorManager.java From tez with Apache License 2.0 | 5 votes |
@Test(timeout = 5000) public void testCustomTaskCommSpecified() throws IOException, TezException { AppContext appContext = mock(AppContext.class); TaskHeartbeatHandler thh = mock(TaskHeartbeatHandler.class); ContainerHeartbeatHandler chh = mock(ContainerHeartbeatHandler.class); String customTaskCommName = "customTaskComm"; List<NamedEntityDescriptor> taskCommDescriptors = new LinkedList<>(); ByteBuffer bb = ByteBuffer.allocate(4); bb.putInt(0, 3); UserPayload customPayload = UserPayload.create(bb); taskCommDescriptors.add( new NamedEntityDescriptor(customTaskCommName, FakeTaskComm.class.getName()) .setUserPayload(customPayload)); TaskCommManagerForMultipleCommTest tcm = new TaskCommManagerForMultipleCommTest(appContext, thh, chh, taskCommDescriptors); try { tcm.init(new Configuration(false)); tcm.start(); assertEquals(1, tcm.getNumTaskComms()); assertFalse(tcm.getYarnTaskCommCreated()); assertFalse(tcm.getUberTaskCommCreated()); assertEquals(customTaskCommName, tcm.getTaskCommName(0)); assertEquals(bb, tcm.getTaskCommContext(0).getInitialUserPayload().getPayload()); } finally { tcm.stop(); } }
Example #28
Source File: RPCUtil.java From tez with Apache License 2.0 | 5 votes |
/** * Utility method that unwraps and returns appropriate exceptions. * * @param se * ServiceException * @return An instance of the actual exception, which will be a subclass of * {@link TezException} or {@link IOException} */ public static Void unwrapAndThrowNonIOException(ServiceException se) throws TezException { try { return unwrapAndThrowException(se); } catch (IOException ioe) { throw new TezException(ioe); } }
Example #29
Source File: TestTaskExecution2.java From tez with Apache License 2.0 | 5 votes |
@Test(timeout = 5000) public void testSignalFatalAndThrow() throws IOException, InterruptedException, TezException, ExecutionException { ListeningExecutorService executor = null; try { ExecutorService rawExecutor = Executors.newFixedThreadPool(1); executor = MoreExecutors.listeningDecorator(rawExecutor); ApplicationId appId = ApplicationId.newInstance(10000, 1); TaskExecutionTestHelpers.TezTaskUmbilicalForTest umbilical = new TaskExecutionTestHelpers.TezTaskUmbilicalForTest(); TaskReporter taskReporter = createTaskReporter(appId, umbilical); TezTaskRunner2 taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor, TestProcessor.CONF_SIGNAL_FATAL_AND_THROW); // Setup the executor Future<TaskRunner2Result> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable2ForTest(taskRunner)); // Signal the processor to go through TestProcessor.awaitStart(); TestProcessor.signal(); TaskRunner2Result result = taskRunnerFuture.get(); verifyTaskRunnerResult(result, EndReason.TASK_ERROR, createProcessorIOException(), false, TaskFailureType.FATAL); TestProcessor.awaitCompletion(); assertNull(taskReporter.currentCallable); umbilical.verifyTaskFailedEvent( FAILURE_START_STRING, IOException.class.getName() + ": " + IOException.class.getSimpleName(), TaskFailureType.FATAL); assertTrue(TestProcessor.wasAborted()); } finally { executor.shutdownNow(); } }
Example #30
Source File: SimpleHistoryParser.java From tez with Apache License 2.0 | 5 votes |
protected void parse(String dagId, JSONObjectSource source) throws JSONException, TezException, IOException { Map<String, JSONObject> vertexJsonMap = Maps.newHashMap(); Map<String, JSONObject> taskJsonMap = Maps.newHashMap(); Map<String, JSONObject> attemptJsonMap = Maps.newHashMap(); readEventsFromSource(dagId, source, vertexJsonMap, taskJsonMap, attemptJsonMap); postProcessMaps(vertexJsonMap, taskJsonMap, attemptJsonMap); }