org.apache.tez.dag.api.DAG Java Examples
The following examples show how to use
org.apache.tez.dag.api.DAG.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestAMRecovery.java From tez with Apache License 2.0 | 6 votes |
/** * Fine-grained recovery task-level, In a vertex (v1), task 0 is done task 1 * is not started. History flush happens. AM dies. Once AM is recovered, task 0 is * not re-run. Task 1 is re-run. (SCATTER_GATHER) * * @throws Exception */ @Test(timeout = 120000) public void testVertexPartiallyFinished_ScatterGather() throws Exception { DAG dag = createDAG("VertexPartiallyFinished_ScatterGather", ControlledShuffleVertexManager.class, DataMovementType.SCATTER_GATHER, true); TezCounters counters = runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); assertEquals(4, counters.findCounter(DAGCounter.NUM_SUCCEEDED_TASKS).getValue()); assertEquals(2, counters.findCounter(TestCounter.Counter_1).getValue()); List<HistoryEvent> historyEvents1 = readRecoveryLog(1); List<HistoryEvent> historyEvents2 = readRecoveryLog(2); printHistoryEvents(historyEvents1, 1); printHistoryEvents(historyEvents1, 2); // task_0 of v1 is finished in attempt 1, task_1 of v1 is not finished in // attempt 1 assertEquals(1, findTaskAttemptFinishedEvent(historyEvents1, 0, 0).size()); assertEquals(0, findTaskAttemptFinishedEvent(historyEvents1, 0, 1).size()); // task_0 of v1 is finished in attempt 1 and not rerun, task_1 of v1 is // finished in attempt 2 assertEquals(1, findTaskAttemptFinishedEvent(historyEvents2, 0, 0).size()); assertEquals(1, findTaskAttemptFinishedEvent(historyEvents2, 0, 1).size()); }
Example #2
Source File: BroadcastLoadGen.java From tez with Apache License 2.0 | 6 votes |
private DAG createDAG(int numGenTasks, int totalSourceDataSize, int numFetcherTasks) { int bytesPerSource = totalSourceDataSize / numGenTasks; LOG.info("DataPerSourceTask(bytes)=" + bytesPerSource); ByteBuffer payload = ByteBuffer.allocate(4); payload.putInt(0, bytesPerSource); Vertex broadcastVertex = Vertex.create("DataGen", ProcessorDescriptor.create(InputGenProcessor.class.getName()) .setUserPayload(UserPayload.create(payload)), numGenTasks); Vertex fetchVertex = Vertex.create("FetchVertex", ProcessorDescriptor.create(InputFetchProcessor.class.getName()), numFetcherTasks); UnorderedKVEdgeConfig edgeConf = UnorderedKVEdgeConfig.newBuilder(NullWritable.class .getName(), IntWritable.class.getName()).setCompression(false, null, null).build(); DAG dag = DAG.create("BroadcastLoadGen"); dag.addVertex(broadcastVertex).addVertex(fetchVertex).addEdge( Edge.create(broadcastVertex, fetchVertex, edgeConf.createDefaultBroadcastEdgeProperty())); return dag; }
Example #3
Source File: SimpleTestDAG3Vertices.java From incubator-tez with Apache License 2.0 | 6 votes |
public static DAG createDAG(String name, Configuration conf) throws Exception { byte[] payload = null; int taskCount = TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT; if (conf != null) { taskCount = conf.getInt(TEZ_SIMPLE_DAG_NUM_TASKS, TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT); payload = TezUtils.createUserPayloadFromConf(conf); } DAG dag = new DAG(name); Vertex v1 = new Vertex("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource); Vertex v2 = new Vertex("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource); Vertex v3 = new Vertex("v3", TestProcessor.getProcDesc(payload), taskCount, defaultResource); dag.addVertex(v1).addVertex(v2).addEdge(new Edge(v1, v2, new EdgeProperty(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, TestOutput.getOutputDesc(payload), TestInput.getInputDesc(payload)))); dag.addVertex(v3).addEdge(new Edge(v2, v3, new EdgeProperty(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, TestOutput.getOutputDesc(payload), TestInput.getInputDesc(payload)))); return dag; }
Example #4
Source File: TestTezClient.java From tez with Apache License 2.0 | 6 votes |
@Test(timeout = 5000) public void testClientResubmit() throws Exception { TezClientForTest client = configureAndCreateTezClient(null, true, null); client.start(); Map<String, LocalResource> lrDAG = Collections.singletonMap("LR1", LocalResource.newInstance( URL.newInstance("file", "localhost", 0, "/test1"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1)); Vertex vertex1 = Vertex.create("Vertex1", ProcessorDescriptor.create("P1"), 1, Resource.newInstance(1, 1)); vertex1.setTaskLaunchCmdOpts("-XX:+UseParallelGC -XX:+UseG1GC"); Vertex vertex2 = Vertex.create("Vertex2", ProcessorDescriptor.create("P2"), 1, Resource.newInstance(1, 1)); vertex2.setTaskLaunchCmdOpts("-XX:+UseParallelGC -XX:+UseG1GC"); DAG dag = DAG.create("DAG").addVertex(vertex1).addVertex(vertex2).addTaskLocalFiles(lrDAG); for (int i = 0; i < 3; ++i) { try { client.submitDAG(dag); Assert.fail("Expected TezUncheckedException here."); } catch(TezUncheckedException ex) { Assert.assertTrue(ex.getMessage().contains("Invalid/conflicting GC options found")); } } client.stop(); }
Example #5
Source File: TestFaultTolerance.java From tez with Apache License 2.0 | 6 votes |
@Test (timeout=60000) public void testBasicInputFailureWithoutExit() throws Exception { Configuration testConf = new Configuration(false); testConf.setBoolean(TestInput.getVertexConfName( TestInput.TEZ_FAILING_INPUT_DO_FAIL, "v2"), true); testConf.set(TestInput.getVertexConfName( TestInput.TEZ_FAILING_INPUT_FAILING_TASK_INDEX, "v2"), "1"); testConf.set(TestInput.getVertexConfName( TestInput.TEZ_FAILING_INPUT_FAILING_TASK_ATTEMPT, "v2"), "0"); testConf.set(TestInput.getVertexConfName( TestInput.TEZ_FAILING_INPUT_FAILING_INPUT_INDEX, "v2"), "0"); testConf.set(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v2"), "0,1"); testConf.setInt(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v2", 1), 4); testConf.setInt(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v2", 0), 3); DAG dag = SimpleTestDAG.createDAG("testBasicInputFailureWithoutExit", testConf); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); }
Example #6
Source File: TestJobSubmissionTez.java From spork with Apache License 2.0 | 6 votes |
@Override public void checkDefaultParallelResult(PhysicalPlan pp, PigContext pc) throws Exception { TezOperPlan tezPlan = buildTezPlan(pp, pc); LoaderProcessor loaderStorer = new LoaderProcessor(tezPlan, pc); loaderStorer.visit(); ParallelismSetter parallelismSetter = new ParallelismSetter(tezPlan, pc); parallelismSetter.visit(); DAG tezDag = getTezDAG(tezPlan, pc); TezDagBuilder dagBuilder = new TezDagBuilder(pc, tezPlan, tezDag, null); dagBuilder.visit(); for (Vertex v : tezDag.getVertices()) { if (!v.getInputVertices().isEmpty()) { Configuration conf = TezUtils.createConfFromUserPayload(v.getProcessorDescriptor().getUserPayload()); int parallel = v.getParallelism(); assertEquals(parallel, 100); Util.assertConfLong(conf, "pig.info.reducers.default.parallel", 100); Util.assertConfLong(conf, "pig.info.reducers.requested.parallel", -1); Util.assertConfLong(conf, "pig.info.reducers.estimated.parallel", -1); } } }
Example #7
Source File: TestJobSubmissionTez.java From spork with Apache License 2.0 | 6 votes |
@Override public void checkJobControlCompilerErrResult(PhysicalPlan pp, PigContext pc) throws Exception { TezOperPlan tezPlan = buildTezPlan(pp, pc); LoaderProcessor loaderStorer = new LoaderProcessor(tezPlan, pc); loaderStorer.visit(); ParallelismSetter parallelismSetter = new ParallelismSetter(tezPlan, pc); parallelismSetter.visit(); DAG tezDag = getTezDAG(tezPlan, pc); TezDagBuilder dagBuilder = new TezDagBuilder(pc, tezPlan, tezDag, null); try { dagBuilder.visit(); } catch (VisitorException jce) { assertTrue(((JobCreationException)jce.getCause()).getErrorCode() == 1068); } }
Example #8
Source File: TestFaultTolerance.java From incubator-tez with Apache License 2.0 | 6 votes |
@Test (timeout=60000) public void testBasicTaskFailure() throws Exception { Configuration testConf = new Configuration(false); testConf.setBoolean(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_DO_FAIL, "v1"), true); testConf.set(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_TASK_INDEX, "v1"), "0"); testConf.setInt(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_UPTO_TASK_ATTEMPT, "v1"), 0); //verify value at v2 task1 testConf.set(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v2"), "1"); //value of v2 task1 is 4. //v1 attempt0 has value of 1 (attempt index + 1). //v1 attempt1 has value of 2 (attempt index + 1). //v3 attempt0 verifies value of 1 + 2 (values from input vertices) // + 1 (attempt index + 1) = 4 testConf.setInt(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v2", 1), 4); DAG dag = SimpleTestDAG.createDAG("testBasicTaskFailure", testConf); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); }
Example #9
Source File: TestMockDAGAppMaster.java From tez with Apache License 2.0 | 6 votes |
private DAG createDAG(String dagName, boolean uv12CommitFail, boolean v3CommitFail) { DAG dag = DAG.create(dagName); Vertex v1 = Vertex.create("v1", ProcessorDescriptor.create("Proc"), 1); Vertex v2 = Vertex.create("v2", ProcessorDescriptor.create("Proc"), 1); Vertex v3 = Vertex.create("v3", ProcessorDescriptor.create("Proc"), 1); VertexGroup uv12 = dag.createVertexGroup("uv12", v1, v2); DataSinkDescriptor uv12DataSink = DataSinkDescriptor.create( OutputDescriptor.create("dummy output"), createOutputCommitterDesc(uv12CommitFail), null); uv12.addDataSink("uv12Out", uv12DataSink); DataSinkDescriptor v3DataSink = DataSinkDescriptor.create( OutputDescriptor.create("dummy output"), createOutputCommitterDesc(v3CommitFail), null); v3.addDataSink("v3Out", v3DataSink); GroupInputEdge e1 = GroupInputEdge.create(uv12, v3, EdgeProperty.create( DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("dummy output class"), InputDescriptor.create("dummy input class")), InputDescriptor .create("merge.class")); dag.addVertex(v1) .addVertex(v2) .addVertex(v3) .addEdge(e1); return dag; }
Example #10
Source File: TestFaultTolerance.java From incubator-tez with Apache License 2.0 | 6 votes |
@Test (timeout=60000) public void testTaskMultipleFailures() throws Exception { Configuration testConf = new Configuration(false); testConf.setBoolean(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_DO_FAIL, "v1"), true); testConf.set(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_TASK_INDEX, "v1"), "0,1"); testConf.setInt(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_UPTO_TASK_ATTEMPT, "v1"), 1); //v1 task0,1 attempt 2 succeed. Input sum = 6. Plus one (v2 attempt0). //ending sum is 7. testConf.set(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v2"), "0"); testConf.setInt(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v2", 0), 7); DAG dag = SimpleTestDAG.createDAG("testTaskMultipleFailures", testConf); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); }
Example #11
Source File: TestFaultTolerance.java From tez with Apache License 2.0 | 6 votes |
@Test (timeout=60000) public void testBasicInputFailureWithoutExitDeadline() throws Exception { Configuration testConf = new Configuration(false); testConf.setInt(SimpleTestDAG.TEZ_SIMPLE_DAG_NUM_TASKS, 3); // 1 error < 0.4 fail fraction testConf.setBoolean(TestInput.getVertexConfName( TestInput.TEZ_FAILING_INPUT_DO_FAIL, "v2"), true); testConf.set(TestInput.getVertexConfName( TestInput.TEZ_FAILING_INPUT_FAILING_TASK_INDEX, "v2"), "2"); testConf.set(TestInput.getVertexConfName( TestInput.TEZ_FAILING_INPUT_FAILING_TASK_ATTEMPT, "v2"), "0"); testConf.set(TestInput.getVertexConfName( TestInput.TEZ_FAILING_INPUT_FAILING_INPUT_INDEX, "v2"), "0"); DAG dag = SimpleTestDAG.createDAG("testBasicInputFailureWithoutExitDeadline", testConf); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); }
Example #12
Source File: TestDAGRecovery2.java From incubator-tez with Apache License 2.0 | 6 votes |
@Test(timeout=120000) public void testSessionDisableMultiAttempts() throws Exception { tezSession.stop(); Path remoteStagingDir = remoteFs.makeQualified(new Path(TEST_ROOT_DIR, String .valueOf(new Random().nextInt(100000)))); TezClientUtils.ensureStagingDirExists(conf, remoteStagingDir); TezConfiguration tezConf = createSessionConfig(remoteStagingDir); tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true); tezConf.setBoolean(TezConfiguration.DAG_RECOVERY_ENABLED, false); TezClient session = new TezClient("TestDAGRecovery2SingleAttemptOnly", tezConf); session.start(); // DAG should fail as it never completes on the first attempt DAG dag = MultiAttemptDAG.createDAG("TestSingleAttemptDAG", null); runDAGAndVerify(dag, State.FAILED, session); session.stop(); }
Example #13
Source File: TestFaultTolerance.java From incubator-tez with Apache License 2.0 | 6 votes |
/** * Test cascading input failure with exit. Expecting success. * v1 -- v2 -- v3 * v3 all-tasks attempt0 input0 fails. v3 attempt0 exits. Triggering v2 rerun. * v2 task0 attempt1 input0 fails. v2 attempt1 exits. Triggering v1 rerun. * v1 attempt1 rerun and succeeds. v2 accepts v1 attempt1 output. v2 attempt2 succeeds. * v3 attempt1 accepts v2 attempt2 output. * * AM vertex succeeded order is v1, v2, v3, v1, v2, v3. * @throws Exception */ @Test (timeout=60000) public void testCascadingInputFailureWithExitSuccess() throws Exception { Configuration testConf = new Configuration(false); setCascadingInputFailureConfig(testConf, true); //v2 task0 attempt2 value = v1 task0 attempt1 (2) + v1 task1 attempt0 (1) + 3 = 6 //v3 all-tasks attempt1 takes v2 task0 attempt2 value (6) + v2 task1 attempt0 (3) + 2 = 11 testConf.set(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v3"), "0,1"); testConf.setInt(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v3", 0), 11); testConf.setInt(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v3", 1), 11); DAG dag = SimpleTestDAG3Vertices.createDAG( "testCascadingInputFailureWithExitSuccess", testConf); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); }
Example #14
Source File: TestDAGRecovery2.java From tez with Apache License 2.0 | 6 votes |
@Test(timeout=120000) public void testSessionDisableMultiAttempts() throws Exception { tezSession.stop(); Path remoteStagingDir = remoteFs.makeQualified(new Path(TEST_ROOT_DIR, String .valueOf(new Random().nextInt(100000)))); TezClientUtils.ensureStagingDirExists(conf, remoteStagingDir); TezConfiguration tezConf = createSessionConfig(remoteStagingDir); tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true); tezConf.setBoolean(TezConfiguration.DAG_RECOVERY_ENABLED, false); TezClient session = TezClient.create("TestDAGRecovery2SingleAttemptOnly", tezConf); session.start(); // DAG should fail as it never completes on the first attempt DAG dag = MultiAttemptDAG.createDAG("TestSingleAttemptDAG", null); runDAGAndVerify(dag, State.FAILED, session); session.stop(); }
Example #15
Source File: TestAMRecovery.java From tez with Apache License 2.0 | 6 votes |
/** * Fine-grained recovery task-level, In a vertex (v1), task 0 is done task 1 * is also done. History flush happens. AM dies. Once AM is recovered, task 0 * and Task 1 is not re-run. (Broadcast) * * @throws Exception */ @Test(timeout = 120000) public void testVertexCompletelyFinished_Broadcast() throws Exception { DAG dag = createDAG("VertexCompletelyFinished_Broadcast", ControlledImmediateStartVertexManager.class, DataMovementType.BROADCAST, false); TezCounters counters = runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); assertEquals(4, counters.findCounter(DAGCounter.NUM_SUCCEEDED_TASKS).getValue()); assertEquals(2, counters.findCounter(TestCounter.Counter_1).getValue()); List<HistoryEvent> historyEvents1 = readRecoveryLog(1); List<HistoryEvent> historyEvents2 = readRecoveryLog(2); printHistoryEvents(historyEvents1, 1); printHistoryEvents(historyEvents1, 2); // task_0 of v1 is finished in attempt 1, task_1 of v1 is not finished in // attempt 1 assertEquals(1, findTaskAttemptFinishedEvent(historyEvents1, 0, 0).size()); assertEquals(1, findTaskAttemptFinishedEvent(historyEvents1, 0, 1).size()); // task_0 of v1 is finished in attempt 1 and not rerun, task_1 of v1 is // finished in attempt 2 assertEquals(1, findTaskAttemptFinishedEvent(historyEvents2, 0, 0).size()); assertEquals(1, findTaskAttemptFinishedEvent(historyEvents2, 0, 1).size()); }
Example #16
Source File: TestFaultTolerance.java From incubator-tez with Apache License 2.0 | 6 votes |
/** * Test cascading input failure without exit. Expecting success. * v1 -- v2 -- v3 * v3 all-tasks attempt0 input0 fails. Wait. Triggering v2 rerun. * v2 task0 attempt1 input0 fails. Wait. Triggering v1 rerun. * v1 attempt1 rerun and succeeds. v2 accepts v1 attempt1 output. v2 attempt1 succeeds. * v3 attempt0 accepts v2 attempt1 output. * * AM vertex succeeded order is v1, v2, v1, v2, v3. * @throws Exception */ @Test (timeout=60000) public void testCascadingInputFailureWithoutExitSuccess() throws Exception { Configuration testConf = new Configuration(false); setCascadingInputFailureConfig(testConf, false); //v2 task0 attempt1 value = v1 task0 attempt1 (2) + v1 task1 attempt0 (1) + 2 = 5 //v3 all-tasks attempt0 takes v2 task0 attempt1 value (5) + v2 task1 attempt0 (3) + 1 = 9 testConf.set(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v3"), "0,1"); testConf.setInt(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v3", 0), 9); testConf.setInt(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v3", 1), 9); DAG dag = SimpleTestDAG3Vertices.createDAG( "testCascadingInputFailureWithoutExitSuccess", testConf); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); }
Example #17
Source File: TestFaultTolerance.java From incubator-tez with Apache License 2.0 | 6 votes |
@Test (timeout=60000) public void testMultipleInputFailureWithoutExit() throws Exception { Configuration testConf = new Configuration(false); testConf.setBoolean(TestInput.getVertexConfName( TestInput.TEZ_FAILING_INPUT_DO_FAIL, "v2"), true); testConf.set(TestInput.getVertexConfName( TestInput.TEZ_FAILING_INPUT_FAILING_TASK_INDEX, "v2"), "0,1"); testConf.set(TestInput.getVertexConfName( TestInput.TEZ_FAILING_INPUT_FAILING_TASK_ATTEMPT, "v2"), "0"); testConf.set(TestInput.getVertexConfName( TestInput.TEZ_FAILING_INPUT_FAILING_INPUT_INDEX, "v2"), "-1"); //v2 task0 attempt0 input0,1 fails. wait. //v1 task0 attempt1 reruns. v1 task1 attempt1 reruns. //2 + 2 + 1 = 5 //same number for v2 task1 testConf.set(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v2"), "0,1"); testConf.setInt(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v2", 0), 5); testConf.setInt(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v2", 1), 5); DAG dag = SimpleTestDAG.createDAG("testMultipleInputFailureWithoutExit", testConf); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); }
Example #18
Source File: TestAnalyzer.java From tez with Apache License 2.0 | 6 votes |
private List<StepCheck[]> testBasicTaskFailure() throws Exception { Configuration testConf = new Configuration(false); testConf.setInt(SimpleTestDAG.TEZ_SIMPLE_DAG_NUM_TASKS, 1); testConf.setBoolean(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_DO_FAIL, "v1"), true); testConf.set(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_TASK_INDEX, "v1"), "0"); testConf.setInt(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_UPTO_TASK_ATTEMPT, "v1"), 0); StepCheck[] check = { createStep("v1 : 000000_0", CriticalPathDependency.INIT_DEPENDENCY), createStep("v1 : 000000_1", CriticalPathDependency.RETRY_DEPENDENCY), createStep("v2 : 000000_0", CriticalPathDependency.DATA_DEPENDENCY), }; DAG dag = SimpleTestDAG.createDAG("testBasicTaskFailure", testConf); runDAG(dag, DAGStatus.State.SUCCEEDED); return Collections.singletonList(check); }
Example #19
Source File: TestMemoryWithEvents.java From tez with Apache License 2.0 | 6 votes |
private void testMemory(DAG dag, boolean sendDMEvents) throws Exception { StopWatch stopwatch = new StopWatch(); stopwatch.start(); TezConfiguration tezconf = new TezConfiguration(defaultConf); MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null, false, false, numThreads, 1000); tezClient.start(); MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp(); MockContainerLauncher mockLauncher = mockApp.getContainerLauncher(); mockLauncher.startScheduling(false); mockApp.eventsDelegate = new TestMockDAGAppMaster.TestEventsDelegate(); mockApp.doSleep = false; DAGClient dagClient = tezClient.submitDAG(dag); mockLauncher.waitTillContainersLaunched(); mockLauncher.startScheduling(true); DAGStatus status = dagClient.waitForCompletion(); Assert.assertEquals(DAGStatus.State.SUCCEEDED, status.getState()); checkMemory(dag.getName(), mockApp); stopwatch.stop(); System.out.println("Time taken(ms): " + stopwatch.now(TimeUnit.MILLISECONDS)); tezClient.stop(); }
Example #20
Source File: TestAMRecovery.java From tez with Apache License 2.0 | 6 votes |
/** * v1 --> v2 <br> * v1 has a customized VM to control whether to schedule only one second task when it is partiallyFinished test case. * v2 has a customized VM which could control when to kill AM * * @param vertexManagerClass * @param dmType * @param failOnParitialCompleted * @return * @throws IOException */ private DAG createDAG(String dagName, Class vertexManagerClass, DataMovementType dmType, boolean failOnParitialCompleted) throws IOException { if (failOnParitialCompleted) { tezConf.set(FAIL_ON_PARTIAL_FINISHED, "true"); } else { tezConf.set(FAIL_ON_PARTIAL_FINISHED, "false"); } DAG dag = DAG.create(dagName); UserPayload payload = UserPayload.create(null); Vertex v1 = Vertex.create("v1", MyProcessor.getProcDesc(), 2); v1.setVertexManagerPlugin(VertexManagerPluginDescriptor.create( ScheduleControlledVertexManager.class.getName()).setUserPayload( TezUtils.createUserPayloadFromConf(tezConf))); Vertex v2 = Vertex.create("v2", DoNothingProcessor.getProcDesc(), 2); v2.setVertexManagerPlugin(VertexManagerPluginDescriptor.create( vertexManagerClass.getName()).setUserPayload( TezUtils.createUserPayloadFromConf(tezConf))); dag.addVertex(v1).addVertex(v2); dag.addEdge(Edge.create(v1, v2, EdgeProperty.create(dmType, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, TestOutput.getOutputDesc(payload), TestInput.getInputDesc(payload)))); return dag; }
Example #21
Source File: IntersectDataGen.java From incubator-tez with Apache License 2.0 | 5 votes |
private void setupURIsForCredentials(DAG dag, Path... paths) throws IOException { List<URI> uris = new LinkedList<URI>(); for (Path path : paths) { FileSystem fs = path.getFileSystem(getConf()); Path qPath = fs.makeQualified(path); uris.add(qPath.toUri()); } dag.addURIsForCredentials(uris); }
Example #22
Source File: TestDAGRecovery2.java From incubator-tez with Apache License 2.0 | 5 votes |
@Test(timeout=120000) public void testFailingCommitter() throws Exception { DAG dag = SimpleVTestDAG.createDAG("FailingCommitterDAG", null); OutputDescriptor od = new OutputDescriptor(MultiAttemptDAG.NoOpOutput.class.getName()); od.setUserPayload(new MultiAttemptDAG.FailingOutputCommitter.FailingOutputCommitterConfig(true) .toUserPayload()); dag.getVertex("v3").addOutput("FailingOutput", od, MultiAttemptDAG.FailingOutputCommitter.class); runDAGAndVerify(dag, State.FAILED); }
Example #23
Source File: TwoLevelsFailingDAG.java From incubator-tez with Apache License 2.0 | 5 votes |
public static DAG createDAG(String name, Configuration conf) throws Exception { if (conf != null) { payload = TezUtils.createUserPayloadFromConf(conf); } dag = new DAG(name); addDAGVerticesAndEdges(); return dag; }
Example #24
Source File: SimpleReverseVTestDAG.java From incubator-tez with Apache License 2.0 | 5 votes |
public static DAG createDAG(String name, Configuration conf) throws Exception { byte[] payload = null; int taskCount = TEZ_SIMPLE_REVERSE_V_DAG_NUM_TASKS_DEFAULT; if (conf != null) { taskCount = conf.getInt(TEZ_SIMPLE_REVERSE_V_DAG_NUM_TASKS, TEZ_SIMPLE_REVERSE_V_DAG_NUM_TASKS_DEFAULT); payload = TezUtils.createUserPayloadFromConf(conf); } DAG dag = new DAG(name); Vertex v1 = new Vertex("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource); Vertex v2 = new Vertex("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource); Vertex v3 = new Vertex("v3", TestProcessor.getProcDesc(payload), taskCount, defaultResource); dag.addVertex(v1).addVertex(v2).addVertex(v3); dag.addEdge(new Edge(v1, v2, new EdgeProperty(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, TestOutput.getOutputDesc(payload), TestInput.getInputDesc(payload)))); dag.addEdge(new Edge(v1, v3, new EdgeProperty(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, TestOutput.getOutputDesc(payload), TestInput.getInputDesc(payload)))); return dag; }
Example #25
Source File: MultiAttemptDAG.java From incubator-tez with Apache License 2.0 | 5 votes |
public static DAG createDAG(String name, Configuration conf) throws Exception { byte[] payload = null; int taskCount = MULTI_ATTEMPT_DAG_VERTEX_NUM_TASKS_DEFAULT; if (conf != null) { taskCount = conf.getInt(MULTI_ATTEMPT_DAG_VERTEX_NUM_TASKS, MULTI_ATTEMPT_DAG_VERTEX_NUM_TASKS_DEFAULT); payload = TezUtils.createUserPayloadFromConf(conf); } DAG dag = new DAG(name); Vertex v1 = new Vertex("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource); Vertex v2 = new Vertex("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource); Vertex v3 = new Vertex("v3", TestProcessor.getProcDesc(payload), taskCount, defaultResource); // Make each vertex manager fail on appropriate attempt v1.setVertexManagerPlugin(new VertexManagerPluginDescriptor( FailOnAttemptVertexManagerPlugin.class.getName()) .setUserPayload(new String("1").getBytes())); v2.setVertexManagerPlugin(new VertexManagerPluginDescriptor( FailOnAttemptVertexManagerPlugin.class.getName()) .setUserPayload(new String("2").getBytes())); v3.setVertexManagerPlugin(new VertexManagerPluginDescriptor( FailOnAttemptVertexManagerPlugin.class.getName()) .setUserPayload(new String("3").getBytes())); dag.addVertex(v1).addVertex(v2).addVertex(v3); dag.addEdge(new Edge(v1, v2, new EdgeProperty(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, TestOutput.getOutputDesc(payload), TestInput.getInputDesc(payload)))); dag.addEdge(new Edge(v2, v3, new EdgeProperty(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, TestOutput.getOutputDesc(payload), TestInput.getInputDesc(payload)))); return dag; }
Example #26
Source File: TestFaultTolerance.java From tez with Apache License 2.0 | 5 votes |
@Test (timeout=60000) public void testTaskMultipleFailuresDAGFail() throws Exception { Configuration testConf = new Configuration(false); testConf.setBoolean(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_DO_FAIL, "v1"), true); testConf.set(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_TASK_INDEX, "v1"), "0"); testConf.setInt(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_UPTO_TASK_ATTEMPT, "v1"), -1); DAG dag = SimpleTestDAG.createDAG("testTaskMultipleFailuresDAGFail", testConf); runDAGAndVerify(dag, DAGStatus.State.FAILED); }
Example #27
Source File: SortMergeJoinExample.java From tez with Apache License 2.0 | 5 votes |
@Override protected int runJob(String[] args, TezConfiguration tezConf, TezClient tezClient) throws Exception { String inputDir1 = args[0]; String inputDir2 = args[1]; int numPartitions = Integer.parseInt(args[2]); String outputDir = args[3]; Path inputPath1 = new Path(inputDir1); Path inputPath2 = new Path(inputDir2); Path outputPath = new Path(outputDir); // Verify output path existence FileSystem fs = outputPath.getFileSystem(tezConf); outputPath = fs.makeQualified(outputPath); if (fs.exists(outputPath)) { System.err.println("Output directory: " + outputDir + " already exists"); return 3; } if (numPartitions <= 0) { System.err.println("NumPartitions must be > 0"); return 4; } DAG dag = createDag(tezConf, inputPath1, inputPath2, outputPath, numPartitions); LOG.info("Running SortMergeJoinExample"); return runDag(dag, isCountersLog(), LOG); }
Example #28
Source File: TestMockDAGAppMaster.java From tez with Apache License 2.0 | 5 votes |
@Test (timeout = 5000) public void testLocalResourceSetup() throws Exception { TezConfiguration tezconf = new TezConfiguration(defaultConf); MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null); tezClient.start(); MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp(); MockContainerLauncher mockLauncher = mockApp.getContainerLauncher(); mockLauncher.startScheduling(false); Map<String, LocalResource> lrDAG = Maps.newHashMap(); String lrName1 = "LR1"; lrDAG.put(lrName1, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1)); Map<String, LocalResource> lrVertex = Maps.newHashMap(); String lrName2 = "LR2"; lrVertex.put(lrName2, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test1"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1)); DAG dag = DAG.create("testLocalResourceSetup").addTaskLocalFiles(lrDAG); Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), 5).addTaskLocalFiles(lrVertex); dag.addVertex(vA); DAGClient dagClient = tezClient.submitDAG(dag); mockLauncher.waitTillContainersLaunched(); ContainerData cData = mockLauncher.getContainers().values().iterator().next(); ContainerLaunchContext launchContext = cData.launchContext; Map<String, LocalResource> taskLR = launchContext.getLocalResources(); // verify tasks are launched with both DAG and task resources. Assert.assertTrue(taskLR.containsKey(lrName1)); Assert.assertTrue(taskLR.containsKey(lrName2)); mockLauncher.startScheduling(true); dagClient.waitForCompletion(); Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState()); tezClient.stop(); }
Example #29
Source File: TestFaultTolerance.java From tez with Apache License 2.0 | 5 votes |
@Test (timeout=60000) public void testThreeLevelsFailingDAG2VerticesHaveFailedAttemptsDAGSucceeds() throws Exception { Configuration testConf = new Configuration(); //set maximum number of task attempts to 4 testConf.setInt(TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS, 4); //l2v1 failure testConf.setBoolean(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_DO_FAIL, "l2v1"), true); testConf.set(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_TASK_INDEX, "l2v1"), "1"); //3 attempts fail testConf.setInt(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_UPTO_TASK_ATTEMPT, "l2v1"), 2); //l3v1 failure testConf.setBoolean(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_DO_FAIL, "l3v1"), true); testConf.set(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_TASK_INDEX, "l3v1"), "0"); //3 attempts fail testConf.setInt(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_UPTO_TASK_ATTEMPT, "l3v1"), 2); //l2v1: task0 attempt0 succeeds. task1 attempt3 succeeds. //l3v1 finally task0 attempt3 will succeed. //l1v1 outputs 1. l1v2 outputs 2. //l2v1 task0 attempt0 output = 2. //l2v2 output: attempt0 (l1v2+self = 2+1) * 3 tasks = 9 //l3v1 task0 attempt3 = l2v1 (2) + l2v2 (9) + self (4) = 15 testConf.set(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "l3v1"), "0"); testConf.setInt(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "l3v1", 0), 15); DAG dag = ThreeLevelsFailingDAG.createDAG("testThreeLevelsFailingDAG2VerticesHaveFailedAttemptsDAGSucceeds", testConf); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); }
Example #30
Source File: TestFaultTolerance.java From incubator-tez with Apache License 2.0 | 5 votes |
/** * Input failure of v3 causes rerun of both both v1 and v2 vertices. * v1 v2 * \ / * v3 * * @throws Exception */ @Test (timeout=60000) public void testInputFailureCausesRerunOfTwoVerticesWithoutExit() throws Exception { Configuration testConf = new Configuration(false); testConf.setBoolean(TestInput.getVertexConfName( TestInput.TEZ_FAILING_INPUT_DO_FAIL, "v3"), true); testConf.setBoolean(TestInput.getVertexConfName( TestInput.TEZ_FAILING_INPUT_DO_FAIL_AND_EXIT, "v3"), false); testConf.set(TestInput.getVertexConfName( TestInput.TEZ_FAILING_INPUT_FAILING_TASK_INDEX, "v3"), "0,1"); testConf.set(TestInput.getVertexConfName( TestInput.TEZ_FAILING_INPUT_FAILING_TASK_ATTEMPT, "v3"), "0"); testConf.set(TestInput.getVertexConfName( TestInput.TEZ_FAILING_INPUT_FAILING_INPUT_INDEX, "v3"), "-1"); testConf.set(TestInput.getVertexConfName( TestInput.TEZ_FAILING_INPUT_FAILING_UPTO_INPUT_ATTEMPT, "v3"), "1"); //v3 attempt0: //v1 task0,1 attempt2 = 6. v2 task0,1 attempt2 = 6. //total = 6 + 6 + 1 = 13 testConf.set(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v3"), "0"); testConf.setInt(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v3", 0), 13); DAG dag = SimpleVTestDAG.createDAG( "testInputFailureCausesRerunOfTwoVerticesWithoutExit", testConf); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); }