org.apache.tez.dag.api.DAG Java Examples

/**
 * Fine-grained recovery task-level, In a vertex (v1), task 0 is done task 1
 * is not started. History flush happens. AM dies. Once AM is recovered, task 0 is
 * not re-run. Task 1 is re-run. (SCATTER_GATHER)
 *
 * @throws Exception
 */
@Test(timeout = 120000)
public void testVertexPartiallyFinished_ScatterGather() throws Exception {
  DAG dag =
      createDAG("VertexPartiallyFinished_ScatterGather", ControlledShuffleVertexManager.class,
          DataMovementType.SCATTER_GATHER, true);
  TezCounters counters = runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED);
  assertEquals(4, counters.findCounter(DAGCounter.NUM_SUCCEEDED_TASKS).getValue());
  assertEquals(2, counters.findCounter(TestCounter.Counter_1).getValue());

  List<HistoryEvent> historyEvents1 = readRecoveryLog(1);
  List<HistoryEvent> historyEvents2 = readRecoveryLog(2);
  printHistoryEvents(historyEvents1, 1);
  printHistoryEvents(historyEvents1, 2);
  // task_0 of v1 is finished in attempt 1, task_1 of v1 is not finished in
  // attempt 1
  assertEquals(1, findTaskAttemptFinishedEvent(historyEvents1, 0, 0).size());
  assertEquals(0, findTaskAttemptFinishedEvent(historyEvents1, 0, 1).size());

  // task_0 of v1 is finished in attempt 1 and not rerun, task_1 of v1 is
  // finished in attempt 2
  assertEquals(1, findTaskAttemptFinishedEvent(historyEvents2, 0, 0).size());
  assertEquals(1, findTaskAttemptFinishedEvent(historyEvents2, 0, 1).size());

}
 

private DAG createDAG(int numGenTasks, int totalSourceDataSize, int numFetcherTasks) {
  int bytesPerSource = totalSourceDataSize / numGenTasks;
  LOG.info("DataPerSourceTask(bytes)=" + bytesPerSource);
  ByteBuffer payload = ByteBuffer.allocate(4);
  payload.putInt(0, bytesPerSource);

  Vertex broadcastVertex = Vertex.create("DataGen",
      ProcessorDescriptor.create(InputGenProcessor.class.getName())
          .setUserPayload(UserPayload.create(payload)), numGenTasks);
  Vertex fetchVertex = Vertex.create("FetchVertex",
      ProcessorDescriptor.create(InputFetchProcessor.class.getName()), numFetcherTasks);
  UnorderedKVEdgeConfig edgeConf = UnorderedKVEdgeConfig.newBuilder(NullWritable.class
  .getName(), IntWritable.class.getName()).setCompression(false, null, null).build();

  DAG dag = DAG.create("BroadcastLoadGen");
  dag.addVertex(broadcastVertex).addVertex(fetchVertex).addEdge(
      Edge.create(broadcastVertex, fetchVertex, edgeConf.createDefaultBroadcastEdgeProperty()));
  return dag;
}
 

public static DAG createDAG(String name, 
    Configuration conf) throws Exception {
  byte[] payload = null;
  int taskCount = TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT;
  if (conf != null) {
    taskCount = conf.getInt(TEZ_SIMPLE_DAG_NUM_TASKS, TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT);
    payload = TezUtils.createUserPayloadFromConf(conf);
  }
  DAG dag = new DAG(name);
  Vertex v1 = new Vertex("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  Vertex v2 = new Vertex("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  Vertex v3 = new Vertex("v3", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  dag.addVertex(v1).addVertex(v2).addEdge(new Edge(v1, v2, 
      new EdgeProperty(DataMovementType.SCATTER_GATHER, 
          DataSourceType.PERSISTED, 
          SchedulingType.SEQUENTIAL, 
          TestOutput.getOutputDesc(payload), 
          TestInput.getInputDesc(payload))));
  dag.addVertex(v3).addEdge(new Edge(v2, v3, 
          new EdgeProperty(DataMovementType.SCATTER_GATHER, 
              DataSourceType.PERSISTED, 
              SchedulingType.SEQUENTIAL, 
              TestOutput.getOutputDesc(payload), 
              TestInput.getInputDesc(payload))));
  return dag;
}
 

@Test(timeout = 5000)
public void testClientResubmit() throws Exception {
  TezClientForTest client = configureAndCreateTezClient(null, true, null);
  client.start();
  Map<String, LocalResource> lrDAG = Collections.singletonMap("LR1",
      LocalResource.newInstance(
          URL.newInstance("file", "localhost", 0, "/test1"),
          LocalResourceType.FILE,
          LocalResourceVisibility.PUBLIC, 1, 1));
  Vertex vertex1 = Vertex.create("Vertex1", ProcessorDescriptor.create("P1"), 1,
      Resource.newInstance(1, 1));
  vertex1.setTaskLaunchCmdOpts("-XX:+UseParallelGC -XX:+UseG1GC");
  Vertex vertex2 = Vertex.create("Vertex2", ProcessorDescriptor.create("P2"), 1,
      Resource.newInstance(1, 1));
  vertex2.setTaskLaunchCmdOpts("-XX:+UseParallelGC -XX:+UseG1GC");
  DAG dag = DAG.create("DAG").addVertex(vertex1).addVertex(vertex2).addTaskLocalFiles(lrDAG);
  for (int i = 0; i < 3; ++i) {
    try {
      client.submitDAG(dag);
      Assert.fail("Expected TezUncheckedException here.");
    } catch(TezUncheckedException ex) {
      Assert.assertTrue(ex.getMessage().contains("Invalid/conflicting GC options found"));
    }
  }
  client.stop();
}
 

@Test (timeout=60000)
public void testBasicInputFailureWithoutExit() throws Exception {
  Configuration testConf = new Configuration(false);
  testConf.setBoolean(TestInput.getVertexConfName(
      TestInput.TEZ_FAILING_INPUT_DO_FAIL, "v2"), true);
  testConf.set(TestInput.getVertexConfName(
      TestInput.TEZ_FAILING_INPUT_FAILING_TASK_INDEX, "v2"), "1");
  testConf.set(TestInput.getVertexConfName(
      TestInput.TEZ_FAILING_INPUT_FAILING_TASK_ATTEMPT, "v2"), "0");
  testConf.set(TestInput.getVertexConfName(
      TestInput.TEZ_FAILING_INPUT_FAILING_INPUT_INDEX, "v2"), "0");
  
  testConf.set(TestProcessor.getVertexConfName(
          TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v2"), "0,1");
  testConf.setInt(TestProcessor.getVertexConfName(
          TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v2", 1), 4);
  testConf.setInt(TestProcessor.getVertexConfName(
          TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v2", 0), 3);
  DAG dag = SimpleTestDAG.createDAG("testBasicInputFailureWithoutExit", testConf);
  runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED);
}
 

@Override
public void checkDefaultParallelResult(PhysicalPlan pp, PigContext pc) throws Exception {
    TezOperPlan tezPlan = buildTezPlan(pp, pc);

    LoaderProcessor loaderStorer = new LoaderProcessor(tezPlan, pc);
    loaderStorer.visit();

    ParallelismSetter parallelismSetter = new ParallelismSetter(tezPlan, pc);
    parallelismSetter.visit();

    DAG tezDag = getTezDAG(tezPlan, pc);
    TezDagBuilder dagBuilder = new TezDagBuilder(pc, tezPlan, tezDag, null);
    dagBuilder.visit();
    for (Vertex v : tezDag.getVertices()) {
        if (!v.getInputVertices().isEmpty()) {
            Configuration conf = TezUtils.createConfFromUserPayload(v.getProcessorDescriptor().getUserPayload());
            int parallel = v.getParallelism();
            assertEquals(parallel, 100);
            Util.assertConfLong(conf, "pig.info.reducers.default.parallel", 100);
            Util.assertConfLong(conf, "pig.info.reducers.requested.parallel", -1);
            Util.assertConfLong(conf, "pig.info.reducers.estimated.parallel", -1);
        }
    }
}
 

@Override
public void checkJobControlCompilerErrResult(PhysicalPlan pp, PigContext pc) throws Exception {
    TezOperPlan tezPlan = buildTezPlan(pp, pc);

    LoaderProcessor loaderStorer = new LoaderProcessor(tezPlan, pc);
    loaderStorer.visit();

    ParallelismSetter parallelismSetter = new ParallelismSetter(tezPlan, pc);
    parallelismSetter.visit();

    DAG tezDag = getTezDAG(tezPlan, pc);
    TezDagBuilder dagBuilder = new TezDagBuilder(pc, tezPlan, tezDag, null);
    try {
        dagBuilder.visit();
    } catch (VisitorException jce) {
        assertTrue(((JobCreationException)jce.getCause()).getErrorCode() == 1068);
    }
}
 

@Test (timeout=60000)
public void testBasicTaskFailure() throws Exception {
  Configuration testConf = new Configuration(false);
  testConf.setBoolean(TestProcessor.getVertexConfName(
      TestProcessor.TEZ_FAILING_PROCESSOR_DO_FAIL, "v1"), true);
  testConf.set(TestProcessor.getVertexConfName(
      TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_TASK_INDEX, "v1"), "0");
  testConf.setInt(TestProcessor.getVertexConfName(
      TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_UPTO_TASK_ATTEMPT, "v1"), 0);
  
  //verify value at v2 task1
  testConf.set(TestProcessor.getVertexConfName(
      TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v2"), "1");
  //value of v2 task1 is 4.
  //v1 attempt0 has value of 1 (attempt index + 1). 
  //v1 attempt1 has value of 2 (attempt index + 1).
  //v3 attempt0 verifies value of 1 + 2 (values from input vertices) 
  // + 1 (attempt index + 1) = 4
  testConf.setInt(TestProcessor.getVertexConfName(
      TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v2", 1), 4);

  DAG dag = SimpleTestDAG.createDAG("testBasicTaskFailure", testConf);
  runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED);
}
 

private DAG createDAG(String dagName, boolean uv12CommitFail, boolean v3CommitFail) {
  DAG dag = DAG.create(dagName);
  Vertex v1 = Vertex.create("v1", ProcessorDescriptor.create("Proc"), 1);
  Vertex v2 = Vertex.create("v2", ProcessorDescriptor.create("Proc"), 1);
  Vertex v3 = Vertex.create("v3", ProcessorDescriptor.create("Proc"), 1);
  VertexGroup uv12 = dag.createVertexGroup("uv12", v1, v2);
  DataSinkDescriptor uv12DataSink = DataSinkDescriptor.create(
      OutputDescriptor.create("dummy output"), createOutputCommitterDesc(uv12CommitFail), null);
  uv12.addDataSink("uv12Out", uv12DataSink);
  DataSinkDescriptor v3DataSink = DataSinkDescriptor.create(
      OutputDescriptor.create("dummy output"), createOutputCommitterDesc(v3CommitFail), null);
  v3.addDataSink("v3Out", v3DataSink);

  GroupInputEdge e1 = GroupInputEdge.create(uv12, v3, EdgeProperty.create(
      DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
      SchedulingType.SEQUENTIAL,
      OutputDescriptor.create("dummy output class"),
      InputDescriptor.create("dummy input class")), InputDescriptor
      .create("merge.class"));
  dag.addVertex(v1)
    .addVertex(v2)
    .addVertex(v3)
    .addEdge(e1);
  return dag;
}
 

@Test (timeout=60000)
public void testTaskMultipleFailures() throws Exception {
  Configuration testConf = new Configuration(false);
  testConf.setBoolean(TestProcessor.getVertexConfName(
      TestProcessor.TEZ_FAILING_PROCESSOR_DO_FAIL, "v1"), true);
  testConf.set(TestProcessor.getVertexConfName(
      TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_TASK_INDEX, "v1"), "0,1");
  testConf.setInt(TestProcessor.getVertexConfName(
      TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_UPTO_TASK_ATTEMPT, "v1"), 1);
  
  //v1 task0,1 attempt 2 succeed. Input sum = 6. Plus one (v2 attempt0).
  //ending sum is 7.
  testConf.set(TestProcessor.getVertexConfName(
          TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v2"), "0");
  testConf.setInt(TestProcessor.getVertexConfName(
          TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v2", 0), 7);
  
  DAG dag = SimpleTestDAG.createDAG("testTaskMultipleFailures", testConf);
  runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED);
}
 

@Test (timeout=60000)
public void testBasicInputFailureWithoutExitDeadline() throws Exception {
  Configuration testConf = new Configuration(false);
  testConf.setInt(SimpleTestDAG.TEZ_SIMPLE_DAG_NUM_TASKS, 3); // 1 error < 0.4 fail fraction
  testConf.setBoolean(TestInput.getVertexConfName(
      TestInput.TEZ_FAILING_INPUT_DO_FAIL, "v2"), true);
  testConf.set(TestInput.getVertexConfName(
      TestInput.TEZ_FAILING_INPUT_FAILING_TASK_INDEX, "v2"), "2");
  testConf.set(TestInput.getVertexConfName(
      TestInput.TEZ_FAILING_INPUT_FAILING_TASK_ATTEMPT, "v2"), "0");
  testConf.set(TestInput.getVertexConfName(
      TestInput.TEZ_FAILING_INPUT_FAILING_INPUT_INDEX, "v2"), "0");
  
  DAG dag = SimpleTestDAG.createDAG("testBasicInputFailureWithoutExitDeadline", testConf);
  runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED);
}
 

@Test(timeout=120000)
public void testSessionDisableMultiAttempts() throws Exception {
  tezSession.stop();
  Path remoteStagingDir = remoteFs.makeQualified(new Path(TEST_ROOT_DIR, String
      .valueOf(new Random().nextInt(100000))));
  TezClientUtils.ensureStagingDirExists(conf, remoteStagingDir);
  TezConfiguration tezConf = createSessionConfig(remoteStagingDir);
  tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true);
  tezConf.setBoolean(TezConfiguration.DAG_RECOVERY_ENABLED, false);
  TezClient session = new TezClient("TestDAGRecovery2SingleAttemptOnly", tezConf);
  session.start();

  // DAG should fail as it never completes on the first attempt
  DAG dag = MultiAttemptDAG.createDAG("TestSingleAttemptDAG", null);
  runDAGAndVerify(dag, State.FAILED, session);
  session.stop();
}
 

/**
 * Test cascading input failure with exit. Expecting success.
 * v1 -- v2 -- v3
 * v3 all-tasks attempt0 input0 fails. v3 attempt0 exits. Triggering v2 rerun.
 * v2 task0 attempt1 input0 fails. v2 attempt1 exits. Triggering v1 rerun.
 * v1 attempt1 rerun and succeeds. v2 accepts v1 attempt1 output. v2 attempt2 succeeds.
 * v3 attempt1 accepts v2 attempt2 output.
 * 
 * AM vertex succeeded order is v1, v2, v3, v1, v2, v3.
 * @throws Exception
 */
@Test (timeout=60000)
public void testCascadingInputFailureWithExitSuccess() throws Exception {
  Configuration testConf = new Configuration(false);
  setCascadingInputFailureConfig(testConf, true);
  
  //v2 task0 attempt2 value = v1 task0 attempt1 (2) + v1 task1 attempt0 (1) + 3 = 6
  //v3 all-tasks attempt1 takes v2 task0 attempt2 value (6) + v2 task1 attempt0 (3) + 2 = 11
  testConf.set(TestProcessor.getVertexConfName(
          TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v3"), "0,1");
  testConf.setInt(TestProcessor.getVertexConfName(
          TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v3", 0), 11);
  testConf.setInt(TestProcessor.getVertexConfName(
          TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v3", 1), 11);
  
  DAG dag = SimpleTestDAG3Vertices.createDAG(
            "testCascadingInputFailureWithExitSuccess", testConf);
  runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED);
}
 

@Test(timeout=120000)
public void testSessionDisableMultiAttempts() throws Exception {
  tezSession.stop();
  Path remoteStagingDir = remoteFs.makeQualified(new Path(TEST_ROOT_DIR, String
      .valueOf(new Random().nextInt(100000))));
  TezClientUtils.ensureStagingDirExists(conf, remoteStagingDir);
  TezConfiguration tezConf = createSessionConfig(remoteStagingDir);
  tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true);
  tezConf.setBoolean(TezConfiguration.DAG_RECOVERY_ENABLED, false);
  TezClient session = TezClient.create("TestDAGRecovery2SingleAttemptOnly", tezConf);
  session.start();

  // DAG should fail as it never completes on the first attempt
  DAG dag = MultiAttemptDAG.createDAG("TestSingleAttemptDAG", null);
  runDAGAndVerify(dag, State.FAILED, session);
  session.stop();
}
 

/**
 * Fine-grained recovery task-level, In a vertex (v1), task 0 is done task 1
 * is also done. History flush happens. AM dies. Once AM is recovered, task 0
 * and Task 1 is not re-run. (Broadcast)
 *
 * @throws Exception
 */
@Test(timeout = 120000)
public void testVertexCompletelyFinished_Broadcast() throws Exception {
  DAG dag =
      createDAG("VertexCompletelyFinished_Broadcast", ControlledImmediateStartVertexManager.class,
          DataMovementType.BROADCAST, false);
  TezCounters counters = runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED);

  assertEquals(4, counters.findCounter(DAGCounter.NUM_SUCCEEDED_TASKS).getValue());
  assertEquals(2, counters.findCounter(TestCounter.Counter_1).getValue());

  List<HistoryEvent> historyEvents1 = readRecoveryLog(1);
  List<HistoryEvent> historyEvents2 = readRecoveryLog(2);
  printHistoryEvents(historyEvents1, 1);
  printHistoryEvents(historyEvents1, 2);
  // task_0 of v1 is finished in attempt 1, task_1 of v1 is not finished in
  // attempt 1
  assertEquals(1, findTaskAttemptFinishedEvent(historyEvents1, 0, 0).size());
  assertEquals(1, findTaskAttemptFinishedEvent(historyEvents1, 0, 1).size());

  // task_0 of v1 is finished in attempt 1 and not rerun, task_1 of v1 is
  // finished in attempt 2
  assertEquals(1, findTaskAttemptFinishedEvent(historyEvents2, 0, 0).size());
  assertEquals(1, findTaskAttemptFinishedEvent(historyEvents2, 0, 1).size());
}
 

/**
 * Test cascading input failure without exit. Expecting success.
 * v1 -- v2 -- v3
 * v3 all-tasks attempt0 input0 fails. Wait. Triggering v2 rerun.
 * v2 task0 attempt1 input0 fails. Wait. Triggering v1 rerun.
 * v1 attempt1 rerun and succeeds. v2 accepts v1 attempt1 output. v2 attempt1 succeeds.
 * v3 attempt0 accepts v2 attempt1 output.
 * 
 * AM vertex succeeded order is v1, v2, v1, v2, v3.
 * @throws Exception
 */
@Test (timeout=60000)
public void testCascadingInputFailureWithoutExitSuccess() throws Exception {
  Configuration testConf = new Configuration(false);
  setCascadingInputFailureConfig(testConf, false);
  
  //v2 task0 attempt1 value = v1 task0 attempt1 (2) + v1 task1 attempt0 (1) + 2 = 5
  //v3 all-tasks attempt0 takes v2 task0 attempt1 value (5) + v2 task1 attempt0 (3) + 1 = 9
  testConf.set(TestProcessor.getVertexConfName(
          TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v3"), "0,1");
  testConf.setInt(TestProcessor.getVertexConfName(
          TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v3", 0), 9);
  testConf.setInt(TestProcessor.getVertexConfName(
          TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v3", 1), 9);
  
  DAG dag = SimpleTestDAG3Vertices.createDAG(
            "testCascadingInputFailureWithoutExitSuccess", testConf);
  runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED);
}
 

@Test (timeout=60000)
public void testMultipleInputFailureWithoutExit() throws Exception {
  Configuration testConf = new Configuration(false);
  testConf.setBoolean(TestInput.getVertexConfName(
      TestInput.TEZ_FAILING_INPUT_DO_FAIL, "v2"), true);
  testConf.set(TestInput.getVertexConfName(
      TestInput.TEZ_FAILING_INPUT_FAILING_TASK_INDEX, "v2"), "0,1");
  testConf.set(TestInput.getVertexConfName(
      TestInput.TEZ_FAILING_INPUT_FAILING_TASK_ATTEMPT, "v2"), "0");
  testConf.set(TestInput.getVertexConfName(
      TestInput.TEZ_FAILING_INPUT_FAILING_INPUT_INDEX, "v2"), "-1");
  
  //v2 task0 attempt0 input0,1 fails. wait.
  //v1 task0 attempt1 reruns. v1 task1 attempt1 reruns.
  //2 + 2 + 1 = 5
  //same number for v2 task1
  testConf.set(TestProcessor.getVertexConfName(
          TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v2"), "0,1");
  testConf.setInt(TestProcessor.getVertexConfName(
          TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v2", 0), 5);
  testConf.setInt(TestProcessor.getVertexConfName(
          TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v2", 1), 5);
  
  DAG dag = SimpleTestDAG.createDAG("testMultipleInputFailureWithoutExit", testConf);
  runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED);
}
 

private List<StepCheck[]> testBasicTaskFailure() throws Exception {
  Configuration testConf = new Configuration(false);
  testConf.setInt(SimpleTestDAG.TEZ_SIMPLE_DAG_NUM_TASKS, 1);
  testConf.setBoolean(TestProcessor.getVertexConfName(
      TestProcessor.TEZ_FAILING_PROCESSOR_DO_FAIL, "v1"), true);
  testConf.set(TestProcessor.getVertexConfName(
      TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_TASK_INDEX, "v1"), "0");
  testConf.setInt(TestProcessor.getVertexConfName(
      TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_UPTO_TASK_ATTEMPT, "v1"), 0);

  StepCheck[] check = {
      createStep("v1 : 000000_0", CriticalPathDependency.INIT_DEPENDENCY),
      createStep("v1 : 000000_1", CriticalPathDependency.RETRY_DEPENDENCY),
      createStep("v2 : 000000_0", CriticalPathDependency.DATA_DEPENDENCY),
  };
  DAG dag = SimpleTestDAG.createDAG("testBasicTaskFailure", testConf);
  runDAG(dag, DAGStatus.State.SUCCEEDED);
  return Collections.singletonList(check);
}
 

private void testMemory(DAG dag, boolean sendDMEvents) throws Exception {
  StopWatch stopwatch = new StopWatch();
  stopwatch.start();
  TezConfiguration tezconf = new TezConfiguration(defaultConf);

  MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null,
      null, false, false, numThreads, 1000);
  tezClient.start();
  
  MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp();
  MockContainerLauncher mockLauncher = mockApp.getContainerLauncher();
  mockLauncher.startScheduling(false);
  mockApp.eventsDelegate = new TestMockDAGAppMaster.TestEventsDelegate();
  mockApp.doSleep = false;
  DAGClient dagClient = tezClient.submitDAG(dag);
  mockLauncher.waitTillContainersLaunched();
  mockLauncher.startScheduling(true);
  DAGStatus status = dagClient.waitForCompletion();
  Assert.assertEquals(DAGStatus.State.SUCCEEDED, status.getState());
  checkMemory(dag.getName(), mockApp);
  stopwatch.stop();
  System.out.println("Time taken(ms): " + stopwatch.now(TimeUnit.MILLISECONDS));
  tezClient.stop();
}
 

/**
 * v1 --> v2 <br>
 * v1 has a customized VM to control whether to schedule only one second task when it is partiallyFinished test case.
 * v2 has a customized VM which could control when to kill AM
 *
 * @param vertexManagerClass
 * @param dmType
 * @param failOnParitialCompleted
 * @return
 * @throws IOException
 */
private DAG createDAG(String dagName, Class vertexManagerClass, DataMovementType dmType,
    boolean failOnParitialCompleted) throws IOException {
  if (failOnParitialCompleted) {
    tezConf.set(FAIL_ON_PARTIAL_FINISHED, "true");
  } else {
    tezConf.set(FAIL_ON_PARTIAL_FINISHED, "false");
  }
  DAG dag = DAG.create(dagName);
  UserPayload payload = UserPayload.create(null);
  Vertex v1 = Vertex.create("v1", MyProcessor.getProcDesc(), 2);
  v1.setVertexManagerPlugin(VertexManagerPluginDescriptor.create(
      ScheduleControlledVertexManager.class.getName()).setUserPayload(
      TezUtils.createUserPayloadFromConf(tezConf)));
  Vertex v2 = Vertex.create("v2", DoNothingProcessor.getProcDesc(), 2);
  v2.setVertexManagerPlugin(VertexManagerPluginDescriptor.create(
      vertexManagerClass.getName()).setUserPayload(
      TezUtils.createUserPayloadFromConf(tezConf)));

  dag.addVertex(v1).addVertex(v2);
  dag.addEdge(Edge.create(v1, v2, EdgeProperty.create(dmType,
      DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
      TestOutput.getOutputDesc(payload), TestInput.getInputDesc(payload))));
  return dag;
}
 

private void setupURIsForCredentials(DAG dag, Path... paths) throws IOException {
  List<URI> uris = new LinkedList<URI>();
  for (Path path : paths) {
    FileSystem fs = path.getFileSystem(getConf());
    Path qPath = fs.makeQualified(path);
    uris.add(qPath.toUri());
  }
  dag.addURIsForCredentials(uris);
}
 

@Test(timeout=120000)
public void testFailingCommitter() throws Exception {
  DAG dag = SimpleVTestDAG.createDAG("FailingCommitterDAG", null);
  OutputDescriptor od =
      new OutputDescriptor(MultiAttemptDAG.NoOpOutput.class.getName());
  od.setUserPayload(new
      MultiAttemptDAG.FailingOutputCommitter.FailingOutputCommitterConfig(true)
          .toUserPayload());
  dag.getVertex("v3").addOutput("FailingOutput", od,
      MultiAttemptDAG.FailingOutputCommitter.class);
  runDAGAndVerify(dag, State.FAILED);
}
 

public static DAG createDAG(String name, 
        Configuration conf) throws Exception {
    if (conf != null) {
      payload = TezUtils.createUserPayloadFromConf(conf);
    } 
    dag = new DAG(name);
    addDAGVerticesAndEdges();
    return dag;
}
 

public static DAG createDAG(String name, 
    Configuration conf) throws Exception {
  byte[] payload = null;
  int taskCount = TEZ_SIMPLE_REVERSE_V_DAG_NUM_TASKS_DEFAULT;
  if (conf != null) {
    taskCount = conf.getInt(TEZ_SIMPLE_REVERSE_V_DAG_NUM_TASKS, TEZ_SIMPLE_REVERSE_V_DAG_NUM_TASKS_DEFAULT);
    payload = TezUtils.createUserPayloadFromConf(conf);
  }
  DAG dag = new DAG(name);
  Vertex v1 = new Vertex("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  Vertex v2 = new Vertex("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  Vertex v3 = new Vertex("v3", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  dag.addVertex(v1).addVertex(v2).addVertex(v3);
  dag.addEdge(new Edge(v1, v2, 
      new EdgeProperty(DataMovementType.SCATTER_GATHER, 
          DataSourceType.PERSISTED, 
          SchedulingType.SEQUENTIAL, 
          TestOutput.getOutputDesc(payload), 
          TestInput.getInputDesc(payload))));
  dag.addEdge(new Edge(v1, v3, 
          new EdgeProperty(DataMovementType.SCATTER_GATHER, 
              DataSourceType.PERSISTED, 
              SchedulingType.SEQUENTIAL, 
              TestOutput.getOutputDesc(payload), 
              TestInput.getInputDesc(payload))));
  return dag;
}
 

public static DAG createDAG(String name,
    Configuration conf) throws Exception {
  byte[] payload = null;
  int taskCount = MULTI_ATTEMPT_DAG_VERTEX_NUM_TASKS_DEFAULT;
  if (conf != null) {
    taskCount = conf.getInt(MULTI_ATTEMPT_DAG_VERTEX_NUM_TASKS, MULTI_ATTEMPT_DAG_VERTEX_NUM_TASKS_DEFAULT);
    payload = TezUtils.createUserPayloadFromConf(conf);
  }
  DAG dag = new DAG(name);
  Vertex v1 = new Vertex("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  Vertex v2 = new Vertex("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  Vertex v3 = new Vertex("v3", TestProcessor.getProcDesc(payload), taskCount, defaultResource);

  // Make each vertex manager fail on appropriate attempt
  v1.setVertexManagerPlugin(new VertexManagerPluginDescriptor(
      FailOnAttemptVertexManagerPlugin.class.getName())
      .setUserPayload(new String("1").getBytes()));
  v2.setVertexManagerPlugin(new VertexManagerPluginDescriptor(
      FailOnAttemptVertexManagerPlugin.class.getName())
      .setUserPayload(new String("2").getBytes()));
  v3.setVertexManagerPlugin(new VertexManagerPluginDescriptor(
      FailOnAttemptVertexManagerPlugin.class.getName())
      .setUserPayload(new String("3").getBytes()));
  dag.addVertex(v1).addVertex(v2).addVertex(v3);
  dag.addEdge(new Edge(v1, v2,
      new EdgeProperty(DataMovementType.SCATTER_GATHER,
          DataSourceType.PERSISTED,
          SchedulingType.SEQUENTIAL,
          TestOutput.getOutputDesc(payload),
          TestInput.getInputDesc(payload))));
  dag.addEdge(new Edge(v2, v3,
      new EdgeProperty(DataMovementType.SCATTER_GATHER,
          DataSourceType.PERSISTED,
          SchedulingType.SEQUENTIAL,
          TestOutput.getOutputDesc(payload),
          TestInput.getInputDesc(payload))));
  return dag;
}
 

@Test (timeout=60000)
public void testTaskMultipleFailuresDAGFail() throws Exception {
  Configuration testConf = new Configuration(false);
  testConf.setBoolean(TestProcessor.getVertexConfName(
      TestProcessor.TEZ_FAILING_PROCESSOR_DO_FAIL, "v1"), true);
  testConf.set(TestProcessor.getVertexConfName(
      TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_TASK_INDEX, "v1"), "0");
  testConf.setInt(TestProcessor.getVertexConfName(
      TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_UPTO_TASK_ATTEMPT, "v1"), -1);
  
  DAG dag = SimpleTestDAG.createDAG("testTaskMultipleFailuresDAGFail", testConf);
  runDAGAndVerify(dag, DAGStatus.State.FAILED);
}
 

@Override
protected int runJob(String[] args, TezConfiguration tezConf,
    TezClient tezClient) throws Exception {

  String inputDir1 = args[0];
  String inputDir2 = args[1];
  int numPartitions = Integer.parseInt(args[2]);
  String outputDir = args[3];

  Path inputPath1 = new Path(inputDir1);
  Path inputPath2 = new Path(inputDir2);
  Path outputPath = new Path(outputDir);

  // Verify output path existence
  FileSystem fs = outputPath.getFileSystem(tezConf);
  outputPath = fs.makeQualified(outputPath);
  if (fs.exists(outputPath)) {
    System.err.println("Output directory: " + outputDir + " already exists");
    return 3;
  }
  if (numPartitions <= 0) {
    System.err.println("NumPartitions must be > 0");
    return 4;
  }
  DAG dag =
      createDag(tezConf, inputPath1, inputPath2, outputPath, numPartitions);
  LOG.info("Running SortMergeJoinExample");
  return runDag(dag, isCountersLog(), LOG);
}
 

@Test (timeout = 5000)
public void testLocalResourceSetup() throws Exception {
  TezConfiguration tezconf = new TezConfiguration(defaultConf);
  
  MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null);
  tezClient.start();
  
  MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp();
  MockContainerLauncher mockLauncher = mockApp.getContainerLauncher();
  mockLauncher.startScheduling(false);
  
  Map<String, LocalResource> lrDAG = Maps.newHashMap();
  String lrName1 = "LR1";
  lrDAG.put(lrName1, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"),
      LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));
  Map<String, LocalResource> lrVertex = Maps.newHashMap();
  String lrName2 = "LR2";
  lrVertex.put(lrName2, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test1"),
      LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));

  DAG dag = DAG.create("testLocalResourceSetup").addTaskLocalFiles(lrDAG);
  Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), 5).addTaskLocalFiles(lrVertex);
  dag.addVertex(vA);

  DAGClient dagClient = tezClient.submitDAG(dag);
  mockLauncher.waitTillContainersLaunched();
  ContainerData cData = mockLauncher.getContainers().values().iterator().next();
  ContainerLaunchContext launchContext = cData.launchContext;
  Map<String, LocalResource> taskLR = launchContext.getLocalResources();
  // verify tasks are launched with both DAG and task resources.
  Assert.assertTrue(taskLR.containsKey(lrName1));
  Assert.assertTrue(taskLR.containsKey(lrName2));
  
  mockLauncher.startScheduling(true);
  dagClient.waitForCompletion();
  Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
  tezClient.stop();
}
 

@Test (timeout=60000)
public void testThreeLevelsFailingDAG2VerticesHaveFailedAttemptsDAGSucceeds() throws Exception {
  Configuration testConf = new Configuration();
  //set maximum number of task attempts to 4
  testConf.setInt(TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS, 4);
  //l2v1 failure
  testConf.setBoolean(TestProcessor.getVertexConfName(
          TestProcessor.TEZ_FAILING_PROCESSOR_DO_FAIL, "l2v1"), true);
  testConf.set(TestProcessor.getVertexConfName(
          TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_TASK_INDEX, "l2v1"), "1");
  //3 attempts fail
  testConf.setInt(TestProcessor.getVertexConfName(
          TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_UPTO_TASK_ATTEMPT, "l2v1"), 2);
  
  //l3v1 failure
  testConf.setBoolean(TestProcessor.getVertexConfName(
          TestProcessor.TEZ_FAILING_PROCESSOR_DO_FAIL, "l3v1"), true);
  testConf.set(TestProcessor.getVertexConfName(
          TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_TASK_INDEX, "l3v1"), "0");
  //3 attempts fail
  testConf.setInt(TestProcessor.getVertexConfName(
          TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_UPTO_TASK_ATTEMPT, "l3v1"), 2);
  
  //l2v1: task0 attempt0 succeeds. task1 attempt3 succeeds. 
  //l3v1 finally task0 attempt3 will succeed.
  //l1v1 outputs 1. l1v2 outputs 2.
  //l2v1 task0 attempt0 output = 2. 
  //l2v2 output: attempt0 (l1v2+self = 2+1) * 3 tasks = 9
  //l3v1 task0 attempt3 = l2v1 (2) + l2v2 (9) + self (4) = 15
  testConf.set(TestProcessor.getVertexConfName(
          TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "l3v1"), "0");
  testConf.setInt(TestProcessor.getVertexConfName(
          TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "l3v1", 0), 15);
  
  DAG dag = ThreeLevelsFailingDAG.createDAG("testThreeLevelsFailingDAG2VerticesHaveFailedAttemptsDAGSucceeds", testConf);
  runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED);
}
 

/**
 * Input failure of v3 causes rerun of both both v1 and v2 vertices. 
 *   v1  v2
 *    \ /
 *    v3
 * 
 * @throws Exception
 */
@Test (timeout=60000)
public void testInputFailureCausesRerunOfTwoVerticesWithoutExit() throws Exception {
  Configuration testConf = new Configuration(false);
  testConf.setBoolean(TestInput.getVertexConfName(
      TestInput.TEZ_FAILING_INPUT_DO_FAIL, "v3"), true);
  testConf.setBoolean(TestInput.getVertexConfName(
      TestInput.TEZ_FAILING_INPUT_DO_FAIL_AND_EXIT, "v3"), false);
  testConf.set(TestInput.getVertexConfName(
      TestInput.TEZ_FAILING_INPUT_FAILING_TASK_INDEX, "v3"), "0,1");
  testConf.set(TestInput.getVertexConfName(
      TestInput.TEZ_FAILING_INPUT_FAILING_TASK_ATTEMPT, "v3"), "0");
  testConf.set(TestInput.getVertexConfName(
      TestInput.TEZ_FAILING_INPUT_FAILING_INPUT_INDEX, "v3"), "-1");
  testConf.set(TestInput.getVertexConfName(
      TestInput.TEZ_FAILING_INPUT_FAILING_UPTO_INPUT_ATTEMPT, "v3"), "1");
  
  //v3 attempt0:
  //v1 task0,1 attempt2 = 6. v2 task0,1 attempt2 = 6.
  //total = 6 + 6 + 1 = 13
  testConf.set(TestProcessor.getVertexConfName(
          TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v3"), "0");
  testConf.setInt(TestProcessor.getVertexConfName(
          TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v3", 0), 13);
  
  DAG dag = SimpleVTestDAG.createDAG(
          "testInputFailureCausesRerunOfTwoVerticesWithoutExit", testConf);
  runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED);
}