Java Code Examples for org.apache.tez.dag.api.client.DAGClient#waitForCompletionWithStatusUpdates()

private void runAndVerifyForNonFatalErrors(TezClient tezClient, String componentName,
                                           Vertex.VertexExecutionContext lhsContext) throws
    TezException,
    InterruptedException, IOException {
  LOG.info("Running JoinValidate with componentName reportNonFatalException");
  JoinValidateConfigured joinValidate =
      new JoinValidateConfigured(EXECUTION_CONTEXT_DEFAULT, lhsContext,
          EXECUTION_CONTEXT_EXT_SERVICE_PUSH,
          EXECUTION_CONTEXT_EXT_SERVICE_PUSH, componentName);

  DAG dag = joinValidate
      .createDag(new TezConfiguration(extServiceTestHelper.getConfForJobs()),
          HASH_JOIN_EXPECTED_RESULT_PATH,
          HASH_JOIN_OUTPUT_PATH, 3);

  DAGClient dagClient = tezClient.submitDAG(dag);

  DAGStatus dagStatus =
      dagClient.waitForCompletionWithStatusUpdates(Sets.newHashSet(StatusGetOpts.GET_COUNTERS));
  assertEquals(DAGStatus.State.FAILED, dagStatus.getState());

  boolean foundDiag = false;
  for (String diag : dagStatus.getDiagnostics()) {
    if (diag.contains(ErrorPluginConfiguration.REPORT_NONFATAL_ERROR_MESSAGE) &&
        diag.contains(ServicePluginErrorDefaults.SERVICE_UNAVAILABLE.name())) {
      foundDiag = true;
      break;
    }
  }
  assertTrue(foundDiag);
}
 

/**
 * @param dag           the dag to execute
 * @param printCounters whether to print counters or not
 * @param logger        the logger to use while printing diagnostics
 * @return Zero indicates success, non-zero indicates failure
 * @throws TezException
 * @throws InterruptedException
 * @throws IOException
 */
public int runDag(DAG dag, boolean printCounters, Logger logger) throws TezException,
    InterruptedException, IOException {
  tezClientInternal.waitTillReady();

  CallerContext callerContext = CallerContext.create("TezExamples",
      "Tez Example DAG: " + dag.getName());
  ApplicationId appId = tezClientInternal.getAppMasterApplicationId();
  if (hadoopShim == null) {
    Configuration conf = (getConf() == null ? new Configuration(false) : getConf());
    hadoopShim = new HadoopShimsLoader(conf).getHadoopShim();
  }

  if (appId != null) {
    TezUtilsInternal.setHadoopCallerContext(hadoopShim, appId);
    callerContext.setCallerIdAndType(appId.toString(), "TezExampleApplication");
  }
  dag.setCallerContext(callerContext);

  DAGClient dagClient = tezClientInternal.submitDAG(dag);
  Set<StatusGetOpts> getOpts = Sets.newHashSet();
  if (printCounters) {
    getOpts.add(StatusGetOpts.GET_COUNTERS);
  }

  DAGStatus dagStatus;
  dagStatus = dagClient.waitForCompletionWithStatusUpdates(getOpts);

  if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) {
    logger.info("DAG diagnostics: " + dagStatus.getDiagnostics());
    return -1;
  }
  return 0;
}
 

TezCounters runDAGAndVerify(DAG dag, DAGStatus.State finalState) throws Exception {
  tezSession.waitTillReady();
  DAGClient dagClient = tezSession.submitDAG(dag);
  DAGStatus dagStatus =
      dagClient.waitForCompletionWithStatusUpdates(EnumSet
          .of(StatusGetOpts.GET_COUNTERS));
  Assert.assertEquals(finalState, dagStatus.getState());
  return dagStatus.getDAGCounters();
}
 

private int execute(String[] args, TezConfiguration tezConf, TezClient tezClient)
        throws IOException, TezException, InterruptedException {
    LOG.info("Running TopK DataGen");

    UserGroupInformation.setConfiguration(tezConf);

    String outDir = args[0];
    long outDirSize = Long.parseLong(args[1]);

    int numExtraColumns = 0;
    if (args.length > 2) {
        numExtraColumns = Integer.parseInt(args[2]);
    }
    int numTasks = 5;
    if (args.length > 3) {
        numTasks = Integer.parseInt(args[3]);
    }

    Path outPath = new Path(outDir);

    // Verify output path existence
    FileSystem fs = FileSystem.get(tezConf);
    int res = 0;
    res = checkOutputDirectory(fs, outPath);
    if (res != 0) {
        return 3;
    }

    if (numTasks <= 0) {
        System.err.println("NumTasks must be > 0");
        return 4;
    }

    DAG dag = createDag(tezConf, outPath, outDirSize, numExtraColumns, numTasks);

    tezClient.waitTillReady();
    DAGClient dagClient = tezClient.submitDAG(dag);
    DAGStatus dagStatus = dagClient.waitForCompletionWithStatusUpdates(null);
    if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) {
        LOG.info("DAG diagnostics: " + dagStatus.getDiagnostics());
        return -1;
    }
    return 0;

}
 

@Override
protected int runJob(String[] args, TezConfiguration tezConf,
    TezClient tezClient) throws Exception {

  LOG.info("Running JoinValidate");

  String lhsDir = args[0];
  String rhsDir = args[1];
  int numPartitions = 1;
  if (args.length == 3) {
    numPartitions = Integer.parseInt(args[2]);
  }

  if (numPartitions <= 0) {
    System.err.println("NumPartitions must be > 0");
    return 4;
  }

  Path lhsPath = new Path(lhsDir);
  Path rhsPath = new Path(rhsDir);

  DAG dag = createDag(tezConf, lhsPath, rhsPath, numPartitions);

  tezClient.waitTillReady();
  DAGClient dagClient = tezClient.submitDAG(dag);
  Set<StatusGetOpts> getOpts = Sets.newHashSet();
  if (isCountersLog()) {
    getOpts.add(StatusGetOpts.GET_COUNTERS);
  }
  DAGStatus dagStatus = dagClient.waitForCompletionWithStatusUpdates(getOpts);
  if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) {
    LOG.info("DAG diagnostics: " + dagStatus.getDiagnostics());
    return -1;
  } else {
    dagStatus = dagClient.getDAGStatus(Sets.newHashSet(StatusGetOpts.GET_COUNTERS));
    TezCounter counter = dagStatus.getDAGCounters().findCounter(COUNTER_GROUP_NAME,
        MISSING_KEY_COUNTER_NAME);
    if (counter == null) {
      LOG.info("Unable to determing equality");
      return -2;
    } else {
      if (counter.getValue() != 0) {
        LOG.info("Validate failed. The two sides are not equivalent");
        return -3;
      } else {
        LOG.info("Validation successful. The two sides are equivalent");
        return 0;
      }
    }
  }

}
 

@Override
public int run(String[] args) throws Exception {
  this.tezConf = new TezConfiguration(getConf());
  String dagName = "pipelinedShuffleTest";
  DAG dag = DAG.create(dagName);

  Vertex m1_Vertex = Vertex.create("mapper1",
      ProcessorDescriptor.create(DataGenerator.class.getName()), 1);

  Vertex m2_Vertex = Vertex.create("mapper2",
      ProcessorDescriptor.create(DataGenerator.class.getName()), 1);

  Vertex reducerVertex = Vertex.create("reducer",
      ProcessorDescriptor.create(SimpleReduceProcessor.class.getName()), 1);

  Edge mapper1_to_reducer = Edge.create(m1_Vertex, reducerVertex,
      OrderedPartitionedKVEdgeConfig
          .newBuilder(Text.class.getName(), Text.class.getName(),
              HashPartitioner.class.getName())
          .setFromConfiguration(tezConf).build().createDefaultEdgeProperty());

  Edge mapper2_to_reducer = Edge.create(m2_Vertex, reducerVertex,
      OrderedPartitionedKVEdgeConfig
          .newBuilder(Text.class.getName(), Text.class.getName(),
              HashPartitioner.class.getName())
          .setFromConfiguration(tezConf).build().createDefaultEdgeProperty());

  dag.addVertex(m1_Vertex);
  dag.addVertex(m2_Vertex);
  dag.addVertex(reducerVertex);

  dag.addEdge(mapper1_to_reducer).addEdge(mapper2_to_reducer);

  TezClient client = TezClient.create(dagName, tezConf);
  client.start();
  client.waitTillReady();

  DAGClient dagClient = client.submitDAG(dag);
  Set<StatusGetOpts> getOpts = Sets.newHashSet();
  getOpts.add(StatusGetOpts.GET_COUNTERS);

  DAGStatus dagStatus = dagClient.waitForCompletionWithStatusUpdates(getOpts);

  System.out.println(dagStatus.getDAGCounters());
  TezCounters counters = dagStatus.getDAGCounters();

  //Ensure that atleast 10 spills were there in this job.
  assertTrue(counters.findCounter(TaskCounter.SHUFFLE_CHUNK_COUNT).getValue() > 10);

  if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) {
    System.out.println("DAG diagnostics: " + dagStatus.getDiagnostics());
    return -1;
  }
  return 0;
}
 

private String runWordCount(String tokenizerProcessor, String summationProcessor,
    String dagName, boolean withTimeline)
    throws Exception {
  //HDFS path
  Path outputLoc = new Path("/tmp/outPath_" + System.currentTimeMillis());

  DataSourceDescriptor dataSource = MRInput.createConfigBuilder(conf,
      TextInputFormat.class, inputLoc.toString()).build();

  DataSinkDescriptor dataSink =
      MROutput.createConfigBuilder(conf, TextOutputFormat.class, outputLoc.toString()).build();

  Vertex tokenizerVertex = Vertex.create(TOKENIZER, ProcessorDescriptor.create(
      tokenizerProcessor)).addDataSource(INPUT, dataSource);

  OrderedPartitionedKVEdgeConfig edgeConf = OrderedPartitionedKVEdgeConfig
      .newBuilder(Text.class.getName(), IntWritable.class.getName(),
          HashPartitioner.class.getName()).build();

  Vertex summationVertex = Vertex.create(SUMMATION,
      ProcessorDescriptor.create(summationProcessor), 1).addDataSink(OUTPUT, dataSink);

  // Create DAG and add the vertices. Connect the producer and consumer vertices via the edge
  DAG dag = DAG.create(dagName);
  dag.addVertex(tokenizerVertex).addVertex(summationVertex).addEdge(
      Edge.create(tokenizerVertex, summationVertex, edgeConf.createDefaultEdgeProperty()));

  TezClient tezClient = getTezClient(withTimeline);

  // Update Caller Context
  CallerContext callerContext = CallerContext.create("TezExamples", "Tez WordCount Example Job");
  ApplicationId appId = tezClient.getAppMasterApplicationId();
  if (appId == null) {
    appId = ApplicationId.newInstance(1001l, 1);
  }
  callerContext.setCallerIdAndType(appId.toString(), "TezApplication");
  dag.setCallerContext(callerContext);

  DAGClient client = tezClient.submitDAG(dag);
  client.waitForCompletionWithStatusUpdates(Sets.newHashSet(StatusGetOpts.GET_COUNTERS));
  TezDAGID tezDAGID = TezDAGID.getInstance(tezClient.getAppMasterApplicationId(), 1);

  if (tezClient != null) {
    tezClient.stop();
  }
  return tezDAGID.toString();
}