Java Code Examples for org.apache.tez.dag.api.DAG#getVertices()

The following examples show how to use org.apache.tez.dag.api.DAG#getVertices() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: TestJobSubmissionTez.java From spork with Apache License 2.0

6 votes

@Override
public void checkDefaultParallelResult(PhysicalPlan pp, PigContext pc) throws Exception {
    TezOperPlan tezPlan = buildTezPlan(pp, pc);

    LoaderProcessor loaderStorer = new LoaderProcessor(tezPlan, pc);
    loaderStorer.visit();

    ParallelismSetter parallelismSetter = new ParallelismSetter(tezPlan, pc);
    parallelismSetter.visit();

    DAG tezDag = getTezDAG(tezPlan, pc);
    TezDagBuilder dagBuilder = new TezDagBuilder(pc, tezPlan, tezDag, null);
    dagBuilder.visit();
    for (Vertex v : tezDag.getVertices()) {
        if (!v.getInputVertices().isEmpty()) {
            Configuration conf = TezUtils.createConfFromUserPayload(v.getProcessorDescriptor().getUserPayload());
            int parallel = v.getParallelism();
            assertEquals(parallel, 100);
            Util.assertConfLong(conf, "pig.info.reducers.default.parallel", 100);
            Util.assertConfLong(conf, "pig.info.reducers.requested.parallel", -1);
            Util.assertConfLong(conf, "pig.info.reducers.estimated.parallel", -1);
        }
    }
}

Example 2

Source File: TestGroupConstParallelTez.java From spork with Apache License 2.0

6 votes

@Override
public void checkGroupConstWithParallelResult(PhysicalPlan pp, PigContext pc) throws Exception {
    TezOperPlan tezPlan = buildTezPlan(pp, pc);

    LoaderProcessor loaderStorer = new LoaderProcessor(tezPlan, pc);
    loaderStorer.visit();

    ParallelismSetter parallelismSetter = new ParallelismSetter(tezPlan, pc);
    parallelismSetter.visit();

    DAG tezDag = getTezDAG(tezPlan, pc);
    TezDagBuilder dagBuilder = new TezDagBuilder(pc, tezPlan, tezDag, null);
    dagBuilder.visit();
    for (Vertex v : tezDag.getVertices()) {
        if (!v.getInputVertices().isEmpty()) {
            assertEquals(v.getParallelism(), 1);
        }
    }
}

Example 3

Source File: TestGroupConstParallelTez.java From spork with Apache License 2.0

6 votes

@Override
public void checkGroupNonConstWithParallelResult(PhysicalPlan pp, PigContext pc) throws Exception {
    TezOperPlan tezPlan = buildTezPlan(pp, pc);

    LoaderProcessor loaderStorer = new LoaderProcessor(tezPlan, pc);
    loaderStorer.visit();

    ParallelismSetter parallelismSetter = new ParallelismSetter(tezPlan, pc);
    parallelismSetter.visit();

    DAG tezDag = getTezDAG(tezPlan, pc);
    TezDagBuilder dagBuilder = new TezDagBuilder(pc, tezPlan, tezDag, null);
    dagBuilder.visit();
    for (Vertex v : tezDag.getVertices()) {
        if (!v.getInputVertices().isEmpty()) {
            assertEquals(v.getParallelism(), 100);
        }
    }
}

Example 4

Source File: TezLauncher.java From spork with Apache License 2.0

5 votes

public boolean notifyFinishedOrFailed() {
    DAGStatus dagStatus = runningJob.getDAGStatus();
    if (dagStatus == null) {
        return false;
    }
    if (dagStatus.getState() == DAGStatus.State.SUCCEEDED) {
        Map<Enum, Long> warningAggMap = new HashMap<Enum, Long>();
        DAG dag = runningJob.getDAG();
        for (Vertex v : dag.getVertices()) {
            TezVertexStats tts = tezStats.getVertexStats(dag.getName(), v.getName());
            if (tts == null) {
                continue; //vertex groups
            }
            Map<String, Map<String, Long>> counterGroups = tts.getCounters();
            if (counterGroups == null) {
                log.warn("Counters are not available for vertex " + v.getName() + ". Not computing warning aggregates.");
            } else {
                computeWarningAggregate(counterGroups, warningAggMap);
            }
        }
        if (aggregateWarning) {
            CompilationMessageCollector.logAggregate(warningAggMap, MessageType.Warning, log);
        }
        return true;
    }
    return false;
}

Example 5

Source File: TezClient.java From incubator-tez with Apache License 2.0

4 votes

private synchronized DAGClient submitDAGSession(DAG dag)
  throws TezException, IOException, InterruptedException {
  Preconditions.checkState(isSession == true, 
      "submitDAG with additional resources applies to only session mode. " + 
      "In non-session mode please specify all resources in the initial configuration");
  
  verifySessionStateForSubmission();

  String dagId = null;
  LOG.info("Submitting dag to TezSession"
    + ", sessionName=" + clientName
    + ", applicationId=" + sessionAppId
    + ", dagName=" + dag.getName());
  
  if (!additionalLocalResources.isEmpty()) {
    for (LocalResource lr : additionalLocalResources.values()) {
      Preconditions.checkArgument(lr.getType() == LocalResourceType.FILE, "LocalResourceType: "
          + lr.getType() + " is not supported, only " + LocalResourceType.FILE + " is supported");
    }
  }

  // Obtain DAG specific credentials.
  TezClientUtils.setupDAGCredentials(dag, sessionCredentials, amConfig.getTezConfiguration());

  // TODO TEZ-1229 - fix jar resources
  // setup env
  for (Vertex v : dag.getVertices()) {
    Map<String, String> taskEnv = v.getTaskEnvironment();
    TezYARNUtils.setupDefaultEnv(taskEnv, amConfig.getTezConfiguration(),
        TezConfiguration.TEZ_TASK_LAUNCH_ENV, TezConfiguration.TEZ_TASK_LAUNCH_ENV_DEFAULT);
    TezClientUtils.setDefaultLaunchCmdOpts(v, amConfig.getTezConfiguration());
  }
  
  DAGPlan dagPlan = dag.createDag(amConfig.getTezConfiguration());
  SubmitDAGRequestProto.Builder requestBuilder = SubmitDAGRequestProto.newBuilder();
  requestBuilder.setDAGPlan(dagPlan).build();
  if (!additionalLocalResources.isEmpty()) {
    requestBuilder.setAdditionalAmResources(DagTypeConverters
        .convertFromLocalResources(additionalLocalResources));
  }
  
  additionalLocalResources.clear();

  DAGClientAMProtocolBlockingPB proxy = waitForProxy();
  if (proxy == null) {
    try {
      LOG.warn("DAG submission to session timed out, stopping session");
      stop();
    } catch (Throwable t) {
      LOG.info("Got an exception when trying to stop session", t);
    }
    throw new DAGSubmissionTimedOut("Could not submit DAG to Tez Session"
        + ", timed out after " + clientTimeout + " seconds");
  }

  try {
    SubmitDAGResponseProto response = proxy.submitDAG(null, requestBuilder.build());
    // the following check is only for testing since the final class
    // SubmitDAGResponseProto cannot be mocked
    if (response != null) {
      dagId = response.getDagId();
    }
  } catch (ServiceException e) {
    throw new TezException(e);
  }
  LOG.info("Submitted dag to TezSession"
      + ", sessionName=" + clientName
      + ", applicationId=" + sessionAppId
      + ", dagName=" + dag.getName());
  return new DAGClientRPCImpl(sessionAppId, dagId,
      amConfig.getTezConfiguration());
}