org.apache.tez.dag.api.UserPayload Java Examples

static OutputContext createOutputContext() throws IOException {
  OutputContext outputContext = mock(OutputContext.class);
  Configuration conf = new TezConfiguration();
  UserPayload payLoad = TezUtils.createUserPayloadFromConf(conf);
  String[] workingDirs = new String[]{"workDir1"};
  OutputStatisticsReporter statsReporter = mock(OutputStatisticsReporter.class);
  TezCounters counters = new TezCounters();

  doReturn("destinationVertex").when(outputContext).getDestinationVertexName();
  doReturn(payLoad).when(outputContext).getUserPayload();
  doReturn(workingDirs).when(outputContext).getWorkDirs();
  doReturn(200 * 1024 * 1024l).when(outputContext).getTotalMemoryAvailableToTask();
  doReturn(counters).when(outputContext).getCounters();
  doReturn(statsReporter).when(outputContext).getStatisticsReporter();
  doReturn(new Configuration(false)).when(outputContext).getContainerConfiguration();
  return outputContext;
}
 

@Test(timeout = 5000)
public void testSerializationFair() throws Exception {
  List<String> sourceVertices = new ArrayList<>();
  sourceVertices.add("v1");
  sourceVertices.add("v2");
  sourceVertices.add("v3");
  CartesianProductConfig config =
    new CartesianProductConfig(sourceVertices);
  UserPayload payload = config.toUserPayload(conf);
  CartesianProductConfig parsedConfig = CartesianProductConfig.fromUserPayload(payload);
  assertConfigEquals(config, parsedConfig);

  // fair cartesian product config should have null in numPartitions fields
  try {
    config = new CartesianProductConfig(false, new int[]{}, new String[]{"v0","v1"},null);
    config.checkNumPartitions();
  } catch (Exception e) {
    return;
  }
  throw new Exception();
}
 

@Override
protected ContainerLauncherManager createContainerLauncherManager(
    List<NamedEntityDescriptor> containerLauncherDescirptors,
    boolean isLocal)
    throws UnknownHostException {
  UserPayload userPayload;
  try {
    userPayload = TezUtils.createUserPayloadFromConf(new Configuration(false));
  } catch (IOException e) {
    throw new TezUncheckedException(e);
  }
  ContainerLauncherManager clManager = new ContainerLauncherManager(getContext());
  ContainerLauncherContext containerLauncherContext =
      new ContainerLauncherContextImpl(getContext(), clManager, getTaskCommunicatorManager(), userPayload, 0);
  containerLauncher = new MockContainerLauncher(launcherGoFlag, containerLauncherContext);
  clManager.setContainerLauncher(containerLauncher);
  return clManager;
}
 

@Test(timeout = 5000)
public void testSerializationPartitioned() throws IOException {
  Map<String, Integer> vertexPartitionMap = new HashMap<>();
  vertexPartitionMap.put("v1", 2);
  vertexPartitionMap.put("v2", 3);
  vertexPartitionMap.put("v3", 4);
  String filterClassName = "filter";
  byte[] bytes = new byte[10];
  (new Random()).nextBytes(bytes);
  CartesianProductFilterDescriptor filterDescriptor =
    new CartesianProductFilterDescriptor(filterClassName)
      .setUserPayload(UserPayload.create(ByteBuffer.wrap(bytes)));
  CartesianProductConfig config =
    new CartesianProductConfig(vertexPartitionMap, filterDescriptor);
  UserPayload payload = config.toUserPayload(conf);
  CartesianProductConfig parsedConfig = CartesianProductConfig.fromUserPayload(payload);
  assertConfigEquals(config, parsedConfig);
}
 

static FairEdgeConfiguration fromUserPayload(UserPayload payload)
    throws InvalidProtocolBufferException {
  HashMap<Integer, DestinationTaskInputsProperty> routingTable = new HashMap<>();
  FairShuffleEdgeManagerConfigPayloadProto proto =
      FairShuffleEdgeManagerConfigPayloadProto.parseFrom(
          ByteString.copyFrom(payload.getPayload()));
  int numBuckets = proto.getNumBuckets();
  if (proto.getDestinationTaskPropsList() != null) {
    for (int i = 0; i < proto.getDestinationTaskPropsList().size(); i++) {
      FairShuffleEdgeManagerDestinationTaskPropProto propProto =
          proto.getDestinationTaskPropsList().get(i);
      routingTable.put(
          propProto.getDestinationTaskIndex(),
          new DestinationTaskInputsProperty(
              propProto.getPartitions().getFirstIndex(),
              propProto.getPartitions().getNumOfIndexes(),
              propProto.getSourceTasks().getFirstIndex(),
              propProto.getSourceTasks().getNumOfIndexes()));
    }
  }
  return new FairEdgeConfiguration(numBuckets, routingTable);
}
 

/**
 * Creates the user payload to be set on the OutputDescriptor for MROutput
 */
private UserPayload createUserPayload() {
  // set which api is being used always
  conf.setBoolean(MRJobConfig.NEW_API_REDUCER_CONFIG, useNewApi);
  conf.setBoolean(MRJobConfig.NEW_API_MAPPER_CONFIG, useNewApi);
  if (outputFormatProvided) {
    if (useNewApi) {
      conf.set(MRJobConfig.OUTPUT_FORMAT_CLASS_ATTR, outputFormat.getName());
    } else {
      conf.set("mapred.output.format.class", outputFormat.getName());
    }
  }
  MRHelpers.translateMRConfToTez(conf);
  try {
    return TezUtils.createUserPayloadFromConf(conf);
  } catch (IOException e) {
    throw new TezUncheckedException(e);
  }
}
 

private void _testPipelinedShuffle(String sorterName) throws Exception {
  conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 3);

  conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_SORTER_CLASS, sorterName);
  conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_ENABLE_FINAL_MERGE_IN_OUTPUT, false);
  conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_PIPELINED_SHUFFLE_ENABLED, true);
  OutputContext context = createTezOutputContext();
  UserPayload payLoad = TezUtils.createUserPayloadFromConf(conf);
  doReturn(payLoad).when(context).getUserPayload();
  sortedOutput = new OrderedPartitionedKVOutput(context, partitions);

  sortedOutput.initialize();
  sortedOutput.start();

  assertFalse(sortedOutput.finalMergeEnabled);
  assertTrue(sortedOutput.pipelinedShuffle);
}
 

private DataSourceDescriptor createGeneratorDataSource() throws IOException {
  setupBasicConf(conf);
  MRHelpers.translateMRConfToTez(conf);
  
  Collection<URI> uris = maybeGetURIsForCredentials();

  UserPayload payload = MRInputHelpersInternal.createMRInputPayload(
      conf, groupSplitsInAM, sortSplitsInAM);

  DataSourceDescriptor ds = DataSourceDescriptor.create(
      InputDescriptor.create(inputClassName).setUserPayload(payload),
      InputInitializerDescriptor.create(MRInputAMSplitGenerator.class.getName()), null);

  if (conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT,
      TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT_DEFAULT)) {
    ds.getInputDescriptor().setHistoryText(TezUtils.convertToHistoryText(conf));
  }

  if (uris != null) {
    ds.addURIsForCredentials(uris);
  }
  return ds;
}
 

@Test (timeout = 5000)
public void testNewAPI_MapperOnly() throws Exception {
  String outputPath = TEST_DIR.getAbsolutePath();
  Job job = Job.getInstance();
  job.setOutputKeyClass(NullWritable.class);
  job.setOutputValueClass(Text.class);
  job.setOutputFormatClass(SequenceFileOutputFormat.class);
  SequenceFileOutputFormat.setOutputPath(job, new Path(outputPath));
  job.getConfiguration().setBoolean("mapred.mapper.new-api", true);
  // the output is attached to mapper
  job.getConfiguration().setBoolean(MRConfig.IS_MAP_PROCESSOR, true);
  UserPayload vertexPayload = TezUtils.createUserPayloadFromConf(job.getConfiguration());
  OutputDescriptor od = OutputDescriptor.create(MROutputLegacy.class.getName())
      .setUserPayload(vertexPayload);
  DataSinkDescriptor sink = DataSinkDescriptor.create(od,
      OutputCommitterDescriptor.create(MROutputCommitter.class.getName()), null);

  OutputContext outputContext = createMockOutputContext(sink.getOutputDescriptor().getUserPayload());
  MROutputLegacy output = new MROutputLegacy(outputContext, 2);
  output.initialize();
  assertEquals(true, output.useNewApi);
  assertEquals(SequenceFileOutputFormat.class, output.newOutputFormat.getClass());
  assertNull(output.oldOutputFormat);
  assertEquals(NullWritable.class, output.newApiTaskAttemptContext.getOutputKeyClass());
  assertEquals(Text.class, output.newApiTaskAttemptContext.getOutputValueClass());
  assertNull(output.oldApiTaskAttemptContext);
  assertNotNull(output.newRecordWriter);
  assertNull(output.oldRecordWriter);
  assertEquals(FileOutputCommitter.class, output.committer.getClass());
}
 

private InputContext createTezInputContext() {
  TezCounters counters = new TezCounters();
  InputContext inputContext = mock(InputContext.class);
  doReturn(1024 * 1024 * 100l).when(inputContext).getTotalMemoryAvailableToTask();
  doReturn(counters).when(inputContext).getCounters();
  doReturn(1).when(inputContext).getInputIndex();
  doReturn("srcVertex").when(inputContext).getSourceVertexName();
  doReturn(1).when(inputContext).getTaskVertexIndex();
  doReturn(UserPayload.create(ByteBuffer.wrap(new byte[1024]))).when(inputContext).getUserPayload();
  return inputContext;
}
 

public ContainerLauncherContextImpl(AppContext appContext, ContainerLauncherManager containerLauncherManager,
                                    TaskCommunicatorManagerInterface tal,
                                    UserPayload initialUserPayload, int containerLauncherIndex) {
  Objects.requireNonNull(appContext, "AppContext cannot be null");
  Objects.requireNonNull(appContext, "ContainerLauncherManager cannot be null");
  Objects.requireNonNull(tal, "TaskCommunicator cannot be null");
  this.context = appContext;
  this.containerLauncherManager = containerLauncherManager;
  this.tal = tal;
  this.initialUserPayload = initialUserPayload;
  this.containerLauncherIndex = containerLauncherIndex;
}
 

@Test(timeout = 5000)
public void testCustomTaskCommSpecified() throws IOException, TezException {

  AppContext appContext = mock(AppContext.class);
  TaskHeartbeatHandler thh = mock(TaskHeartbeatHandler.class);
  ContainerHeartbeatHandler chh = mock(ContainerHeartbeatHandler.class);

  String customTaskCommName = "customTaskComm";
  List<NamedEntityDescriptor> taskCommDescriptors = new LinkedList<>();
  ByteBuffer bb = ByteBuffer.allocate(4);
  bb.putInt(0, 3);
  UserPayload customPayload = UserPayload.create(bb);
  taskCommDescriptors.add(
      new NamedEntityDescriptor(customTaskCommName, FakeTaskComm.class.getName())
          .setUserPayload(customPayload));

  TaskCommManagerForMultipleCommTest tcm =
      new TaskCommManagerForMultipleCommTest(appContext, thh, chh, taskCommDescriptors);

  try {
    tcm.init(new Configuration(false));
    tcm.start();

    assertEquals(1, tcm.getNumTaskComms());
    assertFalse(tcm.getYarnTaskCommCreated());
    assertFalse(tcm.getUberTaskCommCreated());

    assertEquals(customTaskCommName, tcm.getTaskCommName(0));
    assertEquals(bb, tcm.getTaskCommContext(0).getInitialUserPayload().getPayload());

  } finally {
    tcm.stop();
  }
}
 

@SuppressWarnings("unchecked")
public static ErrorPluginConfiguration toErrorPluginConfiguration(UserPayload userPayload) throws
    IOException, ClassNotFoundException {

  byte[] b = new byte[userPayload.getPayload().remaining()];
  userPayload.getPayload().get(b);
  ByteArrayInputStream bais = new ByteArrayInputStream(b);
  ObjectInputStream ois = new ObjectInputStream(bais);

  HashMap<String, String> map = (HashMap) ois.readObject();
  ErrorPluginConfiguration conf = new ErrorPluginConfiguration(map);
  return conf;
}
 

public static VertexManagerPluginDescriptor getVMDesc(ExceptionLocation exLocation) throws IOException {
  Configuration conf = new Configuration();
  conf.set(Test_ExceptionLocation, exLocation.name());
  UserPayload payload = TezUtils.createUserPayloadFromConf(conf);
  return VertexManagerPluginDescriptor.create(InputReadyVertexManagerWithException.class.getName())
          .setUserPayload(payload);
}
 

@Test
public void testMergeConfig() throws IOException, TezException {
  Configuration baseConf = new Configuration(false);
  baseConf.set("base-key", "base-value");

  Configuration payloadConf = new Configuration(false);
  payloadConf.set("local-key", "local-value");

  InputContext inputContext = mock(InputContext.class);

  UserPayload payLoad = TezUtils.createUserPayloadFromConf(payloadConf);
  String[] workingDirs = new String[]{"workDir1"};
  TezCounters counters = new TezCounters();


  doReturn(payLoad).when(inputContext).getUserPayload();
  doReturn(workingDirs).when(inputContext).getWorkDirs();
  doReturn(counters).when(inputContext).getCounters();
  doReturn(baseConf).when(inputContext).getContainerConfiguration();

  OrderedGroupedKVInput input = new OrderedGroupedKVInput(inputContext, 1);
  input.initialize();

  Configuration mergedConf = input.conf;
  assertEquals("base-value", mergedConf.get("base-key"));
  assertEquals("local-value", mergedConf.get("local-key"));
}
 

private DAG createDAG(TezConfiguration conf, int numTasks, int maxSleepTimeMillis,
                      int payloadSize, String mode) throws IOException {

  Map<String, LocalResource> localResourceMap = new HashMap<String, LocalResource>();
  UserPayload payload =
      createUserPayload(conf, maxSleepTimeMillis, payloadSize, mode, localResourceMap);

  Vertex vertex = Vertex.create("RPCLoadVertex",
      ProcessorDescriptor.create(RPCSleepProcessor.class.getName()).setUserPayload(
          payload), numTasks).addTaskLocalFiles(localResourceMap);

  return DAG.create("RPCLoadGen").addVertex(vertex);
}
 

private DAG createDAG(TezConfiguration tezConf) throws IOException {
  InputDescriptor inputDescriptor = InputDescriptor.create(FakeInput.class.getName());
  InputInitializerDescriptor inputInitializerDescriptor =
    InputInitializerDescriptor.create(FakeInputInitializer.class.getName());
  DataSourceDescriptor dataSourceDescriptor =
    DataSourceDescriptor.create(inputDescriptor, inputInitializerDescriptor, null);

  Vertex v1 = Vertex.create(VERTEX1, ProcessorDescriptor.create(TokenProcessor.class.getName()));
  v1.addDataSource(INPUT, dataSourceDescriptor);
  Vertex v2 = Vertex.create(VERTEX2, ProcessorDescriptor.create(TokenProcessor.class.getName()));
  v2.addDataSource(INPUT, dataSourceDescriptor);

  OutputDescriptor outputDescriptor = OutputDescriptor.create(FakeOutput.class.getName());
  OutputCommitterDescriptor outputCommitterDescriptor =
    OutputCommitterDescriptor.create(FakeOutputCommitter.class.getName());
  DataSinkDescriptor dataSinkDescriptor =
    DataSinkDescriptor.create(outputDescriptor, outputCommitterDescriptor, null);

  CartesianProductConfig cartesianProductConfig =
    new CartesianProductConfig(Arrays.asList(sourceVertices));
  UserPayload userPayload = cartesianProductConfig.toUserPayload(tezConf);

  Vertex v3 = Vertex.create(VERTEX3, ProcessorDescriptor.create(JoinProcessor.class.getName()));
  v3.addDataSink(OUTPUT, dataSinkDescriptor);
  v3.setVertexManagerPlugin(
    VertexManagerPluginDescriptor.create(CartesianProductVertexManager.class.getName())
                                 .setUserPayload(userPayload));

  EdgeManagerPluginDescriptor edgeManagerDescriptor =
    EdgeManagerPluginDescriptor.create(CartesianProductEdgeManager.class.getName());
  edgeManagerDescriptor.setUserPayload(userPayload);
  UnorderedPartitionedKVEdgeConfig edgeConf =
    UnorderedPartitionedKVEdgeConfig.newBuilder(Text.class.getName(), IntWritable.class.getName(),
      RoundRobinPartitioner.class.getName()).build();
  EdgeProperty edgeProperty = edgeConf.createDefaultCustomEdgeProperty(edgeManagerDescriptor);

  return DAG.create("CrossProduct").addVertex(v1).addVertex(v2).addVertex(v3)
    .addEdge(Edge.create(v1, v3, edgeProperty)).addEdge(Edge.create(v2, v3, edgeProperty));
}
 

@VisibleForTesting
static void processSchedulerDescriptors(List<NamedEntityDescriptor> descriptors, boolean isLocal,
                                        UserPayload defaultPayload,
                                        BiMap<String, Integer> schedulerPluginMap) {
  if (isLocal) {
    boolean foundUberServiceName = false;
    for (NamedEntityDescriptor descriptor : descriptors) {
      if (descriptor.getEntityName().equals(TezConstants.getTezUberServicePluginName())) {
        foundUberServiceName = true;
        break;
      }
    }
    Preconditions.checkState(foundUberServiceName);
  } else {
    boolean foundYarn = false;
    for (int i = 0; i < descriptors.size(); i++) {
      if (descriptors.get(i).getEntityName().equals(TezConstants.getTezYarnServicePluginName())) {
        foundYarn = true;
      }
    }
    if (!foundYarn) {
      NamedEntityDescriptor yarnDescriptor =
          new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null)
              .setUserPayload(defaultPayload);
      addDescriptor(descriptors, schedulerPluginMap, yarnDescriptor);
    }
  }
}
 

@Private
public TezOutputContextImpl(Configuration conf, String[] workDirs, int appAttemptNumber,
    TezUmbilical tezUmbilical, String dagName,
    String taskVertexName,
    String destinationVertexName,
    int vertexParallelism,
    TezTaskAttemptID taskAttemptID, int outputIndex,
    @Nullable UserPayload userPayload, LogicalIOProcessorRuntimeTask runtimeTask,
    Map<String, ByteBuffer> serviceConsumerMetadata,
    Map<String, String> auxServiceEnv, MemoryDistributor memDist,
    OutputDescriptor outputDescriptor, ObjectRegistry objectRegistry,
    ExecutionContext executionContext, long memAvailable, TezExecutors sharedExecutor) {
  super(conf, workDirs, appAttemptNumber, dagName, taskVertexName, 
      vertexParallelism, taskAttemptID,
      wrapCounters(runtimeTask, taskVertexName, destinationVertexName, conf),
      runtimeTask, tezUmbilical, serviceConsumerMetadata,
      auxServiceEnv, memDist, outputDescriptor, objectRegistry, executionContext, memAvailable,
      sharedExecutor);
  Objects.requireNonNull(outputIndex, "outputIndex is null");
  Objects.requireNonNull(destinationVertexName, "destinationVertexName is null");
  this.userPayload = userPayload;
  this.outputIndex = outputIndex;
  this.destinationVertexName = destinationVertexName;
  this.sourceInfo = new EventMetaData(EventProducerConsumerType.OUTPUT,
      taskVertexName, destinationVertexName, taskAttemptID);
  runtimeTask.getTaskStatistics().addIO(destinationVertexName);
  statsReporter = new OutputStatisticsReporterImpl();
}
 

private DataSourceDescriptor createDistributorDataSource() throws IOException {
  InputSplitInfo inputSplitInfo;
  setupBasicConf(conf);
  try {
    inputSplitInfo = MRInputHelpers.generateInputSplitsToMem(conf, false, true, 0);
  } catch (Exception e) {
    throw new TezUncheckedException(e);
  }
  MRHelpers.translateMRConfToTez(conf);

  UserPayload payload = MRInputHelpersInternal.createMRInputPayload(conf,
      inputSplitInfo.getSplitsProto());
  Credentials credentials = null;
  if (getCredentialsForSourceFilesystem && inputSplitInfo.getCredentials() != null) {
    credentials = inputSplitInfo.getCredentials();
  }
  DataSourceDescriptor ds = DataSourceDescriptor.create(
      InputDescriptor.create(inputClassName).setUserPayload(payload),
      InputInitializerDescriptor.create(MRInputSplitDistributor.class.getName()),
      inputSplitInfo.getNumTasks(), credentials,
      VertexLocationHint.create(inputSplitInfo.getTaskLocationHints()), null);
  if (conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT,
      TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT_DEFAULT)) {
    ds.getInputDescriptor().setHistoryText(TezUtils.convertToHistoryText(conf));
  }

  return ds;
}
 

@InterfaceAudience.Private
public void fromUserPayload(UserPayload payload) {
  try {
    this.conf = TezUtils.createConfFromUserPayload(payload);
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}
 

public static DAG createDAG(String name, 
    Configuration conf) throws Exception {
  UserPayload payload = UserPayload.create(null);
  int taskCount = TEZ_SIMPLE_V_DAG_NUM_TASKS_DEFAULT;
  if (conf != null) {
    taskCount = conf.getInt(TEZ_SIMPLE_V_DAG_NUM_TASKS, TEZ_SIMPLE_V_DAG_NUM_TASKS_DEFAULT);
    payload = TezUtils.createUserPayloadFromConf(conf);
  }
  DAG dag = DAG.create(name);
  Vertex v1 = Vertex.create("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  Vertex v2 = Vertex.create("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  Vertex v3 = Vertex.create("v3", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  dag.addVertex(v1).addVertex(v2).addVertex(v3);
  dag.addEdge(Edge.create(v1, v3,
      EdgeProperty.create(DataMovementType.SCATTER_GATHER,
          DataSourceType.PERSISTED,
          SchedulingType.SEQUENTIAL,
          TestOutput.getOutputDesc(payload),
          TestInput.getInputDesc(payload))));
  dag.addEdge(Edge.create(v2, v3,
      EdgeProperty.create(DataMovementType.SCATTER_GATHER,
          DataSourceType.PERSISTED,
          SchedulingType.SEQUENTIAL,
          TestOutput.getOutputDesc(payload),
          TestInput.getInputDesc(payload))));
  return dag;
}
 

@Test (timeout=2000)
public void testPayloadToAndFromConf() throws IOException {
  Configuration conf = getConf();
  Assert.assertEquals(conf.size(), 6);
  UserPayload bConf = TezUtils.createUserPayloadFromConf(conf);
  conf.clear();
  Assert.assertEquals(conf.size(), 0);
  conf = TezUtils.createConfFromUserPayload(bConf);
  Assert.assertEquals(conf.size(), 6);
  checkConf(conf);
}
 

@Test(timeout = 5000)
public void testVertexManagerPluginCtorAccessUserPayload() throws IOException, TezException {
  byte[] randomUserPayload = {1,2,3};
  UserPayload userPayload = UserPayload.create(ByteBuffer.wrap(randomUserPayload));
  VertexManager vm =
      new VertexManager(
          VertexManagerPluginDescriptor.create(CheckUserPayloadVertexManagerPlugin.class
              .getName()).setUserPayload(userPayload), UserGroupInformation.getCurrentUser(),
          mockVertex, mockAppContext, mock(StateChangeNotifier.class));
}
 

@InterfaceAudience.Private
public void fromUserPayload(UserPayload payload) {
  try {
    this.conf = TezUtils.createConfFromUserPayload(payload);
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}
 

@InterfaceAudience.Private
public void fromUserPayload(UserPayload payload) {
  try {
    this.conf = TezUtils.createConfFromUserPayload(payload);
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}
 

private OutputContext createMockOutputContext(UserPayload payload, Configuration baseConf) {
  OutputContext outputContext = mock(OutputContext.class);
  ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
  when(outputContext.getUserPayload()).thenReturn(payload);
  when(outputContext.getApplicationId()).thenReturn(appId);
  when(outputContext.getTaskVertexIndex()).thenReturn(1);
  when(outputContext.getTaskAttemptNumber()).thenReturn(1);
  when(outputContext.getCounters()).thenReturn(new TezCounters());
  when(outputContext.getContainerConfiguration()).thenReturn(baseConf);
  return outputContext;
}
 

@Override
public void initialize(CartesianProductConfigProto config) throws TezReflectionException {
  this.sourceVertices = config.getSourcesList();
  this.numPartitions = Ints.toArray(config.getNumPartitionsList());
  this.minFraction = config.hasMinFraction() ? config.getMinFraction()
    : CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_SLOW_START_MIN_FRACTION_DEFAULT;
  this.maxFraction = config.hasMaxFraction() ? config.getMaxFraction()
    : CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_SLOW_START_MAX_FRACTION_DEFAULT;

  if (config.hasFilterClassName()) {
    UserPayload userPayload = config.hasFilterUserPayload()
      ? UserPayload.create(ByteBuffer.wrap(config.getFilterUserPayload().toByteArray())) : null;
    try {
      filter = ReflectionUtils.createClazzInstance(config.getFilterClassName(),
        new Class[]{UserPayload.class}, new UserPayload[]{userPayload});
    } catch (TezReflectionException e) {
      LOG.error("Creating filter failed");
      throw e;
    }
  }

  for (String sourceVertex : sourceVertices) {
    sourceTaskCompleted.put(sourceVertex, new BitSet());
  }
  for (String vertex : getContext().getInputVertexEdgeProperties().keySet()) {
    if (sourceVertices.indexOf(vertex) != -1) {
      getContext().registerForVertexStateUpdates(vertex, EnumSet.of(VertexState.CONFIGURED));
      numCPSrcNotInConfiguredState++;
    } else {
      getContext().registerForVertexStateUpdates(vertex, EnumSet.of(VertexState.RUNNING));
      numBroadcastSrcNotInRunningState++;
    }
  }
  getContext().vertexReconfigurationPlanned();
}
 

/**
 * Get a UserPayload representation of the Configuration
 * @return a {@link org.apache.tez.dag.api.UserPayload} instance
 */
public UserPayload toUserPayload() {
  try {
    return TezUtils.createUserPayloadFromConf(conf);
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}
 

@InterfaceAudience.Private
public void fromUserPayload(UserPayload payload) {
  try {
    this.conf = TezUtils.createConfFromUserPayload(payload);
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}