Java Code Examples for org.apache.tez.common.TezUtils#createConfFromUserPayload()

public TezTestServiceContainerLauncher(ContainerLauncherContext containerLauncherContext) {
  super(containerLauncherContext);
  try {
    conf = TezUtils.createConfFromUserPayload(getContext().getInitialUserPayload());
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
  int numThreads = conf.getInt(
      TezTestServiceConfConstants.TEZ_TEST_SERVICE_AM_COMMUNICATOR_NUM_THREADS,
      TezTestServiceConfConstants.TEZ_TEST_SERVICE_AM_COMMUNICATOR_NUM_THREADS_DEFAULT);

  this.servicePort = conf.getInt(
      TezTestServiceConfConstants.TEZ_TEST_SERVICE_RPC_PORT, -1);
  Preconditions.checkArgument(servicePort > 0,
      TezTestServiceConfConstants.TEZ_TEST_SERVICE_RPC_PORT + " must be set");
  this.communicator = new TezTestServiceCommunicator(numThreads);
  this.tokenIdentifier = getContext().getApplicationAttemptId().getApplicationId().toString();
  this.appAttemptId = getContext().getApplicationAttemptId();
}
 

@Private
@VisibleForTesting
YarnTaskSchedulerService(TaskSchedulerContext taskSchedulerContext,
    TezAMRMClientAsync<CookieContainerRequest> client) {
  super(taskSchedulerContext);
  this.containerSignatureMatcher = taskSchedulerContext.getContainerSignatureMatcher();
  this.amRmClient = client;
  this.appHostName = taskSchedulerContext.getAppHostName();
  this.appHostPort = taskSchedulerContext.getAppClientPort();
  this.appTrackingUrl = taskSchedulerContext.getAppTrackingUrl();
  try {
    this.conf = TezUtils.createConfFromUserPayload(taskSchedulerContext.getInitialUserPayload());
  } catch (IOException e) {
    throw new TezUncheckedException(
        "Failed to deserialize payload for " + YarnTaskSchedulerService.class.getSimpleName(),
        e);
  }
}
 

@Override
public synchronized List<Event> initialize() throws IOException {
  this.startTime = System.nanoTime();
  this.conf = TezUtils.createConfFromUserPayload(getContext().getUserPayload());
  // Initializing this parametr in this conf since it is used in multiple
  // places (wherever LocalDirAllocator is used) - TezTaskOutputFiles,
  // TezMerger, etc.
  this.conf.setStrings(TezRuntimeFrameworkConfigs.LOCAL_DIRS, getContext().getWorkDirs());
  this.memoryUpdateCallbackHandler = new MemoryUpdateCallbackHandler();
  getContext().requestInitialMemory(
      ExternalSorter.getInitialMemoryRequirement(conf,
          getContext().getTotalMemoryAvailableToTask()), memoryUpdateCallbackHandler);

  sendEmptyPartitionDetails = this.conf.getBoolean(
      TezJobConfig.TEZ_RUNTIME_EMPTY_PARTITION_INFO_VIA_EVENTS_ENABLED,
      TezJobConfig.TEZ_RUNTIME_EMPTY_PARTITION_INFO_VIA_EVENTS_ENABLED_DEFAULT);
  return Collections.emptyList();
}
 

@Override
public synchronized List<Event> initialize()
    throws Exception {
  this.conf = TezUtils.createConfFromUserPayload(getContext()
      .getUserPayload());
  this.conf.setStrings(TezRuntimeFrameworkConfigs.LOCAL_DIRS,
      getContext().getWorkDirs());

  getContext().requestInitialMemory(0l, null); // mandatory call

  this.dataViaEventsEnabled = conf.getBoolean(
      TezJobConfig.TEZ_RUNTIME_BROADCAST_DATA_VIA_EVENTS_ENABLED,
      TezJobConfig.TEZ_RUNTIME_BROADCAST_DATA_VIA_EVENTS_ENABLED_DEFAULT);
  this.dataViaEventsMaxSize = conf.getInt(
      TezJobConfig.TEZ_RUNTIME_BROADCAST_DATA_VIA_EVENTS_MAX_SIZE,
      TezJobConfig.TEZ_RUNTIME_BROADCAST_DATA_VIA_EVENTS_MAX_SIZE_DEFAULT);
  
  LOG.info(this.getClass().getSimpleName() + " running with params -> "
      + "dataViaEventsEnabled: " + dataViaEventsEnabled
      + ", dataViaEventsMaxSize: " + dataViaEventsMaxSize);
  
  this.kvWriter = new FileBasedKVWriter(getContext(), conf);
  return Collections.emptyList();
}
 

public TezContainerLauncherImpl(ContainerLauncherContext containerLauncherContext) {
  super(containerLauncherContext);
  try {
    this.conf = TezUtils.createConfFromUserPayload(containerLauncherContext.getInitialUserPayload());
  } catch (IOException e) {
    throw new TezUncheckedException(
        "Failed to parse user payload for " + TezContainerLauncherImpl.class.getSimpleName(), e);
  }
  conf.setInt(
      CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY,
      0);
  this.limitOnPoolSize = conf.getInt(
      TezConfiguration.TEZ_AM_CONTAINERLAUNCHER_THREAD_COUNT_LIMIT,
      TezConfiguration.TEZ_AM_CONTAINERLAUNCHER_THREAD_COUNT_LIMIT_DEFAULT);
  LOG.info("Upper limit on the thread pool size is " + this.limitOnPoolSize);
}
 

@Override
public void initialize() {
  super.initialize();
  try {
    conf =
        TezUtils.createConfFromUserPayload(getContext().getUserPayload());
  } catch (IOException e) {
    e.printStackTrace();
  }
}
 

public MRCombiner(TaskContext taskContext) throws IOException {
  final Configuration userConf = TezUtils.createConfFromUserPayload(taskContext.getUserPayload());
  useNewApi = ConfigUtils.useNewApi(userConf);
  if (useNewApi) {
    conf = new JobConf(userConf);
  } else {
    conf = userConf;
  }

  assert(taskContext instanceof InputContext || taskContext instanceof OutputContext);
  if (taskContext instanceof OutputContext) {
    this.keyClass = ConfigUtils.getIntermediateOutputKeyClass(conf);
    this.valClass = ConfigUtils.getIntermediateOutputValueClass(conf);
    this.comparator = ConfigUtils.getIntermediateOutputKeyComparator(conf);
    this.reporter = new MRTaskReporter((OutputContext)taskContext);
  } else {
    this.keyClass = ConfigUtils.getIntermediateInputKeyClass(conf);
    this.valClass = ConfigUtils.getIntermediateInputValueClass(conf);
    this.comparator = ConfigUtils.getIntermediateInputKeyComparator(conf);
    this.reporter = new MRTaskReporter((InputContext)taskContext);
  }

  combineInputRecordsCounter = taskContext.getCounters().findCounter(TaskCounter.COMBINE_INPUT_RECORDS);
  combineOutputRecordsCounter = taskContext.getCounters().findCounter(TaskCounter.COMBINE_OUTPUT_RECORDS);
  
  boolean isMap = conf.getBoolean(MRConfig.IS_MAP_PROCESSOR,false);
  this.mrTaskAttemptID = new TaskAttemptID(
      new TaskID(String.valueOf(taskContext.getApplicationId()
          .getClusterTimestamp()), taskContext.getApplicationId().getId(),
          isMap ? TaskType.MAP : TaskType.REDUCE,
          taskContext.getTaskIndex()), taskContext.getTaskAttemptNumber());
  
  LOG.info("Using combineKeyClass: " + keyClass + ", combineValueClass: " + valClass + ", combineComparator: " +comparator + ", useNewApi: " + useNewApi);
}
 

public YarnTaskSchedulerService(TaskSchedulerContext taskSchedulerContext) {
  super(taskSchedulerContext);
  this.containerSignatureMatcher = taskSchedulerContext.getContainerSignatureMatcher();
  this.amRmClient = TezAMRMClientAsync.createAMRMClientAsync(1000, this);
  this.appHostName = taskSchedulerContext.getAppHostName();
  this.appHostPort = taskSchedulerContext.getAppClientPort();
  this.appTrackingUrl = taskSchedulerContext.getAppTrackingUrl();
  try {
    this.conf = TezUtils.createConfFromUserPayload(taskSchedulerContext.getInitialUserPayload());
  } catch (IOException e) {
    throw new TezUncheckedException(
        "Failed to deserialize payload for " + YarnTaskSchedulerService.class.getSimpleName(),
        e);
  }
}
 

public MRCombiner(TezTaskContext taskContext) throws IOException {
  this.conf = TezUtils.createConfFromUserPayload(taskContext.getUserPayload());

  assert(taskContext instanceof TezInputContext || taskContext instanceof TezOutputContext);
  if (taskContext instanceof TezOutputContext) {
    this.keyClass = ConfigUtils.getIntermediateOutputKeyClass(conf);
    this.valClass = ConfigUtils.getIntermediateOutputValueClass(conf);
    this.comparator = ConfigUtils.getIntermediateOutputKeyComparator(conf);
    this.reporter = new MRTaskReporter((TezOutputContext)taskContext);
  } else {
    this.keyClass = ConfigUtils.getIntermediateInputKeyClass(conf);
    this.valClass = ConfigUtils.getIntermediateInputValueClass(conf);
    this.comparator = ConfigUtils.getIntermediateInputKeyComparator(conf);
    this.reporter = new MRTaskReporter((TezInputContext)taskContext);
  }

  this.useNewApi = ConfigUtils.useNewApi(conf);
  
  combineInputKeyCounter = taskContext.getCounters().findCounter(TaskCounter.COMBINE_INPUT_RECORDS);
  combineInputValueCounter = taskContext.getCounters().findCounter(TaskCounter.COMBINE_OUTPUT_RECORDS);
  
  boolean isMap = conf.getBoolean(MRConfig.IS_MAP_PROCESSOR,false);
  this.mrTaskAttemptID = new TaskAttemptID(
      new TaskID(String.valueOf(taskContext.getApplicationId()
          .getClusterTimestamp()), taskContext.getApplicationId().getId(),
          isMap ? TaskType.MAP : TaskType.REDUCE,
          taskContext.getTaskIndex()), taskContext.getTaskAttemptNumber());
  
  LOG.info("Using combineKeyClass: " + keyClass + ", combineValueClass: " + valClass + ", combineComparator: " +comparator + ", useNewApi: " + useNewApi);
}
 

public void verifyResults(InputDescriptor entityDescriptor, InputDescriptor deserialized, UserPayload payload,
                           String confVal) throws IOException {
  Assert.assertEquals(entityDescriptor.getClassName(), deserialized.getClassName());
  // History text is not serialized when sending to tasks
  Assert.assertNull(deserialized.getHistoryText());
  Assert.assertArrayEquals(payload.deepCopyAsArray(), deserialized.getUserPayload().deepCopyAsArray());
  Configuration deserializedConf = TezUtils.createConfFromUserPayload(deserialized.getUserPayload());
  Assert.assertEquals(confVal, deserializedConf.get("testKey"));
}
 

@InterfaceAudience.Private
public void fromUserPayload(UserPayload payload) {
  try {
    this.conf = TezUtils.createConfFromUserPayload(payload);
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}
 

@Override
public List<Event> initialize() throws IOException {
  getContext().requestInitialMemory(0l, null); // mandatory call.
  getContext().inputIsReady();
  this.conf = TezUtils.createConfFromUserPayload(getContext().getUserPayload());

  if (getNumPhysicalInputs() == 0) {
    return Collections.emptyList();
  }

  LocalShuffle localShuffle = new LocalShuffle(getContext(), conf, getNumPhysicalInputs());
  rawIter = localShuffle.run();
  createValuesIterator();
  return Collections.emptyList();
}
 

@Override
public void initialize() {
  super.initialize();
  try {
    conf =
        TezUtils.createConfFromUserPayload(getContext().getUserPayload());
  } catch (IOException e) {
    e.printStackTrace();
  }
}
 

@Override
public void initialize() {
  super.initialize();
  try {
    conf =
        TezUtils.createConfFromUserPayload(getContext().getUserPayload());
  } catch (IOException e) {
    e.printStackTrace();
  }
}
 

@Override
public synchronized List<Event> initialize(TezOutputContext outputContext) throws Exception {
  this.outputContext = outputContext;
  this.conf = TezUtils.createConfFromUserPayload(outputContext.getUserPayload());
  this.conf.setStrings(TezRuntimeFrameworkConfigs.LOCAL_DIRS, outputContext.getWorkDirs());
  this.conf.setInt(TezRuntimeFrameworkConfigs.TEZ_RUNTIME_NUM_EXPECTED_PARTITIONS,
      this.numPhysicalOutputs);
  this.memoryUpdateCallbackHandler = new MemoryUpdateCallbackHandler();
  outputContext.requestInitialMemory(
      UnorderedPartitionedKVWriter.getInitialMemoryRequirement(conf,
          outputContext.getTotalMemoryAvailableToTask()), memoryUpdateCallbackHandler);
  return Collections.emptyList();
}
 

/**
 * Updates the statistics after a DAG is finished.
 */
public void accumulateStats(TezJob tezJob) throws IOException {
    //For Oozie Pig action job id matching compatibility with MR mode
    appId = tezJob.getApplicationId().toString().replace("application", "job");
    setConf(tezJob.getConfiguration());
    DAG dag = tezJob.getDAG();
    hdfsBytesRead = -1;
    hdfsBytesWritten = -1;
    TezCounters tezCounters = tezJob.getDAGCounters();
    if (tezCounters != null) {
        counters = covertToHadoopCounters(tezCounters);

        CounterGroup dagGrp = tezCounters.getGroup(DAG_COUNTER_GROUP);
        totalTasks = (int) dagGrp.findCounter("TOTAL_LAUNCHED_TASKS").getValue();

        CounterGroup fsGrp = tezCounters.getGroup(FS_COUNTER_GROUP);
        fileBytesRead = fsGrp.findCounter("FILE_BYTES_READ").getValue();
        fileBytesWritten = fsGrp.findCounter("FILE_BYTES_WRITTEN").getValue();
        hdfsBytesRead = fsGrp.findCounter("HDFS_BYTES_READ").getValue();
        hdfsBytesWritten = fsGrp.findCounter("HDFS_BYTES_WRITTEN").getValue();
    } else {
        LOG.warn("Failed to get counters for DAG: " + dag.getName());
    }

    for (Entry<String, TezVertexStats> entry : tezVertexStatsMap.entrySet()) {
        Vertex v = dag.getVertex(entry.getKey());
        if (v != null && tezVertexStatsMap.containsKey(v.getName())) {
            TezVertexStats vertexStats = entry.getValue();
            UserPayload payload = v.getProcessorDescriptor().getUserPayload();
            Configuration conf = TezUtils.createConfFromUserPayload(payload);
            vertexStats.setConf(conf);

            VertexStatus status = tezJob.getVertexStatus(v.getName());
            if (status == null) {
                LOG.warn("Failed to get status for vertex: " + v.getName());
                continue;
            }
            vertexStats.accumulateStats(status, v.getParallelism());
            if(vertexStats.getInputs() != null && !vertexStats.getInputs().isEmpty()) {
                inputs.addAll(vertexStats.getInputs());
            }
            if(vertexStats.getOutputs() != null  && !vertexStats.getOutputs().isEmpty()) {
                outputs.addAll(vertexStats.getOutputs());
            }
            /*if (vertexStats.getHdfsBytesRead() >= 0) {
                hdfsBytesRead = (hdfsBytesRead == -1) ? 0 : hdfsBytesRead;
                hdfsBytesRead += vertexStats.getHdfsBytesRead();
            }
            if (vertexStats.getHdfsBytesWritten() >= 0) {
                hdfsBytesWritten = (hdfsBytesWritten == -1) ? 0 : hdfsBytesWritten;
                hdfsBytesWritten += vertexStats.getHdfsBytesWritten();
            }*/
            if (!vertexStats.getMultiStoreCounters().isEmpty()) {
                multiStoreCounters.putAll(vertexStats.getMultiStoreCounters());
            }
            numberMaps += vertexStats.getNumberMaps();
            numberReduces += vertexStats.getNumberReduces();
            mapInputRecords += vertexStats.getMapInputRecords();
            mapOutputRecords += vertexStats.getMapOutputRecords();
            reduceInputRecords += vertexStats.getReduceInputRecords();
            reduceOutputRecords += vertexStats.getReduceOutputRecords();
            spillCount += vertexStats.getSMMSpillCount();
            activeSpillCountObj  += vertexStats.getProactiveSpillCountObjects();
            activeSpillCountRecs += vertexStats.getProactiveSpillCountRecs();
        }
    }
}
 

@Override
public void initialize() {
  UserPayload userPayload = getContext().getUserPayload();
  if (userPayload == null || userPayload.getPayload() == null ||
      userPayload.getPayload().limit() == 0) {
    throw new RuntimeException("Could not initialize RootInputVertexManager"
        + " from provided user payload");
  }
  try {
    conf = TezUtils.createConfFromUserPayload(userPayload);
  } catch (IOException e) {
    throw new RuntimeException("Could not initialize RootInputVertexManager"
      + " from provided user payload", e);
  }
  slowStartEnabled = conf.getBoolean(
    TEZ_ROOT_INPUT_VERTEX_MANAGER_ENABLE_SLOW_START,
    TEZ_ROOT_INPUT_VERTEX_MANAGER_ENABLE_SLOW_START_DEFAULT);

  if(slowStartEnabled) {
    slowStartMinFraction = conf.getFloat(
      TEZ_ROOT_INPUT_VERTEX_MANAGER_MIN_SRC_FRACTION,
      TEZ_ROOT_INPUT_VERTEX_MANAGER_MIN_SRC_FRACTION_DEFAULT);
    slowStartMaxFraction = conf.getFloat(
      TEZ_ROOT_INPUT_VERTEX_MANAGER_MAX_SRC_FRACTION,
      Math.max(slowStartMinFraction,
          TEZ_ROOT_INPUT_VERTEX_MANAGER_MAX_SRC_FRACTION_DEFAULT));
  } else {
    slowStartMinFraction = 0;
    slowStartMaxFraction = 0;
  }
  if (slowStartMinFraction < 0 || slowStartMaxFraction > 1
      || slowStartMaxFraction < slowStartMinFraction) {
    throw new IllegalArgumentException(
        "Invalid values for slowStartMinFraction"
        + "/slowStartMaxFraction. Min "
        + "cannot be < 0, max cannot be > 1, and max cannot be < min."
        + ", configuredMin=" + slowStartMinFraction
        + ", configuredMax=" + slowStartMaxFraction);
  }


  updatePendingTasks();
}
 

@LimitedPrivate("Hive, Pig")
@Unstable
public static Configuration createConfFromUserPayload(byte[] bb)
    throws IOException {
  return TezUtils.createConfFromUserPayload(bb);
}
 

@Test(timeout = 5000)
public void testCustomTaskSchedulerSetup() throws IOException {
  Configuration conf = new Configuration(false);
  conf.set("testkey", "testval");
  UserPayload defaultPayload = TezUtils.createUserPayloadFromConf(conf);

  String customSchedulerName = "fakeScheduler";
  List<NamedEntityDescriptor> taskSchedulers = new LinkedList<>();
  ByteBuffer bb = ByteBuffer.allocate(4);
  bb.putInt(0, 3);
  UserPayload userPayload = UserPayload.create(bb);
  taskSchedulers.add(
      new NamedEntityDescriptor(customSchedulerName, FakeTaskScheduler.class.getName())
          .setUserPayload(userPayload));
  taskSchedulers.add(new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null)
      .setUserPayload(defaultPayload));

  TSEHForMultipleSchedulersTest tseh =
      new TSEHForMultipleSchedulersTest(mockAppContext, mockClientService, mockEventHandler,
          mockSigMatcher, mockWebUIService, taskSchedulers, false);

  tseh.init(conf);
  tseh.start();

  // Verify that the YARN task scheduler is installed by default
  assertTrue(tseh.getYarnSchedulerCreated());
  assertFalse(tseh.getUberSchedulerCreated());
  assertEquals(2, tseh.getNumCreateInvocations());

  // Verify the order of the schedulers
  assertEquals(customSchedulerName, tseh.getTaskSchedulerName(0));
  assertEquals(TezConstants.getTezYarnServicePluginName(), tseh.getTaskSchedulerName(1));

  // Verify the payload setup for the custom task scheduler
  assertNotNull(tseh.getTaskSchedulerContext(0));
  assertEquals(bb, tseh.getTaskSchedulerContext(0).getInitialUserPayload().getPayload());

  // Verify the payload on the yarn scheduler
  assertNotNull(tseh.getTaskSchedulerContext(1));
  Configuration parsed = TezUtils.createConfFromUserPayload(tseh.getTaskSchedulerContext(1).getInitialUserPayload());
  assertEquals("testval", parsed.get("testkey"));
}
 

@SuppressWarnings("unchecked")
@Override
public void initialize() throws Exception {
    // Reset any static variables to avoid conflict in container-reuse.
    sampleVertex = null;
    sampleMap = null;

    // Reset static variables cleared for avoiding OOM.
    new JVMReuseImpl().cleanupStaticData();

    UserPayload payload = getContext().getUserPayload();
    conf = TezUtils.createConfFromUserPayload(payload);
    PigContext.setPackageImportList((ArrayList<String>) ObjectSerializer
            .deserialize(conf.get("udf.import.list")));
    PigContext pc = (PigContext) ObjectSerializer.deserialize(conf.get("pig.pigContext"));

    // To determine front-end in UDFContext
    conf.set(MRConfiguration.JOB_APPLICATION_ATTEMPT_ID, getContext().getUniqueIdentifier());
    conf.set(PigConstants.TASK_INDEX, Integer.toString(getContext().getTaskIndex()));
    UDFContext.getUDFContext().addJobConf(conf);
    UDFContext.getUDFContext().deserialize();

    String execPlanString = conf.get(PLAN);
    execPlan = (PhysicalPlan) ObjectSerializer.deserialize(execPlanString);
    SchemaTupleBackend.initialize(conf, pc);
    PigMapReduce.sJobContext = HadoopShims.createJobContext(conf, new org.apache.hadoop.mapreduce.JobID());

    // Set the job conf as a thread-local member of PigMapReduce
    // for backwards compatibility with the existing code base.
    PigMapReduce.sJobConfInternal.set(conf);

    Utils.setDefaultTimeZone(conf);

    boolean aggregateWarning = "true".equalsIgnoreCase(pc.getProperties().getProperty("aggregate.warning"));
    PigStatusReporter pigStatusReporter = PigStatusReporter.getInstance();
    pigStatusReporter.setContext(new TezTaskContext(getContext()));
    pigHadoopLogger = PigHadoopLogger.getInstance();
    pigHadoopLogger.setReporter(pigStatusReporter);
    pigHadoopLogger.setAggregate(aggregateWarning);
    PhysicalOperator.setPigLogger(pigHadoopLogger);

    LinkedList<TezTaskConfigurable> tezTCs = PlanHelper.getPhysicalOperators(execPlan, TezTaskConfigurable.class);
    for (TezTaskConfigurable tezTC : tezTCs){
        tezTC.initialize(getContext());
    }
}