org.apache.tez.mapreduce.hadoop.DeprecatedKeys Java Examples

The following examples show how to use org.apache.tez.mapreduce.hadoop.DeprecatedKeys. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: YARNRunner.java    From incubator-tez with Apache License 2.0 6 votes vote down vote up
private TezConfiguration getDAGAMConfFromMRConf() {
  TezConfiguration finalConf = new TezConfiguration(this.tezConf);
  Map<String, String> mrParamToDAGParamMap = DeprecatedKeys
      .getMRToDAGParamMap();

  for (Entry<String, String> entry : mrParamToDAGParamMap.entrySet()) {
    if (finalConf.get(entry.getKey()) != null) {
      finalConf.set(entry.getValue(), finalConf.get(entry.getKey()));
      finalConf.unset(entry.getKey());
      if (LOG.isDebugEnabled()) {
        LOG.debug("MR->DAG Translating MR key: " + entry.getKey()
            + " to Tez key: " + entry.getValue() + " with value "
            + finalConf.get(entry.getValue()));
      }
    }
  }
  return finalConf;
}
 
Example #2
Source File: YARNRunner.java    From tez with Apache License 2.0 6 votes vote down vote up
private TezConfiguration getDAGAMConfFromMRConf() {
  TezConfiguration finalConf = new TezConfiguration(this.tezConf);
  Map<String, String> mrParamToDAGParamMap = DeprecatedKeys
      .getMRToDAGParamMap();

  for (Entry<String, String> entry : mrParamToDAGParamMap.entrySet()) {
    if (finalConf.get(entry.getKey()) != null) {
      finalConf.set(entry.getValue(), finalConf.get(entry.getKey()));
      finalConf.unset(entry.getKey());
      if (LOG.isDebugEnabled()) {
        LOG.debug("MR->DAG Translating MR key: " + entry.getKey()
            + " to Tez key: " + entry.getValue() + " with value "
            + finalConf.get(entry.getValue()));
      }
    }
  }
  return finalConf;
}
 
Example #3
Source File: MRToTezHelper.java    From spork with Apache License 2.0 5 votes vote down vote up
/**
 * Convert MR settings to Tez settings and set on conf.
 *
 * @param conf  Configuration on which MR equivalent Tez settings should be set
 * @param mrConf Configuration that contains MR settings
 */
private static void convertMRToTezRuntimeConf(Configuration conf, Configuration mrConf) {
    for (Entry<String, String> dep : DeprecatedKeys.getMRToTezRuntimeParamMap().entrySet()) {
        if (mrConf.get(dep.getKey()) != null) {
            conf.unset(dep.getKey());
            LOG.info("Setting " + dep.getValue() + " to "
                    + mrConf.get(dep.getKey()) + " from MR setting "
                    + dep.getKey());
            conf.setIfUnset(dep.getValue(), mrConf.get(dep.getKey()));
        }
    }
}
 
Example #4
Source File: MRToTezHelper.java    From spork with Apache License 2.0 4 votes vote down vote up
public static TezConfiguration getDAGAMConfFromMRConf(
        Configuration tezConf) {

    // Set Tez parameters based on MR parameters.
    TezConfiguration dagAMConf = new TezConfiguration(tezConf);
    Map<String, String> mrParamToDAGParamMap = DeprecatedKeys
            .getMRToDAGParamMap();

    for (Entry<String, String> entry : mrParamToDAGParamMap.entrySet()) {
        if (dagAMConf.get(entry.getKey()) != null) {
            dagAMConf.set(entry.getValue(), dagAMConf.get(entry.getKey()));
            dagAMConf.unset(entry.getKey());
            if (LOG.isDebugEnabled()) {
                LOG.debug("MR->DAG Translating MR key: " + entry.getKey()
                        + " to Tez key: " + entry.getValue()
                        + " with value " + dagAMConf.get(entry.getValue()));
            }
        }
    }

    String env = tezConf.get(MRJobConfig.MR_AM_ADMIN_USER_ENV);
    if (tezConf.get(MRJobConfig.MR_AM_ENV) != null) {
        env = (env == null) ? tezConf.get(MRJobConfig.MR_AM_ENV)
                            : env + "," + tezConf.get(MRJobConfig.MR_AM_ENV);
    }

    if (env != null) {
        dagAMConf.setIfUnset(TezConfiguration.TEZ_AM_LAUNCH_ENV, env);
    }

    dagAMConf.setIfUnset(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS,
            org.apache.tez.mapreduce.hadoop.MRHelpers
                    .getJavaOptsForMRAM(tezConf));

    String queueName = tezConf.get(JobContext.QUEUE_NAME,
            YarnConfiguration.DEFAULT_QUEUE_NAME);
    dagAMConf.setIfUnset(TezConfiguration.TEZ_QUEUE_NAME, queueName);

    int amMemMB = tezConf.getInt(MRJobConfig.MR_AM_VMEM_MB,
            MRJobConfig.DEFAULT_MR_AM_VMEM_MB);
    int amCores = tezConf.getInt(MRJobConfig.MR_AM_CPU_VCORES,
            MRJobConfig.DEFAULT_MR_AM_CPU_VCORES);
    dagAMConf.setIfUnset(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, ""
            + amMemMB);
    dagAMConf.setIfUnset(TezConfiguration.TEZ_AM_RESOURCE_CPU_VCORES, ""
            + amCores);

    dagAMConf.setIfUnset(TezConfiguration.TEZ_AM_VIEW_ACLS,
            tezConf.get(MRJobConfig.JOB_ACL_VIEW_JOB, MRJobConfig.DEFAULT_JOB_ACL_VIEW_JOB));

    dagAMConf.setIfUnset(TezConfiguration.TEZ_AM_MODIFY_ACLS,
            tezConf.get(MRJobConfig.JOB_ACL_MODIFY_JOB, MRJobConfig.DEFAULT_JOB_ACL_MODIFY_JOB));

    dagAMConf.setIfUnset(TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS, ""
            + dagAMConf.getInt(MRJobConfig.MR_AM_MAX_ATTEMPTS,
                    MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS));

    if (tezConf.get(MRConfiguration.JOB_CREDENTIALS_BINARY) != null) {
        dagAMConf.setIfUnset(TezConfiguration.TEZ_CREDENTIALS_PATH,
                tezConf.get(MRConfiguration.JOB_CREDENTIALS_BINARY));
    }

    //TODO: Strip out all MR settings

    return dagAMConf;
}
 
Example #5
Source File: MRTask.java    From incubator-tez with Apache License 2.0 4 votes vote down vote up
public void initialize(TezProcessorContext context) throws IOException,
InterruptedException {

  DeprecatedKeys.init();

  processorContext = context;
  counters = context.getCounters();
  this.taskAttemptId = new TaskAttemptID(
      new TaskID(
          Long.toString(context.getApplicationId().getClusterTimestamp()),
          context.getApplicationId().getId(),
          (isMap ? TaskType.MAP : TaskType.REDUCE),
          context.getTaskIndex()),
        context.getTaskAttemptNumber());

  byte[] userPayload = context.getUserPayload();
  Configuration conf = TezUtils.createConfFromUserPayload(userPayload);
  if (conf instanceof JobConf) {
    this.jobConf = (JobConf)conf;
  } else {
    this.jobConf = new JobConf(conf);
  }
  jobConf.set(Constants.TEZ_RUNTIME_TASK_ATTEMPT_ID,
      taskAttemptId.toString());
  jobConf.set(MRJobConfig.TASK_ATTEMPT_ID,
    taskAttemptId.toString());
  jobConf.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID,
      context.getDAGAttemptNumber());

  LOG.info("MRTask.inited: taskAttemptId = " + taskAttemptId.toString());

  // TODO Post MRR
  // A single file per vertex will likely be a better solution. Does not
  // require translation - client can take care of this. Will work independent
  // of whether the configuration is for intermediate tasks or not. Has the
  // overhead of localizing multiple files per job - i.e. the client would
  // need to write these files to hdfs, add them as local resources per
  // vertex. A solution like this may be more practical once it's possible to
  // submit configuration parameters to the AM and effectively tasks via RPC.

  jobConf.set(MRJobConfig.VERTEX_NAME, processorContext.getTaskVertexName());

  if (LOG.isDebugEnabled() && userPayload != null) {
    Iterator<Entry<String, String>> iter = jobConf.iterator();
    String taskIdStr = taskAttemptId.getTaskID().toString();
    while (iter.hasNext()) {
      Entry<String, String> confEntry = iter.next();
      LOG.debug("TaskConf Entry"
          + ", taskId=" + taskIdStr
          + ", key=" + confEntry.getKey()
          + ", value=" + confEntry.getValue());
    }
  }

  configureMRTask();
}
 
Example #6
Source File: MRTask.java    From tez with Apache License 2.0 4 votes vote down vote up
@Override
public void initialize() throws IOException,
InterruptedException {

  DeprecatedKeys.init();

  processorContext = getContext();
  counters = processorContext.getCounters();
  this.taskAttemptId = new TaskAttemptID(
      new TaskID(
          Long.toString(processorContext.getApplicationId().getClusterTimestamp()),
          processorContext.getApplicationId().getId(),
          (isMap ? TaskType.MAP : TaskType.REDUCE),
          processorContext.getTaskIndex()),
      processorContext.getTaskAttemptNumber());

  UserPayload userPayload = processorContext.getUserPayload();
  Configuration conf = TezUtils.createConfFromUserPayload(userPayload);
  if (conf instanceof JobConf) {
    this.jobConf = (JobConf)conf;
  } else {
    this.jobConf = new JobConf(conf);
  }
  jobConf.set(Constants.TEZ_RUNTIME_TASK_ATTEMPT_ID,
      taskAttemptId.toString());
  jobConf.set(MRJobConfig.TASK_ATTEMPT_ID,
    taskAttemptId.toString());
  jobConf.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID,
      processorContext.getDAGAttemptNumber());

  LOG.info("MRTask.inited: taskAttemptId = " + taskAttemptId.toString());

  // TODO Post MRR
  // A single file per vertex will likely be a better solution. Does not
  // require translation - client can take care of this. Will work independent
  // of whether the configuration is for intermediate tasks or not. Has the
  // overhead of localizing multiple files per job - i.e. the client would
  // need to write these files to hdfs, add them as local resources per
  // vertex. A solution like this may be more practical once it's possible to
  // submit configuration parameters to the AM and effectively tasks via RPC.

  jobConf.set(MRJobConfig.VERTEX_NAME, processorContext.getTaskVertexName());

  if (LOG.isDebugEnabled() && userPayload != null) {
    Iterator<Entry<String, String>> iter = jobConf.iterator();
    String taskIdStr = taskAttemptId.getTaskID().toString();
    while (iter.hasNext()) {
      Entry<String, String> confEntry = iter.next();
      LOG.debug("TaskConf Entry"
          + ", taskId=" + taskIdStr
          + ", key=" + confEntry.getKey()
          + ", value=" + confEntry.getValue());
    }
  }

  configureMRTask();
}