Java Code Examples for org.apache.tez.dag.api.TezConfiguration#get()

The following examples show how to use org.apache.tez.dag.api.TezConfiguration#get() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: YARNRunner.java From incubator-tez with Apache License 2.0

6 votes

private TezConfiguration getDAGAMConfFromMRConf() {
  TezConfiguration finalConf = new TezConfiguration(this.tezConf);
  Map<String, String> mrParamToDAGParamMap = DeprecatedKeys
      .getMRToDAGParamMap();

  for (Entry<String, String> entry : mrParamToDAGParamMap.entrySet()) {
    if (finalConf.get(entry.getKey()) != null) {
      finalConf.set(entry.getValue(), finalConf.get(entry.getKey()));
      finalConf.unset(entry.getKey());
      if (LOG.isDebugEnabled()) {
        LOG.debug("MR->DAG Translating MR key: " + entry.getKey()
            + " to Tez key: " + entry.getValue() + " with value "
            + finalConf.get(entry.getValue()));
      }
    }
  }
  return finalConf;
}

Example 2

Source File: YARNRunner.java From tez with Apache License 2.0

6 votes

private TezConfiguration getDAGAMConfFromMRConf() {
  TezConfiguration finalConf = new TezConfiguration(this.tezConf);
  Map<String, String> mrParamToDAGParamMap = DeprecatedKeys
      .getMRToDAGParamMap();

  for (Entry<String, String> entry : mrParamToDAGParamMap.entrySet()) {
    if (finalConf.get(entry.getKey()) != null) {
      finalConf.set(entry.getValue(), finalConf.get(entry.getKey()));
      finalConf.unset(entry.getKey());
      if (LOG.isDebugEnabled()) {
        LOG.debug("MR->DAG Translating MR key: " + entry.getKey()
            + " to Tez key: " + entry.getValue() + " with value "
            + finalConf.get(entry.getValue()));
      }
    }
  }
  return finalConf;
}

Example 3

Source File: TezClientUtils.java From tez with Apache License 2.0

6 votes

@Private
@VisibleForTesting
static String constructAMLaunchOpts(TezConfiguration tezConf, Resource capability) {
  String defaultOpts = tezConf.get(TezConfiguration.TEZ_AM_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS,
      TezConfiguration.TEZ_AM_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS_DEFAULT);
  Path tmpDir = new Path(Environment.PWD.$(),
      YarnConfiguration.DEFAULT_CONTAINER_TEMP_DIR);
  String amOpts = "-Djava.io.tmpdir=" + tmpDir + " ";

  if (defaultOpts != null && !defaultOpts.isEmpty()) {
    amOpts = amOpts + defaultOpts + " ";
  }
  amOpts = amOpts + tezConf.get(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS,
      TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS_DEFAULT);

  amOpts = maybeAddDefaultMemoryJavaOpts(amOpts, capability,
      tezConf.getDouble(TezConfiguration.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION,
          TezConfiguration.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION_DEFAULT));

  return amOpts;
}

Example 4

Source File: TezClientUtils.java From incubator-tez with Apache License 2.0

5 votes

static void setDefaultLaunchCmdOpts(Vertex v, TezConfiguration conf) {
  String vOpts = v.getTaskLaunchCmdOpts();
  String vConfigOpts = conf.get(TezConfiguration.TEZ_TASK_LAUNCH_CMD_OPTS,
      TezConfiguration.TEZ_TASK_LAUNCH_CMD_OPTS_DEFAULT);
  if (vConfigOpts != null && vConfigOpts.length() > 0) {
    vOpts += (" " + vConfigOpts);
  }
  
  vOpts = maybeAddDefaultLoggingJavaOpts(conf.get(
      TezConfiguration.TEZ_TASK_LOG_LEVEL,
      TezConfiguration.TEZ_TASK_LOG_LEVEL_DEFAULT), vOpts);
  v.setTaskLaunchCmdOpts(vOpts);
}

Example 5

Source File: TezClientUtils.java From tez with Apache License 2.0

5 votes

static Credentials prepareAmLaunchCredentials(AMConfiguration amConfig, Credentials sessionCreds,
    TezConfiguration conf, Path binaryConfPath) throws IOException {
  // Setup security tokens
  Credentials amLaunchCredentials = new Credentials();

  // Add SimpleHistoryLoggingService logDir creds to the list of session credentials
  // If it is on HDFS
  String simpleHistoryLogDir = conf.get(TezConfiguration.TEZ_SIMPLE_HISTORY_LOGGING_DIR);
  if (simpleHistoryLogDir != null && !simpleHistoryLogDir.isEmpty()) {
    Path simpleHistoryLogDirPath = new Path(simpleHistoryLogDir);
    TokenCache.obtainTokensForFileSystems(sessionCreds, new Path[] { simpleHistoryLogDirPath },
        conf);
  }

  // Add Staging dir creds to the list of session credentials.
  TokenCache.obtainTokensForFileSystems(sessionCreds, new Path[] {binaryConfPath }, conf);

  populateTokenCache(conf, sessionCreds);

  // Add session specific credentials to the AM credentials.
  amLaunchCredentials.mergeAll(sessionCreds);

  if (amConfig.getCredentials() != null) {
    amLaunchCredentials.mergeAll(amConfig.getCredentials());
  }
  TezCommonUtils.logCredentials(LOG, amLaunchCredentials, "amLaunch");
  return amLaunchCredentials;
}

Example 6

Source File: DAGClientTimelineImpl.java From tez with Apache License 2.0

5 votes

public DAGClientTimelineImpl(ApplicationId appId, String dagId, TezConfiguration conf,
                             FrameworkClient frameworkClient, int connTimeout)
    throws TezException {

  if (!TimelineReaderFactory.isTimelineClientSupported()) {
    throw new TezException("Reading from secure timeline is supported only for hadoop 2.6 and above.");
  }

  this.appId = appId;
  this.dagId = dagId;
  this.frameworkClient = frameworkClient;

  String scheme;
  String webAppAddress;
  boolean useHttps = webappHttpsOnly(conf);
  if (useHttps) {
    scheme = HTTPS_SCHEME;
    webAppAddress = conf.get(ATSConstants.TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS_CONF_NAME);
  } else {
    scheme = HTTP_SCHEME;
    webAppAddress = conf.get(ATSConstants.TIMELINE_SERVICE_WEBAPP_HTTP_ADDRESS_CONF_NAME);
  }
  if (webAppAddress == null) {
    throw new TezException("Failed to get ATS webapp address");
  }

  baseUri = Joiner.on("").join(scheme, webAppAddress, ATSConstants.RESOURCE_URI_BASE);

  timelineReaderStrategy =
      TimelineReaderFactory.getTimelineReaderStrategy(conf, useHttps, connTimeout);
}

Example 7

Source File: MRToTezHelper.java From spork with Apache License 2.0

4 votes

public static TezConfiguration getDAGAMConfFromMRConf(
        Configuration tezConf) {

    // Set Tez parameters based on MR parameters.
    TezConfiguration dagAMConf = new TezConfiguration(tezConf);
    Map<String, String> mrParamToDAGParamMap = DeprecatedKeys
            .getMRToDAGParamMap();

    for (Entry<String, String> entry : mrParamToDAGParamMap.entrySet()) {
        if (dagAMConf.get(entry.getKey()) != null) {
            dagAMConf.set(entry.getValue(), dagAMConf.get(entry.getKey()));
            dagAMConf.unset(entry.getKey());
            if (LOG.isDebugEnabled()) {
                LOG.debug("MR->DAG Translating MR key: " + entry.getKey()
                        + " to Tez key: " + entry.getValue()
                        + " with value " + dagAMConf.get(entry.getValue()));
            }
        }
    }

    String env = tezConf.get(MRJobConfig.MR_AM_ADMIN_USER_ENV);
    if (tezConf.get(MRJobConfig.MR_AM_ENV) != null) {
        env = (env == null) ? tezConf.get(MRJobConfig.MR_AM_ENV)
                            : env + "," + tezConf.get(MRJobConfig.MR_AM_ENV);
    }

    if (env != null) {
        dagAMConf.setIfUnset(TezConfiguration.TEZ_AM_LAUNCH_ENV, env);
    }

    dagAMConf.setIfUnset(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS,
            org.apache.tez.mapreduce.hadoop.MRHelpers
                    .getJavaOptsForMRAM(tezConf));

    String queueName = tezConf.get(JobContext.QUEUE_NAME,
            YarnConfiguration.DEFAULT_QUEUE_NAME);
    dagAMConf.setIfUnset(TezConfiguration.TEZ_QUEUE_NAME, queueName);

    int amMemMB = tezConf.getInt(MRJobConfig.MR_AM_VMEM_MB,
            MRJobConfig.DEFAULT_MR_AM_VMEM_MB);
    int amCores = tezConf.getInt(MRJobConfig.MR_AM_CPU_VCORES,
            MRJobConfig.DEFAULT_MR_AM_CPU_VCORES);
    dagAMConf.setIfUnset(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, ""
            + amMemMB);
    dagAMConf.setIfUnset(TezConfiguration.TEZ_AM_RESOURCE_CPU_VCORES, ""
            + amCores);

    dagAMConf.setIfUnset(TezConfiguration.TEZ_AM_VIEW_ACLS,
            tezConf.get(MRJobConfig.JOB_ACL_VIEW_JOB, MRJobConfig.DEFAULT_JOB_ACL_VIEW_JOB));

    dagAMConf.setIfUnset(TezConfiguration.TEZ_AM_MODIFY_ACLS,
            tezConf.get(MRJobConfig.JOB_ACL_MODIFY_JOB, MRJobConfig.DEFAULT_JOB_ACL_MODIFY_JOB));

    dagAMConf.setIfUnset(TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS, ""
            + dagAMConf.getInt(MRJobConfig.MR_AM_MAX_ATTEMPTS,
                    MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS));

    if (tezConf.get(MRConfiguration.JOB_CREDENTIALS_BINARY) != null) {
        dagAMConf.setIfUnset(TezConfiguration.TEZ_CREDENTIALS_PATH,
                tezConf.get(MRConfiguration.JOB_CREDENTIALS_BINARY));
    }

    //TODO: Strip out all MR settings

    return dagAMConf;
}

Example 8

Source File: CartesianProductConfig.java From tez with Apache License 2.0

4 votes

protected CartesianProductConfigProto toProto(TezConfiguration conf) {
  CartesianProductConfigProto.Builder builder =
    CartesianProductConfigProto.newBuilder();
  builder.setIsPartitioned(this.isPartitioned)
    .addAllSources(Arrays.asList(sources));

  if (isPartitioned) {
    builder.addAllNumPartitions(Ints.asList(numPartitions));
    if (filterDescriptor != null) {
      builder.setFilterClassName(filterDescriptor.getClassName());
      UserPayload filterUesrPayload = filterDescriptor.getUserPayload();
      if (filterUesrPayload != null) {
        builder.setFilterUserPayload(ByteString.copyFrom(filterUesrPayload.getPayload()));
      }
    }
  }

  if (conf != null) {
    builder.setMinFraction(conf.getFloat(
      CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_SLOW_START_MIN_FRACTION,
      CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_SLOW_START_MIN_FRACTION_DEFAULT));
    builder.setMaxFraction(conf.getFloat(
      CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_SLOW_START_MAX_FRACTION,
      CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_SLOW_START_MAX_FRACTION_DEFAULT));
    builder.setMaxParallelism(conf.getInt(
      CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_MAX_PARALLELISM,
      CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_MAX_PARALLELISM_DEFAULT));
    builder.setMinOpsPerWorker(conf.getLong(
      CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_MIN_OPS_PER_WORKER,
      CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_MIN_OPS_PER_WORKER_DEFAULT));
    builder.setEnableGrouping(conf.getBoolean(
      CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_ENABLE_GROUPING,
      CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_ENABLE_GROUPING_DEFAULT));
    if (conf.get(CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_GROUPING_FRACTION) != null) {
      builder.setGroupingFraction(Float.parseFloat(
        conf.get(CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_GROUPING_FRACTION)));
      Preconditions.checkArgument(0 < builder.getGroupingFraction() &&
        builder.getGroupingFraction() <= 1, "grouping fraction should be larger than 0 and less" +
        " or equal to 1, current value: " + builder.getGroupingFraction());
    }
    if (conf.get(CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_NUM_PARTITIONS) != null) {
      builder.setNumPartitionsForFairCase(Integer.parseInt(
        conf.get(CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_NUM_PARTITIONS)));
      Preconditions.checkArgument(builder.getNumPartitionsForFairCase() > 0,
        "Number of partitions for fair cartesian product should be positive integer");
    }
  }
  Preconditions.checkArgument(builder.getMinFraction() <= builder.getMaxFraction(),
    "min fraction(" + builder.getMinFraction() + ") should be less than max fraction(" +
      builder.getMaxFraction() + ") in cartesian product slow start");
  Preconditions.checkArgument(builder.getMaxParallelism() > 0,
    "max parallelism must be positive, currently is " + builder.getMaxParallelism());
  Preconditions.checkArgument(builder.getMinOpsPerWorker() > 0,
    "Min ops per worker must be positive, currently is " + builder.getMinOpsPerWorker());

  return builder.build();
}