Java Code Examples for org.apache.hadoop.yarn.conf.YarnConfiguration#DEFAULT_CONTAINER_TEMP_DIR

The following examples show how to use org.apache.hadoop.yarn.conf.YarnConfiguration#DEFAULT_CONTAINER_TEMP_DIR . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: TezClientUtils.java From tez with Apache License 2.0

6 votes

@Private
@VisibleForTesting
static String constructAMLaunchOpts(TezConfiguration tezConf, Resource capability) {
  String defaultOpts = tezConf.get(TezConfiguration.TEZ_AM_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS,
      TezConfiguration.TEZ_AM_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS_DEFAULT);
  Path tmpDir = new Path(Environment.PWD.$(),
      YarnConfiguration.DEFAULT_CONTAINER_TEMP_DIR);
  String amOpts = "-Djava.io.tmpdir=" + tmpDir + " ";

  if (defaultOpts != null && !defaultOpts.isEmpty()) {
    amOpts = amOpts + defaultOpts + " ";
  }
  amOpts = amOpts + tezConf.get(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS,
      TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS_DEFAULT);

  amOpts = maybeAddDefaultMemoryJavaOpts(amOpts, capability,
      tezConf.getDouble(TezConfiguration.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION,
          TezConfiguration.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION_DEFAULT));

  return amOpts;
}

Example 2

Source File: LaunchContainerRunnable.java From Bats with Apache License 2.0

4 votes

/**
 * Build the command to launch the child VM in the container
 *
 * @param jvmID
 * @return
 */
public List<CharSequence> getChildVMCommand(String jvmID)
{

  List<CharSequence> vargs = new ArrayList<>(8);

  if (!StringUtils.isBlank(System.getenv(Environment.JAVA_HOME.key()))) {
    // node manager provides JAVA_HOME
    vargs.add(Environment.JAVA_HOME.$() + "/bin/java");
  } else {
    vargs.add("java");
  }

  String jvmOpts = dag.getAttributes().get(LogicalPlan.CONTAINER_JVM_OPTIONS);
  if (jvmOpts == null) {
    if (dag.isDebug()) {
      vargs.add(JAVA_REMOTE_DEBUG_OPTS);
    }
  } else {
    Map<String, String> params = new HashMap<>();
    params.put("applicationId", Integer.toString(container.getId().getApplicationAttemptId().getApplicationId().getId()));
    params.put("containerId", Integer.toString(container.getId().getId()));
    StrSubstitutor sub = new StrSubstitutor(params, "%(", ")");
    vargs.add(sub.replace(jvmOpts));
    if (dag.isDebug() && !jvmOpts.contains("-agentlib:jdwp=")) {
      vargs.add(JAVA_REMOTE_DEBUG_OPTS);
    }
  }

  List<DAG.OperatorMeta> operatorMetaList = Lists.newArrayList();
  int bufferServerMemory = 0;
  for (PTOperator operator : sca.getContainer().getOperators()) {
    bufferServerMemory += operator.getBufferServerMemory();
    operatorMetaList.add(operator.getOperatorMeta());
  }
  Context.ContainerOptConfigurator containerOptConfigurator = dag.getAttributes().get(LogicalPlan.CONTAINER_OPTS_CONFIGURATOR);
  jvmOpts = containerOptConfigurator.getJVMOptions(operatorMetaList);
  jvmOpts = parseJvmOpts(jvmOpts, ((long)bufferServerMemory) * MB_TO_B);
  LOG.info("Jvm opts {} for container {}",jvmOpts,container.getId());
  vargs.add(jvmOpts);

  Path childTmpDir = new Path(Environment.PWD.$(), YarnConfiguration.DEFAULT_CONTAINER_TEMP_DIR);
  vargs.add(String.format("-D%s=%s", StreamingContainer.PROP_APP_PATH, dag.assertAppPath()));
  vargs.add("-Djava.io.tmpdir=" + childTmpDir);
  vargs.add(String.format("-D%scid=%s", StreamingApplication.DT_PREFIX, jvmID));
  vargs.add("-Dhadoop.root.logger=" + (dag.isDebug() ? "DEBUG" : "INFO") + ",RFA");
  vargs.add("-Dhadoop.log.dir=" + ApplicationConstants.LOG_DIR_EXPANSION_VAR);
  StramClientUtils.addAttributeToArgs(LogicalPlan.APPLICATION_NAME, dag, vargs);
  StramClientUtils.addAttributeToArgs(LogicalPlan.LOGGER_APPENDER, dag, vargs);

  String loggersLevel = System.getProperty(StramUtils.DT_LOGGERS_LEVEL);
  if (loggersLevel != null) {
    vargs.add(String.format("-D%s=%s", StramUtils.DT_LOGGERS_LEVEL, loggersLevel));
  }
  // Add main class and its arguments
  vargs.add(StreamingContainer.class.getName());  // main of Child

  vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout");
  vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr");

  // Final commmand
  StringBuilder mergedCommand = new StringBuilder(256);
  for (CharSequence str : vargs) {
    mergedCommand.append(str).append(" ");
  }
  List<CharSequence> vargsFinal = new ArrayList<>(1);
  vargsFinal.add(mergedCommand.toString());
  return vargsFinal;

}

Example 3

Source File: MapReduceChildJVM.java From hadoop with Apache License 2.0

4 votes

public static List<String> getVMCommand(
    InetSocketAddress taskAttemptListenerAddr, Task task, 
    JVMId jvmID) {

  TaskAttemptID attemptID = task.getTaskID();
  JobConf conf = task.conf;

  Vector<String> vargs = new Vector<String>(8);

  vargs.add(MRApps.crossPlatformifyMREnv(task.conf, Environment.JAVA_HOME)
      + "/bin/java");

  // Add child (task) java-vm options.
  //
  // The following symbols if present in mapred.{map|reduce}.child.java.opts 
  // value are replaced:
  // + @taskid@ is interpolated with value of TaskID.
  // Other occurrences of @ will not be altered.
  //
  // Example with multiple arguments and substitutions, showing
  // jvm GC logging, and start of a passwordless JVM JMX agent so can
  // connect with jconsole and the likes to watch child memory, threads
  // and get thread dumps.
  //
  //  <property>
  //    <name>mapred.map.child.java.opts</name>
  //    <value>-Xmx 512M -verbose:gc -Xloggc:/tmp/@[email protected] \
  //           -Dcom.sun.management.jmxremote.authenticate=false \
  //           -Dcom.sun.management.jmxremote.ssl=false \
  //    </value>
  //  </property>
  //
  //  <property>
  //    <name>mapred.reduce.child.java.opts</name>
  //    <value>-Xmx 1024M -verbose:gc -Xloggc:/tmp/@[email protected] \
  //           -Dcom.sun.management.jmxremote.authenticate=false \
  //           -Dcom.sun.management.jmxremote.ssl=false \
  //    </value>
  //  </property>
  //
  String javaOpts = getChildJavaOpts(conf, task.isMapTask());
  javaOpts = javaOpts.replace("@taskid@", attemptID.toString());
  String [] javaOptsSplit = javaOpts.split(" ");
  for (int i = 0; i < javaOptsSplit.length; i++) {
    vargs.add(javaOptsSplit[i]);
  }

  Path childTmpDir = new Path(MRApps.crossPlatformifyMREnv(conf, Environment.PWD),
      YarnConfiguration.DEFAULT_CONTAINER_TEMP_DIR);
  vargs.add("-Djava.io.tmpdir=" + childTmpDir);
  MRApps.addLog4jSystemProperties(task, vargs, conf);

  if (conf.getProfileEnabled()) {
    if (conf.getProfileTaskRange(task.isMapTask()
                                 ).isIncluded(task.getPartition())) {
      final String profileParams = conf.get(task.isMapTask()
          ? MRJobConfig.TASK_MAP_PROFILE_PARAMS
          : MRJobConfig.TASK_REDUCE_PROFILE_PARAMS, conf.getProfileParams());
      vargs.add(String.format(profileParams,
          getTaskLogFile(TaskLog.LogName.PROFILE)));
    }
  }

  // Add main class and its arguments 
  vargs.add(YarnChild.class.getName());  // main of Child
  // pass TaskAttemptListener's address
  vargs.add(taskAttemptListenerAddr.getAddress().getHostAddress()); 
  vargs.add(Integer.toString(taskAttemptListenerAddr.getPort())); 
  vargs.add(attemptID.toString());                      // pass task identifier

  // Finally add the jvmID
  vargs.add(String.valueOf(jvmID.getId()));
  vargs.add("1>" + getTaskLogFile(TaskLog.LogName.STDOUT));
  vargs.add("2>" + getTaskLogFile(TaskLog.LogName.STDERR));

  // Final commmand
  StringBuilder mergedCommand = new StringBuilder();
  for (CharSequence str : vargs) {
    mergedCommand.append(str).append(" ");
  }
  Vector<String> vargsFinal = new Vector<String>(1);
  vargsFinal.add(mergedCommand.toString());
  return vargsFinal;
}

Example 4

Source File: MapReduceChildJVM.java From big-c with Apache License 2.0

4 votes

public static List<String> getVMCommand(
    InetSocketAddress taskAttemptListenerAddr, Task task, 
    JVMId jvmID) {

  TaskAttemptID attemptID = task.getTaskID();
  JobConf conf = task.conf;

  Vector<String> vargs = new Vector<String>(8);

  vargs.add(MRApps.crossPlatformifyMREnv(task.conf, Environment.JAVA_HOME)
      + "/bin/java");

  // Add child (task) java-vm options.
  //
  // The following symbols if present in mapred.{map|reduce}.child.java.opts 
  // value are replaced:
  // + @taskid@ is interpolated with value of TaskID.
  // Other occurrences of @ will not be altered.
  //
  // Example with multiple arguments and substitutions, showing
  // jvm GC logging, and start of a passwordless JVM JMX agent so can
  // connect with jconsole and the likes to watch child memory, threads
  // and get thread dumps.
  //
  //  <property>
  //    <name>mapred.map.child.java.opts</name>
  //    <value>-Xmx 512M -verbose:gc -Xloggc:/tmp/@[email protected] \
  //           -Dcom.sun.management.jmxremote.authenticate=false \
  //           -Dcom.sun.management.jmxremote.ssl=false \
  //    </value>
  //  </property>
  //
  //  <property>
  //    <name>mapred.reduce.child.java.opts</name>
  //    <value>-Xmx 1024M -verbose:gc -Xloggc:/tmp/@[email protected] \
  //           -Dcom.sun.management.jmxremote.authenticate=false \
  //           -Dcom.sun.management.jmxremote.ssl=false \
  //    </value>
  //  </property>
  //
  String javaOpts = getChildJavaOpts(conf, task.isMapTask());
  javaOpts = javaOpts.replace("@taskid@", attemptID.toString());
  String [] javaOptsSplit = javaOpts.split(" ");
  for (int i = 0; i < javaOptsSplit.length; i++) {
    vargs.add(javaOptsSplit[i]);
  }

  Path childTmpDir = new Path(MRApps.crossPlatformifyMREnv(conf, Environment.PWD),
      YarnConfiguration.DEFAULT_CONTAINER_TEMP_DIR);
  vargs.add("-Djava.io.tmpdir=" + childTmpDir);
  MRApps.addLog4jSystemProperties(task, vargs, conf);

  if (conf.getProfileEnabled()) {
    if (conf.getProfileTaskRange(task.isMapTask()
                                 ).isIncluded(task.getPartition())) {
      final String profileParams = conf.get(task.isMapTask()
          ? MRJobConfig.TASK_MAP_PROFILE_PARAMS
          : MRJobConfig.TASK_REDUCE_PROFILE_PARAMS, conf.getProfileParams());
      vargs.add(String.format(profileParams,
          getTaskLogFile(TaskLog.LogName.PROFILE)));
    }
  }

  // Add main class and its arguments 
  vargs.add(YarnChild.class.getName());  // main of Child
  // pass TaskAttemptListener's address
  vargs.add(taskAttemptListenerAddr.getAddress().getHostAddress()); 
  vargs.add(Integer.toString(taskAttemptListenerAddr.getPort())); 
  vargs.add(attemptID.toString());                      // pass task identifier

  // Finally add the jvmID
  vargs.add(String.valueOf(jvmID.getId()));
  vargs.add("1>" + getTaskLogFile(TaskLog.LogName.STDOUT));
  vargs.add("2>" + getTaskLogFile(TaskLog.LogName.STDERR));

  // Final commmand
  StringBuilder mergedCommand = new StringBuilder();
  for (CharSequence str : vargs) {
    mergedCommand.append(str).append(" ");
  }
  Vector<String> vargsFinal = new Vector<String>(1);
  vargsFinal.add(mergedCommand.toString());
  return vargsFinal;
}

Example 5

Source File: LaunchContainerRunnable.java From attic-apex-core with Apache License 2.0

4 votes

/**
 * Build the command to launch the child VM in the container
 *
 * @param jvmID
 * @return
 */
public List<CharSequence> getChildVMCommand(String jvmID)
{

  List<CharSequence> vargs = new ArrayList<>(8);

  if (!StringUtils.isBlank(System.getenv(Environment.JAVA_HOME.key()))) {
    // node manager provides JAVA_HOME
    vargs.add(Environment.JAVA_HOME.$() + "/bin/java");
  } else {
    vargs.add("java");
  }

  String jvmOpts = dag.getAttributes().get(LogicalPlan.CONTAINER_JVM_OPTIONS);
  if (jvmOpts == null) {
    if (dag.isDebug()) {
      vargs.add(JAVA_REMOTE_DEBUG_OPTS);
    }
  } else {
    Map<String, String> params = new HashMap<>();
    params.put("applicationId", Integer.toString(container.getId().getApplicationAttemptId().getApplicationId().getId()));
    params.put("containerId", Integer.toString(container.getId().getId()));
    StrSubstitutor sub = new StrSubstitutor(params, "%(", ")");
    vargs.add(sub.replace(jvmOpts));
    if (dag.isDebug() && !jvmOpts.contains("-agentlib:jdwp=")) {
      vargs.add(JAVA_REMOTE_DEBUG_OPTS);
    }
  }

  List<DAG.OperatorMeta> operatorMetaList = Lists.newArrayList();
  int bufferServerMemory = 0;
  for (PTOperator operator : sca.getContainer().getOperators()) {
    bufferServerMemory += operator.getBufferServerMemory();
    operatorMetaList.add(operator.getOperatorMeta());
  }
  Context.ContainerOptConfigurator containerOptConfigurator = dag.getAttributes().get(LogicalPlan.CONTAINER_OPTS_CONFIGURATOR);
  jvmOpts = containerOptConfigurator.getJVMOptions(operatorMetaList);
  jvmOpts = parseJvmOpts(jvmOpts, ((long)bufferServerMemory) * MB_TO_B);
  LOG.info("Jvm opts {} for container {}",jvmOpts,container.getId());
  vargs.add(jvmOpts);

  Path childTmpDir = new Path(Environment.PWD.$(), YarnConfiguration.DEFAULT_CONTAINER_TEMP_DIR);
  vargs.add(String.format("-D%s=%s", StreamingContainer.PROP_APP_PATH, dag.assertAppPath()));
  vargs.add("-Djava.io.tmpdir=" + childTmpDir);
  vargs.add(String.format("-D%scid=%s", StreamingApplication.DT_PREFIX, jvmID));
  vargs.add("-Dhadoop.root.logger=" + (dag.isDebug() ? "DEBUG" : "INFO") + ",RFA");
  vargs.add("-Dhadoop.log.dir=" + ApplicationConstants.LOG_DIR_EXPANSION_VAR);
  StramClientUtils.addAttributeToArgs(LogicalPlan.APPLICATION_NAME, dag, vargs);
  StramClientUtils.addAttributeToArgs(LogicalPlan.LOGGER_APPENDER, dag, vargs);

  String loggersLevel = System.getProperty(StramUtils.DT_LOGGERS_LEVEL);
  if (loggersLevel != null) {
    vargs.add(String.format("-D%s=%s", StramUtils.DT_LOGGERS_LEVEL, loggersLevel));
  }
  // Add main class and its arguments
  vargs.add(StreamingContainer.class.getName());  // main of Child

  vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout");
  vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr");

  // Final commmand
  StringBuilder mergedCommand = new StringBuilder(256);
  for (CharSequence str : vargs) {
    mergedCommand.append(str).append(" ");
  }
  List<CharSequence> vargsFinal = new ArrayList<>(1);
  vargsFinal.add(mergedCommand.toString());
  return vargsFinal;

}

Example 6

Source File: TezRuntimeChildJVM.java From incubator-tez with Apache License 2.0

4 votes

public static List<String> getVMCommand(
    InetSocketAddress taskAttemptListenerAddr,
    String containerIdentifier,
    String tokenIdentifier,
    int applicationAttemptNumber,
    String javaOpts) {

  Vector<String> vargs = new Vector<String>(9);

  vargs.add(Environment.JAVA_HOME.$() + "/bin/java");

  //set custom javaOpts
  vargs.add(javaOpts);

  Path childTmpDir = new Path(Environment.PWD.$(),
      YarnConfiguration.DEFAULT_CONTAINER_TEMP_DIR);
  vargs.add("-Djava.io.tmpdir=" + childTmpDir);

  // Add main class and its arguments
  vargs.add(TezChild.class.getName());  // main of Child

  // pass TaskAttemptListener's address
  vargs.add(taskAttemptListenerAddr.getAddress().getHostAddress());
  vargs.add(Integer.toString(taskAttemptListenerAddr.getPort()));
  vargs.add(containerIdentifier);
  vargs.add(tokenIdentifier);
  vargs.add(Integer.toString(applicationAttemptNumber));

  vargs.add("1>" + getTaskLogFile(LogName.STDOUT));
  vargs.add("2>" + getTaskLogFile(LogName.STDERR));

  // TODO Is this StringBuilder really required ? YARN already accepts a list of commands.
  // Final commmand
  StringBuilder mergedCommand = new StringBuilder();
  for (CharSequence str : vargs) {
    mergedCommand.append(str).append(" ");
  }
  Vector<String> vargsFinal = new Vector<String>(1);
  vargsFinal.add(mergedCommand.toString());
  return vargsFinal;
}

Example 7

Source File: TezRuntimeChildJVM.java From tez with Apache License 2.0

4 votes

public static List<String> getVMCommand(
    InetSocketAddress taskAttemptListenerAddr,
    String containerIdentifier,
    String tokenIdentifier,
    int applicationAttemptNumber,
    String javaOpts) {

  Vector<String> vargs = new Vector<String>(9);

  vargs.add(Environment.JAVA_HOME.$() + "/bin/java");

  //set custom javaOpts
  vargs.add(javaOpts);

  Path childTmpDir = new Path(Environment.PWD.$(),
      YarnConfiguration.DEFAULT_CONTAINER_TEMP_DIR);
  vargs.add("-Djava.io.tmpdir=" + childTmpDir);

  // Add main class and its arguments
  vargs.add(TezChild.class.getName());  // main of Child

  // pass TaskAttemptListener's address
  vargs.add(taskAttemptListenerAddr.getAddress().getHostName());
  vargs.add(Integer.toString(taskAttemptListenerAddr.getPort()));
  vargs.add(containerIdentifier);
  vargs.add(tokenIdentifier);
  vargs.add(Integer.toString(applicationAttemptNumber));

  vargs.add("1>" + getTaskLogFile(LogName.STDOUT));
  vargs.add("2>" + getTaskLogFile(LogName.STDERR));

  // TODO Is this StringBuilder really required ? YARN already accepts a list of commands.
  // Final commmand
  StringBuilder mergedCommand = new StringBuilder();
  for (CharSequence str : vargs) {
    mergedCommand.append(str).append(" ");
  }
  Vector<String> vargsFinal = new Vector<String>(1);
  vargsFinal.add(mergedCommand.toString());
  return vargsFinal;
}

Example 8

Source File: TestTezClientUtils.java From tez with Apache License 2.0

4 votes

@Test(timeout = 5000)
public void testAMCommandOpts() {
  Path tmpDir = new Path(Environment.PWD.$(),
      YarnConfiguration.DEFAULT_CONTAINER_TEMP_DIR);
  String tmpOpts = "-Djava.io.tmpdir=" + tmpDir;
  TezConfiguration tezConf = new TezConfiguration();
  String amCommandOpts = "-Xmx 200m -Dtest.property";
  tezConf.set(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS, amCommandOpts);

  // Test1: Rely on defaults for cluster-default opts
  String amOptsConstructed =
      TezClientUtils.constructAMLaunchOpts(tezConf, Resource.newInstance(1024, 1));
  assertEquals(tmpOpts + " "
      + TezConfiguration.TEZ_AM_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS_DEFAULT + " "
      + amCommandOpts,
      amOptsConstructed);

  // Test2: Setup cluster-default command opts explicitly
  String clusterDefaultCommandOpts =
      "-server -Djava.net.preferIPv4Stack=true -XX:+PrintGCDetails -verbose:gc ";
  tezConf.set(TezConfiguration.TEZ_AM_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS, clusterDefaultCommandOpts);
  amOptsConstructed =
      TezClientUtils.constructAMLaunchOpts(tezConf, Resource.newInstance(1024, 1));
  assertEquals(tmpOpts + " " + clusterDefaultCommandOpts + " " + amCommandOpts, amOptsConstructed);


  // Test3: Don't setup Xmx explicitly
  final double factor = 0.8;
  amCommandOpts = "-Dtest.property";
  tezConf.set(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS, amCommandOpts);
  amOptsConstructed =
      TezClientUtils.constructAMLaunchOpts(tezConf, Resource.newInstance(1024, 1));
  // It's OK for the Xmx value to show up before cluster default options, since Xmx will not be replaced if it already exists.
  assertEquals(
      " -Xmx" + ((int) (1024 * factor)) + "m" + " " + tmpOpts + " " + clusterDefaultCommandOpts + " " +
          amCommandOpts,
      amOptsConstructed);

  // Test4: Ensure admin options with Xmx does not cause them to be overridden. This should almost never be done though.
  clusterDefaultCommandOpts =
      "-server -Djava.net.preferIPv4Stack=true -XX:+PrintGCDetails -verbose:gc -Xmx200m";
  tezConf.set(TezConfiguration.TEZ_AM_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS, clusterDefaultCommandOpts);
  amOptsConstructed =
      TezClientUtils.constructAMLaunchOpts(tezConf, Resource.newInstance(1024, 1));
  assertEquals(tmpOpts + " " + clusterDefaultCommandOpts + " " + amCommandOpts, amOptsConstructed);
}