Java Code Examples for org.apache.hadoop.mapreduce.v2.util.MRApps#addLog4jSystemProperties()

The following examples show how to use org.apache.hadoop.mapreduce.v2.util.MRApps#addLog4jSystemProperties() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: MapReduceChildJVM.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
public static List<String> getVMCommand(
    InetSocketAddress taskAttemptListenerAddr, Task task, 
    JVMId jvmID) {

  TaskAttemptID attemptID = task.getTaskID();
  JobConf conf = task.conf;

  Vector<String> vargs = new Vector<String>(8);

  vargs.add(MRApps.crossPlatformifyMREnv(task.conf, Environment.JAVA_HOME)
      + "/bin/java");

  // Add child (task) java-vm options.
  //
  // The following symbols if present in mapred.{map|reduce}.child.java.opts 
  // value are replaced:
  // + @taskid@ is interpolated with value of TaskID.
  // Other occurrences of @ will not be altered.
  //
  // Example with multiple arguments and substitutions, showing
  // jvm GC logging, and start of a passwordless JVM JMX agent so can
  // connect with jconsole and the likes to watch child memory, threads
  // and get thread dumps.
  //
  //  <property>
  //    <name>mapred.map.child.java.opts</name>
  //    <value>-Xmx 512M -verbose:gc -Xloggc:/tmp/@[email protected] \
  //           -Dcom.sun.management.jmxremote.authenticate=false \
  //           -Dcom.sun.management.jmxremote.ssl=false \
  //    </value>
  //  </property>
  //
  //  <property>
  //    <name>mapred.reduce.child.java.opts</name>
  //    <value>-Xmx 1024M -verbose:gc -Xloggc:/tmp/@[email protected] \
  //           -Dcom.sun.management.jmxremote.authenticate=false \
  //           -Dcom.sun.management.jmxremote.ssl=false \
  //    </value>
  //  </property>
  //
  String javaOpts = getChildJavaOpts(conf, task.isMapTask());
  javaOpts = javaOpts.replace("@taskid@", attemptID.toString());
  String [] javaOptsSplit = javaOpts.split(" ");
  for (int i = 0; i < javaOptsSplit.length; i++) {
    vargs.add(javaOptsSplit[i]);
  }

  Path childTmpDir = new Path(MRApps.crossPlatformifyMREnv(conf, Environment.PWD),
      YarnConfiguration.DEFAULT_CONTAINER_TEMP_DIR);
  vargs.add("-Djava.io.tmpdir=" + childTmpDir);
  MRApps.addLog4jSystemProperties(task, vargs, conf);

  if (conf.getProfileEnabled()) {
    if (conf.getProfileTaskRange(task.isMapTask()
                                 ).isIncluded(task.getPartition())) {
      final String profileParams = conf.get(task.isMapTask()
          ? MRJobConfig.TASK_MAP_PROFILE_PARAMS
          : MRJobConfig.TASK_REDUCE_PROFILE_PARAMS, conf.getProfileParams());
      vargs.add(String.format(profileParams,
          getTaskLogFile(TaskLog.LogName.PROFILE)));
    }
  }

  // Add main class and its arguments 
  vargs.add(YarnChild.class.getName());  // main of Child
  // pass TaskAttemptListener's address
  vargs.add(taskAttemptListenerAddr.getAddress().getHostAddress()); 
  vargs.add(Integer.toString(taskAttemptListenerAddr.getPort())); 
  vargs.add(attemptID.toString());                      // pass task identifier

  // Finally add the jvmID
  vargs.add(String.valueOf(jvmID.getId()));
  vargs.add("1>" + getTaskLogFile(TaskLog.LogName.STDOUT));
  vargs.add("2>" + getTaskLogFile(TaskLog.LogName.STDERR));

  // Final commmand
  StringBuilder mergedCommand = new StringBuilder();
  for (CharSequence str : vargs) {
    mergedCommand.append(str).append(" ");
  }
  Vector<String> vargsFinal = new Vector<String>(1);
  vargsFinal.add(mergedCommand.toString());
  return vargsFinal;
}
Example 2
Source File: MapReduceChildJVM.java    From big-c with Apache License 2.0 4 votes vote down vote up 
public static List<String> getVMCommand(
    InetSocketAddress taskAttemptListenerAddr, Task task, 
    JVMId jvmID) {

  TaskAttemptID attemptID = task.getTaskID();
  JobConf conf = task.conf;

  Vector<String> vargs = new Vector<String>(8);

  vargs.add(MRApps.crossPlatformifyMREnv(task.conf, Environment.JAVA_HOME)
      + "/bin/java");

  // Add child (task) java-vm options.
  //
  // The following symbols if present in mapred.{map|reduce}.child.java.opts 
  // value are replaced:
  // + @taskid@ is interpolated with value of TaskID.
  // Other occurrences of @ will not be altered.
  //
  // Example with multiple arguments and substitutions, showing
  // jvm GC logging, and start of a passwordless JVM JMX agent so can
  // connect with jconsole and the likes to watch child memory, threads
  // and get thread dumps.
  //
  //  <property>
  //    <name>mapred.map.child.java.opts</name>
  //    <value>-Xmx 512M -verbose:gc -Xloggc:/tmp/@[email protected] \
  //           -Dcom.sun.management.jmxremote.authenticate=false \
  //           -Dcom.sun.management.jmxremote.ssl=false \
  //    </value>
  //  </property>
  //
  //  <property>
  //    <name>mapred.reduce.child.java.opts</name>
  //    <value>-Xmx 1024M -verbose:gc -Xloggc:/tmp/@[email protected] \
  //           -Dcom.sun.management.jmxremote.authenticate=false \
  //           -Dcom.sun.management.jmxremote.ssl=false \
  //    </value>
  //  </property>
  //
  String javaOpts = getChildJavaOpts(conf, task.isMapTask());
  javaOpts = javaOpts.replace("@taskid@", attemptID.toString());
  String [] javaOptsSplit = javaOpts.split(" ");
  for (int i = 0; i < javaOptsSplit.length; i++) {
    vargs.add(javaOptsSplit[i]);
  }

  Path childTmpDir = new Path(MRApps.crossPlatformifyMREnv(conf, Environment.PWD),
      YarnConfiguration.DEFAULT_CONTAINER_TEMP_DIR);
  vargs.add("-Djava.io.tmpdir=" + childTmpDir);
  MRApps.addLog4jSystemProperties(task, vargs, conf);

  if (conf.getProfileEnabled()) {
    if (conf.getProfileTaskRange(task.isMapTask()
                                 ).isIncluded(task.getPartition())) {
      final String profileParams = conf.get(task.isMapTask()
          ? MRJobConfig.TASK_MAP_PROFILE_PARAMS
          : MRJobConfig.TASK_REDUCE_PROFILE_PARAMS, conf.getProfileParams());
      vargs.add(String.format(profileParams,
          getTaskLogFile(TaskLog.LogName.PROFILE)));
    }
  }

  // Add main class and its arguments 
  vargs.add(YarnChild.class.getName());  // main of Child
  // pass TaskAttemptListener's address
  vargs.add(taskAttemptListenerAddr.getAddress().getHostAddress()); 
  vargs.add(Integer.toString(taskAttemptListenerAddr.getPort())); 
  vargs.add(attemptID.toString());                      // pass task identifier

  // Finally add the jvmID
  vargs.add(String.valueOf(jvmID.getId()));
  vargs.add("1>" + getTaskLogFile(TaskLog.LogName.STDOUT));
  vargs.add("2>" + getTaskLogFile(TaskLog.LogName.STDERR));

  // Final commmand
  StringBuilder mergedCommand = new StringBuilder();
  for (CharSequence str : vargs) {
    mergedCommand.append(str).append(" ");
  }
  Vector<String> vargsFinal = new Vector<String>(1);
  vargsFinal.add(mergedCommand.toString());
  return vargsFinal;
}