org.apache.hadoop.yarn.api.ApplicationConstants.Environment Java Examples

The following examples show how to use org.apache.hadoop.yarn.api.ApplicationConstants.Environment. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: LoggerUtil.java    From attic-apex-core with Apache License 2.0 6 votes vote down vote up
/**
 * Makes MDC properties
 */
public static void setupMDC(String service)
{
  MDC.put("apex.service", service);

  String value = StramClientUtils.getHostName();
  MDC.put("apex.node", value == null ? "unknown" : value);

  value = System.getenv(Environment.USER.key());
  if (value != null) {
    MDC.put("apex.user", value);
  }

  value = System.getenv(Environment.CONTAINER_ID.name());
  if (value != null) {
    ContainerId containerId = ConverterUtils.toContainerId(value);
    ApplicationId applicationId = containerId.getApplicationAttemptId().getApplicationId();
    MDC.put("apex.containerId", containerId.toString());
    MDC.put("apex.applicationId", applicationId.toString());
  }

  value = System.getProperty(APPLICATION_NAME.getLongName());
  if (value != null) {
    MDC.put("apex.application", value);
  }
}
 
Example #2
Source File: TestMRHelpers.java    From incubator-tez with Apache License 2.0 6 votes vote down vote up
private void testCommonEnvSettingsForMRTasks(Map<String, String> env) {
    Assert.assertTrue(env.containsKey("foo"));
    Assert.assertTrue(env.containsKey("bar"));
    Assert.assertTrue(env.containsKey(Environment.LD_LIBRARY_PATH.name()));
    Assert.assertTrue(env.containsKey(Environment.SHELL.name()));
    Assert.assertTrue(env.containsKey("HADOOP_ROOT_LOGGER"));
    Assert.assertEquals("$PWD:$TEZ_ADMIN_ENV_TEST/lib/native",
        env.get(Environment.LD_LIBRARY_PATH.name()));

//    TEZ-273 will reinstate this or similar. 
//    for (String val : YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH) {
//      Assert.assertTrue(env.get(Environment.CLASSPATH.name()).contains(val));
//    }
//    Assert.assertTrue(0 ==
//        env.get(Environment.CLASSPATH.name()).indexOf(Environment.PWD.$()));
  }
 
Example #3
Source File: TezChild.java    From incubator-tez with Apache License 2.0 6 votes vote down vote up
/**
 * Setup
 * 
 * @param containerTask
 *          the new task specification. Must be a valid task
 * @param childUGI
 *          the old UGI instance being used
 * @return
 */
UserGroupInformation handleNewTaskCredentials(ContainerTask containerTask,
    UserGroupInformation childUGI) {
  // Re-use the UGI only if the Credentials have not changed.
  Preconditions.checkState(containerTask.shouldDie() != true);
  Preconditions.checkState(containerTask.getTaskSpec() != null);
  if (containerTask.haveCredentialsChanged()) {
    LOG.info("Refreshing UGI since Credentials have changed");
    Credentials taskCreds = containerTask.getCredentials();
    if (taskCreds != null) {
      LOG.info("Credentials : #Tokens=" + taskCreds.numberOfTokens() + ", #SecretKeys="
          + taskCreds.numberOfSecretKeys());
      childUGI = UserGroupInformation.createRemoteUser(System
          .getenv(ApplicationConstants.Environment.USER.toString()));
      childUGI.addCredentials(containerTask.getCredentials());
    } else {
      LOG.info("Not loading any credentials, since no credentials provided");
    }
  }
  return childUGI;
}
 
Example #4
Source File: TezYARNUtils.java    From incubator-tez with Apache License 2.0 6 votes vote down vote up
public static String getFrameworkClasspath(Configuration conf) {
  Map<String, String> environment = new HashMap<String, String>();

  TezYARNUtils.addToEnvironment(environment,
      Environment.CLASSPATH.name(),
      Environment.PWD.$(),
      File.pathSeparator);

  TezYARNUtils.addToEnvironment(environment,
      Environment.CLASSPATH.name(),
      Environment.PWD.$() + File.separator + "*",
      File.pathSeparator);

  // Add YARN/COMMON/HDFS jars and conf locations to path
  for (String c : conf.getStrings(
      YarnConfiguration.YARN_APPLICATION_CLASSPATH,
      YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
    TezYARNUtils.addToEnvironment(environment, Environment.CLASSPATH.name(),
        c.trim(), File.pathSeparator);
  }
  return StringInterner.weakIntern(environment.get(Environment.CLASSPATH.name()));
}
 
Example #5
Source File: TezYARNUtils.java    From incubator-tez with Apache License 2.0 6 votes vote down vote up
public static void setupDefaultEnv(Map<String, String> env,
    Configuration conf,  String confEnvKey, String confEnvKeyDefault) {
  // Setup the CLASSPATH in environment
  // i.e. add { Hadoop jars, job jar, CWD } to classpath.
  String classpath = getFrameworkClasspath(conf);
  TezYARNUtils.addToEnvironment(env,
      ApplicationConstants.Environment.CLASSPATH.name(),
      classpath, File.pathSeparator);

  // set any env from config if it is not set already
  TezYARNUtils.setEnvIfAbsentFromInputString(env, conf.get(
      confEnvKey, confEnvKeyDefault));
  
  // Append pwd to LD_LIBRARY_PATH
  // Done separately here because this is known to work platform independent
  TezYARNUtils.addToEnvironment(env, Environment.LD_LIBRARY_PATH.name(),
      Environment.PWD.$(), File.pathSeparator);
}
 
Example #6
Source File: MRApps.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Creates a {@link ApplicationClassLoader} if
 * {@link MRJobConfig#MAPREDUCE_JOB_CLASSLOADER} is set to true, and
 * the APP_CLASSPATH environment variable is set.
 * @param conf
 * @return the created job classloader, or null if the job classloader is not
 * enabled or the APP_CLASSPATH environment variable is not set
 * @throws IOException
 */
public static ClassLoader createJobClassLoader(Configuration conf)
    throws IOException {
  ClassLoader jobClassLoader = null;
  if (conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_CLASSLOADER, false)) {
    String appClasspath = System.getenv(Environment.APP_CLASSPATH.key());
    if (appClasspath == null) {
      LOG.warn("Not creating job classloader since APP_CLASSPATH is not set.");
    } else {
      LOG.info("Creating job classloader");
      if (LOG.isDebugEnabled()) {
        LOG.debug("APP_CLASSPATH=" + appClasspath);
      }
      String[] systemClasses = getSystemClasses(conf);
      jobClassLoader = createJobClassLoader(appClasspath,
          systemClasses);
    }
  }
  return jobClassLoader;
}
 
Example #7
Source File: Client.java    From Scribengin with GNU Affero General Public License v3.0 6 votes vote down vote up
private void setupAppMasterEnv(Map<String, String> appMasterEnv) {
  StringBuilder classPathEnv = new StringBuilder();
  classPathEnv.append(Environment.CLASSPATH.$()).append(File.pathSeparatorChar);
  classPathEnv.append("./*");

  for (String c : conf.getStrings(
      YarnConfiguration.YARN_APPLICATION_CLASSPATH,
      YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
    classPathEnv.append(File.pathSeparatorChar);
    classPathEnv.append(c.trim());
  }

  String envStr = classPathEnv.toString();
  LOG.info("env: " + envStr);
  appMasterEnv.put(Environment.CLASSPATH.name(), envStr);
}
 
Example #8
Source File: YarnManager.java    From Scribengin with GNU Affero General Public License v3.0 6 votes vote down vote up
public void startContainer(Container container, VMConfig appVMConfig) throws YarnException, IOException {
  String command = appVMConfig.buildCommand();
  ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class);
  if(vmConfig.getVmResources().size() > 0) {
    appVMConfig.getVmResources().putAll(vmConfig.getVmResources());
    VMResources vmResources = new VMResources(conf, appVMConfig);
    ctx.setLocalResources(vmResources);
  }
  Map<String, String> appEnv = new HashMap<String, String>();
  boolean miniClusterEnv = vmConfig.getEnvironment() == VMConfig.Environment.YARN_MINICLUSTER;
  setupAppClasspath(miniClusterEnv , conf, appEnv);
  ctx.setEnvironment(appEnv);
  
  StringBuilder sb = new StringBuilder();
  List<String> commands = Collections.singletonList(
      sb.append(command).
      append(" 1> ").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR).append("/stdout").
      append(" 2> ").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR).append("/stderr").toString()
  );
  ctx.setCommands(commands);
  nmClient.startContainer(container, ctx);
  //TODO: update vm descriptor status
}
 
Example #9
Source File: YarnManager.java    From Scribengin with GNU Affero General Public License v3.0 6 votes vote down vote up
void setupAppClasspath(boolean miniClusterEnv, Configuration conf, Map<String, String> appMasterEnv) {
  if(miniClusterEnv) {
    String cps = System.getProperty("java.class.path") ;
    String[] cp = cps.split(":") ;
    for(String selCp : cp) {
      Apps.addToEnvironment(appMasterEnv, Environment.CLASSPATH.name(), selCp, ":");
    }
  } else {
    StringBuilder classPathEnv = new StringBuilder();
    classPathEnv.append(Environment.CLASSPATH.$()).append(File.pathSeparatorChar);
    classPathEnv.append("./*");

    String[] classpath = conf.getStrings(
        YarnConfiguration.YARN_APPLICATION_CLASSPATH,
        YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH
    ) ;
    for (String selClasspath : classpath) {
      classPathEnv.append(File.pathSeparatorChar);
      classPathEnv.append(selClasspath.trim());
    }
    appMasterEnv.put(Environment.CLASSPATH.name(), classPathEnv.toString());
    System.err.println("CLASSPATH: " + classPathEnv);
  }
  Apps.addToEnvironment(appMasterEnv, Environment.CLASSPATH.name(), Environment.PWD.$() + File.separator + "*", ":");
}
 
Example #10
Source File: TestMRHelpers.java    From tez with Apache License 2.0 6 votes vote down vote up
private void testCommonEnvSettingsForMRTasks(Map<String, String> env) {
    Assert.assertTrue(env.containsKey("foo"));
    Assert.assertTrue(env.containsKey("bar"));
    Assert.assertTrue(env.containsKey(Environment.LD_LIBRARY_PATH.name()));
    Assert.assertTrue(env.containsKey(Environment.SHELL.name()));
    Assert.assertTrue(env.containsKey("HADOOP_ROOT_LOGGER"));

    /* On non-windows platform ensure that LD_LIBRARY_PATH is being set and PWD is present.
     * on windows platform LD_LIBRARY_PATH is not applicable. check the PATH is being appended
     * by the user setting (ex user may set HADOOP_HOME\\bin.
     */
    if (!Shell.WINDOWS) {
      Assert.assertEquals("$PWD:$TEZ_ADMIN_ENV_TEST/lib/native",
          env.get(Environment.LD_LIBRARY_PATH.name()));
    } else {
      Assert.assertTrue(env.get(Environment.PATH.name()).contains(";%TEZ_ADMIN_ENV%\\bin"));
    }

//    TEZ-273 will reinstate this or similar. 
//    for (String val : YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH) {
//      Assert.assertTrue(env.get(Environment.CLASSPATH.name()).contains(val));
//    }
//    Assert.assertTrue(0 ==
//        env.get(Environment.CLASSPATH.name()).indexOf(Environment.PWD.$()));
  }
 
Example #11
Source File: LocalContainerLauncher.java    From tez with Apache License 2.0 6 votes vote down vote up
private TezChild createTezChild(Configuration defaultConf, ContainerId containerId,
                                String tokenIdentifier, int attemptNumber, String[] localDirs,
                                TezTaskUmbilicalProtocol tezTaskUmbilicalProtocol,
                                Credentials credentials) throws
    InterruptedException, TezException, IOException {
  Map<String, String> containerEnv = new HashMap<String, String>();
  containerEnv.putAll(localEnv);
  // Use the user from env if it's available.
  String user = isLocalMode ? System.getenv(Environment.USER.name()) : context.getUser();
  containerEnv.put(Environment.USER.name(), user);

  long memAvailable;
  synchronized (this) { // needed to fix findbugs Inconsistent synchronization warning
    memAvailable = Runtime.getRuntime().maxMemory() / numExecutors;
  }
  TezChild tezChild =
      TezChild.newTezChild(defaultConf, null, 0, containerId.toString(), tokenIdentifier,
          attemptNumber, localDirs, workingDirectory, containerEnv, "", executionContext, credentials,
          memAvailable, context.getUser(), tezTaskUmbilicalProtocol, false,
          context.getHadoopShim());
  return tezChild;
}
 
Example #12
Source File: MRApps.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Creates a {@link ApplicationClassLoader} if
 * {@link MRJobConfig#MAPREDUCE_JOB_CLASSLOADER} is set to true, and
 * the APP_CLASSPATH environment variable is set.
 * @param conf
 * @return the created job classloader, or null if the job classloader is not
 * enabled or the APP_CLASSPATH environment variable is not set
 * @throws IOException
 */
public static ClassLoader createJobClassLoader(Configuration conf)
    throws IOException {
  ClassLoader jobClassLoader = null;
  if (conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_CLASSLOADER, false)) {
    String appClasspath = System.getenv(Environment.APP_CLASSPATH.key());
    if (appClasspath == null) {
      LOG.warn("Not creating job classloader since APP_CLASSPATH is not set.");
    } else {
      LOG.info("Creating job classloader");
      if (LOG.isDebugEnabled()) {
        LOG.debug("APP_CLASSPATH=" + appClasspath);
      }
      String[] systemClasses = getSystemClasses(conf);
      jobClassLoader = createJobClassLoader(appClasspath,
          systemClasses);
    }
  }
  return jobClassLoader;
}
 
Example #13
Source File: TezClientUtils.java    From tez with Apache License 2.0 6 votes vote down vote up
@Private
@VisibleForTesting
static String constructAMLaunchOpts(TezConfiguration tezConf, Resource capability) {
  String defaultOpts = tezConf.get(TezConfiguration.TEZ_AM_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS,
      TezConfiguration.TEZ_AM_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS_DEFAULT);
  Path tmpDir = new Path(Environment.PWD.$(),
      YarnConfiguration.DEFAULT_CONTAINER_TEMP_DIR);
  String amOpts = "-Djava.io.tmpdir=" + tmpDir + " ";

  if (defaultOpts != null && !defaultOpts.isEmpty()) {
    amOpts = amOpts + defaultOpts + " ";
  }
  amOpts = amOpts + tezConf.get(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS,
      TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS_DEFAULT);

  amOpts = maybeAddDefaultMemoryJavaOpts(amOpts, capability,
      tezConf.getDouble(TezConfiguration.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION,
          TezConfiguration.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION_DEFAULT));

  return amOpts;
}
 
Example #14
Source File: TezYARNUtils.java    From tez with Apache License 2.0 6 votes vote down vote up
public static void setupDefaultEnv(Map<String, String> env, Configuration conf,  String userEnvKey, String userEnvDefault,
    String clusterDefaultEnvKey, String clusterDefaultEnvDefault, boolean usingArchive) {
  // Setup the CLASSPATH in environment
  // i.e. add { Hadoop jars, job jar, CWD } to classpath.
  String classpath = getFrameworkClasspath(conf, usingArchive);
  TezYARNUtils.addToEnvironment(env,
      ApplicationConstants.Environment.CLASSPATH.name(),
      classpath, File.pathSeparator);

  // Pre-pend pwd to LD_LIBRARY_PATH
  // Done separately here because this is known to work platform
  // independent
  TezYARNUtils.addToEnvironment(env,
      Environment.LD_LIBRARY_PATH.name(), Environment.PWD.$(), File.pathSeparator);
  TezYARNUtils.appendToEnvFromInputString(env,
      conf.get(userEnvKey, userEnvDefault), File.pathSeparator);
  // set any env from config if it is not set already
  TezYARNUtils.appendToEnvFromInputString(env,
      conf.get(clusterDefaultEnvKey, clusterDefaultEnvDefault), File.pathSeparator);
}
 
Example #15
Source File: LoggerUtil.java    From Bats with Apache License 2.0 6 votes vote down vote up
/**
 * Makes MDC properties
 */
public static void setupMDC(String service)
{
  MDC.put("apex.service", service);

  String value = StramClientUtils.getHostName();
  MDC.put("apex.node", value == null ? "unknown" : value);

  value = System.getenv(Environment.USER.key());
  if (value != null) {
    MDC.put("apex.user", value);
  }

  value = System.getenv(Environment.CONTAINER_ID.name());
  if (value != null) {
    ContainerId containerId = ConverterUtils.toContainerId(value);
    ApplicationId applicationId = containerId.getApplicationAttemptId().getApplicationId();
    MDC.put("apex.containerId", containerId.toString());
    MDC.put("apex.applicationId", applicationId.toString());
  }

  value = System.getProperty(APPLICATION_NAME.getLongName());
  if (value != null) {
    MDC.put("apex.application", value);
  }
}
 
Example #16
Source File: TestTezYARNUtils.java    From tez with Apache License 2.0 6 votes vote down vote up
@Test(timeout = 5000)
public void testNoHadoopConfInClasspath() {
  Configuration conf = new Configuration(false);
  conf.setBoolean(TezConfiguration.TEZ_CLASSPATH_ADD_HADOOP_CONF, true);
  String classpath = TezYARNUtils.getFrameworkClasspath(conf, true);
  Assert.assertTrue(classpath.contains(Environment.PWD.$()));
  Assert.assertTrue(classpath.contains(Environment.PWD.$() + File.separator + "*"));
  Assert.assertTrue(classpath.contains(TezConstants.TEZ_TAR_LR_NAME + File.separator + "*"));
  Assert.assertTrue(classpath.contains(TezConstants.TEZ_TAR_LR_NAME + File.separator
      + "lib" + File.separator + "*"));
  Assert.assertTrue(classpath.contains(Environment.HADOOP_CONF_DIR.$()));
  Assert.assertTrue(classpath.indexOf(Environment.PWD.$()) <
      classpath.indexOf(TezConstants.TEZ_TAR_LR_NAME));
  Assert.assertTrue(classpath.indexOf(TezConstants.TEZ_TAR_LR_NAME) <
      classpath.indexOf(Environment.HADOOP_CONF_DIR.$()));
}
 
Example #17
Source File: TestTezYARNUtils.java    From tez with Apache License 2.0 6 votes vote down vote up
@Test(timeout = 5000)
public void testSetupDefaultEnvironment() {
  Configuration conf = new Configuration(false);
  conf.set(TezConfiguration.TEZ_AM_LAUNCH_ENV, "LD_LIBRARY_PATH=USER_PATH,USER_KEY=USER_VALUE");
  conf.set(TezConfiguration.TEZ_AM_LAUNCH_CLUSTER_DEFAULT_ENV, "LD_LIBRARY_PATH=DEFAULT_PATH,DEFAULT_KEY=DEFAULT_VALUE");

  Map<String, String> environment = new TreeMap<String, String>();
  TezYARNUtils.setupDefaultEnv(environment, conf,
      TezConfiguration.TEZ_AM_LAUNCH_ENV,
      TezConfiguration.TEZ_AM_LAUNCH_ENV_DEFAULT,
      TezConfiguration.TEZ_AM_LAUNCH_CLUSTER_DEFAULT_ENV,
      TezConfiguration.TEZ_AM_LAUNCH_CLUSTER_DEFAULT_ENV_DEFAULT, false);

  String value1 = environment.get("USER_KEY");
  Assert.assertEquals("User env should merge with default env", "USER_VALUE", value1);
  String value2 = environment.get("DEFAULT_KEY");
  Assert.assertEquals("User env should merge with default env", "DEFAULT_VALUE", value2);
  String value3 = environment.get("LD_LIBRARY_PATH");
  Assert.assertEquals("User env should append default env",
      Environment.PWD.$() + File.pathSeparator + "USER_PATH" + File.pathSeparator + "DEFAULT_PATH", value3);
  }
 
Example #18
Source File: TezYARNUtils.java    From tez with Apache License 2.0 5 votes vote down vote up
private static void addUserSpecifiedClasspath(StringBuilder classpathBuilder,
    Configuration conf) {
  // Add any additional user-specified classpath
  String additionalClasspath = conf.get(TezConfiguration.TEZ_CLUSTER_ADDITIONAL_CLASSPATH_PREFIX);
  if (additionalClasspath != null && !additionalClasspath.trim().isEmpty()) {
    classpathBuilder.append(additionalClasspath)
        .append(File.pathSeparator);
  }

  // Add PWD:PWD/*
  classpathBuilder.append(Environment.PWD.$())
      .append(File.pathSeparator)
      .append(Environment.PWD.$() + File.separator + "*")
      .append(File.pathSeparator);
}
 
Example #19
Source File: TestTezYARNUtils.java    From tez with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 20000)
public void testUserClasspathFirstFalse() {
  Configuration conf = new Configuration(false);
  conf.setBoolean(TezConfiguration.TEZ_USER_CLASSPATH_FIRST, false);
  conf.set(TezConfiguration.TEZ_CLUSTER_ADDITIONAL_CLASSPATH_PREFIX, "foobar");
  String classpath = TezYARNUtils.getFrameworkClasspath(conf, true);
  Assert.assertTrue(classpath.contains("foobar"));
  Assert.assertTrue(classpath.indexOf("foobar") >
      classpath.indexOf(TezConstants.TEZ_TAR_LR_NAME));
  Assert.assertTrue(classpath.indexOf("foobar") >
      classpath.indexOf(Environment.PWD.$()));
}
 
Example #20
Source File: TestTezYARNUtils.java    From tez with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 5000)
public void testAuxClasspath() {
  Configuration conf = new Configuration(false);
  conf.set(TezConfiguration.TEZ_CLUSTER_ADDITIONAL_CLASSPATH_PREFIX, "foobar");
  String classpath = TezYARNUtils.getFrameworkClasspath(conf, true);
  Assert.assertTrue(classpath.contains("foobar"));
  Assert.assertTrue(classpath.indexOf("foobar") <
      classpath.indexOf(TezConstants.TEZ_TAR_LR_NAME));
  Assert.assertTrue(classpath.indexOf("foobar") <
      classpath.indexOf(Environment.PWD.$()));
}
 
Example #21
Source File: TestTezYARNUtils.java    From tez with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 5000)
public void testBasicArchiveClasspath() {
  Configuration conf = new Configuration(false);
  String classpath = TezYARNUtils.getFrameworkClasspath(conf, true);
  Assert.assertTrue(classpath.contains(Environment.PWD.$()));
  Assert.assertTrue(classpath.contains(Environment.PWD.$() + File.separator + "*"));
  Assert.assertTrue(classpath.contains(TezConstants.TEZ_TAR_LR_NAME + File.separator + "*"));
  Assert.assertTrue(classpath.contains(TezConstants.TEZ_TAR_LR_NAME + File.separator
      + "lib" + File.separator + "*"));
  Assert.assertTrue(!classpath.contains(Environment.HADOOP_CONF_DIR.$()));
  Assert.assertTrue(classpath.indexOf(Environment.PWD.$()) <
      classpath.indexOf(TezConstants.TEZ_TAR_LR_NAME));
}
 
Example #22
Source File: Client.java    From Scribengin with GNU Affero General Public License v3.0 5 votes vote down vote up
private String getAppMasterCommand() {
  StringBuilder sb = new StringBuilder();
  sb.append(Environment.JAVA_HOME.$()).append("/bin/java").append(" ");
  sb.append("-Xmx").append(this.applicationMasterMem).append("M").append(" ");
  sb.append(this.applicationMasterClassName).append(" ");
  sb.append("--").append(Constants.OPT_CONTAINER_MEM).append(" ").append(this.containerMem)
      .append(" ");
  sb.append("--").append(Constants.OPT_KAFKA_SEED_BROKERS).append(" ")
      .append(StringUtils.join(this.kafkaSeedBrokers, " ")).append(" ");
  sb.append("--").append(Constants.OPT_KAFKA_TOPIC).append(" ")
      .append(StringUtils.join(this.topicList, " ")).append(" ");
  sb.append(" --").append(Constants.OPT_YARN_SITE_XML).append(" ").append(this.yarnSiteXml);
  sb.append(" --").append(Constants.OPT_PRE_COMMIT_PATH_PREFIX).append(" ").append(this.preCommitPrefix);
  sb.append(" --").append(Constants.OPT_COMMIT_PATH_PREFIX).append(" ").append(this.commitPrefix);
  sb.append(" --").append(Constants.OPT_CHECK_POINT_INTERVAL).append(" ").append(Long.toString(this.commitCheckPointInterval));
  
  if(this.hdfsPath != null){
    sb.append(" --").append(Constants.OPT_HDFS_PATH).append(" ").append(this.hdfsPath);
  }
  if(this.cleanstart){
    sb.append(" --").append(Constants.OPT_CLEAN_START);
  }
  
  
  sb.append(" 1> ").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR).append("/stdout")
      .append(" ");
  sb.append(" 2> ").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR).append("/stderr");
  String r = sb.toString();
  LOG.info("ApplicationConstants.getCommand() : " + r);
  return r;
}
 
Example #23
Source File: MRHelpers.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
/**
 * Setup classpath and other environment variables
 * @param conf Configuration to retrieve settings from
 * @param environment Environment to update
 * @param isMap Whether task is a map or reduce task
 */
public static void updateEnvironmentForMRTasks(Configuration conf,
    Map<String, String> environment, boolean isMap) {
  // Shell
  environment.put(Environment.SHELL.name(), conf.get(
      MRJobConfig.MAPRED_ADMIN_USER_SHELL, MRJobConfig.DEFAULT_SHELL));

  // Add pwd to LD_LIBRARY_PATH, add this before adding anything else
  TezYARNUtils.addToEnvironment(environment, Environment.LD_LIBRARY_PATH.name(),
      Environment.PWD.$(), File.pathSeparator);

  // Add the env variables passed by the admin
  TezYARNUtils.appendToEnvFromInputString(environment, conf.get(
      MRJobConfig.MAPRED_ADMIN_USER_ENV,
      MRJobConfig.DEFAULT_MAPRED_ADMIN_USER_ENV),
      File.pathSeparator);

  // Add the env variables passed by the user
  String mapredChildEnv = (isMap ?
      conf.get(MRJobConfig.MAP_ENV, "")
      : conf.get(MRJobConfig.REDUCE_ENV, ""));
  TezYARNUtils.appendToEnvFromInputString(environment, mapredChildEnv, File.pathSeparator);

  // Set logging level in the environment.
  environment.put(
      "HADOOP_ROOT_LOGGER",
      getChildLogLevel(conf, isMap) + ",CLA");
}
 
Example #24
Source File: Client.java    From hadoop-mini-clusters with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("deprecation")
private void setupAppMasterEnv(Map<String, String> appMasterEnv) {
    for (String c : conf.getStrings(
            YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
        Apps.addToEnvironment(appMasterEnv, Environment.CLASSPATH.name(),
                c.trim());
    }
    Apps.addToEnvironment(appMasterEnv,
            Environment.CLASSPATH.name(),
            Environment.PWD.$() + File.separator + "*");
}
 
Example #25
Source File: TezUtils.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
public static String getContainerLogDir() {
  String logDirsStr  = System.getenv(Environment.LOG_DIRS.name());
  if (logDirsStr == null || logDirsStr.isEmpty()) {
    return null;
  }
  String[] logDirs = StringUtils.split(logDirsStr, ',');
  if (logDirs.length == 0) {
    return null;
  }
  int logIndex = RANDOM.nextInt(logDirs.length);
  return logDirs[logIndex];
}
 
Example #26
Source File: SolrClient.java    From yarn-proto with Apache License 2.0 5 votes vote down vote up
protected void setupAppMasterEnv(CommandLine cli, Map<String, String> appMasterEnv) throws IOException {
  for (String c : conf.getStrings(
          YarnConfiguration.YARN_APPLICATION_CLASSPATH,
          YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
    Apps.addToEnvironment(appMasterEnv, Environment.CLASSPATH.name(), c.trim());
  }
  Apps.addToEnvironment(appMasterEnv,
          Environment.CLASSPATH.name(),
          Environment.PWD.$() + File.separator + "*");

  if (cli.hasOption("extclasspath")) {
    String extclasspathArg = cli.getOptionValue("extclasspath");
    File extclasspathFile = new File(extclasspathArg);
    StringBuilder sb = new StringBuilder();
    BufferedReader reader = null;
    String line = null;
    try {
      reader = new BufferedReader(
        new InputStreamReader(new FileInputStream(extclasspathFile), Charset.forName("UTF-8")));
      while ((line = reader.readLine()) != null)
        sb.append(line.trim()).append(":");
    } finally {
      if (reader != null) {
        reader.close();
      }
    }
    for (String part : sb.toString().split(":")) {
      if (part.length() > 0)
        Apps.addToEnvironment(appMasterEnv, Environment.CLASSPATH.name(), part);
    }
  }
}
 
Example #27
Source File: YarnTaskExecutorRunner.java    From flink with Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
static void setupConfigurationAndInstallSecurityContext(Configuration configuration, String currDir, Map<String, String> variables) throws Exception {
	final String localDirs = variables.get(Environment.LOCAL_DIRS.key());
	LOG.info("Current working/local Directory: {}", localDirs);

	BootstrapTools.updateTmpDirectoriesInConfiguration(configuration, localDirs);

	setupConfigurationFromVariables(configuration, currDir, variables);

	SecurityUtils.install(new SecurityConfiguration(configuration));
}
 
Example #28
Source File: MyClient.java    From yarn-beginners-examples with Apache License 2.0 5 votes vote down vote up
private Map<String, String> getAMEnvironment(Map<String, LocalResource> localResources
    , FileSystem fs) throws IOException{
  Map<String, String> env = new HashMap<String, String>();

  // Set ApplicationMaster jar file
  LocalResource appJarResource = localResources.get(Constants.AM_JAR_NAME);
  Path hdfsAppJarPath = new Path(fs.getHomeDirectory(), appJarResource.getResource().getFile());
  FileStatus hdfsAppJarStatus = fs.getFileStatus(hdfsAppJarPath);
  long hdfsAppJarLength = hdfsAppJarStatus.getLen();
  long hdfsAppJarTimestamp = hdfsAppJarStatus.getModificationTime();

  env.put(Constants.AM_JAR_PATH, hdfsAppJarPath.toString());
  env.put(Constants.AM_JAR_TIMESTAMP, Long.toString(hdfsAppJarTimestamp));
  env.put(Constants.AM_JAR_LENGTH, Long.toString(hdfsAppJarLength));

  // Add AppMaster.jar location to classpath
  // At some point we should not be required to add
  // the hadoop specific classpaths to the env.
  // It should be provided out of the box.
  // For now setting all required classpaths including
  // the classpath to "." for the application jar
  StringBuilder classPathEnv = new StringBuilder(Environment.CLASSPATH.$$())
      .append(ApplicationConstants.CLASS_PATH_SEPARATOR).append("./*");
  for (String c : conf.getStrings(
      YarnConfiguration.YARN_APPLICATION_CLASSPATH,
      YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH)) {
    classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR);
    classPathEnv.append(c.trim());
  }
  env.put("CLASSPATH", classPathEnv.toString());

  return env;
}
 
Example #29
Source File: Utils.java    From flink with Apache License 2.0 5 votes vote down vote up
public static void setupYarnClassPath(Configuration conf, Map<String, String> appMasterEnv) {
	addToEnvironment(
		appMasterEnv,
		Environment.CLASSPATH.name(),
		appMasterEnv.get(ENV_FLINK_CLASSPATH));
	String[] applicationClassPathEntries = conf.getStrings(
		YarnConfiguration.YARN_APPLICATION_CLASSPATH,
		YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH);
	for (String c : applicationClassPathEntries) {
		addToEnvironment(appMasterEnv, Environment.CLASSPATH.name(), c.trim());
	}
}
 
Example #30
Source File: YarnTaskRunnerLauncherImpl.java    From incubator-tajo with Apache License 2.0 5 votes vote down vote up
/**
 * Lock this on initialClasspath so that there is only one fork in the AM for
 * getting the initial class-path. TODO: We already construct
 * a parent CLC and use it for all the containers, so this should go away
 * once the mr-generated-classpath stuff is gone.
 */
private static String getInitialClasspath(Configuration conf) {
  synchronized (classpathLock) {
    if (initialClasspathFlag.get()) {
      return initialClasspath;
    }
    Map<String, String> env = new HashMap<String, String>();

    initialClasspath = env.get(Environment.CLASSPATH.name());
    initialClasspathFlag.set(true);
    return initialClasspath;
  }
}