Java Code Examples for org.apache.hadoop.yarn.util.Apps

The following examples show how to use org.apache.hadoop.yarn.util.Apps. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: TestContainerLaunch.java    License: Apache License 2.0 6 votes vote down vote up
@Test(timeout = 10000)
public void testEnvExpansion() throws IOException {
  Path logPath = new Path("/nm/container/logs");
  String input =
      Apps.crossPlatformify("HADOOP_HOME") + "/share/hadoop/common/*"
          + ApplicationConstants.CLASS_PATH_SEPARATOR
          + Apps.crossPlatformify("HADOOP_HOME") + "/share/hadoop/common/lib/*"
          + ApplicationConstants.CLASS_PATH_SEPARATOR
          + Apps.crossPlatformify("HADOOP_LOG_HOME")
          + ApplicationConstants.LOG_DIR_EXPANSION_VAR;

  String res = ContainerLaunch.expandEnvironment(input, logPath);

  if (Shell.WINDOWS) {
    Assert.assertEquals("%HADOOP_HOME%/share/hadoop/common/*;"
        + "%HADOOP_HOME%/share/hadoop/common/lib/*;"
        + "%HADOOP_LOG_HOME%/nm/container/logs", res);
  } else {
    Assert.assertEquals("$HADOOP_HOME/share/hadoop/common/*:"
        + "$HADOOP_HOME/share/hadoop/common/lib/*:"
        + "$HADOOP_LOG_HOME/nm/container/logs", res);
  }
  System.out.println(res);
}
 
Example 2
Source Project: big-c   Source File: TestContainerLaunch.java    License: Apache License 2.0 6 votes vote down vote up
@Test(timeout = 10000)
public void testEnvExpansion() throws IOException {
  Path logPath = new Path("/nm/container/logs");
  String input =
      Apps.crossPlatformify("HADOOP_HOME") + "/share/hadoop/common/*"
          + ApplicationConstants.CLASS_PATH_SEPARATOR
          + Apps.crossPlatformify("HADOOP_HOME") + "/share/hadoop/common/lib/*"
          + ApplicationConstants.CLASS_PATH_SEPARATOR
          + Apps.crossPlatformify("HADOOP_LOG_HOME")
          + ApplicationConstants.LOG_DIR_EXPANSION_VAR;

  String res = ContainerLaunch.expandEnvironment(input, logPath);

  if (Shell.WINDOWS) {
    Assert.assertEquals("%HADOOP_HOME%/share/hadoop/common/*;"
        + "%HADOOP_HOME%/share/hadoop/common/lib/*;"
        + "%HADOOP_LOG_HOME%/nm/container/logs", res);
  } else {
    Assert.assertEquals("$HADOOP_HOME/share/hadoop/common/*:"
        + "$HADOOP_HOME/share/hadoop/common/lib/*:"
        + "$HADOOP_LOG_HOME/nm/container/logs", res);
  }
  System.out.println(res);
}
 
Example 3
void setupAppClasspath(boolean miniClusterEnv, Configuration conf, Map<String, String> appMasterEnv) {
  if(miniClusterEnv) {
    String cps = System.getProperty("java.class.path") ;
    String[] cp = cps.split(":") ;
    for(String selCp : cp) {
      Apps.addToEnvironment(appMasterEnv, Environment.CLASSPATH.name(), selCp, ":");
    }
  } else {
    StringBuilder classPathEnv = new StringBuilder();
    classPathEnv.append(Environment.CLASSPATH.$()).append(File.pathSeparatorChar);
    classPathEnv.append("./*");

    String[] classpath = conf.getStrings(
        YarnConfiguration.YARN_APPLICATION_CLASSPATH,
        YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH
    ) ;
    for (String selClasspath : classpath) {
      classPathEnv.append(File.pathSeparatorChar);
      classPathEnv.append(selClasspath.trim());
    }
    appMasterEnv.put(Environment.CLASSPATH.name(), classPathEnv.toString());
    System.err.println("CLASSPATH: " + classPathEnv);
  }
  Apps.addToEnvironment(appMasterEnv, Environment.CLASSPATH.name(), Environment.PWD.$() + File.separator + "*", ":");
}
 
Example 4
Source Project: sylph   Source File: YarnJobContainer.java    License: Apache License 2.0 5 votes vote down vote up
public JobContainer build()
{
    final YarnJobContainer yarnJobContainer = new YarnJobContainer(yarnClient, submitter);
    if (lastRunId != null) {
        yarnJobContainer.yarnAppId = Apps.toAppID(lastRunId);
        yarnJobContainer.setStatus(RUNNING);
        try {
            yarnJobContainer.webUi = yarnClient.getApplicationReport(yarnJobContainer.yarnAppId).getTrackingUrl();
        }
        catch (Exception e) {
            logger.warn(e.getMessage());
        }
    }

    //----create JobContainer Proxy
    return AopFactory.proxy(JobContainer.class)
            .byInstance(yarnJobContainer)
            .around(proxyContext -> {
                /*
                 * 通过这个 修改当前YarnClient的ClassLoader的为当前runner的加载器
                 * 默认hadoop Configuration使用jvm的AppLoader,会出现 akka.version not setting的错误 原因是找不到akka相关jar包
                 * 原因是hadoop Configuration 初始化: this.classLoader = Thread.currentThread().getContextClassLoader();
                 * */
                try (Closeables ignored = Closeables.openThreadContextClassLoader(jobClassLoader)) {
                    return proxyContext.proceed();
                }
            });
}
 
Example 5
Source Project: hadoop   Source File: MRApps.java    License: Apache License 2.0 5 votes vote down vote up
public static void setEnvFromInputString(Map<String, String> env,
    String envString, Configuration conf) {
  String classPathSeparator =
      conf.getBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM,
        MRConfig.DEFAULT_MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM)
          ? ApplicationConstants.CLASS_PATH_SEPARATOR : File.pathSeparator;
  Apps.setEnvFromInputString(env, envString, classPathSeparator);
}
 
Example 6
Source Project: hadoop   Source File: MRApps.java    License: Apache License 2.0 5 votes vote down vote up
@Public
@Unstable
public static void addToEnvironment(Map<String, String> environment,
    String variable, String value, Configuration conf) {
  String classPathSeparator =
      conf.getBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM,
        MRConfig.DEFAULT_MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM)
          ? ApplicationConstants.CLASS_PATH_SEPARATOR : File.pathSeparator;
  Apps.addToEnvironment(environment, variable, value, classPathSeparator);
}
 
Example 7
Source Project: big-c   Source File: MRApps.java    License: Apache License 2.0 5 votes vote down vote up
public static void setEnvFromInputString(Map<String, String> env,
    String envString, Configuration conf) {
  String classPathSeparator =
      conf.getBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM,
        MRConfig.DEFAULT_MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM)
          ? ApplicationConstants.CLASS_PATH_SEPARATOR : File.pathSeparator;
  Apps.setEnvFromInputString(env, envString, classPathSeparator);
}
 
Example 8
Source Project: big-c   Source File: MRApps.java    License: Apache License 2.0 5 votes vote down vote up
@Public
@Unstable
public static void addToEnvironment(Map<String, String> environment,
    String variable, String value, Configuration conf) {
  String classPathSeparator =
      conf.getBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM,
        MRConfig.DEFAULT_MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM)
          ? ApplicationConstants.CLASS_PATH_SEPARATOR : File.pathSeparator;
  Apps.addToEnvironment(environment, variable, value, classPathSeparator);
}
 
Example 9
Source Project: ignite   Source File: IgniteYarnClient.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * @param envs Environment variables.
 * @param conf Yarn configuration.
 */
private static void setupAppMasterEnv(Map<String, String> envs, YarnConfiguration conf) {
    for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
        YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH))
        Apps.addToEnvironment(envs, Environment.CLASSPATH.name(),
                c.trim(), File.pathSeparator);

    Apps.addToEnvironment(envs,
            Environment.CLASSPATH.name(),
            Environment.PWD.$() + File.separator + "*",
            File.pathSeparator);
}
 
Example 10
Source Project: hadoop-mini-clusters   Source File: Client.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("deprecation")
private void setupAppMasterEnv(Map<String, String> appMasterEnv) {
    for (String c : conf.getStrings(
            YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
        Apps.addToEnvironment(appMasterEnv, Environment.CLASSPATH.name(),
                c.trim());
    }
    Apps.addToEnvironment(appMasterEnv,
            Environment.CLASSPATH.name(),
            Environment.PWD.$() + File.separator + "*");
}
 
Example 11
Source Project: yarn-proto   Source File: SolrClient.java    License: Apache License 2.0 5 votes vote down vote up
protected void setupAppMasterEnv(CommandLine cli, Map<String, String> appMasterEnv) throws IOException {
  for (String c : conf.getStrings(
          YarnConfiguration.YARN_APPLICATION_CLASSPATH,
          YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
    Apps.addToEnvironment(appMasterEnv, Environment.CLASSPATH.name(), c.trim());
  }
  Apps.addToEnvironment(appMasterEnv,
          Environment.CLASSPATH.name(),
          Environment.PWD.$() + File.separator + "*");

  if (cli.hasOption("extclasspath")) {
    String extclasspathArg = cli.getOptionValue("extclasspath");
    File extclasspathFile = new File(extclasspathArg);
    StringBuilder sb = new StringBuilder();
    BufferedReader reader = null;
    String line = null;
    try {
      reader = new BufferedReader(
        new InputStreamReader(new FileInputStream(extclasspathFile), Charset.forName("UTF-8")));
      while ((line = reader.readLine()) != null)
        sb.append(line.trim()).append(":");
    } finally {
      if (reader != null) {
        reader.close();
      }
    }
    for (String part : sb.toString().split(":")) {
      if (part.length() > 0)
        Apps.addToEnvironment(appMasterEnv, Environment.CLASSPATH.name(), part);
    }
  }
}
 
Example 12
Source Project: stratosphere   Source File: Utils.java    License: Apache License 2.0 4 votes vote down vote up
public static void setupEnv(Configuration conf, Map<String, String> appMasterEnv) {
	for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
		Apps.addToEnvironment(appMasterEnv, Environment.CLASSPATH.name(), c.trim());
	}
	Apps.addToEnvironment(appMasterEnv, Environment.CLASSPATH.name(), Environment.PWD.$() + File.separator + "*");
}