org.apache.hadoop.yarn.util.Apps Java Examples

The following examples show how to use org.apache.hadoop.yarn.util.Apps. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestContainerLaunch.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test(timeout = 10000)
public void testEnvExpansion() throws IOException {
  Path logPath = new Path("/nm/container/logs");
  String input =
      Apps.crossPlatformify("HADOOP_HOME") + "/share/hadoop/common/*"
          + ApplicationConstants.CLASS_PATH_SEPARATOR
          + Apps.crossPlatformify("HADOOP_HOME") + "/share/hadoop/common/lib/*"
          + ApplicationConstants.CLASS_PATH_SEPARATOR
          + Apps.crossPlatformify("HADOOP_LOG_HOME")
          + ApplicationConstants.LOG_DIR_EXPANSION_VAR;

  String res = ContainerLaunch.expandEnvironment(input, logPath);

  if (Shell.WINDOWS) {
    Assert.assertEquals("%HADOOP_HOME%/share/hadoop/common/*;"
        + "%HADOOP_HOME%/share/hadoop/common/lib/*;"
        + "%HADOOP_LOG_HOME%/nm/container/logs", res);
  } else {
    Assert.assertEquals("$HADOOP_HOME/share/hadoop/common/*:"
        + "$HADOOP_HOME/share/hadoop/common/lib/*:"
        + "$HADOOP_LOG_HOME/nm/container/logs", res);
  }
  System.out.println(res);
}
 
Example #2
Source File: TestContainerLaunch.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test(timeout = 10000)
public void testEnvExpansion() throws IOException {
  Path logPath = new Path("/nm/container/logs");
  String input =
      Apps.crossPlatformify("HADOOP_HOME") + "/share/hadoop/common/*"
          + ApplicationConstants.CLASS_PATH_SEPARATOR
          + Apps.crossPlatformify("HADOOP_HOME") + "/share/hadoop/common/lib/*"
          + ApplicationConstants.CLASS_PATH_SEPARATOR
          + Apps.crossPlatformify("HADOOP_LOG_HOME")
          + ApplicationConstants.LOG_DIR_EXPANSION_VAR;

  String res = ContainerLaunch.expandEnvironment(input, logPath);

  if (Shell.WINDOWS) {
    Assert.assertEquals("%HADOOP_HOME%/share/hadoop/common/*;"
        + "%HADOOP_HOME%/share/hadoop/common/lib/*;"
        + "%HADOOP_LOG_HOME%/nm/container/logs", res);
  } else {
    Assert.assertEquals("$HADOOP_HOME/share/hadoop/common/*:"
        + "$HADOOP_HOME/share/hadoop/common/lib/*:"
        + "$HADOOP_LOG_HOME/nm/container/logs", res);
  }
  System.out.println(res);
}
 
Example #3
Source File: YarnManager.java    From Scribengin with GNU Affero General Public License v3.0 6 votes vote down vote up
void setupAppClasspath(boolean miniClusterEnv, Configuration conf, Map<String, String> appMasterEnv) {
  if(miniClusterEnv) {
    String cps = System.getProperty("java.class.path") ;
    String[] cp = cps.split(":") ;
    for(String selCp : cp) {
      Apps.addToEnvironment(appMasterEnv, Environment.CLASSPATH.name(), selCp, ":");
    }
  } else {
    StringBuilder classPathEnv = new StringBuilder();
    classPathEnv.append(Environment.CLASSPATH.$()).append(File.pathSeparatorChar);
    classPathEnv.append("./*");

    String[] classpath = conf.getStrings(
        YarnConfiguration.YARN_APPLICATION_CLASSPATH,
        YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH
    ) ;
    for (String selClasspath : classpath) {
      classPathEnv.append(File.pathSeparatorChar);
      classPathEnv.append(selClasspath.trim());
    }
    appMasterEnv.put(Environment.CLASSPATH.name(), classPathEnv.toString());
    System.err.println("CLASSPATH: " + classPathEnv);
  }
  Apps.addToEnvironment(appMasterEnv, Environment.CLASSPATH.name(), Environment.PWD.$() + File.separator + "*", ":");
}
 
Example #4
Source File: YarnJobContainer.java    From sylph with Apache License 2.0 5 votes vote down vote up
public JobContainer build()
{
    final YarnJobContainer yarnJobContainer = new YarnJobContainer(yarnClient, submitter);
    if (lastRunId != null) {
        yarnJobContainer.yarnAppId = Apps.toAppID(lastRunId);
        yarnJobContainer.setStatus(RUNNING);
        try {
            yarnJobContainer.webUi = yarnClient.getApplicationReport(yarnJobContainer.yarnAppId).getTrackingUrl();
        }
        catch (Exception e) {
            logger.warn(e.getMessage());
        }
    }

    //----create JobContainer Proxy
    return AopFactory.proxy(JobContainer.class)
            .byInstance(yarnJobContainer)
            .around(proxyContext -> {
                /*
                 * 通过这个 修改当前YarnClient的ClassLoader的为当前runner的加载器
                 * 默认hadoop Configuration使用jvm的AppLoader,会出现 akka.version not setting的错误 原因是找不到akka相关jar包
                 * 原因是hadoop Configuration 初始化: this.classLoader = Thread.currentThread().getContextClassLoader();
                 * */
                try (Closeables ignored = Closeables.openThreadContextClassLoader(jobClassLoader)) {
                    return proxyContext.proceed();
                }
            });
}
 
Example #5
Source File: MRApps.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public static void setEnvFromInputString(Map<String, String> env,
    String envString, Configuration conf) {
  String classPathSeparator =
      conf.getBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM,
        MRConfig.DEFAULT_MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM)
          ? ApplicationConstants.CLASS_PATH_SEPARATOR : File.pathSeparator;
  Apps.setEnvFromInputString(env, envString, classPathSeparator);
}
 
Example #6
Source File: MRApps.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Public
@Unstable
public static void addToEnvironment(Map<String, String> environment,
    String variable, String value, Configuration conf) {
  String classPathSeparator =
      conf.getBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM,
        MRConfig.DEFAULT_MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM)
          ? ApplicationConstants.CLASS_PATH_SEPARATOR : File.pathSeparator;
  Apps.addToEnvironment(environment, variable, value, classPathSeparator);
}
 
Example #7
Source File: MRApps.java    From big-c with Apache License 2.0 5 votes vote down vote up
public static void setEnvFromInputString(Map<String, String> env,
    String envString, Configuration conf) {
  String classPathSeparator =
      conf.getBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM,
        MRConfig.DEFAULT_MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM)
          ? ApplicationConstants.CLASS_PATH_SEPARATOR : File.pathSeparator;
  Apps.setEnvFromInputString(env, envString, classPathSeparator);
}
 
Example #8
Source File: MRApps.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Public
@Unstable
public static void addToEnvironment(Map<String, String> environment,
    String variable, String value, Configuration conf) {
  String classPathSeparator =
      conf.getBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM,
        MRConfig.DEFAULT_MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM)
          ? ApplicationConstants.CLASS_PATH_SEPARATOR : File.pathSeparator;
  Apps.addToEnvironment(environment, variable, value, classPathSeparator);
}
 
Example #9
Source File: IgniteYarnClient.java    From ignite with Apache License 2.0 5 votes vote down vote up
/**
 * @param envs Environment variables.
 * @param conf Yarn configuration.
 */
private static void setupAppMasterEnv(Map<String, String> envs, YarnConfiguration conf) {
    for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
        YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH))
        Apps.addToEnvironment(envs, Environment.CLASSPATH.name(),
                c.trim(), File.pathSeparator);

    Apps.addToEnvironment(envs,
            Environment.CLASSPATH.name(),
            Environment.PWD.$() + File.separator + "*",
            File.pathSeparator);
}
 
Example #10
Source File: Client.java    From hadoop-mini-clusters with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("deprecation")
private void setupAppMasterEnv(Map<String, String> appMasterEnv) {
    for (String c : conf.getStrings(
            YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
        Apps.addToEnvironment(appMasterEnv, Environment.CLASSPATH.name(),
                c.trim());
    }
    Apps.addToEnvironment(appMasterEnv,
            Environment.CLASSPATH.name(),
            Environment.PWD.$() + File.separator + "*");
}
 
Example #11
Source File: SolrClient.java    From yarn-proto with Apache License 2.0 5 votes vote down vote up
protected void setupAppMasterEnv(CommandLine cli, Map<String, String> appMasterEnv) throws IOException {
  for (String c : conf.getStrings(
          YarnConfiguration.YARN_APPLICATION_CLASSPATH,
          YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
    Apps.addToEnvironment(appMasterEnv, Environment.CLASSPATH.name(), c.trim());
  }
  Apps.addToEnvironment(appMasterEnv,
          Environment.CLASSPATH.name(),
          Environment.PWD.$() + File.separator + "*");

  if (cli.hasOption("extclasspath")) {
    String extclasspathArg = cli.getOptionValue("extclasspath");
    File extclasspathFile = new File(extclasspathArg);
    StringBuilder sb = new StringBuilder();
    BufferedReader reader = null;
    String line = null;
    try {
      reader = new BufferedReader(
        new InputStreamReader(new FileInputStream(extclasspathFile), Charset.forName("UTF-8")));
      while ((line = reader.readLine()) != null)
        sb.append(line.trim()).append(":");
    } finally {
      if (reader != null) {
        reader.close();
      }
    }
    for (String part : sb.toString().split(":")) {
      if (part.length() > 0)
        Apps.addToEnvironment(appMasterEnv, Environment.CLASSPATH.name(), part);
    }
  }
}
 
Example #12
Source File: Utils.java    From stratosphere with Apache License 2.0 4 votes vote down vote up
public static void setupEnv(Configuration conf, Map<String, String> appMasterEnv) {
	for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
		Apps.addToEnvironment(appMasterEnv, Environment.CLASSPATH.name(), c.trim());
	}
	Apps.addToEnvironment(appMasterEnv, Environment.CLASSPATH.name(), Environment.PWD.$() + File.separator + "*");
}