org.apache.hadoop.yarn.api.ApplicationConstants Java Examples

The following examples show how to use org.apache.hadoop.yarn.api.ApplicationConstants. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestMRApps.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test (timeout = 120000)
public void testSetClasspathWithNoUserPrecendence() {
  Configuration conf = new Configuration();
  conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true);
  conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, false);
  Map<String, String> env = new HashMap<String, String>();
  try {
    MRApps.setClasspath(env, conf);
  } catch (Exception e) {
    fail("Got exception while setting classpath");
  }
  String env_str = env.get("CLASSPATH");
  String expectedClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR,
    Arrays.asList("job.jar/job.jar", "job.jar/classes/", "job.jar/lib/*",
      ApplicationConstants.Environment.PWD.$$() + "/*"));
  assertTrue("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, and job.jar is not in"
    + " the classpath!", env_str.contains(expectedClasspath));
  assertFalse("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, but taking effect!",
    env_str.startsWith(expectedClasspath));
}
 
Example #2
Source File: ContainerLaunch.java    From big-c with Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
public static String expandEnvironment(String var,
    Path containerLogDir) {
  var = var.replace(ApplicationConstants.LOG_DIR_EXPANSION_VAR,
    containerLogDir.toString());
  var =  var.replace(ApplicationConstants.CLASS_PATH_SEPARATOR,
    File.pathSeparator);

  // replace parameter expansion marker. e.g. {{VAR}} on Windows is replaced
  // as %VAR% and on Linux replaced as "$VAR"
  if (Shell.WINDOWS) {
    var = var.replaceAll("(\\{\\{)|(\\}\\})", "%");
  } else {
    var = var.replace(ApplicationConstants.PARAMETER_EXPANSION_LEFT, "$");
    var = var.replace(ApplicationConstants.PARAMETER_EXPANSION_RIGHT, "");
  }
  return var;
}
 
Example #3
Source File: YarnClusterResourceManager.java    From samza with Apache License 2.0 6 votes vote down vote up
/**
 * Runs a process as specified by the command builder on the container.
 * @param processorId id of the samza processor to run (passed as a command line parameter to the process)
 * @param container the yarn container to run the processor on.
 * @param cmdBuilder the command builder that encapsulates the command, and the context
 * @throws IOException on IO exceptions running the container
 */
public void runProcessor(String processorId, Container container, CommandBuilder cmdBuilder) throws IOException {
  String containerIdStr = ConverterUtils.toString(container.getId());
  String cmdPath = "./__package/";
  cmdBuilder.setCommandPath(cmdPath);
  String command = cmdBuilder.buildCommand();

  Map<String, String> env = getEscapedEnvironmentVariablesMap(cmdBuilder);
  env.put(ShellCommandConfig.ENV_EXECUTION_ENV_CONTAINER_ID, Util.envVarEscape(container.getId().toString()));

  Path packagePath = new Path(yarnConfig.getPackagePath());
  String formattedCommand =
      getFormattedCommand(ApplicationConstants.LOG_DIR_EXPANSION_VAR, command, ApplicationConstants.STDOUT,
          ApplicationConstants.STDERR);

  log.info("Running Processor ID: {} on Container ID: {} on host: {} using command: {} and env: {} and package path: {}",
      processorId, containerIdStr, container.getNodeHttpAddress(), formattedCommand, env, packagePath);
  state.pendingProcessors.put(processorId, new YarnContainer(container));

  startContainer(packagePath, container, env, formattedCommand);

  log.info("Made start request for Processor ID: {} on Container ID: {} on host: {} (http://{}/node/containerlogs/{}).",
      processorId, containerIdStr, container.getNodeId().getHost(), container.getNodeHttpAddress(), containerIdStr);
}
 
Example #4
Source File: TestMRApps.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test (timeout = 120000)
public void testSetClasspathWithUserPrecendence() {
   Configuration conf = new Configuration();
   conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true);
   conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, true);
   Map<String, String> env = new HashMap<String, String>();
   try {
     MRApps.setClasspath(env, conf);
   } catch (Exception e) {
     fail("Got exception while setting classpath");
   }
   String env_str = env.get("CLASSPATH");
   String expectedClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR,
     Arrays.asList(ApplicationConstants.Environment.PWD.$$(), "job.jar/job.jar",
       "job.jar/classes/", "job.jar/lib/*",
       ApplicationConstants.Environment.PWD.$$() + "/*"));
   assertTrue("MAPREDUCE_JOB_USER_CLASSPATH_FIRST set, but not taking effect!",
     env_str.startsWith(expectedClasspath));
 }
 
Example #5
Source File: LaunchContainerThread.java    From TensorFlowOnYARN with Apache License 2.0 6 votes vote down vote up
private String makeContainerCommand(long containerMemory, String clusterSpec,
    String jobName, int taskIndex) {
  String[] commands = new String[]{
      ApplicationConstants.Environment.JAVA_HOME.$$() + "/bin/java",
      "-Xmx" + containerMemory + "m",
      TFServerRunner.class.getName() + " ",
      Utils.mkOption(Constants.OPT_CLUSTER_SPEC, clusterSpec),
      Utils.mkOption(Constants.OPT_JOB_NAME, jobName),
      Utils.mkOption(Constants.OPT_TASK_INDEX, taskIndex),
      "1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/TFServerRunner." +
          ApplicationConstants.STDOUT,
      "2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/TFServerRunner." +
          ApplicationConstants.STDERR
  };

  return Utils.mkString(commands, " ");
}
 
Example #6
Source File: LaunchCluster.java    From TensorFlowOnYARN with Apache License 2.0 6 votes vote down vote up
private String makeAppMasterCommand(String tfLib, String tfJar) {
  String[] commands = new String[]{
      ApplicationConstants.Environment.JAVA_HOME.$$() + "/bin/java",
      // Set Xmx based on am memory size
      "-Xmx" + amMemory + "m",
      // Set class name
      ApplicationMaster.class.getName(),
      Utils.mkOption(Constants.OPT_TF_CONTAINER_MEMORY, containerMemory),
      Utils.mkOption(Constants.OPT_TF_CONTAINER_VCORES, containerVCores),
      Utils.mkOption(Constants.OPT_TF_WORKER_NUM, workerNum),
      Utils.mkOption(Constants.OPT_TF_PS_NUM, psNum),
      Utils.mkOption(Constants.OPT_TF_LIB, tfLib),
      Utils.mkOption(Constants.OPT_TF_JAR, tfJar),
      "1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stdout",
      "2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stderr"
  };
  return Utils.mkString(commands, " ");
}
 
Example #7
Source File: TezYARNUtils.java    From incubator-tez with Apache License 2.0 6 votes vote down vote up
public static void setupDefaultEnv(Map<String, String> env,
    Configuration conf,  String confEnvKey, String confEnvKeyDefault) {
  // Setup the CLASSPATH in environment
  // i.e. add { Hadoop jars, job jar, CWD } to classpath.
  String classpath = getFrameworkClasspath(conf);
  TezYARNUtils.addToEnvironment(env,
      ApplicationConstants.Environment.CLASSPATH.name(),
      classpath, File.pathSeparator);

  // set any env from config if it is not set already
  TezYARNUtils.setEnvIfAbsentFromInputString(env, conf.get(
      confEnvKey, confEnvKeyDefault));
  
  // Append pwd to LD_LIBRARY_PATH
  // Done separately here because this is known to work platform independent
  TezYARNUtils.addToEnvironment(env, Environment.LD_LIBRARY_PATH.name(),
      Environment.PWD.$(), File.pathSeparator);
}
 
Example #8
Source File: TestMRApps.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test (timeout = 120000)
public void testSetClasspathWithNoUserPrecendence() {
  Configuration conf = new Configuration();
  conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true);
  conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, false);
  Map<String, String> env = new HashMap<String, String>();
  try {
    MRApps.setClasspath(env, conf);
  } catch (Exception e) {
    fail("Got exception while setting classpath");
  }
  String env_str = env.get("CLASSPATH");
  String expectedClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR,
    Arrays.asList("job.jar/job.jar", "job.jar/classes/", "job.jar/lib/*",
      ApplicationConstants.Environment.PWD.$$() + "/*"));
  assertTrue("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, and job.jar is not in"
    + " the classpath!", env_str.contains(expectedClasspath));
  assertFalse("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, but taking effect!",
    env_str.startsWith(expectedClasspath));
}
 
Example #9
Source File: TezTestServiceContainerLauncher.java    From tez with Apache License 2.0 6 votes vote down vote up
private RunContainerRequestProto constructRunContainerRequest(ContainerLaunchRequest launchRequest) throws
    IOException {
  RunContainerRequestProto.Builder builder = RunContainerRequestProto.newBuilder();
  Preconditions.checkArgument(launchRequest.getTaskCommunicatorName().equals(
      TezConstants.getTezYarnServicePluginName()));
  InetSocketAddress address = (InetSocketAddress) getContext().getTaskCommunicatorMetaInfo(launchRequest.getTaskCommunicatorName());
  builder.setAmHost(address.getHostName()).setAmPort(address.getPort());
  builder.setAppAttemptNumber(appAttemptId.getAttemptId());
  builder.setApplicationIdString(appAttemptId.getApplicationId().toString());
  builder.setTokenIdentifier(tokenIdentifier);
  builder.setContainerIdString(launchRequest.getContainerId().toString());
  builder.setCredentialsBinary(
      ByteString.copyFrom(launchRequest.getContainerLaunchContext().getTokens()));
  // TODO Avoid reading this from the environment
  builder.setUser(System.getenv(ApplicationConstants.Environment.USER.name()));
  return builder.build();
}
 
Example #10
Source File: TestMRApps.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test (timeout = 120000)
public void testSetClasspathWithJobClassloader() throws IOException {
  Configuration conf = new Configuration();
  conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true);
  conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_CLASSLOADER, true);
  Map<String, String> env = new HashMap<String, String>();
  MRApps.setClasspath(env, conf);
  String cp = env.get("CLASSPATH");
  String appCp = env.get("APP_CLASSPATH");
  assertFalse("MAPREDUCE_JOB_CLASSLOADER true, but job.jar is in the"
    + " classpath!", cp.contains("jar" + ApplicationConstants.CLASS_PATH_SEPARATOR + "job"));
  assertFalse("MAPREDUCE_JOB_CLASSLOADER true, but PWD is in the classpath!",
    cp.contains("PWD"));
  String expectedAppClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR,
    Arrays.asList(ApplicationConstants.Environment.PWD.$$(), "job.jar/job.jar",
      "job.jar/classes/", "job.jar/lib/*",
      ApplicationConstants.Environment.PWD.$$() + "/*"));
  assertEquals("MAPREDUCE_JOB_CLASSLOADER true, but job.jar is not in the app"
    + " classpath!", expectedAppClasspath, appCp);
}
 
Example #11
Source File: TezYARNUtils.java    From tez with Apache License 2.0 6 votes vote down vote up
public static void setupDefaultEnv(Map<String, String> env, Configuration conf,  String userEnvKey, String userEnvDefault,
    String clusterDefaultEnvKey, String clusterDefaultEnvDefault, boolean usingArchive) {
  // Setup the CLASSPATH in environment
  // i.e. add { Hadoop jars, job jar, CWD } to classpath.
  String classpath = getFrameworkClasspath(conf, usingArchive);
  TezYARNUtils.addToEnvironment(env,
      ApplicationConstants.Environment.CLASSPATH.name(),
      classpath, File.pathSeparator);

  // Pre-pend pwd to LD_LIBRARY_PATH
  // Done separately here because this is known to work platform
  // independent
  TezYARNUtils.addToEnvironment(env,
      Environment.LD_LIBRARY_PATH.name(), Environment.PWD.$(), File.pathSeparator);
  TezYARNUtils.appendToEnvFromInputString(env,
      conf.get(userEnvKey, userEnvDefault), File.pathSeparator);
  // set any env from config if it is not set already
  TezYARNUtils.appendToEnvFromInputString(env,
      conf.get(clusterDefaultEnvKey, clusterDefaultEnvDefault), File.pathSeparator);
}
 
Example #12
Source File: ContainerLaunch.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
public static String expandEnvironment(String var,
    Path containerLogDir) {
  var = var.replace(ApplicationConstants.LOG_DIR_EXPANSION_VAR,
    containerLogDir.toString());
  var =  var.replace(ApplicationConstants.CLASS_PATH_SEPARATOR,
    File.pathSeparator);

  // replace parameter expansion marker. e.g. {{VAR}} on Windows is replaced
  // as %VAR% and on Linux replaced as "$VAR"
  if (Shell.WINDOWS) {
    var = var.replaceAll("(\\{\\{)|(\\}\\})", "%");
  } else {
    var = var.replace(ApplicationConstants.PARAMETER_EXPANSION_LEFT, "$");
    var = var.replace(ApplicationConstants.PARAMETER_EXPANSION_RIGHT, "");
  }
  return var;
}
 
Example #13
Source File: YarnSessionClusterEntrypoint.java    From flink with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) {
	// startup checks and logging
	EnvironmentInformation.logEnvironmentInfo(LOG, YarnSessionClusterEntrypoint.class.getSimpleName(), args);
	SignalHandler.register(LOG);
	JvmShutdownSafeguard.installAsShutdownHook(LOG);

	Map<String, String> env = System.getenv();

	final String workingDirectory = env.get(ApplicationConstants.Environment.PWD.key());
	Preconditions.checkArgument(
		workingDirectory != null,
		"Working directory variable (%s) not set",
		ApplicationConstants.Environment.PWD.key());

	try {
		YarnEntrypointUtils.logYarnEnvironmentInformation(env, LOG);
	} catch (IOException e) {
		LOG.warn("Could not log YARN environment information.", e);
	}

	Configuration configuration = YarnEntrypointUtils.loadConfiguration(workingDirectory, env);

	YarnSessionClusterEntrypoint yarnSessionClusterEntrypoint = new YarnSessionClusterEntrypoint(configuration);

	ClusterEntrypoint.runClusterEntrypoint(yarnSessionClusterEntrypoint);
}
 
Example #14
Source File: StramWebServices.java    From Bats with Apache License 2.0 6 votes vote down vote up
@GET
@Path(PATH_PHYSICAL_PLAN_CONTAINERS + "/{containerId}")
@Produces(MediaType.APPLICATION_JSON)
public JSONObject getContainer(@PathParam("containerId") String containerId) throws Exception
{
  init();
  ContainerInfo ci = null;
  if (containerId.equals(System.getenv(ApplicationConstants.Environment.CONTAINER_ID.toString()))) {
    ci = dagManager.getAppMasterContainerInfo();
  } else {
    for (ContainerInfo containerInfo : dagManager.getCompletedContainerInfo()) {
      if (containerInfo.id.equals(containerId)) {
        ci = containerInfo;
      }
    }
    if (ci == null) {
      StreamingContainerAgent sca = dagManager.getContainerAgent(containerId);
      if (sca == null) {
        throw new NotFoundException();
      }
      ci = sca.getContainerInfo();
    }
  }
  return new JSONObject(objectMapper.writeValueAsString(ci));
}
 
Example #15
Source File: TezChild.java    From incubator-tez with Apache License 2.0 6 votes vote down vote up
/**
 * Setup
 * 
 * @param containerTask
 *          the new task specification. Must be a valid task
 * @param childUGI
 *          the old UGI instance being used
 * @return
 */
UserGroupInformation handleNewTaskCredentials(ContainerTask containerTask,
    UserGroupInformation childUGI) {
  // Re-use the UGI only if the Credentials have not changed.
  Preconditions.checkState(containerTask.shouldDie() != true);
  Preconditions.checkState(containerTask.getTaskSpec() != null);
  if (containerTask.haveCredentialsChanged()) {
    LOG.info("Refreshing UGI since Credentials have changed");
    Credentials taskCreds = containerTask.getCredentials();
    if (taskCreds != null) {
      LOG.info("Credentials : #Tokens=" + taskCreds.numberOfTokens() + ", #SecretKeys="
          + taskCreds.numberOfSecretKeys());
      childUGI = UserGroupInformation.createRemoteUser(System
          .getenv(ApplicationConstants.Environment.USER.toString()));
      childUGI.addCredentials(containerTask.getCredentials());
    } else {
      LOG.info("Not loading any credentials, since no credentials provided");
    }
  }
  return childUGI;
}
 
Example #16
Source File: TestMRApps.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test (timeout = 120000)
public void testSetClasspathWithJobClassloader() throws IOException {
  Configuration conf = new Configuration();
  conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true);
  conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_CLASSLOADER, true);
  Map<String, String> env = new HashMap<String, String>();
  MRApps.setClasspath(env, conf);
  String cp = env.get("CLASSPATH");
  String appCp = env.get("APP_CLASSPATH");
  assertFalse("MAPREDUCE_JOB_CLASSLOADER true, but job.jar is in the"
    + " classpath!", cp.contains("jar" + ApplicationConstants.CLASS_PATH_SEPARATOR + "job"));
  assertFalse("MAPREDUCE_JOB_CLASSLOADER true, but PWD is in the classpath!",
    cp.contains("PWD"));
  String expectedAppClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR,
    Arrays.asList(ApplicationConstants.Environment.PWD.$$(), "job.jar/job.jar",
      "job.jar/classes/", "job.jar/lib/*",
      ApplicationConstants.Environment.PWD.$$() + "/*"));
  assertEquals("MAPREDUCE_JOB_CLASSLOADER true, but job.jar is not in the app"
    + " classpath!", expectedAppClasspath, appCp);
}
 
Example #17
Source File: StreamingContainerAgent.java    From attic-apex-core with Apache License 2.0 5 votes vote down vote up
public ContainerInfo getContainerInfo()
{
  ContainerInfo ci = new ContainerInfo();
  ci.id = container.getExternalId();
  ci.host = container.host;
  ci.state = container.getState().name();
  ci.jvmName = this.jvmName;
  ci.numOperators = container.getOperators().size();
  ci.operators = new TreeMap<>();

  for (PTOperator ptOperator : container.getOperators()) {
    ci.operators.put(ptOperator.getId(), ptOperator.getName());
  }

  ci.memoryMBAllocated = container.getAllocatedMemoryMB();
  ci.lastHeartbeat = lastHeartbeatMillis;
  ci.memoryMBFree = this.memoryMBFree;
  ci.gcCollectionCount = this.gcCollectionCount;
  ci.gcCollectionTime = this.gcCollectionTime;
  ci.startedTime = container.getStartedTime();
  ci.finishedTime = container.getFinishedTime();
  if (this.container.nodeHttpAddress != null) {
    YarnConfiguration conf = new YarnConfiguration();
    ci.containerLogsUrl = ConfigUtils
        .getSchemePrefix(conf) + this.container.nodeHttpAddress + "/node/containerlogs/" + ci.id + "/" + System
        .getenv(ApplicationConstants.Environment.USER.toString());
    ci.rawContainerLogsUrl = ConfigUtils
        .getRawContainerLogsUrl(conf, container.nodeHttpAddress, container.getPlan().getLogicalPlan().getAttributes()
            .get(LogicalPlan.APPLICATION_ID), ci.id);
  }
  return ci;
}
 
Example #18
Source File: StramWebServices.java    From attic-apex-core with Apache License 2.0 5 votes vote down vote up
@GET
@Path(PATH_PHYSICAL_PLAN_CONTAINERS + "/{containerId}/" + PATH_STACKTRACE)
@Produces(MediaType.APPLICATION_JSON)
public JSONObject getContainerStackTrace(@PathParam("containerId") String containerId) throws Exception
{
  init();

  if (containerId.equals(System.getenv(ApplicationConstants.Environment.CONTAINER_ID.toString()))) {
    return StramUtils.getStackTrace();
  }

  StreamingContainerAgent sca = dagManager.getContainerAgent(containerId);

  if (sca == null) {
    throw new NotFoundException("Container not found.");
  }

  if (!sca.getContainerInfo().state.equals("ACTIVE")) {
    throw new NotFoundException("Container is not active.");
  }

  for (int i = 0; i < STACK_TRACE_ATTEMPTS; ++i) {
    String result = sca.getStackTrace();

    if (result != null) {
      return new JSONObject(result);
    }

    Thread.sleep(STACK_TRACE_WAIT_TIME);
  }

  throw new TimeoutException("Not able to get the stack trace");
}
 
Example #19
Source File: YarnEntrypointUtilsTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Nonnull
private static Configuration loadConfiguration(Configuration initialConfiguration, Map<String, String> env) throws IOException {
	final File workingDirectory = TEMPORARY_FOLDER.newFolder();
	env.put(ApplicationConstants.Environment.NM_HOST.key(), "foobar");
	BootstrapTools.writeConfiguration(initialConfiguration, new File(workingDirectory, "flink-conf.yaml"));
	return YarnEntrypointUtils.loadConfiguration(workingDirectory.getAbsolutePath(), env);
}
 
Example #20
Source File: HDInsightJobSubmissionHandler.java    From reef with Apache License 2.0 5 votes vote down vote up
/**
 * Assembles the command to execute the Driver in list form.
 */
private List<String> getCommandList(
    final JobSubmissionEvent jobSubmissionEvent) {

  return new JavaLaunchCommandBuilder()
      .setJavaPath("%JAVA_HOME%/bin/java")
      .setConfigurationFilePaths(Collections.singletonList(this.filenames.getDriverConfigurationPath()))
      .setClassPath(this.classpath.getDriverClasspath())
      .setMemory(jobSubmissionEvent.getDriverMemory().get())
      .setStandardErr(ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + this.filenames.getDriverStderrFileName())
      .setStandardOut(ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + this.filenames.getDriverStdoutFileName())
      .build();
}
 
Example #21
Source File: TestVertexImpl2.java    From tez with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 5000)
public void testTaskLoggingOptsPerLogger() {

  Configuration conf = new TezConfiguration();
  conf.set(TezConfiguration.TEZ_TASK_LOG_LEVEL, "DEBUG;org.apache.hadoop.ipc=INFO;org.apache.hadoop.server=INFO");

  LogTestInfoHolder testInfo = new LogTestInfoHolder();
  VertexWrapper vertexWrapper = createVertexWrapperForLogTests(testInfo, conf);

  List<String> expectedCommands = new LinkedList<String>();
  expectedCommands.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator");
  expectedCommands.add("-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE);
  expectedCommands.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "=" +
      ApplicationConstants.LOG_DIR_EXPANSION_VAR);
  expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "DEBUG" + "," +
      TezConstants.TEZ_CONTAINER_LOGGER_NAME);

  for (int i = 0 ; i < testInfo.numTasks ; i++) {
    ContainerContext containerContext = vertexWrapper
        .vertex.getContainerContext(i);
    String javaOpts = containerContext.getJavaOpts();
    assertTrue(javaOpts.contains(testInfo.initialJavaOpts));
    for (String expectedCmd : expectedCommands) {
      assertTrue(javaOpts.contains(expectedCmd));
    }

    Map<String, String> env = containerContext.getEnvironment();
    String val = env.get(testInfo.envKey);
    assertEquals(testInfo.envVal, val);
    String logEnvVal = env.get(TezConstants.TEZ_CONTAINER_LOG_PARAMS);
    assertEquals("org.apache.hadoop.ipc=INFO;org.apache.hadoop.server=INFO", logEnvVal);
  }
}
 
Example #22
Source File: TestMRApps.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test (timeout = 120000)
public void testSetClasspathWithArchives () throws IOException {
  File testTGZ = new File(testWorkDir, "test.tgz");
  FileOutputStream out = new FileOutputStream(testTGZ);
  out.write(0);
  out.close();
  Configuration conf = new Configuration();
  conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true);
  Job job = Job.getInstance(conf);
  conf = job.getConfiguration();
  String testTGZQualifiedPath = FileSystem.getLocal(conf).makeQualified(new Path(
    testTGZ.getAbsolutePath())).toString();
  conf.set(MRJobConfig.CLASSPATH_ARCHIVES, testTGZQualifiedPath);
  conf.set(MRJobConfig.CACHE_ARCHIVES, testTGZQualifiedPath + "#testTGZ");
  Map<String, String> environment = new HashMap<String, String>();
  MRApps.setClasspath(environment, conf);
  assertTrue(environment.get("CLASSPATH").startsWith(
    ApplicationConstants.Environment.PWD.$$() + ApplicationConstants.CLASS_PATH_SEPARATOR));
  String confClasspath = job.getConfiguration().get(
      YarnConfiguration.YARN_APPLICATION_CLASSPATH,
      StringUtils.join(",",
          YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH));
  if (confClasspath != null) {
    confClasspath = confClasspath.replaceAll(",\\s*", ApplicationConstants.CLASS_PATH_SEPARATOR)
      .trim();
  }
  assertTrue(environment.get("CLASSPATH").contains(confClasspath));
  assertTrue(environment.get("CLASSPATH").contains("testTGZ"));
}
 
Example #23
Source File: TezClientUtils.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
@Private
@VisibleForTesting
public static void addLog4jSystemProperties(String logLevel,
    List<String> vargs) {
  vargs.add("-Dlog4j.configuration="
      + TezConfiguration.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE);
  vargs.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "="
      + ApplicationConstants.LOG_DIR_EXPANSION_VAR);
  vargs.add("-D" + TezConfiguration.TEZ_ROOT_LOGGER_NAME + "=" + logLevel
      + "," + TezConfiguration.TEZ_CONTAINER_LOGGER_NAME);
}
 
Example #24
Source File: StramWebServices.java    From attic-apex-core with Apache License 2.0 5 votes vote down vote up
@POST // not supported by WebAppProxyServlet, can only be called directly
@Path(PATH_PHYSICAL_PLAN_CONTAINERS + "/{containerId}/kill")
@Produces(MediaType.APPLICATION_JSON)
public JSONObject killContainer(@PathParam("containerId") String containerId)
{
  init();
  JSONObject response = new JSONObject();
  if (containerId.equals(System.getenv(ApplicationConstants.Environment.CONTAINER_ID.toString()))) {
    LOG.info("Received a kill request on application master container. Exiting.");
    new Thread()
    {
      @Override
      public void run()
      {
        try {
          Thread.sleep(3000);
          System.exit(1);
        } catch (InterruptedException ex) {
          LOG.info("Received interrupt, aborting exit.");
        }
      }

    }.start();
  } else {
    dagManager.stopContainer(containerId);
  }
  return response;
}
 
Example #25
Source File: StreamingContainerAgent.java    From Bats with Apache License 2.0 5 votes vote down vote up
public ContainerInfo getContainerInfo()
{
  ContainerInfo ci = new ContainerInfo();
  ci.id = container.getExternalId();
  ci.host = container.host;
  ci.state = container.getState().name();
  ci.jvmName = this.jvmName;
  ci.numOperators = container.getOperators().size();
  ci.operators = new TreeMap<>();

  for (PTOperator ptOperator : container.getOperators()) {
    ci.operators.put(ptOperator.getId(), ptOperator.getName());
  }

  ci.memoryMBAllocated = container.getAllocatedMemoryMB();
  ci.lastHeartbeat = lastHeartbeatMillis;
  ci.memoryMBFree = this.memoryMBFree;
  ci.gcCollectionCount = this.gcCollectionCount;
  ci.gcCollectionTime = this.gcCollectionTime;
  ci.startedTime = container.getStartedTime();
  ci.finishedTime = container.getFinishedTime();
  if (this.container.nodeHttpAddress != null) {
    YarnConfiguration conf = new YarnConfiguration();
    ci.containerLogsUrl = ConfigUtils
        .getSchemePrefix(conf) + this.container.nodeHttpAddress + "/node/containerlogs/" + ci.id + "/" + System
        .getenv(ApplicationConstants.Environment.USER.toString());
    ci.rawContainerLogsUrl = ConfigUtils
        .getRawContainerLogsUrl(conf, container.nodeHttpAddress, container.getPlan().getLogicalPlan().getAttributes()
            .get(LogicalPlan.APPLICATION_ID), ci.id);
  }
  return ci;
}
 
Example #26
Source File: TestMRApps.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test (timeout = 120000)
public void testSetClasspath() throws IOException {
  Configuration conf = new Configuration();
  conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true);
  Job job = Job.getInstance(conf);
  Map<String, String> environment = new HashMap<String, String>();
  MRApps.setClasspath(environment, job.getConfiguration());
  assertTrue(environment.get("CLASSPATH").startsWith(
    ApplicationConstants.Environment.PWD.$$()
        + ApplicationConstants.CLASS_PATH_SEPARATOR));
  String yarnAppClasspath = job.getConfiguration().get(
      YarnConfiguration.YARN_APPLICATION_CLASSPATH,
      StringUtils.join(",",
          YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH));
  if (yarnAppClasspath != null) {
    yarnAppClasspath =
        yarnAppClasspath.replaceAll(",\\s*",
          ApplicationConstants.CLASS_PATH_SEPARATOR).trim();
  }
  assertTrue(environment.get("CLASSPATH").contains(yarnAppClasspath));
  String mrAppClasspath = 
      job.getConfiguration().get(
          MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH,
          MRJobConfig.DEFAULT_MAPREDUCE_CROSS_PLATFORM_APPLICATION_CLASSPATH);
  if (mrAppClasspath != null) {
    mrAppClasspath =
        mrAppClasspath.replaceAll(",\\s*",
          ApplicationConstants.CLASS_PATH_SEPARATOR).trim();
  }
  assertTrue(environment.get("CLASSPATH").contains(mrAppClasspath));
}
 
Example #27
Source File: TestMRApps.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test (timeout = 3000000)
public void testSetClasspathWithFramework() throws IOException {
  final String FRAMEWORK_NAME = "some-framework-name";
  final String FRAMEWORK_PATH = "some-framework-path#" + FRAMEWORK_NAME;
  Configuration conf = new Configuration();
  conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true);
  conf.set(MRJobConfig.MAPREDUCE_APPLICATION_FRAMEWORK_PATH, FRAMEWORK_PATH);
  Map<String, String> env = new HashMap<String, String>();
  try {
    MRApps.setClasspath(env, conf);
    fail("Failed to catch framework path set without classpath change");
  } catch (IllegalArgumentException e) {
    assertTrue("Unexpected IllegalArgumentException",
        e.getMessage().contains("Could not locate MapReduce framework name '"
            + FRAMEWORK_NAME + "'"));
  }

  env.clear();
  final String FRAMEWORK_CLASSPATH = FRAMEWORK_NAME + "/*.jar";
  conf.set(MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH, FRAMEWORK_CLASSPATH);
  MRApps.setClasspath(env, conf);
  final String stdClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR,
      Arrays.asList("job.jar/job.jar", "job.jar/classes/", "job.jar/lib/*",
          ApplicationConstants.Environment.PWD.$$() + "/*"));
  String expectedClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR,
      Arrays.asList(ApplicationConstants.Environment.PWD.$$(),
          FRAMEWORK_CLASSPATH, stdClasspath));
  assertEquals("Incorrect classpath with framework and no user precedence",
      expectedClasspath, env.get("CLASSPATH"));

  env.clear();
  conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, true);
  MRApps.setClasspath(env, conf);
  expectedClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR,
      Arrays.asList(ApplicationConstants.Environment.PWD.$$(),
          stdClasspath, FRAMEWORK_CLASSPATH));
  assertEquals("Incorrect classpath with framework and user precedence",
      expectedClasspath, env.get("CLASSPATH"));
}
 
Example #28
Source File: TestMRApps.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test (timeout = 120000)
public void testSetClasspathWithArchives () throws IOException {
  File testTGZ = new File(testWorkDir, "test.tgz");
  FileOutputStream out = new FileOutputStream(testTGZ);
  out.write(0);
  out.close();
  Configuration conf = new Configuration();
  conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true);
  Job job = Job.getInstance(conf);
  conf = job.getConfiguration();
  String testTGZQualifiedPath = FileSystem.getLocal(conf).makeQualified(new Path(
    testTGZ.getAbsolutePath())).toString();
  conf.set(MRJobConfig.CLASSPATH_ARCHIVES, testTGZQualifiedPath);
  conf.set(MRJobConfig.CACHE_ARCHIVES, testTGZQualifiedPath + "#testTGZ");
  Map<String, String> environment = new HashMap<String, String>();
  MRApps.setClasspath(environment, conf);
  assertTrue(environment.get("CLASSPATH").startsWith(
    ApplicationConstants.Environment.PWD.$$() + ApplicationConstants.CLASS_PATH_SEPARATOR));
  String confClasspath = job.getConfiguration().get(
      YarnConfiguration.YARN_APPLICATION_CLASSPATH,
      StringUtils.join(",",
          YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH));
  if (confClasspath != null) {
    confClasspath = confClasspath.replaceAll(",\\s*", ApplicationConstants.CLASS_PATH_SEPARATOR)
      .trim();
  }
  assertTrue(environment.get("CLASSPATH").contains(confClasspath));
  assertTrue(environment.get("CLASSPATH").contains("testTGZ"));
}
 
Example #29
Source File: TestMRApps.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test (timeout = 120000)
public void testSetClasspath() throws IOException {
  Configuration conf = new Configuration();
  conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true);
  Job job = Job.getInstance(conf);
  Map<String, String> environment = new HashMap<String, String>();
  MRApps.setClasspath(environment, job.getConfiguration());
  assertTrue(environment.get("CLASSPATH").startsWith(
    ApplicationConstants.Environment.PWD.$$()
        + ApplicationConstants.CLASS_PATH_SEPARATOR));
  String yarnAppClasspath = job.getConfiguration().get(
      YarnConfiguration.YARN_APPLICATION_CLASSPATH,
      StringUtils.join(",",
          YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH));
  if (yarnAppClasspath != null) {
    yarnAppClasspath =
        yarnAppClasspath.replaceAll(",\\s*",
          ApplicationConstants.CLASS_PATH_SEPARATOR).trim();
  }
  assertTrue(environment.get("CLASSPATH").contains(yarnAppClasspath));
  String mrAppClasspath = 
      job.getConfiguration().get(
          MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH,
          MRJobConfig.DEFAULT_MAPREDUCE_CROSS_PLATFORM_APPLICATION_CLASSPATH);
  if (mrAppClasspath != null) {
    mrAppClasspath =
        mrAppClasspath.replaceAll(",\\s*",
          ApplicationConstants.CLASS_PATH_SEPARATOR).trim();
  }
  assertTrue(environment.get("CLASSPATH").contains(mrAppClasspath));
}
 
Example #30
Source File: YarnResourceManager.java    From flink with Apache License 2.0 5 votes vote down vote up
private ContainerLaunchContext createTaskExecutorLaunchContext(
	String containerId,
	String host,
	TaskExecutorProcessSpec taskExecutorProcessSpec) throws Exception {

	// init the ContainerLaunchContext
	final String currDir = env.get(ApplicationConstants.Environment.PWD.key());

	final ContaineredTaskManagerParameters taskManagerParameters =
			ContaineredTaskManagerParameters.create(flinkConfig, taskExecutorProcessSpec);

	log.info("TaskExecutor {} will be started on {} with {}.",
		containerId,
		host,
		taskExecutorProcessSpec);

	final Configuration taskManagerConfig = BootstrapTools.cloneConfiguration(flinkConfig);
	taskManagerConfig.set(TaskManagerOptions.TASK_MANAGER_RESOURCE_ID, containerId);

	final String taskManagerDynamicProperties =
		BootstrapTools.getDynamicPropertiesAsString(flinkClientConfig, taskManagerConfig);

	log.debug("TaskManager configuration: {}", taskManagerConfig);

	ContainerLaunchContext taskExecutorLaunchContext = Utils.createTaskExecutorContext(
		flinkConfig,
		yarnConfig,
		env,
		taskManagerParameters,
		taskManagerDynamicProperties,
		currDir,
		YarnTaskExecutorRunner.class,
		log);

	taskExecutorLaunchContext.getEnvironment()
			.put(ENV_FLINK_NODE_ID, host);
	return taskExecutorLaunchContext;
}