Java Code Examples for org.apache.hadoop.yarn.api.ApplicationConstants

The following examples show how to use org.apache.hadoop.yarn.api.ApplicationConstants. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Bats   Source File: StramWebServices.java    License: Apache License 2.0 6 votes vote down vote up
@GET
@Path(PATH_PHYSICAL_PLAN_CONTAINERS + "/{containerId}")
@Produces(MediaType.APPLICATION_JSON)
public JSONObject getContainer(@PathParam("containerId") String containerId) throws Exception
{
  init();
  ContainerInfo ci = null;
  if (containerId.equals(System.getenv(ApplicationConstants.Environment.CONTAINER_ID.toString()))) {
    ci = dagManager.getAppMasterContainerInfo();
  } else {
    for (ContainerInfo containerInfo : dagManager.getCompletedContainerInfo()) {
      if (containerInfo.id.equals(containerId)) {
        ci = containerInfo;
      }
    }
    if (ci == null) {
      StreamingContainerAgent sca = dagManager.getContainerAgent(containerId);
      if (sca == null) {
        throw new NotFoundException();
      }
      ci = sca.getContainerInfo();
    }
  }
  return new JSONObject(objectMapper.writeValueAsString(ci));
}
 
Example 2
Source Project: tez   Source File: TezYARNUtils.java    License: Apache License 2.0 6 votes vote down vote up
public static void setupDefaultEnv(Map<String, String> env, Configuration conf,  String userEnvKey, String userEnvDefault,
    String clusterDefaultEnvKey, String clusterDefaultEnvDefault, boolean usingArchive) {
  // Setup the CLASSPATH in environment
  // i.e. add { Hadoop jars, job jar, CWD } to classpath.
  String classpath = getFrameworkClasspath(conf, usingArchive);
  TezYARNUtils.addToEnvironment(env,
      ApplicationConstants.Environment.CLASSPATH.name(),
      classpath, File.pathSeparator);

  // Pre-pend pwd to LD_LIBRARY_PATH
  // Done separately here because this is known to work platform
  // independent
  TezYARNUtils.addToEnvironment(env,
      Environment.LD_LIBRARY_PATH.name(), Environment.PWD.$(), File.pathSeparator);
  TezYARNUtils.appendToEnvFromInputString(env,
      conf.get(userEnvKey, userEnvDefault), File.pathSeparator);
  // set any env from config if it is not set already
  TezYARNUtils.appendToEnvFromInputString(env,
      conf.get(clusterDefaultEnvKey, clusterDefaultEnvDefault), File.pathSeparator);
}
 
Example 3
Source Project: samza   Source File: YarnClusterResourceManager.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Runs a process as specified by the command builder on the container.
 * @param processorId id of the samza processor to run (passed as a command line parameter to the process)
 * @param container the yarn container to run the processor on.
 * @param cmdBuilder the command builder that encapsulates the command, and the context
 * @throws IOException on IO exceptions running the container
 */
public void runProcessor(String processorId, Container container, CommandBuilder cmdBuilder) throws IOException {
  String containerIdStr = ConverterUtils.toString(container.getId());
  String cmdPath = "./__package/";
  cmdBuilder.setCommandPath(cmdPath);
  String command = cmdBuilder.buildCommand();

  Map<String, String> env = getEscapedEnvironmentVariablesMap(cmdBuilder);
  env.put(ShellCommandConfig.ENV_EXECUTION_ENV_CONTAINER_ID, Util.envVarEscape(container.getId().toString()));

  Path packagePath = new Path(yarnConfig.getPackagePath());
  String formattedCommand =
      getFormattedCommand(ApplicationConstants.LOG_DIR_EXPANSION_VAR, command, ApplicationConstants.STDOUT,
          ApplicationConstants.STDERR);

  log.info("Running Processor ID: {} on Container ID: {} on host: {} using command: {} and env: {} and package path: {}",
      processorId, containerIdStr, container.getNodeHttpAddress(), formattedCommand, env, packagePath);
  state.pendingProcessors.put(processorId, new YarnContainer(container));

  startContainer(packagePath, container, env, formattedCommand);

  log.info("Made start request for Processor ID: {} on Container ID: {} on host: {} (http://{}/node/containerlogs/{}).",
      processorId, containerIdStr, container.getNodeId().getHost(), container.getNodeHttpAddress(), containerIdStr);
}
 
Example 4
Source Project: TensorFlowOnYARN   Source File: LaunchContainerThread.java    License: Apache License 2.0 6 votes vote down vote up
private String makeContainerCommand(long containerMemory, String clusterSpec,
    String jobName, int taskIndex) {
  String[] commands = new String[]{
      ApplicationConstants.Environment.JAVA_HOME.$$() + "/bin/java",
      "-Xmx" + containerMemory + "m",
      TFServerRunner.class.getName() + " ",
      Utils.mkOption(Constants.OPT_CLUSTER_SPEC, clusterSpec),
      Utils.mkOption(Constants.OPT_JOB_NAME, jobName),
      Utils.mkOption(Constants.OPT_TASK_INDEX, taskIndex),
      "1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/TFServerRunner." +
          ApplicationConstants.STDOUT,
      "2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/TFServerRunner." +
          ApplicationConstants.STDERR
  };

  return Utils.mkString(commands, " ");
}
 
Example 5
Source Project: TensorFlowOnYARN   Source File: LaunchCluster.java    License: Apache License 2.0 6 votes vote down vote up
private String makeAppMasterCommand(String tfLib, String tfJar) {
  String[] commands = new String[]{
      ApplicationConstants.Environment.JAVA_HOME.$$() + "/bin/java",
      // Set Xmx based on am memory size
      "-Xmx" + amMemory + "m",
      // Set class name
      ApplicationMaster.class.getName(),
      Utils.mkOption(Constants.OPT_TF_CONTAINER_MEMORY, containerMemory),
      Utils.mkOption(Constants.OPT_TF_CONTAINER_VCORES, containerVCores),
      Utils.mkOption(Constants.OPT_TF_WORKER_NUM, workerNum),
      Utils.mkOption(Constants.OPT_TF_PS_NUM, psNum),
      Utils.mkOption(Constants.OPT_TF_LIB, tfLib),
      Utils.mkOption(Constants.OPT_TF_JAR, tfJar),
      "1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stdout",
      "2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stderr"
  };
  return Utils.mkString(commands, " ");
}
 
Example 6
Source Project: incubator-tez   Source File: TezYARNUtils.java    License: Apache License 2.0 6 votes vote down vote up
public static void setupDefaultEnv(Map<String, String> env,
    Configuration conf,  String confEnvKey, String confEnvKeyDefault) {
  // Setup the CLASSPATH in environment
  // i.e. add { Hadoop jars, job jar, CWD } to classpath.
  String classpath = getFrameworkClasspath(conf);
  TezYARNUtils.addToEnvironment(env,
      ApplicationConstants.Environment.CLASSPATH.name(),
      classpath, File.pathSeparator);

  // set any env from config if it is not set already
  TezYARNUtils.setEnvIfAbsentFromInputString(env, conf.get(
      confEnvKey, confEnvKeyDefault));
  
  // Append pwd to LD_LIBRARY_PATH
  // Done separately here because this is known to work platform independent
  TezYARNUtils.addToEnvironment(env, Environment.LD_LIBRARY_PATH.name(),
      Environment.PWD.$(), File.pathSeparator);
}
 
Example 7
Source Project: tez   Source File: TezTestServiceContainerLauncher.java    License: Apache License 2.0 6 votes vote down vote up
private RunContainerRequestProto constructRunContainerRequest(ContainerLaunchRequest launchRequest) throws
    IOException {
  RunContainerRequestProto.Builder builder = RunContainerRequestProto.newBuilder();
  Preconditions.checkArgument(launchRequest.getTaskCommunicatorName().equals(
      TezConstants.getTezYarnServicePluginName()));
  InetSocketAddress address = (InetSocketAddress) getContext().getTaskCommunicatorMetaInfo(launchRequest.getTaskCommunicatorName());
  builder.setAmHost(address.getHostName()).setAmPort(address.getPort());
  builder.setAppAttemptNumber(appAttemptId.getAttemptId());
  builder.setApplicationIdString(appAttemptId.getApplicationId().toString());
  builder.setTokenIdentifier(tokenIdentifier);
  builder.setContainerIdString(launchRequest.getContainerId().toString());
  builder.setCredentialsBinary(
      ByteString.copyFrom(launchRequest.getContainerLaunchContext().getTokens()));
  // TODO Avoid reading this from the environment
  builder.setUser(System.getenv(ApplicationConstants.Environment.USER.name()));
  return builder.build();
}
 
Example 8
Source Project: hadoop   Source File: ContainerLaunch.java    License: Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
public static String expandEnvironment(String var,
    Path containerLogDir) {
  var = var.replace(ApplicationConstants.LOG_DIR_EXPANSION_VAR,
    containerLogDir.toString());
  var =  var.replace(ApplicationConstants.CLASS_PATH_SEPARATOR,
    File.pathSeparator);

  // replace parameter expansion marker. e.g. {{VAR}} on Windows is replaced
  // as %VAR% and on Linux replaced as "$VAR"
  if (Shell.WINDOWS) {
    var = var.replaceAll("(\\{\\{)|(\\}\\})", "%");
  } else {
    var = var.replace(ApplicationConstants.PARAMETER_EXPANSION_LEFT, "$");
    var = var.replace(ApplicationConstants.PARAMETER_EXPANSION_RIGHT, "");
  }
  return var;
}
 
Example 9
Source Project: incubator-tez   Source File: TezChild.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Setup
 * 
 * @param containerTask
 *          the new task specification. Must be a valid task
 * @param childUGI
 *          the old UGI instance being used
 * @return
 */
UserGroupInformation handleNewTaskCredentials(ContainerTask containerTask,
    UserGroupInformation childUGI) {
  // Re-use the UGI only if the Credentials have not changed.
  Preconditions.checkState(containerTask.shouldDie() != true);
  Preconditions.checkState(containerTask.getTaskSpec() != null);
  if (containerTask.haveCredentialsChanged()) {
    LOG.info("Refreshing UGI since Credentials have changed");
    Credentials taskCreds = containerTask.getCredentials();
    if (taskCreds != null) {
      LOG.info("Credentials : #Tokens=" + taskCreds.numberOfTokens() + ", #SecretKeys="
          + taskCreds.numberOfSecretKeys());
      childUGI = UserGroupInformation.createRemoteUser(System
          .getenv(ApplicationConstants.Environment.USER.toString()));
      childUGI.addCredentials(containerTask.getCredentials());
    } else {
      LOG.info("Not loading any credentials, since no credentials provided");
    }
  }
  return childUGI;
}
 
Example 10
Source Project: flink   Source File: YarnSessionClusterEntrypoint.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) {
	// startup checks and logging
	EnvironmentInformation.logEnvironmentInfo(LOG, YarnSessionClusterEntrypoint.class.getSimpleName(), args);
	SignalHandler.register(LOG);
	JvmShutdownSafeguard.installAsShutdownHook(LOG);

	Map<String, String> env = System.getenv();

	final String workingDirectory = env.get(ApplicationConstants.Environment.PWD.key());
	Preconditions.checkArgument(
		workingDirectory != null,
		"Working directory variable (%s) not set",
		ApplicationConstants.Environment.PWD.key());

	try {
		YarnEntrypointUtils.logYarnEnvironmentInformation(env, LOG);
	} catch (IOException e) {
		LOG.warn("Could not log YARN environment information.", e);
	}

	Configuration configuration = YarnEntrypointUtils.loadConfiguration(workingDirectory, env);

	YarnSessionClusterEntrypoint yarnSessionClusterEntrypoint = new YarnSessionClusterEntrypoint(configuration);

	ClusterEntrypoint.runClusterEntrypoint(yarnSessionClusterEntrypoint);
}
 
Example 11
Source Project: hadoop   Source File: TestMRApps.java    License: Apache License 2.0 6 votes vote down vote up
@Test (timeout = 120000)
public void testSetClasspathWithUserPrecendence() {
   Configuration conf = new Configuration();
   conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true);
   conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, true);
   Map<String, String> env = new HashMap<String, String>();
   try {
     MRApps.setClasspath(env, conf);
   } catch (Exception e) {
     fail("Got exception while setting classpath");
   }
   String env_str = env.get("CLASSPATH");
   String expectedClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR,
     Arrays.asList(ApplicationConstants.Environment.PWD.$$(), "job.jar/job.jar",
       "job.jar/classes/", "job.jar/lib/*",
       ApplicationConstants.Environment.PWD.$$() + "/*"));
   assertTrue("MAPREDUCE_JOB_USER_CLASSPATH_FIRST set, but not taking effect!",
     env_str.startsWith(expectedClasspath));
 }
 
Example 12
Source Project: hadoop   Source File: TestMRApps.java    License: Apache License 2.0 6 votes vote down vote up
@Test (timeout = 120000)
public void testSetClasspathWithNoUserPrecendence() {
  Configuration conf = new Configuration();
  conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true);
  conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, false);
  Map<String, String> env = new HashMap<String, String>();
  try {
    MRApps.setClasspath(env, conf);
  } catch (Exception e) {
    fail("Got exception while setting classpath");
  }
  String env_str = env.get("CLASSPATH");
  String expectedClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR,
    Arrays.asList("job.jar/job.jar", "job.jar/classes/", "job.jar/lib/*",
      ApplicationConstants.Environment.PWD.$$() + "/*"));
  assertTrue("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, and job.jar is not in"
    + " the classpath!", env_str.contains(expectedClasspath));
  assertFalse("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, but taking effect!",
    env_str.startsWith(expectedClasspath));
}
 
Example 13
Source Project: hadoop   Source File: TestMRApps.java    License: Apache License 2.0 6 votes vote down vote up
@Test (timeout = 120000)
public void testSetClasspathWithJobClassloader() throws IOException {
  Configuration conf = new Configuration();
  conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true);
  conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_CLASSLOADER, true);
  Map<String, String> env = new HashMap<String, String>();
  MRApps.setClasspath(env, conf);
  String cp = env.get("CLASSPATH");
  String appCp = env.get("APP_CLASSPATH");
  assertFalse("MAPREDUCE_JOB_CLASSLOADER true, but job.jar is in the"
    + " classpath!", cp.contains("jar" + ApplicationConstants.CLASS_PATH_SEPARATOR + "job"));
  assertFalse("MAPREDUCE_JOB_CLASSLOADER true, but PWD is in the classpath!",
    cp.contains("PWD"));
  String expectedAppClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR,
    Arrays.asList(ApplicationConstants.Environment.PWD.$$(), "job.jar/job.jar",
      "job.jar/classes/", "job.jar/lib/*",
      ApplicationConstants.Environment.PWD.$$() + "/*"));
  assertEquals("MAPREDUCE_JOB_CLASSLOADER true, but job.jar is not in the app"
    + " classpath!", expectedAppClasspath, appCp);
}
 
Example 14
Source Project: big-c   Source File: ContainerLaunch.java    License: Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
public static String expandEnvironment(String var,
    Path containerLogDir) {
  var = var.replace(ApplicationConstants.LOG_DIR_EXPANSION_VAR,
    containerLogDir.toString());
  var =  var.replace(ApplicationConstants.CLASS_PATH_SEPARATOR,
    File.pathSeparator);

  // replace parameter expansion marker. e.g. {{VAR}} on Windows is replaced
  // as %VAR% and on Linux replaced as "$VAR"
  if (Shell.WINDOWS) {
    var = var.replaceAll("(\\{\\{)|(\\}\\})", "%");
  } else {
    var = var.replace(ApplicationConstants.PARAMETER_EXPANSION_LEFT, "$");
    var = var.replace(ApplicationConstants.PARAMETER_EXPANSION_RIGHT, "");
  }
  return var;
}
 
Example 15
Source Project: big-c   Source File: TestMRApps.java    License: Apache License 2.0 6 votes vote down vote up
@Test (timeout = 120000)
public void testSetClasspathWithNoUserPrecendence() {
  Configuration conf = new Configuration();
  conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true);
  conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, false);
  Map<String, String> env = new HashMap<String, String>();
  try {
    MRApps.setClasspath(env, conf);
  } catch (Exception e) {
    fail("Got exception while setting classpath");
  }
  String env_str = env.get("CLASSPATH");
  String expectedClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR,
    Arrays.asList("job.jar/job.jar", "job.jar/classes/", "job.jar/lib/*",
      ApplicationConstants.Environment.PWD.$$() + "/*"));
  assertTrue("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, and job.jar is not in"
    + " the classpath!", env_str.contains(expectedClasspath));
  assertFalse("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, but taking effect!",
    env_str.startsWith(expectedClasspath));
}
 
Example 16
Source Project: big-c   Source File: TestMRApps.java    License: Apache License 2.0 6 votes vote down vote up
@Test (timeout = 120000)
public void testSetClasspathWithJobClassloader() throws IOException {
  Configuration conf = new Configuration();
  conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true);
  conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_CLASSLOADER, true);
  Map<String, String> env = new HashMap<String, String>();
  MRApps.setClasspath(env, conf);
  String cp = env.get("CLASSPATH");
  String appCp = env.get("APP_CLASSPATH");
  assertFalse("MAPREDUCE_JOB_CLASSLOADER true, but job.jar is in the"
    + " classpath!", cp.contains("jar" + ApplicationConstants.CLASS_PATH_SEPARATOR + "job"));
  assertFalse("MAPREDUCE_JOB_CLASSLOADER true, but PWD is in the classpath!",
    cp.contains("PWD"));
  String expectedAppClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR,
    Arrays.asList(ApplicationConstants.Environment.PWD.$$(), "job.jar/job.jar",
      "job.jar/classes/", "job.jar/lib/*",
      ApplicationConstants.Environment.PWD.$$() + "/*"));
  assertEquals("MAPREDUCE_JOB_CLASSLOADER true, but job.jar is not in the app"
    + " classpath!", expectedAppClasspath, appCp);
}
 
Example 17
Source Project: Bats   Source File: StreamingContainerAgent.java    License: Apache License 2.0 5 votes vote down vote up
public ContainerInfo getContainerInfo()
{
  ContainerInfo ci = new ContainerInfo();
  ci.id = container.getExternalId();
  ci.host = container.host;
  ci.state = container.getState().name();
  ci.jvmName = this.jvmName;
  ci.numOperators = container.getOperators().size();
  ci.operators = new TreeMap<>();

  for (PTOperator ptOperator : container.getOperators()) {
    ci.operators.put(ptOperator.getId(), ptOperator.getName());
  }

  ci.memoryMBAllocated = container.getAllocatedMemoryMB();
  ci.lastHeartbeat = lastHeartbeatMillis;
  ci.memoryMBFree = this.memoryMBFree;
  ci.gcCollectionCount = this.gcCollectionCount;
  ci.gcCollectionTime = this.gcCollectionTime;
  ci.startedTime = container.getStartedTime();
  ci.finishedTime = container.getFinishedTime();
  if (this.container.nodeHttpAddress != null) {
    YarnConfiguration conf = new YarnConfiguration();
    ci.containerLogsUrl = ConfigUtils
        .getSchemePrefix(conf) + this.container.nodeHttpAddress + "/node/containerlogs/" + ci.id + "/" + System
        .getenv(ApplicationConstants.Environment.USER.toString());
    ci.rawContainerLogsUrl = ConfigUtils
        .getRawContainerLogsUrl(conf, container.nodeHttpAddress, container.getPlan().getLogicalPlan().getAttributes()
            .get(LogicalPlan.APPLICATION_ID), ci.id);
  }
  return ci;
}
 
Example 18
Source Project: Bats   Source File: StramWebServices.java    License: Apache License 2.0 5 votes vote down vote up
@GET
@Path(PATH_PHYSICAL_PLAN_CONTAINERS + "/{containerId}/" + PATH_STACKTRACE)
@Produces(MediaType.APPLICATION_JSON)
public JSONObject getContainerStackTrace(@PathParam("containerId") String containerId) throws Exception
{
  init();

  if (containerId.equals(System.getenv(ApplicationConstants.Environment.CONTAINER_ID.toString()))) {
    return StramUtils.getStackTrace();
  }

  StreamingContainerAgent sca = dagManager.getContainerAgent(containerId);

  if (sca == null) {
    throw new NotFoundException("Container not found.");
  }

  if (!sca.getContainerInfo().state.equals("ACTIVE")) {
    throw new NotFoundException("Container is not active.");
  }

  for (int i = 0; i < STACK_TRACE_ATTEMPTS; ++i) {
    String result = sca.getStackTrace();

    if (result != null) {
      return new JSONObject(result);
    }

    Thread.sleep(STACK_TRACE_WAIT_TIME);
  }

  throw new TimeoutException("Not able to get the stack trace");
}
 
Example 19
Source Project: Bats   Source File: StramWebServices.java    License: Apache License 2.0 5 votes vote down vote up
@POST // not supported by WebAppProxyServlet, can only be called directly
@Path(PATH_PHYSICAL_PLAN_CONTAINERS + "/{containerId}/kill")
@Produces(MediaType.APPLICATION_JSON)
public JSONObject killContainer(@PathParam("containerId") String containerId)
{
  init();
  JSONObject response = new JSONObject();
  if (containerId.equals(System.getenv(ApplicationConstants.Environment.CONTAINER_ID.toString()))) {
    LOG.info("Received a kill request on application master container. Exiting.");
    new Thread()
    {
      @Override
      public void run()
      {
        try {
          Thread.sleep(3000);
          System.exit(1);
        } catch (InterruptedException ex) {
          LOG.info("Received interrupt, aborting exit.");
        }
      }

    }.start();
  } else {
    dagManager.stopContainer(containerId);
  }
  return response;
}
 
Example 20
Source Project: Flink-CEPplus   Source File: YarnResourceManager.java    License: Apache License 2.0 5 votes vote down vote up
private ContainerLaunchContext createTaskExecutorLaunchContext(Resource resource, String containerId, String host)
		throws Exception {
	// init the ContainerLaunchContext
	final String currDir = env.get(ApplicationConstants.Environment.PWD.key());

	final ContaineredTaskManagerParameters taskManagerParameters =
			ContaineredTaskManagerParameters.create(flinkConfig, resource.getMemory(), numberOfTaskSlots);

	log.debug("TaskExecutor {} will be started with container size {} MB, JVM heap size {} MB, " +
			"JVM direct memory limit {} MB",
			containerId,
			taskManagerParameters.taskManagerTotalMemoryMB(),
			taskManagerParameters.taskManagerHeapSizeMB(),
			taskManagerParameters.taskManagerDirectMemoryLimitMB());

	Configuration taskManagerConfig = BootstrapTools.cloneConfiguration(flinkConfig);

	log.debug("TaskManager configuration: {}", taskManagerConfig);

	ContainerLaunchContext taskExecutorLaunchContext = Utils.createTaskExecutorContext(
		flinkConfig,
		yarnConfig,
		env,
		taskManagerParameters,
		taskManagerConfig,
		currDir,
		YarnTaskExecutorRunner.class,
		log);

	// set a special environment variable to uniquely identify this container
	taskExecutorLaunchContext.getEnvironment()
			.put(ENV_FLINK_CONTAINER_ID, containerId);
	taskExecutorLaunchContext.getEnvironment()
			.put(ENV_FLINK_NODE_ID, host);
	return taskExecutorLaunchContext;
}
 
Example 21
Source Project: incubator-gobblin   Source File: GobblinYarnAppLauncher.java    License: Apache License 2.0 5 votes vote down vote up
private LogCopier buildLogCopier(Config config, Path sinkLogDir, Path appWorkDir) throws IOException {
  FileSystem rawLocalFs = this.closer.register(new RawLocalFileSystem());
  rawLocalFs.initialize(URI.create(ConfigurationKeys.LOCAL_FS_URI), new Configuration());

  LogCopier.Builder builder = LogCopier.newBuilder()
          .useSrcFileSystem(this.fs)
          .useDestFileSystem(rawLocalFs)
          .readFrom(getHdfsLogDir(appWorkDir))
          .writeTo(sinkLogDir)
          .acceptsLogFileExtensions(ImmutableSet.of(ApplicationConstants.STDOUT, ApplicationConstants.STDERR));
  return builder.build();
}
 
Example 22
Source Project: flink   Source File: YarnLogConfigUtil.java    License: Apache License 2.0 5 votes vote down vote up
private static String getLog4jCommand(final String logConfigFilePath) {
	final boolean hasLog4j = logConfigFilePath.endsWith(CONFIG_FILE_LOG4J_NAME);
	if (!hasLog4j) {
		return "";
	}

	return new StringBuilder("-Dlog.file=\"" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/jobmanager.log\"")
			.append(" -Dlog4j.configuration=file:" + CONFIG_FILE_LOG4J_NAME)
			.append(" -Dlog4j.configurationFile=file:" + CONFIG_FILE_LOG4J_NAME)
			.toString();
}
 
Example 23
Source Project: Flink-CEPplus   Source File: YarnJobClusterEntrypoint.java    License: Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) {
	// startup checks and logging
	EnvironmentInformation.logEnvironmentInfo(LOG, YarnJobClusterEntrypoint.class.getSimpleName(), args);
	SignalHandler.register(LOG);
	JvmShutdownSafeguard.installAsShutdownHook(LOG);

	Map<String, String> env = System.getenv();

	final String workingDirectory = env.get(ApplicationConstants.Environment.PWD.key());
	Preconditions.checkArgument(
		workingDirectory != null,
		"Working directory variable (%s) not set",
		ApplicationConstants.Environment.PWD.key());

	try {
		YarnEntrypointUtils.logYarnEnvironmentInformation(env, LOG);
	} catch (IOException e) {
		LOG.warn("Could not log YARN environment information.", e);
	}

	Configuration configuration = YarnEntrypointUtils.loadConfiguration(workingDirectory, env, LOG);

	YarnJobClusterEntrypoint yarnJobClusterEntrypoint = new YarnJobClusterEntrypoint(
		configuration,
		workingDirectory);

	ClusterEntrypoint.runClusterEntrypoint(yarnJobClusterEntrypoint);
}
 
Example 24
Source Project: flink   Source File: YarnResourceManager.java    License: Apache License 2.0 5 votes vote down vote up
private ContainerLaunchContext createTaskExecutorLaunchContext(Resource resource, String containerId, String host)
		throws Exception {
	// init the ContainerLaunchContext
	final String currDir = env.get(ApplicationConstants.Environment.PWD.key());

	final ContaineredTaskManagerParameters taskManagerParameters =
			ContaineredTaskManagerParameters.create(flinkConfig, resource.getMemory(), numberOfTaskSlots);

	log.debug("TaskExecutor {} will be started with container size {} MB, JVM heap size {} MB, " +
			"JVM direct memory limit {} MB",
			containerId,
			taskManagerParameters.taskManagerTotalMemoryMB(),
			taskManagerParameters.taskManagerHeapSizeMB(),
			taskManagerParameters.taskManagerDirectMemoryLimitMB());

	Configuration taskManagerConfig = BootstrapTools.cloneConfiguration(flinkConfig);

	log.debug("TaskManager configuration: {}", taskManagerConfig);

	ContainerLaunchContext taskExecutorLaunchContext = Utils.createTaskExecutorContext(
		flinkConfig,
		yarnConfig,
		env,
		taskManagerParameters,
		taskManagerConfig,
		currDir,
		YarnTaskExecutorRunner.class,
		log);

	// set a special environment variable to uniquely identify this container
	taskExecutorLaunchContext.getEnvironment()
			.put(ENV_FLINK_CONTAINER_ID, containerId);
	taskExecutorLaunchContext.getEnvironment()
			.put(ENV_FLINK_NODE_ID, host);
	return taskExecutorLaunchContext;
}
 
Example 25
Source Project: flink   Source File: YarnJobClusterEntrypoint.java    License: Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) {
	// startup checks and logging
	EnvironmentInformation.logEnvironmentInfo(LOG, YarnJobClusterEntrypoint.class.getSimpleName(), args);
	SignalHandler.register(LOG);
	JvmShutdownSafeguard.installAsShutdownHook(LOG);

	Map<String, String> env = System.getenv();

	final String workingDirectory = env.get(ApplicationConstants.Environment.PWD.key());
	Preconditions.checkArgument(
		workingDirectory != null,
		"Working directory variable (%s) not set",
		ApplicationConstants.Environment.PWD.key());

	try {
		YarnEntrypointUtils.logYarnEnvironmentInformation(env, LOG);
	} catch (IOException e) {
		LOG.warn("Could not log YARN environment information.", e);
	}

	Configuration configuration = YarnEntrypointUtils.loadConfiguration(workingDirectory, env, LOG);

	YarnJobClusterEntrypoint yarnJobClusterEntrypoint = new YarnJobClusterEntrypoint(
		configuration,
		workingDirectory);

	ClusterEntrypoint.runClusterEntrypoint(yarnJobClusterEntrypoint);
}
 
Example 26
Source Project: garmadon   Source File: ContainerHeader.java    License: Apache License 2.0 5 votes vote down vote up
private Header.SerializedHeader createCachedHeader() {
    String user = System.getenv(ApplicationConstants.Environment.USER.name());
    String containerIdString = System.getenv(ApplicationConstants.Environment.CONTAINER_ID.name());
    String host = System.getenv(ApplicationConstants.Environment.NM_HOST.name());

    // Get applicationID
    ContainerId containerId = ConverterUtils.toContainerId(containerIdString);
    ApplicationAttemptId appAttemptID = containerId.getApplicationAttemptId();
    ApplicationId appId = appAttemptID.getApplicationId();

    //build the header for the whole application once
    return Header.newBuilder()
            .withId(appId.toString())
            .addTag(Header.Tag.YARN_APPLICATION.name())
            .addTags(System.getProperty("garmadon.tags"))
            .withHostname(host)
            .withApplicationID(appId.toString())
            .withAttemptID(appAttemptID.toString())
            .withUser(user)
            .withContainerID(containerIdString)
            .withPid(HeaderUtils.getPid())
            .withFramework(framework.toString())
            .withComponent(component.name())
            .withExecutorId(executorId)
            .withMainClass(mainClass)
            .buildSerializedHeader();
}
 
Example 27
Source Project: incubator-gobblin   Source File: YarnService.java    License: Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
protected String buildContainerCommand(Container container, String helixInstanceName) {
  String containerProcessName = GobblinYarnTaskRunner.class.getSimpleName();
  StringBuilder containerCommand = new StringBuilder()
      .append(ApplicationConstants.Environment.JAVA_HOME.$()).append("/bin/java")
      .append(" -Xmx").append((int) (container.getResource().getMemory() * this.jvmMemoryXmxRatio) -
          this.jvmMemoryOverheadMbs).append("M")
      .append(" -D").append(GobblinYarnConfigurationKeys.JVM_USER_TIMEZONE_CONFIG).append("=").append(this.containerTimezone)
      .append(" -D").append(GobblinYarnConfigurationKeys.GOBBLIN_YARN_CONTAINER_LOG_DIR_NAME).append("=").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR)
      .append(" -D").append(GobblinYarnConfigurationKeys.GOBBLIN_YARN_CONTAINER_LOG_FILE_NAME).append("=").append(containerProcessName).append(".").append(ApplicationConstants.STDOUT)
      .append(" ").append(JvmUtils.formatJvmArguments(this.containerJvmArgs))
      .append(" ").append(GobblinYarnTaskRunner.class.getName())
      .append(" --").append(GobblinClusterConfigurationKeys.APPLICATION_NAME_OPTION_NAME)
      .append(" ").append(this.applicationName)
      .append(" --").append(GobblinClusterConfigurationKeys.APPLICATION_ID_OPTION_NAME)
      .append(" ").append(this.applicationId)
      .append(" --").append(GobblinClusterConfigurationKeys.HELIX_INSTANCE_NAME_OPTION_NAME)
      .append(" ").append(helixInstanceName);

  if (!Strings.isNullOrEmpty(this.helixInstanceTags)) {
    containerCommand.append(" --").append(GobblinClusterConfigurationKeys.HELIX_INSTANCE_TAGS_OPTION_NAME)
        .append(" ").append(helixInstanceTags);
  }
  return containerCommand.append(" 1>").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR).append(File.separator).append(
        containerProcessName).append(".").append(ApplicationConstants.STDOUT)
      .append(" 2>").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR).append(File.separator).append(
        containerProcessName).append(".").append(ApplicationConstants.STDERR).toString();
}
 
Example 28
Source Project: TensorFlowOnYARN   Source File: LaunchContainerThread.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void run() {
  try {
    Map<String, String> env = Utils.setJavaEnv(appMaster.getConfiguration());
    String current = ApplicationConstants.Environment.LD_LIBRARY_PATH.$$();
    env.put("LD_LIBRARY_PATH", current + ":" + "`pwd`");

    Map<String, Path> files = new HashMap<>();
    files.put(Constants.TF_JAR_NAME, new Path(tfJar));
    files.put(Constants.TF_LIB_NAME, new Path(tfLib));

    FileSystem fs = FileSystem.get(appMaster.getConfiguration());
    Map<String, LocalResource> localResources =
        Utils.makeLocalResources(fs, files);

    String command = makeContainerCommand(
        containerMemory, clusterSpec.toBase64EncodedJsonString(),
        taskInfo.jobName, taskInfo.taskIndex);

    LOG.info("Launching a new container."
        + ", containerId=" + container.getId()
        + ", containerNode=" + container.getNodeId().getHost()
        + ":" + container.getNodeId().getPort()
        + ", containerNodeURI=" + container.getNodeHttpAddress()
        + ", containerResourceMemory="
        + container.getResource().getMemorySize()
        + ", containerResourceVirtualCores="
        + container.getResource().getVirtualCores()
        + ", command: " + command);
    ContainerLaunchContext ctx = ContainerLaunchContext.newInstance(
        localResources, env, Lists.newArrayList(command), null, null, null, null);
    appMaster.addContainer(container);
    appMaster.getNMClientAsync().startContainerAsync(container, ctx);
  } catch (Exception e) {
    throw new RuntimeException(e);
  }
}
 
Example 29
Source Project: tez   Source File: TezClientUtils.java    License: Apache License 2.0 5 votes vote down vote up
@Private
@VisibleForTesting
public static void addLog4jSystemProperties(String logLevel,
    List<String> vargs) {
  vargs.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator");
  vargs.add("-Dlog4j.configuration="
      + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE);
  vargs.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "="
      + ApplicationConstants.LOG_DIR_EXPANSION_VAR);
  vargs.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + logLevel
      + "," + TezConstants.TEZ_CONTAINER_LOGGER_NAME);
}
 
Example 30
Source Project: hadoop   Source File: LinuxContainerExecutor.java    License: Apache License 2.0 5 votes vote down vote up
protected String getContainerExecutorExecutablePath(Configuration conf) {
  String yarnHomeEnvVar =
      System.getenv(ApplicationConstants.Environment.HADOOP_YARN_HOME.key());
  File hadoopBin = new File(yarnHomeEnvVar, "bin");
  String defaultPath =
      new File(hadoopBin, "container-executor").getAbsolutePath();
  return null == conf
      ? defaultPath
      : conf.get(YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH,
      defaultPath);
}