Java Code Examples for org.apache.hadoop.yarn.api.records.ContainerLaunchContext#getEnvironment()

The following examples show how to use org.apache.hadoop.yarn.api.records.ContainerLaunchContext#getEnvironment() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: TestMapReduceChildJVM.java From hadoop with Apache License 2.0

6 votes

@Override
protected ContainerLauncher createContainerLauncher(AppContext context) {
  return new MockContainerLauncher() {
    @Override
    public void handle(ContainerLauncherEvent event) {
      if (event.getType() == EventType.CONTAINER_REMOTE_LAUNCH) {
        ContainerRemoteLaunchEvent launchEvent = (ContainerRemoteLaunchEvent) event;
        ContainerLaunchContext launchContext =
            launchEvent.getContainerLaunchContext();
        String cmdString = launchContext.getCommands().toString();
        LOG.info("launchContext " + cmdString);
        myCommandLine = cmdString;
        cmdEnvironment = launchContext.getEnvironment();
      }
      super.handle(event);
    }
  };
}

Example 2

Source File: StreamAppmaster.java From spring-cloud-deployer-yarn with Apache License 2.0

6 votes

@Override
public ContainerLaunchContext preLaunch(Container container, ContainerLaunchContext context) {
	ContainerCluster containerCluster = findContainerClusterByContainerId(container.getId());
	if (containerCluster == null) {
		return context;
	}
	containerIdMap.put(container.getId(), containerCluster.getId());

	Map<String, String> environment = context.getEnvironment();
	Map<String, String> indexEnv = new HashMap<>();
	indexEnv.putAll(environment);
	Integer reservedIndex;
	synchronized (indexTracker) {
		reservedIndex = indexTracker.reserveIndex(container.getId(), containerCluster);
	}
	indexEnv.put("SPRING_CLOUD_APPLICATION_GUID", container.getId().toString());
	indexEnv.put("SPRING_APPLICATION_INDEX", Integer.toString(reservedIndex));
	indexEnv.put("INSTANCE_INDEX", Integer.toString(reservedIndex));
	context.setEnvironment(indexEnv);
	return context;
}

Example 3

Source File: TestMapReduceChildJVM.java From big-c with Apache License 2.0

6 votes

@Override
protected ContainerLauncher createContainerLauncher(AppContext context) {
  return new MockContainerLauncher() {
    @Override
    public void handle(ContainerLauncherEvent event) {
      if (event.getType() == EventType.CONTAINER_REMOTE_LAUNCH) {
        ContainerRemoteLaunchEvent launchEvent = (ContainerRemoteLaunchEvent) event;
        ContainerLaunchContext launchContext =
            launchEvent.getContainerLaunchContext();
        String cmdString = launchContext.getCommands().toString();
        LOG.info("launchContext " + cmdString);
        myCommandLine = cmdString;
        cmdEnvironment = launchContext.getEnvironment();
      }
      super.handle(event);
    }
  };
}

Example 4

Source File: AMLauncher.java From hadoop with Apache License 2.0

5 votes

private void setupTokens(
    ContainerLaunchContext container, ContainerId containerID)
    throws IOException {
  Map<String, String> environment = container.getEnvironment();
  environment.put(ApplicationConstants.APPLICATION_WEB_PROXY_BASE_ENV,
      application.getWebProxyBase());
  // Set AppSubmitTime and MaxAppAttempts to be consumable by the AM.
  ApplicationId applicationId =
      application.getAppAttemptId().getApplicationId();
  environment.put(
      ApplicationConstants.APP_SUBMIT_TIME_ENV,
      String.valueOf(rmContext.getRMApps()
          .get(applicationId)
          .getSubmitTime()));
  environment.put(ApplicationConstants.MAX_APP_ATTEMPTS_ENV,
      String.valueOf(rmContext.getRMApps().get(
          applicationId).getMaxAppAttempts()));

  Credentials credentials = new Credentials();
  DataInputByteBuffer dibb = new DataInputByteBuffer();
  if (container.getTokens() != null) {
    // TODO: Don't do this kind of checks everywhere.
    dibb.reset(container.getTokens());
    credentials.readTokenStorageStream(dibb);
  }

  // Add AMRMToken
  Token<AMRMTokenIdentifier> amrmToken = createAndSetAMRMToken();
  if (amrmToken != null) {
    credentials.addToken(amrmToken.getService(), amrmToken);
  }
  DataOutputBuffer dob = new DataOutputBuffer();
  credentials.writeTokenStorageToStream(dob);
  container.setTokens(ByteBuffer.wrap(dob.getData(), 0, dob.getLength()));
}

Example 5

Source File: TestYARNRunner.java From hadoop with Apache License 2.0

5 votes

@Test
public void testAMStandardEnv() throws Exception {
  final String ADMIN_LIB_PATH = "foo";
  final String USER_LIB_PATH = "bar";
  final String USER_SHELL = "shell";
  JobConf jobConf = new JobConf();

  jobConf.set(MRJobConfig.MR_AM_ADMIN_USER_ENV, "LD_LIBRARY_PATH=" +
      ADMIN_LIB_PATH);
  jobConf.set(MRJobConfig.MR_AM_ENV, "LD_LIBRARY_PATH="
      + USER_LIB_PATH);
  jobConf.set(MRJobConfig.MAPRED_ADMIN_USER_SHELL, USER_SHELL);

  YARNRunner yarnRunner = new YARNRunner(jobConf);
  ApplicationSubmissionContext appSubCtx =
      buildSubmitContext(yarnRunner, jobConf);

  // make sure PWD is first in the lib path
  ContainerLaunchContext clc = appSubCtx.getAMContainerSpec();
  Map<String, String> env = clc.getEnvironment();
  String libPath = env.get(Environment.LD_LIBRARY_PATH.name());
  assertNotNull("LD_LIBRARY_PATH not set", libPath);
  String cps = jobConf.getBoolean(
      MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM,
      MRConfig.DEFAULT_MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM)
      ? ApplicationConstants.CLASS_PATH_SEPARATOR : File.pathSeparator;
  assertEquals("Bad AM LD_LIBRARY_PATH setting",
      MRApps.crossPlatformifyMREnv(conf, Environment.PWD)
      + cps + ADMIN_LIB_PATH + cps + USER_LIB_PATH, libPath);

  // make sure SHELL is set
  String shell = env.get(Environment.SHELL.name());
  assertNotNull("SHELL not set", shell);
  assertEquals("Bad SHELL setting", USER_SHELL, shell);
}

Example 6

Source File: AMLauncher.java From big-c with Apache License 2.0

5 votes

private void setupTokens(
    ContainerLaunchContext container, ContainerId containerID)
    throws IOException {
  Map<String, String> environment = container.getEnvironment();
  environment.put(ApplicationConstants.APPLICATION_WEB_PROXY_BASE_ENV,
      application.getWebProxyBase());
  // Set AppSubmitTime and MaxAppAttempts to be consumable by the AM.
  ApplicationId applicationId =
      application.getAppAttemptId().getApplicationId();
  environment.put(
      ApplicationConstants.APP_SUBMIT_TIME_ENV,
      String.valueOf(rmContext.getRMApps()
          .get(applicationId)
          .getSubmitTime()));
  environment.put(ApplicationConstants.MAX_APP_ATTEMPTS_ENV,
      String.valueOf(rmContext.getRMApps().get(
          applicationId).getMaxAppAttempts()));

  Credentials credentials = new Credentials();
  DataInputByteBuffer dibb = new DataInputByteBuffer();
  if (container.getTokens() != null) {
    // TODO: Don't do this kind of checks everywhere.
    dibb.reset(container.getTokens());
    credentials.readTokenStorageStream(dibb);
  }

  // Add AMRMToken
  Token<AMRMTokenIdentifier> amrmToken = createAndSetAMRMToken();
  if (amrmToken != null) {
    credentials.addToken(amrmToken.getService(), amrmToken);
  }
  DataOutputBuffer dob = new DataOutputBuffer();
  credentials.writeTokenStorageToStream(dob);
  container.setTokens(ByteBuffer.wrap(dob.getData(), 0, dob.getLength()));
}

Example 7

Source File: TestYARNRunner.java From big-c with Apache License 2.0

5 votes

@Test
public void testAMStandardEnv() throws Exception {
  final String ADMIN_LIB_PATH = "foo";
  final String USER_LIB_PATH = "bar";
  final String USER_SHELL = "shell";
  JobConf jobConf = new JobConf();

  jobConf.set(MRJobConfig.MR_AM_ADMIN_USER_ENV, "LD_LIBRARY_PATH=" +
      ADMIN_LIB_PATH);
  jobConf.set(MRJobConfig.MR_AM_ENV, "LD_LIBRARY_PATH="
      + USER_LIB_PATH);
  jobConf.set(MRJobConfig.MAPRED_ADMIN_USER_SHELL, USER_SHELL);

  YARNRunner yarnRunner = new YARNRunner(jobConf);
  ApplicationSubmissionContext appSubCtx =
      buildSubmitContext(yarnRunner, jobConf);

  // make sure PWD is first in the lib path
  ContainerLaunchContext clc = appSubCtx.getAMContainerSpec();
  Map<String, String> env = clc.getEnvironment();
  String libPath = env.get(Environment.LD_LIBRARY_PATH.name());
  assertNotNull("LD_LIBRARY_PATH not set", libPath);
  String cps = jobConf.getBoolean(
      MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM,
      MRConfig.DEFAULT_MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM)
      ? ApplicationConstants.CLASS_PATH_SEPARATOR : File.pathSeparator;
  assertEquals("Bad AM LD_LIBRARY_PATH setting",
      MRApps.crossPlatformifyMREnv(conf, Environment.PWD)
      + cps + ADMIN_LIB_PATH + cps + USER_LIB_PATH, libPath);

  // make sure SHELL is set
  String shell = env.get(Environment.SHELL.name());
  assertNotNull("SHELL not set", shell);
  assertEquals("Bad SHELL setting", USER_SHELL, shell);
}

Example 8

Source File: ApplicationMaster.java From ignite with Apache License 2.0

4 votes

/** {@inheritDoc} */
@Override public synchronized void onContainersAllocated(List<Container> conts) {
    for (Container c : conts) {
        if (checkContainer(c)) {
            log.log(Level.INFO, "Container {0} allocated", c.getId());

            try {
                ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class);

                if (UserGroupInformation.isSecurityEnabled())
                    // Set the tokens to the newly allocated container:
                    ctx.setTokens(allTokens.duplicate());

                Map<String, String> env = new HashMap<>(ctx.getEnvironment());

                Map<String, String> systemEnv = System.getenv();

                for (String key : systemEnv.keySet()) {
                    if (key.matches("^IGNITE_[_0-9A-Z]+$"))
                        env.put(key, systemEnv.get(key));
                }

                env.put("IGNITE_TCP_DISCOVERY_ADDRESSES", getAddress(c.getNodeId().getHost()));

                if (props.jvmOpts() != null && !props.jvmOpts().isEmpty())
                    env.put("JVM_OPTS", props.jvmOpts());

                ctx.setEnvironment(env);

                Map<String, LocalResource> resources = new HashMap<>();

                resources.put("ignite", IgniteYarnUtils.setupFile(ignitePath, fs, LocalResourceType.ARCHIVE));
                resources.put("ignite-config.xml", IgniteYarnUtils.setupFile(cfgPath, fs, LocalResourceType.FILE));

                if (props.licencePath() != null)
                    resources.put("gridgain-license.xml",
                        IgniteYarnUtils.setupFile(new Path(props.licencePath()), fs, LocalResourceType.FILE));

                if (props.userLibs() != null)
                    resources.put("libs", IgniteYarnUtils.setupFile(new Path(props.userLibs()), fs,
                        LocalResourceType.FILE));

                ctx.setLocalResources(resources);

                ctx.setCommands(
                    Collections.singletonList(
                        (props.licencePath() != null ? "cp gridgain-license.xml ./ignite/*/ || true && " : "")
                        + "cp -r ./libs/* ./ignite/*/libs/ || true && "
                        + "./ignite/*/bin/ignite.sh "
                        + "./ignite-config.xml"
                        + " -J-Xmx" + ((int)props.memoryPerNode()) + "m"
                        + " -J-Xms" + ((int)props.memoryPerNode()) + "m"
                        + IgniteYarnUtils.YARN_LOG_OUT
                    ));

                log.log(Level.INFO, "Launching container: {0}.", c.getId());

                nmClient.startContainer(c, ctx);

                containers.put(c.getId(),
                    new IgniteContainer(
                        c.getId(),
                        c.getNodeId(),
                        c.getResource().getVirtualCores(),
                        c.getResource().getMemory()));
            }
            catch (Exception ex) {
                log.log(Level.WARNING, "Error launching container " + c.getId(), ex);
            }
        }
        else {
            log.log(Level.WARNING, "Container {0} check failed. Releasing...", c.getId());

            rmClient.releaseAssignedContainer(c.getId());
        }
    }
}

Example 9

Source File: AMContainerHelpers.java From incubator-tez with Apache License 2.0

4 votes

@VisibleForTesting
public static ContainerLaunchContext createContainerLaunchContext(
    TezDAGID tezDAGID,
    Map<ApplicationAccessType, String> acls,
    ContainerId containerId,
    Map<String, LocalResource> localResources,
    Map<String, String> vertexEnv,
    String javaOpts,
    InetSocketAddress taskAttemptListenerAddress, Credentials credentials,
    AppContext appContext, Resource containerResource,
    Configuration conf) {

  ContainerLaunchContext commonContainerSpec = null;
  synchronized (commonContainerSpecLock) {
    if (!commonContainerSpecs.containsKey(tezDAGID)) {
      commonContainerSpec =
          createCommonContainerLaunchContext(acls, credentials);
      commonContainerSpecs.put(tezDAGID, commonContainerSpec);
    } else {
      commonContainerSpec = commonContainerSpecs.get(tezDAGID);
    }

    // Ensure that we remove container specs for previous AMs to reduce
    // memory footprint
    if (lastDAGID == null) {
      lastDAGID = tezDAGID;
    } else if (!lastDAGID.equals(tezDAGID)) {
      commonContainerSpecs.remove(lastDAGID);
      lastDAGID = tezDAGID;
    }
  }

  // Fill in the fields needed per-container that are missing in the common
  // spec.
  Map<String, LocalResource> lResources =
      new TreeMap<String, LocalResource>();
  lResources.putAll(commonContainerSpec.getLocalResources());
  lResources.putAll(localResources);

  // Setup environment by cloning from common env.
  Map<String, String> env = commonContainerSpec.getEnvironment();
  Map<String, String> myEnv = new HashMap<String, String>(env.size());
  myEnv.putAll(env);
  myEnv.putAll(vertexEnv);

  String modifiedJavaOpts = TezClientUtils.maybeAddDefaultMemoryJavaOpts(javaOpts,
      containerResource, conf.getDouble(TezConfiguration.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION,
          TezConfiguration.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION_DEFAULT));
  if (LOG.isDebugEnabled()) {
    if (!modifiedJavaOpts.equals(javaOpts)) {
      LOG.debug("Modified java opts for container"
        + ", containerId=" + containerId
        + ", originalJavaOpts=" + javaOpts
        + ", modifiedJavaOpts=" + modifiedJavaOpts);
    }
  }

  List<String> commands = TezRuntimeChildJVM.getVMCommand(
      taskAttemptListenerAddress, containerId.toString(),
      appContext.getApplicationID().toString(),
      appContext.getApplicationAttemptId().getAttemptId(), modifiedJavaOpts);

  // Duplicate the ByteBuffers for access by multiple containers.
  Map<String, ByteBuffer> myServiceData = new HashMap<String, ByteBuffer>();
  for (Entry<String, ByteBuffer> entry : commonContainerSpec.getServiceData()
      .entrySet()) {
    myServiceData.put(entry.getKey(), entry.getValue().duplicate());
  }

  // Construct the actual Container
  ContainerLaunchContext container =
      ContainerLaunchContext.newInstance(lResources, myEnv, commands,
          myServiceData, commonContainerSpec.getTokens().duplicate(), acls);

  return container;
}

Example 10

Source File: AMContainerHelpers.java From tez with Apache License 2.0

4 votes

@VisibleForTesting
public static ContainerLaunchContext createContainerLaunchContext(
    TezDAGID tezDAGID,
    Map<ApplicationAccessType, String> acls,
    ContainerId containerId,
    Map<String, LocalResource> localResources,
    Map<String, String> vertexEnv,
    String javaOpts,
    InetSocketAddress taskAttemptListenerAddress, Credentials credentials,
    AppContext appContext, Resource containerResource,
    Configuration conf, String auxiliaryService) {

  ContainerLaunchContext commonContainerSpec = null;
  synchronized (commonContainerSpecLock) {
    if (!commonContainerSpecs.containsKey(tezDAGID)) {
      commonContainerSpec =
          createCommonContainerLaunchContext(acls, credentials, auxiliaryService);
      commonContainerSpecs.put(tezDAGID, commonContainerSpec);
    } else {
      commonContainerSpec = commonContainerSpecs.get(tezDAGID);
    }

    // Ensure that we remove container specs for previous AMs to reduce
    // memory footprint
    if (lastDAGID == null) {
      lastDAGID = tezDAGID;
    } else if (!lastDAGID.equals(tezDAGID)) {
      commonContainerSpecs.remove(lastDAGID);
      lastDAGID = tezDAGID;
    }
  }

  // Setup environment by cloning from common env.
  Map<String, String> env = commonContainerSpec.getEnvironment();
  Map<String, String> myEnv = new HashMap<String, String>(env.size());
  myEnv.putAll(env);
  myEnv.putAll(vertexEnv);

  String modifiedJavaOpts = TezClientUtils.maybeAddDefaultMemoryJavaOpts(javaOpts,
      containerResource, conf.getDouble(TezConfiguration.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION,
          TezConfiguration.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION_DEFAULT));
  if (LOG.isDebugEnabled()) {
    if (!modifiedJavaOpts.equals(javaOpts)) {
      LOG.debug("Modified java opts for container"
        + ", containerId=" + containerId
        + ", originalJavaOpts=" + javaOpts
        + ", modifiedJavaOpts=" + modifiedJavaOpts);
    }
  }

  List<String> commands = TezRuntimeChildJVM.getVMCommand(
      taskAttemptListenerAddress, containerId.toString(),
      appContext.getApplicationID().toString(),
      appContext.getApplicationAttemptId().getAttemptId(), modifiedJavaOpts);

  // Duplicate the ByteBuffers for access by multiple containers.
  Map<String, ByteBuffer> myServiceData = new HashMap<String, ByteBuffer>();
  for (Entry<String, ByteBuffer> entry : commonContainerSpec.getServiceData()
      .entrySet()) {
    myServiceData.put(entry.getKey(), entry.getValue().duplicate());
  }

  // Construct the actual Container
  ContainerLaunchContext container =
      ContainerLaunchContext.newInstance(localResources, myEnv, commands,
          myServiceData, commonContainerSpec.getTokens().duplicate(), acls);

  return container;
}