Java Code Examples for org.apache.hadoop.yarn.api.records.ContainerLaunchContext#newInstance()

The following examples show how to use org.apache.hadoop.yarn.api.records.ContainerLaunchContext#newInstance() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: LaunchCluster.java    From TensorFlowOnYARN with Apache License 2.0 6 votes vote down vote up
public boolean run() throws Exception {
  YarnClientApplication app = createApplication();
  ApplicationId appId = app.getNewApplicationResponse().getApplicationId();

  // Copy the application jar to the filesystem
  FileSystem fs = FileSystem.get(conf);
  String appIdStr = appId.toString();
  Path dstJarPath = Utils.copyLocalFileToDfs(fs, appIdStr, new Path(tfJar), Constants.TF_JAR_NAME);
  Path dstLibPath = Utils.copyLocalFileToDfs(fs, appIdStr, new Path(tfLib),
      Constants.TF_LIB_NAME);
  Map<String, Path> files = new HashMap<>();
  files.put(Constants.TF_JAR_NAME, dstJarPath);
  Map<String, LocalResource> localResources = Utils.makeLocalResources(fs, files);
  Map<String, String> javaEnv = Utils.setJavaEnv(conf);
  String command = makeAppMasterCommand(dstLibPath.toString(), dstJarPath.toString());
  LOG.info("Make ApplicationMaster command: " + command);
  ContainerLaunchContext launchContext = ContainerLaunchContext.newInstance(
      localResources, javaEnv, Lists.newArrayList(command), null, null, null);
  Resource resource = Resource.newInstance(amMemory, amVCores);
  submitApplication(app, appName, launchContext, resource, amQueue);
  return awaitApplication(appId);
}
 
Example 2
Source File: LaunchContainerThread.java    From TensorFlowOnYARN with Apache License 2.0 5 votes vote down vote up
@Override
public void run() {
  try {
    Map<String, String> env = Utils.setJavaEnv(appMaster.getConfiguration());
    String current = ApplicationConstants.Environment.LD_LIBRARY_PATH.$$();
    env.put("LD_LIBRARY_PATH", current + ":" + "`pwd`");

    Map<String, Path> files = new HashMap<>();
    files.put(Constants.TF_JAR_NAME, new Path(tfJar));
    files.put(Constants.TF_LIB_NAME, new Path(tfLib));

    FileSystem fs = FileSystem.get(appMaster.getConfiguration());
    Map<String, LocalResource> localResources =
        Utils.makeLocalResources(fs, files);

    String command = makeContainerCommand(
        containerMemory, clusterSpec.toBase64EncodedJsonString(),
        taskInfo.jobName, taskInfo.taskIndex);

    LOG.info("Launching a new container."
        + ", containerId=" + container.getId()
        + ", containerNode=" + container.getNodeId().getHost()
        + ":" + container.getNodeId().getPort()
        + ", containerNodeURI=" + container.getNodeHttpAddress()
        + ", containerResourceMemory="
        + container.getResource().getMemorySize()
        + ", containerResourceVirtualCores="
        + container.getResource().getVirtualCores()
        + ", command: " + command);
    ContainerLaunchContext ctx = ContainerLaunchContext.newInstance(
        localResources, env, Lists.newArrayList(command), null, null, null, null);
    appMaster.addContainer(container);
    appMaster.getNMClientAsync().startContainerAsync(container, ctx);
  } catch (Exception e) {
    throw new RuntimeException(e);
  }
}
 
Example 3
Source File: QueueACLsTestBase.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private ApplicationId submitAppAndGetAppId(String submitter,
    String queueName, boolean setupACLs) throws Exception {

  GetNewApplicationRequest newAppRequest =
      GetNewApplicationRequest.newInstance();

  ApplicationClientProtocol submitterClient = getRMClientForUser(submitter);
  ApplicationId applicationId =
      submitterClient.getNewApplication(newAppRequest).getApplicationId();

  Resource resource = BuilderUtils.newResource(1024, 1);
  Map<ApplicationAccessType, String> acls = createACLs(submitter, setupACLs);
  ContainerLaunchContext amContainerSpec =
      ContainerLaunchContext.newInstance(null, null, null, null, null, acls);

  ApplicationSubmissionContext appSubmissionContext =
      ApplicationSubmissionContext.newInstance(applicationId,
        "applicationName", queueName, null, amContainerSpec, false, true, 1,
        resource, "applicationType");
  appSubmissionContext.setApplicationId(applicationId);
  appSubmissionContext.setQueue(queueName);

  SubmitApplicationRequest submitRequest =
      SubmitApplicationRequest.newInstance(appSubmissionContext);
  submitterClient.submitApplication(submitRequest);
  resourceManager.waitForState(applicationId, RMAppState.ACCEPTED);
  return applicationId;
}
 
Example 4
Source File: TestDelegationTokenRenewer.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test(timeout=20000)
public void testAppSubmissionWithInvalidDelegationToken() throws Exception {
  Configuration conf = new Configuration();
  conf.set(
      CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
      "kerberos");
  UserGroupInformation.setConfiguration(conf);
  MockRM rm = new MockRM(conf) {
    @Override
    protected void doSecureLogin() throws IOException {
      // Skip the login.
    }
  };
  ByteBuffer tokens = ByteBuffer.wrap("BOGUS".getBytes()); 
  ContainerLaunchContext amContainer =
      ContainerLaunchContext.newInstance(
          new HashMap<String, LocalResource>(), new HashMap<String, String>(),
          new ArrayList<String>(), new HashMap<String, ByteBuffer>(), tokens,
          new HashMap<ApplicationAccessType, String>());
  ApplicationSubmissionContext appSubContext =
      ApplicationSubmissionContext.newInstance(
          ApplicationId.newInstance(1234121, 0),
          "BOGUS", "default", Priority.UNDEFINED, amContainer, false,
          true, 1, Resource.newInstance(1024, 1, 1), "BOGUS");
  SubmitApplicationRequest request =
      SubmitApplicationRequest.newInstance(appSubContext);
  try {
    rm.getClientRMService().submitApplication(request);
    fail("Error was excepted.");
  } catch (YarnException e) {
    Assert.assertTrue(e.getMessage().contains(
        "Bad header found in token storage"));
  }
}
 
Example 5
Source File: QueueACLsTestBase.java    From big-c with Apache License 2.0 5 votes vote down vote up
private ApplicationId submitAppAndGetAppId(String submitter,
    String queueName, boolean setupACLs) throws Exception {

  GetNewApplicationRequest newAppRequest =
      GetNewApplicationRequest.newInstance();

  ApplicationClientProtocol submitterClient = getRMClientForUser(submitter);
  ApplicationId applicationId =
      submitterClient.getNewApplication(newAppRequest).getApplicationId();

  Resource resource = BuilderUtils.newResource(1024, 1);
  Map<ApplicationAccessType, String> acls = createACLs(submitter, setupACLs);
  ContainerLaunchContext amContainerSpec =
      ContainerLaunchContext.newInstance(null, null, null, null, null, acls);

  ApplicationSubmissionContext appSubmissionContext =
      ApplicationSubmissionContext.newInstance(applicationId,
        "applicationName", queueName, null, amContainerSpec, false, true, 1,
        resource, "applicationType");
  appSubmissionContext.setApplicationId(applicationId);
  appSubmissionContext.setQueue(queueName);

  SubmitApplicationRequest submitRequest =
      SubmitApplicationRequest.newInstance(appSubmissionContext);
  submitterClient.submitApplication(submitRequest);
  resourceManager.waitForState(applicationId, RMAppState.ACCEPTED);
  return applicationId;
}
 
Example 6
Source File: TestDelegationTokenRenewer.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test(timeout=20000)
public void testAppSubmissionWithInvalidDelegationToken() throws Exception {
  Configuration conf = new Configuration();
  conf.set(
      CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
      "kerberos");
  UserGroupInformation.setConfiguration(conf);
  MockRM rm = new MockRM(conf) {
    @Override
    protected void doSecureLogin() throws IOException {
      // Skip the login.
    }
  };
  ByteBuffer tokens = ByteBuffer.wrap("BOGUS".getBytes()); 
  ContainerLaunchContext amContainer =
      ContainerLaunchContext.newInstance(
          new HashMap<String, LocalResource>(), new HashMap<String, String>(),
          new ArrayList<String>(), new HashMap<String, ByteBuffer>(), tokens,
          new HashMap<ApplicationAccessType, String>());
  ApplicationSubmissionContext appSubContext =
      ApplicationSubmissionContext.newInstance(
          ApplicationId.newInstance(1234121, 0),
          "BOGUS", "default", Priority.UNDEFINED, amContainer, false,
          true, 1, Resource.newInstance(1024, 1), "BOGUS");
  SubmitApplicationRequest request =
      SubmitApplicationRequest.newInstance(appSubContext);
  try {
    rm.getClientRMService().submitApplication(request);
    fail("Error was excepted.");
  } catch (YarnException e) {
    Assert.assertTrue(e.getMessage().contains(
        "Bad header found in token storage"));
  }
}
 
Example 7
Source File: TaskAttemptImpl.java    From hadoop with Apache License 2.0 4 votes vote down vote up
static ContainerLaunchContext createContainerLaunchContext(
    Map<ApplicationAccessType, String> applicationACLs,
    Configuration conf, Token<JobTokenIdentifier> jobToken, Task remoteTask,
    final org.apache.hadoop.mapred.JobID oldJobId,
    WrappedJvmID jvmID,
    TaskAttemptListener taskAttemptListener,
    Credentials credentials) {

  synchronized (commonContainerSpecLock) {
    if (commonContainerSpec == null) {
      commonContainerSpec = createCommonContainerLaunchContext(
          applicationACLs, conf, jobToken, oldJobId, credentials);
    }
  }

  // Fill in the fields needed per-container that are missing in the common
  // spec.

  boolean userClassesTakesPrecedence =
    conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, false);

  // Setup environment by cloning from common env.
  Map<String, String> env = commonContainerSpec.getEnvironment();
  Map<String, String> myEnv = new HashMap<String, String>(env.size());
  myEnv.putAll(env);
  if (userClassesTakesPrecedence) {
    myEnv.put(Environment.CLASSPATH_PREPEND_DISTCACHE.name(), "true");
  }
  MapReduceChildJVM.setVMEnv(myEnv, remoteTask);

  // Set up the launch command
  List<String> commands = MapReduceChildJVM.getVMCommand(
      taskAttemptListener.getAddress(), remoteTask, jvmID);

  // Duplicate the ByteBuffers for access by multiple containers.
  Map<String, ByteBuffer> myServiceData = new HashMap<String, ByteBuffer>();
  for (Entry<String, ByteBuffer> entry : commonContainerSpec
              .getServiceData().entrySet()) {
    myServiceData.put(entry.getKey(), entry.getValue().duplicate());
  }

  // Construct the actual Container
  ContainerLaunchContext container = ContainerLaunchContext.newInstance(
      commonContainerSpec.getLocalResources(), myEnv, commands,
      myServiceData, commonContainerSpec.getTokens().duplicate(),
      applicationACLs);

  return container;
}
 
Example 8
Source File: TaskAttemptImpl.java    From big-c with Apache License 2.0 4 votes vote down vote up
static ContainerLaunchContext createContainerLaunchContext(
    Map<ApplicationAccessType, String> applicationACLs,
    Configuration conf, Token<JobTokenIdentifier> jobToken, Task remoteTask,
    final org.apache.hadoop.mapred.JobID oldJobId,
    WrappedJvmID jvmID,
    TaskAttemptListener taskAttemptListener,
    Credentials credentials) {

  synchronized (commonContainerSpecLock) {
    if (commonContainerSpec == null) {
      commonContainerSpec = createCommonContainerLaunchContext(
          applicationACLs, conf, jobToken, oldJobId, credentials);
    }
  }

  // Fill in the fields needed per-container that are missing in the common
  // spec.

  boolean userClassesTakesPrecedence =
    conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, false);

  // Setup environment by cloning from common env.
  Map<String, String> env = commonContainerSpec.getEnvironment();
  Map<String, String> myEnv = new HashMap<String, String>(env.size());
  myEnv.putAll(env);
  if (userClassesTakesPrecedence) {
    myEnv.put(Environment.CLASSPATH_PREPEND_DISTCACHE.name(), "true");
  }
  MapReduceChildJVM.setVMEnv(myEnv, remoteTask);

  // Set up the launch command
  List<String> commands = MapReduceChildJVM.getVMCommand(
      taskAttemptListener.getAddress(), remoteTask, jvmID);

  // Duplicate the ByteBuffers for access by multiple containers.
  Map<String, ByteBuffer> myServiceData = new HashMap<String, ByteBuffer>();
  for (Entry<String, ByteBuffer> entry : commonContainerSpec
              .getServiceData().entrySet()) {
    myServiceData.put(entry.getKey(), entry.getValue().duplicate());
  }

  // Construct the actual Container
  ContainerLaunchContext container = ContainerLaunchContext.newInstance(
      commonContainerSpec.getLocalResources(), myEnv, commands,
      myServiceData, commonContainerSpec.getTokens().duplicate(),
      applicationACLs);

  return container;
}
 
Example 9
Source File: AMContainerHelpers.java    From incubator-tez with Apache License 2.0 4 votes vote down vote up
/**
 * Create the common {@link ContainerLaunchContext} for all attempts.
 *
 * @param applicationACLs
 */
private static ContainerLaunchContext createCommonContainerLaunchContext(
    Map<ApplicationAccessType, String> applicationACLs,
    Credentials credentials) {

  // Application resources
  Map<String, LocalResource> localResources =
      new HashMap<String, LocalResource>();

  // Application environment
  Map<String, String> environment = new HashMap<String, String>();

  // Service data
  Map<String, ByteBuffer> serviceData = new HashMap<String, ByteBuffer>();

  // Tokens
  
  // Setup up task credentials buffer
  ByteBuffer containerCredentialsBuffer = ByteBuffer.wrap(new byte[] {});
  try {
    Credentials containerCredentials = new Credentials();
    
    // All Credentials need to be set so that YARN can localize the resources
    // correctly, even though they may not be used by all tasks which will run
    // on this container.

    LOG.info("Adding #" + credentials.numberOfTokens() + " tokens and #"
        + credentials.numberOfSecretKeys() + " secret keys for NM use for launching container");
    containerCredentials.addAll(credentials);

    DataOutputBuffer containerTokens_dob = new DataOutputBuffer();
    containerCredentials.writeTokenStorageToStream(containerTokens_dob);
    containerCredentialsBuffer = ByteBuffer.wrap(containerTokens_dob.getData(), 0,
        containerTokens_dob.getLength());

    // Add shuffle token
    LOG.info("Putting shuffle token in serviceData");
    serviceData.put(TezConfiguration.TEZ_SHUFFLE_HANDLER_SERVICE_ID,
        serializeServiceData(TokenCache.getSessionToken(containerCredentials)));
  } catch (IOException e) {
    throw new TezUncheckedException(e);
  }
  // Construct the actual Container
  // The null fields are per-container and will be constructed for each
  // container separately.
  ContainerLaunchContext container =
      ContainerLaunchContext.newInstance(localResources, environment, null,
          serviceData, containerCredentialsBuffer, applicationACLs);
  return container;
}
 
Example 10
Source File: AMContainerHelpers.java    From incubator-tez with Apache License 2.0 4 votes vote down vote up
@VisibleForTesting
public static ContainerLaunchContext createContainerLaunchContext(
    TezDAGID tezDAGID,
    Map<ApplicationAccessType, String> acls,
    ContainerId containerId,
    Map<String, LocalResource> localResources,
    Map<String, String> vertexEnv,
    String javaOpts,
    InetSocketAddress taskAttemptListenerAddress, Credentials credentials,
    AppContext appContext, Resource containerResource,
    Configuration conf) {

  ContainerLaunchContext commonContainerSpec = null;
  synchronized (commonContainerSpecLock) {
    if (!commonContainerSpecs.containsKey(tezDAGID)) {
      commonContainerSpec =
          createCommonContainerLaunchContext(acls, credentials);
      commonContainerSpecs.put(tezDAGID, commonContainerSpec);
    } else {
      commonContainerSpec = commonContainerSpecs.get(tezDAGID);
    }

    // Ensure that we remove container specs for previous AMs to reduce
    // memory footprint
    if (lastDAGID == null) {
      lastDAGID = tezDAGID;
    } else if (!lastDAGID.equals(tezDAGID)) {
      commonContainerSpecs.remove(lastDAGID);
      lastDAGID = tezDAGID;
    }
  }

  // Fill in the fields needed per-container that are missing in the common
  // spec.
  Map<String, LocalResource> lResources =
      new TreeMap<String, LocalResource>();
  lResources.putAll(commonContainerSpec.getLocalResources());
  lResources.putAll(localResources);

  // Setup environment by cloning from common env.
  Map<String, String> env = commonContainerSpec.getEnvironment();
  Map<String, String> myEnv = new HashMap<String, String>(env.size());
  myEnv.putAll(env);
  myEnv.putAll(vertexEnv);

  String modifiedJavaOpts = TezClientUtils.maybeAddDefaultMemoryJavaOpts(javaOpts,
      containerResource, conf.getDouble(TezConfiguration.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION,
          TezConfiguration.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION_DEFAULT));
  if (LOG.isDebugEnabled()) {
    if (!modifiedJavaOpts.equals(javaOpts)) {
      LOG.debug("Modified java opts for container"
        + ", containerId=" + containerId
        + ", originalJavaOpts=" + javaOpts
        + ", modifiedJavaOpts=" + modifiedJavaOpts);
    }
  }

  List<String> commands = TezRuntimeChildJVM.getVMCommand(
      taskAttemptListenerAddress, containerId.toString(),
      appContext.getApplicationID().toString(),
      appContext.getApplicationAttemptId().getAttemptId(), modifiedJavaOpts);

  // Duplicate the ByteBuffers for access by multiple containers.
  Map<String, ByteBuffer> myServiceData = new HashMap<String, ByteBuffer>();
  for (Entry<String, ByteBuffer> entry : commonContainerSpec.getServiceData()
      .entrySet()) {
    myServiceData.put(entry.getKey(), entry.getValue().duplicate());
  }

  // Construct the actual Container
  ContainerLaunchContext container =
      ContainerLaunchContext.newInstance(lResources, myEnv, commands,
          myServiceData, commonContainerSpec.getTokens().duplicate(), acls);

  return container;
}
 
Example 11
Source File: AMContainerHelpers.java    From tez with Apache License 2.0 4 votes vote down vote up
/**
 * Create the common {@link ContainerLaunchContext} for all attempts.
 *
 * @param applicationACLs
 * @param auxiliaryService
 */
private static ContainerLaunchContext createCommonContainerLaunchContext(
    Map<ApplicationAccessType, String> applicationACLs,
    Credentials credentials, String auxiliaryService) {

  // Application environment
  Map<String, String> environment = new HashMap<String, String>();

  // Service data
  Map<String, ByteBuffer> serviceData = new HashMap<String, ByteBuffer>();

  // Tokens
  
  // Setup up task credentials buffer
  ByteBuffer containerCredentialsBuffer = ByteBuffer.wrap(new byte[] {});
  try {
    Credentials containerCredentials = new Credentials();
    
    // All Credentials need to be set so that YARN can localize the resources
    // correctly, even though they may not be used by all tasks which will run
    // on this container.

    if (LOG.isDebugEnabled()) {
      LOG.debug("Adding #" + credentials.numberOfTokens() + " tokens and #"
          + credentials.numberOfSecretKeys() + " secret keys for NM use for launching container in common CLC");
    }
    containerCredentials.addAll(credentials);

    DataOutputBuffer containerTokens_dob = new DataOutputBuffer();
    containerCredentials.writeTokenStorageToStream(containerTokens_dob);
    containerCredentialsBuffer = ByteBuffer.wrap(containerTokens_dob.getData(), 0,
        containerTokens_dob.getLength());

    // Add shuffle token
    if (LOG.isDebugEnabled()) {
      LOG.debug("Putting shuffle token in serviceData in common CLC");
    }
    serviceData.put(auxiliaryService,
        TezCommonUtils.serializeServiceData(TokenCache.getSessionToken(containerCredentials)));
  } catch (IOException e) {
    throw new TezUncheckedException(e);
  }
  // Construct the actual Container
  // The null fields are per-container and will be constructed for each
  // container separately.
  ContainerLaunchContext container =
      ContainerLaunchContext.newInstance(null, environment, null,
          serviceData, containerCredentialsBuffer, applicationACLs);
  return container;
}
 
Example 12
Source File: AMContainerHelpers.java    From tez with Apache License 2.0 4 votes vote down vote up
@VisibleForTesting
public static ContainerLaunchContext createContainerLaunchContext(
    TezDAGID tezDAGID,
    Map<ApplicationAccessType, String> acls,
    ContainerId containerId,
    Map<String, LocalResource> localResources,
    Map<String, String> vertexEnv,
    String javaOpts,
    InetSocketAddress taskAttemptListenerAddress, Credentials credentials,
    AppContext appContext, Resource containerResource,
    Configuration conf, String auxiliaryService) {

  ContainerLaunchContext commonContainerSpec = null;
  synchronized (commonContainerSpecLock) {
    if (!commonContainerSpecs.containsKey(tezDAGID)) {
      commonContainerSpec =
          createCommonContainerLaunchContext(acls, credentials, auxiliaryService);
      commonContainerSpecs.put(tezDAGID, commonContainerSpec);
    } else {
      commonContainerSpec = commonContainerSpecs.get(tezDAGID);
    }

    // Ensure that we remove container specs for previous AMs to reduce
    // memory footprint
    if (lastDAGID == null) {
      lastDAGID = tezDAGID;
    } else if (!lastDAGID.equals(tezDAGID)) {
      commonContainerSpecs.remove(lastDAGID);
      lastDAGID = tezDAGID;
    }
  }

  // Setup environment by cloning from common env.
  Map<String, String> env = commonContainerSpec.getEnvironment();
  Map<String, String> myEnv = new HashMap<String, String>(env.size());
  myEnv.putAll(env);
  myEnv.putAll(vertexEnv);

  String modifiedJavaOpts = TezClientUtils.maybeAddDefaultMemoryJavaOpts(javaOpts,
      containerResource, conf.getDouble(TezConfiguration.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION,
          TezConfiguration.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION_DEFAULT));
  if (LOG.isDebugEnabled()) {
    if (!modifiedJavaOpts.equals(javaOpts)) {
      LOG.debug("Modified java opts for container"
        + ", containerId=" + containerId
        + ", originalJavaOpts=" + javaOpts
        + ", modifiedJavaOpts=" + modifiedJavaOpts);
    }
  }

  List<String> commands = TezRuntimeChildJVM.getVMCommand(
      taskAttemptListenerAddress, containerId.toString(),
      appContext.getApplicationID().toString(),
      appContext.getApplicationAttemptId().getAttemptId(), modifiedJavaOpts);

  // Duplicate the ByteBuffers for access by multiple containers.
  Map<String, ByteBuffer> myServiceData = new HashMap<String, ByteBuffer>();
  for (Entry<String, ByteBuffer> entry : commonContainerSpec.getServiceData()
      .entrySet()) {
    myServiceData.put(entry.getKey(), entry.getValue().duplicate());
  }

  // Construct the actual Container
  ContainerLaunchContext container =
      ContainerLaunchContext.newInstance(localResources, myEnv, commands,
          myServiceData, commonContainerSpec.getTokens().duplicate(), acls);

  return container;
}