Java Code Examples for org.apache.hadoop.yarn.api.records.ContainerLaunchContext

The following examples show how to use org.apache.hadoop.yarn.api.records.ContainerLaunchContext. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: big-c   Source File: NMClientAsyncImpl.java    License: Apache License 2.0 6 votes vote down vote up
public void startContainerAsync(
    Container container, ContainerLaunchContext containerLaunchContext) {
  if (containers.putIfAbsent(container.getId(),
      new StatefulContainer(this, container.getId())) != null) {
    callbackHandler.onStartContainerError(container.getId(),
        RPCUtil.getRemoteException("Container " + container.getId() +
            " is already started or scheduled to start"));
  }
  try {
    events.put(new StartContainerEvent(container, containerLaunchContext));
  } catch (InterruptedException e) {
    LOG.warn("Exception when scheduling the event of starting Container " +
        container.getId());
    callbackHandler.onStartContainerError(container.getId(), e);
  }
}
 
Example 2
Source Project: zeppelin   Source File: YarnRemoteInterpreterProcess.java    License: Apache License 2.0 6 votes vote down vote up
private ApplicationSubmissionContext createApplicationSubmissionContext(
        ApplicationSubmissionContext appContext) throws Exception {

  setResources(appContext);
  setPriority(appContext);
  setQueue(appContext);
  appContext.setApplicationId(appId);
  setApplicationName(appContext);
  appContext.setApplicationType("ZEPPELIN INTERPRETER");
  appContext.setMaxAppAttempts(1);

  ContainerLaunchContext amContainer = setUpAMLaunchContext();
  appContext.setAMContainerSpec(amContainer);
  appContext.setCancelTokensWhenComplete(true);
  return appContext;
}
 
Example 3
Source Project: hadoop   Source File: TestContainerManagerSecurity.java    License: Apache License 2.0 6 votes vote down vote up
private void startContainer(final YarnRPC rpc,
    org.apache.hadoop.yarn.api.records.Token nmToken,
    org.apache.hadoop.yarn.api.records.Token containerToken,
    NodeId nodeId, String user) throws Exception {

  ContainerLaunchContext context =
      Records.newRecord(ContainerLaunchContext.class);
  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(context,containerToken);
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  ContainerManagementProtocol proxy = null;
  try {
    proxy = getContainerManagementProtocolProxy(rpc, nmToken, nodeId, user);
    StartContainersResponse response = proxy.startContainers(allRequests);
    for(SerializedException ex : response.getFailedRequests().values()){
      parseAndThrowException(ex.deSerialize());
    }
  } finally {
    if (proxy != null) {
      rpc.stopProxy(proxy, conf);
    }
  }
}
 
Example 4
Source Project: big-c   Source File: TestYARNRunner.java    License: Apache License 2.0 6 votes vote down vote up
@Test(timeout=20000)
public void testAMProfiler() throws Exception {
  JobConf jobConf = new JobConf();

  jobConf.setBoolean(MRJobConfig.MR_AM_PROFILE, true);

  YARNRunner yarnRunner = new YARNRunner(jobConf);

  ApplicationSubmissionContext submissionContext =
      buildSubmitContext(yarnRunner, jobConf);

  ContainerLaunchContext containerSpec = submissionContext.getAMContainerSpec();
  List<String> commands = containerSpec.getCommands();

  for(String command : commands) {
    if (command != null) {
      if (command.contains(PROFILE_PARAMS)) {
        return;
      }
    }
  }
  throw new IllegalStateException("Profiler opts not found!");
}
 
Example 5
Source Project: hadoop   Source File: ContainerManagerImpl.java    License: Apache License 2.0 6 votes vote down vote up
private Credentials parseCredentials(ContainerLaunchContext launchContext)
    throws IOException {
  Credentials credentials = new Credentials();
  // //////////// Parse credentials
  ByteBuffer tokens = launchContext.getTokens();

  if (tokens != null) {
    DataInputByteBuffer buf = new DataInputByteBuffer();
    tokens.rewind();
    buf.reset(tokens);
    credentials.readTokenStorageStream(buf);
    if (LOG.isDebugEnabled()) {
      for (Token<? extends TokenIdentifier> tk : credentials.getAllTokens()) {
        LOG.debug(tk.getService() + " = " + tk.toString());
      }
    }
  }
  // //////////// End of parsing credentials
  return credentials;
}
 
Example 6
Source Project: big-c   Source File: TestContainerManagerRecovery.java    License: Apache License 2.0 6 votes vote down vote up
private StartContainersResponse startContainer(Context context,
    final ContainerManagerImpl cm, ContainerId cid,
    ContainerLaunchContext clc, LogAggregationContext logAggregationContext)
        throws Exception {
  UserGroupInformation user = UserGroupInformation.createRemoteUser(
      cid.getApplicationAttemptId().toString());
  StartContainerRequest scReq = StartContainerRequest.newInstance(
      clc, TestContainerManager.createContainerToken(cid, 0,
          context.getNodeId(), user.getShortUserName(),
          context.getContainerTokenSecretManager(), logAggregationContext));
  final List<StartContainerRequest> scReqList =
      new ArrayList<StartContainerRequest>();
  scReqList.add(scReq);
  NMTokenIdentifier nmToken = new NMTokenIdentifier(
      cid.getApplicationAttemptId(), context.getNodeId(),
      user.getShortUserName(),
      context.getNMTokenSecretManager().getCurrentKey().getKeyId());
  user.addTokenIdentifier(nmToken);
  return user.doAs(new PrivilegedExceptionAction<StartContainersResponse>() {
    @Override
    public StartContainersResponse run() throws Exception {
      return cm.startContainers(
          StartContainersRequest.newInstance(scReqList));
    }
  });
}
 
Example 7
Source Project: hadoop   Source File: TestContainerLaunch.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("rawtypes")
@Test (timeout = 10000)
public void testCallFailureWithNullLocalizedResources() {
  Container container = mock(Container.class);
  when(container.getContainerId()).thenReturn(ContainerId.newContainerId(
      ApplicationAttemptId.newInstance(ApplicationId.newInstance(
          System.currentTimeMillis(), 1), 1), 1));
  ContainerLaunchContext clc = mock(ContainerLaunchContext.class);
  when(clc.getCommands()).thenReturn(Collections.<String>emptyList());
  when(container.getLaunchContext()).thenReturn(clc);
  when(container.getLocalizedResources()).thenReturn(null);
  Dispatcher dispatcher = mock(Dispatcher.class);
  EventHandler eventHandler = new EventHandler() {
    public void handle(Event event) {
      Assert.assertTrue(event instanceof ContainerExitEvent);
      ContainerExitEvent exitEvent = (ContainerExitEvent) event;
      Assert.assertEquals(ContainerEventType.CONTAINER_EXITED_WITH_FAILURE,
          exitEvent.getType());
    }
  };
  when(dispatcher.getEventHandler()).thenReturn(eventHandler);
  ContainerLaunch launch = new ContainerLaunch(context, new Configuration(),
      dispatcher, exec, null, container, dirsHandler, containerManager);
  launch.call();
}
 
Example 8
Source Project: hadoop   Source File: TestContainerManagerRecovery.java    License: Apache License 2.0 6 votes vote down vote up
private StartContainersResponse startContainer(Context context,
    final ContainerManagerImpl cm, ContainerId cid,
    ContainerLaunchContext clc, LogAggregationContext logAggregationContext)
        throws Exception {
  UserGroupInformation user = UserGroupInformation.createRemoteUser(
      cid.getApplicationAttemptId().toString());
  StartContainerRequest scReq = StartContainerRequest.newInstance(
      clc, TestContainerManager.createContainerToken(cid, 0,
          context.getNodeId(), user.getShortUserName(),
          context.getContainerTokenSecretManager(), logAggregationContext));
  final List<StartContainerRequest> scReqList =
      new ArrayList<StartContainerRequest>();
  scReqList.add(scReq);
  NMTokenIdentifier nmToken = new NMTokenIdentifier(
      cid.getApplicationAttemptId(), context.getNodeId(),
      user.getShortUserName(),
      context.getNMTokenSecretManager().getCurrentKey().getKeyId());
  user.addTokenIdentifier(nmToken);
  return user.doAs(new PrivilegedExceptionAction<StartContainersResponse>() {
    @Override
    public StartContainersResponse run() throws Exception {
      return cm.startContainers(
          StartContainersRequest.newInstance(scReqList));
    }
  });
}
 
Example 9
@Override
public ContainerLaunchContext preLaunch(Container container, ContainerLaunchContext context) {
	ContainerCluster containerCluster = findContainerClusterByContainerId(container.getId());
	if (containerCluster == null) {
		return context;
	}
	containerIdMap.put(container.getId(), containerCluster.getId());

	Map<String, String> environment = context.getEnvironment();
	Map<String, String> indexEnv = new HashMap<>();
	indexEnv.putAll(environment);
	Integer reservedIndex;
	synchronized (indexTracker) {
		reservedIndex = indexTracker.reserveIndex(container.getId(), containerCluster);
	}
	indexEnv.put("SPRING_CLOUD_APPLICATION_GUID", container.getId().toString());
	indexEnv.put("SPRING_APPLICATION_INDEX", Integer.toString(reservedIndex));
	indexEnv.put("INSTANCE_INDEX", Integer.toString(reservedIndex));
	context.setEnvironment(indexEnv);
	return context;
}
 
Example 10
Source Project: big-c   Source File: ContainerImpl.java    License: Apache License 2.0 6 votes vote down vote up
public ContainerImpl(Context context,Configuration conf, Dispatcher dispatcher,
    NMStateStoreService stateStore, ContainerLaunchContext launchContext,
    Credentials creds, NodeManagerMetrics metrics,
    ContainerTokenIdentifier containerTokenIdentifier,Set<Integer> cpuCores) {
  this.daemonConf = conf;
  this.dispatcher = dispatcher;
  this.stateStore = stateStore;
  this.launchContext = launchContext;
  this.containerTokenIdentifier = containerTokenIdentifier;
  this.containerId = containerTokenIdentifier.getContainerID();
  this.resource = containerTokenIdentifier.getResource();
  this.currentResource = resource;
  this.diagnostics = new StringBuilder();
  this.credentials = creds;
  this.metrics = metrics;
  user = containerTokenIdentifier.getApplicationSubmitter();
  ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
  this.readLock = readWriteLock.readLock();
  this.writeLock = readWriteLock.writeLock();
  this.cpuCores  = cpuCores;
  this.context = context;

  stateMachine = stateMachineFactory.make(this);
}
 
Example 11
Source Project: hadoop   Source File: TestRMAppTransitions.java    License: Apache License 2.0 6 votes vote down vote up
@Test (timeout = 30000)
public void testAppRecoverPath() throws IOException {
  LOG.info("--- START: testAppRecoverPath ---");
  ApplicationSubmissionContext sub =
      Records.newRecord(ApplicationSubmissionContext.class);
  ContainerLaunchContext clc =
      Records.newRecord(ContainerLaunchContext.class);
  Credentials credentials = new Credentials();
  DataOutputBuffer dob = new DataOutputBuffer();
  credentials.writeTokenStorageToStream(dob);
  ByteBuffer securityTokens =
      ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
  clc.setTokens(securityTokens);
  sub.setAMContainerSpec(clc);
  testCreateAppSubmittedRecovery(sub);
}
 
Example 12
Source Project: big-c   Source File: TestContainerLaunch.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("rawtypes")
@Test (timeout = 10000)
public void testCallFailureWithNullLocalizedResources() {
  Container container = mock(Container.class);
  when(container.getContainerId()).thenReturn(ContainerId.newContainerId(
      ApplicationAttemptId.newInstance(ApplicationId.newInstance(
          System.currentTimeMillis(), 1), 1), 1));
  ContainerLaunchContext clc = mock(ContainerLaunchContext.class);
  when(clc.getCommands()).thenReturn(Collections.<String>emptyList());
  when(container.getLaunchContext()).thenReturn(clc);
  when(container.getLocalizedResources()).thenReturn(null);
  Dispatcher dispatcher = mock(Dispatcher.class);
  EventHandler eventHandler = new EventHandler() {
    public void handle(Event event) {
      Assert.assertTrue(event instanceof ContainerExitEvent);
      ContainerExitEvent exitEvent = (ContainerExitEvent) event;
      Assert.assertEquals(ContainerEventType.CONTAINER_EXITED_WITH_FAILURE,
          exitEvent.getType());
    }
  };
  when(dispatcher.getEventHandler()).thenReturn(eventHandler);
  ContainerLaunch launch = new ContainerLaunch(context, new Configuration(),
      dispatcher, exec, null, container, dirsHandler, containerManager);
  launch.call();
}
 
Example 13
Source Project: hadoop   Source File: NMClientAsyncImpl.java    License: Apache License 2.0 6 votes vote down vote up
public void startContainerAsync(
    Container container, ContainerLaunchContext containerLaunchContext) {
  if (containers.putIfAbsent(container.getId(),
      new StatefulContainer(this, container.getId())) != null) {
    callbackHandler.onStartContainerError(container.getId(),
        RPCUtil.getRemoteException("Container " + container.getId() +
            " is already started or scheduled to start"));
  }
  try {
    events.put(new StartContainerEvent(container, containerLaunchContext));
  } catch (InterruptedException e) {
    LOG.warn("Exception when scheduling the event of starting Container " +
        container.getId());
    callbackHandler.onStartContainerError(container.getId(), e);
  }
}
 
Example 14
Source Project: big-c   Source File: TestLinuxContainerExecutorWithMocks.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testContainerLaunch() throws IOException {
  String appSubmitter = "nobody";
  String cmd = String.valueOf(
      LinuxContainerExecutor.Commands.LAUNCH_CONTAINER.getValue());
  String appId = "APP_ID";
  String containerId = "CONTAINER_ID";
  Container container = mock(Container.class);
  ContainerId cId = mock(ContainerId.class);
  ContainerLaunchContext context = mock(ContainerLaunchContext.class);
  HashMap<String, String> env = new HashMap<String,String>();
  
  when(container.getContainerId()).thenReturn(cId);
  when(container.getLaunchContext()).thenReturn(context);
  
  when(cId.toString()).thenReturn(containerId);
  
  when(context.getEnvironment()).thenReturn(env);
  
  Path scriptPath = new Path("file:///bin/echo");
  Path tokensPath = new Path("file:///dev/null");
  Path workDir = new Path("/tmp");
  Path pidFile = new Path(workDir, "pid.txt");

  mockExec.activateContainer(cId, pidFile);
  int ret = mockExec.launchContainer(container, scriptPath, tokensPath,
      appSubmitter, appId, workDir, dirsHandler.getLocalDirs(),
      dirsHandler.getLogDirs());
  assertEquals(0, ret);
  assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
      appSubmitter, cmd, appId, containerId,
      workDir.toString(), "/bin/echo", "/dev/null", pidFile.toString(),
      StringUtils.join(",", dirsHandler.getLocalDirs()),
      StringUtils.join(",", dirsHandler.getLogDirs()), "cgroups=none"),
      readMockParams());
  
}
 
Example 15
Source Project: big-c   Source File: YarnClientImpl.java    License: Apache License 2.0 5 votes vote down vote up
private void addTimelineDelegationToken(
    ContainerLaunchContext clc) throws YarnException, IOException {
  Credentials credentials = new Credentials();
  DataInputByteBuffer dibb = new DataInputByteBuffer();
  ByteBuffer tokens = clc.getTokens();
  if (tokens != null) {
    dibb.reset(tokens);
    credentials.readTokenStorageStream(dibb);
    tokens.rewind();
  }
  // If the timeline delegation token is already in the CLC, no need to add
  // one more
  for (org.apache.hadoop.security.token.Token<? extends TokenIdentifier> token : credentials
      .getAllTokens()) {
    if (token.getKind().equals(TimelineDelegationTokenIdentifier.KIND_NAME)) {
      return;
    }
  }
  org.apache.hadoop.security.token.Token<TimelineDelegationTokenIdentifier>
      timelineDelegationToken = getTimelineDelegationToken();
  if (timelineDelegationToken == null) {
    return;
  }
  credentials.addToken(timelineService, timelineDelegationToken);
  if (LOG.isDebugEnabled()) {
    LOG.debug("Add timline delegation token into credentials: "
        + timelineDelegationToken);
  }
  DataOutputBuffer dob = new DataOutputBuffer();
  credentials.writeTokenStorageToStream(dob);
  tokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
  clc.setTokens(tokens);
}
 
Example 16
Source Project: flink   Source File: YarnResourceManager.java    License: Apache License 2.0 5 votes vote down vote up
private ContainerLaunchContext createTaskExecutorLaunchContext(Resource resource, String containerId, String host)
		throws Exception {
	// init the ContainerLaunchContext
	final String currDir = env.get(ApplicationConstants.Environment.PWD.key());

	final ContaineredTaskManagerParameters taskManagerParameters =
			ContaineredTaskManagerParameters.create(flinkConfig, resource.getMemory(), numberOfTaskSlots);

	log.debug("TaskExecutor {} will be started with container size {} MB, JVM heap size {} MB, " +
			"JVM direct memory limit {} MB",
			containerId,
			taskManagerParameters.taskManagerTotalMemoryMB(),
			taskManagerParameters.taskManagerHeapSizeMB(),
			taskManagerParameters.taskManagerDirectMemoryLimitMB());

	Configuration taskManagerConfig = BootstrapTools.cloneConfiguration(flinkConfig);

	log.debug("TaskManager configuration: {}", taskManagerConfig);

	ContainerLaunchContext taskExecutorLaunchContext = Utils.createTaskExecutorContext(
		flinkConfig,
		yarnConfig,
		env,
		taskManagerParameters,
		taskManagerConfig,
		currDir,
		YarnTaskExecutorRunner.class,
		log);

	// set a special environment variable to uniquely identify this container
	taskExecutorLaunchContext.getEnvironment()
			.put(ENV_FLINK_CONTAINER_ID, containerId);
	taskExecutorLaunchContext.getEnvironment()
			.put(ENV_FLINK_NODE_ID, host);
	return taskExecutorLaunchContext;
}
 
Example 17
Source Project: incubator-tajo   Source File: TajoContainerProxy.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void launch(ContainerLaunchContext containerLaunchContext) {
  context.getResourceAllocator().addContainer(containerID, this);

  this.hostName = container.getNodeId().getHost();
  this.port = ((TajoWorkerContainer)container).getWorkerResource().getPullServerPort();
  this.state = ContainerState.RUNNING;

  if (LOG.isDebugEnabled()) {
    LOG.debug("Launch Container:" + executionBlockId + "," + containerID.getId() + "," +
        container.getId() + "," + container.getNodeId() + ", pullServer=" + port);
  }

  assignExecutionBlock(executionBlockId, container);
}
 
Example 18
Source Project: spring-cloud-deployer-yarn   Source File: TaskAppmaster.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public ContainerLaunchContext preLaunch(Container container, ContainerLaunchContext context) {
	Map<String, LocalResource> resources = new HashMap<String, LocalResource>();
	resources.putAll(context.getLocalResources());
	resources.putAll(artifactResourceLocalizer.getResources());
	context.setLocalResources(resources);
	return context;
}
 
Example 19
Source Project: flink   Source File: YarnResourceManagerTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests that YarnResourceManager will not request more containers than needs during
 * callback from Yarn when container is Completed.
 */
@Test
public void testOnContainerCompleted() throws Exception {
	new Context() {{
		runTest(() -> {
			CompletableFuture<?> registerSlotRequestFuture = resourceManager.runInMainThread(() -> {
				rmServices.slotManager.registerSlotRequest(
					new SlotRequest(new JobID(), new AllocationID(), resourceProfile1, taskHost));
				return null;
			});

			// wait for the registerSlotRequest completion
			registerSlotRequestFuture.get();

			// Callback from YARN when container is allocated.
			Container testingContainer = mockContainer("container", 1234, 1, resourceManager.getContainerResource());

			doReturn(Collections.singletonList(Collections.singletonList(resourceManager.getContainerRequest())))
				.when(mockResourceManagerClient).getMatchingRequests(any(Priority.class), anyString(), any(Resource.class));

			resourceManager.onContainersAllocated(ImmutableList.of(testingContainer));
			verify(mockResourceManagerClient).addContainerRequest(any(AMRMClient.ContainerRequest.class));
			verify(mockResourceManagerClient).removeContainerRequest(any(AMRMClient.ContainerRequest.class));
			verify(mockNMClient).startContainer(eq(testingContainer), any(ContainerLaunchContext.class));

			// Callback from YARN when container is Completed, pending request can not be fulfilled by pending
			// containers, need to request new container.
			ContainerStatus testingContainerStatus = mockContainerStatus(testingContainer.getId());

			resourceManager.onContainersCompleted(ImmutableList.of(testingContainerStatus));
			verify(mockResourceManagerClient, times(2)).addContainerRequest(any(AMRMClient.ContainerRequest.class));

			// Callback from YARN when container is Completed happened before global fail, pending request
			// slot is already fulfilled by pending containers, no need to request new container.
			resourceManager.onContainersCompleted(ImmutableList.of(testingContainerStatus));
			verify(mockResourceManagerClient, times(2)).addContainerRequest(any(AMRMClient.ContainerRequest.class));
		});
	}};
}
 
Example 20
Source Project: reef   Source File: YARNResourceLaunchHandler.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void onNext(final ResourceLaunchEvent resourceLaunchEvent) {
  try {

    final String containerId = resourceLaunchEvent.getIdentifier();
    LOG.log(Level.FINEST, "TIME: Start ResourceLaunch {0}", containerId);
    final Container container = this.containers.get(containerId);
    LOG.log(Level.FINEST, "Setting up container launch container for id={0}", container.getId());
    final Map<String, LocalResource> localResources =
        this.evaluatorSetupHelper.getResources(resourceLaunchEvent);

    final List<String> command = getLaunchCommand(resourceLaunchEvent, container.getResource().getMemory());
    if (LOG.isLoggable(Level.INFO)) {
      LOG.log(Level.INFO,
          "TIME: Run ResourceLaunchProto {0} command: {1} resources: {2}",
          new Object[] {containerId, StringUtils.join(command, ' '), localResources});
    }

    final byte[] securityTokensBuffer = this.tokenProvider.getTokens();
    final ContainerLaunchContext ctx = YarnTypes.getContainerLaunchContext(
        command, localResources, securityTokensBuffer, YarnUtilities.getApplicationId());
    this.yarnContainerManager.get().submit(container, ctx);

    LOG.log(Level.FINEST, "TIME: End ResourceLaunch {0}", containerId);

  } catch (final IOException e) {
    LOG.log(Level.WARNING, "Error handling resource launch message: " + resourceLaunchEvent, e);
    throw new RuntimeException(e);
  }
}
 
Example 21
Source Project: hadoop   Source File: StartContainerRequestPBImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public ContainerLaunchContext getContainerLaunchContext() {
  StartContainerRequestProtoOrBuilder p = viaProto ? proto : builder;
  if (this.containerLaunchContext != null) {
    return this.containerLaunchContext;
  }
  if (!p.hasContainerLaunchContext()) {
    return null;
  }
  this.containerLaunchContext = convertFromProtoFormat(p.getContainerLaunchContext());
  return this.containerLaunchContext;
}
 
Example 22
Source Project: big-c   Source File: BuilderUtils.java    License: Apache License 2.0 5 votes vote down vote up
public static ContainerLaunchContext newContainerLaunchContext(
    Map<String, LocalResource> localResources,
    Map<String, String> environment, List<String> commands,
    Map<String, ByteBuffer> serviceData, ByteBuffer tokens,
    Map<ApplicationAccessType, String> acls) {
  ContainerLaunchContext container = recordFactory
      .newRecordInstance(ContainerLaunchContext.class);
  container.setLocalResources(localResources);
  container.setEnvironment(environment);
  container.setCommands(commands);
  container.setServiceData(serviceData);
  container.setTokens(tokens);
  container.setApplicationACLs(acls);
  return container;
}
 
Example 23
Source Project: incubator-gobblin   Source File: GobblinYarnAppLauncher.java    License: Apache License 2.0 5 votes vote down vote up
private void setupSecurityTokens(ContainerLaunchContext containerLaunchContext) throws IOException {
  Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials();

  // Pass on the credentials from the hadoop token file if present.
  // The value in the token file takes precedence.
  if (System.getenv(HADOOP_TOKEN_FILE_LOCATION) != null) {
    Credentials tokenFileCredentials = Credentials.readTokenStorageFile(new File(System.getenv(HADOOP_TOKEN_FILE_LOCATION)),
        new Configuration());
    credentials.addAll(tokenFileCredentials);
  }

  String tokenRenewer = this.yarnConfiguration.get(YarnConfiguration.RM_PRINCIPAL);
  if (tokenRenewer == null || tokenRenewer.length() == 0) {
    throw new IOException("Failed to get master Kerberos principal for the RM to use as renewer");
  }

  // For now, only getting tokens for the default file-system.
  Token<?> tokens[] = this.fs.addDelegationTokens(tokenRenewer, credentials);
  if (tokens != null) {
    for (Token<?> token : tokens) {
      LOGGER.info("Got delegation token for " + this.fs.getUri() + "; " + token);
    }
  }

  Closer closer = Closer.create();
  try {
    DataOutputBuffer dataOutputBuffer = closer.register(new DataOutputBuffer());
    credentials.writeTokenStorageToStream(dataOutputBuffer);
    ByteBuffer fsTokens = ByteBuffer.wrap(dataOutputBuffer.getData(), 0, dataOutputBuffer.getLength());
    containerLaunchContext.setTokens(fsTokens);
  } catch (Throwable t) {
    throw closer.rethrow(t);
  } finally {
    closer.close();
  }
}
 
Example 24
Source Project: flink   Source File: YarnResourceManager.java    License: Apache License 2.0 5 votes vote down vote up
private ContainerLaunchContext createTaskExecutorLaunchContext(
	String containerId,
	String host,
	TaskExecutorProcessSpec taskExecutorProcessSpec) throws Exception {

	// init the ContainerLaunchContext
	final String currDir = env.get(ApplicationConstants.Environment.PWD.key());

	final ContaineredTaskManagerParameters taskManagerParameters =
			ContaineredTaskManagerParameters.create(flinkConfig, taskExecutorProcessSpec);

	log.info("TaskExecutor {} will be started on {} with {}.",
		containerId,
		host,
		taskExecutorProcessSpec);

	final Configuration taskManagerConfig = BootstrapTools.cloneConfiguration(flinkConfig);
	taskManagerConfig.set(TaskManagerOptions.TASK_MANAGER_RESOURCE_ID, containerId);

	final String taskManagerDynamicProperties =
		BootstrapTools.getDynamicPropertiesAsString(flinkClientConfig, taskManagerConfig);

	log.debug("TaskManager configuration: {}", taskManagerConfig);

	ContainerLaunchContext taskExecutorLaunchContext = Utils.createTaskExecutorContext(
		flinkConfig,
		yarnConfig,
		env,
		taskManagerParameters,
		taskManagerDynamicProperties,
		currDir,
		YarnTaskExecutorRunner.class,
		log);

	taskExecutorLaunchContext.getEnvironment()
			.put(ENV_FLINK_NODE_ID, host);
	return taskExecutorLaunchContext;
}
 
Example 25
Source Project: big-c   Source File: QueueACLsTestBase.java    License: Apache License 2.0 5 votes vote down vote up
private ApplicationId submitAppAndGetAppId(String submitter,
    String queueName, boolean setupACLs) throws Exception {

  GetNewApplicationRequest newAppRequest =
      GetNewApplicationRequest.newInstance();

  ApplicationClientProtocol submitterClient = getRMClientForUser(submitter);
  ApplicationId applicationId =
      submitterClient.getNewApplication(newAppRequest).getApplicationId();

  Resource resource = BuilderUtils.newResource(1024, 1);
  Map<ApplicationAccessType, String> acls = createACLs(submitter, setupACLs);
  ContainerLaunchContext amContainerSpec =
      ContainerLaunchContext.newInstance(null, null, null, null, null, acls);

  ApplicationSubmissionContext appSubmissionContext =
      ApplicationSubmissionContext.newInstance(applicationId,
        "applicationName", queueName, null, amContainerSpec, false, true, 1,
        resource, "applicationType");
  appSubmissionContext.setApplicationId(applicationId);
  appSubmissionContext.setQueue(queueName);

  SubmitApplicationRequest submitRequest =
      SubmitApplicationRequest.newInstance(appSubmissionContext);
  submitterClient.submitApplication(submitRequest);
  resourceManager.waitForState(applicationId, RMAppState.ACCEPTED);
  return applicationId;
}
 
Example 26
Source Project: twill   Source File: Hadoop21YarnNMClient.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Cancellable start(YarnContainerInfo containerInfo, YarnLaunchContext launchContext) {
  try {
    Container container = containerInfo.getContainer();
    nmClient.startContainer(container, launchContext.<ContainerLaunchContext>getLaunchContext());
    return new ContainerTerminator(container, nmClient);
  } catch (Exception e) {
    LOG.error("Error in launching process", e);
    throw Throwables.propagate(e);
  }

}
 
Example 27
Source Project: hadoop   Source File: SharedCacheUploadEvent.java    License: Apache License 2.0 5 votes vote down vote up
public SharedCacheUploadEvent(Map<LocalResourceRequest,Path> resources,
    ContainerLaunchContext context, String user,
    SharedCacheUploadEventType eventType) {
  super(eventType);
  this.resources = resources;
  this.context = context;
  this.user = user;
}
 
Example 28
Source Project: big-c   Source File: TestAppManager.java    License: Apache License 2.0 5 votes vote down vote up
private static ContainerLaunchContext mockContainerLaunchContext(
    RecordFactory recordFactory) {
  ContainerLaunchContext amContainer = recordFactory.newRecordInstance(
      ContainerLaunchContext.class);
  amContainer.setApplicationACLs(new HashMap<ApplicationAccessType, String>());;
  return amContainer;
}
 
Example 29
Source Project: tez   Source File: ContainerLauncherLaunchRequestEvent.java    License: Apache License 2.0 5 votes vote down vote up
public ContainerLauncherLaunchRequestEvent(ContainerLaunchContext clc,
                                           Container container, int launcherId, int schedulerId,
                                           int taskCommId) {
  super(container.getId(), container.getNodeId(), container
      .getContainerToken(), ContainerLauncherEventType.CONTAINER_LAUNCH_REQUEST,
      launcherId, schedulerId, taskCommId);
  this.clc = clc;
  this.container = container;
}
 
Example 30
Source Project: hadoop   Source File: TestApplication.java    License: Apache License 2.0 5 votes vote down vote up
private Container createMockedContainer(ApplicationId appId, int containerId) {
  ApplicationAttemptId appAttemptId =
      BuilderUtils.newApplicationAttemptId(appId, 1);
  ContainerId cId = BuilderUtils.newContainerId(appAttemptId, containerId);
  Container c = mock(Container.class);
  when(c.getContainerId()).thenReturn(cId);
  ContainerLaunchContext launchContext = mock(ContainerLaunchContext.class);
  when(c.getLaunchContext()).thenReturn(launchContext);
  when(launchContext.getApplicationACLs()).thenReturn(
      new HashMap<ApplicationAccessType, String>());
  return c;
}