org.apache.hadoop.yarn.api.records.ContainerLaunchContext Java Examples

The following examples show how to use org.apache.hadoop.yarn.api.records.ContainerLaunchContext. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestContainerManagerRecovery.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private StartContainersResponse startContainer(Context context,
    final ContainerManagerImpl cm, ContainerId cid,
    ContainerLaunchContext clc, LogAggregationContext logAggregationContext)
        throws Exception {
  UserGroupInformation user = UserGroupInformation.createRemoteUser(
      cid.getApplicationAttemptId().toString());
  StartContainerRequest scReq = StartContainerRequest.newInstance(
      clc, TestContainerManager.createContainerToken(cid, 0,
          context.getNodeId(), user.getShortUserName(),
          context.getContainerTokenSecretManager(), logAggregationContext));
  final List<StartContainerRequest> scReqList =
      new ArrayList<StartContainerRequest>();
  scReqList.add(scReq);
  NMTokenIdentifier nmToken = new NMTokenIdentifier(
      cid.getApplicationAttemptId(), context.getNodeId(),
      user.getShortUserName(),
      context.getNMTokenSecretManager().getCurrentKey().getKeyId());
  user.addTokenIdentifier(nmToken);
  return user.doAs(new PrivilegedExceptionAction<StartContainersResponse>() {
    @Override
    public StartContainersResponse run() throws Exception {
      return cm.startContainers(
          StartContainersRequest.newInstance(scReqList));
    }
  });
}
 
Example #2
Source File: ContainerImpl.java    From big-c with Apache License 2.0 6 votes vote down vote up
public ContainerImpl(Context context,Configuration conf, Dispatcher dispatcher,
    NMStateStoreService stateStore, ContainerLaunchContext launchContext,
    Credentials creds, NodeManagerMetrics metrics,
    ContainerTokenIdentifier containerTokenIdentifier,Set<Integer> cpuCores) {
  this.daemonConf = conf;
  this.dispatcher = dispatcher;
  this.stateStore = stateStore;
  this.launchContext = launchContext;
  this.containerTokenIdentifier = containerTokenIdentifier;
  this.containerId = containerTokenIdentifier.getContainerID();
  this.resource = containerTokenIdentifier.getResource();
  this.currentResource = resource;
  this.diagnostics = new StringBuilder();
  this.credentials = creds;
  this.metrics = metrics;
  user = containerTokenIdentifier.getApplicationSubmitter();
  ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
  this.readLock = readWriteLock.readLock();
  this.writeLock = readWriteLock.writeLock();
  this.cpuCores  = cpuCores;
  this.context = context;

  stateMachine = stateMachineFactory.make(this);
}
 
Example #3
Source File: TestRMAppTransitions.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test (timeout = 30000)
public void testAppRecoverPath() throws IOException {
  LOG.info("--- START: testAppRecoverPath ---");
  ApplicationSubmissionContext sub =
      Records.newRecord(ApplicationSubmissionContext.class);
  ContainerLaunchContext clc =
      Records.newRecord(ContainerLaunchContext.class);
  Credentials credentials = new Credentials();
  DataOutputBuffer dob = new DataOutputBuffer();
  credentials.writeTokenStorageToStream(dob);
  ByteBuffer securityTokens =
      ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
  clc.setTokens(securityTokens);
  sub.setAMContainerSpec(clc);
  testCreateAppSubmittedRecovery(sub);
}
 
Example #4
Source File: TestContainerManagerRecovery.java    From big-c with Apache License 2.0 6 votes vote down vote up
private StartContainersResponse startContainer(Context context,
    final ContainerManagerImpl cm, ContainerId cid,
    ContainerLaunchContext clc, LogAggregationContext logAggregationContext)
        throws Exception {
  UserGroupInformation user = UserGroupInformation.createRemoteUser(
      cid.getApplicationAttemptId().toString());
  StartContainerRequest scReq = StartContainerRequest.newInstance(
      clc, TestContainerManager.createContainerToken(cid, 0,
          context.getNodeId(), user.getShortUserName(),
          context.getContainerTokenSecretManager(), logAggregationContext));
  final List<StartContainerRequest> scReqList =
      new ArrayList<StartContainerRequest>();
  scReqList.add(scReq);
  NMTokenIdentifier nmToken = new NMTokenIdentifier(
      cid.getApplicationAttemptId(), context.getNodeId(),
      user.getShortUserName(),
      context.getNMTokenSecretManager().getCurrentKey().getKeyId());
  user.addTokenIdentifier(nmToken);
  return user.doAs(new PrivilegedExceptionAction<StartContainersResponse>() {
    @Override
    public StartContainersResponse run() throws Exception {
      return cm.startContainers(
          StartContainersRequest.newInstance(scReqList));
    }
  });
}
 
Example #5
Source File: YarnRemoteInterpreterProcess.java    From zeppelin with Apache License 2.0 6 votes vote down vote up
private ApplicationSubmissionContext createApplicationSubmissionContext(
        ApplicationSubmissionContext appContext) throws Exception {

  setResources(appContext);
  setPriority(appContext);
  setQueue(appContext);
  appContext.setApplicationId(appId);
  setApplicationName(appContext);
  appContext.setApplicationType("ZEPPELIN INTERPRETER");
  appContext.setMaxAppAttempts(1);

  ContainerLaunchContext amContainer = setUpAMLaunchContext();
  appContext.setAMContainerSpec(amContainer);
  appContext.setCancelTokensWhenComplete(true);
  return appContext;
}
 
Example #6
Source File: ContainerManagerImpl.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private Credentials parseCredentials(ContainerLaunchContext launchContext)
    throws IOException {
  Credentials credentials = new Credentials();
  // //////////// Parse credentials
  ByteBuffer tokens = launchContext.getTokens();

  if (tokens != null) {
    DataInputByteBuffer buf = new DataInputByteBuffer();
    tokens.rewind();
    buf.reset(tokens);
    credentials.readTokenStorageStream(buf);
    if (LOG.isDebugEnabled()) {
      for (Token<? extends TokenIdentifier> tk : credentials.getAllTokens()) {
        LOG.debug(tk.getService() + " = " + tk.toString());
      }
    }
  }
  // //////////// End of parsing credentials
  return credentials;
}
 
Example #7
Source File: TestYARNRunner.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test(timeout=20000)
public void testAMProfiler() throws Exception {
  JobConf jobConf = new JobConf();

  jobConf.setBoolean(MRJobConfig.MR_AM_PROFILE, true);

  YARNRunner yarnRunner = new YARNRunner(jobConf);

  ApplicationSubmissionContext submissionContext =
      buildSubmitContext(yarnRunner, jobConf);

  ContainerLaunchContext containerSpec = submissionContext.getAMContainerSpec();
  List<String> commands = containerSpec.getCommands();

  for(String command : commands) {
    if (command != null) {
      if (command.contains(PROFILE_PARAMS)) {
        return;
      }
    }
  }
  throw new IllegalStateException("Profiler opts not found!");
}
 
Example #8
Source File: TestContainerLaunch.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("rawtypes")
@Test (timeout = 10000)
public void testCallFailureWithNullLocalizedResources() {
  Container container = mock(Container.class);
  when(container.getContainerId()).thenReturn(ContainerId.newContainerId(
      ApplicationAttemptId.newInstance(ApplicationId.newInstance(
          System.currentTimeMillis(), 1), 1), 1));
  ContainerLaunchContext clc = mock(ContainerLaunchContext.class);
  when(clc.getCommands()).thenReturn(Collections.<String>emptyList());
  when(container.getLaunchContext()).thenReturn(clc);
  when(container.getLocalizedResources()).thenReturn(null);
  Dispatcher dispatcher = mock(Dispatcher.class);
  EventHandler eventHandler = new EventHandler() {
    public void handle(Event event) {
      Assert.assertTrue(event instanceof ContainerExitEvent);
      ContainerExitEvent exitEvent = (ContainerExitEvent) event;
      Assert.assertEquals(ContainerEventType.CONTAINER_EXITED_WITH_FAILURE,
          exitEvent.getType());
    }
  };
  when(dispatcher.getEventHandler()).thenReturn(eventHandler);
  ContainerLaunch launch = new ContainerLaunch(context, new Configuration(),
      dispatcher, exec, null, container, dirsHandler, containerManager);
  launch.call();
}
 
Example #9
Source File: TestContainerLaunch.java    From big-c with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("rawtypes")
@Test (timeout = 10000)
public void testCallFailureWithNullLocalizedResources() {
  Container container = mock(Container.class);
  when(container.getContainerId()).thenReturn(ContainerId.newContainerId(
      ApplicationAttemptId.newInstance(ApplicationId.newInstance(
          System.currentTimeMillis(), 1), 1), 1));
  ContainerLaunchContext clc = mock(ContainerLaunchContext.class);
  when(clc.getCommands()).thenReturn(Collections.<String>emptyList());
  when(container.getLaunchContext()).thenReturn(clc);
  when(container.getLocalizedResources()).thenReturn(null);
  Dispatcher dispatcher = mock(Dispatcher.class);
  EventHandler eventHandler = new EventHandler() {
    public void handle(Event event) {
      Assert.assertTrue(event instanceof ContainerExitEvent);
      ContainerExitEvent exitEvent = (ContainerExitEvent) event;
      Assert.assertEquals(ContainerEventType.CONTAINER_EXITED_WITH_FAILURE,
          exitEvent.getType());
    }
  };
  when(dispatcher.getEventHandler()).thenReturn(eventHandler);
  ContainerLaunch launch = new ContainerLaunch(context, new Configuration(),
      dispatcher, exec, null, container, dirsHandler, containerManager);
  launch.call();
}
 
Example #10
Source File: TestContainerManagerSecurity.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private void startContainer(final YarnRPC rpc,
    org.apache.hadoop.yarn.api.records.Token nmToken,
    org.apache.hadoop.yarn.api.records.Token containerToken,
    NodeId nodeId, String user) throws Exception {

  ContainerLaunchContext context =
      Records.newRecord(ContainerLaunchContext.class);
  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(context,containerToken);
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  ContainerManagementProtocol proxy = null;
  try {
    proxy = getContainerManagementProtocolProxy(rpc, nmToken, nodeId, user);
    StartContainersResponse response = proxy.startContainers(allRequests);
    for(SerializedException ex : response.getFailedRequests().values()){
      parseAndThrowException(ex.deSerialize());
    }
  } finally {
    if (proxy != null) {
      rpc.stopProxy(proxy, conf);
    }
  }
}
 
Example #11
Source File: NMClientAsyncImpl.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public void startContainerAsync(
    Container container, ContainerLaunchContext containerLaunchContext) {
  if (containers.putIfAbsent(container.getId(),
      new StatefulContainer(this, container.getId())) != null) {
    callbackHandler.onStartContainerError(container.getId(),
        RPCUtil.getRemoteException("Container " + container.getId() +
            " is already started or scheduled to start"));
  }
  try {
    events.put(new StartContainerEvent(container, containerLaunchContext));
  } catch (InterruptedException e) {
    LOG.warn("Exception when scheduling the event of starting Container " +
        container.getId());
    callbackHandler.onStartContainerError(container.getId(), e);
  }
}
 
Example #12
Source File: StreamAppmaster.java    From spring-cloud-deployer-yarn with Apache License 2.0 6 votes vote down vote up
@Override
public ContainerLaunchContext preLaunch(Container container, ContainerLaunchContext context) {
	ContainerCluster containerCluster = findContainerClusterByContainerId(container.getId());
	if (containerCluster == null) {
		return context;
	}
	containerIdMap.put(container.getId(), containerCluster.getId());

	Map<String, String> environment = context.getEnvironment();
	Map<String, String> indexEnv = new HashMap<>();
	indexEnv.putAll(environment);
	Integer reservedIndex;
	synchronized (indexTracker) {
		reservedIndex = indexTracker.reserveIndex(container.getId(), containerCluster);
	}
	indexEnv.put("SPRING_CLOUD_APPLICATION_GUID", container.getId().toString());
	indexEnv.put("SPRING_APPLICATION_INDEX", Integer.toString(reservedIndex));
	indexEnv.put("INSTANCE_INDEX", Integer.toString(reservedIndex));
	context.setEnvironment(indexEnv);
	return context;
}
 
Example #13
Source File: NMClientAsyncImpl.java    From big-c with Apache License 2.0 6 votes vote down vote up
public void startContainerAsync(
    Container container, ContainerLaunchContext containerLaunchContext) {
  if (containers.putIfAbsent(container.getId(),
      new StatefulContainer(this, container.getId())) != null) {
    callbackHandler.onStartContainerError(container.getId(),
        RPCUtil.getRemoteException("Container " + container.getId() +
            " is already started or scheduled to start"));
  }
  try {
    events.put(new StartContainerEvent(container, containerLaunchContext));
  } catch (InterruptedException e) {
    LOG.warn("Exception when scheduling the event of starting Container " +
        container.getId());
    callbackHandler.onStartContainerError(container.getId(), e);
  }
}
 
Example #14
Source File: SharedCacheUploadEvent.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public SharedCacheUploadEvent(Map<LocalResourceRequest,Path> resources,
    ContainerLaunchContext context, String user,
    SharedCacheUploadEventType eventType) {
  super(eventType);
  this.resources = resources;
  this.context = context;
  this.user = user;
}
 
Example #15
Source File: YarnResourceManagerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that YarnResourceManager will not request more containers than needs during
 * callback from Yarn when container is Completed.
 */
@Test
public void testOnContainerCompleted() throws Exception {
	new Context() {{
		runTest(() -> {
			CompletableFuture<?> registerSlotRequestFuture = resourceManager.runInMainThread(() -> {
				rmServices.slotManager.registerSlotRequest(
					new SlotRequest(new JobID(), new AllocationID(), resourceProfile1, taskHost));
				return null;
			});

			// wait for the registerSlotRequest completion
			registerSlotRequestFuture.get();

			// Callback from YARN when container is allocated.
			Container testingContainer = mockContainer("container", 1234, 1, resourceManager.getContainerResource());

			doReturn(Collections.singletonList(Collections.singletonList(resourceManager.getContainerRequest())))
				.when(mockResourceManagerClient).getMatchingRequests(any(Priority.class), anyString(), any(Resource.class));

			resourceManager.onContainersAllocated(ImmutableList.of(testingContainer));
			verify(mockResourceManagerClient).addContainerRequest(any(AMRMClient.ContainerRequest.class));
			verify(mockResourceManagerClient).removeContainerRequest(any(AMRMClient.ContainerRequest.class));
			verify(mockNMClient).startContainer(eq(testingContainer), any(ContainerLaunchContext.class));

			// Callback from YARN when container is Completed, pending request can not be fulfilled by pending
			// containers, need to request new container.
			ContainerStatus testingContainerStatus = mockContainerStatus(testingContainer.getId());

			resourceManager.onContainersCompleted(ImmutableList.of(testingContainerStatus));
			verify(mockResourceManagerClient, times(2)).addContainerRequest(any(AMRMClient.ContainerRequest.class));

			// Callback from YARN when container is Completed happened before global fail, pending request
			// slot is already fulfilled by pending containers, no need to request new container.
			resourceManager.onContainersCompleted(ImmutableList.of(testingContainerStatus));
			verify(mockResourceManagerClient, times(2)).addContainerRequest(any(AMRMClient.ContainerRequest.class));
		});
	}};
}
 
Example #16
Source File: TestAppManager.java    From big-c with Apache License 2.0 5 votes vote down vote up
private static ContainerLaunchContext mockContainerLaunchContext(
    RecordFactory recordFactory) {
  ContainerLaunchContext amContainer = recordFactory.newRecordInstance(
      ContainerLaunchContext.class);
  amContainer.setApplicationACLs(new HashMap<ApplicationAccessType, String>());;
  return amContainer;
}
 
Example #17
Source File: QueueACLsTestBase.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private ApplicationId submitAppAndGetAppId(String submitter,
    String queueName, boolean setupACLs) throws Exception {

  GetNewApplicationRequest newAppRequest =
      GetNewApplicationRequest.newInstance();

  ApplicationClientProtocol submitterClient = getRMClientForUser(submitter);
  ApplicationId applicationId =
      submitterClient.getNewApplication(newAppRequest).getApplicationId();

  Resource resource = BuilderUtils.newResource(1024, 1);
  Map<ApplicationAccessType, String> acls = createACLs(submitter, setupACLs);
  ContainerLaunchContext amContainerSpec =
      ContainerLaunchContext.newInstance(null, null, null, null, null, acls);

  ApplicationSubmissionContext appSubmissionContext =
      ApplicationSubmissionContext.newInstance(applicationId,
        "applicationName", queueName, null, amContainerSpec, false, true, 1,
        resource, "applicationType");
  appSubmissionContext.setApplicationId(applicationId);
  appSubmissionContext.setQueue(queueName);

  SubmitApplicationRequest submitRequest =
      SubmitApplicationRequest.newInstance(appSubmissionContext);
  submitterClient.submitApplication(submitRequest);
  resourceManager.waitForState(applicationId, RMAppState.ACCEPTED);
  return applicationId;
}
 
Example #18
Source File: AMLauncher.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private void launch() throws IOException, YarnException {
  connect();
  ContainerId masterContainerID = masterContainer.getId();
  ApplicationSubmissionContext applicationContext =
    application.getSubmissionContext();
  LOG.info("Setting up container " + masterContainer
      + " for AM " + application.getAppAttemptId());  
  ContainerLaunchContext launchContext =
      createAMContainerLaunchContext(applicationContext, masterContainerID);

  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(launchContext,
        masterContainer.getContainerToken());
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);

  StartContainersResponse response =
      containerMgrProxy.startContainers(allRequests);
  if (response.getFailedRequests() != null
      && response.getFailedRequests().containsKey(masterContainerID)) {
    Throwable t =
        response.getFailedRequests().get(masterContainerID).deSerialize();
    parseAndThrowException(t);
  } else {
    LOG.info("Done launching container " + masterContainer + " for AM "
        + application.getAppAttemptId());
  }
}
 
Example #19
Source File: Hadoop21YarnNMClient.java    From twill with Apache License 2.0 5 votes vote down vote up
@Override
public Cancellable start(YarnContainerInfo containerInfo, YarnLaunchContext launchContext) {
  try {
    Container container = containerInfo.getContainer();
    nmClient.startContainer(container, launchContext.<ContainerLaunchContext>getLaunchContext());
    return new ContainerTerminator(container, nmClient);
  } catch (Exception e) {
    LOG.error("Error in launching process", e);
    throw Throwables.propagate(e);
  }

}
 
Example #20
Source File: YarnResourceManager.java    From flink with Apache License 2.0 5 votes vote down vote up
private ContainerLaunchContext createTaskExecutorLaunchContext(
	String containerId,
	String host,
	TaskExecutorProcessSpec taskExecutorProcessSpec) throws Exception {

	// init the ContainerLaunchContext
	final String currDir = env.get(ApplicationConstants.Environment.PWD.key());

	final ContaineredTaskManagerParameters taskManagerParameters =
			ContaineredTaskManagerParameters.create(flinkConfig, taskExecutorProcessSpec);

	log.info("TaskExecutor {} will be started on {} with {}.",
		containerId,
		host,
		taskExecutorProcessSpec);

	final Configuration taskManagerConfig = BootstrapTools.cloneConfiguration(flinkConfig);
	taskManagerConfig.set(TaskManagerOptions.TASK_MANAGER_RESOURCE_ID, containerId);

	final String taskManagerDynamicProperties =
		BootstrapTools.getDynamicPropertiesAsString(flinkClientConfig, taskManagerConfig);

	log.debug("TaskManager configuration: {}", taskManagerConfig);

	ContainerLaunchContext taskExecutorLaunchContext = Utils.createTaskExecutorContext(
		flinkConfig,
		yarnConfig,
		env,
		taskManagerParameters,
		taskManagerDynamicProperties,
		currDir,
		YarnTaskExecutorRunner.class,
		log);

	taskExecutorLaunchContext.getEnvironment()
			.put(ENV_FLINK_NODE_ID, host);
	return taskExecutorLaunchContext;
}
 
Example #21
Source File: YarnServiceTestWithExpiration.java    From incubator-gobblin with Apache License 2.0 5 votes vote down vote up
protected ContainerLaunchContext newContainerLaunchContext(Container container, String helixInstanceName)
    throws IOException {
  try {
    Thread.sleep(1000);
  } catch (InterruptedException e) {
    Thread.currentThread().interrupt();
  }
  return BuilderUtils.newContainerLaunchContext(Collections.emptyMap(), Collections.emptyMap(),
      Arrays.asList("sleep", "60000"), Collections.emptyMap(), null, Collections.emptyMap());
}
 
Example #22
Source File: ContainerLauncherLaunchRequestEvent.java    From tez with Apache License 2.0 5 votes vote down vote up
public ContainerLauncherLaunchRequestEvent(ContainerLaunchContext clc,
                                           Container container, int launcherId, int schedulerId,
                                           int taskCommId) {
  super(container.getId(), container.getNodeId(), container
      .getContainerToken(), ContainerLauncherEventType.CONTAINER_LAUNCH_REQUEST,
      launcherId, schedulerId, taskCommId);
  this.clc = clc;
  this.container = container;
}
 
Example #23
Source File: TestNMClientAsync.java    From big-c with Apache License 2.0 5 votes vote down vote up
private NMClient mockNMClient(int mode)
    throws YarnException, IOException {
  NMClient client = mock(NMClient.class);
  switch (mode) {
    case 0:
      when(client.startContainer(any(Container.class),
          any(ContainerLaunchContext.class))).thenReturn(
              Collections.<String, ByteBuffer>emptyMap());
      when(client.getContainerStatus(any(ContainerId.class),
          any(NodeId.class))).thenReturn(
              recordFactory.newRecordInstance(ContainerStatus.class));
      doNothing().when(client).stopContainer(any(ContainerId.class),
          any(NodeId.class));
      break;
    case 1:
      doThrow(RPCUtil.getRemoteException("Start Exception")).when(client)
          .startContainer(any(Container.class),
              any(ContainerLaunchContext.class));
      doThrow(RPCUtil.getRemoteException("Query Exception")).when(client)
          .getContainerStatus(any(ContainerId.class), any(NodeId.class));
      doThrow(RPCUtil.getRemoteException("Stop Exception")).when(client)
          .stopContainer(any(ContainerId.class), any(NodeId.class));
      break;
    case 2:
      when(client.startContainer(any(Container.class),
          any(ContainerLaunchContext.class))).thenReturn(
              Collections.<String, ByteBuffer>emptyMap());
      when(client.getContainerStatus(any(ContainerId.class),
          any(NodeId.class))).thenReturn(
              recordFactory.newRecordInstance(ContainerStatus.class));
      doThrow(RPCUtil.getRemoteException("Stop Exception")).when(client)
          .stopContainer(any(ContainerId.class), any(NodeId.class));
  }
  return client;
}
 
Example #24
Source File: TestApplication.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private Container createMockedContainer(ApplicationId appId, int containerId) {
  ApplicationAttemptId appAttemptId =
      BuilderUtils.newApplicationAttemptId(appId, 1);
  ContainerId cId = BuilderUtils.newContainerId(appAttemptId, containerId);
  Container c = mock(Container.class);
  when(c.getContainerId()).thenReturn(cId);
  ContainerLaunchContext launchContext = mock(ContainerLaunchContext.class);
  when(c.getLaunchContext()).thenReturn(launchContext);
  when(launchContext.getApplicationACLs()).thenReturn(
      new HashMap<ApplicationAccessType, String>());
  return c;
}
 
Example #25
Source File: TestDelegationTokenRenewer.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test(timeout=20000)
public void testAppSubmissionWithInvalidDelegationToken() throws Exception {
  Configuration conf = new Configuration();
  conf.set(
      CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
      "kerberos");
  UserGroupInformation.setConfiguration(conf);
  MockRM rm = new MockRM(conf) {
    @Override
    protected void doSecureLogin() throws IOException {
      // Skip the login.
    }
  };
  ByteBuffer tokens = ByteBuffer.wrap("BOGUS".getBytes()); 
  ContainerLaunchContext amContainer =
      ContainerLaunchContext.newInstance(
          new HashMap<String, LocalResource>(), new HashMap<String, String>(),
          new ArrayList<String>(), new HashMap<String, ByteBuffer>(), tokens,
          new HashMap<ApplicationAccessType, String>());
  ApplicationSubmissionContext appSubContext =
      ApplicationSubmissionContext.newInstance(
          ApplicationId.newInstance(1234121, 0),
          "BOGUS", "default", Priority.UNDEFINED, amContainer, false,
          true, 1, Resource.newInstance(1024, 1), "BOGUS");
  SubmitApplicationRequest request =
      SubmitApplicationRequest.newInstance(appSubContext);
  try {
    rm.getClientRMService().submitApplication(request);
    fail("Error was excepted.");
  } catch (YarnException e) {
    Assert.assertTrue(e.getMessage().contains(
        "Bad header found in token storage"));
  }
}
 
Example #26
Source File: TestYarnClient.java    From big-c with Apache License 2.0 5 votes vote down vote up
private ApplicationId createApp(YarnClient rmClient, boolean unmanaged) 
  throws Exception {
  YarnClientApplication newApp = rmClient.createApplication();

  ApplicationId appId = newApp.getNewApplicationResponse().getApplicationId();

  // Create launch context for app master
  ApplicationSubmissionContext appContext
    = Records.newRecord(ApplicationSubmissionContext.class);

  // set the application id
  appContext.setApplicationId(appId);

  // set the application name
  appContext.setApplicationName("test");

  // Set the priority for the application master
  Priority pri = Records.newRecord(Priority.class);
  pri.setPriority(1);
  appContext.setPriority(pri);

  // Set the queue to which this application is to be submitted in the RM
  appContext.setQueue("default");

  // Set up the container launch context for the application master
  ContainerLaunchContext amContainer
    = Records.newRecord(ContainerLaunchContext.class);
  appContext.setAMContainerSpec(amContainer);
  appContext.setResource(Resource.newInstance(1024, 1));
  appContext.setUnmanagedAM(unmanaged);

  // Submit the application to the applications manager
  rmClient.submitApplication(appContext);

  return appId;
}
 
Example #27
Source File: TestDockerContainerExecutorWithMocks.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test(expected = IllegalArgumentException.class)
public void testContainerLaunchNullImage() throws IOException {
  String appSubmitter = "nobody";
  String appId = "APP_ID";
  String containerId = "CONTAINER_ID";
  String testImage = "";

  Container container = mock(Container.class, RETURNS_DEEP_STUBS);
  ContainerId cId = mock(ContainerId.class, RETURNS_DEEP_STUBS);
  ContainerLaunchContext context = mock(ContainerLaunchContext.class);
  HashMap<String, String> env = new HashMap<String,String>();

  when(container.getContainerId()).thenReturn(cId);
  when(container.getLaunchContext()).thenReturn(context);
  when(cId.getApplicationAttemptId().getApplicationId().toString()).thenReturn(appId);
  when(cId.toString()).thenReturn(containerId);

  when(context.getEnvironment()).thenReturn(env);
  env.put(YarnConfiguration.NM_DOCKER_CONTAINER_EXECUTOR_IMAGE_NAME, testImage);
  dockerContainerExecutor.getConf()
      .set(YarnConfiguration.NM_DOCKER_CONTAINER_EXECUTOR_IMAGE_NAME, testImage);
  Path scriptPath = new Path("file:///bin/echo");
  Path tokensPath = new Path("file:///dev/null");

  Path pidFile = new Path(workDir, "pid.txt");

  dockerContainerExecutor.activateContainer(cId, pidFile);
  dockerContainerExecutor.launchContainer(new ContainerStartContext.Builder()
      .setContainer(container)
      .setNmPrivateContainerScriptPath(scriptPath)
      .setNmPrivateTokensPath(tokensPath)
      .setUser(appSubmitter)
      .setAppId(appId)
      .setContainerWorkDir(workDir)
      .setLocalDirs(dirsHandler.getLocalDirs())
      .setLogDirs(dirsHandler.getLogDirs())
      .build());
}
 
Example #28
Source File: TestNMClientAsync.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test (timeout = 10000)
public void testOutOfOrder() throws Exception {
  CyclicBarrier barrierA = new CyclicBarrier(2);
  CyclicBarrier barrierB = new CyclicBarrier(2);
  CyclicBarrier barrierC = new CyclicBarrier(2);
  asyncClient = new MockNMClientAsync2(barrierA, barrierB, barrierC);
  asyncClient.init(new Configuration());
  asyncClient.start();

  final Container container = mockContainer(1);
  final ContainerLaunchContext clc =
      recordFactory.newRecordInstance(ContainerLaunchContext.class);

  // start container from another thread
  Thread t = new Thread() {
    @Override
    public void run() {
      asyncClient.startContainerAsync(container, clc);
    }
  };
  t.start();

  barrierA.await();
  asyncClient.stopContainerAsync(container.getId(), container.getNodeId());
  barrierC.await();

  Assert.assertFalse("Starting and stopping should be out of order",
      ((TestCallbackHandler2) asyncClient.getCallbackHandler())
          .exceptionOccurred.get());
}
 
Example #29
Source File: ContainerImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public ContainerLaunchContext getLaunchContext() {
  this.readLock.lock();
  try {
    return launchContext;
  } finally {
    this.readLock.unlock();
  }
}
 
Example #30
Source File: YarnTypes.java    From reef with Apache License 2.0 5 votes vote down vote up
/**
 * @return a ContainerLaunchContext with the given commands and LocalResources.
 */
public static ContainerLaunchContext getContainerLaunchContext(
    final List<String> commands,
    final Map<String, LocalResource> localResources,
    final byte[] securityTokenBuffer) {
  return getContainerLaunchContext(commands, localResources, securityTokenBuffer,
      new HashMap<String, String>(), null);
}