org.apache.hadoop.yarn.server.utils.BuilderUtils Java Examples

The following examples show how to use org.apache.hadoop.yarn.server.utils.BuilderUtils. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestClientRMService.java    From big-c with Apache License 2.0 6 votes vote down vote up
private void checkTokenCancellation(ClientRMService rmService,
    UserGroupInformation owner, UserGroupInformation renewer)
    throws IOException, YarnException {
  RMDelegationTokenIdentifier tokenIdentifier =
      new RMDelegationTokenIdentifier(new Text(owner.getUserName()),
        new Text(renewer.getUserName()), null);
  Token<?> token =
      new Token<RMDelegationTokenIdentifier>(tokenIdentifier, dtsm);
  org.apache.hadoop.yarn.api.records.Token dToken =
      BuilderUtils.newDelegationToken(token.getIdentifier(), token.getKind()
        .toString(), token.getPassword(), token.getService().toString());
  CancelDelegationTokenRequest request =
      Records.newRecord(CancelDelegationTokenRequest.class);
  request.setDelegationToken(dToken);
  rmService.cancelDelegationToken(request);
}
 
Example #2
Source File: TestRMAppAttemptTransitions.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testAMCrashAtAllocated() {
  Container amContainer = allocateApplicationAttempt();
  String containerDiagMsg = "some error";
  int exitCode = 123;
  ContainerStatus cs =
      BuilderUtils.newContainerStatus(amContainer.getId(),
        ContainerState.COMPLETE, containerDiagMsg, exitCode);
  NodeId anyNodeId = NodeId.newInstance("host", 1234);
  applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
    applicationAttempt.getAppAttemptId(), cs, anyNodeId));
  assertEquals(YarnApplicationAttemptState.ALLOCATED,
      applicationAttempt.createApplicationAttemptState());
  sendAttemptUpdateSavedEvent(applicationAttempt);
  assertEquals(RMAppAttemptState.FAILED,
    applicationAttempt.getAppAttemptState());
  verifyTokenCount(applicationAttempt.getAppAttemptId(), 1);
  verifyApplicationAttemptFinished(RMAppAttemptState.FAILED);
  boolean shouldCheckURL = (applicationAttempt.getTrackingUrl() != null);
  verifyAMCrashAtAllocatedDiagnosticInfo(applicationAttempt.getDiagnostics(),
    exitCode, shouldCheckURL);
}
 
Example #3
Source File: TestLogAggregationService.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test(timeout=20000)
public void testStopAfterError() throws Exception {
  DeletionService delSrvc = mock(DeletionService.class);

  // get the AppLogAggregationImpl thread to crash
  LocalDirsHandlerService mockedDirSvc = mock(LocalDirsHandlerService.class);
  when(mockedDirSvc.getLogDirs()).thenThrow(new RuntimeException());
  
  LogAggregationService logAggregationService =
      new LogAggregationService(dispatcher, this.context, delSrvc,
                                mockedDirSvc);
  logAggregationService.init(this.conf);
  logAggregationService.start();

  ApplicationId application1 = BuilderUtils.newApplicationId(1234, 1);
  logAggregationService.handle(new LogHandlerAppStartedEvent(
          application1, this.user, null,
          ContainerLogsRetentionPolicy.ALL_CONTAINERS, this.acls));

  logAggregationService.stop();
  assertEquals(0, logAggregationService.getNumAggregators());
  logAggregationService.close();
}
 
Example #4
Source File: TestClientRMService.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private void checkTokenCancellation(ClientRMService rmService,
    UserGroupInformation owner, UserGroupInformation renewer)
    throws IOException, YarnException {
  RMDelegationTokenIdentifier tokenIdentifier =
      new RMDelegationTokenIdentifier(new Text(owner.getUserName()),
        new Text(renewer.getUserName()), null);
  Token<?> token =
      new Token<RMDelegationTokenIdentifier>(tokenIdentifier, dtsm);
  org.apache.hadoop.yarn.api.records.Token dToken =
      BuilderUtils.newDelegationToken(token.getIdentifier(), token.getKind()
        .toString(), token.getPassword(), token.getService().toString());
  CancelDelegationTokenRequest request =
      Records.newRecord(CancelDelegationTokenRequest.class);
  request.setDelegationToken(dToken);
  rmService.cancelDelegationToken(request);
}
 
Example #5
Source File: TestRMAppAttemptTransitions.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testSuccessfulFinishingToFinished() {
  Container amContainer = allocateApplicationAttempt();
  launchApplicationAttempt(amContainer);
  runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
  FinalApplicationStatus finalStatus = FinalApplicationStatus.SUCCEEDED;
  String trackingUrl = "mytrackingurl";
  String diagnostics = "Successful";
  unregisterApplicationAttempt(amContainer, finalStatus, trackingUrl,
      diagnostics);
  NodeId anyNodeId = NodeId.newInstance("host", 1234);
  applicationAttempt.handle(
      new RMAppAttemptContainerFinishedEvent(
          applicationAttempt.getAppAttemptId(),
          BuilderUtils.newContainerStatus(amContainer.getId(),
              ContainerState.COMPLETE, "", 0), anyNodeId));
  testAppAttemptFinishedState(amContainer, finalStatus, trackingUrl,
      diagnostics, 0, false);
}
 
Example #6
Source File: TestNMWebServicesApps.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private HashMap<String, String> addAppContainers(Application app) 
    throws IOException {
  Dispatcher dispatcher = new AsyncDispatcher();
  ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
      app.getAppId(), 1);
  Container container1 = new MockContainer(appAttemptId, dispatcher, conf,
      app.getUser(), app.getAppId(), 1);
  Container container2 = new MockContainer(appAttemptId, dispatcher, conf,
      app.getUser(), app.getAppId(), 2);
  nmContext.getContainers()
      .put(container1.getContainerId(), container1);
  nmContext.getContainers()
      .put(container2.getContainerId(), container2);

  app.getContainers().put(container1.getContainerId(), container1);
  app.getContainers().put(container2.getContainerId(), container2);
  HashMap<String, String> hash = new HashMap<String, String>();
  hash.put(container1.getContainerId().toString(), container1
      .getContainerId().toString());
  hash.put(container2.getContainerId().toString(), container2
      .getContainerId().toString());
  return hash;
}
 
Example #7
Source File: ClientRMService.java    From big-c with Apache License 2.0 6 votes vote down vote up
private NodeReport createNodeReports(RMNode rmNode) {    
  SchedulerNodeReport schedulerNodeReport = 
      scheduler.getNodeReport(rmNode.getNodeID());
  Resource used = BuilderUtils.newResource(0, 0);
  int numContainers = 0;
  if (schedulerNodeReport != null) {
    used = schedulerNodeReport.getUsedResource();
    numContainers = schedulerNodeReport.getNumContainers();
  } 
  
  NodeReport report =
      BuilderUtils.newNodeReport(rmNode.getNodeID(), rmNode.getState(),
          rmNode.getHttpAddress(), rmNode.getRackName(), used,
          rmNode.getTotalCapability(), numContainers,
          rmNode.getHealthReport(), rmNode.getLastHealthReportTime(),
          rmNode.getNodeLabels());

  return report;
}
 
Example #8
Source File: TestClientRMService.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testForceKillNonExistingApplication() throws YarnException {
  RMContext rmContext = mock(RMContext.class);
  when(rmContext.getRMApps()).thenReturn(
      new ConcurrentHashMap<ApplicationId, RMApp>());
  ClientRMService rmService = new ClientRMService(rmContext, null, null,
      null, null, null);
  ApplicationId applicationId =
      BuilderUtils.newApplicationId(System.currentTimeMillis(), 0);
  KillApplicationRequest request =
      KillApplicationRequest.newInstance(applicationId);
  try {
    rmService.forceKillApplication(request);
    Assert.fail();
  } catch (ApplicationNotFoundException ex) {
    Assert.assertEquals(ex.getMessage(),
        "Trying to kill an absent " +
            "application " + request.getApplicationId());
  }
}
 
Example #9
Source File: TestResourceTrackerService.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testNodeRegistrationSuccess() throws Exception {
  writeToHostsFile("host2");
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, hostFile
      .getAbsolutePath());
  rm = new MockRM(conf);
  rm.start();

  ResourceTrackerService resourceTrackerService = rm.getResourceTrackerService();
  RegisterNodeManagerRequest req = Records.newRecord(
      RegisterNodeManagerRequest.class);
  NodeId nodeId = NodeId.newInstance("host2", 1234);
  Resource capability = BuilderUtils.newResource(1024, 1, 1);
  req.setResource(capability);
  req.setNodeId(nodeId);
  req.setHttpPort(1234);
  req.setNMVersion(YarnVersionInfo.getVersion());
  // trying to register a invalid node.
  RegisterNodeManagerResponse response = resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.NORMAL,response.getNodeAction());
}
 
Example #10
Source File: TestNMReconnect.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testReconnect() throws Exception {
  String hostname1 = "localhost1";
  Resource capability = BuilderUtils.newResource(1024, 1);

  RegisterNodeManagerRequest request1 = recordFactory
      .newRecordInstance(RegisterNodeManagerRequest.class);
  NodeId nodeId1 = NodeId.newInstance(hostname1, 0);
  request1.setNodeId(nodeId1);
  request1.setHttpPort(0);
  request1.setResource(capability);
  resourceTrackerService.registerNodeManager(request1);

  Assert.assertEquals(RMNodeEventType.STARTED, rmNodeEvents.get(0).getType());

  rmNodeEvents.clear();
  resourceTrackerService.registerNodeManager(request1);
  Assert.assertEquals(RMNodeEventType.RECONNECTED,
      rmNodeEvents.get(0).getType());

  rmNodeEvents.clear();
  resourceTrackerService.registerNodeManager(request1);
  capability = BuilderUtils.newResource(1024, 2);
  request1.setResource(capability);
  Assert.assertEquals(RMNodeEventType.RECONNECTED,
      rmNodeEvents.get(0).getType());
}
 
Example #11
Source File: MockApp.java    From big-c with Apache License 2.0 5 votes vote down vote up
public MockApp(String user, long clusterTimeStamp, int uniqId) {
  super();
  this.user = user;
  // Add an application and the corresponding containers
  RecordFactory recordFactory = RecordFactoryProvider
      .getRecordFactory(new Configuration());
  this.appId = BuilderUtils.newApplicationId(recordFactory, clusterTimeStamp,
      uniqId);
  appState = ApplicationState.NEW;
}
 
Example #12
Source File: TestProxyUriUtils.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testGetProxyUriFromPluginsReturnsNullIfNoPlugins()
    throws URISyntaxException {
  ApplicationId id = BuilderUtils.newApplicationId(6384623l, 5);
  List<TrackingUriPlugin> list =
      Lists.newArrayListWithExpectedSize(0);
  assertNull(ProxyUriUtils.getUriFromTrackingPlugins(id, list));
}
 
Example #13
Source File: TestApplicationMasterLauncher.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public StartContainersResponse
    startContainers(StartContainersRequest requests)
        throws YarnException {
  StartContainerRequest request = requests.getStartContainerRequests().get(0);
  LOG.info("Container started by MyContainerManager: " + request);
  launched = true;
  Map<String, String> env =
      request.getContainerLaunchContext().getEnvironment();

  Token containerToken = request.getContainerToken();
  ContainerTokenIdentifier tokenId = null;

  try {
    tokenId = BuilderUtils.newContainerTokenIdentifier(containerToken);
  } catch (IOException e) {
    throw RPCUtil.getRemoteException(e);
  }

  ContainerId containerId = tokenId.getContainerID();
  containerIdAtContainerManager = containerId.toString();
  attemptIdAtContainerManager =
      containerId.getApplicationAttemptId().toString();
  nmHostAtContainerManager = tokenId.getNmHostAddress();
  submitTimeAtContainerManager =
      Long.parseLong(env.get(ApplicationConstants.APP_SUBMIT_TIME_ENV));
  maxAppAttempts =
      Integer.parseInt(env.get(ApplicationConstants.MAX_APP_ATTEMPTS_ENV));
  return StartContainersResponse.newInstance(
    new HashMap<String, ByteBuffer>(), new ArrayList<ContainerId>(),
    new HashMap<ContainerId, SerializedException>());
}
 
Example #14
Source File: TestProxyUriUtils.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testGetProxyUriFromPluginsReturnsNullIfNoPlugins()
    throws URISyntaxException {
  ApplicationId id = BuilderUtils.newApplicationId(6384623l, 5);
  List<TrackingUriPlugin> list =
      Lists.newArrayListWithExpectedSize(0);
  assertNull(ProxyUriUtils.getUriFromTrackingPlugins(id, list));
}
 
Example #15
Source File: TestClientRMService.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test (expected = ApplicationNotFoundException.class)
public void testMoveAbsentApplication() throws YarnException {
  RMContext rmContext = mock(RMContext.class);
  when(rmContext.getRMApps()).thenReturn(
      new ConcurrentHashMap<ApplicationId, RMApp>());
  ClientRMService rmService = new ClientRMService(rmContext, null, null,
      null, null, null);
  ApplicationId applicationId =
      BuilderUtils.newApplicationId(System.currentTimeMillis(), 0);
  MoveApplicationAcrossQueuesRequest request =
      MoveApplicationAcrossQueuesRequest.newInstance(applicationId, "newqueue");
  rmService.moveApplicationAcrossQueues(request);
}
 
Example #16
Source File: TestRMNodeTransitions.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test(timeout=20000)
public void testUpdateHeartbeatResponseForCleanup() {
  RMNodeImpl node = getRunningNode();
  NodeId nodeId = node.getNodeID();

  // Expire a container
ContainerId completedContainerId = BuilderUtils.newContainerId(
		BuilderUtils.newApplicationAttemptId(
				BuilderUtils.newApplicationId(0, 0), 0), 0);
  node.handle(new RMNodeCleanContainerEvent(nodeId, completedContainerId));
  Assert.assertEquals(1, node.getContainersToCleanUp().size());

  // Finish an application
  ApplicationId finishedAppId = BuilderUtils.newApplicationId(0, 1);
  node.handle(new RMNodeCleanAppEvent(nodeId, finishedAppId));
  Assert.assertEquals(1, node.getAppsToCleanup().size());

  // Verify status update does not clear containers/apps to cleanup
  // but updating heartbeat response for cleanup does
  RMNodeStatusEvent statusEvent = getMockRMNodeStatusEvent();
  node.handle(statusEvent);
  Assert.assertEquals(1, node.getContainersToCleanUp().size());
  Assert.assertEquals(1, node.getAppsToCleanup().size());
  NodeHeartbeatResponse hbrsp = Records.newRecord(NodeHeartbeatResponse.class);
  node.updateNodeHeartbeatResponseForCleanup(hbrsp);
  Assert.assertEquals(0, node.getContainersToCleanUp().size());
  Assert.assertEquals(0, node.getAppsToCleanup().size());
  Assert.assertEquals(1, hbrsp.getContainersToCleanup().size());
  Assert.assertEquals(completedContainerId, hbrsp.getContainersToCleanup().get(0));
  Assert.assertEquals(1, hbrsp.getApplicationsToCleanup().size());
  Assert.assertEquals(finishedAppId, hbrsp.getApplicationsToCleanup().get(0));
}
 
Example #17
Source File: FSAppAttempt.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Create and return a container object reflecting an allocation for the
 * given appliction on the given node with the given capability and
 * priority.
 */
public Container createContainer(
    FSSchedulerNode node, Resource capability, Priority priority) {

  NodeId nodeId = node.getRMNode().getNodeID();
  ContainerId containerId = BuilderUtils.newContainerId(
      getApplicationAttemptId(), getNewContainerId());

  // Create the container
  Container container =
      BuilderUtils.newContainer(containerId, nodeId, node.getRMNode()
          .getHttpAddress(), capability, priority, null);

  return container;
}
 
Example #18
Source File: TestRMAppAttemptTransitions.java    From big-c with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
private Container allocateApplicationAttempt() {
  scheduleApplicationAttempt();
  
  // Mock the allocation of AM container 
  Container container = mock(Container.class);
  Resource resource = BuilderUtils.newResource(2048, 1);
  when(container.getId()).thenReturn(
      BuilderUtils.newContainerId(applicationAttempt.getAppAttemptId(), 1));
  when(container.getResource()).thenReturn(resource);
  Allocation allocation = mock(Allocation.class);
  when(allocation.getContainers()).
      thenReturn(Collections.singletonList(container));
  when(
      scheduler.allocate(
          any(ApplicationAttemptId.class), 
          any(List.class), 
          any(List.class), 
          any(List.class), 
          any(List.class))).
  thenReturn(allocation);
  RMContainer rmContainer = mock(RMContainerImpl.class);
  when(scheduler.getRMContainer(container.getId())).
      thenReturn(rmContainer);
  
  applicationAttempt.handle(
      new RMAppAttemptContainerAllocatedEvent(
          applicationAttempt.getAppAttemptId()));
  
  assertEquals(RMAppAttemptState.ALLOCATED_SAVING, 
      applicationAttempt.getAppAttemptState());
  applicationAttempt.handle(
      new RMAppAttemptEvent(applicationAttempt.getAppAttemptId(),
          RMAppAttemptEventType.ATTEMPT_NEW_SAVED));
  
  testAppAttemptAllocatedState(container);
  
  return container;
}
 
Example #19
Source File: HostLocalTest.java    From attic-apex-core with Apache License 2.0 5 votes vote down vote up
@Test
public void testNodeLocal()
{
  LogicalPlan dag = new LogicalPlan();
  dag.getAttributes().put(com.datatorrent.api.Context.DAGContext.APPLICATION_PATH, new File("target", HostLocalTest.class.getName()).getAbsolutePath());
  dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());


  GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
  dag.setOperatorAttribute(o1,OperatorContext.MEMORY_MB,256);

  GenericTestOperator partitioned = dag.addOperator("partitioned", GenericTestOperator.class);
  dag.setOperatorAttribute(partitioned,OperatorContext.MEMORY_MB,256);
  dag.getMeta(partitioned).getAttributes().put(OperatorContext.LOCALITY_HOST, "host1");

  dag.addStream("o1_outport1", o1.outport1, partitioned.inport1).setLocality(Locality.NODE_LOCAL);

  StreamingContainerManager scm = new StreamingContainerManager(dag);

  ResourceRequestHandler rr = new ResourceRequestHandler();

  int containerMem = 1000;
  Map<String, NodeReport> nodeReports = Maps.newHashMap();
  NodeReport nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host1", 0),
      NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);
  nodeReports.put(nr.getNodeId().getHost(), nr);
  nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host2", 0),
      NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);
  nodeReports.put(nr.getNodeId().getHost(), nr);

  // set resources
  rr.updateNodeReports(Lists.newArrayList(nodeReports.values()));

  for (ContainerStartRequest csr : scm.containerStartRequests) {
    String host = rr.getHost(csr, true);
    csr.container.host = host;
    Assert.assertEquals("Hosts set to host1", "host1", host);
  }

}
 
Example #20
Source File: HostLocalTest.java    From attic-apex-core with Apache License 2.0 5 votes vote down vote up
@Test
public void testUnavailableResources()
{
  LogicalPlan dag = new LogicalPlan();
  dag.getAttributes().put(com.datatorrent.api.Context.DAGContext.APPLICATION_PATH, new File("target", HostLocalTest.class.getName()).getAbsolutePath());
  dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());

  GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
  dag.getMeta(o1).getAttributes().put(OperatorContext.LOCALITY_HOST, "host2");

  GenericTestOperator partitioned = dag.addOperator("partitioned", GenericTestOperator.class);
  dag.addStream("o1_outport1", o1.outport1, partitioned.inport1).setLocality(Locality.CONTAINER_LOCAL);
  dag.setOperatorAttribute(o1, OperatorContext.MEMORY_MB, 256);
  dag.setOperatorAttribute(o1, OperatorContext.VCORES, 2);
  dag.setOperatorAttribute(partitioned, OperatorContext.VCORES, 1);

  StreamingContainerManager scm = new StreamingContainerManager(dag);

  ResourceRequestHandler rr = new ResourceRequestHandler();

  int containerMem = 1000;
  Map<String, NodeReport> nodeReports = Maps.newHashMap();
  NodeReport nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host1", 0),
      NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);
  nodeReports.put(nr.getNodeId().getHost(), nr);
  nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host2", 0),
      NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);
  nodeReports.put(nr.getNodeId().getHost(), nr);

  // set resources
  rr.updateNodeReports(Lists.newArrayList(nodeReports.values()));
  Assert.assertEquals("number of containers is 1", 1, scm.containerStartRequests.size());
  for (ContainerStartRequest csr : scm.containerStartRequests) {
    String host = rr.getHost(csr, true);
    Assert.assertEquals("number of vcores", 3, csr.container.getRequiredVCores());
    Assert.assertNull("Host is null", host);
  }
}
 
Example #21
Source File: TestRMNodeTransitions.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private RMNodeImpl getRebootedNode() {
  NodeId nodeId = BuilderUtils.newNodeId("localhost", 0);
  Resource capability = Resource.newInstance(4096, 4, 4);
  RMNodeImpl node = new RMNodeImpl(nodeId, rmContext,null, 0, 0,
      null, capability, null);
  node.handle(new RMNodeStartedEvent(node.getNodeID(), null, null));
  Assert.assertEquals(NodeState.RUNNING, node.getState());
  node.handle(new RMNodeEvent(node.getNodeID(), RMNodeEventType.REBOOTING));
  Assert.assertEquals(NodeState.REBOOTED, node.getState());
  return node;
}
 
Example #22
Source File: TestDelegationTokenRenewer.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test(timeout=60000)
public void testAppRejectionWithCancelledDelegationToken() throws Exception {
  MyFS dfs = (MyFS)FileSystem.get(conf);
  LOG.info("dfs="+(Object)dfs.hashCode() + ";conf="+conf.hashCode());

  MyToken token = dfs.getDelegationToken("user1");
  token.cancelToken();

  Credentials ts = new Credentials();
  ts.addToken(token.getKind(), token);
  
  // register the tokens for renewal
  ApplicationId appId =  BuilderUtils.newApplicationId(0, 0);
  delegationTokenRenewer.addApplicationAsync(appId, ts, true, "user");
  int waitCnt = 20;
  while (waitCnt-- >0) {
    if (!eventQueue.isEmpty()) {
      Event evt = eventQueue.take();
      if (evt.getType() == RMAppEventType.APP_REJECTED) {
        Assert.assertTrue(
            ((RMAppEvent) evt).getApplicationId().equals(appId));
        return;
      }
    } else {
      Thread.sleep(500);
    }
  }
  fail("App submission with a cancelled token should have failed");
}
 
Example #23
Source File: LeafQueue.java    From hadoop with Apache License 2.0 5 votes vote down vote up
Container createContainer(FiCaSchedulerApp application, FiCaSchedulerNode node, 
    Resource capability, Priority priority) {

  NodeId nodeId = node.getRMNode().getNodeID();
  ContainerId containerId = BuilderUtils.newContainerId(application
      .getApplicationAttemptId(), application.getNewContainerId());

  // Create the container
  Container container =
      BuilderUtils.newContainer(containerId, nodeId, node.getRMNode()
        .getHttpAddress(), capability, priority, null);

  return container;
}
 
Example #24
Source File: TestQueueMetrics.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private static AppSchedulingInfo mockApp(String user) {
  AppSchedulingInfo app = mock(AppSchedulingInfo.class);
  when(app.getUser()).thenReturn(user);
  ApplicationId appId = BuilderUtils.newApplicationId(1, 1);
  ApplicationAttemptId id = BuilderUtils.newApplicationAttemptId(appId, 1);
  when(app.getApplicationAttemptId()).thenReturn(id);
  return app;
}
 
Example #25
Source File: TestRMAppAttemptTransitions.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testRunningToFailed() {
  Container amContainer = allocateApplicationAttempt();
  launchApplicationAttempt(amContainer);
  runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
  String containerDiagMsg = "some error";
  int exitCode = 123;
  ContainerStatus cs = BuilderUtils.newContainerStatus(amContainer.getId(),
      ContainerState.COMPLETE, containerDiagMsg, exitCode);
  ApplicationAttemptId appAttemptId = applicationAttempt.getAppAttemptId();
  NodeId anyNodeId = NodeId.newInstance("host", 1234);
  applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
      appAttemptId, cs, anyNodeId));

  // ignored ContainerFinished and Expire at FinalSaving if we were supposed
  // to Failed state.
  assertEquals(RMAppAttemptState.FINAL_SAVING,
    applicationAttempt.getAppAttemptState());
  applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
    applicationAttempt.getAppAttemptId(), BuilderUtils.newContainerStatus(
      amContainer.getId(), ContainerState.COMPLETE, "", 0), anyNodeId));
  applicationAttempt.handle(new RMAppAttemptEvent(
    applicationAttempt.getAppAttemptId(), RMAppAttemptEventType.EXPIRE));
  assertEquals(RMAppAttemptState.FINAL_SAVING,
    applicationAttempt.getAppAttemptState()); 
  assertEquals(YarnApplicationAttemptState.RUNNING,
      applicationAttempt.createApplicationAttemptState());
  sendAttemptUpdateSavedEvent(applicationAttempt);
  assertEquals(RMAppAttemptState.FAILED,
      applicationAttempt.getAppAttemptState());
  assertEquals(0, applicationAttempt.getJustFinishedContainers().size());
  assertEquals(amContainer, applicationAttempt.getMasterContainer());
  assertEquals(0, application.getRanNodes().size());
  String rmAppPageUrl = pjoin(RM_WEBAPP_ADDR, "cluster", "app",
      applicationAttempt.getAppAttemptId().getApplicationId());
  assertEquals(rmAppPageUrl, applicationAttempt.getOriginalTrackingUrl());
  assertEquals(rmAppPageUrl, applicationAttempt.getTrackingUrl());
  verifyAMHostAndPortInvalidated();
  verifyApplicationAttemptFinished(RMAppAttemptState.FAILED);
}
 
Example #26
Source File: QueueACLsTestBase.java    From big-c with Apache License 2.0 5 votes vote down vote up
private ApplicationId submitAppAndGetAppId(String submitter,
    String queueName, boolean setupACLs) throws Exception {

  GetNewApplicationRequest newAppRequest =
      GetNewApplicationRequest.newInstance();

  ApplicationClientProtocol submitterClient = getRMClientForUser(submitter);
  ApplicationId applicationId =
      submitterClient.getNewApplication(newAppRequest).getApplicationId();

  Resource resource = BuilderUtils.newResource(1024, 1);
  Map<ApplicationAccessType, String> acls = createACLs(submitter, setupACLs);
  ContainerLaunchContext amContainerSpec =
      ContainerLaunchContext.newInstance(null, null, null, null, null, acls);

  ApplicationSubmissionContext appSubmissionContext =
      ApplicationSubmissionContext.newInstance(applicationId,
        "applicationName", queueName, null, amContainerSpec, false, true, 1,
        resource, "applicationType");
  appSubmissionContext.setApplicationId(applicationId);
  appSubmissionContext.setQueue(queueName);

  SubmitApplicationRequest submitRequest =
      SubmitApplicationRequest.newInstance(appSubmissionContext);
  submitterClient.submitApplication(submitRequest);
  resourceManager.waitForState(applicationId, RMAppState.ACCEPTED);
  return applicationId;
}
 
Example #27
Source File: TestContainerLaunch.java    From big-c with Apache License 2.0 5 votes vote down vote up
protected Token createContainerToken(ContainerId cId, Priority priority,
    long createTime) throws InvalidToken {
  Resource r = BuilderUtils.newResource(1024, 1);
  ContainerTokenIdentifier containerTokenIdentifier =
      new ContainerTokenIdentifier(cId, context.getNodeId().toString(), user,
        r, System.currentTimeMillis() + 10000L, 123, DUMMY_RM_IDENTIFIER,
        priority, createTime);
  Token containerToken =
      BuilderUtils.newContainerToken(
        context.getNodeId(),
        context.getContainerTokenSecretManager().retrievePassword(
          containerTokenIdentifier), containerTokenIdentifier);
  return containerToken;
}
 
Example #28
Source File: ContainerImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public ContainerStatus cloneAndGetContainerStatus() {
  this.readLock.lock();
  try {
    ContainerStatus containerStatus = BuilderUtils.newContainerStatus(this.containerId,
  	        getCurrentState(), diagnostics.toString(), exitCode);
    containerStatus.setCpuCores(cpuCores);
    return containerStatus;
  } finally {
    this.readLock.unlock();
  }
}
 
Example #29
Source File: TestRMAppAttemptTransitions.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testRunningToKilled() {
  Container amContainer = allocateApplicationAttempt();
  launchApplicationAttempt(amContainer);
  runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
  applicationAttempt.handle(
      new RMAppAttemptEvent(
          applicationAttempt.getAppAttemptId(),
          RMAppAttemptEventType.KILL));

  // ignored ContainerFinished and Expire at FinalSaving if we were supposed
  // to Killed state.
  assertEquals(RMAppAttemptState.FINAL_SAVING,
    applicationAttempt.getAppAttemptState());
  NodeId anyNodeId = NodeId.newInstance("host", 1234);
  applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
    applicationAttempt.getAppAttemptId(), BuilderUtils.newContainerStatus(
      amContainer.getId(), ContainerState.COMPLETE, "", 0), anyNodeId));
  applicationAttempt.handle(new RMAppAttemptEvent(
    applicationAttempt.getAppAttemptId(), RMAppAttemptEventType.EXPIRE));
  assertEquals(RMAppAttemptState.FINAL_SAVING,
    applicationAttempt.getAppAttemptState()); 
  assertEquals(YarnApplicationAttemptState.RUNNING,
      applicationAttempt.createApplicationAttemptState());
  sendAttemptUpdateSavedEvent(applicationAttempt);
  assertEquals(RMAppAttemptState.KILLED,
      applicationAttempt.getAppAttemptState());
  assertEquals(0, applicationAttempt.getJustFinishedContainers().size());
  assertEquals(amContainer, applicationAttempt.getMasterContainer());
  assertEquals(0, application.getRanNodes().size());
  String rmAppPageUrl = pjoin(RM_WEBAPP_ADDR, "cluster", "app",
      applicationAttempt.getAppAttemptId().getApplicationId());
  assertEquals(rmAppPageUrl, applicationAttempt.getOriginalTrackingUrl());
  assertEquals(rmAppPageUrl, applicationAttempt.getTrackingUrl());
  verifyTokenCount(applicationAttempt.getAppAttemptId(), 1);
  verifyAMHostAndPortInvalidated();
  verifyApplicationAttemptFinished(RMAppAttemptState.KILLED);
}
 
Example #30
Source File: TestRMNodeTransitions.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test (timeout = 5000)
public void testStatusChange(){
  //Start the node
  node.handle(new RMNodeStartedEvent(null, null, null));
  //Add info to the queue first
  node.setNextHeartBeat(false);

  ContainerId completedContainerId1 = BuilderUtils.newContainerId(
      BuilderUtils.newApplicationAttemptId(
          BuilderUtils.newApplicationId(0, 0), 0), 0);
  ContainerId completedContainerId2 = BuilderUtils.newContainerId(
      BuilderUtils.newApplicationAttemptId(
          BuilderUtils.newApplicationId(1, 1), 1), 1);
      
  RMNodeStatusEvent statusEvent1 = getMockRMNodeStatusEvent();
  RMNodeStatusEvent statusEvent2 = getMockRMNodeStatusEvent();

  ContainerStatus containerStatus1 = mock(ContainerStatus.class);
  ContainerStatus containerStatus2 = mock(ContainerStatus.class);

  doReturn(completedContainerId1).when(containerStatus1).getContainerId();
  doReturn(Collections.singletonList(containerStatus1))
      .when(statusEvent1).getContainers();
   
  doReturn(completedContainerId2).when(containerStatus2).getContainerId();
  doReturn(Collections.singletonList(containerStatus2))
      .when(statusEvent2).getContainers();

  verify(scheduler,times(1)).handle(any(NodeUpdateSchedulerEvent.class)); 
  node.handle(statusEvent1);
  node.handle(statusEvent2);
  verify(scheduler,times(1)).handle(any(NodeUpdateSchedulerEvent.class));
  Assert.assertEquals(2, node.getQueueSize());
  node.handle(new RMNodeEvent(node.getNodeID(), RMNodeEventType.EXPIRE));
  Assert.assertEquals(0, node.getQueueSize());
}