org.apache.hadoop.yarn.util.Records Java Examples

@Public
@Stable
public static ApplicationSubmissionContext newInstance(
    ApplicationId applicationId, String applicationName, String queue,
    ContainerLaunchContext amContainer, boolean isUnmanagedAM,
    boolean cancelTokensWhenComplete, int maxAppAttempts,
    String applicationType, boolean keepContainers,
    String appLabelExpression, ResourceRequest resourceRequest) {
  ApplicationSubmissionContext context =
      Records.newRecord(ApplicationSubmissionContext.class);
  context.setApplicationId(applicationId);
  context.setApplicationName(applicationName);
  context.setQueue(queue);
  context.setAMContainerSpec(amContainer);
  context.setUnmanagedAM(isUnmanagedAM);
  context.setCancelTokensWhenComplete(cancelTokensWhenComplete);
  context.setMaxAppAttempts(maxAppAttempts);
  context.setApplicationType(applicationType);
  context.setKeepContainersAcrossApplicationAttempts(keepContainers);
  context.setNodeLabelExpression(appLabelExpression);
  context.setAMContainerResourceRequest(resourceRequest);
  return context;
}
 

@Public
@Stable
public static AllocateResponse newInstance(int responseId,
    List<ContainerStatus> completedContainers,
    List<Container> allocatedContainers, List<NodeReport> updatedNodes,
    Resource availResources, AMCommand command, int numClusterNodes,
    PreemptionMessage preempt, List<NMToken> nmTokens) {
  AllocateResponse response = Records.newRecord(AllocateResponse.class);
  response.setNumClusterNodes(numClusterNodes);
  response.setResponseId(responseId);
  response.setCompletedContainersStatuses(completedContainers);
  response.setAllocatedContainers(allocatedContainers);
  response.setUpdatedNodes(updatedNodes);
  response.setAvailableResources(availResources);
  response.setAMCommand(command);
  response.setPreemptionMessage(preempt);
  response.setNMTokens(nmTokens);
  return response;
}
 

@Override
public long renew(Token<?> token, Configuration conf) throws IOException,
    InterruptedException {

  org.apache.hadoop.yarn.api.records.Token dToken =
      org.apache.hadoop.yarn.api.records.Token.newInstance(
        token.getIdentifier(), token.getKind().toString(),
        token.getPassword(), token.getService().toString());

  MRClientProtocol histProxy = instantiateHistoryProxy(conf,
      SecurityUtil.getTokenServiceAddr(token));
  try {
    RenewDelegationTokenRequest request = Records
        .newRecord(RenewDelegationTokenRequest.class);
    request.setDelegationToken(dToken);
    return histProxy.renewDelegationToken(request).getNextExpirationTime();
  } finally {
    stopHistoryProxy(histProxy);
  }

}
 

@Override
public List<ApplicationAttemptReport> getApplicationAttempts(
    ApplicationId appId) throws YarnException, IOException {
  try {
    GetApplicationAttemptsRequest request = Records
        .newRecord(GetApplicationAttemptsRequest.class);
    request.setApplicationId(appId);
    GetApplicationAttemptsResponse response = rmClient
        .getApplicationAttempts(request);
    return response.getApplicationAttemptList();
  } catch (YarnException e) {
    if (!historyServiceEnabled) {
      // Just throw it as usual if historyService is not enabled.
      throw e;
    }
    // Even if history-service is enabled, treat all exceptions still the same
    // except the following
    if (e.getClass() != ApplicationNotFoundException.class) {
      throw e;
    }
    return historyClient.getApplicationAttempts(appId);
  }
}
 

@Test
public void testNodeRegistrationFailure() throws Exception {
  writeToHostsFile("host1");
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, hostFile
      .getAbsolutePath());
  rm = new MockRM(conf);
  rm.start();
  
  ResourceTrackerService resourceTrackerService = rm.getResourceTrackerService();
  RegisterNodeManagerRequest req = Records.newRecord(
      RegisterNodeManagerRequest.class);
  NodeId nodeId = NodeId.newInstance("host2", 1234);
  req.setNodeId(nodeId);
  req.setHttpPort(1234);
  // trying to register a invalid node.
  RegisterNodeManagerResponse response = resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.SHUTDOWN,response.getNodeAction());
  Assert
    .assertEquals(
      "Disallowed NodeManager from  host2, Sending SHUTDOWN signal to the NodeManager.",
      response.getDiagnosticsMessage());
}
 

@Override
public ApplicationReport getApplicationReport(ApplicationId appId)
    throws YarnException, IOException {
  GetApplicationReportResponse response = null;
  try {
    GetApplicationReportRequest request = Records
        .newRecord(GetApplicationReportRequest.class);
    request.setApplicationId(appId);
    response = rmClient.getApplicationReport(request);
  } catch (YarnException e) {
    if (!historyServiceEnabled) {
      // Just throw it as usual if historyService is not enabled.
      throw e;
    }
    // Even if history-service is enabled, treat all exceptions still the same
    // except the following
    if (!(e.getClass() == ApplicationNotFoundException.class)) {
      throw e;
    }
    return historyClient.getApplicationReport(appId);
  }
  return response.getApplicationReport();
}
 

private void checkTokenCancellation(ClientRMService rmService,
    UserGroupInformation owner, UserGroupInformation renewer)
    throws IOException, YarnException {
  RMDelegationTokenIdentifier tokenIdentifier =
      new RMDelegationTokenIdentifier(new Text(owner.getUserName()),
        new Text(renewer.getUserName()), null);
  Token<?> token =
      new Token<RMDelegationTokenIdentifier>(tokenIdentifier, dtsm);
  org.apache.hadoop.yarn.api.records.Token dToken =
      BuilderUtils.newDelegationToken(token.getIdentifier(), token.getKind()
        .toString(), token.getPassword(), token.getService().toString());
  CancelDelegationTokenRequest request =
      Records.newRecord(CancelDelegationTokenRequest.class);
  request.setDelegationToken(dToken);
  rmService.cancelDelegationToken(request);
}
 

@Override
public void release(ApplicationId applicationId, String resourceKey)
    throws YarnException {
  ReleaseSharedCacheResourceRequest request = Records.newRecord(
      ReleaseSharedCacheResourceRequest.class);
  request.setAppId(applicationId);
  request.setResourceKey(resourceKey);
  try {
    // We do not care about the response because it is empty.
    this.scmClient.release(request);
  } catch (Exception e) {
    // Just catching IOException isn't enough.
    // RPC call can throw ConnectionException.
    throw new YarnException(e);
  }
}
 

private ApplicationReport getApplicationReport(
    YarnApplicationState yarnApplicationState,
    FinalApplicationStatus finalApplicationStatus) {
  ApplicationReport appReport = Mockito.mock(ApplicationReport.class);
  ApplicationResourceUsageReport appResources = Mockito
      .mock(ApplicationResourceUsageReport.class);
  Mockito.when(appReport.getApplicationId()).thenReturn(
      ApplicationId.newInstance(0, 0));
  Mockito.when(appResources.getNeededResources()).thenReturn(
      Records.newRecord(Resource.class));
  Mockito.when(appResources.getReservedResources()).thenReturn(
      Records.newRecord(Resource.class));
  Mockito.when(appResources.getUsedResources()).thenReturn(
      Records.newRecord(Resource.class));
  Mockito.when(appReport.getApplicationResourceUsageReport()).thenReturn(
      appResources);
  Mockito.when(appReport.getYarnApplicationState()).thenReturn(
      yarnApplicationState);
  Mockito.when(appReport.getFinalApplicationStatus()).thenReturn(
      finalApplicationStatus);

  return appReport;
}
 

@Override
public CancelDelegationTokenResponse cancelDelegationToken(
    CancelDelegationTokenRequest request) throws IOException {
  if (!isAllowedDelegationTokenOp()) {
    throw new IOException(
        "Delegation Token can be cancelled only with kerberos authentication");
  }

  org.apache.hadoop.yarn.api.records.Token protoToken = request.getDelegationToken();
  Token<MRDelegationTokenIdentifier> token =
      new Token<MRDelegationTokenIdentifier>(
          protoToken.getIdentifier().array(), protoToken.getPassword()
          .array(), new Text(protoToken.getKind()), new Text(
          protoToken.getService()));

  String user = UserGroupInformation.getCurrentUser().getUserName();
  jhsDTSecretManager.cancelToken(token, user);
  return Records.newRecord(CancelDelegationTokenResponse.class);
}
 

private void startContainer(final YarnRPC rpc,
    org.apache.hadoop.yarn.api.records.Token nmToken,
    org.apache.hadoop.yarn.api.records.Token containerToken,
    NodeId nodeId, String user) throws Exception {

  ContainerLaunchContext context =
      Records.newRecord(ContainerLaunchContext.class);
  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(context,containerToken);
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  ContainerManagementProtocol proxy = null;
  try {
    proxy = getContainerManagementProtocolProxy(rpc, nmToken, nodeId, user);
    StartContainersResponse response = proxy.startContainers(allRequests);
    for(SerializedException ex : response.getFailedRequests().values()){
      parseAndThrowException(ex.deSerialize());
    }
  } finally {
    if (proxy != null) {
      rpc.stopProxy(proxy, conf);
    }
  }
}
 

@Override
public CancelDelegationTokenResponse cancelDelegationToken(
    CancelDelegationTokenRequest request) throws IOException {
    if (!isAllowedDelegationTokenOp()) {
      throw new IOException(
          "Delegation Token can be cancelled only with kerberos authentication");
    }

    org.apache.hadoop.yarn.api.records.Token protoToken = request.getDelegationToken();
    Token<MRDelegationTokenIdentifier> token =
        new Token<MRDelegationTokenIdentifier>(
            protoToken.getIdentifier().array(), protoToken.getPassword()
                .array(), new Text(protoToken.getKind()), new Text(
                protoToken.getService()));

    String user = UserGroupInformation.getCurrentUser().getUserName();
    jhsDTSecretManager.cancelToken(token, user);
    return Records.newRecord(CancelDelegationTokenResponse.class);
}
 

@Override
public RenewDelegationTokenResponse renewDelegationToken(
    RenewDelegationTokenRequest request) throws IOException {
  if (!isAllowedDelegationTokenOp()) {
    throw new IOException(
        "Delegation Token can be renewed only with kerberos authentication");
  }

  org.apache.hadoop.yarn.api.records.Token protoToken = request.getDelegationToken();
  Token<MRDelegationTokenIdentifier> token =
      new Token<MRDelegationTokenIdentifier>(
          protoToken.getIdentifier().array(), protoToken.getPassword()
          .array(), new Text(protoToken.getKind()), new Text(
          protoToken.getService()));

  String user = UserGroupInformation.getCurrentUser().getShortUserName();
  long nextExpTime = jhsDTSecretManager.renewToken(token, user);
  RenewDelegationTokenResponse renewResponse = Records
      .newRecord(RenewDelegationTokenResponse.class);
  renewResponse.setNextExpirationTime(nextExpTime);
  return renewResponse;
}
 

public ResourceRequest createResourceReq(String resource, int memory, int priority,
    int containers, String labelExpression) throws Exception {
  ResourceRequest req = Records.newRecord(ResourceRequest.class);
  req.setResourceName(resource);
  req.setNumContainers(containers);
  Priority pri = Records.newRecord(Priority.class);
  pri.setPriority(priority);
  req.setPriority(pri);
  Resource capability = Records.newRecord(Resource.class);
  capability.setMemory(memory);
  req.setCapability(capability);
  if (labelExpression != null) {
   req.setNodeLabelExpression(labelExpression); 
  }
  return req;
}
 

private void checkTokenRenewal(UserGroupInformation owner,
    UserGroupInformation renewer) throws IOException, YarnException {
  RMDelegationTokenIdentifier tokenIdentifier =
      new RMDelegationTokenIdentifier(
          new Text(owner.getUserName()), new Text(renewer.getUserName()), null);
  Token<?> token =
      new Token<RMDelegationTokenIdentifier>(tokenIdentifier, dtsm);
  org.apache.hadoop.yarn.api.records.Token dToken = BuilderUtils.newDelegationToken(
      token.getIdentifier(), token.getKind().toString(),
      token.getPassword(), token.getService().toString());
  RenewDelegationTokenRequest request =
      Records.newRecord(RenewDelegationTokenRequest.class);
  request.setDelegationToken(dToken);

  RMContext rmContext = mock(RMContext.class);
  ClientRMService rmService = new ClientRMService(
      rmContext, null, null, null, null, dtsm);
  rmService.renewDelegationToken(request);
}
 

@Private
@Unstable
public static GetApplicationReportResponse newInstance(
    ApplicationReport ApplicationReport) {
  GetApplicationReportResponse response =
      Records.newRecord(GetApplicationReportResponse.class);
  response.setApplicationReport(ApplicationReport);
  return response;
}
 

public static TaskReport newTaskReport(TaskId id) {
  TaskReport report = Records.newRecord(TaskReport.class);
  report.setTaskId(id);
  report
      .setStartTime(System.currentTimeMillis() - (int) (Math.random() * DT));
  report.setFinishTime(System.currentTimeMillis()
      + (int) (Math.random() * DT) + 1);
  report.setProgress((float) Math.random());
  report.setStatus("Moving average: " + Math.random());
  report.setCounters(TypeConverter.toYarn(newCounters()));
  report.setTaskState(TASK_STATES.next());
  return report;
}
 

@Private
@Deprecated
@Unstable
public static ContainerId newInstance(ApplicationAttemptId appAttemptId,
    int containerId) {
  ContainerId id = Records.newRecord(ContainerId.class);
  id.setContainerId(containerId);
  id.setApplicationAttemptId(appAttemptId);
  id.build();
  return id;
}
 

@Private
@Unstable
public static PreemptionContract newInstance(
    List<PreemptionResourceRequest> req, Set<PreemptionContainer> containers) {
  PreemptionContract contract = Records.newRecord(PreemptionContract.class);
  contract.setResourceRequest(req);
  contract.setContainers(containers);
  return contract;
}
 

@Public
@Unstable
public static ApplicationFinishData newInstance(ApplicationId applicationId,
    long finishTime, String diagnosticsInfo,
    FinalApplicationStatus finalApplicationStatus,
    YarnApplicationState yarnApplicationState) {
  ApplicationFinishData appFD =
      Records.newRecord(ApplicationFinishData.class);
  appFD.setApplicationId(applicationId);
  appFD.setFinishTime(finishTime);
  appFD.setDiagnosticsInfo(diagnosticsInfo);
  appFD.setFinalApplicationStatus(finalApplicationStatus);
  appFD.setYarnApplicationState(yarnApplicationState);
  return appFD;
}
 

public static RemoveFromClusterNodeLabelsRequest newInstance(
    Set<String> labels) {
  RemoveFromClusterNodeLabelsRequest request =
      Records.newRecord(RemoveFromClusterNodeLabelsRequest.class);
  request.setNodeLabels(labels);
  return request;
}
 

@Public
@Unstable
public static ReservationDeleteRequest newInstance(ReservationId reservationId) {
  ReservationDeleteRequest request =
      Records.newRecord(ReservationDeleteRequest.class);
  request.setReservationId(reservationId);
  return request;
}
 

@Public
@Unstable
public static ReservationRequest newInstance(Resource capability,
    int numContainers, int concurrency, long duration) {
  ReservationRequest request = Records.newRecord(ReservationRequest.class);
  request.setCapability(capability);
  request.setNumContainers(numContainers);
  request.setConcurrency(concurrency);
  request.setDuration(duration);
  return request;
}
 

@Override
public MoveApplicationAcrossQueuesResponse moveApplicationAcrossQueues(
    MoveApplicationAcrossQueuesRequest request) throws YarnException {
  resetStartFailoverFlag(true);

  // make sure failover has been triggered
  Assert.assertTrue(waittingForFailOver());

  return Records.newRecord(MoveApplicationAcrossQueuesResponse.class);
}
 

@SuppressWarnings("deprecation")
public synchronized void submit() throws IOException, YarnException {
  ApplicationSubmissionContext context = recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
  context.setApplicationId(this.applicationId);
  context.setQueue(this.queue);
  
  // Set up the container launch context for the application master
  ContainerLaunchContext amContainer
      = Records.newRecord(ContainerLaunchContext.class);
  context.setAMContainerSpec(amContainer);
  context.setResource(Resources.createResource(
      YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB));
  
  SubmitApplicationRequest request = recordFactory
      .newRecordInstance(SubmitApplicationRequest.class);
  request.setApplicationSubmissionContext(context);
  final ResourceScheduler scheduler = resourceManager.getResourceScheduler();
  
  resourceManager.getClientRMService().submitApplication(request);

  // Notify scheduler
  AppAddedSchedulerEvent addAppEvent =
      new AppAddedSchedulerEvent(this.applicationId, this.queue, "user");
  scheduler.handle(addAppEvent);
  AppAttemptAddedSchedulerEvent addAttemptEvent =
      new AppAttemptAddedSchedulerEvent(this.applicationAttemptId, false);
  scheduler.handle(addAttemptEvent);
}
 

@Override
public void killApplication(ApplicationId applicationId)
    throws YarnException, IOException {
  KillApplicationRequest request =
      Records.newRecord(KillApplicationRequest.class);
  request.setApplicationId(applicationId);

  try {
    int pollCount = 0;
    long startTime = System.currentTimeMillis();

    while (true) {
      KillApplicationResponse response =
          rmClient.forceKillApplication(request);
      if (response.getIsKillCompleted()) {
        LOG.info("Killed application " + applicationId);
        break;
      }

      long elapsedMillis = System.currentTimeMillis() - startTime;
      if (enforceAsyncAPITimeout() &&
          elapsedMillis >= this.asyncApiPollTimeoutMillis) {
        throw new YarnException("Timed out while waiting for application " +
          applicationId + " to be killed.");
      }

      if (++pollCount % 10 == 0) {
        LOG.info("Waiting for application " + applicationId + " to be killed.");
      }
      Thread.sleep(asyncApiPollIntervalMillis);
    }
  } catch (InterruptedException e) {
    LOG.error("Interrupted while waiting for application " + applicationId
        + " to be killed.");
  }
}
 

@Private
@Unstable
public static CancelDelegationTokenResponse newInstance() {
  CancelDelegationTokenResponse response =
      Records.newRecord(CancelDelegationTokenResponse.class);
  return response;
}
 

@Private
@Unstable
public static ReservationId newInstance(long clusterTimestamp, long id) {
  ReservationId reservationId = Records.newRecord(ReservationId.class);
  reservationId.setClusterTimestamp(clusterTimestamp);
  reservationId.setId(id);
  reservationId.build();
  return reservationId;
}
 

@Private
@Unstable
public static PreemptionMessage newInstance(StrictPreemptionContract set,
    PreemptionContract contract) {
  PreemptionMessage message = Records.newRecord(PreemptionMessage.class);
  message.setStrictContract(set);
  message.setContract(contract);
  return message;
}
 

@Public
@Unstable
public static ReservationRequests newInstance(
    List<ReservationRequest> reservationResources,
    ReservationRequestInterpreter type) {
  ReservationRequests reservationRequests =
      Records.newRecord(ReservationRequests.class);
  reservationRequests.setReservationResources(reservationResources);
  reservationRequests.setInterpreter(type);
  return reservationRequests;
}