org.apache.hadoop.yarn.util.Records Java Examples

The following examples show how to use org.apache.hadoop.yarn.util.Records. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ApplicationSubmissionContext.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Public
@Stable
public static ApplicationSubmissionContext newInstance(
    ApplicationId applicationId, String applicationName, String queue,
    ContainerLaunchContext amContainer, boolean isUnmanagedAM,
    boolean cancelTokensWhenComplete, int maxAppAttempts,
    String applicationType, boolean keepContainers,
    String appLabelExpression, ResourceRequest resourceRequest) {
  ApplicationSubmissionContext context =
      Records.newRecord(ApplicationSubmissionContext.class);
  context.setApplicationId(applicationId);
  context.setApplicationName(applicationName);
  context.setQueue(queue);
  context.setAMContainerSpec(amContainer);
  context.setUnmanagedAM(isUnmanagedAM);
  context.setCancelTokensWhenComplete(cancelTokensWhenComplete);
  context.setMaxAppAttempts(maxAppAttempts);
  context.setApplicationType(applicationType);
  context.setKeepContainersAcrossApplicationAttempts(keepContainers);
  context.setNodeLabelExpression(appLabelExpression);
  context.setAMContainerResourceRequest(resourceRequest);
  return context;
}
 
Example #2
Source File: AllocateResponse.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Public
@Stable
public static AllocateResponse newInstance(int responseId,
    List<ContainerStatus> completedContainers,
    List<Container> allocatedContainers, List<NodeReport> updatedNodes,
    Resource availResources, AMCommand command, int numClusterNodes,
    PreemptionMessage preempt, List<NMToken> nmTokens) {
  AllocateResponse response = Records.newRecord(AllocateResponse.class);
  response.setNumClusterNodes(numClusterNodes);
  response.setResponseId(responseId);
  response.setCompletedContainersStatuses(completedContainers);
  response.setAllocatedContainers(allocatedContainers);
  response.setUpdatedNodes(updatedNodes);
  response.setAvailableResources(availResources);
  response.setAMCommand(command);
  response.setPreemptionMessage(preempt);
  response.setNMTokens(nmTokens);
  return response;
}
 
Example #3
Source File: MRDelegationTokenRenewer.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
public long renew(Token<?> token, Configuration conf) throws IOException,
    InterruptedException {

  org.apache.hadoop.yarn.api.records.Token dToken =
      org.apache.hadoop.yarn.api.records.Token.newInstance(
        token.getIdentifier(), token.getKind().toString(),
        token.getPassword(), token.getService().toString());

  MRClientProtocol histProxy = instantiateHistoryProxy(conf,
      SecurityUtil.getTokenServiceAddr(token));
  try {
    RenewDelegationTokenRequest request = Records
        .newRecord(RenewDelegationTokenRequest.class);
    request.setDelegationToken(dToken);
    return histProxy.renewDelegationToken(request).getNextExpirationTime();
  } finally {
    stopHistoryProxy(histProxy);
  }

}
 
Example #4
Source File: YarnClientImpl.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
public List<ApplicationAttemptReport> getApplicationAttempts(
    ApplicationId appId) throws YarnException, IOException {
  try {
    GetApplicationAttemptsRequest request = Records
        .newRecord(GetApplicationAttemptsRequest.class);
    request.setApplicationId(appId);
    GetApplicationAttemptsResponse response = rmClient
        .getApplicationAttempts(request);
    return response.getApplicationAttemptList();
  } catch (YarnException e) {
    if (!historyServiceEnabled) {
      // Just throw it as usual if historyService is not enabled.
      throw e;
    }
    // Even if history-service is enabled, treat all exceptions still the same
    // except the following
    if (e.getClass() != ApplicationNotFoundException.class) {
      throw e;
    }
    return historyClient.getApplicationAttempts(appId);
  }
}
 
Example #5
Source File: TestResourceTrackerService.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testNodeRegistrationFailure() throws Exception {
  writeToHostsFile("host1");
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, hostFile
      .getAbsolutePath());
  rm = new MockRM(conf);
  rm.start();
  
  ResourceTrackerService resourceTrackerService = rm.getResourceTrackerService();
  RegisterNodeManagerRequest req = Records.newRecord(
      RegisterNodeManagerRequest.class);
  NodeId nodeId = NodeId.newInstance("host2", 1234);
  req.setNodeId(nodeId);
  req.setHttpPort(1234);
  // trying to register a invalid node.
  RegisterNodeManagerResponse response = resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.SHUTDOWN,response.getNodeAction());
  Assert
    .assertEquals(
      "Disallowed NodeManager from  host2, Sending SHUTDOWN signal to the NodeManager.",
      response.getDiagnosticsMessage());
}
 
Example #6
Source File: YarnClientImpl.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
public ApplicationReport getApplicationReport(ApplicationId appId)
    throws YarnException, IOException {
  GetApplicationReportResponse response = null;
  try {
    GetApplicationReportRequest request = Records
        .newRecord(GetApplicationReportRequest.class);
    request.setApplicationId(appId);
    response = rmClient.getApplicationReport(request);
  } catch (YarnException e) {
    if (!historyServiceEnabled) {
      // Just throw it as usual if historyService is not enabled.
      throw e;
    }
    // Even if history-service is enabled, treat all exceptions still the same
    // except the following
    if (!(e.getClass() == ApplicationNotFoundException.class)) {
      throw e;
    }
    return historyClient.getApplicationReport(appId);
  }
  return response.getApplicationReport();
}
 
Example #7
Source File: TestClientRMService.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private void checkTokenCancellation(ClientRMService rmService,
    UserGroupInformation owner, UserGroupInformation renewer)
    throws IOException, YarnException {
  RMDelegationTokenIdentifier tokenIdentifier =
      new RMDelegationTokenIdentifier(new Text(owner.getUserName()),
        new Text(renewer.getUserName()), null);
  Token<?> token =
      new Token<RMDelegationTokenIdentifier>(tokenIdentifier, dtsm);
  org.apache.hadoop.yarn.api.records.Token dToken =
      BuilderUtils.newDelegationToken(token.getIdentifier(), token.getKind()
        .toString(), token.getPassword(), token.getService().toString());
  CancelDelegationTokenRequest request =
      Records.newRecord(CancelDelegationTokenRequest.class);
  request.setDelegationToken(dToken);
  rmService.cancelDelegationToken(request);
}
 
Example #8
Source File: SharedCacheClientImpl.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
public void release(ApplicationId applicationId, String resourceKey)
    throws YarnException {
  ReleaseSharedCacheResourceRequest request = Records.newRecord(
      ReleaseSharedCacheResourceRequest.class);
  request.setAppId(applicationId);
  request.setResourceKey(resourceKey);
  try {
    // We do not care about the response because it is empty.
    this.scmClient.release(request);
  } catch (Exception e) {
    // Just catching IOException isn't enough.
    // RPC call can throw ConnectionException.
    throw new YarnException(e);
  }
}
 
Example #9
Source File: TestResourceMgrDelegate.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private ApplicationReport getApplicationReport(
    YarnApplicationState yarnApplicationState,
    FinalApplicationStatus finalApplicationStatus) {
  ApplicationReport appReport = Mockito.mock(ApplicationReport.class);
  ApplicationResourceUsageReport appResources = Mockito
      .mock(ApplicationResourceUsageReport.class);
  Mockito.when(appReport.getApplicationId()).thenReturn(
      ApplicationId.newInstance(0, 0));
  Mockito.when(appResources.getNeededResources()).thenReturn(
      Records.newRecord(Resource.class));
  Mockito.when(appResources.getReservedResources()).thenReturn(
      Records.newRecord(Resource.class));
  Mockito.when(appResources.getUsedResources()).thenReturn(
      Records.newRecord(Resource.class));
  Mockito.when(appReport.getApplicationResourceUsageReport()).thenReturn(
      appResources);
  Mockito.when(appReport.getYarnApplicationState()).thenReturn(
      yarnApplicationState);
  Mockito.when(appReport.getFinalApplicationStatus()).thenReturn(
      finalApplicationStatus);

  return appReport;
}
 
Example #10
Source File: HistoryClientService.java    From XLearning with Apache License 2.0 6 votes vote down vote up
@Override
public CancelDelegationTokenResponse cancelDelegationToken(
    CancelDelegationTokenRequest request) throws IOException {
  if (!isAllowedDelegationTokenOp()) {
    throw new IOException(
        "Delegation Token can be cancelled only with kerberos authentication");
  }

  org.apache.hadoop.yarn.api.records.Token protoToken = request.getDelegationToken();
  Token<MRDelegationTokenIdentifier> token =
      new Token<MRDelegationTokenIdentifier>(
          protoToken.getIdentifier().array(), protoToken.getPassword()
          .array(), new Text(protoToken.getKind()), new Text(
          protoToken.getService()));

  String user = UserGroupInformation.getCurrentUser().getUserName();
  jhsDTSecretManager.cancelToken(token, user);
  return Records.newRecord(CancelDelegationTokenResponse.class);
}
 
Example #11
Source File: TestContainerManagerSecurity.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private void startContainer(final YarnRPC rpc,
    org.apache.hadoop.yarn.api.records.Token nmToken,
    org.apache.hadoop.yarn.api.records.Token containerToken,
    NodeId nodeId, String user) throws Exception {

  ContainerLaunchContext context =
      Records.newRecord(ContainerLaunchContext.class);
  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(context,containerToken);
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  ContainerManagementProtocol proxy = null;
  try {
    proxy = getContainerManagementProtocolProxy(rpc, nmToken, nodeId, user);
    StartContainersResponse response = proxy.startContainers(allRequests);
    for(SerializedException ex : response.getFailedRequests().values()){
      parseAndThrowException(ex.deSerialize());
    }
  } finally {
    if (proxy != null) {
      rpc.stopProxy(proxy, conf);
    }
  }
}
 
Example #12
Source File: HistoryClientService.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
public CancelDelegationTokenResponse cancelDelegationToken(
    CancelDelegationTokenRequest request) throws IOException {
    if (!isAllowedDelegationTokenOp()) {
      throw new IOException(
          "Delegation Token can be cancelled only with kerberos authentication");
    }

    org.apache.hadoop.yarn.api.records.Token protoToken = request.getDelegationToken();
    Token<MRDelegationTokenIdentifier> token =
        new Token<MRDelegationTokenIdentifier>(
            protoToken.getIdentifier().array(), protoToken.getPassword()
                .array(), new Text(protoToken.getKind()), new Text(
                protoToken.getService()));

    String user = UserGroupInformation.getCurrentUser().getUserName();
    jhsDTSecretManager.cancelToken(token, user);
    return Records.newRecord(CancelDelegationTokenResponse.class);
}
 
Example #13
Source File: HistoryClientService.java    From XLearning with Apache License 2.0 6 votes vote down vote up
@Override
public RenewDelegationTokenResponse renewDelegationToken(
    RenewDelegationTokenRequest request) throws IOException {
  if (!isAllowedDelegationTokenOp()) {
    throw new IOException(
        "Delegation Token can be renewed only with kerberos authentication");
  }

  org.apache.hadoop.yarn.api.records.Token protoToken = request.getDelegationToken();
  Token<MRDelegationTokenIdentifier> token =
      new Token<MRDelegationTokenIdentifier>(
          protoToken.getIdentifier().array(), protoToken.getPassword()
          .array(), new Text(protoToken.getKind()), new Text(
          protoToken.getService()));

  String user = UserGroupInformation.getCurrentUser().getShortUserName();
  long nextExpTime = jhsDTSecretManager.renewToken(token, user);
  RenewDelegationTokenResponse renewResponse = Records
      .newRecord(RenewDelegationTokenResponse.class);
  renewResponse.setNextExpirationTime(nextExpTime);
  return renewResponse;
}
 
Example #14
Source File: MockAM.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public ResourceRequest createResourceReq(String resource, int memory, int priority,
    int containers, String labelExpression) throws Exception {
  ResourceRequest req = Records.newRecord(ResourceRequest.class);
  req.setResourceName(resource);
  req.setNumContainers(containers);
  Priority pri = Records.newRecord(Priority.class);
  pri.setPriority(priority);
  req.setPriority(pri);
  Resource capability = Records.newRecord(Resource.class);
  capability.setMemory(memory);
  req.setCapability(capability);
  if (labelExpression != null) {
   req.setNodeLabelExpression(labelExpression); 
  }
  return req;
}
 
Example #15
Source File: TestClientRMService.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private void checkTokenRenewal(UserGroupInformation owner,
    UserGroupInformation renewer) throws IOException, YarnException {
  RMDelegationTokenIdentifier tokenIdentifier =
      new RMDelegationTokenIdentifier(
          new Text(owner.getUserName()), new Text(renewer.getUserName()), null);
  Token<?> token =
      new Token<RMDelegationTokenIdentifier>(tokenIdentifier, dtsm);
  org.apache.hadoop.yarn.api.records.Token dToken = BuilderUtils.newDelegationToken(
      token.getIdentifier(), token.getKind().toString(),
      token.getPassword(), token.getService().toString());
  RenewDelegationTokenRequest request =
      Records.newRecord(RenewDelegationTokenRequest.class);
  request.setDelegationToken(dToken);

  RMContext rmContext = mock(RMContext.class);
  ClientRMService rmService = new ClientRMService(
      rmContext, null, null, null, null, dtsm);
  rmService.renewDelegationToken(request);
}
 
Example #16
Source File: GetApplicationReportResponse.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Private
@Unstable
public static GetApplicationReportResponse newInstance(
    ApplicationReport ApplicationReport) {
  GetApplicationReportResponse response =
      Records.newRecord(GetApplicationReportResponse.class);
  response.setApplicationReport(ApplicationReport);
  return response;
}
 
Example #17
Source File: MockJobs.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public static TaskReport newTaskReport(TaskId id) {
  TaskReport report = Records.newRecord(TaskReport.class);
  report.setTaskId(id);
  report
      .setStartTime(System.currentTimeMillis() - (int) (Math.random() * DT));
  report.setFinishTime(System.currentTimeMillis()
      + (int) (Math.random() * DT) + 1);
  report.setProgress((float) Math.random());
  report.setStatus("Moving average: " + Math.random());
  report.setCounters(TypeConverter.toYarn(newCounters()));
  report.setTaskState(TASK_STATES.next());
  return report;
}
 
Example #18
Source File: ContainerId.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Private
@Deprecated
@Unstable
public static ContainerId newInstance(ApplicationAttemptId appAttemptId,
    int containerId) {
  ContainerId id = Records.newRecord(ContainerId.class);
  id.setContainerId(containerId);
  id.setApplicationAttemptId(appAttemptId);
  id.build();
  return id;
}
 
Example #19
Source File: PreemptionContract.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Private
@Unstable
public static PreemptionContract newInstance(
    List<PreemptionResourceRequest> req, Set<PreemptionContainer> containers) {
  PreemptionContract contract = Records.newRecord(PreemptionContract.class);
  contract.setResourceRequest(req);
  contract.setContainers(containers);
  return contract;
}
 
Example #20
Source File: ApplicationFinishData.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Public
@Unstable
public static ApplicationFinishData newInstance(ApplicationId applicationId,
    long finishTime, String diagnosticsInfo,
    FinalApplicationStatus finalApplicationStatus,
    YarnApplicationState yarnApplicationState) {
  ApplicationFinishData appFD =
      Records.newRecord(ApplicationFinishData.class);
  appFD.setApplicationId(applicationId);
  appFD.setFinishTime(finishTime);
  appFD.setDiagnosticsInfo(diagnosticsInfo);
  appFD.setFinalApplicationStatus(finalApplicationStatus);
  appFD.setYarnApplicationState(yarnApplicationState);
  return appFD;
}
 
Example #21
Source File: RemoveFromClusterNodeLabelsRequest.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public static RemoveFromClusterNodeLabelsRequest newInstance(
    Set<String> labels) {
  RemoveFromClusterNodeLabelsRequest request =
      Records.newRecord(RemoveFromClusterNodeLabelsRequest.class);
  request.setNodeLabels(labels);
  return request;
}
 
Example #22
Source File: ReservationDeleteRequest.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Public
@Unstable
public static ReservationDeleteRequest newInstance(ReservationId reservationId) {
  ReservationDeleteRequest request =
      Records.newRecord(ReservationDeleteRequest.class);
  request.setReservationId(reservationId);
  return request;
}
 
Example #23
Source File: ReservationRequest.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Public
@Unstable
public static ReservationRequest newInstance(Resource capability,
    int numContainers, int concurrency, long duration) {
  ReservationRequest request = Records.newRecord(ReservationRequest.class);
  request.setCapability(capability);
  request.setNumContainers(numContainers);
  request.setConcurrency(concurrency);
  request.setDuration(duration);
  return request;
}
 
Example #24
Source File: ProtocolHATestBase.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public MoveApplicationAcrossQueuesResponse moveApplicationAcrossQueues(
    MoveApplicationAcrossQueuesRequest request) throws YarnException {
  resetStartFailoverFlag(true);

  // make sure failover has been triggered
  Assert.assertTrue(waittingForFailOver());

  return Records.newRecord(MoveApplicationAcrossQueuesResponse.class);
}
 
Example #25
Source File: Application.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("deprecation")
public synchronized void submit() throws IOException, YarnException {
  ApplicationSubmissionContext context = recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
  context.setApplicationId(this.applicationId);
  context.setQueue(this.queue);
  
  // Set up the container launch context for the application master
  ContainerLaunchContext amContainer
      = Records.newRecord(ContainerLaunchContext.class);
  context.setAMContainerSpec(amContainer);
  context.setResource(Resources.createResource(
      YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB));
  
  SubmitApplicationRequest request = recordFactory
      .newRecordInstance(SubmitApplicationRequest.class);
  request.setApplicationSubmissionContext(context);
  final ResourceScheduler scheduler = resourceManager.getResourceScheduler();
  
  resourceManager.getClientRMService().submitApplication(request);

  // Notify scheduler
  AppAddedSchedulerEvent addAppEvent =
      new AppAddedSchedulerEvent(this.applicationId, this.queue, "user");
  scheduler.handle(addAppEvent);
  AppAttemptAddedSchedulerEvent addAttemptEvent =
      new AppAttemptAddedSchedulerEvent(this.applicationAttemptId, false);
  scheduler.handle(addAttemptEvent);
}
 
Example #26
Source File: YarnClientImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public void killApplication(ApplicationId applicationId)
    throws YarnException, IOException {
  KillApplicationRequest request =
      Records.newRecord(KillApplicationRequest.class);
  request.setApplicationId(applicationId);

  try {
    int pollCount = 0;
    long startTime = System.currentTimeMillis();

    while (true) {
      KillApplicationResponse response =
          rmClient.forceKillApplication(request);
      if (response.getIsKillCompleted()) {
        LOG.info("Killed application " + applicationId);
        break;
      }

      long elapsedMillis = System.currentTimeMillis() - startTime;
      if (enforceAsyncAPITimeout() &&
          elapsedMillis >= this.asyncApiPollTimeoutMillis) {
        throw new YarnException("Timed out while waiting for application " +
          applicationId + " to be killed.");
      }

      if (++pollCount % 10 == 0) {
        LOG.info("Waiting for application " + applicationId + " to be killed.");
      }
      Thread.sleep(asyncApiPollIntervalMillis);
    }
  } catch (InterruptedException e) {
    LOG.error("Interrupted while waiting for application " + applicationId
        + " to be killed.");
  }
}
 
Example #27
Source File: CancelDelegationTokenResponse.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Private
@Unstable
public static CancelDelegationTokenResponse newInstance() {
  CancelDelegationTokenResponse response =
      Records.newRecord(CancelDelegationTokenResponse.class);
  return response;
}
 
Example #28
Source File: ReservationId.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Private
@Unstable
public static ReservationId newInstance(long clusterTimestamp, long id) {
  ReservationId reservationId = Records.newRecord(ReservationId.class);
  reservationId.setClusterTimestamp(clusterTimestamp);
  reservationId.setId(id);
  reservationId.build();
  return reservationId;
}
 
Example #29
Source File: PreemptionMessage.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Private
@Unstable
public static PreemptionMessage newInstance(StrictPreemptionContract set,
    PreemptionContract contract) {
  PreemptionMessage message = Records.newRecord(PreemptionMessage.class);
  message.setStrictContract(set);
  message.setContract(contract);
  return message;
}
 
Example #30
Source File: ReservationRequests.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Public
@Unstable
public static ReservationRequests newInstance(
    List<ReservationRequest> reservationResources,
    ReservationRequestInterpreter type) {
  ReservationRequests reservationRequests =
      Records.newRecord(ReservationRequests.class);
  reservationRequests.setReservationResources(reservationResources);
  reservationRequests.setInterpreter(type);
  return reservationRequests;
}