Java Code Examples for org.apache.hadoop.yarn.util.Records

The following examples show how to use org.apache.hadoop.yarn.util.Records. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: ApplicationSubmissionContext.java    License: Apache License 2.0 6 votes vote down vote up
@Public
@Stable
public static ApplicationSubmissionContext newInstance(
    ApplicationId applicationId, String applicationName, String queue,
    ContainerLaunchContext amContainer, boolean isUnmanagedAM,
    boolean cancelTokensWhenComplete, int maxAppAttempts,
    String applicationType, boolean keepContainers,
    String appLabelExpression, ResourceRequest resourceRequest) {
  ApplicationSubmissionContext context =
      Records.newRecord(ApplicationSubmissionContext.class);
  context.setApplicationId(applicationId);
  context.setApplicationName(applicationName);
  context.setQueue(queue);
  context.setAMContainerSpec(amContainer);
  context.setUnmanagedAM(isUnmanagedAM);
  context.setCancelTokensWhenComplete(cancelTokensWhenComplete);
  context.setMaxAppAttempts(maxAppAttempts);
  context.setApplicationType(applicationType);
  context.setKeepContainersAcrossApplicationAttempts(keepContainers);
  context.setNodeLabelExpression(appLabelExpression);
  context.setAMContainerResourceRequest(resourceRequest);
  return context;
}
 
Example 2
Source Project: hadoop   Source File: AllocateResponse.java    License: Apache License 2.0 6 votes vote down vote up
@Public
@Stable
public static AllocateResponse newInstance(int responseId,
    List<ContainerStatus> completedContainers,
    List<Container> allocatedContainers, List<NodeReport> updatedNodes,
    Resource availResources, AMCommand command, int numClusterNodes,
    PreemptionMessage preempt, List<NMToken> nmTokens) {
  AllocateResponse response = Records.newRecord(AllocateResponse.class);
  response.setNumClusterNodes(numClusterNodes);
  response.setResponseId(responseId);
  response.setCompletedContainersStatuses(completedContainers);
  response.setAllocatedContainers(allocatedContainers);
  response.setUpdatedNodes(updatedNodes);
  response.setAvailableResources(availResources);
  response.setAMCommand(command);
  response.setPreemptionMessage(preempt);
  response.setNMTokens(nmTokens);
  return response;
}
 
Example 3
Source Project: hadoop   Source File: SharedCacheClientImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void release(ApplicationId applicationId, String resourceKey)
    throws YarnException {
  ReleaseSharedCacheResourceRequest request = Records.newRecord(
      ReleaseSharedCacheResourceRequest.class);
  request.setAppId(applicationId);
  request.setResourceKey(resourceKey);
  try {
    // We do not care about the response because it is empty.
    this.scmClient.release(request);
  } catch (Exception e) {
    // Just catching IOException isn't enough.
    // RPC call can throw ConnectionException.
    throw new YarnException(e);
  }
}
 
Example 4
Source Project: hadoop   Source File: TestResourceMgrDelegate.java    License: Apache License 2.0 6 votes vote down vote up
private ApplicationReport getApplicationReport(
    YarnApplicationState yarnApplicationState,
    FinalApplicationStatus finalApplicationStatus) {
  ApplicationReport appReport = Mockito.mock(ApplicationReport.class);
  ApplicationResourceUsageReport appResources = Mockito
      .mock(ApplicationResourceUsageReport.class);
  Mockito.when(appReport.getApplicationId()).thenReturn(
      ApplicationId.newInstance(0, 0));
  Mockito.when(appResources.getNeededResources()).thenReturn(
      Records.newRecord(Resource.class));
  Mockito.when(appResources.getReservedResources()).thenReturn(
      Records.newRecord(Resource.class));
  Mockito.when(appResources.getUsedResources()).thenReturn(
      Records.newRecord(Resource.class));
  Mockito.when(appReport.getApplicationResourceUsageReport()).thenReturn(
      appResources);
  Mockito.when(appReport.getYarnApplicationState()).thenReturn(
      yarnApplicationState);
  Mockito.when(appReport.getFinalApplicationStatus()).thenReturn(
      finalApplicationStatus);

  return appReport;
}
 
Example 5
Source Project: XLearning   Source File: HistoryClientService.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public RenewDelegationTokenResponse renewDelegationToken(
    RenewDelegationTokenRequest request) throws IOException {
  if (!isAllowedDelegationTokenOp()) {
    throw new IOException(
        "Delegation Token can be renewed only with kerberos authentication");
  }

  org.apache.hadoop.yarn.api.records.Token protoToken = request.getDelegationToken();
  Token<MRDelegationTokenIdentifier> token =
      new Token<MRDelegationTokenIdentifier>(
          protoToken.getIdentifier().array(), protoToken.getPassword()
          .array(), new Text(protoToken.getKind()), new Text(
          protoToken.getService()));

  String user = UserGroupInformation.getCurrentUser().getShortUserName();
  long nextExpTime = jhsDTSecretManager.renewToken(token, user);
  RenewDelegationTokenResponse renewResponse = Records
      .newRecord(RenewDelegationTokenResponse.class);
  renewResponse.setNextExpirationTime(nextExpTime);
  return renewResponse;
}
 
Example 6
Source Project: XLearning   Source File: HistoryClientService.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public CancelDelegationTokenResponse cancelDelegationToken(
    CancelDelegationTokenRequest request) throws IOException {
  if (!isAllowedDelegationTokenOp()) {
    throw new IOException(
        "Delegation Token can be cancelled only with kerberos authentication");
  }

  org.apache.hadoop.yarn.api.records.Token protoToken = request.getDelegationToken();
  Token<MRDelegationTokenIdentifier> token =
      new Token<MRDelegationTokenIdentifier>(
          protoToken.getIdentifier().array(), protoToken.getPassword()
          .array(), new Text(protoToken.getKind()), new Text(
          protoToken.getService()));

  String user = UserGroupInformation.getCurrentUser().getUserName();
  jhsDTSecretManager.cancelToken(token, user);
  return Records.newRecord(CancelDelegationTokenResponse.class);
}
 
Example 7
Source Project: hadoop   Source File: TestContainerManagerSecurity.java    License: Apache License 2.0 6 votes vote down vote up
private void startContainer(final YarnRPC rpc,
    org.apache.hadoop.yarn.api.records.Token nmToken,
    org.apache.hadoop.yarn.api.records.Token containerToken,
    NodeId nodeId, String user) throws Exception {

  ContainerLaunchContext context =
      Records.newRecord(ContainerLaunchContext.class);
  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(context,containerToken);
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  ContainerManagementProtocol proxy = null;
  try {
    proxy = getContainerManagementProtocolProxy(rpc, nmToken, nodeId, user);
    StartContainersResponse response = proxy.startContainers(allRequests);
    for(SerializedException ex : response.getFailedRequests().values()){
      parseAndThrowException(ex.deSerialize());
    }
  } finally {
    if (proxy != null) {
      rpc.stopProxy(proxy, conf);
    }
  }
}
 
Example 8
Source Project: hadoop   Source File: HistoryClientService.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public CancelDelegationTokenResponse cancelDelegationToken(
    CancelDelegationTokenRequest request) throws IOException {
    if (!isAllowedDelegationTokenOp()) {
      throw new IOException(
          "Delegation Token can be cancelled only with kerberos authentication");
    }

    org.apache.hadoop.yarn.api.records.Token protoToken = request.getDelegationToken();
    Token<MRDelegationTokenIdentifier> token =
        new Token<MRDelegationTokenIdentifier>(
            protoToken.getIdentifier().array(), protoToken.getPassword()
                .array(), new Text(protoToken.getKind()), new Text(
                protoToken.getService()));

    String user = UserGroupInformation.getCurrentUser().getUserName();
    jhsDTSecretManager.cancelToken(token, user);
    return Records.newRecord(CancelDelegationTokenResponse.class);
}
 
Example 9
Source Project: hadoop   Source File: MockAM.java    License: Apache License 2.0 6 votes vote down vote up
public ResourceRequest createResourceReq(String resource, int memory, int priority,
    int containers, String labelExpression) throws Exception {
  ResourceRequest req = Records.newRecord(ResourceRequest.class);
  req.setResourceName(resource);
  req.setNumContainers(containers);
  Priority pri = Records.newRecord(Priority.class);
  pri.setPriority(priority);
  req.setPriority(pri);
  Resource capability = Records.newRecord(Resource.class);
  capability.setMemory(memory);
  req.setCapability(capability);
  if (labelExpression != null) {
   req.setNodeLabelExpression(labelExpression); 
  }
  return req;
}
 
Example 10
Source Project: hadoop   Source File: TestClientRMService.java    License: Apache License 2.0 6 votes vote down vote up
private void checkTokenRenewal(UserGroupInformation owner,
    UserGroupInformation renewer) throws IOException, YarnException {
  RMDelegationTokenIdentifier tokenIdentifier =
      new RMDelegationTokenIdentifier(
          new Text(owner.getUserName()), new Text(renewer.getUserName()), null);
  Token<?> token =
      new Token<RMDelegationTokenIdentifier>(tokenIdentifier, dtsm);
  org.apache.hadoop.yarn.api.records.Token dToken = BuilderUtils.newDelegationToken(
      token.getIdentifier(), token.getKind().toString(),
      token.getPassword(), token.getService().toString());
  RenewDelegationTokenRequest request =
      Records.newRecord(RenewDelegationTokenRequest.class);
  request.setDelegationToken(dToken);

  RMContext rmContext = mock(RMContext.class);
  ClientRMService rmService = new ClientRMService(
      rmContext, null, null, null, null, dtsm);
  rmService.renewDelegationToken(request);
}
 
Example 11
Source Project: hadoop   Source File: TestClientRMService.java    License: Apache License 2.0 6 votes vote down vote up
private void checkTokenCancellation(ClientRMService rmService,
    UserGroupInformation owner, UserGroupInformation renewer)
    throws IOException, YarnException {
  RMDelegationTokenIdentifier tokenIdentifier =
      new RMDelegationTokenIdentifier(new Text(owner.getUserName()),
        new Text(renewer.getUserName()), null);
  Token<?> token =
      new Token<RMDelegationTokenIdentifier>(tokenIdentifier, dtsm);
  org.apache.hadoop.yarn.api.records.Token dToken =
      BuilderUtils.newDelegationToken(token.getIdentifier(), token.getKind()
        .toString(), token.getPassword(), token.getService().toString());
  CancelDelegationTokenRequest request =
      Records.newRecord(CancelDelegationTokenRequest.class);
  request.setDelegationToken(dToken);
  rmService.cancelDelegationToken(request);
}
 
Example 12
Source Project: hadoop   Source File: TestResourceTrackerService.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testNodeRegistrationFailure() throws Exception {
  writeToHostsFile("host1");
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, hostFile
      .getAbsolutePath());
  rm = new MockRM(conf);
  rm.start();
  
  ResourceTrackerService resourceTrackerService = rm.getResourceTrackerService();
  RegisterNodeManagerRequest req = Records.newRecord(
      RegisterNodeManagerRequest.class);
  NodeId nodeId = NodeId.newInstance("host2", 1234);
  req.setNodeId(nodeId);
  req.setHttpPort(1234);
  // trying to register a invalid node.
  RegisterNodeManagerResponse response = resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.SHUTDOWN,response.getNodeAction());
  Assert
    .assertEquals(
      "Disallowed NodeManager from  host2, Sending SHUTDOWN signal to the NodeManager.",
      response.getDiagnosticsMessage());
}
 
Example 13
Source Project: hadoop   Source File: YarnClientImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public List<ApplicationAttemptReport> getApplicationAttempts(
    ApplicationId appId) throws YarnException, IOException {
  try {
    GetApplicationAttemptsRequest request = Records
        .newRecord(GetApplicationAttemptsRequest.class);
    request.setApplicationId(appId);
    GetApplicationAttemptsResponse response = rmClient
        .getApplicationAttempts(request);
    return response.getApplicationAttemptList();
  } catch (YarnException e) {
    if (!historyServiceEnabled) {
      // Just throw it as usual if historyService is not enabled.
      throw e;
    }
    // Even if history-service is enabled, treat all exceptions still the same
    // except the following
    if (e.getClass() != ApplicationNotFoundException.class) {
      throw e;
    }
    return historyClient.getApplicationAttempts(appId);
  }
}
 
Example 14
Source Project: hadoop   Source File: MRDelegationTokenRenewer.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public long renew(Token<?> token, Configuration conf) throws IOException,
    InterruptedException {

  org.apache.hadoop.yarn.api.records.Token dToken =
      org.apache.hadoop.yarn.api.records.Token.newInstance(
        token.getIdentifier(), token.getKind().toString(),
        token.getPassword(), token.getService().toString());

  MRClientProtocol histProxy = instantiateHistoryProxy(conf,
      SecurityUtil.getTokenServiceAddr(token));
  try {
    RenewDelegationTokenRequest request = Records
        .newRecord(RenewDelegationTokenRequest.class);
    request.setDelegationToken(dToken);
    return histProxy.renewDelegationToken(request).getNextExpirationTime();
  } finally {
    stopHistoryProxy(histProxy);
  }

}
 
Example 15
Source Project: hadoop   Source File: YarnClientImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public ApplicationReport getApplicationReport(ApplicationId appId)
    throws YarnException, IOException {
  GetApplicationReportResponse response = null;
  try {
    GetApplicationReportRequest request = Records
        .newRecord(GetApplicationReportRequest.class);
    request.setApplicationId(appId);
    response = rmClient.getApplicationReport(request);
  } catch (YarnException e) {
    if (!historyServiceEnabled) {
      // Just throw it as usual if historyService is not enabled.
      throw e;
    }
    // Even if history-service is enabled, treat all exceptions still the same
    // except the following
    if (!(e.getClass() == ApplicationNotFoundException.class)) {
      throw e;
    }
    return historyClient.getApplicationReport(appId);
  }
  return response.getApplicationReport();
}
 
Example 16
Source Project: hadoop   Source File: GetDelegationTokenRequest.java    License: Apache License 2.0 5 votes vote down vote up
@Public
@Stable
public static GetDelegationTokenRequest newInstance(String renewer) {
  GetDelegationTokenRequest request =
      Records.newRecord(GetDelegationTokenRequest.class);
  request.setRenewer(renewer);
  return request;
}
 
Example 17
Source Project: Flink-CEPplus   Source File: Utils.java    License: Apache License 2.0 5 votes vote down vote up
private static LocalResource registerLocalResource(FileSystem fs, Path remoteRsrcPath) throws IOException {
	LocalResource localResource = Records.newRecord(LocalResource.class);
	FileStatus jarStat = fs.getFileStatus(remoteRsrcPath);
	localResource.setResource(ConverterUtils.getYarnUrlFromURI(remoteRsrcPath.toUri()));
	localResource.setSize(jarStat.getLen());
	localResource.setTimestamp(jarStat.getModificationTime());
	localResource.setType(LocalResourceType.FILE);
	localResource.setVisibility(LocalResourceVisibility.APPLICATION);
	return localResource;
}
 
Example 18
Source Project: flink   Source File: Utils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates a YARN resource for the remote object at the given location.
 *
 * @param remoteRsrcPath	remote location of the resource
 * @param resourceSize		size of the resource
 * @param resourceModificationTime last modification time of the resource
 *
 * @return YARN resource
 */
private static LocalResource registerLocalResource(
		Path remoteRsrcPath,
		long resourceSize,
		long resourceModificationTime) {
	LocalResource localResource = Records.newRecord(LocalResource.class);
	localResource.setResource(ConverterUtils.getYarnUrlFromURI(remoteRsrcPath.toUri()));
	localResource.setSize(resourceSize);
	localResource.setTimestamp(resourceModificationTime);
	localResource.setType(LocalResourceType.FILE);
	localResource.setVisibility(LocalResourceVisibility.APPLICATION);
	return localResource;
}
 
Example 19
Source Project: hadoop   Source File: GetQueueUserAclsInfoRequest.java    License: Apache License 2.0 5 votes vote down vote up
@Public
@Stable
public static GetQueueUserAclsInfoRequest newInstance() {
  GetQueueUserAclsInfoRequest request =
      Records.newRecord(GetQueueUserAclsInfoRequest.class);
  return request;
}
 
Example 20
Source Project: hadoop   Source File: ContainerResourceIncrease.java    License: Apache License 2.0 5 votes vote down vote up
@Public
public static ContainerResourceIncrease newInstance(
    ContainerId existingContainerId, Resource targetCapability, Token token) {
  ContainerResourceIncrease context = Records
      .newRecord(ContainerResourceIncrease.class);
  context.setContainerId(existingContainerId);
  context.setCapability(targetCapability);
  context.setContainerToken(token);
  return context;
}
 
Example 21
Source Project: hadoop   Source File: FinishApplicationMasterResponse.java    License: Apache License 2.0 5 votes vote down vote up
@Private
@Unstable
public static FinishApplicationMasterResponse newInstance(
    boolean isRemovedFromRMStateStore) {
  FinishApplicationMasterResponse response =
      Records.newRecord(FinishApplicationMasterResponse.class);
  response.setIsUnregistered(isRemovedFromRMStateStore);
  return response;
}
 
Example 22
Source Project: ambari-metrics   Source File: ApplicationStartData.java    License: Apache License 2.0 5 votes vote down vote up
@Public
@Unstable
public static ApplicationStartData newInstance(ApplicationId applicationId,
    String applicationName, String applicationType, String queue,
    String user, long submitTime, long startTime) {
  ApplicationStartData appSD = Records.newRecord(ApplicationStartData.class);
  appSD.setApplicationId(applicationId);
  appSD.setApplicationName(applicationName);
  appSD.setApplicationType(applicationType);
  appSD.setQueue(queue);
  appSD.setUser(user);
  appSD.setSubmitTime(submitTime);
  appSD.setStartTime(startTime);
  return appSD;
}
 
Example 23
Source Project: ambari-metrics   Source File: ApplicationFinishData.java    License: Apache License 2.0 5 votes vote down vote up
@Public
@Unstable
public static ApplicationFinishData newInstance(ApplicationId applicationId,
    long finishTime, String diagnosticsInfo,
    FinalApplicationStatus finalApplicationStatus,
    YarnApplicationState yarnApplicationState) {
  ApplicationFinishData appFD =
      Records.newRecord(ApplicationFinishData.class);
  appFD.setApplicationId(applicationId);
  appFD.setFinishTime(finishTime);
  appFD.setDiagnosticsInfo(diagnosticsInfo);
  appFD.setFinalApplicationStatus(finalApplicationStatus);
  appFD.setYarnApplicationState(yarnApplicationState);
  return appFD;
}
 
Example 24
Source Project: hadoop   Source File: Container.java    License: Apache License 2.0 5 votes vote down vote up
@Private
@Unstable
public static Container newInstance(ContainerId containerId, NodeId nodeId,
    String nodeHttpAddress, Resource resource, Priority priority,
    Token containerToken) {
  Container container = Records.newRecord(Container.class);
  container.setId(containerId);
  container.setNodeId(nodeId);
  container.setNodeHttpAddress(nodeHttpAddress);
  container.setResource(resource);
  container.setPriority(priority);
  container.setContainerToken(containerToken);
  return container;
}
 
Example 25
Source Project: hadoop   Source File: GetApplicationAttemptsResponse.java    License: Apache License 2.0 5 votes vote down vote up
@Public
@Unstable
public static GetApplicationAttemptsResponse newInstance(
    List<ApplicationAttemptReport> applicationAttempts) {
  GetApplicationAttemptsResponse response =
      Records.newRecord(GetApplicationAttemptsResponse.class);
  response.setApplicationAttemptList(applicationAttempts);
  return response;
}
 
Example 26
Source Project: hadoop   Source File: GetClusterNodesResponse.java    License: Apache License 2.0 5 votes vote down vote up
@Private
@Unstable
public static GetClusterNodesResponse
    newInstance(List<NodeReport> nodeReports) {
  GetClusterNodesResponse response =
      Records.newRecord(GetClusterNodesResponse.class);
  response.setNodeReports(nodeReports);
  return response;
}
 
Example 27
Source Project: hadoop   Source File: GetContainerReportRequest.java    License: Apache License 2.0 5 votes vote down vote up
@Public
@Unstable
public static GetContainerReportRequest newInstance(ContainerId containerId) {
  GetContainerReportRequest request =
      Records.newRecord(GetContainerReportRequest.class);
  request.setContainerId(containerId);
  return request;
}
 
Example 28
Source Project: hadoop   Source File: Resources.java    License: Apache License 2.0 5 votes vote down vote up
public static Resource createResource(int memory, int cores, int gcores) {
  Resource resource = Records.newRecord(Resource.class);
  resource.setMemory(memory);
  resource.setVirtualCores(cores);
  resource.setGpuCores(gcores);
  return resource;
}
 
Example 29
Source Project: hadoop   Source File: MasterKeyData.java    License: Apache License 2.0 5 votes vote down vote up
public MasterKeyData(int serialNo, SecretKey secretKey) {
  this.masterKeyRecord = Records.newRecord(MasterKey.class);
  this.masterKeyRecord.setKeyId(serialNo);
  this.generatedSecretKey = secretKey;
  this.masterKeyRecord.setBytes(ByteBuffer.wrap(generatedSecretKey
    .getEncoded()));
}
 
Example 30
Source Project: hadoop   Source File: NodeHeartbeatRequest.java    License: Apache License 2.0 5 votes vote down vote up
public static NodeHeartbeatRequest newInstance(NodeStatus nodeStatus,
    MasterKey lastKnownContainerTokenMasterKey,
    MasterKey lastKnownNMTokenMasterKey) {
  NodeHeartbeatRequest nodeHeartbeatRequest =
      Records.newRecord(NodeHeartbeatRequest.class);
  nodeHeartbeatRequest.setNodeStatus(nodeStatus);
  nodeHeartbeatRequest
      .setLastKnownContainerTokenMasterKey(lastKnownContainerTokenMasterKey);
  nodeHeartbeatRequest
      .setLastKnownNMTokenMasterKey(lastKnownNMTokenMasterKey);
  return nodeHeartbeatRequest;
}