org.apache.hadoop.yarn.util.Records Java Examples

The following examples show how to use org.apache.hadoop.yarn.util.Records. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source Project: hadoop   Author: naver   File: ApplicationSubmissionContext.java    License: Apache License 2.0 6 votes vote down vote up
@Public
@Stable
public static ApplicationSubmissionContext newInstance(
    ApplicationId applicationId, String applicationName, String queue,
    ContainerLaunchContext amContainer, boolean isUnmanagedAM,
    boolean cancelTokensWhenComplete, int maxAppAttempts,
    String applicationType, boolean keepContainers,
    String appLabelExpression, ResourceRequest resourceRequest) {
  ApplicationSubmissionContext context =
      Records.newRecord(ApplicationSubmissionContext.class);
  context.setApplicationId(applicationId);
  context.setApplicationName(applicationName);
  context.setQueue(queue);
  context.setAMContainerSpec(amContainer);
  context.setUnmanagedAM(isUnmanagedAM);
  context.setCancelTokensWhenComplete(cancelTokensWhenComplete);
  context.setMaxAppAttempts(maxAppAttempts);
  context.setApplicationType(applicationType);
  context.setKeepContainersAcrossApplicationAttempts(keepContainers);
  context.setNodeLabelExpression(appLabelExpression);
  context.setAMContainerResourceRequest(resourceRequest);
  return context;
}
 
Example #2
Source Project: hadoop   Author: naver   File: AllocateResponse.java    License: Apache License 2.0 6 votes vote down vote up
@Public
@Stable
public static AllocateResponse newInstance(int responseId,
    List<ContainerStatus> completedContainers,
    List<Container> allocatedContainers, List<NodeReport> updatedNodes,
    Resource availResources, AMCommand command, int numClusterNodes,
    PreemptionMessage preempt, List<NMToken> nmTokens) {
  AllocateResponse response = Records.newRecord(AllocateResponse.class);
  response.setNumClusterNodes(numClusterNodes);
  response.setResponseId(responseId);
  response.setCompletedContainersStatuses(completedContainers);
  response.setAllocatedContainers(allocatedContainers);
  response.setUpdatedNodes(updatedNodes);
  response.setAvailableResources(availResources);
  response.setAMCommand(command);
  response.setPreemptionMessage(preempt);
  response.setNMTokens(nmTokens);
  return response;
}
 
Example #3
Source Project: hadoop   Author: naver   File: SharedCacheClientImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void release(ApplicationId applicationId, String resourceKey)
    throws YarnException {
  ReleaseSharedCacheResourceRequest request = Records.newRecord(
      ReleaseSharedCacheResourceRequest.class);
  request.setAppId(applicationId);
  request.setResourceKey(resourceKey);
  try {
    // We do not care about the response because it is empty.
    this.scmClient.release(request);
  } catch (Exception e) {
    // Just catching IOException isn't enough.
    // RPC call can throw ConnectionException.
    throw new YarnException(e);
  }
}
 
Example #4
Source Project: hadoop   Author: naver   File: TestResourceMgrDelegate.java    License: Apache License 2.0 6 votes vote down vote up
private ApplicationReport getApplicationReport(
    YarnApplicationState yarnApplicationState,
    FinalApplicationStatus finalApplicationStatus) {
  ApplicationReport appReport = Mockito.mock(ApplicationReport.class);
  ApplicationResourceUsageReport appResources = Mockito
      .mock(ApplicationResourceUsageReport.class);
  Mockito.when(appReport.getApplicationId()).thenReturn(
      ApplicationId.newInstance(0, 0));
  Mockito.when(appResources.getNeededResources()).thenReturn(
      Records.newRecord(Resource.class));
  Mockito.when(appResources.getReservedResources()).thenReturn(
      Records.newRecord(Resource.class));
  Mockito.when(appResources.getUsedResources()).thenReturn(
      Records.newRecord(Resource.class));
  Mockito.when(appReport.getApplicationResourceUsageReport()).thenReturn(
      appResources);
  Mockito.when(appReport.getYarnApplicationState()).thenReturn(
      yarnApplicationState);
  Mockito.when(appReport.getFinalApplicationStatus()).thenReturn(
      finalApplicationStatus);

  return appReport;
}
 
Example #5
Source Project: XLearning   Author: Qihoo360   File: HistoryClientService.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public RenewDelegationTokenResponse renewDelegationToken(
    RenewDelegationTokenRequest request) throws IOException {
  if (!isAllowedDelegationTokenOp()) {
    throw new IOException(
        "Delegation Token can be renewed only with kerberos authentication");
  }

  org.apache.hadoop.yarn.api.records.Token protoToken = request.getDelegationToken();
  Token<MRDelegationTokenIdentifier> token =
      new Token<MRDelegationTokenIdentifier>(
          protoToken.getIdentifier().array(), protoToken.getPassword()
          .array(), new Text(protoToken.getKind()), new Text(
          protoToken.getService()));

  String user = UserGroupInformation.getCurrentUser().getShortUserName();
  long nextExpTime = jhsDTSecretManager.renewToken(token, user);
  RenewDelegationTokenResponse renewResponse = Records
      .newRecord(RenewDelegationTokenResponse.class);
  renewResponse.setNextExpirationTime(nextExpTime);
  return renewResponse;
}
 
Example #6
Source Project: XLearning   Author: Qihoo360   File: HistoryClientService.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public CancelDelegationTokenResponse cancelDelegationToken(
    CancelDelegationTokenRequest request) throws IOException {
  if (!isAllowedDelegationTokenOp()) {
    throw new IOException(
        "Delegation Token can be cancelled only with kerberos authentication");
  }

  org.apache.hadoop.yarn.api.records.Token protoToken = request.getDelegationToken();
  Token<MRDelegationTokenIdentifier> token =
      new Token<MRDelegationTokenIdentifier>(
          protoToken.getIdentifier().array(), protoToken.getPassword()
          .array(), new Text(protoToken.getKind()), new Text(
          protoToken.getService()));

  String user = UserGroupInformation.getCurrentUser().getUserName();
  jhsDTSecretManager.cancelToken(token, user);
  return Records.newRecord(CancelDelegationTokenResponse.class);
}
 
Example #7
Source Project: hadoop   Author: naver   File: TestContainerManagerSecurity.java    License: Apache License 2.0 6 votes vote down vote up
private void startContainer(final YarnRPC rpc,
    org.apache.hadoop.yarn.api.records.Token nmToken,
    org.apache.hadoop.yarn.api.records.Token containerToken,
    NodeId nodeId, String user) throws Exception {

  ContainerLaunchContext context =
      Records.newRecord(ContainerLaunchContext.class);
  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(context,containerToken);
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  ContainerManagementProtocol proxy = null;
  try {
    proxy = getContainerManagementProtocolProxy(rpc, nmToken, nodeId, user);
    StartContainersResponse response = proxy.startContainers(allRequests);
    for(SerializedException ex : response.getFailedRequests().values()){
      parseAndThrowException(ex.deSerialize());
    }
  } finally {
    if (proxy != null) {
      rpc.stopProxy(proxy, conf);
    }
  }
}
 
Example #8
Source Project: hadoop   Author: naver   File: HistoryClientService.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public CancelDelegationTokenResponse cancelDelegationToken(
    CancelDelegationTokenRequest request) throws IOException {
    if (!isAllowedDelegationTokenOp()) {
      throw new IOException(
          "Delegation Token can be cancelled only with kerberos authentication");
    }

    org.apache.hadoop.yarn.api.records.Token protoToken = request.getDelegationToken();
    Token<MRDelegationTokenIdentifier> token =
        new Token<MRDelegationTokenIdentifier>(
            protoToken.getIdentifier().array(), protoToken.getPassword()
                .array(), new Text(protoToken.getKind()), new Text(
                protoToken.getService()));

    String user = UserGroupInformation.getCurrentUser().getUserName();
    jhsDTSecretManager.cancelToken(token, user);
    return Records.newRecord(CancelDelegationTokenResponse.class);
}
 
Example #9
Source Project: hadoop   Author: naver   File: MockAM.java    License: Apache License 2.0 6 votes vote down vote up
public ResourceRequest createResourceReq(String resource, int memory, int priority,
    int containers, String labelExpression) throws Exception {
  ResourceRequest req = Records.newRecord(ResourceRequest.class);
  req.setResourceName(resource);
  req.setNumContainers(containers);
  Priority pri = Records.newRecord(Priority.class);
  pri.setPriority(priority);
  req.setPriority(pri);
  Resource capability = Records.newRecord(Resource.class);
  capability.setMemory(memory);
  req.setCapability(capability);
  if (labelExpression != null) {
   req.setNodeLabelExpression(labelExpression); 
  }
  return req;
}
 
Example #10
Source Project: hadoop   Author: naver   File: TestClientRMService.java    License: Apache License 2.0 6 votes vote down vote up
private void checkTokenRenewal(UserGroupInformation owner,
    UserGroupInformation renewer) throws IOException, YarnException {
  RMDelegationTokenIdentifier tokenIdentifier =
      new RMDelegationTokenIdentifier(
          new Text(owner.getUserName()), new Text(renewer.getUserName()), null);
  Token<?> token =
      new Token<RMDelegationTokenIdentifier>(tokenIdentifier, dtsm);
  org.apache.hadoop.yarn.api.records.Token dToken = BuilderUtils.newDelegationToken(
      token.getIdentifier(), token.getKind().toString(),
      token.getPassword(), token.getService().toString());
  RenewDelegationTokenRequest request =
      Records.newRecord(RenewDelegationTokenRequest.class);
  request.setDelegationToken(dToken);

  RMContext rmContext = mock(RMContext.class);
  ClientRMService rmService = new ClientRMService(
      rmContext, null, null, null, null, dtsm);
  rmService.renewDelegationToken(request);
}
 
Example #11
Source Project: hadoop   Author: naver   File: TestClientRMService.java    License: Apache License 2.0 6 votes vote down vote up
private void checkTokenCancellation(ClientRMService rmService,
    UserGroupInformation owner, UserGroupInformation renewer)
    throws IOException, YarnException {
  RMDelegationTokenIdentifier tokenIdentifier =
      new RMDelegationTokenIdentifier(new Text(owner.getUserName()),
        new Text(renewer.getUserName()), null);
  Token<?> token =
      new Token<RMDelegationTokenIdentifier>(tokenIdentifier, dtsm);
  org.apache.hadoop.yarn.api.records.Token dToken =
      BuilderUtils.newDelegationToken(token.getIdentifier(), token.getKind()
        .toString(), token.getPassword(), token.getService().toString());
  CancelDelegationTokenRequest request =
      Records.newRecord(CancelDelegationTokenRequest.class);
  request.setDelegationToken(dToken);
  rmService.cancelDelegationToken(request);
}
 
Example #12
Source Project: hadoop   Author: naver   File: TestResourceTrackerService.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testNodeRegistrationFailure() throws Exception {
  writeToHostsFile("host1");
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, hostFile
      .getAbsolutePath());
  rm = new MockRM(conf);
  rm.start();
  
  ResourceTrackerService resourceTrackerService = rm.getResourceTrackerService();
  RegisterNodeManagerRequest req = Records.newRecord(
      RegisterNodeManagerRequest.class);
  NodeId nodeId = NodeId.newInstance("host2", 1234);
  req.setNodeId(nodeId);
  req.setHttpPort(1234);
  // trying to register a invalid node.
  RegisterNodeManagerResponse response = resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.SHUTDOWN,response.getNodeAction());
  Assert
    .assertEquals(
      "Disallowed NodeManager from  host2, Sending SHUTDOWN signal to the NodeManager.",
      response.getDiagnosticsMessage());
}
 
Example #13
Source Project: hadoop   Author: naver   File: YarnClientImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public List<ApplicationAttemptReport> getApplicationAttempts(
    ApplicationId appId) throws YarnException, IOException {
  try {
    GetApplicationAttemptsRequest request = Records
        .newRecord(GetApplicationAttemptsRequest.class);
    request.setApplicationId(appId);
    GetApplicationAttemptsResponse response = rmClient
        .getApplicationAttempts(request);
    return response.getApplicationAttemptList();
  } catch (YarnException e) {
    if (!historyServiceEnabled) {
      // Just throw it as usual if historyService is not enabled.
      throw e;
    }
    // Even if history-service is enabled, treat all exceptions still the same
    // except the following
    if (e.getClass() != ApplicationNotFoundException.class) {
      throw e;
    }
    return historyClient.getApplicationAttempts(appId);
  }
}
 
Example #14
Source Project: hadoop   Author: naver   File: MRDelegationTokenRenewer.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public long renew(Token<?> token, Configuration conf) throws IOException,
    InterruptedException {

  org.apache.hadoop.yarn.api.records.Token dToken =
      org.apache.hadoop.yarn.api.records.Token.newInstance(
        token.getIdentifier(), token.getKind().toString(),
        token.getPassword(), token.getService().toString());

  MRClientProtocol histProxy = instantiateHistoryProxy(conf,
      SecurityUtil.getTokenServiceAddr(token));
  try {
    RenewDelegationTokenRequest request = Records
        .newRecord(RenewDelegationTokenRequest.class);
    request.setDelegationToken(dToken);
    return histProxy.renewDelegationToken(request).getNextExpirationTime();
  } finally {
    stopHistoryProxy(histProxy);
  }

}
 
Example #15
Source Project: hadoop   Author: naver   File: YarnClientImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public ApplicationReport getApplicationReport(ApplicationId appId)
    throws YarnException, IOException {
  GetApplicationReportResponse response = null;
  try {
    GetApplicationReportRequest request = Records
        .newRecord(GetApplicationReportRequest.class);
    request.setApplicationId(appId);
    response = rmClient.getApplicationReport(request);
  } catch (YarnException e) {
    if (!historyServiceEnabled) {
      // Just throw it as usual if historyService is not enabled.
      throw e;
    }
    // Even if history-service is enabled, treat all exceptions still the same
    // except the following
    if (!(e.getClass() == ApplicationNotFoundException.class)) {
      throw e;
    }
    return historyClient.getApplicationReport(appId);
  }
  return response.getApplicationReport();
}
 
Example #16
Source Project: hadoop   Author: naver   File: GetDelegationTokenRequest.java    License: Apache License 2.0 5 votes vote down vote up
@Public
@Stable
public static GetDelegationTokenRequest newInstance(String renewer) {
  GetDelegationTokenRequest request =
      Records.newRecord(GetDelegationTokenRequest.class);
  request.setRenewer(renewer);
  return request;
}
 
Example #17
Source Project: Flink-CEPplus   Author: ljygz   File: Utils.java    License: Apache License 2.0 5 votes vote down vote up
private static LocalResource registerLocalResource(FileSystem fs, Path remoteRsrcPath) throws IOException {
	LocalResource localResource = Records.newRecord(LocalResource.class);
	FileStatus jarStat = fs.getFileStatus(remoteRsrcPath);
	localResource.setResource(ConverterUtils.getYarnUrlFromURI(remoteRsrcPath.toUri()));
	localResource.setSize(jarStat.getLen());
	localResource.setTimestamp(jarStat.getModificationTime());
	localResource.setType(LocalResourceType.FILE);
	localResource.setVisibility(LocalResourceVisibility.APPLICATION);
	return localResource;
}
 
Example #18
Source Project: flink   Author: flink-tpc-ds   File: Utils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates a YARN resource for the remote object at the given location.
 *
 * @param remoteRsrcPath	remote location of the resource
 * @param resourceSize		size of the resource
 * @param resourceModificationTime last modification time of the resource
 *
 * @return YARN resource
 */
private static LocalResource registerLocalResource(
		Path remoteRsrcPath,
		long resourceSize,
		long resourceModificationTime) {
	LocalResource localResource = Records.newRecord(LocalResource.class);
	localResource.setResource(ConverterUtils.getYarnUrlFromURI(remoteRsrcPath.toUri()));
	localResource.setSize(resourceSize);
	localResource.setTimestamp(resourceModificationTime);
	localResource.setType(LocalResourceType.FILE);
	localResource.setVisibility(LocalResourceVisibility.APPLICATION);
	return localResource;
}
 
Example #19
Source Project: hadoop   Author: naver   File: GetQueueUserAclsInfoRequest.java    License: Apache License 2.0 5 votes vote down vote up
@Public
@Stable
public static GetQueueUserAclsInfoRequest newInstance() {
  GetQueueUserAclsInfoRequest request =
      Records.newRecord(GetQueueUserAclsInfoRequest.class);
  return request;
}
 
Example #20
Source Project: hadoop   Author: naver   File: ContainerResourceIncrease.java    License: Apache License 2.0 5 votes vote down vote up
@Public
public static ContainerResourceIncrease newInstance(
    ContainerId existingContainerId, Resource targetCapability, Token token) {
  ContainerResourceIncrease context = Records
      .newRecord(ContainerResourceIncrease.class);
  context.setContainerId(existingContainerId);
  context.setCapability(targetCapability);
  context.setContainerToken(token);
  return context;
}
 
Example #21
Source Project: hadoop   Author: naver   File: FinishApplicationMasterResponse.java    License: Apache License 2.0 5 votes vote down vote up
@Private
@Unstable
public static FinishApplicationMasterResponse newInstance(
    boolean isRemovedFromRMStateStore) {
  FinishApplicationMasterResponse response =
      Records.newRecord(FinishApplicationMasterResponse.class);
  response.setIsUnregistered(isRemovedFromRMStateStore);
  return response;
}
 
Example #22
Source Project: ambari-metrics   Author: apache   File: ApplicationStartData.java    License: Apache License 2.0 5 votes vote down vote up
@Public
@Unstable
public static ApplicationStartData newInstance(ApplicationId applicationId,
    String applicationName, String applicationType, String queue,
    String user, long submitTime, long startTime) {
  ApplicationStartData appSD = Records.newRecord(ApplicationStartData.class);
  appSD.setApplicationId(applicationId);
  appSD.setApplicationName(applicationName);
  appSD.setApplicationType(applicationType);
  appSD.setQueue(queue);
  appSD.setUser(user);
  appSD.setSubmitTime(submitTime);
  appSD.setStartTime(startTime);
  return appSD;
}
 
Example #23
Source Project: ambari-metrics   Author: apache   File: ApplicationFinishData.java    License: Apache License 2.0 5 votes vote down vote up
@Public
@Unstable
public static ApplicationFinishData newInstance(ApplicationId applicationId,
    long finishTime, String diagnosticsInfo,
    FinalApplicationStatus finalApplicationStatus,
    YarnApplicationState yarnApplicationState) {
  ApplicationFinishData appFD =
      Records.newRecord(ApplicationFinishData.class);
  appFD.setApplicationId(applicationId);
  appFD.setFinishTime(finishTime);
  appFD.setDiagnosticsInfo(diagnosticsInfo);
  appFD.setFinalApplicationStatus(finalApplicationStatus);
  appFD.setYarnApplicationState(yarnApplicationState);
  return appFD;
}
 
Example #24
Source Project: hadoop   Author: naver   File: Container.java    License: Apache License 2.0 5 votes vote down vote up
@Private
@Unstable
public static Container newInstance(ContainerId containerId, NodeId nodeId,
    String nodeHttpAddress, Resource resource, Priority priority,
    Token containerToken) {
  Container container = Records.newRecord(Container.class);
  container.setId(containerId);
  container.setNodeId(nodeId);
  container.setNodeHttpAddress(nodeHttpAddress);
  container.setResource(resource);
  container.setPriority(priority);
  container.setContainerToken(containerToken);
  return container;
}
 
Example #25
Source Project: hadoop   Author: naver   File: GetApplicationAttemptsResponse.java    License: Apache License 2.0 5 votes vote down vote up
@Public
@Unstable
public static GetApplicationAttemptsResponse newInstance(
    List<ApplicationAttemptReport> applicationAttempts) {
  GetApplicationAttemptsResponse response =
      Records.newRecord(GetApplicationAttemptsResponse.class);
  response.setApplicationAttemptList(applicationAttempts);
  return response;
}
 
Example #26
Source Project: hadoop   Author: naver   File: GetClusterNodesResponse.java    License: Apache License 2.0 5 votes vote down vote up
@Private
@Unstable
public static GetClusterNodesResponse
    newInstance(List<NodeReport> nodeReports) {
  GetClusterNodesResponse response =
      Records.newRecord(GetClusterNodesResponse.class);
  response.setNodeReports(nodeReports);
  return response;
}
 
Example #27
Source Project: hadoop   Author: naver   File: GetContainerReportRequest.java    License: Apache License 2.0 5 votes vote down vote up
@Public
@Unstable
public static GetContainerReportRequest newInstance(ContainerId containerId) {
  GetContainerReportRequest request =
      Records.newRecord(GetContainerReportRequest.class);
  request.setContainerId(containerId);
  return request;
}
 
Example #28
Source Project: hadoop   Author: naver   File: Resources.java    License: Apache License 2.0 5 votes vote down vote up
public static Resource createResource(int memory, int cores, int gcores) {
  Resource resource = Records.newRecord(Resource.class);
  resource.setMemory(memory);
  resource.setVirtualCores(cores);
  resource.setGpuCores(gcores);
  return resource;
}
 
Example #29
Source Project: hadoop   Author: naver   File: MasterKeyData.java    License: Apache License 2.0 5 votes vote down vote up
public MasterKeyData(int serialNo, SecretKey secretKey) {
  this.masterKeyRecord = Records.newRecord(MasterKey.class);
  this.masterKeyRecord.setKeyId(serialNo);
  this.generatedSecretKey = secretKey;
  this.masterKeyRecord.setBytes(ByteBuffer.wrap(generatedSecretKey
    .getEncoded()));
}
 
Example #30
Source Project: hadoop   Author: naver   File: NodeHeartbeatRequest.java    License: Apache License 2.0 5 votes vote down vote up
public static NodeHeartbeatRequest newInstance(NodeStatus nodeStatus,
    MasterKey lastKnownContainerTokenMasterKey,
    MasterKey lastKnownNMTokenMasterKey) {
  NodeHeartbeatRequest nodeHeartbeatRequest =
      Records.newRecord(NodeHeartbeatRequest.class);
  nodeHeartbeatRequest.setNodeStatus(nodeStatus);
  nodeHeartbeatRequest
      .setLastKnownContainerTokenMasterKey(lastKnownContainerTokenMasterKey);
  nodeHeartbeatRequest
      .setLastKnownNMTokenMasterKey(lastKnownNMTokenMasterKey);
  return nodeHeartbeatRequest;
}