org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse Java Examples

The following examples show how to use org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: YarnService.java    From incubator-gobblin with Apache License 2.0 6 votes vote down vote up
@Override
protected void startUp() throws Exception {
  LOGGER.info("Starting the YarnService");

  // Register itself with the EventBus for container-related requests
  this.eventBus.register(this);

  this.amrmClientAsync.start();
  this.nmClientAsync.start();

  // The ApplicationMaster registration response is used to determine the maximum resource capacity of the cluster
  RegisterApplicationMasterResponse response = this.amrmClientAsync.registerApplicationMaster(
      GobblinClusterUtils.getHostname(), -1, "");
  LOGGER.info("ApplicationMaster registration response: " + response);
  this.maxResourceCapacity = Optional.of(response.getMaximumResourceCapability());

  LOGGER.info("Requesting initial containers");
  requestInitialContainers(this.initialContainers);
}
 
Example #2
Source File: YarnResourceManager.java    From flink with Apache License 2.0 6 votes vote down vote up
private void updateMatchingStrategy(final RegisterApplicationMasterResponse registerApplicationMasterResponse) {
	final Optional<Set<String>> schedulerResourceTypesOptional =
		registerApplicationMasterResponseReflector.getSchedulerResourceTypeNames(registerApplicationMasterResponse);

	final WorkerSpecContainerResourceAdapter.MatchingStrategy strategy;
	if (schedulerResourceTypesOptional.isPresent()) {
		Set<String> types = schedulerResourceTypesOptional.get();
		log.info("Register application master response contains scheduler resource types: {}.", types);
		matchingStrategy = types.contains("CPU") ?
			WorkerSpecContainerResourceAdapter.MatchingStrategy.MATCH_VCORE :
			WorkerSpecContainerResourceAdapter.MatchingStrategy.IGNORE_VCORE;
	} else {
		log.info("Register application master response does not contain scheduler resource types, use '{}'.",
			YarnConfigOptionsInternal.MATCH_CONTAINER_VCORES.key());
	}
	log.info("Container matching strategy: {}.", matchingStrategy);
}
 
Example #3
Source File: ProtocolHATestBase.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public RegisterApplicationMasterResponse registerApplicationMaster(
    RegisterApplicationMasterRequest request) throws YarnException,
    IOException {
  resetStartFailoverFlag(true);
  // make sure failover has been triggered
  Assert.assertTrue(waittingForFailOver());
  return createFakeRegisterApplicationMasterResponse();
}
 
Example #4
Source File: ApplicationMasterProtocolPBClientImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public RegisterApplicationMasterResponse registerApplicationMaster(
    RegisterApplicationMasterRequest request) throws YarnException,
    IOException {
  RegisterApplicationMasterRequestProto requestProto =
      ((RegisterApplicationMasterRequestPBImpl) request).getProto();
  try {
    return new RegisterApplicationMasterResponsePBImpl(
      proxy.registerApplicationMaster(null, requestProto));
  } catch (ServiceException e) {
    RPCUtil.unwrapAndThrowException(e);
    return null;
  }
}
 
Example #5
Source File: ApplicationMaster.java    From TensorFlowOnYARN with Apache License 2.0 5 votes vote down vote up
@Override
public void init(CommandLine cliParser) throws Exception {
  LOG.info("Starting ApplicationMaster");

  // Args have to be initialized first
  this.args = new ApplicationMasterArgs(cliParser);
  this.clusterSpec = ClusterSpec.empty();

  String hostname = System.getenv(Environment.NM_HOST.name());
  int rpcPort = setupRPCService(hostname);
  RegisterApplicationMasterResponse response = setupRMConnection(hostname, rpcPort);
  setupPreviousRunningContainers(response);
  setupContainerResource(response);
  setupNMConnection();
}
 
Example #6
Source File: TestRPCFactories.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public RegisterApplicationMasterResponse registerApplicationMaster(
    RegisterApplicationMasterRequest request) throws YarnException,
    IOException {
  // TODO Auto-generated method stub
  return null;
}
 
Example #7
Source File: YarnResourceManager.java    From flink with Apache License 2.0 5 votes vote down vote up
private void getContainersFromPreviousAttempts(final RegisterApplicationMasterResponse registerApplicationMasterResponse) {
	final List<Container> containersFromPreviousAttempts =
		new RegisterApplicationMasterResponseReflector(log).getContainersFromPreviousAttempts(registerApplicationMasterResponse);

	log.info("Recovered {} containers from previous attempts ({}).", containersFromPreviousAttempts.size(), containersFromPreviousAttempts);

	for (final Container container : containersFromPreviousAttempts) {
		workerNodeMap.put(new ResourceID(container.getId().toString()), new YarnWorkerNode(container));
	}
}
 
Example #8
Source File: TestApplicationMasterServiceProtocolOnHA.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 15000)
public void testRegisterApplicationMasterOnHA() throws YarnException,
    IOException {
  RegisterApplicationMasterRequest request =
      RegisterApplicationMasterRequest.newInstance("localhost", 0, "");
  RegisterApplicationMasterResponse response =
      amClient.registerApplicationMaster(request);
  Assert.assertEquals(response,
      this.cluster.createFakeRegisterApplicationMasterResponse());
}
 
Example #9
Source File: RMCommunicator.java    From hadoop with Apache License 2.0 5 votes vote down vote up
protected void register() {
  //Register
  InetSocketAddress serviceAddr = null;
  if (clientService != null ) {
    serviceAddr = clientService.getBindAddress();
  }
  try {
    RegisterApplicationMasterRequest request =
      recordFactory.newRecordInstance(RegisterApplicationMasterRequest.class);
    if (serviceAddr != null) {
      request.setHost(serviceAddr.getHostName());
      request.setRpcPort(serviceAddr.getPort());
      request.setTrackingUrl(MRWebAppUtil
          .getAMWebappScheme(getConfig())
          + serviceAddr.getHostName() + ":" + clientService.getHttpPort());
    }
    RegisterApplicationMasterResponse response =
      scheduler.registerApplicationMaster(request);
    isApplicationMasterRegistered = true;
    maxContainerCapability = response.getMaximumResourceCapability();
    this.context.getClusterInfo().setMaxContainerCapability(
        maxContainerCapability);
    if (UserGroupInformation.isSecurityEnabled()) {
      setClientToAMToken(response.getClientToAMTokenMasterKey());        
    }
    this.applicationACLs = response.getApplicationACLs();
    LOG.info("maxContainerCapability: " + maxContainerCapability);
    String queue = response.getQueue();
    LOG.info("queue: " + queue);
    job.setQueueName(queue);
    this.schedulerResourceTypes.addAll(response.getSchedulerResourceTypes());
  } catch (Exception are) {
    LOG.error("Exception while registering", are);
    throw new YarnRuntimeException(are);
  }
}
 
Example #10
Source File: TestRMContainerAllocator.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public RegisterApplicationMasterResponse registerApplicationMaster(
    RegisterApplicationMasterRequest request) throws YarnException,
    IOException {
  return RegisterApplicationMasterResponse.newInstance(
      Resource.newInstance(512, 1),
      Resource.newInstance(512000, 1024),
      Collections.<ApplicationAccessType,String>emptyMap(),
      ByteBuffer.wrap("fake_key".getBytes()),
      Collections.<Container>emptyList(),
      "default",
      Collections.<NMToken>emptyList());
}
 
Example #11
Source File: Hadoop21YarnAMClient.java    From twill with Apache License 2.0 5 votes vote down vote up
@Override
protected void startUp() throws Exception {
  Preconditions.checkNotNull(trackerAddr, "Tracker address not set.");
  Preconditions.checkNotNull(trackerUrl, "Tracker URL not set.");

  amrmClient.start();
  RegisterApplicationMasterResponse response = amrmClient.registerApplicationMaster(trackerAddr.getHostName(),
                                                                                    trackerAddr.getPort(),
                                                                                    trackerUrl.toString());
  maxCapability = response.getMaximumResourceCapability();
  nmClient.startAndWait();
}
 
Example #12
Source File: ApplicationMasterProtocolPBClientImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public RegisterApplicationMasterResponse registerApplicationMaster(
    RegisterApplicationMasterRequest request) throws YarnException,
    IOException {
  RegisterApplicationMasterRequestProto requestProto =
      ((RegisterApplicationMasterRequestPBImpl) request).getProto();
  try {
    return new RegisterApplicationMasterResponsePBImpl(
      proxy.registerApplicationMaster(null, requestProto));
  } catch (ServiceException e) {
    RPCUtil.unwrapAndThrowException(e);
    return null;
  }
}
 
Example #13
Source File: TestRPCFactories.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public RegisterApplicationMasterResponse registerApplicationMaster(
    RegisterApplicationMasterRequest request) throws YarnException,
    IOException {
  // TODO Auto-generated method stub
  return null;
}
 
Example #14
Source File: MockAM.java    From big-c with Apache License 2.0 5 votes vote down vote up
public RegisterApplicationMasterResponse registerAppAttempt(boolean wait)
    throws Exception {
  if (wait) {
    waitForState(RMAppAttemptState.LAUNCHED);
  }
  responseId = 0;
  final RegisterApplicationMasterRequest req =
      Records.newRecord(RegisterApplicationMasterRequest.class);
  req.setHost("");
  req.setRpcPort(1);
  req.setTrackingUrl("");
  if (ugi == null) {
    ugi = UserGroupInformation.createRemoteUser(
        attemptId.toString());
    Token<AMRMTokenIdentifier> token =
        context.getRMApps().get(attemptId.getApplicationId())
            .getRMAppAttempt(attemptId).getAMRMToken();
    ugi.addTokenIdentifier(token.decodeIdentifier());
  }
  try {
    return ugi
      .doAs(
          new PrivilegedExceptionAction<RegisterApplicationMasterResponse>() {
        @Override
        public RegisterApplicationMasterResponse run() throws Exception {
          return amRMProtocol.registerApplicationMaster(req);
        }
      });
  } catch (UndeclaredThrowableException e) {
    throw (Exception) e.getCause();
  }
}
 
Example #15
Source File: AMRMClientImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public RegisterApplicationMasterResponse registerApplicationMaster(
    String appHostName, int appHostPort, String appTrackingUrl)
    throws YarnException, IOException {
  this.appHostName = appHostName;
  this.appHostPort = appHostPort;
  this.appTrackingUrl = appTrackingUrl;
  Preconditions.checkArgument(appHostName != null,
      "The host name should not be null");
  Preconditions.checkArgument(appHostPort >= -1, "Port number of the host"
      + " should be any integers larger than or equal to -1");

  return registerApplicationMaster();
}
 
Example #16
Source File: AMRMClientImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
private RegisterApplicationMasterResponse registerApplicationMaster()
    throws YarnException, IOException {
  RegisterApplicationMasterRequest request =
      RegisterApplicationMasterRequest.newInstance(this.appHostName,
          this.appHostPort, this.appTrackingUrl);
  RegisterApplicationMasterResponse response =
      rmClient.registerApplicationMaster(request);
  synchronized (this) {
    lastResponseId = 0;
    if (!response.getNMTokensFromPreviousAttempts().isEmpty()) {
      populateNMTokens(response.getNMTokensFromPreviousAttempts());
    }
  }
  return response;
}
 
Example #17
Source File: AMRMClientAsyncImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Registers this application master with the resource manager. On successful
 * registration, starts the heartbeating thread.
 * @throws YarnException
 * @throws IOException
 */
public RegisterApplicationMasterResponse registerApplicationMaster(
    String appHostName, int appHostPort, String appTrackingUrl)
    throws YarnException, IOException {
  RegisterApplicationMasterResponse response = client
      .registerApplicationMaster(appHostName, appHostPort, appTrackingUrl);
  heartbeatThread.start();
  return response;
}
 
Example #18
Source File: YarnResourceManager.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private void getContainersFromPreviousAttempts(final RegisterApplicationMasterResponse registerApplicationMasterResponse) {
	final List<Container> containersFromPreviousAttempts =
		new RegisterApplicationMasterResponseReflector(log).getContainersFromPreviousAttempts(registerApplicationMasterResponse);

	log.info("Recovered {} containers from previous attempts ({}).", containersFromPreviousAttempts.size(), containersFromPreviousAttempts);

	for (final Container container : containersFromPreviousAttempts) {
		workerNodeMap.put(new ResourceID(container.getId().toString()), new YarnWorkerNode(container));
	}
}
 
Example #19
Source File: ProtocolHATestBase.java    From big-c with Apache License 2.0 5 votes vote down vote up
public RegisterApplicationMasterResponse
createFakeRegisterApplicationMasterResponse() {
  Resource minCapability = Resource.newInstance(2048, 2);
  Resource maxCapability = Resource.newInstance(4096, 4);
  Map<ApplicationAccessType, String> acls =
      new HashMap<ApplicationAccessType, String>();
  acls.put(ApplicationAccessType.MODIFY_APP, "*");
  ByteBuffer key = ByteBuffer.wrap("fake_key".getBytes());
  return RegisterApplicationMasterResponse.newInstance(minCapability,
      maxCapability, acls, key, new ArrayList<Container>(), "root",
      new ArrayList<NMToken>());
}
 
Example #20
Source File: TestApplicationMasterServiceProtocolOnHA.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 15000)
public void testRegisterApplicationMasterOnHA() throws YarnException,
    IOException {
  RegisterApplicationMasterRequest request =
      RegisterApplicationMasterRequest.newInstance("localhost", 0, "");
  RegisterApplicationMasterResponse response =
      amClient.registerApplicationMaster(request);
  Assert.assertEquals(response,
      this.cluster.createFakeRegisterApplicationMasterResponse());
}
 
Example #21
Source File: RMCommunicator.java    From big-c with Apache License 2.0 5 votes vote down vote up
protected void register() {
  //Register
  InetSocketAddress serviceAddr = null;
  if (clientService != null ) {
    serviceAddr = clientService.getBindAddress();
  }
  try {
    RegisterApplicationMasterRequest request =
      recordFactory.newRecordInstance(RegisterApplicationMasterRequest.class);
    if (serviceAddr != null) {
      request.setHost(serviceAddr.getHostName());
      request.setRpcPort(serviceAddr.getPort());
      request.setTrackingUrl(MRWebAppUtil
          .getAMWebappScheme(getConfig())
          + serviceAddr.getHostName() + ":" + clientService.getHttpPort());
    }
    RegisterApplicationMasterResponse response =
      scheduler.registerApplicationMaster(request);
    isApplicationMasterRegistered = true;
    maxContainerCapability = response.getMaximumResourceCapability();
    this.context.getClusterInfo().setMaxContainerCapability(
        maxContainerCapability);
    if (UserGroupInformation.isSecurityEnabled()) {
      setClientToAMToken(response.getClientToAMTokenMasterKey());        
    }
    this.applicationACLs = response.getApplicationACLs();
    LOG.info("maxContainerCapability: " + maxContainerCapability);
    String queue = response.getQueue();
    LOG.info("queue: " + queue);
    job.setQueueName(queue);
    this.schedulerResourceTypes.addAll(response.getSchedulerResourceTypes());
  } catch (Exception are) {
    LOG.error("Exception while registering", are);
    throw new YarnRuntimeException(are);
  }
}
 
Example #22
Source File: TestRMContainerAllocator.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public RegisterApplicationMasterResponse registerApplicationMaster(
    RegisterApplicationMasterRequest request) throws YarnException,
    IOException {
  return RegisterApplicationMasterResponse.newInstance(
      Resource.newInstance(512, 1),
      Resource.newInstance(512000, 1024),
      Collections.<ApplicationAccessType,String>emptyMap(),
      ByteBuffer.wrap("fake_key".getBytes()),
      Collections.<Container>emptyList(),
      "default",
      Collections.<NMToken>emptyList());
}
 
Example #23
Source File: YarnResourceManager.java    From flink with Apache License 2.0 5 votes vote down vote up
private void getContainersFromPreviousAttempts(final RegisterApplicationMasterResponse registerApplicationMasterResponse) {
	final List<Container> containersFromPreviousAttempts =
		registerApplicationMasterResponseReflector.getContainersFromPreviousAttempts(registerApplicationMasterResponse);

	log.info("Recovered {} containers from previous attempts ({}).", containersFromPreviousAttempts.size(), containersFromPreviousAttempts);

	for (final Container container : containersFromPreviousAttempts) {
		workerNodeMap.put(new ResourceID(container.getId().toString()), new YarnWorkerNode(container));
	}
}
 
Example #24
Source File: ApplicationMasterRegistration.java    From reef with Apache License 2.0 5 votes vote down vote up
/**
 * Set the registration information. This is a set-once field.
 *
 * @param registration
 */
synchronized void setRegistration(final RegisterApplicationMasterResponse registration) {
  if (this.isPresent()) {
    throw new RuntimeException("Trying to re-register the AM");
  }
  this.registration = Optional.of(registration);
}
 
Example #25
Source File: TestTaskSchedulerHelpers.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
public RegisterApplicationMasterResponse registerApplicationMaster(
    String appHostName, int appHostPort, String appTrackingUrl) {
  RegisterApplicationMasterResponse mockRegResponse = mock(RegisterApplicationMasterResponse.class);
  Resource mockMaxResource = mock(Resource.class);
  Map<ApplicationAccessType, String> mockAcls = mock(Map.class);
  when(mockRegResponse.getMaximumResourceCapability()).thenReturn(
      mockMaxResource);
  when(mockRegResponse.getApplicationACLs()).thenReturn(mockAcls);
  return mockRegResponse;
}
 
Example #26
Source File: YarnManager.java    From Scribengin with GNU Affero General Public License v3.0 5 votes vote down vote up
@Inject
public void onInit(VMConfig vmConfig) throws Exception {
  logger.info("Start init(VMConfig vmConfig)");
  this.vmConfig = vmConfig;
  try {
    this.yarnConfig = vmConfig.getHadoopProperties();
    conf = new YarnConfiguration() ;
    vmConfig.overrideHadoopConfiguration(conf);
    
    amrmClient = AMRMClient.createAMRMClient();
    amrmClientAsync = AMRMClientAsync.createAMRMClientAsync(amrmClient, 1000, new AMRMCallbackHandler());
    amrmClientAsync.init(conf);
    amrmClientAsync.start();
    
    nmClient = NMClient.createNMClient();
    nmClient.init(conf);
    nmClient.start();
    // Register with RM
    String appHostName = InetAddress.getLocalHost().getHostAddress()  ;

    RegisterApplicationMasterResponse registerResponse = amrmClientAsync.registerApplicationMaster(appHostName, 0, "");
    System.out.println("amrmClientAsync.registerApplicationMaster");
  } catch(Throwable t) {
    logger.error("Error: " , t);
    t.printStackTrace();
  }
  logger.info("Finish init(VMConfig vmConfig)");
}
 
Example #27
Source File: HadoopShim27.java    From tez with Apache License 2.0 5 votes vote down vote up
@Override
public Set<String> getSupportedResourceTypes(RegisterApplicationMasterResponse response) {
  EnumSet<SchedulerResourceTypes> supportedResourceTypes = response.getSchedulerResourceTypes();
  Set<String> supportedTypes = new HashSet<String>();
  for (SchedulerResourceTypes resourceType : supportedResourceTypes) {
    supportedTypes.add(resourceType.name());
  }
  return supportedTypes;
}
 
Example #28
Source File: HadoopShim28.java    From tez with Apache License 2.0 5 votes vote down vote up
@Override
public Set<String> getSupportedResourceTypes(RegisterApplicationMasterResponse response) {
  EnumSet<SchedulerResourceTypes> supportedResourceTypes = response.getSchedulerResourceTypes();
  Set<String> supportedTypes = new HashSet<String>();
  for (SchedulerResourceTypes resourceType : supportedResourceTypes) {
    supportedTypes.add(resourceType.name());
  }
  return supportedTypes;
}
 
Example #29
Source File: TestTaskSchedulerHelpers.java    From tez with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
public RegisterApplicationMasterResponse registerApplicationMaster(
    String appHostName, int appHostPort, String appTrackingUrl) {
  mockRegResponse = mock(RegisterApplicationMasterResponse.class);
  Resource mockMaxResource = mock(Resource.class);
  Map<ApplicationAccessType, String> mockAcls = mock(Map.class);
  when(mockRegResponse.getMaximumResourceCapability()).thenReturn(
      mockMaxResource);
  when(mockRegResponse.getApplicationACLs()).thenReturn(mockAcls);
  return mockRegResponse;
}
 
Example #30
Source File: TestDagAwareYarnTaskScheduler.java    From tez with Apache License 2.0 5 votes vote down vote up
@Override
public RegisterApplicationMasterResponse registerApplicationMaster(String appHostName, int appHostPort,
    String appTrackingUrl) {
  mockRegResponse = mock(RegisterApplicationMasterResponse.class);
  Resource mockMaxResource = Resources.createResource(1024*1024, 1024);
  Map<ApplicationAccessType, String> mockAcls = Collections.emptyMap();
  when(mockRegResponse.getMaximumResourceCapability()).thenReturn(
      mockMaxResource);
  when(mockRegResponse.getApplicationACLs()).thenReturn(mockAcls);
  when(mockRegResponse.getSchedulerResourceTypes()).thenReturn(
      EnumSet.of(SchedulerResourceTypes.MEMORY, SchedulerResourceTypes.CPU));
  return mockRegResponse;
}