org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext Java Examples

The following examples show how to use org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: AbstractYarnClusterDescriptor.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
private void setApplicationTags(final ApplicationSubmissionContext appContext) throws InvocationTargetException,
	IllegalAccessException {

	final ApplicationSubmissionContextReflector reflector = ApplicationSubmissionContextReflector.getInstance();
	final String tagsString = flinkConfiguration.getString(YarnConfigOptions.APPLICATION_TAGS);

	final Set<String> applicationTags = new HashSet<>();

	// Trim whitespace and cull empty tags
	for (final String tag : tagsString.split(",")) {
		final String trimmedTag = tag.trim();
		if (!trimmedTag.isEmpty()) {
			applicationTags.add(trimmedTag);
		}
	}

	reflector.setApplicationTags(appContext, applicationTags);
}
 
Example #2
Source File: TestAppManager.java    From big-c with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("deprecation")
@Before
public void setUp() {
  long now = System.currentTimeMillis();

  rmContext = mockRMContext(1, now - 10);
  ResourceScheduler scheduler = mockResourceScheduler();
  Configuration conf = new Configuration();
  ApplicationMasterService masterService =
      new ApplicationMasterService(rmContext, scheduler);
  appMonitor = new TestRMAppManager(rmContext,
      new ClientToAMTokenSecretManagerInRM(), scheduler, masterService,
      new ApplicationACLsManager(conf), conf);

  appId = MockApps.newAppID(1);
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  asContext =
      recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
  asContext.setApplicationId(appId);
  asContext.setAMContainerSpec(mockContainerLaunchContext(recordFactory));
  asContext.setResource(mockResource());
  setupDispatcher(rmContext, conf);
}
 
Example #3
Source File: BuilderUtils.java    From big-c with Apache License 2.0 6 votes vote down vote up
public static ApplicationSubmissionContext newApplicationSubmissionContext(
    ApplicationId applicationId, String applicationName, String queue,
    Priority priority, ContainerLaunchContext amContainer,
    boolean isUnmanagedAM, boolean cancelTokensWhenComplete,
    int maxAppAttempts, Resource resource, String applicationType) {
  ApplicationSubmissionContext context =
      recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
  context.setApplicationId(applicationId);
  context.setApplicationName(applicationName);
  context.setQueue(queue);
  context.setPriority(priority);
  context.setAMContainerSpec(amContainer);
  context.setUnmanagedAM(isUnmanagedAM);
  context.setCancelTokensWhenComplete(cancelTokensWhenComplete);
  context.setMaxAppAttempts(maxAppAttempts);
  context.setResource(resource);
  context.setApplicationType(applicationType);
  return context;
}
 
Example #4
Source File: SchedulerApplicationAttempt.java    From big-c with Apache License 2.0 6 votes vote down vote up
public SchedulerApplicationAttempt(ApplicationAttemptId applicationAttemptId, 
    String user, Queue queue, ActiveUsersManager activeUsersManager,
    RMContext rmContext) {
  Preconditions.checkNotNull(rmContext, "RMContext should not be null");
  this.rmContext = rmContext;
  this.appSchedulingInfo = 
      new AppSchedulingInfo(applicationAttemptId, user, queue,  
          activeUsersManager, rmContext.getEpoch());
  this.queue = queue;
  this.pendingRelease = new HashSet<ContainerId>();
  this.attemptId = applicationAttemptId;
  if (rmContext.getRMApps() != null &&
      rmContext.getRMApps()
          .containsKey(applicationAttemptId.getApplicationId())) {
    ApplicationSubmissionContext appSubmissionContext =
        rmContext.getRMApps().get(applicationAttemptId.getApplicationId())
            .getApplicationSubmissionContext();
    if (appSubmissionContext != null) {
      unmanagedAM = appSubmissionContext.getUnmanagedAM();
      this.logAggregationContext =
          appSubmissionContext.getLogAggregationContext();
    }
  }
}
 
Example #5
Source File: AMLauncher.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private ContainerLaunchContext createAMContainerLaunchContext(
    ApplicationSubmissionContext applicationMasterContext,
    ContainerId containerID) throws IOException {

  // Construct the actual Container
  ContainerLaunchContext container = 
      applicationMasterContext.getAMContainerSpec();
  LOG.info("Command to launch container "
      + containerID
      + " : "
      + StringUtils.arrayToString(container.getCommands().toArray(
          new String[0])));
  
  // Finalize the container
  setupTokens(container, containerID);
  
  return container;
}
 
Example #6
Source File: TezClientUtils.java    From tez with Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
// this method will set the app priority only if the priority config has been defined
public static void setApplicationPriority(ApplicationSubmissionContext context,
    AMConfiguration amConfig) {
  if (amConfig.getTezConfiguration().get(TezConfiguration.TEZ_AM_APPLICATION_PRIORITY) != null) {
    // since we already checked not null condition, we are guaranteed that default value
    // (0 in this case for getInt) will not be returned/used.
    // The idea is to not use any default priority from TEZ side, if none provided in config;
    // let YARN determine the priority of the submitted application
    int priority = amConfig.getTezConfiguration().getInt(TezConfiguration.TEZ_AM_APPLICATION_PRIORITY, 0);
    context.setPriority(Priority.newInstance(priority));
    if (LOG.isDebugEnabled()) {
      LOG.debug("Settting TEZ application priority, applicationId= " + context.getApplicationId() +
          ", priority= " + context.getPriority().getPriority());
    }
  }
}
 
Example #7
Source File: TestRMAppTransitions.java    From hadoop with Apache License 2.0 6 votes vote down vote up
protected RMApp testCreateAppFinished(
    ApplicationSubmissionContext submissionContext,
    String diagnostics) throws IOException {
  // unmanaged AMs don't use the FINISHING state
  RMApp application = null;
  if (submissionContext != null && submissionContext.getUnmanagedAM()) {
    application = testCreateAppRunning(submissionContext);
  } else {
    application = testCreateAppFinishing(submissionContext);
  }
  // RUNNING/FINISHING => FINISHED event RMAppEventType.ATTEMPT_FINISHED
  RMAppEvent finishedEvent = new RMAppFinishedAttemptEvent(
      application.getApplicationId(), diagnostics);
  application.handle(finishedEvent);
  assertAppState(RMAppState.FINISHED, application);
  assertTimesAtFinish(application);
  // finished without a proper unregister implies failed
  assertFinalAppStatus(FinalApplicationStatus.FAILED, application);
  Assert.assertTrue("Finished app missing diagnostics",
      application.getDiagnostics().indexOf(diagnostics) != -1);
  return application;
}
 
Example #8
Source File: RMHATestBase.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
protected void submitApplication(
    ApplicationSubmissionContext submissionContext, long submitTime,
    String user) throws YarnException {
  //Do nothing, just add the application to RMContext
  RMAppImpl application =
      new RMAppImpl(submissionContext.getApplicationId(), this.rmContext,
          this.conf, submissionContext.getApplicationName(), user,
          submissionContext.getQueue(), submissionContext,
          this.rmContext.getScheduler(),
          this.rmContext.getApplicationMasterService(),
          submitTime, submissionContext.getApplicationType(),
          submissionContext.getApplicationTags(), null);
  this.rmContext.getRMApps().put(submissionContext.getApplicationId(),
      application);
  //Do not send RMAppEventType.START event
  //so the state of Application will not reach to NEW_SAVING state.
}
 
Example #9
Source File: RMAppAttemptImpl.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public RMAppAttemptImpl(ApplicationAttemptId appAttemptId,
    RMContext rmContext, YarnScheduler scheduler,
    ApplicationMasterService masterService,
    ApplicationSubmissionContext submissionContext,
    Configuration conf, boolean maybeLastAttempt, ResourceRequest amReq) {
  this.conf = conf;
  this.applicationAttemptId = appAttemptId;
  this.rmContext = rmContext;
  this.eventHandler = rmContext.getDispatcher().getEventHandler();
  this.submissionContext = submissionContext;
  this.scheduler = scheduler;
  this.masterService = masterService;

  ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
  this.readLock = lock.readLock();
  this.writeLock = lock.writeLock();

  this.proxiedTrackingUrl = generateProxyUriWithScheme();
  this.maybeLastAttempt = maybeLastAttempt;
  this.stateMachine = stateMachineFactory.make(this);

  this.attemptMetrics =
      new RMAppAttemptMetrics(applicationAttemptId, rmContext);
  
  this.amReq = amReq;
}
 
Example #10
Source File: AbstractYarnClusterDescriptor.java    From flink with Apache License 2.0 6 votes vote down vote up
private void setApplicationTags(final ApplicationSubmissionContext appContext) throws InvocationTargetException,
	IllegalAccessException {

	final ApplicationSubmissionContextReflector reflector = ApplicationSubmissionContextReflector.getInstance();
	final String tagsString = flinkConfiguration.getString(YarnConfigOptions.APPLICATION_TAGS);

	final Set<String> applicationTags = new HashSet<>();

	// Trim whitespace and cull empty tags
	for (final String tag : tagsString.split(",")) {
		final String trimmedTag = tag.trim();
		if (!trimmedTag.isEmpty()) {
			applicationTags.add(trimmedTag);
		}
	}

	reflector.setApplicationTags(appContext, applicationTags);
}
 
Example #11
Source File: TezClient.java    From tez with Apache License 2.0 6 votes vote down vote up
private ApplicationSubmissionContext setupApplicationContext() throws IOException, YarnException {
  TezClientUtils.processTezLocalCredentialsFile(sessionCredentials,
          amConfig.getTezConfiguration());

  Map<String, LocalResource> tezJarResources = getTezJarResources(sessionCredentials);
  // Add session token for shuffle
  TezClientUtils.createSessionToken(sessionAppId.toString(),
          jobTokenSecretManager, sessionCredentials);

  ApplicationSubmissionContext appContext =
          TezClientUtils.createApplicationSubmissionContext(
                  sessionAppId,
                  null, clientName, amConfig,
                  tezJarResources, sessionCredentials, usingTezArchiveDeploy, apiVersionInfo,
                  servicePluginsDescriptor, javaOptsChecker);

  // Set Tez Sessions to not retry on AM crashes if recovery is disabled
  if (!amConfig.getTezConfiguration().getBoolean(
          TezConfiguration.DAG_RECOVERY_ENABLED,
          TezConfiguration.DAG_RECOVERY_ENABLED_DEFAULT)) {
    appContext.setMaxAppAttempts(1);
  }
  return appContext;
}
 
Example #12
Source File: YarnClusterDescriptor.java    From flink with Apache License 2.0 6 votes vote down vote up
private void setApplicationTags(final ApplicationSubmissionContext appContext) throws InvocationTargetException,
		IllegalAccessException {

	final ApplicationSubmissionContextReflector reflector = ApplicationSubmissionContextReflector.getInstance();
	final String tagsString = flinkConfiguration.getString(YarnConfigOptions.APPLICATION_TAGS);

	final Set<String> applicationTags = new HashSet<>();

	// Trim whitespace and cull empty tags
	for (final String tag : tagsString.split(",")) {
		final String trimmedTag = tag.trim();
		if (!trimmedTag.isEmpty()) {
			applicationTags.add(trimmedTag);
		}
	}

	reflector.setApplicationTags(appContext, applicationTags);
}
 
Example #13
Source File: TestYARNRunner.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test(timeout=20000)
public void testAMProfiler() throws Exception {
  JobConf jobConf = new JobConf();

  jobConf.setBoolean(MRJobConfig.MR_AM_PROFILE, true);

  YARNRunner yarnRunner = new YARNRunner(jobConf);

  ApplicationSubmissionContext submissionContext =
      buildSubmitContext(yarnRunner, jobConf);

  ContainerLaunchContext containerSpec = submissionContext.getAMContainerSpec();
  List<String> commands = containerSpec.getCommands();

  for(String command : commands) {
    if (command != null) {
      if (command.contains(PROFILE_PARAMS)) {
        return;
      }
    }
  }
  throw new IllegalStateException("Profiler opts not found!");
}
 
Example #14
Source File: RMStateStoreTestBase.java    From hadoop with Apache License 2.0 6 votes vote down vote up
protected RMApp storeApp(RMStateStore store, ApplicationId appId,
    long submitTime,
    long startTime) throws Exception {
  ApplicationSubmissionContext context =
      new ApplicationSubmissionContextPBImpl();
  context.setApplicationId(appId);

  RMApp mockApp = mock(RMApp.class);
  when(mockApp.getApplicationId()).thenReturn(appId);
  when(mockApp.getSubmitTime()).thenReturn(submitTime);
  when(mockApp.getStartTime()).thenReturn(startTime);
  when(mockApp.getApplicationSubmissionContext()).thenReturn(context);
  when(mockApp.getUser()).thenReturn("test");
  store.storeNewApplication(mockApp);
  return mockApp;
}
 
Example #15
Source File: TestTezClientUtils.java    From tez with Apache License 2.0 6 votes vote down vote up
@Test(timeout = 2000)
// this test checks if the priority field is set properly in the
// ApplicationSubmissionContext
public void testAppSubmissionContextForPriority() throws Exception {
  TezConfiguration tezConf = new TezConfiguration();
  tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, STAGING_DIR.getAbsolutePath());
  int testpriority = 999;
  ApplicationId appId = ApplicationId.newInstance(1000, 1);
  Credentials credentials = new Credentials();
  TezClientUtils.createSessionToken(appId.toString(),
      new JobTokenSecretManager(), credentials);
  tezConf.setBoolean(TezConfiguration.TEZ_IGNORE_LIB_URIS, true);
  Map<String, LocalResource> m = new HashMap<String, LocalResource>();
  tezConf.setInt(TezConfiguration.TEZ_AM_APPLICATION_PRIORITY, testpriority);
  AMConfiguration amConf =
      new AMConfiguration(tezConf, new HashMap<String, LocalResource>(), credentials);
  ApplicationSubmissionContext appcontext;
  appcontext = TezClientUtils.createApplicationSubmissionContext(
      appId, null, "dagname",
      amConf, m,
      credentials, false,
      new TezApiVersionInfo(), null, null);
  assertEquals(testpriority, appcontext.getPriority().getPriority());
}
 
Example #16
Source File: TestRMAppTransitions.java    From hadoop with Apache License 2.0 5 votes vote down vote up
protected RMApp createNewTestApp(ApplicationSubmissionContext submissionContext) {
  ApplicationId applicationId = MockApps.newAppID(appId++);
  String user = MockApps.newUserName();
  String name = MockApps.newAppName();
  String queue = MockApps.newQueue();
  // ensure max application attempts set to known value
  conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, maxAppAttempts);
  scheduler = mock(YarnScheduler.class);

  ApplicationMasterService masterService =
      new ApplicationMasterService(rmContext, scheduler);
  
  if(submissionContext == null) {
    submissionContext = new ApplicationSubmissionContextPBImpl();
  }
  // applicationId will not be used because RMStateStore is mocked,
  // but applicationId is still set for safety
  submissionContext.setApplicationId(applicationId);

  RMApp application =
      new RMAppImpl(applicationId, rmContext, conf, name, user, queue,
        submissionContext, scheduler, masterService,
        System.currentTimeMillis(), "YARN", null, null);

  testAppStartState(applicationId, user, name, queue, application);
  this.rmContext.getRMApps().putIfAbsent(application.getApplicationId(),
      application);
  return application;
}
 
Example #17
Source File: TestRMAppTransitions.java    From hadoop with Apache License 2.0 5 votes vote down vote up
protected RMApp testCreateAppAccepted(
    ApplicationSubmissionContext submissionContext) throws IOException {
  RMApp application = testCreateAppSubmittedNoRecovery(submissionContext);
// SUBMITTED => ACCEPTED event RMAppEventType.APP_ACCEPTED
  RMAppEvent event = 
      new RMAppEvent(application.getApplicationId(), 
          RMAppEventType.APP_ACCEPTED);
  application.handle(event);
  assertStartTimeSet(application);
  assertAppState(RMAppState.ACCEPTED, application);
  return application;
}
 
Example #18
Source File: TestDelegationTokenRenewer.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test(timeout=20000)
public void testAppSubmissionWithInvalidDelegationToken() throws Exception {
  Configuration conf = new Configuration();
  conf.set(
      CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
      "kerberos");
  UserGroupInformation.setConfiguration(conf);
  MockRM rm = new MockRM(conf) {
    @Override
    protected void doSecureLogin() throws IOException {
      // Skip the login.
    }
  };
  ByteBuffer tokens = ByteBuffer.wrap("BOGUS".getBytes()); 
  ContainerLaunchContext amContainer =
      ContainerLaunchContext.newInstance(
          new HashMap<String, LocalResource>(), new HashMap<String, String>(),
          new ArrayList<String>(), new HashMap<String, ByteBuffer>(), tokens,
          new HashMap<ApplicationAccessType, String>());
  ApplicationSubmissionContext appSubContext =
      ApplicationSubmissionContext.newInstance(
          ApplicationId.newInstance(1234121, 0),
          "BOGUS", "default", Priority.UNDEFINED, amContainer, false,
          true, 1, Resource.newInstance(1024, 1), "BOGUS");
  SubmitApplicationRequest request =
      SubmitApplicationRequest.newInstance(appSubContext);
  try {
    rm.getClientRMService().submitApplication(request);
    fail("Error was excepted.");
  } catch (YarnException e) {
    Assert.assertTrue(e.getMessage().contains(
        "Bad header found in token storage"));
  }
}
 
Example #19
Source File: TestYARNRunner.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testNodeLabelExp() throws Exception {
  JobConf jobConf = new JobConf();

  jobConf.set(MRJobConfig.JOB_NODE_LABEL_EXP, "GPU");
  jobConf.set(MRJobConfig.AM_NODE_LABEL_EXP, "highMem");

  YARNRunner yarnRunner = new YARNRunner(jobConf);
  ApplicationSubmissionContext appSubCtx =
      buildSubmitContext(yarnRunner, jobConf);

  assertEquals(appSubCtx.getNodeLabelExpression(), "GPU");
  assertEquals(appSubCtx.getAMContainerResourceRequest()
      .getNodeLabelExpression(), "highMem");
}
 
Example #20
Source File: FairSchedulerTestBase.java    From hadoop with Apache License 2.0 5 votes vote down vote up
protected void createApplicationWithAMResource(ApplicationAttemptId attId,
    String queue, String user, Resource amResource) {
  RMContext rmContext = resourceManager.getRMContext();
  RMApp rmApp = new RMAppImpl(attId.getApplicationId(), rmContext, conf,
      null, null, null, ApplicationSubmissionContext.newInstance(null, null,
      null, null, null, false, false, 0, amResource, null), null, null,
      0, null, null, null);
  rmContext.getRMApps().put(attId.getApplicationId(), rmApp);
  AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent(
      attId.getApplicationId(), queue, user);
  scheduler.handle(appAddedEvent);
  AppAttemptAddedSchedulerEvent attempAddedEvent =
      new AppAttemptAddedSchedulerEvent(attId, false);
  scheduler.handle(attempAddedEvent);
}
 
Example #21
Source File: TestRMAppTransitions.java    From big-c with Apache License 2.0 5 votes vote down vote up
protected RMApp testCreateAppFinalSaving(
    ApplicationSubmissionContext submissionContext) throws IOException {
  RMApp application = testCreateAppRunning(submissionContext);
  RMAppEvent finishingEvent =
      new RMAppEvent(application.getApplicationId(),
        RMAppEventType.ATTEMPT_UNREGISTERED);
  application.handle(finishingEvent);
  assertAppState(RMAppState.FINAL_SAVING, application);
  assertAppFinalStateSaved(application);
  return application;
}
 
Example #22
Source File: YarnClientImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public YarnClientApplication createApplication()
    throws YarnException, IOException {
  ApplicationSubmissionContext context = Records.newRecord
      (ApplicationSubmissionContext.class);
  GetNewApplicationResponse newApp = getNewApplication();
  ApplicationId appId = newApp.getApplicationId();
  context.setApplicationId(appId);
  return new YarnClientApplication(newApp, context);
}
 
Example #23
Source File: SubmitApplicationRequestPBImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public void setApplicationSubmissionContext(ApplicationSubmissionContext applicationSubmissionContext) {
  maybeInitBuilder();
  if (applicationSubmissionContext == null) 
    builder.clearApplicationSubmissionContext();
  this.applicationSubmissionContext = applicationSubmissionContext;
}
 
Example #24
Source File: SubmitApplicationRequestPBImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public ApplicationSubmissionContext getApplicationSubmissionContext() {
  SubmitApplicationRequestProtoOrBuilder p = viaProto ? proto : builder;
  if (this.applicationSubmissionContext != null) {
    return this.applicationSubmissionContext;
  }
  if (!p.hasApplicationSubmissionContext()) {
    return null;
  }
  this.applicationSubmissionContext = convertFromProtoFormat(p.getApplicationSubmissionContext());
  return this.applicationSubmissionContext;
}
 
Example #25
Source File: TestRMAppTransitions.java    From big-c with Apache License 2.0 5 votes vote down vote up
protected RMApp testCreateAppAccepted(
    ApplicationSubmissionContext submissionContext) throws IOException {
  RMApp application = testCreateAppSubmittedNoRecovery(submissionContext);
// SUBMITTED => ACCEPTED event RMAppEventType.APP_ACCEPTED
  RMAppEvent event = 
      new RMAppEvent(application.getApplicationId(), 
          RMAppEventType.APP_ACCEPTED);
  application.handle(event);
  assertStartTimeSet(application);
  assertAppState(RMAppState.ACCEPTED, application);
  return application;
}
 
Example #26
Source File: YarnClusterDescriptor.java    From flink with Apache License 2.0 5 votes vote down vote up
private void activateHighAvailabilitySupport(ApplicationSubmissionContext appContext) throws
		InvocationTargetException, IllegalAccessException {

	ApplicationSubmissionContextReflector reflector = ApplicationSubmissionContextReflector.getInstance();

	reflector.setKeepContainersAcrossApplicationAttempts(appContext, true);

	reflector.setAttemptFailuresValidityInterval(
			appContext,
			flinkConfiguration.getLong(YarnConfigOptions.APPLICATION_ATTEMPT_FAILURE_VALIDITY_INTERVAL));
}
 
Example #27
Source File: ApplicationStateDataPBImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public ApplicationSubmissionContext getApplicationSubmissionContext() {
  ApplicationStateDataProtoOrBuilder p = viaProto ? proto : builder;
  if(applicationSubmissionContext != null) {
    return applicationSubmissionContext;
  }
  if (!p.hasApplicationSubmissionContext()) {
    return null;
  }
  applicationSubmissionContext = 
      new ApplicationSubmissionContextPBImpl(
          p.getApplicationSubmissionContext());
  return applicationSubmissionContext;
}
 
Example #28
Source File: TestTezClientUtils.java    From tez with Apache License 2.0 5 votes vote down vote up
@Test(timeout=1000)
public void testSetApplicationTags() {
  TezConfiguration conf = new TezConfiguration(false);
  conf.set(TezConfiguration.TEZ_APPLICATION_TAGS, "foo,bar");
  AMConfiguration amconfig = new AMConfiguration(conf, null, null);
  ApplicationSubmissionContext appContext = Records
      .newRecord(ApplicationSubmissionContext.class);
  Collection<String> tagsFromConf =
      amconfig.getTezConfiguration().getTrimmedStringCollection(
      TezConfiguration.TEZ_APPLICATION_TAGS);
  appContext.setApplicationTags(new HashSet<String>(tagsFromConf));
  assertTrue(appContext.getApplicationTags().contains("foo"));
  assertTrue(appContext.getApplicationTags().contains("bar"));
}
 
Example #29
Source File: TezClient.java    From tez with Apache License 2.0 5 votes vote down vote up
/**
 * Start the client. This establishes a connection to the YARN cluster.
 * In session mode, this start the App Master thats runs all the DAGs in the
 * session.
 * @throws TezException
 * @throws IOException
 */
public synchronized void start() throws TezException, IOException {
  amConfig.setYarnConfiguration(new YarnConfiguration(amConfig.getTezConfiguration()));
  startFrameworkClient();
  setupJavaOptsChecker();

  if (isSession) {
    LOG.info("Session mode. Starting session.");
    TezClientUtils.processTezLocalCredentialsFile(sessionCredentials,
        amConfig.getTezConfiguration());

    clientTimeout = amConfig.getTezConfiguration().getInt(
        TezConfiguration.TEZ_SESSION_CLIENT_TIMEOUT_SECS,
        TezConfiguration.TEZ_SESSION_CLIENT_TIMEOUT_SECS_DEFAULT);

    try {
      if (sessionAppId == null) {
        sessionAppId = createApplication();
      }

      ApplicationSubmissionContext appContext = setupApplicationContext();
      frameworkClient.submitApplication(appContext);
      ApplicationReport appReport = frameworkClient.getApplicationReport(sessionAppId);
      LOG.info("The url to track the Tez Session: " + appReport.getTrackingUrl());
      sessionStarted.set(true);
    } catch (YarnException e) {
      throw new TezException(e);
    }

    startClientHeartbeat();
    this.stagingFs = FileSystem.get(amConfig.getTezConfiguration());
  }
}
 
Example #30
Source File: TestZKRMStateStore.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testDuplicateRMAppDeletion() throws Exception {
  TestZKRMStateStoreTester zkTester = new TestZKRMStateStoreTester();
  long submitTime = System.currentTimeMillis();
  long startTime = System.currentTimeMillis() + 1234;
  RMStateStore store = zkTester.getRMStateStore();
  TestDispatcher dispatcher = new TestDispatcher();
  store.setRMDispatcher(dispatcher);

  ApplicationAttemptId attemptIdRemoved = ConverterUtils
      .toApplicationAttemptId("appattempt_1352994193343_0002_000001");
  ApplicationId appIdRemoved = attemptIdRemoved.getApplicationId();
  storeApp(store, appIdRemoved, submitTime, startTime);
  storeAttempt(store, attemptIdRemoved,
      "container_1352994193343_0002_01_000001", null, null, dispatcher);

  ApplicationSubmissionContext context =
      new ApplicationSubmissionContextPBImpl();
  context.setApplicationId(appIdRemoved);
  ApplicationStateData appStateRemoved =
      ApplicationStateData.newInstance(
          submitTime, startTime, context, "user1");
  appStateRemoved.attempts.put(attemptIdRemoved, null);
  store.removeApplicationStateInternal(appStateRemoved);
  try {
    store.removeApplicationStateInternal(appStateRemoved);
  } catch (KeeperException.NoNodeException nne) {
    Assert.fail("NoNodeException should not happen.");
  }
  store.close();
}