Java Code Examples for org.apache.hadoop.yarn.api.records.ApplicationId

The following examples show how to use org.apache.hadoop.yarn.api.records.ApplicationId. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: ZKRMStateStore.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public synchronized void updateApplicationStateInternal(ApplicationId appId,
    ApplicationStateData appStateDataPB) throws Exception {
  String nodeUpdatePath = getNodePath(rmAppRoot, appId.toString());

  if (LOG.isDebugEnabled()) {
    LOG.debug("Storing final state info for app: " + appId + " at: "
        + nodeUpdatePath);
  }
  byte[] appStateData = appStateDataPB.getProto().toByteArray();

  if (existsWithRetries(nodeUpdatePath, false) != null) {
    setDataWithRetries(nodeUpdatePath, appStateData, -1);
  } else {
    createWithRetries(nodeUpdatePath, appStateData, zkAcl,
      CreateMode.PERSISTENT);
    LOG.debug(appId + " znode didn't exist. Created a new znode to"
        + " update the application state.");
  }
}
 
Example 2
Source Project: incubator-tez   Source File: TestTaskExecution.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSingleSuccessfulTask() throws IOException, InterruptedException, TezException,
    ExecutionException {
  ListeningExecutorService executor = null;
  try {
    ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
    executor = MoreExecutors.listeningDecorator(rawExecutor);
    ApplicationId appId = ApplicationId.newInstance(10000, 1);
    TezTaskUmbilicalForTest umbilical = new TezTaskUmbilicalForTest();
    TaskReporter taskReporter = createTaskReporter(appId, umbilical);

    TezTaskRunner taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor,
        TestProcessor.CONF_EMPTY);
    // Setup the executor
    Future<Boolean> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable(taskRunner));
    // Signal the processor to go through
    TestProcessor.signal();
    boolean result = taskRunnerFuture.get();
    assertTrue(result);
    assertNull(taskReporter.currentCallable);
    umbilical.verifyTaskSuccessEvent();
  } finally {
    executor.shutdownNow();
  }
}
 
Example 3
Source Project: big-c   Source File: TestRMContainerAllocator.java    License: Apache License 2.0 6 votes vote down vote up
private static AppContext createAppContext(
    ApplicationAttemptId appAttemptId, Job job) {
  AppContext context = mock(AppContext.class);
  ApplicationId appId = appAttemptId.getApplicationId();
  when(context.getApplicationID()).thenReturn(appId);
  when(context.getApplicationAttemptId()).thenReturn(appAttemptId);
  when(context.getJob(isA(JobId.class))).thenReturn(job);
  when(context.getClusterInfo()).thenReturn(
    new ClusterInfo(Resource.newInstance(10240, 1)));
  when(context.getEventHandler()).thenReturn(new EventHandler() {
    @Override
    public void handle(Event event) {
      // Only capture interesting events.
      if (event instanceof TaskAttemptContainerAssignedEvent) {
        events.add((TaskAttemptContainerAssignedEvent) event);
      } else if (event instanceof TaskAttemptKillEvent) {
        taskAttemptKillEvents.add((TaskAttemptKillEvent)event);
      } else if (event instanceof JobUpdatedNodesEvent) {
        jobUpdatedNodeEvents.add((JobUpdatedNodesEvent)event);
      } else if (event instanceof JobEvent) {
        jobEvents.add((JobEvent)event);
      }
    }
  });
  return context;
}
 
Example 4
Source Project: tez   Source File: TestHistoryEventsProtoConversion.java    License: Apache License 2.0 6 votes vote down vote up
private void testDAGSubmittedEvent() throws Exception {
  DAGSubmittedEvent event = new DAGSubmittedEvent(TezDAGID.getInstance(
      ApplicationId.newInstance(0, 1), 1), 1001l,
      DAGPlan.newBuilder().setName("foo").build(),
      ApplicationAttemptId.newInstance(
          ApplicationId.newInstance(0, 1), 1), null, "", null, null, QUEUE_NAME);
  DAGSubmittedEvent deserializedEvent = (DAGSubmittedEvent)
      testProtoConversion(event);
  Assert.assertEquals(event.getApplicationAttemptId(),
      deserializedEvent.getApplicationAttemptId());
  Assert.assertEquals(event.getDagID(),
      deserializedEvent.getDagID());
  Assert.assertEquals(event.getDAGName(),
      deserializedEvent.getDAGName());
  Assert.assertEquals(event.getSubmitTime(),
      deserializedEvent.getSubmitTime());
  Assert.assertEquals(event.getDAGPlan(),
      deserializedEvent.getDAGPlan());
  Assert.assertEquals(event.getQueueName(), deserializedEvent.getQueueName());
  logEvents(event, deserializedEvent);
}
 
Example 5
Source Project: big-c   Source File: ActiveUsersManager.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * An application has new outstanding requests.
 * 
 * @param user application user 
 * @param applicationId activated application
 */
@Lock({Queue.class, SchedulerApplicationAttempt.class})
synchronized public void activateApplication(
    String user, ApplicationId applicationId) {
  Set<ApplicationId> userApps = usersApplications.get(user);
  if (userApps == null) {
    userApps = new HashSet<ApplicationId>();
    usersApplications.put(user, userApps);
    ++activeUsers;
    metrics.incrActiveUsers();
    LOG.debug("User " + user + " added to activeUsers, currently: " + 
        activeUsers);
  }
  if (userApps.add(applicationId)) {
    metrics.activateApp(user);
  }
}
 
Example 6
Source Project: big-c   Source File: RMNodeImpl.java    License: Apache License 2.0 6 votes vote down vote up
private static void handleRunningAppOnNode(RMNodeImpl rmNode,
    RMContext context, ApplicationId appId, NodeId nodeId) {
  RMApp app = context.getRMApps().get(appId);

  // if we failed getting app by appId, maybe something wrong happened, just
  // add the app to the finishedApplications list so that the app can be
  // cleaned up on the NM
  if (null == app) {
    LOG.warn("Cannot get RMApp by appId=" + appId
        + ", just added it to finishedApplications list for cleanup");
    rmNode.finishedApplications.add(appId);
    return;
  }

  context.getDispatcher().getEventHandler()
      .handle(new RMAppRunningOnNodeEvent(appId, nodeId));
}
 
Example 7
Source Project: spring-cloud-deployer-yarn   Source File: TaskLauncherIT.java    License: Apache License 2.0 6 votes vote down vote up
private ApplicationId assertWaitApp(long timeout, TimeUnit unit, YarnCloudAppService yarnCloudAppService) throws Exception {
	ApplicationId applicationId = null;
	Collection<CloudAppInstanceInfo> instances;
	long end = System.currentTimeMillis() + unit.toMillis(timeout);

	do {
		instances = yarnCloudAppService.getInstances(CloudAppType.TASK);
		if (instances.size() == 1) {
			CloudAppInstanceInfo cloudAppInstanceInfo = instances.iterator().next();
			if (StringUtils.hasText(cloudAppInstanceInfo.getAddress())) {
				applicationId = ConverterUtils.toApplicationId(cloudAppInstanceInfo.getApplicationId());
				break;
			}
		}
		Thread.sleep(1000);
	} while (System.currentTimeMillis() < end);

	assertThat(applicationId, notNullValue());
	return applicationId;
}
 
Example 8
Source Project: incubator-gobblin   Source File: GobblinYarnAppLauncher.java    License: Apache License 2.0 6 votes vote down vote up
private void addContainerLocalResources(ApplicationId applicationId) throws IOException {
  Path appWorkDir = GobblinClusterUtils.getAppWorkDirPathFromConfig(this.config, this.fs, this.applicationName, applicationId.toString());

  Path containerWorkDir = new Path(appWorkDir, GobblinYarnConfigurationKeys.CONTAINER_WORK_DIR_NAME);
  LOGGER.info("Configured Container work directory to: {}", containerWorkDir.toString());

  FileSystem localFs = FileSystem.getLocal(new Configuration());

  if (this.config.hasPath(GobblinYarnConfigurationKeys.CONTAINER_JARS_KEY)) {
    Path appJarsDestDir = new Path(containerWorkDir, GobblinYarnConfigurationKeys.APP_JARS_DIR_NAME);
    addAppJars(this.config.getString(GobblinYarnConfigurationKeys.CONTAINER_JARS_KEY),
        Optional.<Map<String, LocalResource>>absent(), appJarsDestDir, localFs);
  }
  if (this.config.hasPath(GobblinYarnConfigurationKeys.CONTAINER_FILES_LOCAL_KEY)) {
    Path appFilesDestDir = new Path(containerWorkDir, GobblinYarnConfigurationKeys.APP_FILES_DIR_NAME);
    addAppLocalFiles(this.config.getString(GobblinYarnConfigurationKeys.CONTAINER_FILES_LOCAL_KEY),
        Optional.<Map<String, LocalResource>>absent(), appFilesDestDir, localFs);
  }
}
 
Example 9
Source Project: flink   Source File: FlinkYarnSessionCliTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that the configuration settings are used to create the
 * {@link ClusterSpecification}.
 */
@Test
public void testConfigurationClusterSpecification() throws Exception {
	final Configuration configuration = new Configuration();
	final int jobManagerMemory = 1337;
	configuration.set(JobManagerOptions.TOTAL_PROCESS_MEMORY, MemorySize.ofMebiBytes(jobManagerMemory));
	final int taskManagerMemory = 7331;
	configuration.set(TaskManagerOptions.TOTAL_PROCESS_MEMORY, MemorySize.ofMebiBytes(taskManagerMemory));
	final int slotsPerTaskManager = 42;
	configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, slotsPerTaskManager);

	final String[] args = {};
	final FlinkYarnSessionCli flinkYarnSessionCli = createFlinkYarnSessionCli(configuration);

	CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);

	Configuration executorConfig = flinkYarnSessionCli.applyCommandLineOptionsToConfiguration(commandLine);
	ClusterClientFactory<ApplicationId> clientFactory = getClusterClientFactory(executorConfig);
	ClusterSpecification clusterSpecification = clientFactory.getClusterSpecification(executorConfig);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(jobManagerMemory));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(taskManagerMemory));
	assertThat(clusterSpecification.getSlotsPerTaskManager(), is(slotsPerTaskManager));
}
 
Example 10
Source Project: incubator-tez   Source File: TestTaskExecution.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testGetTaskShouldDie() throws InterruptedException, ExecutionException {
  ListeningExecutorService executor = null;
  try {
    ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
    executor = MoreExecutors.listeningDecorator(rawExecutor);
    ApplicationId appId = ApplicationId.newInstance(10000, 1);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
    ContainerId containerId = ContainerId.newInstance(appAttemptId, 1);

    TezTaskUmbilicalForTest umbilical = new TezTaskUmbilicalForTest();
    ContainerContext containerContext = new ContainerContext(containerId.toString());

    ContainerReporter containerReporter = new ContainerReporter(umbilical, containerContext, 100);
    ListenableFuture<ContainerTask> getTaskFuture = executor.submit(containerReporter);

    getTaskFuture.get();
    assertEquals(1, umbilical.getTaskInvocations);

  } finally {
    executor.shutdownNow();
  }
}
 
Example 11
protected ApplicationInfo waitState(ApplicationId applicationId, long timeout, TimeUnit unit, YarnApplicationState... applicationStates) throws Exception {
	YarnApplicationState state = null;
	ApplicationReport report = null;
	long end = System.currentTimeMillis() + unit.toMillis(timeout);

	// break label for inner loop
	done:
	do {
		report = findApplicationReport(getYarnClient(), applicationId);
		if (report == null) {
			break;
		}
		state = report.getYarnApplicationState();
		for (YarnApplicationState stateCheck : applicationStates) {
			if (state.equals(stateCheck)) {
				break done;
			}
		}
		Thread.sleep(1000);
	} while (System.currentTimeMillis() < end);
	return new ApplicationInfo(applicationId, report);
}
 
Example 12
Source Project: tez   Source File: OutputCommitterContextImpl.java    License: Apache License 2.0 6 votes vote down vote up
public OutputCommitterContextImpl(ApplicationId applicationId,
    int dagAttemptNumber,
    String dagName,
    String vertexName,
    RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor> output,
    int vertexIdx) {
  Objects.requireNonNull(applicationId, "applicationId is null");
  Objects.requireNonNull(dagName, "dagName is null");
  Objects.requireNonNull(vertexName, "vertexName is null");
  Objects.requireNonNull(output, "output is null");
  this.applicationId = applicationId;
  this.dagAttemptNumber = dagAttemptNumber;
  this.dagName = dagName;
  this.vertexName = vertexName;
  this.output = output;
  this.vertexIdx = vertexIdx;
}
 
Example 13
Source Project: hadoop   Source File: TestLocalizedResource.java    License: Apache License 2.0 5 votes vote down vote up
static ContainerId getMockContainer(long id) {
  ApplicationId appId = mock(ApplicationId.class);
  when(appId.getClusterTimestamp()).thenReturn(314159265L);
  when(appId.getId()).thenReturn(3);
  ApplicationAttemptId appAttemptId = mock(ApplicationAttemptId.class);
  when(appAttemptId.getApplicationId()).thenReturn(appId);
  when(appAttemptId.getAttemptId()).thenReturn(0);
  ContainerId container = mock(ContainerId.class);
  when(container.getContainerId()).thenReturn(id);
  when(container.getApplicationAttemptId()).thenReturn(appAttemptId);
  return container;
}
 
Example 14
Source Project: big-c   Source File: KillApplicationRequestPBImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public ApplicationId getApplicationId() {
  KillApplicationRequestProtoOrBuilder p = viaProto ? proto : builder;
  if (this.applicationId != null) {
    return this.applicationId;
  }
  if (!p.hasApplicationId()) {
    return null;
  }
  this.applicationId = convertFromProtoFormat(p.getApplicationId());
  return this.applicationId;
}
 
Example 15
Source Project: hadoop   Source File: NodeStatusUpdaterImpl.java    License: Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
protected List<ContainerStatus> getContainerStatuses() throws IOException {
  List<ContainerStatus> containerStatuses = new ArrayList<ContainerStatus>();
  for (Container container : this.context.getContainers().values()) {
    ContainerId containerId = container.getContainerId();
    ApplicationId applicationId = containerId.getApplicationAttemptId()
        .getApplicationId();
    org.apache.hadoop.yarn.api.records.ContainerStatus containerStatus =
        container.cloneAndGetContainerStatus();
    if (containerStatus.getState() == ContainerState.COMPLETE) {
      if (isApplicationStopped(applicationId)) {
        if (LOG.isDebugEnabled()) {
          LOG.debug(applicationId + " is completing, " + " remove "
              + containerId + " from NM context.");
        }
        context.getContainers().remove(containerId);
        pendingCompletedContainers.put(containerId, containerStatus);
      } else {
        if (!isContainerRecentlyStopped(containerId)) {
          pendingCompletedContainers.put(containerId, containerStatus);
          // Adding to finished containers cache. Cache will keep it around at
          // least for #durationToTrackStoppedContainers duration. In the
          // subsequent call to stop container it will get removed from cache.
          addCompletedContainer(containerId);
        }
      }
    } else {
      containerStatuses.add(containerStatus);
    }
  }
  containerStatuses.addAll(pendingCompletedContainers.values());
  if (LOG.isDebugEnabled()) {
    LOG.debug("Sending out " + containerStatuses.size()
        + " container statuses: " + containerStatuses);
  }
  return containerStatuses;
}
 
Example 16
Source Project: big-c   Source File: TestApplicationACLs.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("deprecation")
private ApplicationId submitAppAndGetAppId(AccessControlList viewACL,
    AccessControlList modifyACL) throws Exception {
  SubmitApplicationRequest submitRequest = recordFactory
      .newRecordInstance(SubmitApplicationRequest.class);
  ApplicationSubmissionContext context = recordFactory
      .newRecordInstance(ApplicationSubmissionContext.class);

  ApplicationId applicationId = rmClient.getNewApplication(
      recordFactory.newRecordInstance(GetNewApplicationRequest.class))
      .getApplicationId();
  context.setApplicationId(applicationId);

  Map<ApplicationAccessType, String> acls
      = new HashMap<ApplicationAccessType, String>();
  acls.put(ApplicationAccessType.VIEW_APP, viewACL.getAclString());
  acls.put(ApplicationAccessType.MODIFY_APP, modifyACL.getAclString());

  ContainerLaunchContext amContainer = recordFactory
      .newRecordInstance(ContainerLaunchContext.class);
  Resource resource = BuilderUtils.newResource(1024, 1);
  context.setResource(resource);
  amContainer.setApplicationACLs(acls);
  context.setAMContainerSpec(amContainer);
  submitRequest.setApplicationSubmissionContext(context);
  rmClient.submitApplication(submitRequest);
  resourceManager.waitForState(applicationId, RMAppState.ACCEPTED);
  return applicationId;
}
 
Example 17
Source Project: hadoop   Source File: BuilderUtils.java    License: Apache License 2.0 5 votes vote down vote up
public static ContainerId newContainerId(int appId, int appAttemptId,
    long timestamp, long id) {
  ApplicationId applicationId = newApplicationId(timestamp, appId);
  ApplicationAttemptId applicationAttemptId = newApplicationAttemptId(
      applicationId, appAttemptId);
  ContainerId cId = newContainerId(applicationAttemptId, id);
  return cId;
}
 
Example 18
Source Project: hadoop   Source File: TestClientRMService.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testGetApplicationReport() throws Exception {
  YarnScheduler yarnScheduler = mock(YarnScheduler.class);
  RMContext rmContext = mock(RMContext.class);
  mockRMContext(yarnScheduler, rmContext);

  ApplicationId appId1 = getApplicationId(1);

  ApplicationACLsManager mockAclsManager = mock(ApplicationACLsManager.class);
  when(
      mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(),
          ApplicationAccessType.VIEW_APP, null, appId1)).thenReturn(true);

  ClientRMService rmService = new ClientRMService(rmContext, yarnScheduler,
      null, mockAclsManager, null, null);
  try {
    RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
    GetApplicationReportRequest request = recordFactory
        .newRecordInstance(GetApplicationReportRequest.class);
    request.setApplicationId(appId1);
    GetApplicationReportResponse response = 
        rmService.getApplicationReport(request);
    ApplicationReport report = response.getApplicationReport();
    ApplicationResourceUsageReport usageReport = 
        report.getApplicationResourceUsageReport();
    Assert.assertEquals(10, usageReport.getMemorySeconds());
    Assert.assertEquals(3, usageReport.getVcoreSeconds());
    Assert.assertEquals(3, usageReport.getGcoreSeconds());
  } finally {
    rmService.close();
  }
}
 
Example 19
Source Project: hadoop   Source File: LogHandlerAppStartedEvent.java    License: Apache License 2.0 5 votes vote down vote up
public LogHandlerAppStartedEvent(ApplicationId appId, String user,
    Credentials credentials, ContainerLogsRetentionPolicy retentionPolicy,
    Map<ApplicationAccessType, String> appAcls,
    LogAggregationContext logAggregationContext) {
  super(LogHandlerEventType.APPLICATION_STARTED);
  this.applicationId = appId;
  this.user = user;
  this.credentials = credentials;
  this.retentionPolicy = retentionPolicy;
  this.appAcls = appAcls;
  this.logAggregationContext = logAggregationContext;
}
 
Example 20
Source Project: hadoop   Source File: TestYarnClient.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout = 10000)
public void testGetContainers() throws YarnException, IOException {
  Configuration conf = new Configuration();
  conf.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED,
      true);
  
  final YarnClient client = new MockYarnClient();
  client.init(conf);
  client.start();

  ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
  ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
      applicationId, 1);
  List<ContainerReport> reports = client.getContainers(appAttemptId);
  Assert.assertNotNull(reports);
  Assert.assertEquals(reports.get(0).getContainerId(),
      (ContainerId.newContainerId(appAttemptId, 1)));
  Assert.assertEquals(reports.get(1).getContainerId(),
      (ContainerId.newContainerId(appAttemptId, 2)));
  Assert.assertEquals(reports.get(2).getContainerId(),
      (ContainerId.newContainerId(appAttemptId, 3)));
  
  //First2 containers should come from RM with updated state information and 
  // 3rd container is not there in RM and should
  Assert.assertEquals(ContainerState.RUNNING,
      (reports.get(0).getContainerState()));
  Assert.assertEquals(ContainerState.RUNNING,
      (reports.get(1).getContainerState()));
  Assert.assertEquals(ContainerState.COMPLETE,
      (reports.get(2).getContainerState()));
  client.stop();
}
 
Example 21
Source Project: big-c   Source File: TestAHSWebServices.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSingleContainer() throws Exception {
  ApplicationId appId = ApplicationId.newInstance(0, 1);
  ApplicationAttemptId appAttemptId =
      ApplicationAttemptId.newInstance(appId, 1);
  ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1);
  WebResource r = resource();
  ClientResponse response =
      r.path("ws").path("v1").path("applicationhistory").path("apps")
        .path(appId.toString()).path("appattempts")
        .path(appAttemptId.toString()).path("containers")
        .path(containerId.toString())
        .queryParam("user.name", USERS[round])
        .accept(MediaType.APPLICATION_JSON)
        .get(ClientResponse.class);
  if (round == 1) {
    assertEquals(
        Status.FORBIDDEN, response.getClientResponseStatus());
    return;
  }
  assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
  JSONObject json = response.getEntity(JSONObject.class);
  assertEquals("incorrect number of elements", 1, json.length());
  JSONObject container = json.getJSONObject("container");
  assertEquals(containerId.toString(), container.getString("containerId"));
  assertEquals("test diagnostics info", container.getString("diagnosticsInfo"));
  assertEquals("-1", container.getString("allocatedMB"));
  assertEquals("-1", container.getString("allocatedVCores"));
  assertEquals(NodeId.newInstance("test host", 100).toString(),
    container.getString("assignedNodeId"));
  assertEquals("-1", container.getString("priority"));
  Configuration conf = new YarnConfiguration();
  assertEquals(WebAppUtils.getHttpSchemePrefix(conf) +
      WebAppUtils.getAHSWebAppURLWithoutScheme(conf) +
      "/applicationhistory/logs/test host:100/container_0_0001_01_000001/" +
      "container_0_0001_01_000001/user1", container.getString("logUrl"));
  assertEquals(ContainerState.COMPLETE.toString(),
    container.getString("containerState"));
}
 
Example 22
@Override
protected ApplicationSubmissionContext getSubmissionContext(ApplicationId applicationId) {
	ApplicationSubmissionContext submissionContext = super.getSubmissionContext(applicationId);
	// force app fail after first appmaster failure regardles
	// what's set on yarn level.
	// TODO: should make configurable in spring yarn if possible
	submissionContext.setMaxAppAttempts(1);
	return submissionContext;
}
 
Example 23
Source Project: big-c   Source File: ApplicationStartDataPBImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public ApplicationId getApplicationId() {
  if (this.applicationId != null) {
    return this.applicationId;
  }
  ApplicationStartDataProtoOrBuilder p = viaProto ? proto : builder;
  if (!p.hasApplicationId()) {
    return null;
  }
  this.applicationId = convertFromProtoFormat(p.getApplicationId());
  return this.applicationId;
}
 
Example 24
Source Project: big-c   Source File: WebServices.java    License: Apache License 2.0 5 votes vote down vote up
protected static ApplicationId parseApplicationId(String appId) {
  if (appId == null || appId.isEmpty()) {
    throw new NotFoundException("appId, " + appId + ", is empty or null");
  }
  ApplicationId aid = ConverterUtils.toApplicationId(appId);
  if (aid == null) {
    throw new NotFoundException("appId is null");
  }
  return aid;
}
 
Example 25
Source Project: big-c   Source File: TestYARNRunner.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() throws Exception {
  resourceMgrDelegate = mock(ResourceMgrDelegate.class);
  conf = new YarnConfiguration();
  conf.set(YarnConfiguration.RM_PRINCIPAL, "mapred/[email protected]");
  clientCache = new ClientCache(conf, resourceMgrDelegate);
  clientCache = spy(clientCache);
  yarnRunner = new YARNRunner(conf, resourceMgrDelegate, clientCache);
  yarnRunner = spy(yarnRunner);
  submissionContext = mock(ApplicationSubmissionContext.class);
  doAnswer(
      new Answer<ApplicationSubmissionContext>() {
        @Override
        public ApplicationSubmissionContext answer(InvocationOnMock invocation)
            throws Throwable {
          return submissionContext;
        }
      }
      ).when(yarnRunner).createApplicationSubmissionContext(any(Configuration.class),
          any(String.class), any(Credentials.class));

  appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
  jobId = TypeConverter.fromYarn(appId);
  if (testWorkDir.exists()) {
    FileContext.getLocalFSFileContext().delete(new Path(testWorkDir.toString()), true);
  }
  testWorkDir.mkdirs();
}
 
Example 26
Source Project: big-c   Source File: GetNewApplicationResponsePBImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void setApplicationId(ApplicationId applicationId) {
  maybeInitBuilder();
  if (applicationId == null) 
    builder.clearApplicationId();
  this.applicationId = applicationId;
}
 
Example 27
Source Project: big-c   Source File: NodeStatusPBImpl.java    License: Apache License 2.0 5 votes vote down vote up
private synchronized void initKeepAliveApplications() {
  if (this.keepAliveApplications != null) {
    return;
  }
  NodeStatusProtoOrBuilder p = viaProto ? proto : builder;
  List<ApplicationIdProto> list = p.getKeepAliveApplicationsList();
  this.keepAliveApplications = new ArrayList<ApplicationId>();

  for (ApplicationIdProto c : list) {
    this.keepAliveApplications.add(convertFromProtoFormat(c));
  }
  
}
 
Example 28
Source Project: big-c   Source File: TestDelegationTokenRenewer.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout=20000)
public void testAppSubmissionWithInvalidDelegationToken() throws Exception {
  Configuration conf = new Configuration();
  conf.set(
      CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
      "kerberos");
  UserGroupInformation.setConfiguration(conf);
  MockRM rm = new MockRM(conf) {
    @Override
    protected void doSecureLogin() throws IOException {
      // Skip the login.
    }
  };
  ByteBuffer tokens = ByteBuffer.wrap("BOGUS".getBytes()); 
  ContainerLaunchContext amContainer =
      ContainerLaunchContext.newInstance(
          new HashMap<String, LocalResource>(), new HashMap<String, String>(),
          new ArrayList<String>(), new HashMap<String, ByteBuffer>(), tokens,
          new HashMap<ApplicationAccessType, String>());
  ApplicationSubmissionContext appSubContext =
      ApplicationSubmissionContext.newInstance(
          ApplicationId.newInstance(1234121, 0),
          "BOGUS", "default", Priority.UNDEFINED, amContainer, false,
          true, 1, Resource.newInstance(1024, 1), "BOGUS");
  SubmitApplicationRequest request =
      SubmitApplicationRequest.newInstance(appSubContext);
  try {
    rm.getClientRMService().submitApplication(request);
    fail("Error was excepted.");
  } catch (YarnException e) {
    Assert.assertTrue(e.getMessage().contains(
        "Bad header found in token storage"));
  }
}
 
Example 29
Source Project: big-c   Source File: TestProxyUriUtils.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testGetProxyUriNull() throws Exception {
  URI originalUri = null;
  URI proxyUri = new URI("http://proxy.net:8080/");
  ApplicationId id = BuilderUtils.newApplicationId(6384623l, 5);
  URI expected = new URI("http://proxy.net:8080/proxy/application_6384623_0005/");
  URI result = ProxyUriUtils.getProxyUri(originalUri, proxyUri, id);
  assertEquals(expected, result);
}
 
Example 30
Source Project: big-c   Source File: DelegationTokenRenewer.java    License: Apache License 2.0 5 votes vote down vote up
public DelegationTokenRenewerAppSubmitEvent(ApplicationId appId,
    Credentials credentails, boolean shouldCancelAtEnd, String user) {
  super(appId, DelegationTokenRenewerEventType.VERIFY_AND_START_APPLICATION);
  this.credentials = credentails;
  this.shouldCancelAtEnd = shouldCancelAtEnd;
  this.user = user;
}