org.apache.hadoop.yarn.api.records.ApplicationId Java Examples

The following examples show how to use org.apache.hadoop.yarn.api.records.ApplicationId. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestTaskExecution.java    From incubator-tez with Apache License 2.0 6 votes vote down vote up
@Test
public void testGetTaskShouldDie() throws InterruptedException, ExecutionException {
  ListeningExecutorService executor = null;
  try {
    ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
    executor = MoreExecutors.listeningDecorator(rawExecutor);
    ApplicationId appId = ApplicationId.newInstance(10000, 1);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
    ContainerId containerId = ContainerId.newInstance(appAttemptId, 1);

    TezTaskUmbilicalForTest umbilical = new TezTaskUmbilicalForTest();
    ContainerContext containerContext = new ContainerContext(containerId.toString());

    ContainerReporter containerReporter = new ContainerReporter(umbilical, containerContext, 100);
    ListenableFuture<ContainerTask> getTaskFuture = executor.submit(containerReporter);

    getTaskFuture.get();
    assertEquals(1, umbilical.getTaskInvocations);

  } finally {
    executor.shutdownNow();
  }
}
 
Example #2
Source File: ActiveUsersManager.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * An application has new outstanding requests.
 * 
 * @param user application user 
 * @param applicationId activated application
 */
@Lock({Queue.class, SchedulerApplicationAttempt.class})
synchronized public void activateApplication(
    String user, ApplicationId applicationId) {
  Set<ApplicationId> userApps = usersApplications.get(user);
  if (userApps == null) {
    userApps = new HashSet<ApplicationId>();
    usersApplications.put(user, userApps);
    ++activeUsers;
    metrics.incrActiveUsers();
    LOG.debug("User " + user + " added to activeUsers, currently: " + 
        activeUsers);
  }
  if (userApps.add(applicationId)) {
    metrics.activateApp(user);
  }
}
 
Example #3
Source File: RMNodeImpl.java    From big-c with Apache License 2.0 6 votes vote down vote up
private static void handleRunningAppOnNode(RMNodeImpl rmNode,
    RMContext context, ApplicationId appId, NodeId nodeId) {
  RMApp app = context.getRMApps().get(appId);

  // if we failed getting app by appId, maybe something wrong happened, just
  // add the app to the finishedApplications list so that the app can be
  // cleaned up on the NM
  if (null == app) {
    LOG.warn("Cannot get RMApp by appId=" + appId
        + ", just added it to finishedApplications list for cleanup");
    rmNode.finishedApplications.add(appId);
    return;
  }

  context.getDispatcher().getEventHandler()
      .handle(new RMAppRunningOnNodeEvent(appId, nodeId));
}
 
Example #4
Source File: TestHistoryEventsProtoConversion.java    From tez with Apache License 2.0 6 votes vote down vote up
private void testDAGSubmittedEvent() throws Exception {
  DAGSubmittedEvent event = new DAGSubmittedEvent(TezDAGID.getInstance(
      ApplicationId.newInstance(0, 1), 1), 1001l,
      DAGPlan.newBuilder().setName("foo").build(),
      ApplicationAttemptId.newInstance(
          ApplicationId.newInstance(0, 1), 1), null, "", null, null, QUEUE_NAME);
  DAGSubmittedEvent deserializedEvent = (DAGSubmittedEvent)
      testProtoConversion(event);
  Assert.assertEquals(event.getApplicationAttemptId(),
      deserializedEvent.getApplicationAttemptId());
  Assert.assertEquals(event.getDagID(),
      deserializedEvent.getDagID());
  Assert.assertEquals(event.getDAGName(),
      deserializedEvent.getDAGName());
  Assert.assertEquals(event.getSubmitTime(),
      deserializedEvent.getSubmitTime());
  Assert.assertEquals(event.getDAGPlan(),
      deserializedEvent.getDAGPlan());
  Assert.assertEquals(event.getQueueName(), deserializedEvent.getQueueName());
  logEvents(event, deserializedEvent);
}
 
Example #5
Source File: TestRMContainerAllocator.java    From big-c with Apache License 2.0 6 votes vote down vote up
private static AppContext createAppContext(
    ApplicationAttemptId appAttemptId, Job job) {
  AppContext context = mock(AppContext.class);
  ApplicationId appId = appAttemptId.getApplicationId();
  when(context.getApplicationID()).thenReturn(appId);
  when(context.getApplicationAttemptId()).thenReturn(appAttemptId);
  when(context.getJob(isA(JobId.class))).thenReturn(job);
  when(context.getClusterInfo()).thenReturn(
    new ClusterInfo(Resource.newInstance(10240, 1)));
  when(context.getEventHandler()).thenReturn(new EventHandler() {
    @Override
    public void handle(Event event) {
      // Only capture interesting events.
      if (event instanceof TaskAttemptContainerAssignedEvent) {
        events.add((TaskAttemptContainerAssignedEvent) event);
      } else if (event instanceof TaskAttemptKillEvent) {
        taskAttemptKillEvents.add((TaskAttemptKillEvent)event);
      } else if (event instanceof JobUpdatedNodesEvent) {
        jobUpdatedNodeEvents.add((JobUpdatedNodesEvent)event);
      } else if (event instanceof JobEvent) {
        jobEvents.add((JobEvent)event);
      }
    }
  });
  return context;
}
 
Example #6
Source File: TaskLauncherIT.java    From spring-cloud-deployer-yarn with Apache License 2.0 6 votes vote down vote up
private ApplicationId assertWaitApp(long timeout, TimeUnit unit, YarnCloudAppService yarnCloudAppService) throws Exception {
	ApplicationId applicationId = null;
	Collection<CloudAppInstanceInfo> instances;
	long end = System.currentTimeMillis() + unit.toMillis(timeout);

	do {
		instances = yarnCloudAppService.getInstances(CloudAppType.TASK);
		if (instances.size() == 1) {
			CloudAppInstanceInfo cloudAppInstanceInfo = instances.iterator().next();
			if (StringUtils.hasText(cloudAppInstanceInfo.getAddress())) {
				applicationId = ConverterUtils.toApplicationId(cloudAppInstanceInfo.getApplicationId());
				break;
			}
		}
		Thread.sleep(1000);
	} while (System.currentTimeMillis() < end);

	assertThat(applicationId, notNullValue());
	return applicationId;
}
 
Example #7
Source File: OutputCommitterContextImpl.java    From tez with Apache License 2.0 6 votes vote down vote up
public OutputCommitterContextImpl(ApplicationId applicationId,
    int dagAttemptNumber,
    String dagName,
    String vertexName,
    RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor> output,
    int vertexIdx) {
  Objects.requireNonNull(applicationId, "applicationId is null");
  Objects.requireNonNull(dagName, "dagName is null");
  Objects.requireNonNull(vertexName, "vertexName is null");
  Objects.requireNonNull(output, "output is null");
  this.applicationId = applicationId;
  this.dagAttemptNumber = dagAttemptNumber;
  this.dagName = dagName;
  this.vertexName = vertexName;
  this.output = output;
  this.vertexIdx = vertexIdx;
}
 
Example #8
Source File: AbstractCliBootYarnClusterTests.java    From spring-cloud-deployer-yarn with Apache License 2.0 6 votes vote down vote up
protected ApplicationInfo waitState(ApplicationId applicationId, long timeout, TimeUnit unit, YarnApplicationState... applicationStates) throws Exception {
	YarnApplicationState state = null;
	ApplicationReport report = null;
	long end = System.currentTimeMillis() + unit.toMillis(timeout);

	// break label for inner loop
	done:
	do {
		report = findApplicationReport(getYarnClient(), applicationId);
		if (report == null) {
			break;
		}
		state = report.getYarnApplicationState();
		for (YarnApplicationState stateCheck : applicationStates) {
			if (state.equals(stateCheck)) {
				break done;
			}
		}
		Thread.sleep(1000);
	} while (System.currentTimeMillis() < end);
	return new ApplicationInfo(applicationId, report);
}
 
Example #9
Source File: FlinkYarnSessionCliTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that the configuration settings are used to create the
 * {@link ClusterSpecification}.
 */
@Test
public void testConfigurationClusterSpecification() throws Exception {
	final Configuration configuration = new Configuration();
	final int jobManagerMemory = 1337;
	configuration.set(JobManagerOptions.TOTAL_PROCESS_MEMORY, MemorySize.ofMebiBytes(jobManagerMemory));
	final int taskManagerMemory = 7331;
	configuration.set(TaskManagerOptions.TOTAL_PROCESS_MEMORY, MemorySize.ofMebiBytes(taskManagerMemory));
	final int slotsPerTaskManager = 42;
	configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, slotsPerTaskManager);

	final String[] args = {};
	final FlinkYarnSessionCli flinkYarnSessionCli = createFlinkYarnSessionCli(configuration);

	CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);

	Configuration executorConfig = flinkYarnSessionCli.applyCommandLineOptionsToConfiguration(commandLine);
	ClusterClientFactory<ApplicationId> clientFactory = getClusterClientFactory(executorConfig);
	ClusterSpecification clusterSpecification = clientFactory.getClusterSpecification(executorConfig);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(jobManagerMemory));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(taskManagerMemory));
	assertThat(clusterSpecification.getSlotsPerTaskManager(), is(slotsPerTaskManager));
}
 
Example #10
Source File: TestTaskExecution.java    From incubator-tez with Apache License 2.0 6 votes vote down vote up
@Test
public void testSingleSuccessfulTask() throws IOException, InterruptedException, TezException,
    ExecutionException {
  ListeningExecutorService executor = null;
  try {
    ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
    executor = MoreExecutors.listeningDecorator(rawExecutor);
    ApplicationId appId = ApplicationId.newInstance(10000, 1);
    TezTaskUmbilicalForTest umbilical = new TezTaskUmbilicalForTest();
    TaskReporter taskReporter = createTaskReporter(appId, umbilical);

    TezTaskRunner taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor,
        TestProcessor.CONF_EMPTY);
    // Setup the executor
    Future<Boolean> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable(taskRunner));
    // Signal the processor to go through
    TestProcessor.signal();
    boolean result = taskRunnerFuture.get();
    assertTrue(result);
    assertNull(taskReporter.currentCallable);
    umbilical.verifyTaskSuccessEvent();
  } finally {
    executor.shutdownNow();
  }
}
 
Example #11
Source File: GobblinYarnAppLauncher.java    From incubator-gobblin with Apache License 2.0 6 votes vote down vote up
private void addContainerLocalResources(ApplicationId applicationId) throws IOException {
  Path appWorkDir = GobblinClusterUtils.getAppWorkDirPathFromConfig(this.config, this.fs, this.applicationName, applicationId.toString());

  Path containerWorkDir = new Path(appWorkDir, GobblinYarnConfigurationKeys.CONTAINER_WORK_DIR_NAME);
  LOGGER.info("Configured Container work directory to: {}", containerWorkDir.toString());

  FileSystem localFs = FileSystem.getLocal(new Configuration());

  if (this.config.hasPath(GobblinYarnConfigurationKeys.CONTAINER_JARS_KEY)) {
    Path appJarsDestDir = new Path(containerWorkDir, GobblinYarnConfigurationKeys.APP_JARS_DIR_NAME);
    addAppJars(this.config.getString(GobblinYarnConfigurationKeys.CONTAINER_JARS_KEY),
        Optional.<Map<String, LocalResource>>absent(), appJarsDestDir, localFs);
  }
  if (this.config.hasPath(GobblinYarnConfigurationKeys.CONTAINER_FILES_LOCAL_KEY)) {
    Path appFilesDestDir = new Path(containerWorkDir, GobblinYarnConfigurationKeys.APP_FILES_DIR_NAME);
    addAppLocalFiles(this.config.getString(GobblinYarnConfigurationKeys.CONTAINER_FILES_LOCAL_KEY),
        Optional.<Map<String, LocalResource>>absent(), appFilesDestDir, localFs);
  }
}
 
Example #12
Source File: ZKRMStateStore.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
public synchronized void updateApplicationStateInternal(ApplicationId appId,
    ApplicationStateData appStateDataPB) throws Exception {
  String nodeUpdatePath = getNodePath(rmAppRoot, appId.toString());

  if (LOG.isDebugEnabled()) {
    LOG.debug("Storing final state info for app: " + appId + " at: "
        + nodeUpdatePath);
  }
  byte[] appStateData = appStateDataPB.getProto().toByteArray();

  if (existsWithRetries(nodeUpdatePath, false) != null) {
    setDataWithRetries(nodeUpdatePath, appStateData, -1);
  } else {
    createWithRetries(nodeUpdatePath, appStateData, zkAcl,
      CreateMode.PERSISTENT);
    LOG.debug(appId + " znode didn't exist. Created a new znode to"
        + " update the application state.");
  }
}
 
Example #13
Source File: TestRecoveryParser.java    From tez with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 5000)
public void testGetLastCompletedDAG() {
  Map<TezDAGID, DAGSummaryData> summaryDataMap =
      new HashMap<TezDAGID, DAGSummaryData>();
  int lastCompletedDAGId = new Random().nextInt(20) + 1;
  for (int i = 1; i <= lastCompletedDAGId; ++i) {
    ApplicationId appId = ApplicationId.newInstance(1, 1);
    TezDAGID dagId = TezDAGID.getInstance(appId, i);
    summaryDataMap.put(dagId, createDAGSummaryData(dagId, true));
  }

  DAGSummaryData lastCompletedDAG =
      parser.getLastCompletedOrInProgressDAG(summaryDataMap);
  assertEquals(lastCompletedDAGId, lastCompletedDAG.dagId.getId());
}
 
Example #14
Source File: TestRuntimeEstimators.java    From big-c with Apache License 2.0 5 votes vote down vote up
MyAppContext(int numberMaps, int numberReduces) {
  myApplicationID = ApplicationId.newInstance(clock.getTime(), 1);

  myAppAttemptID = ApplicationAttemptId.newInstance(myApplicationID, 0);
  myJobID = recordFactory.newRecordInstance(JobId.class);
  myJobID.setAppId(myApplicationID);

  Job myJob
      = new MyJobImpl(myJobID, numberMaps, numberReduces);

  allJobs = Collections.singletonMap(myJobID, myJob);
}
 
Example #15
Source File: TestYarnClient.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 10000)
public void testGetContainers() throws YarnException, IOException {
  Configuration conf = new Configuration();
  conf.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED,
      true);
  
  final YarnClient client = new MockYarnClient();
  client.init(conf);
  client.start();

  ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
  ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
      applicationId, 1);
  List<ContainerReport> reports = client.getContainers(appAttemptId);
  Assert.assertNotNull(reports);
  Assert.assertEquals(reports.get(0).getContainerId(),
      (ContainerId.newContainerId(appAttemptId, 1)));
  Assert.assertEquals(reports.get(1).getContainerId(),
      (ContainerId.newContainerId(appAttemptId, 2)));
  Assert.assertEquals(reports.get(2).getContainerId(),
      (ContainerId.newContainerId(appAttemptId, 3)));
  
  //First2 containers should come from RM with updated state information and 
  // 3rd container is not there in RM and should
  Assert.assertEquals(ContainerState.RUNNING,
      (reports.get(0).getContainerState()));
  Assert.assertEquals(ContainerState.RUNNING,
      (reports.get(1).getContainerState()));
  Assert.assertEquals(ContainerState.COMPLETE,
      (reports.get(2).getContainerState()));
  client.stop();
}
 
Example #16
Source File: NodeStatusUpdaterImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
protected List<ContainerStatus> getContainerStatuses() throws IOException {
  List<ContainerStatus> containerStatuses = new ArrayList<ContainerStatus>();
  for (Container container : this.context.getContainers().values()) {
    ContainerId containerId = container.getContainerId();
    ApplicationId applicationId = containerId.getApplicationAttemptId()
        .getApplicationId();
    org.apache.hadoop.yarn.api.records.ContainerStatus containerStatus =
        container.cloneAndGetContainerStatus();
    if (containerStatus.getState() == ContainerState.COMPLETE) {
      if (isApplicationStopped(applicationId)) {
        if (LOG.isDebugEnabled()) {
          LOG.debug(applicationId + " is completing, " + " remove "
              + containerId + " from NM context.");
        }
        context.getContainers().remove(containerId);
        pendingCompletedContainers.put(containerId, containerStatus);
      } else {
        if (!isContainerRecentlyStopped(containerId)) {
          pendingCompletedContainers.put(containerId, containerStatus);
          // Adding to finished containers cache. Cache will keep it around at
          // least for #durationToTrackStoppedContainers duration. In the
          // subsequent call to stop container it will get removed from cache.
          addCompletedContainer(containerId);
        }
      }
    } else {
      containerStatuses.add(containerStatus);
    }
  }
  containerStatuses.addAll(pendingCompletedContainers.values());
  if (LOG.isDebugEnabled()) {
    LOG.debug("Sending out " + containerStatuses.size()
        + " container statuses: " + containerStatuses);
  }
  return containerStatuses;
}
 
Example #17
Source File: TestAHSWebServices.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testSingleContainer() throws Exception {
  ApplicationId appId = ApplicationId.newInstance(0, 1);
  ApplicationAttemptId appAttemptId =
      ApplicationAttemptId.newInstance(appId, 1);
  ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1);
  WebResource r = resource();
  ClientResponse response =
      r.path("ws").path("v1").path("applicationhistory").path("apps")
        .path(appId.toString()).path("appattempts")
        .path(appAttemptId.toString()).path("containers")
        .path(containerId.toString())
        .queryParam("user.name", USERS[round])
        .accept(MediaType.APPLICATION_JSON)
        .get(ClientResponse.class);
  if (round == 1) {
    assertEquals(
        Status.FORBIDDEN, response.getClientResponseStatus());
    return;
  }
  assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
  JSONObject json = response.getEntity(JSONObject.class);
  assertEquals("incorrect number of elements", 1, json.length());
  JSONObject container = json.getJSONObject("container");
  assertEquals(containerId.toString(), container.getString("containerId"));
  assertEquals("test diagnostics info", container.getString("diagnosticsInfo"));
  assertEquals("-1", container.getString("allocatedMB"));
  assertEquals("-1", container.getString("allocatedVCores"));
  assertEquals(NodeId.newInstance("test host", 100).toString(),
    container.getString("assignedNodeId"));
  assertEquals("-1", container.getString("priority"));
  Configuration conf = new YarnConfiguration();
  assertEquals(WebAppUtils.getHttpSchemePrefix(conf) +
      WebAppUtils.getAHSWebAppURLWithoutScheme(conf) +
      "/applicationhistory/logs/test host:100/container_0_0001_01_000001/" +
      "container_0_0001_01_000001/user1", container.getString("logUrl"));
  assertEquals(ContainerState.COMPLETE.toString(),
    container.getString("containerState"));
}
 
Example #18
Source File: KillApplicationRequestPBImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public ApplicationId getApplicationId() {
  KillApplicationRequestProtoOrBuilder p = viaProto ? proto : builder;
  if (this.applicationId != null) {
    return this.applicationId;
  }
  if (!p.hasApplicationId()) {
    return null;
  }
  this.applicationId = convertFromProtoFormat(p.getApplicationId());
  return this.applicationId;
}
 
Example #19
Source File: TestTaskExecution2.java    From tez with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 5000)
public void testTaskSelfKill() throws IOException, InterruptedException, TezException,
    ExecutionException {

  ListeningExecutorService executor = null;
  try {
    ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
    executor = MoreExecutors.listeningDecorator(rawExecutor);
    ApplicationId appId = ApplicationId.newInstance(10000, 1);
    TaskExecutionTestHelpers.TezTaskUmbilicalForTest
        umbilical = new TaskExecutionTestHelpers.TezTaskUmbilicalForTest();
    TaskReporter taskReporter = createTaskReporter(appId, umbilical);

    TezTaskRunner2 taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor,
        TestProcessor.CONF_SELF_KILL_AND_COMPLETE);
    // Setup the executor
    Future<TaskRunner2Result> taskRunnerFuture =
        taskExecutor.submit(new TaskRunnerCallable2ForTest(taskRunner));
    // Signal the processor to go through
    TestProcessor.awaitStart();
    TestProcessor.signal();

    TaskRunner2Result result = taskRunnerFuture.get();
    verifyTaskRunnerResult(result, EndReason.TASK_KILL_REQUEST, createProcessorIOException(), false,
        null);

    TestProcessor.awaitCompletion();
    assertNull(taskReporter.currentCallable);
    umbilical.verifyTaskKilledEvent(
        KILL_START_STRING,
        IOException.class.getName() + ": " + IOException.class.getSimpleName());
    assertTrue(TestProcessor.wasAborted());
  } finally {
    executor.shutdownNow();
  }
}
 
Example #20
Source File: NodeHeartbeatResponsePBImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private void addSystemCredentialsToProto() {
  maybeInitBuilder();
  builder.clearSystemCredentialsForApps();
  for (Map.Entry<ApplicationId, ByteBuffer> entry : systemCredentials.entrySet()) {
    builder.addSystemCredentialsForApps(SystemCredentialsForAppsProto.newBuilder()
      .setAppId(convertToProtoFormat(entry.getKey()))
      .setCredentialsForApp(ProtoUtils.convertToProtoFormat(
          entry.getValue().duplicate())));
  }
}
 
Example #21
Source File: WebServices.java    From big-c with Apache License 2.0 5 votes vote down vote up
protected static ApplicationId parseApplicationId(String appId) {
  if (appId == null || appId.isEmpty()) {
    throw new NotFoundException("appId, " + appId + ", is empty or null");
  }
  ApplicationId aid = ConverterUtils.toApplicationId(appId);
  if (aid == null) {
    throw new NotFoundException("appId is null");
  }
  return aid;
}
 
Example #22
Source File: TestYARNRunner.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() throws Exception {
  resourceMgrDelegate = mock(ResourceMgrDelegate.class);
  conf = new YarnConfiguration();
  conf.set(YarnConfiguration.RM_PRINCIPAL, "mapred/host@REALM");
  clientCache = new ClientCache(conf, resourceMgrDelegate);
  clientCache = spy(clientCache);
  yarnRunner = new YARNRunner(conf, resourceMgrDelegate, clientCache);
  yarnRunner = spy(yarnRunner);
  submissionContext = mock(ApplicationSubmissionContext.class);
  doAnswer(
      new Answer<ApplicationSubmissionContext>() {
        @Override
        public ApplicationSubmissionContext answer(InvocationOnMock invocation)
            throws Throwable {
          return submissionContext;
        }
      }
      ).when(yarnRunner).createApplicationSubmissionContext(any(Configuration.class),
          any(String.class), any(Credentials.class));

  appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
  jobId = TypeConverter.fromYarn(appId);
  if (testWorkDir.exists()) {
    FileContext.getLocalFSFileContext().delete(new Path(testWorkDir.toString()), true);
  }
  testWorkDir.mkdirs();
}
 
Example #23
Source File: GetNewApplicationResponsePBImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public void setApplicationId(ApplicationId applicationId) {
  maybeInitBuilder();
  if (applicationId == null) 
    builder.clearApplicationId();
  this.applicationId = applicationId;
}
 
Example #24
Source File: TestRMAppTransitions.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 30000)
public void testAppsRecoveringStates() throws Exception {
  RMState state = new RMState();
  Map<ApplicationId, ApplicationStateData> applicationState =
      state.getApplicationState();
  createRMStateForApplications(applicationState, RMAppState.FINISHED);
  createRMStateForApplications(applicationState, RMAppState.KILLED);
  createRMStateForApplications(applicationState, RMAppState.FAILED);
  for (ApplicationStateData appState : applicationState.values()) {
    testRecoverApplication(appState, state);
  }
}
 
Example #25
Source File: NodeStatusPBImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
private synchronized void initKeepAliveApplications() {
  if (this.keepAliveApplications != null) {
    return;
  }
  NodeStatusProtoOrBuilder p = viaProto ? proto : builder;
  List<ApplicationIdProto> list = p.getKeepAliveApplicationsList();
  this.keepAliveApplications = new ArrayList<ApplicationId>();

  for (ApplicationIdProto c : list) {
    this.keepAliveApplications.add(convertFromProtoFormat(c));
  }
  
}
 
Example #26
Source File: RMNodeStartedEvent.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public RMNodeStartedEvent(NodeId nodeId,
    List<NMContainerStatus> containerReports,
    List<ApplicationId> runningApplications) {
  super(nodeId, RMNodeEventType.STARTED);
  this.containerStatuses = containerReports;
  this.runningApplications = runningApplications;
}
 
Example #27
Source File: TestApplicationACLs.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private void verifyFriendAccess() throws Exception {

    AccessControlList viewACL = new AccessControlList("");
    viewACL.addGroup(FRIENDLY_GROUP);
    AccessControlList modifyACL = new AccessControlList("");
    modifyACL.addUser(FRIEND);
    ApplicationId applicationId = submitAppAndGetAppId(viewACL, modifyACL);

    final GetApplicationReportRequest appReportRequest = recordFactory
        .newRecordInstance(GetApplicationReportRequest.class);
    appReportRequest.setApplicationId(applicationId);
    final KillApplicationRequest finishAppRequest = recordFactory
        .newRecordInstance(KillApplicationRequest.class);
    finishAppRequest.setApplicationId(applicationId);

    ApplicationClientProtocol friendClient = getRMClientForUser(FRIEND);

    // View as the friend
    friendClient.getApplicationReport(appReportRequest);

    // List apps as friend
    Assert.assertEquals("App view by a friend should list the apps!!", 3,
        friendClient.getApplications(
            recordFactory.newRecordInstance(GetApplicationsRequest.class))
            .getApplicationList().size());

    // Kill app as the friend
    friendClient.forceKillApplication(finishAppRequest);
    resourceManager.waitForState(applicationId, RMAppState.KILLED);
  }
 
Example #28
Source File: TestDelegationTokenRenewer.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test(timeout=20000)
public void testAppSubmissionWithInvalidDelegationToken() throws Exception {
  Configuration conf = new Configuration();
  conf.set(
      CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
      "kerberos");
  UserGroupInformation.setConfiguration(conf);
  MockRM rm = new MockRM(conf) {
    @Override
    protected void doSecureLogin() throws IOException {
      // Skip the login.
    }
  };
  ByteBuffer tokens = ByteBuffer.wrap("BOGUS".getBytes()); 
  ContainerLaunchContext amContainer =
      ContainerLaunchContext.newInstance(
          new HashMap<String, LocalResource>(), new HashMap<String, String>(),
          new ArrayList<String>(), new HashMap<String, ByteBuffer>(), tokens,
          new HashMap<ApplicationAccessType, String>());
  ApplicationSubmissionContext appSubContext =
      ApplicationSubmissionContext.newInstance(
          ApplicationId.newInstance(1234121, 0),
          "BOGUS", "default", Priority.UNDEFINED, amContainer, false,
          true, 1, Resource.newInstance(1024, 1), "BOGUS");
  SubmitApplicationRequest request =
      SubmitApplicationRequest.newInstance(appSubContext);
  try {
    rm.getClientRMService().submitApplication(request);
    fail("Error was excepted.");
  } catch (YarnException e) {
    Assert.assertTrue(e.getMessage().contains(
        "Bad header found in token storage"));
  }
}
 
Example #29
Source File: TestProxyUriUtils.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testGetProxyUriNull() throws Exception {
  URI originalUri = null;
  URI proxyUri = new URI("http://proxy.net:8080/");
  ApplicationId id = BuilderUtils.newApplicationId(6384623l, 5);
  URI expected = new URI("http://proxy.net:8080/proxy/application_6384623_0005/");
  URI result = ProxyUriUtils.getProxyUri(originalUri, proxyUri, id);
  assertEquals(expected, result);
}
 
Example #30
Source File: DelegationTokenRenewer.java    From big-c with Apache License 2.0 5 votes vote down vote up
public DelegationTokenRenewerAppSubmitEvent(ApplicationId appId,
    Credentials credentails, boolean shouldCancelAtEnd, String user) {
  super(appId, DelegationTokenRenewerEventType.VERIFY_AND_START_APPLICATION);
  this.credentials = credentails;
  this.shouldCancelAtEnd = shouldCancelAtEnd;
  this.user = user;
}