Java Code Examples for org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider

The following examples show how to use org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: SharedCacheUploader.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * @param resource the local resource that contains the original remote path
 * @param localPath the path in the local filesystem where the resource is
 * localized
 * @param fs the filesystem of the shared cache
 * @param localFs the local filesystem
 */
public SharedCacheUploader(LocalResource resource, Path localPath,
    String user, Configuration conf, SCMUploaderProtocol scmClient,
    FileSystem fs, FileSystem localFs) {
  this.resource = resource;
  this.localPath = localPath;
  this.user = user;
  this.conf = conf;
  this.scmClient = scmClient;
  this.fs = fs;
  this.sharedCacheRootDir =
      conf.get(YarnConfiguration.SHARED_CACHE_ROOT,
          YarnConfiguration.DEFAULT_SHARED_CACHE_ROOT);
  this.nestedLevel = SharedCacheUtil.getCacheDepth(conf);
  this.checksum = SharedCacheChecksumFactory.getChecksum(conf);
  this.localFs = localFs;
  this.recordFactory = RecordFactoryProvider.getRecordFactory(null);
}
 
Example 2
Source Project: hadoop   Source File: MockContainer.java    License: Apache License 2.0 6 votes vote down vote up
public MockContainer(ApplicationAttemptId appAttemptId,
    Dispatcher dispatcher, Configuration conf, String user,
    ApplicationId appId, int uniqId) throws IOException{

  this.user = user;
  this.recordFactory = RecordFactoryProvider.getRecordFactory(conf);
  this.id = BuilderUtils.newContainerId(recordFactory, appId, appAttemptId,
      uniqId);
  this.launchContext = recordFactory
      .newRecordInstance(ContainerLaunchContext.class);
  long currentTime = System.currentTimeMillis();
  this.containerTokenIdentifier =
      BuilderUtils.newContainerTokenIdentifier(BuilderUtils
        .newContainerToken(id, "127.0.0.1", 1234, user,
          BuilderUtils.newResource(1024, 1), currentTime + 10000, 123,
          "password".getBytes(), currentTime));
  this.state = ContainerState.NEW;
}
 
Example 3
Source Project: hadoop   Source File: TestAppManager.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("deprecation")
@Before
public void setUp() {
  long now = System.currentTimeMillis();

  rmContext = mockRMContext(1, now - 10);
  ResourceScheduler scheduler = mockResourceScheduler();
  Configuration conf = new Configuration();
  ApplicationMasterService masterService =
      new ApplicationMasterService(rmContext, scheduler);
  appMonitor = new TestRMAppManager(rmContext,
      new ClientToAMTokenSecretManagerInRM(), scheduler, masterService,
      new ApplicationACLsManager(conf), conf);

  appId = MockApps.newAppID(1);
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  asContext =
      recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
  asContext.setApplicationId(appId);
  asContext.setAMContainerSpec(mockContainerLaunchContext(recordFactory));
  asContext.setResource(mockResource());
  setupDispatcher(rmContext, conf);
}
 
Example 4
Source Project: hadoop   Source File: TestClientRMService.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testNonExistingApplicationReport() throws YarnException {
  RMContext rmContext = mock(RMContext.class);
  when(rmContext.getRMApps()).thenReturn(
      new ConcurrentHashMap<ApplicationId, RMApp>());
  ClientRMService rmService = new ClientRMService(rmContext, null, null,
      null, null, null);
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  GetApplicationReportRequest request = recordFactory
      .newRecordInstance(GetApplicationReportRequest.class);
  request.setApplicationId(ApplicationId.newInstance(0, 0));
  try {
    rmService.getApplicationReport(request);
    Assert.fail();
  } catch (ApplicationNotFoundException ex) {
    Assert.assertEquals(ex.getMessage(),
        "Application with id '" + request.getApplicationId()
            + "' doesn't exist in RM.");
  }
}
 
Example 5
Source Project: hadoop   Source File: TestClientRMService.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testGetApplicationAttemptReport() throws YarnException,
    IOException {
  ClientRMService rmService = createRMService();
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  GetApplicationAttemptReportRequest request = recordFactory
      .newRecordInstance(GetApplicationAttemptReportRequest.class);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
      ApplicationId.newInstance(123456, 1), 1);
  request.setApplicationAttemptId(attemptId);

  try {
    GetApplicationAttemptReportResponse response = rmService
        .getApplicationAttemptReport(request);
    Assert.assertEquals(attemptId, response.getApplicationAttemptReport()
        .getApplicationAttemptId());
  } catch (ApplicationNotFoundException ex) {
    Assert.fail(ex.getMessage());
  }
}
 
Example 6
Source Project: hadoop   Source File: TestClientRMService.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testGetApplicationAttempts() throws YarnException, IOException {
  ClientRMService rmService = createRMService();
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  GetApplicationAttemptsRequest request = recordFactory
      .newRecordInstance(GetApplicationAttemptsRequest.class);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
      ApplicationId.newInstance(123456, 1), 1);
  request.setApplicationId(ApplicationId.newInstance(123456, 1));

  try {
    GetApplicationAttemptsResponse response = rmService
        .getApplicationAttempts(request);
    Assert.assertEquals(1, response.getApplicationAttemptList().size());
    Assert.assertEquals(attemptId, response.getApplicationAttemptList()
        .get(0).getApplicationAttemptId());

  } catch (ApplicationNotFoundException ex) {
    Assert.fail(ex.getMessage());
  }
}
 
Example 7
Source Project: hadoop   Source File: TestClientRMService.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testGetContainerReport() throws YarnException, IOException {
  ClientRMService rmService = createRMService();
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  GetContainerReportRequest request = recordFactory
      .newRecordInstance(GetContainerReportRequest.class);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
      ApplicationId.newInstance(123456, 1), 1);
  ContainerId containerId = ContainerId.newContainerId(attemptId, 1);
  request.setContainerId(containerId);

  try {
    GetContainerReportResponse response = rmService
        .getContainerReport(request);
    Assert.assertEquals(containerId, response.getContainerReport()
        .getContainerId());
  } catch (ApplicationNotFoundException ex) {
    Assert.fail(ex.getMessage());
  }
}
 
Example 8
Source Project: hadoop   Source File: TestClientRMService.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testGetContainers() throws YarnException, IOException {
  ClientRMService rmService = createRMService();
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  GetContainersRequest request = recordFactory
      .newRecordInstance(GetContainersRequest.class);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
      ApplicationId.newInstance(123456, 1), 1);
  ContainerId containerId = ContainerId.newContainerId(attemptId, 1);
  request.setApplicationAttemptId(attemptId);
  try {
    GetContainersResponse response = rmService.getContainers(request);
    Assert.assertEquals(containerId, response.getContainerList().get(0)
        .getContainerId());
  } catch (ApplicationNotFoundException ex) {
    Assert.fail(ex.getMessage());
  }
}
 
Example 9
Source Project: big-c   Source File: SharedCacheUploader.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * @param resource the local resource that contains the original remote path
 * @param localPath the path in the local filesystem where the resource is
 * localized
 * @param fs the filesystem of the shared cache
 * @param localFs the local filesystem
 */
public SharedCacheUploader(LocalResource resource, Path localPath,
    String user, Configuration conf, SCMUploaderProtocol scmClient,
    FileSystem fs, FileSystem localFs) {
  this.resource = resource;
  this.localPath = localPath;
  this.user = user;
  this.conf = conf;
  this.scmClient = scmClient;
  this.fs = fs;
  this.sharedCacheRootDir =
      conf.get(YarnConfiguration.SHARED_CACHE_ROOT,
          YarnConfiguration.DEFAULT_SHARED_CACHE_ROOT);
  this.nestedLevel = SharedCacheUtil.getCacheDepth(conf);
  this.checksum = SharedCacheChecksumFactory.getChecksum(conf);
  this.localFs = localFs;
  this.recordFactory = RecordFactoryProvider.getRecordFactory(null);
}
 
Example 10
Source Project: big-c   Source File: MockContainer.java    License: Apache License 2.0 6 votes vote down vote up
public MockContainer(ApplicationAttemptId appAttemptId,
    Dispatcher dispatcher, Configuration conf, String user,
    ApplicationId appId, int uniqId) throws IOException{

  this.user = user;
  this.recordFactory = RecordFactoryProvider.getRecordFactory(conf);
  this.id = BuilderUtils.newContainerId(recordFactory, appId, appAttemptId,
      uniqId);
  this.launchContext = recordFactory
      .newRecordInstance(ContainerLaunchContext.class);
  long currentTime = System.currentTimeMillis();
  this.containerTokenIdentifier =
      BuilderUtils.newContainerTokenIdentifier(BuilderUtils
        .newContainerToken(id, "127.0.0.1", 1234, user,
          BuilderUtils.newResource(1024, 1), currentTime + 10000, 123,
          "password".getBytes(), currentTime));
  this.state = ContainerState.NEW;
}
 
Example 11
Source Project: big-c   Source File: TestAppManager.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("deprecation")
@Before
public void setUp() {
  long now = System.currentTimeMillis();

  rmContext = mockRMContext(1, now - 10);
  ResourceScheduler scheduler = mockResourceScheduler();
  Configuration conf = new Configuration();
  ApplicationMasterService masterService =
      new ApplicationMasterService(rmContext, scheduler);
  appMonitor = new TestRMAppManager(rmContext,
      new ClientToAMTokenSecretManagerInRM(), scheduler, masterService,
      new ApplicationACLsManager(conf), conf);

  appId = MockApps.newAppID(1);
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  asContext =
      recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
  asContext.setApplicationId(appId);
  asContext.setAMContainerSpec(mockContainerLaunchContext(recordFactory));
  asContext.setResource(mockResource());
  setupDispatcher(rmContext, conf);
}
 
Example 12
Source Project: big-c   Source File: TestClientRMService.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testNonExistingApplicationReport() throws YarnException {
  RMContext rmContext = mock(RMContext.class);
  when(rmContext.getRMApps()).thenReturn(
      new ConcurrentHashMap<ApplicationId, RMApp>());
  ClientRMService rmService = new ClientRMService(rmContext, null, null,
      null, null, null);
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  GetApplicationReportRequest request = recordFactory
      .newRecordInstance(GetApplicationReportRequest.class);
  request.setApplicationId(ApplicationId.newInstance(0, 0));
  try {
    rmService.getApplicationReport(request);
    Assert.fail();
  } catch (ApplicationNotFoundException ex) {
    Assert.assertEquals(ex.getMessage(),
        "Application with id '" + request.getApplicationId()
            + "' doesn't exist in RM.");
  }
}
 
Example 13
Source Project: big-c   Source File: TestClientRMService.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testGetApplicationAttemptReport() throws YarnException,
    IOException {
  ClientRMService rmService = createRMService();
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  GetApplicationAttemptReportRequest request = recordFactory
      .newRecordInstance(GetApplicationAttemptReportRequest.class);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
      ApplicationId.newInstance(123456, 1), 1);
  request.setApplicationAttemptId(attemptId);

  try {
    GetApplicationAttemptReportResponse response = rmService
        .getApplicationAttemptReport(request);
    Assert.assertEquals(attemptId, response.getApplicationAttemptReport()
        .getApplicationAttemptId());
  } catch (ApplicationNotFoundException ex) {
    Assert.fail(ex.getMessage());
  }
}
 
Example 14
Source Project: big-c   Source File: TestClientRMService.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testGetApplicationAttempts() throws YarnException, IOException {
  ClientRMService rmService = createRMService();
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  GetApplicationAttemptsRequest request = recordFactory
      .newRecordInstance(GetApplicationAttemptsRequest.class);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
      ApplicationId.newInstance(123456, 1), 1);
  request.setApplicationId(ApplicationId.newInstance(123456, 1));

  try {
    GetApplicationAttemptsResponse response = rmService
        .getApplicationAttempts(request);
    Assert.assertEquals(1, response.getApplicationAttemptList().size());
    Assert.assertEquals(attemptId, response.getApplicationAttemptList()
        .get(0).getApplicationAttemptId());

  } catch (ApplicationNotFoundException ex) {
    Assert.fail(ex.getMessage());
  }
}
 
Example 15
Source Project: big-c   Source File: TestClientRMService.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testGetContainerReport() throws YarnException, IOException {
  ClientRMService rmService = createRMService();
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  GetContainerReportRequest request = recordFactory
      .newRecordInstance(GetContainerReportRequest.class);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
      ApplicationId.newInstance(123456, 1), 1);
  ContainerId containerId = ContainerId.newContainerId(attemptId, 1);
  request.setContainerId(containerId);

  try {
    GetContainerReportResponse response = rmService
        .getContainerReport(request);
    Assert.assertEquals(containerId, response.getContainerReport()
        .getContainerId());
  } catch (ApplicationNotFoundException ex) {
    Assert.fail(ex.getMessage());
  }
}
 
Example 16
Source Project: big-c   Source File: TestClientRMService.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testGetContainers() throws YarnException, IOException {
  ClientRMService rmService = createRMService();
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  GetContainersRequest request = recordFactory
      .newRecordInstance(GetContainersRequest.class);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
      ApplicationId.newInstance(123456, 1), 1);
  ContainerId containerId = ContainerId.newContainerId(attemptId, 1);
  request.setApplicationAttemptId(attemptId);
  try {
    GetContainersResponse response = rmService.getContainers(request);
    Assert.assertEquals(containerId, response.getContainerList().get(0)
        .getContainerId());
  } catch (ApplicationNotFoundException ex) {
    Assert.fail(ex.getMessage());
  }
}
 
Example 17
@Override
protected NodeStatusUpdater createNodeStatusUpdater(Context context,
                                                    Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
    return new NodeStatusUpdaterImpl(context, dispatcher,
                                     healthChecker, metrics) {
        @Override
        protected ResourceTracker getRMClient() {
            final ResourceTrackerService rt =
                getResourceManager().getResourceTrackerService();
            final RecordFactory recordFactory =
                RecordFactoryProvider.getRecordFactory(null);

            // For in-process communication without RPC
            return Utils.getResourceTracker(rt);
        }

        @Override
        protected void stopRMProxy() { }
     };
}
 
Example 18
Source Project: tez   Source File: TestTezLocalCacheManager.java    License: Apache License 2.0 6 votes vote down vote up
private static LocalResource createFile(String content) throws IOException {
  FileContext fs = FileContext.getLocalFSFileContext();

  java.nio.file.Path tempFile = Files.createTempFile("test-cache-manager", ".txt");
  File temp = tempFile.toFile();
  temp.deleteOnExit();
  Path p = new Path("file:///" + tempFile.toAbsolutePath().toString());

  Files.write(tempFile, content.getBytes());

  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
  URL yarnUrlFromPath = ConverterUtils.getYarnUrlFromPath(p);
  ret.setResource(yarnUrlFromPath);
  ret.setSize(content.getBytes().length);
  ret.setType(LocalResourceType.FILE);
  ret.setVisibility(LocalResourceVisibility.PRIVATE);
  ret.setTimestamp(fs.getFileStatus(p).getModificationTime());
  return ret;
}
 
Example 19
Source Project: hadoop   Source File: ConverterUtils.java    License: Apache License 2.0 5 votes vote down vote up
public static URL getYarnUrlFromURI(URI uri) {
  URL url = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(URL.class);
  if (uri.getHost() != null) {
    url.setHost(uri.getHost());
  }
  if (uri.getUserInfo() != null) {
    url.setUserInfo(uri.getUserInfo());
  }
  url.setPort(uri.getPort());
  url.setScheme(uri.getScheme());
  url.setFile(uri.getPath());
  return url;
}
 
Example 20
Source Project: hadoop   Source File: DockerContainerExecutor.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public synchronized void startLocalizer(LocalizerStartContext ctx)
  throws IOException, InterruptedException {
  Path nmPrivateContainerTokensPath = ctx.getNmPrivateContainerTokens();
  InetSocketAddress nmAddr = ctx.getNmAddr();
  String user = ctx.getUser();
  String appId = ctx.getAppId();
  String locId = ctx.getLocId();
  LocalDirsHandlerService dirsHandler = ctx.getDirsHandler();
  List<String> localDirs = dirsHandler.getLocalDirs();
  List<String> logDirs = dirsHandler.getLogDirs();

  ContainerLocalizer localizer =
    new ContainerLocalizer(lfs, user, appId, locId, getPaths(localDirs),
      RecordFactoryProvider.getRecordFactory(getConf()));

  createUserLocalDirs(localDirs, user);
  createUserCacheDirs(localDirs, user);
  createAppDirs(localDirs, user, appId);
  createAppLogDirs(appId, logDirs, user);

  // randomly choose the local directory
  Path appStorageDir = getWorkingDir(localDirs, user, appId);

  String tokenFn = String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, locId);
  Path tokenDst = new Path(appStorageDir, tokenFn);
  copyFile(nmPrivateContainerTokensPath, tokenDst, user);
  LOG.info("Copying from " + nmPrivateContainerTokensPath + " to " + tokenDst);
  lfs.setWorkingDirectory(appStorageDir);
  LOG.info("CWD set to " + appStorageDir + " = " + lfs.getWorkingDirectory());
  // TODO: DO it over RPC for maintaining similarity?
  localizer.runLocalization(nmAddr);
}
 
Example 21
Source Project: hadoop   Source File: DefaultContainerExecutor.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void startLocalizer(LocalizerStartContext ctx)
    throws IOException, InterruptedException {
  Path nmPrivateContainerTokensPath = ctx.getNmPrivateContainerTokens();
  InetSocketAddress nmAddr = ctx.getNmAddr();
  String user = ctx.getUser();
  String appId = ctx.getAppId();
  String locId = ctx.getLocId();
  LocalDirsHandlerService dirsHandler = ctx.getDirsHandler();

  List<String> localDirs = dirsHandler.getLocalDirs();
  List<String> logDirs = dirsHandler.getLogDirs();
  
  createUserLocalDirs(localDirs, user);
  createUserCacheDirs(localDirs, user);
  createAppDirs(localDirs, user, appId);
  createAppLogDirs(appId, logDirs, user);

  // randomly choose the local directory
  Path appStorageDir = getWorkingDir(localDirs, user, appId);

  String tokenFn = String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, locId);
  Path tokenDst = new Path(appStorageDir, tokenFn);
  copyFile(nmPrivateContainerTokensPath, tokenDst, user);
  LOG.info("Copying from " + nmPrivateContainerTokensPath + " to " + tokenDst);


  FileContext localizerFc = FileContext.getFileContext(
      lfs.getDefaultFileSystem(), getConf());
  localizerFc.setUMask(lfs.getUMask());
  localizerFc.setWorkingDirectory(appStorageDir);
  LOG.info("Localizer CWD set to " + appStorageDir + " = " 
      + localizerFc.getWorkingDirectory());
  ContainerLocalizer localizer =
      new ContainerLocalizer(localizerFc, user, appId, locId, 
          getPaths(localDirs), RecordFactoryProvider.getRecordFactory(getConf()));
  // TODO: DO it over RPC for maintaining similarity?
  localizer.runLocalization(nmAddr);
}
 
Example 22
Source Project: hadoop   Source File: MockApp.java    License: Apache License 2.0 5 votes vote down vote up
public MockApp(String user, long clusterTimeStamp, int uniqId) {
  super();
  this.user = user;
  // Add an application and the corresponding containers
  RecordFactory recordFactory = RecordFactoryProvider
      .getRecordFactory(new Configuration());
  this.appId = BuilderUtils.newApplicationId(recordFactory, clusterTimeStamp,
      uniqId);
  appState = ApplicationState.NEW;
}
 
Example 23
Source Project: hadoop   Source File: TestLocalResource.java    License: Apache License 2.0 5 votes vote down vote up
static org.apache.hadoop.yarn.api.records.LocalResource getYarnResource(Path p, long size,
    long timestamp, LocalResourceType type, LocalResourceVisibility state, String pattern)
    throws URISyntaxException {
  org.apache.hadoop.yarn.api.records.LocalResource ret = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(org.apache.hadoop.yarn.api.records.LocalResource.class);
  ret.setResource(ConverterUtils.getYarnUrlFromURI(p.toUri()));
  ret.setSize(size);
  ret.setTimestamp(timestamp);
  ret.setType(type);
  ret.setVisibility(state);
  ret.setPattern(pattern);
  return ret;
}
 
Example 24
Source Project: hadoop   Source File: NodeManager.java    License: Apache License 2.0 5 votes vote down vote up
public static org.apache.hadoop.yarn.server.api.records.NodeStatus 
createNodeStatus(NodeId nodeId, List<ContainerStatus> containers) {
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  org.apache.hadoop.yarn.server.api.records.NodeStatus nodeStatus = 
      recordFactory.newRecordInstance(org.apache.hadoop.yarn.server.api.records.NodeStatus.class);
  nodeStatus.setNodeId(nodeId);
  nodeStatus.setContainersStatuses(containers);
  NodeHealthStatus nodeHealthStatus = 
    recordFactory.newRecordInstance(NodeHealthStatus.class);
  nodeHealthStatus.setIsNodeHealthy(true);
  nodeStatus.setNodeHealthStatus(nodeHealthStatus);
  return nodeStatus;
}
 
Example 25
Source Project: hadoop   Source File: TestClientRMService.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testGetApplicationReport() throws Exception {
  YarnScheduler yarnScheduler = mock(YarnScheduler.class);
  RMContext rmContext = mock(RMContext.class);
  mockRMContext(yarnScheduler, rmContext);

  ApplicationId appId1 = getApplicationId(1);

  ApplicationACLsManager mockAclsManager = mock(ApplicationACLsManager.class);
  when(
      mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(),
          ApplicationAccessType.VIEW_APP, null, appId1)).thenReturn(true);

  ClientRMService rmService = new ClientRMService(rmContext, yarnScheduler,
      null, mockAclsManager, null, null);
  try {
    RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
    GetApplicationReportRequest request = recordFactory
        .newRecordInstance(GetApplicationReportRequest.class);
    request.setApplicationId(appId1);
    GetApplicationReportResponse response = 
        rmService.getApplicationReport(request);
    ApplicationReport report = response.getApplicationReport();
    ApplicationResourceUsageReport usageReport = 
        report.getApplicationResourceUsageReport();
    Assert.assertEquals(10, usageReport.getMemorySeconds());
    Assert.assertEquals(3, usageReport.getVcoreSeconds());
    Assert.assertEquals(3, usageReport.getGcoreSeconds());
  } finally {
    rmService.close();
  }
}
 
Example 26
Source Project: hadoop   Source File: TestTaskAttemptListenerImpl.java    License: Apache License 2.0 5 votes vote down vote up
private static TaskAttemptCompletionEvent createTce(int eventId,
    boolean isMap, TaskAttemptCompletionEventStatus status) {
  JobId jid = MRBuilderUtils.newJobId(12345, 1, 1);
  TaskId tid = MRBuilderUtils.newTaskId(jid, 0,
      isMap ? org.apache.hadoop.mapreduce.v2.api.records.TaskType.MAP
          : org.apache.hadoop.mapreduce.v2.api.records.TaskType.REDUCE);
  TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(tid, 0);
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  TaskAttemptCompletionEvent tce = recordFactory
      .newRecordInstance(TaskAttemptCompletionEvent.class);
  tce.setEventId(eventId);
  tce.setAttemptId(attemptId);
  tce.setStatus(status);
  return tce;
}
 
Example 27
Source Project: hadoop   Source File: TestMRApps.java    License: Apache License 2.0 5 votes vote down vote up
@Test (timeout = 120000)
public void testTaskIDtoString() {
  TaskId tid = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(TaskId.class);
  tid.setJobId(RecordFactoryProvider.getRecordFactory(null).newRecordInstance(JobId.class));
  tid.getJobId().setAppId(ApplicationId.newInstance(0, 0));
  tid.setTaskType(TaskType.MAP);
  TaskType type = tid.getTaskType();
  System.err.println(type);
  type = TaskType.REDUCE;
  System.err.println(type);
  System.err.println(tid.getTaskType());
  assertEquals("task_0_0000_m_000000", MRApps.toString(tid));
  tid.setTaskType(TaskType.REDUCE);
  assertEquals("task_0_0000_r_000000", MRApps.toString(tid));
}
 
Example 28
Source Project: hadoop   Source File: TestMRApps.java    License: Apache License 2.0 5 votes vote down vote up
@Test (timeout = 120000)
public void testTaskAttemptIDtoString() {
  TaskAttemptId taid = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(TaskAttemptId.class);
  taid.setTaskId(RecordFactoryProvider.getRecordFactory(null).newRecordInstance(TaskId.class));
  taid.getTaskId().setTaskType(TaskType.MAP);
  taid.getTaskId().setJobId(RecordFactoryProvider.getRecordFactory(null).newRecordInstance(JobId.class));
  taid.getTaskId().getJobId().setAppId(ApplicationId.newInstance(0, 0));
  assertEquals("attempt_0_0000_m_000000_0", MRApps.toString(taid));
}
 
Example 29
Source Project: hadoop   Source File: JobHistory.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected void serviceInit(Configuration conf) throws Exception {
  LOG.info("JobHistory Init");
  this.conf = conf;
  this.appID = ApplicationId.newInstance(0, 0);
  this.appAttemptID = RecordFactoryProvider.getRecordFactory(conf)
      .newRecordInstance(ApplicationAttemptId.class);

  moveThreadInterval = conf.getLong(
      JHAdminConfig.MR_HISTORY_MOVE_INTERVAL_MS,
      JHAdminConfig.DEFAULT_MR_HISTORY_MOVE_INTERVAL_MS);

  hsManager = createHistoryFileManager();
  hsManager.init(conf);
  try {
    hsManager.initExisting();
  } catch (IOException e) {
    throw new YarnRuntimeException("Failed to intialize existing directories", e);
  }

  storage = createHistoryStorage();
  
  if (storage instanceof Service) {
    ((Service) storage).init(conf);
  }
  storage.setHistoryFileManager(hsManager);

  super.serviceInit(conf);
}
 
Example 30
Source Project: hadoop   Source File: PartialJob.java    License: Apache License 2.0 5 votes vote down vote up
public PartialJob(JobIndexInfo jobIndexInfo, JobId jobId) {
  this.jobIndexInfo = jobIndexInfo;
  this.jobId = jobId;
  jobReport = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(JobReport.class);
  jobReport.setSubmitTime(jobIndexInfo.getSubmitTime());
  jobReport.setStartTime(jobIndexInfo.getJobStartTime());
  jobReport.setFinishTime(jobIndexInfo.getFinishTime());
  jobReport.setJobState(getState());
}