Java Code Examples for org.apache.hadoop.yarn.factories.RecordFactory#newRecordInstance()

The following examples show how to use org.apache.hadoop.yarn.factories.RecordFactory#newRecordInstance() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: TestClientRMService.java From hadoop with Apache License 2.0

6 votes

@Test
public void testGetApplicationAttemptReport() throws YarnException,
    IOException {
  ClientRMService rmService = createRMService();
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  GetApplicationAttemptReportRequest request = recordFactory
      .newRecordInstance(GetApplicationAttemptReportRequest.class);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
      ApplicationId.newInstance(123456, 1), 1);
  request.setApplicationAttemptId(attemptId);

  try {
    GetApplicationAttemptReportResponse response = rmService
        .getApplicationAttemptReport(request);
    Assert.assertEquals(attemptId, response.getApplicationAttemptReport()
        .getApplicationAttemptId());
  } catch (ApplicationNotFoundException ex) {
    Assert.fail(ex.getMessage());
  }
}

Example 2

Source File: TestClientRMService.java From big-c with Apache License 2.0

6 votes

@Test
public void testGetContainerReport() throws YarnException, IOException {
  ClientRMService rmService = createRMService();
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  GetContainerReportRequest request = recordFactory
      .newRecordInstance(GetContainerReportRequest.class);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
      ApplicationId.newInstance(123456, 1), 1);
  ContainerId containerId = ContainerId.newContainerId(attemptId, 1);
  request.setContainerId(containerId);

  try {
    GetContainerReportResponse response = rmService
        .getContainerReport(request);
    Assert.assertEquals(containerId, response.getContainerReport()
        .getContainerId());
  } catch (ApplicationNotFoundException ex) {
    Assert.fail(ex.getMessage());
  }
}

Example 3

Source File: TestClientRMService.java From big-c with Apache License 2.0

6 votes

@Test
public void testNonExistingApplicationReport() throws YarnException {
  RMContext rmContext = mock(RMContext.class);
  when(rmContext.getRMApps()).thenReturn(
      new ConcurrentHashMap<ApplicationId, RMApp>());
  ClientRMService rmService = new ClientRMService(rmContext, null, null,
      null, null, null);
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  GetApplicationReportRequest request = recordFactory
      .newRecordInstance(GetApplicationReportRequest.class);
  request.setApplicationId(ApplicationId.newInstance(0, 0));
  try {
    rmService.getApplicationReport(request);
    Assert.fail();
  } catch (ApplicationNotFoundException ex) {
    Assert.assertEquals(ex.getMessage(),
        "Application with id '" + request.getApplicationId()
            + "' doesn't exist in RM.");
  }
}

Example 4

Source File: TestClientRMService.java From hadoop with Apache License 2.0

6 votes

@Test
public void testGetContainers() throws YarnException, IOException {
  ClientRMService rmService = createRMService();
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  GetContainersRequest request = recordFactory
      .newRecordInstance(GetContainersRequest.class);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
      ApplicationId.newInstance(123456, 1), 1);
  ContainerId containerId = ContainerId.newContainerId(attemptId, 1);
  request.setApplicationAttemptId(attemptId);
  try {
    GetContainersResponse response = rmService.getContainers(request);
    Assert.assertEquals(containerId, response.getContainerList().get(0)
        .getContainerId());
  } catch (ApplicationNotFoundException ex) {
    Assert.fail(ex.getMessage());
  }
}

Example 5

Source File: TestClientRMService.java From hadoop with Apache License 2.0

6 votes

@Test
public void testGetApplicationAttempts() throws YarnException, IOException {
  ClientRMService rmService = createRMService();
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  GetApplicationAttemptsRequest request = recordFactory
      .newRecordInstance(GetApplicationAttemptsRequest.class);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
      ApplicationId.newInstance(123456, 1), 1);
  request.setApplicationId(ApplicationId.newInstance(123456, 1));

  try {
    GetApplicationAttemptsResponse response = rmService
        .getApplicationAttempts(request);
    Assert.assertEquals(1, response.getApplicationAttemptList().size());
    Assert.assertEquals(attemptId, response.getApplicationAttemptList()
        .get(0).getApplicationAttemptId());

  } catch (ApplicationNotFoundException ex) {
    Assert.fail(ex.getMessage());
  }
}

Example 6

Source File: TestTezLocalCacheManager.java From tez with Apache License 2.0

6 votes

private static LocalResource createFile(String content) throws IOException {
  FileContext fs = FileContext.getLocalFSFileContext();

  java.nio.file.Path tempFile = Files.createTempFile("test-cache-manager", ".txt");
  File temp = tempFile.toFile();
  temp.deleteOnExit();
  Path p = new Path("file:///" + tempFile.toAbsolutePath().toString());

  Files.write(tempFile, content.getBytes());

  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
  URL yarnUrlFromPath = ConverterUtils.getYarnUrlFromPath(p);
  ret.setResource(yarnUrlFromPath);
  ret.setSize(content.getBytes().length);
  ret.setType(LocalResourceType.FILE);
  ret.setVisibility(LocalResourceVisibility.PRIVATE);
  ret.setTimestamp(fs.getFileStatus(p).getModificationTime());
  return ret;
}

Example 7

Source File: TestClientRMService.java From big-c with Apache License 2.0

6 votes

@Test
public void testGetApplicationAttemptReport() throws YarnException,
    IOException {
  ClientRMService rmService = createRMService();
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  GetApplicationAttemptReportRequest request = recordFactory
      .newRecordInstance(GetApplicationAttemptReportRequest.class);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
      ApplicationId.newInstance(123456, 1), 1);
  request.setApplicationAttemptId(attemptId);

  try {
    GetApplicationAttemptReportResponse response = rmService
        .getApplicationAttemptReport(request);
    Assert.assertEquals(attemptId, response.getApplicationAttemptReport()
        .getApplicationAttemptId());
  } catch (ApplicationNotFoundException ex) {
    Assert.fail(ex.getMessage());
  }
}

Example 8

Source File: TestClientRMService.java From hadoop with Apache License 2.0

6 votes

@Test
public void testNonExistingApplicationReport() throws YarnException {
  RMContext rmContext = mock(RMContext.class);
  when(rmContext.getRMApps()).thenReturn(
      new ConcurrentHashMap<ApplicationId, RMApp>());
  ClientRMService rmService = new ClientRMService(rmContext, null, null,
      null, null, null);
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  GetApplicationReportRequest request = recordFactory
      .newRecordInstance(GetApplicationReportRequest.class);
  request.setApplicationId(ApplicationId.newInstance(0, 0));
  try {
    rmService.getApplicationReport(request);
    Assert.fail();
  } catch (ApplicationNotFoundException ex) {
    Assert.assertEquals(ex.getMessage(),
        "Application with id '" + request.getApplicationId()
            + "' doesn't exist in RM.");
  }
}

Example 9

Source File: TestAppManager.java From hadoop with Apache License 2.0

6 votes

@SuppressWarnings("deprecation")
@Before
public void setUp() {
  long now = System.currentTimeMillis();

  rmContext = mockRMContext(1, now - 10);
  ResourceScheduler scheduler = mockResourceScheduler();
  Configuration conf = new Configuration();
  ApplicationMasterService masterService =
      new ApplicationMasterService(rmContext, scheduler);
  appMonitor = new TestRMAppManager(rmContext,
      new ClientToAMTokenSecretManagerInRM(), scheduler, masterService,
      new ApplicationACLsManager(conf), conf);

  appId = MockApps.newAppID(1);
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  asContext =
      recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
  asContext.setApplicationId(appId);
  asContext.setAMContainerSpec(mockContainerLaunchContext(recordFactory));
  asContext.setResource(mockResource());
  setupDispatcher(rmContext, conf);
}

Example 10

Source File: NodeManager.java From hadoop with Apache License 2.0

5 votes

public static org.apache.hadoop.yarn.server.api.records.NodeStatus 
createNodeStatus(NodeId nodeId, List<ContainerStatus> containers) {
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  org.apache.hadoop.yarn.server.api.records.NodeStatus nodeStatus = 
      recordFactory.newRecordInstance(org.apache.hadoop.yarn.server.api.records.NodeStatus.class);
  nodeStatus.setNodeId(nodeId);
  nodeStatus.setContainersStatuses(containers);
  NodeHealthStatus nodeHealthStatus = 
    recordFactory.newRecordInstance(NodeHealthStatus.class);
  nodeHealthStatus.setIsNodeHealthy(true);
  nodeStatus.setNodeHealthStatus(nodeHealthStatus);
  return nodeStatus;
}

Example 11

Source File: TestUtils.java From hadoop with Apache License 2.0

5 votes

public static ResourceRequest createResourceRequest(
    String resourceName, int memory, int numContainers, boolean relaxLocality,
    Priority priority, RecordFactory recordFactory) {
  ResourceRequest request = 
      recordFactory.newRecordInstance(ResourceRequest.class);
  Resource capability = Resources.createResource(memory, 1);
  
  request.setNumContainers(numContainers);
  request.setResourceName(resourceName);
  request.setCapability(capability);
  request.setRelaxLocality(relaxLocality);
  request.setPriority(priority);
  return request;
}

Example 12

Source File: TestUtils.java From big-c with Apache License 2.0

5 votes

public static ResourceRequest createResourceRequest(
    String resourceName, int memory, int numContainers, boolean relaxLocality,
    Priority priority, RecordFactory recordFactory) {
  ResourceRequest request = 
      recordFactory.newRecordInstance(ResourceRequest.class);
  Resource capability = Resources.createResource(memory, 1);
  
  request.setNumContainers(numContainers);
  request.setResourceName(resourceName);
  request.setCapability(capability);
  request.setRelaxLocality(relaxLocality);
  request.setPriority(priority);
  return request;
}

Example 13

Source File: NodeManager.java From big-c with Apache License 2.0

5 votes

public static org.apache.hadoop.yarn.server.api.records.NodeStatus 
createNodeStatus(NodeId nodeId, List<ContainerStatus> containers) {
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  org.apache.hadoop.yarn.server.api.records.NodeStatus nodeStatus = 
      recordFactory.newRecordInstance(org.apache.hadoop.yarn.server.api.records.NodeStatus.class);
  nodeStatus.setNodeId(nodeId);
  nodeStatus.setContainersStatuses(containers);
  NodeHealthStatus nodeHealthStatus = 
    recordFactory.newRecordInstance(NodeHealthStatus.class);
  nodeHealthStatus.setIsNodeHealthy(true);
  nodeStatus.setNodeHealthStatus(nodeHealthStatus);
  return nodeStatus;
}

Example 14

Source File: TestTaskAttemptListenerImpl.java From hadoop with Apache License 2.0

5 votes

private static TaskAttemptCompletionEvent createTce(int eventId,
    boolean isMap, TaskAttemptCompletionEventStatus status) {
  JobId jid = MRBuilderUtils.newJobId(12345, 1, 1);
  TaskId tid = MRBuilderUtils.newTaskId(jid, 0,
      isMap ? org.apache.hadoop.mapreduce.v2.api.records.TaskType.MAP
          : org.apache.hadoop.mapreduce.v2.api.records.TaskType.REDUCE);
  TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(tid, 0);
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  TaskAttemptCompletionEvent tce = recordFactory
      .newRecordInstance(TaskAttemptCompletionEvent.class);
  tce.setEventId(eventId);
  tce.setAttemptId(attemptId);
  tce.setStatus(status);
  return tce;
}

Example 15

Source File: TestYSCRecordFactory.java From big-c with Apache License 2.0

5 votes

@Test
public void testPbRecordFactory() {
  RecordFactory pbRecordFactory = RecordFactoryPBImpl.get();
  try {
    NodeHeartbeatRequest request = pbRecordFactory.newRecordInstance(NodeHeartbeatRequest.class);
    Assert.assertEquals(NodeHeartbeatRequestPBImpl.class, request.getClass());
  } catch (YarnRuntimeException e) {
    e.printStackTrace();
    Assert.fail("Failed to crete record");
  }
  
}

Example 16

Source File: TestRecordFactory.java From hadoop with Apache License 2.0

5 votes

@Test
public void testPbRecordFactory() {
  RecordFactory pbRecordFactory = RecordFactoryPBImpl.get();
  
  try {
    LocalizerHeartbeatResponse response = pbRecordFactory.newRecordInstance(
        LocalizerHeartbeatResponse.class);
    Assert.assertEquals(LocalizerHeartbeatResponsePBImpl.class,
                        response.getClass());
  } catch (YarnRuntimeException e) {
    e.printStackTrace();
    Assert.fail("Failed to crete record");
  }
}

Example 17

Source File: TestClientRMService.java From big-c with Apache License 2.0

5 votes

@Test
public void testGetApplicationReport() throws Exception {
  YarnScheduler yarnScheduler = mock(YarnScheduler.class);
  RMContext rmContext = mock(RMContext.class);
  mockRMContext(yarnScheduler, rmContext);

  ApplicationId appId1 = getApplicationId(1);

  ApplicationACLsManager mockAclsManager = mock(ApplicationACLsManager.class);
  when(
      mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(),
          ApplicationAccessType.VIEW_APP, null, appId1)).thenReturn(true);

  ClientRMService rmService = new ClientRMService(rmContext, yarnScheduler,
      null, mockAclsManager, null, null);
  try {
    RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
    GetApplicationReportRequest request = recordFactory
        .newRecordInstance(GetApplicationReportRequest.class);
    request.setApplicationId(appId1);
    GetApplicationReportResponse response = 
        rmService.getApplicationReport(request);
    ApplicationReport report = response.getApplicationReport();
    ApplicationResourceUsageReport usageReport = 
        report.getApplicationResourceUsageReport();
    Assert.assertEquals(10, usageReport.getMemorySeconds());
    Assert.assertEquals(3, usageReport.getVcoreSeconds());
  } finally {
    rmService.close();
  }
}

Example 18

Source File: TestTaskAttemptListenerImpl.java From big-c with Apache License 2.0

5 votes

private static TaskAttemptCompletionEvent createTce(int eventId,
    boolean isMap, TaskAttemptCompletionEventStatus status) {
  JobId jid = MRBuilderUtils.newJobId(12345, 1, 1);
  TaskId tid = MRBuilderUtils.newTaskId(jid, 0,
      isMap ? org.apache.hadoop.mapreduce.v2.api.records.TaskType.MAP
          : org.apache.hadoop.mapreduce.v2.api.records.TaskType.REDUCE);
  TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(tid, 0);
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  TaskAttemptCompletionEvent tce = recordFactory
      .newRecordInstance(TaskAttemptCompletionEvent.class);
  tce.setEventId(eventId);
  tce.setAttemptId(attemptId);
  tce.setStatus(status);
  return tce;
}

Example 19

Source File: TestNodeHealthService.java From hadoop with Apache License 2.0

4 votes

@Test
public void testNodeHealthService() throws Exception {
  RecordFactory factory = RecordFactoryProvider.getRecordFactory(null);
  NodeHealthStatus healthStatus =
      factory.newRecordInstance(NodeHealthStatus.class);
  Configuration conf = getConfForNodeHealthScript();
  conf.writeXml(new FileOutputStream(nodeHealthConfigFile));
  conf.addResource(nodeHealthConfigFile.getName());
  writeNodeHealthScriptFile("", true);

  LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
  NodeHealthScriptRunner nodeHealthScriptRunner =
      spy(NodeManager.getNodeHealthScriptRunner(conf));
  NodeHealthCheckerService nodeHealthChecker = new NodeHealthCheckerService(
  		nodeHealthScriptRunner, dirsHandler);
  nodeHealthChecker.init(conf);

  doReturn(true).when(nodeHealthScriptRunner).isHealthy();
  doReturn("").when(nodeHealthScriptRunner).getHealthReport();
  setHealthStatus(healthStatus, nodeHealthChecker.isHealthy(),
      nodeHealthChecker.getHealthReport(),
      nodeHealthChecker.getLastHealthReportTime());
  LOG.info("Checking initial healthy condition");
  // Check proper report conditions.
  Assert.assertTrue("Node health status reported unhealthy", healthStatus
      .getIsNodeHealthy());
  Assert.assertTrue("Node health status reported unhealthy", healthStatus
      .getHealthReport().equals(nodeHealthChecker.getHealthReport()));

  doReturn(false).when(nodeHealthScriptRunner).isHealthy();   
  // update health status
  setHealthStatus(healthStatus, nodeHealthChecker.isHealthy(),
      nodeHealthChecker.getHealthReport(),
      nodeHealthChecker.getLastHealthReportTime());
  LOG.info("Checking Healthy--->Unhealthy");
  Assert.assertFalse("Node health status reported healthy", healthStatus
      .getIsNodeHealthy());
  Assert.assertTrue("Node health status reported healthy", healthStatus
      .getHealthReport().equals(nodeHealthChecker.getHealthReport()));

  doReturn(true).when(nodeHealthScriptRunner).isHealthy();
  setHealthStatus(healthStatus, nodeHealthChecker.isHealthy(),
      nodeHealthChecker.getHealthReport(),
      nodeHealthChecker.getLastHealthReportTime());
  LOG.info("Checking UnHealthy--->healthy");
  // Check proper report conditions.
  Assert.assertTrue("Node health status reported unhealthy", healthStatus
      .getIsNodeHealthy());
  Assert.assertTrue("Node health status reported unhealthy", healthStatus
      .getHealthReport().equals(nodeHealthChecker.getHealthReport()));

  // Healthy to timeout transition.
  doReturn(false).when(nodeHealthScriptRunner).isHealthy();
  doReturn(NodeHealthScriptRunner.NODE_HEALTH_SCRIPT_TIMED_OUT_MSG)
      .when(nodeHealthScriptRunner).getHealthReport();
  setHealthStatus(healthStatus, nodeHealthChecker.isHealthy(),
      nodeHealthChecker.getHealthReport(),
      nodeHealthChecker.getLastHealthReportTime());
  LOG.info("Checking Healthy--->timeout");
  Assert.assertFalse("Node health status reported healthy even after timeout",
      healthStatus.getIsNodeHealthy());
  Assert.assertTrue("Node script time out message not propogated",
      healthStatus.getHealthReport().equals(
          NodeHealthScriptRunner.NODE_HEALTH_SCRIPT_TIMED_OUT_MSG
          + NodeHealthCheckerService.SEPARATOR
          + nodeHealthChecker.getDiskHandler().getDisksHealthReport(false)));
}

Example 20

Source File: TestNMWebServer.java From big-c with Apache License 2.0

4 votes

@Test
  public void testNMWebApp() throws IOException, YarnException {
    Context nmContext = new NodeManager.NMContext(null, null, null, null,
        null, null);
    ResourceView resourceView = new ResourceView() {
      @Override
      public long getVmemAllocatedForContainers() {
        return 0;
      }
      @Override
      public long getPmemAllocatedForContainers() {
        return 0;
      }
      @Override
      public long getVCoresAllocatedForContainers() {
        return 0;
      }
      @Override
      public boolean isVmemCheckEnabled() {
        return true;
      }
      @Override
      public boolean isPmemCheckEnabled() {
        return true;
      }
    };
    Configuration conf = new Configuration();
    conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath());
    conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath());
    NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
    healthChecker.init(conf);
    LocalDirsHandlerService dirsHandler = healthChecker.getDiskHandler();

    WebServer server = new WebServer(nmContext, resourceView,
        new ApplicationACLsManager(conf), dirsHandler);
    server.init(conf);
    server.start();

    // Add an application and the corresponding containers
    RecordFactory recordFactory =
        RecordFactoryProvider.getRecordFactory(conf);
    Dispatcher dispatcher = new AsyncDispatcher();
    String user = "nobody";
    long clusterTimeStamp = 1234;
    ApplicationId appId =
        BuilderUtils.newApplicationId(recordFactory, clusterTimeStamp, 1);
    Application app = mock(Application.class);
    when(app.getUser()).thenReturn(user);
    when(app.getAppId()).thenReturn(appId);
    nmContext.getApplications().put(appId, app);
    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
        appId, 1);
    ContainerId container1 =
        BuilderUtils.newContainerId(recordFactory, appId, appAttemptId, 0);
    ContainerId container2 =
        BuilderUtils.newContainerId(recordFactory, appId, appAttemptId, 1);
    NodeManagerMetrics metrics = mock(NodeManagerMetrics.class);
    NMStateStoreService stateStore = new NMNullStateStoreService();
    for (ContainerId containerId : new ContainerId[] { container1,
        container2}) {
      // TODO: Use builder utils
      ContainerLaunchContext launchContext =
          recordFactory.newRecordInstance(ContainerLaunchContext.class);
      long currentTime = System.currentTimeMillis();
      Token containerToken =
          BuilderUtils.newContainerToken(containerId, "127.0.0.1", 1234, user,
            BuilderUtils.newResource(1024, 1), currentTime + 10000L, 123,
            "password".getBytes(), currentTime);
      Container container =
          new ContainerImpl(nmContext, conf, dispatcher, stateStore, launchContext,
            null, metrics,
            BuilderUtils.newContainerTokenIdentifier(containerToken), null, 0, user, false, null) {

            @Override
            public ContainerState getContainerState() {
              return ContainerState.RUNNING;
            };
          };
      nmContext.getContainers().put(containerId, container);
      //TODO: Gross hack. Fix in code.
      ApplicationId applicationId = 
          containerId.getApplicationAttemptId().getApplicationId();
      nmContext.getApplications().get(applicationId).getContainers()
          .put(containerId, container);
      writeContainerLogs(nmContext, containerId, dirsHandler);

    }
    // TODO: Pull logs and test contents.
//    Thread.sleep(1000000);
  }