Java Code Examples for org.apache.hadoop.yarn.factories.RecordFactory

The following examples show how to use org.apache.hadoop.yarn.factories.RecordFactory. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source Project: hadoop Source File: ContainerLocalizer.java License: Apache License 2.0

6 votes

public ContainerLocalizer(FileContext lfs, String user, String appId,
    String localizerId, List<Path> localDirs,
    RecordFactory recordFactory) throws IOException {
  if (null == user) {
    throw new IOException("Cannot initialize for null user");
  }
  if (null == localizerId) {
    throw new IOException("Cannot initialize for null containerId");
  }
  this.lfs = lfs;
  this.user = user;
  this.appId = appId;
  this.localDirs = localDirs;
  this.localizerId = localizerId;
  this.recordFactory = recordFactory;
  this.conf = new Configuration();
  this.appCacheDirContextName = String.format(APPCACHE_CTXT_FMT, appId);
  this.pendingResources = new HashMap<LocalResource,Future<Path>>();
}

Example 2

Source Project: hadoop Source File: TestAppManager.java License: Apache License 2.0

6 votes

@SuppressWarnings("deprecation")
@Before
public void setUp() {
  long now = System.currentTimeMillis();

  rmContext = mockRMContext(1, now - 10);
  ResourceScheduler scheduler = mockResourceScheduler();
  Configuration conf = new Configuration();
  ApplicationMasterService masterService =
      new ApplicationMasterService(rmContext, scheduler);
  appMonitor = new TestRMAppManager(rmContext,
      new ClientToAMTokenSecretManagerInRM(), scheduler, masterService,
      new ApplicationACLsManager(conf), conf);

  appId = MockApps.newAppID(1);
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  asContext =
      recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
  asContext.setApplicationId(appId);
  asContext.setAMContainerSpec(mockContainerLaunchContext(recordFactory));
  asContext.setResource(mockResource());
  setupDispatcher(rmContext, conf);
}

Example 3

Source Project: hadoop Source File: TestClientRMService.java License: Apache License 2.0

6 votes

@Test
public void testNonExistingApplicationReport() throws YarnException {
  RMContext rmContext = mock(RMContext.class);
  when(rmContext.getRMApps()).thenReturn(
      new ConcurrentHashMap<ApplicationId, RMApp>());
  ClientRMService rmService = new ClientRMService(rmContext, null, null,
      null, null, null);
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  GetApplicationReportRequest request = recordFactory
      .newRecordInstance(GetApplicationReportRequest.class);
  request.setApplicationId(ApplicationId.newInstance(0, 0));
  try {
    rmService.getApplicationReport(request);
    Assert.fail();
  } catch (ApplicationNotFoundException ex) {
    Assert.assertEquals(ex.getMessage(),
        "Application with id '" + request.getApplicationId()
            + "' doesn't exist in RM.");
  }
}

Example 4

Source Project: hadoop Source File: TestClientRMService.java License: Apache License 2.0

6 votes

@Test
public void testGetApplicationAttemptReport() throws YarnException,
    IOException {
  ClientRMService rmService = createRMService();
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  GetApplicationAttemptReportRequest request = recordFactory
      .newRecordInstance(GetApplicationAttemptReportRequest.class);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
      ApplicationId.newInstance(123456, 1), 1);
  request.setApplicationAttemptId(attemptId);

  try {
    GetApplicationAttemptReportResponse response = rmService
        .getApplicationAttemptReport(request);
    Assert.assertEquals(attemptId, response.getApplicationAttemptReport()
        .getApplicationAttemptId());
  } catch (ApplicationNotFoundException ex) {
    Assert.fail(ex.getMessage());
  }
}

Example 5

Source Project: hadoop Source File: TestClientRMService.java License: Apache License 2.0

6 votes

@Test
public void testGetApplicationAttempts() throws YarnException, IOException {
  ClientRMService rmService = createRMService();
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  GetApplicationAttemptsRequest request = recordFactory
      .newRecordInstance(GetApplicationAttemptsRequest.class);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
      ApplicationId.newInstance(123456, 1), 1);
  request.setApplicationId(ApplicationId.newInstance(123456, 1));

  try {
    GetApplicationAttemptsResponse response = rmService
        .getApplicationAttempts(request);
    Assert.assertEquals(1, response.getApplicationAttemptList().size());
    Assert.assertEquals(attemptId, response.getApplicationAttemptList()
        .get(0).getApplicationAttemptId());

  } catch (ApplicationNotFoundException ex) {
    Assert.fail(ex.getMessage());
  }
}

Example 6

Source Project: hadoop Source File: TestClientRMService.java License: Apache License 2.0

6 votes

@Test
public void testGetContainerReport() throws YarnException, IOException {
  ClientRMService rmService = createRMService();
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  GetContainerReportRequest request = recordFactory
      .newRecordInstance(GetContainerReportRequest.class);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
      ApplicationId.newInstance(123456, 1), 1);
  ContainerId containerId = ContainerId.newContainerId(attemptId, 1);
  request.setContainerId(containerId);

  try {
    GetContainerReportResponse response = rmService
        .getContainerReport(request);
    Assert.assertEquals(containerId, response.getContainerReport()
        .getContainerId());
  } catch (ApplicationNotFoundException ex) {
    Assert.fail(ex.getMessage());
  }
}

Example 7

Source Project: hadoop Source File: TestClientRMService.java License: Apache License 2.0

6 votes

@Test
public void testGetContainers() throws YarnException, IOException {
  ClientRMService rmService = createRMService();
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  GetContainersRequest request = recordFactory
      .newRecordInstance(GetContainersRequest.class);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
      ApplicationId.newInstance(123456, 1), 1);
  ContainerId containerId = ContainerId.newContainerId(attemptId, 1);
  request.setApplicationAttemptId(attemptId);
  try {
    GetContainersResponse response = rmService.getContainers(request);
    Assert.assertEquals(containerId, response.getContainerList().get(0)
        .getContainerId());
  } catch (ApplicationNotFoundException ex) {
    Assert.fail(ex.getMessage());
  }
}

Example 8

Source Project: big-c Source File: ContainerLocalizer.java License: Apache License 2.0

6 votes

public ContainerLocalizer(FileContext lfs, String user, String appId,
    String localizerId, List<Path> localDirs,
    RecordFactory recordFactory) throws IOException {
  if (null == user) {
    throw new IOException("Cannot initialize for null user");
  }
  if (null == localizerId) {
    throw new IOException("Cannot initialize for null containerId");
  }
  this.lfs = lfs;
  this.user = user;
  this.appId = appId;
  this.localDirs = localDirs;
  this.localizerId = localizerId;
  this.recordFactory = recordFactory;
  this.conf = new Configuration();
  this.appCacheDirContextName = String.format(APPCACHE_CTXT_FMT, appId);
  this.pendingResources = new HashMap<LocalResource,Future<Path>>();
}

Example 9

Source Project: big-c Source File: TestAppManager.java License: Apache License 2.0

6 votes

@SuppressWarnings("deprecation")
@Before
public void setUp() {
  long now = System.currentTimeMillis();

  rmContext = mockRMContext(1, now - 10);
  ResourceScheduler scheduler = mockResourceScheduler();
  Configuration conf = new Configuration();
  ApplicationMasterService masterService =
      new ApplicationMasterService(rmContext, scheduler);
  appMonitor = new TestRMAppManager(rmContext,
      new ClientToAMTokenSecretManagerInRM(), scheduler, masterService,
      new ApplicationACLsManager(conf), conf);

  appId = MockApps.newAppID(1);
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  asContext =
      recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
  asContext.setApplicationId(appId);
  asContext.setAMContainerSpec(mockContainerLaunchContext(recordFactory));
  asContext.setResource(mockResource());
  setupDispatcher(rmContext, conf);
}

Example 10

Source Project: big-c Source File: TestClientRMService.java License: Apache License 2.0

6 votes

@Test
public void testNonExistingApplicationReport() throws YarnException {
  RMContext rmContext = mock(RMContext.class);
  when(rmContext.getRMApps()).thenReturn(
      new ConcurrentHashMap<ApplicationId, RMApp>());
  ClientRMService rmService = new ClientRMService(rmContext, null, null,
      null, null, null);
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  GetApplicationReportRequest request = recordFactory
      .newRecordInstance(GetApplicationReportRequest.class);
  request.setApplicationId(ApplicationId.newInstance(0, 0));
  try {
    rmService.getApplicationReport(request);
    Assert.fail();
  } catch (ApplicationNotFoundException ex) {
    Assert.assertEquals(ex.getMessage(),
        "Application with id '" + request.getApplicationId()
            + "' doesn't exist in RM.");
  }
}

Example 11

Source Project: big-c Source File: TestClientRMService.java License: Apache License 2.0

6 votes

@Test
public void testGetApplicationAttemptReport() throws YarnException,
    IOException {
  ClientRMService rmService = createRMService();
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  GetApplicationAttemptReportRequest request = recordFactory
      .newRecordInstance(GetApplicationAttemptReportRequest.class);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
      ApplicationId.newInstance(123456, 1), 1);
  request.setApplicationAttemptId(attemptId);

  try {
    GetApplicationAttemptReportResponse response = rmService
        .getApplicationAttemptReport(request);
    Assert.assertEquals(attemptId, response.getApplicationAttemptReport()
        .getApplicationAttemptId());
  } catch (ApplicationNotFoundException ex) {
    Assert.fail(ex.getMessage());
  }
}

Example 12

Source Project: big-c Source File: TestClientRMService.java License: Apache License 2.0

6 votes

@Test
public void testGetApplicationAttempts() throws YarnException, IOException {
  ClientRMService rmService = createRMService();
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  GetApplicationAttemptsRequest request = recordFactory
      .newRecordInstance(GetApplicationAttemptsRequest.class);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
      ApplicationId.newInstance(123456, 1), 1);
  request.setApplicationId(ApplicationId.newInstance(123456, 1));

  try {
    GetApplicationAttemptsResponse response = rmService
        .getApplicationAttempts(request);
    Assert.assertEquals(1, response.getApplicationAttemptList().size());
    Assert.assertEquals(attemptId, response.getApplicationAttemptList()
        .get(0).getApplicationAttemptId());

  } catch (ApplicationNotFoundException ex) {
    Assert.fail(ex.getMessage());
  }
}

Example 13

Source Project: big-c Source File: TestClientRMService.java License: Apache License 2.0

6 votes

@Test
public void testGetContainerReport() throws YarnException, IOException {
  ClientRMService rmService = createRMService();
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  GetContainerReportRequest request = recordFactory
      .newRecordInstance(GetContainerReportRequest.class);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
      ApplicationId.newInstance(123456, 1), 1);
  ContainerId containerId = ContainerId.newContainerId(attemptId, 1);
  request.setContainerId(containerId);

  try {
    GetContainerReportResponse response = rmService
        .getContainerReport(request);
    Assert.assertEquals(containerId, response.getContainerReport()
        .getContainerId());
  } catch (ApplicationNotFoundException ex) {
    Assert.fail(ex.getMessage());
  }
}

Example 14

Source Project: big-c Source File: TestClientRMService.java License: Apache License 2.0

6 votes

@Test
public void testGetContainers() throws YarnException, IOException {
  ClientRMService rmService = createRMService();
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  GetContainersRequest request = recordFactory
      .newRecordInstance(GetContainersRequest.class);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
      ApplicationId.newInstance(123456, 1), 1);
  ContainerId containerId = ContainerId.newContainerId(attemptId, 1);
  request.setApplicationAttemptId(attemptId);
  try {
    GetContainersResponse response = rmService.getContainers(request);
    Assert.assertEquals(containerId, response.getContainerList().get(0)
        .getContainerId());
  } catch (ApplicationNotFoundException ex) {
    Assert.fail(ex.getMessage());
  }
}

Example 15

Source Project: spliceengine Source File: MiniYARNClusterSplice.java License: GNU Affero General Public License v3.0

6 votes

@Override
protected NodeStatusUpdater createNodeStatusUpdater(Context context,
                                                    Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
    return new NodeStatusUpdaterImpl(context, dispatcher,
                                     healthChecker, metrics) {
        @Override
        protected ResourceTracker getRMClient() {
            final ResourceTrackerService rt =
                getResourceManager().getResourceTrackerService();
            final RecordFactory recordFactory =
                RecordFactoryProvider.getRecordFactory(null);

            // For in-process communication without RPC
            return Utils.getResourceTracker(rt);
        }

        @Override
        protected void stopRMProxy() { }
     };
}

Example 16

Source Project: tez Source File: TestTezLocalCacheManager.java License: Apache License 2.0

6 votes

private static LocalResource createFile(String content) throws IOException {
  FileContext fs = FileContext.getLocalFSFileContext();

  java.nio.file.Path tempFile = Files.createTempFile("test-cache-manager", ".txt");
  File temp = tempFile.toFile();
  temp.deleteOnExit();
  Path p = new Path("file:///" + tempFile.toAbsolutePath().toString());

  Files.write(tempFile, content.getBytes());

  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
  URL yarnUrlFromPath = ConverterUtils.getYarnUrlFromPath(p);
  ret.setResource(yarnUrlFromPath);
  ret.setSize(content.getBytes().length);
  ret.setType(LocalResourceType.FILE);
  ret.setVisibility(LocalResourceVisibility.PRIVATE);
  ret.setTimestamp(fs.getFileStatus(p).getModificationTime());
  return ret;
}

Example 17

Source Project: hadoop Source File: TestYSCRecordFactory.java License: Apache License 2.0

5 votes

@Test
public void testPbRecordFactory() {
  RecordFactory pbRecordFactory = RecordFactoryPBImpl.get();
  try {
    NodeHeartbeatRequest request = pbRecordFactory.newRecordInstance(NodeHeartbeatRequest.class);
    Assert.assertEquals(NodeHeartbeatRequestPBImpl.class, request.getClass());
  } catch (YarnRuntimeException e) {
    e.printStackTrace();
    Assert.fail("Failed to crete record");
  }
  
}

Example 18

Source Project: hadoop Source File: MockApp.java License: Apache License 2.0

5 votes

public MockApp(String user, long clusterTimeStamp, int uniqId) {
  super();
  this.user = user;
  // Add an application and the corresponding containers
  RecordFactory recordFactory = RecordFactoryProvider
      .getRecordFactory(new Configuration());
  this.appId = BuilderUtils.newApplicationId(recordFactory, clusterTimeStamp,
      uniqId);
  appState = ApplicationState.NEW;
}

Example 19

Source Project: hadoop Source File: TestRecordFactory.java License: Apache License 2.0

5 votes

@Test
public void testPbRecordFactory() {
  RecordFactory pbRecordFactory = RecordFactoryPBImpl.get();
  
  try {
    LocalizerHeartbeatResponse response = pbRecordFactory.newRecordInstance(
        LocalizerHeartbeatResponse.class);
    Assert.assertEquals(LocalizerHeartbeatResponsePBImpl.class,
                        response.getClass());
  } catch (YarnRuntimeException e) {
    e.printStackTrace();
    Assert.fail("Failed to crete record");
  }
}

Example 20

Source Project: hadoop Source File: TestAppManager.java License: Apache License 2.0

5 votes

private static ContainerLaunchContext mockContainerLaunchContext(
    RecordFactory recordFactory) {
  ContainerLaunchContext amContainer = recordFactory.newRecordInstance(
      ContainerLaunchContext.class);
  amContainer.setApplicationACLs(new HashMap<ApplicationAccessType, String>());;
  return amContainer;
}

Example 21

Source Project: hadoop Source File: TestUtils.java License: Apache License 2.0

5 votes

public static ResourceRequest createResourceRequest(
    String resourceName, int memory, int numContainers, boolean relaxLocality,
    Priority priority, RecordFactory recordFactory) {
  ResourceRequest request = 
      recordFactory.newRecordInstance(ResourceRequest.class);
  Resource capability = Resources.createResource(memory, 1);
  
  request.setNumContainers(numContainers);
  request.setResourceName(resourceName);
  request.setCapability(capability);
  request.setRelaxLocality(relaxLocality);
  request.setPriority(priority);
  return request;
}

Example 22

Source Project: hadoop Source File: NodeManager.java License: Apache License 2.0

5 votes

public static org.apache.hadoop.yarn.server.api.records.NodeStatus 
createNodeStatus(NodeId nodeId, List<ContainerStatus> containers) {
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  org.apache.hadoop.yarn.server.api.records.NodeStatus nodeStatus = 
      recordFactory.newRecordInstance(org.apache.hadoop.yarn.server.api.records.NodeStatus.class);
  nodeStatus.setNodeId(nodeId);
  nodeStatus.setContainersStatuses(containers);
  NodeHealthStatus nodeHealthStatus = 
    recordFactory.newRecordInstance(NodeHealthStatus.class);
  nodeHealthStatus.setIsNodeHealthy(true);
  nodeStatus.setNodeHealthStatus(nodeHealthStatus);
  return nodeStatus;
}

Example 23

Source Project: hadoop Source File: TestClientRMService.java License: Apache License 2.0

5 votes

@Test
public void testGetApplicationReport() throws Exception {
  YarnScheduler yarnScheduler = mock(YarnScheduler.class);
  RMContext rmContext = mock(RMContext.class);
  mockRMContext(yarnScheduler, rmContext);

  ApplicationId appId1 = getApplicationId(1);

  ApplicationACLsManager mockAclsManager = mock(ApplicationACLsManager.class);
  when(
      mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(),
          ApplicationAccessType.VIEW_APP, null, appId1)).thenReturn(true);

  ClientRMService rmService = new ClientRMService(rmContext, yarnScheduler,
      null, mockAclsManager, null, null);
  try {
    RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
    GetApplicationReportRequest request = recordFactory
        .newRecordInstance(GetApplicationReportRequest.class);
    request.setApplicationId(appId1);
    GetApplicationReportResponse response = 
        rmService.getApplicationReport(request);
    ApplicationReport report = response.getApplicationReport();
    ApplicationResourceUsageReport usageReport = 
        report.getApplicationResourceUsageReport();
    Assert.assertEquals(10, usageReport.getMemorySeconds());
    Assert.assertEquals(3, usageReport.getVcoreSeconds());
    Assert.assertEquals(3, usageReport.getGcoreSeconds());
  } finally {
    rmService.close();
  }
}

Example 24

Source Project: hadoop Source File: RecordFactoryProvider.java License: Apache License 2.0

5 votes

public static RecordFactory getRecordFactory(Configuration conf) {
  if (conf == null) {
    //Assuming the default configuration has the correct factories set.
    //Users can specify a particular factory by providing a configuration.
    conf = defaultConf;
  }
  String recordFactoryClassName = conf.get(
      YarnConfiguration.IPC_RECORD_FACTORY_CLASS,
      YarnConfiguration.DEFAULT_IPC_RECORD_FACTORY_CLASS);
  return (RecordFactory) getFactoryClassInstance(recordFactoryClassName);
}

Example 25

Source Project: hadoop Source File: TestTaskAttemptListenerImpl.java License: Apache License 2.0

5 votes

private static TaskAttemptCompletionEvent createTce(int eventId,
    boolean isMap, TaskAttemptCompletionEventStatus status) {
  JobId jid = MRBuilderUtils.newJobId(12345, 1, 1);
  TaskId tid = MRBuilderUtils.newTaskId(jid, 0,
      isMap ? org.apache.hadoop.mapreduce.v2.api.records.TaskType.MAP
          : org.apache.hadoop.mapreduce.v2.api.records.TaskType.REDUCE);
  TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(tid, 0);
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  TaskAttemptCompletionEvent tce = recordFactory
      .newRecordInstance(TaskAttemptCompletionEvent.class);
  tce.setEventId(eventId);
  tce.setAttemptId(attemptId);
  tce.setStatus(status);
  return tce;
}

Example 26

Source Project: big-c Source File: TestYSCRecordFactory.java License: Apache License 2.0

5 votes

@Test
public void testPbRecordFactory() {
  RecordFactory pbRecordFactory = RecordFactoryPBImpl.get();
  try {
    NodeHeartbeatRequest request = pbRecordFactory.newRecordInstance(NodeHeartbeatRequest.class);
    Assert.assertEquals(NodeHeartbeatRequestPBImpl.class, request.getClass());
  } catch (YarnRuntimeException e) {
    e.printStackTrace();
    Assert.fail("Failed to crete record");
  }
  
}

Example 27

Source Project: big-c Source File: MockApp.java License: Apache License 2.0

5 votes

public MockApp(String user, long clusterTimeStamp, int uniqId) {
  super();
  this.user = user;
  // Add an application and the corresponding containers
  RecordFactory recordFactory = RecordFactoryProvider
      .getRecordFactory(new Configuration());
  this.appId = BuilderUtils.newApplicationId(recordFactory, clusterTimeStamp,
      uniqId);
  appState = ApplicationState.NEW;
}

Example 28

Source Project: big-c Source File: TestRecordFactory.java License: Apache License 2.0

5 votes

@Test
public void testPbRecordFactory() {
  RecordFactory pbRecordFactory = RecordFactoryPBImpl.get();
  
  try {
    LocalizerHeartbeatResponse response = pbRecordFactory.newRecordInstance(
        LocalizerHeartbeatResponse.class);
    Assert.assertEquals(LocalizerHeartbeatResponsePBImpl.class,
                        response.getClass());
  } catch (YarnRuntimeException e) {
    e.printStackTrace();
    Assert.fail("Failed to crete record");
  }
}

Example 29

Source Project: big-c Source File: TestAppManager.java License: Apache License 2.0

5 votes

private static ContainerLaunchContext mockContainerLaunchContext(
    RecordFactory recordFactory) {
  ContainerLaunchContext amContainer = recordFactory.newRecordInstance(
      ContainerLaunchContext.class);
  amContainer.setApplicationACLs(new HashMap<ApplicationAccessType, String>());;
  return amContainer;
}

Example 30

Source Project: big-c Source File: TestUtils.java License: Apache License 2.0

5 votes

public static ResourceRequest createResourceRequest(
    String resourceName, int memory, int numContainers, boolean relaxLocality,
    Priority priority, RecordFactory recordFactory) {
  ResourceRequest request = 
      recordFactory.newRecordInstance(ResourceRequest.class);
  Resource capability = Resources.createResource(memory, 1);
  
  request.setNumContainers(numContainers);
  request.setResourceName(resourceName);
  request.setCapability(capability);
  request.setRelaxLocality(relaxLocality);
  request.setPriority(priority);
  return request;
}