Java Code Examples for org.apache.hadoop.yarn.server.utils.BuilderUtils#newContainer()
The following examples show how to use
org.apache.hadoop.yarn.server.utils.BuilderUtils#newContainer() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: FSAppAttempt.java From hadoop with Apache License 2.0 | 5 votes |
/** * Create and return a container object reflecting an allocation for the * given appliction on the given node with the given capability and * priority. */ public Container createContainer( FSSchedulerNode node, Resource capability, Priority priority) { NodeId nodeId = node.getRMNode().getNodeID(); ContainerId containerId = BuilderUtils.newContainerId( getApplicationAttemptId(), getNewContainerId()); // Create the container Container container = BuilderUtils.newContainer(containerId, nodeId, node.getRMNode() .getHttpAddress(), capability, priority, null); return container; }
Example 2
Source File: LeafQueue.java From hadoop with Apache License 2.0 | 5 votes |
Container createContainer(FiCaSchedulerApp application, FiCaSchedulerNode node, Resource capability, Priority priority) { NodeId nodeId = node.getRMNode().getNodeID(); ContainerId containerId = BuilderUtils.newContainerId(application .getApplicationAttemptId(), application.getNewContainerId()); // Create the container Container container = BuilderUtils.newContainer(containerId, nodeId, node.getRMNode() .getHttpAddress(), capability, priority, null); return container; }
Example 3
Source File: FSAppAttempt.java From big-c with Apache License 2.0 | 5 votes |
/** * Create and return a container object reflecting an allocation for the * given appliction on the given node with the given capability and * priority. */ public Container createContainer( FSSchedulerNode node, Resource capability, Priority priority) { NodeId nodeId = node.getRMNode().getNodeID(); ContainerId containerId = BuilderUtils.newContainerId( getApplicationAttemptId(), getNewContainerId()); // Create the container Container container = BuilderUtils.newContainer(containerId, nodeId, node.getRMNode() .getHttpAddress(), capability, priority, null); return container; }
Example 4
Source File: LeafQueue.java From big-c with Apache License 2.0 | 5 votes |
Container createContainer(FiCaSchedulerApp application, FiCaSchedulerNode node, Resource capability, Priority priority) { NodeId nodeId = node.getRMNode().getNodeID(); ContainerId containerId = BuilderUtils.newContainerId(application .getApplicationAttemptId(), application.getNewContainerId()); // Create the container Container container = BuilderUtils.newContainer(containerId, nodeId, node.getRMNode() .getHttpAddress(), capability, priority, null); return container; }
Example 5
Source File: FifoScheduler.java From hadoop with Apache License 2.0 | 4 votes |
private int assignContainer(FiCaSchedulerNode node, FiCaSchedulerApp application, Priority priority, int assignableContainers, ResourceRequest request, NodeType type) { LOG.debug("assignContainers:" + " node=" + node.getRMNode().getNodeAddress() + " application=" + application.getApplicationId().getId() + " priority=" + priority.getPriority() + " assignableContainers=" + assignableContainers + " request=" + request + " type=" + type); Resource capability = request.getCapability(); int availableContainers = node.getAvailableResource().getMemory() / capability.getMemory(); // TODO: A buggy // application // with this // zero would // crash the // scheduler. int assignedContainers = Math.min(assignableContainers, availableContainers); if (assignedContainers > 0) { for (int i=0; i < assignedContainers; ++i) { NodeId nodeId = node.getRMNode().getNodeID(); ContainerId containerId = BuilderUtils.newContainerId(application .getApplicationAttemptId(), application.getNewContainerId()); // Create the container Container container = BuilderUtils.newContainer(containerId, nodeId, node.getRMNode() .getHttpAddress(), capability, priority, null); // Allocate! // Inform the application RMContainer rmContainer = application.allocate(type, node, priority, request, container); // Inform the node node.allocateContainer(rmContainer); // Update usage for this container increaseUsedResources(rmContainer); } } return assignedContainers; }
Example 6
Source File: TestRMContainerImpl.java From hadoop with Apache License 2.0 | 4 votes |
@Test public void testReleaseWhileRunning() { DrainDispatcher drainDispatcher = new DrainDispatcher(); EventHandler<RMAppAttemptEvent> appAttemptEventHandler = mock(EventHandler.class); EventHandler generic = mock(EventHandler.class); drainDispatcher.register(RMAppAttemptEventType.class, appAttemptEventHandler); drainDispatcher.register(RMNodeEventType.class, generic); drainDispatcher.init(new YarnConfiguration()); drainDispatcher.start(); NodeId nodeId = BuilderUtils.newNodeId("host", 3425); ApplicationId appId = BuilderUtils.newApplicationId(1, 1); ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId( appId, 1); ContainerId containerId = BuilderUtils.newContainerId(appAttemptId, 1); ContainerAllocationExpirer expirer = mock(ContainerAllocationExpirer.class); Resource resource = BuilderUtils.newResource(512, 1, 1); Priority priority = BuilderUtils.newPriority(5); Container container = BuilderUtils.newContainer(containerId, nodeId, "host:3465", resource, priority, null); ConcurrentMap<ApplicationId, RMApp> rmApps = spy(new ConcurrentHashMap<ApplicationId, RMApp>()); RMApp rmApp = mock(RMApp.class); when(rmApp.getRMAppAttempt((ApplicationAttemptId)Matchers.any())).thenReturn(null); Mockito.doReturn(rmApp).when(rmApps).get((ApplicationId)Matchers.any()); RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class); SystemMetricsPublisher publisher = mock(SystemMetricsPublisher.class); RMContext rmContext = mock(RMContext.class); when(rmContext.getDispatcher()).thenReturn(drainDispatcher); when(rmContext.getContainerAllocationExpirer()).thenReturn(expirer); when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer); when(rmContext.getRMApps()).thenReturn(rmApps); when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher); when(rmContext.getYarnConfiguration()).thenReturn(new YarnConfiguration()); RMContainer rmContainer = new RMContainerImpl(container, appAttemptId, nodeId, "user", rmContext); assertEquals(RMContainerState.NEW, rmContainer.getState()); assertEquals(resource, rmContainer.getAllocatedResource()); assertEquals(nodeId, rmContainer.getAllocatedNode()); assertEquals(priority, rmContainer.getAllocatedPriority()); verify(writer).containerStarted(any(RMContainer.class)); verify(publisher).containerCreated(any(RMContainer.class), anyLong()); rmContainer.handle(new RMContainerEvent(containerId, RMContainerEventType.START)); drainDispatcher.await(); assertEquals(RMContainerState.ALLOCATED, rmContainer.getState()); rmContainer.handle(new RMContainerEvent(containerId, RMContainerEventType.ACQUIRED)); drainDispatcher.await(); assertEquals(RMContainerState.ACQUIRED, rmContainer.getState()); rmContainer.handle(new RMContainerEvent(containerId, RMContainerEventType.LAUNCHED)); drainDispatcher.await(); assertEquals(RMContainerState.RUNNING, rmContainer.getState()); assertEquals("http://host:3465/node/containerlogs/container_1_0001_01_000001/user", rmContainer.getLogURL()); // In RUNNING state. Verify RELEASED and associated actions. reset(appAttemptEventHandler); ContainerStatus containerStatus = SchedulerUtils .createAbnormalContainerStatus(containerId, SchedulerUtils.RELEASED_CONTAINER); rmContainer.handle(new RMContainerFinishedEvent(containerId, containerStatus, RMContainerEventType.RELEASED)); drainDispatcher.await(); assertEquals(RMContainerState.RELEASED, rmContainer.getState()); assertEquals(SchedulerUtils.RELEASED_CONTAINER, rmContainer.getDiagnosticsInfo()); assertEquals(ContainerExitStatus.ABORTED, rmContainer.getContainerExitStatus()); assertEquals(ContainerState.COMPLETE, rmContainer.getContainerState()); verify(writer).containerFinished(any(RMContainer.class)); verify(publisher).containerFinished(any(RMContainer.class), anyLong()); ArgumentCaptor<RMAppAttemptContainerFinishedEvent> captor = ArgumentCaptor .forClass(RMAppAttemptContainerFinishedEvent.class); verify(appAttemptEventHandler).handle(captor.capture()); RMAppAttemptContainerFinishedEvent cfEvent = captor.getValue(); assertEquals(appAttemptId, cfEvent.getApplicationAttemptId()); assertEquals(containerStatus, cfEvent.getContainerStatus()); assertEquals(RMAppAttemptEventType.CONTAINER_FINISHED, cfEvent.getType()); // In RELEASED state. A FINIHSED event may come in. rmContainer.handle(new RMContainerFinishedEvent(containerId, SchedulerUtils .createAbnormalContainerStatus(containerId, "FinishedContainer"), RMContainerEventType.FINISHED)); assertEquals(RMContainerState.RELEASED, rmContainer.getState()); }
Example 7
Source File: TestRMContainerImpl.java From hadoop with Apache License 2.0 | 4 votes |
@Test public void testExpireWhileRunning() { DrainDispatcher drainDispatcher = new DrainDispatcher(); EventHandler<RMAppAttemptEvent> appAttemptEventHandler = mock(EventHandler.class); EventHandler generic = mock(EventHandler.class); drainDispatcher.register(RMAppAttemptEventType.class, appAttemptEventHandler); drainDispatcher.register(RMNodeEventType.class, generic); drainDispatcher.init(new YarnConfiguration()); drainDispatcher.start(); NodeId nodeId = BuilderUtils.newNodeId("host", 3425); ApplicationId appId = BuilderUtils.newApplicationId(1, 1); ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId( appId, 1); ContainerId containerId = BuilderUtils.newContainerId(appAttemptId, 1); ContainerAllocationExpirer expirer = mock(ContainerAllocationExpirer.class); Resource resource = BuilderUtils.newResource(512, 1, 1); Priority priority = BuilderUtils.newPriority(5); Container container = BuilderUtils.newContainer(containerId, nodeId, "host:3465", resource, priority, null); RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class); SystemMetricsPublisher publisher = mock(SystemMetricsPublisher.class); RMContext rmContext = mock(RMContext.class); when(rmContext.getDispatcher()).thenReturn(drainDispatcher); when(rmContext.getContainerAllocationExpirer()).thenReturn(expirer); when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer); when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher); when(rmContext.getYarnConfiguration()).thenReturn(new YarnConfiguration()); RMContainer rmContainer = new RMContainerImpl(container, appAttemptId, nodeId, "user", rmContext); assertEquals(RMContainerState.NEW, rmContainer.getState()); assertEquals(resource, rmContainer.getAllocatedResource()); assertEquals(nodeId, rmContainer.getAllocatedNode()); assertEquals(priority, rmContainer.getAllocatedPriority()); verify(writer).containerStarted(any(RMContainer.class)); verify(publisher).containerCreated(any(RMContainer.class), anyLong()); rmContainer.handle(new RMContainerEvent(containerId, RMContainerEventType.START)); drainDispatcher.await(); assertEquals(RMContainerState.ALLOCATED, rmContainer.getState()); rmContainer.handle(new RMContainerEvent(containerId, RMContainerEventType.ACQUIRED)); drainDispatcher.await(); assertEquals(RMContainerState.ACQUIRED, rmContainer.getState()); rmContainer.handle(new RMContainerEvent(containerId, RMContainerEventType.LAUNCHED)); drainDispatcher.await(); assertEquals(RMContainerState.RUNNING, rmContainer.getState()); assertEquals("http://host:3465/node/containerlogs/container_1_0001_01_000001/user", rmContainer.getLogURL()); // In RUNNING state. Verify EXPIRE and associated actions. reset(appAttemptEventHandler); ContainerStatus containerStatus = SchedulerUtils .createAbnormalContainerStatus(containerId, SchedulerUtils.EXPIRED_CONTAINER); rmContainer.handle(new RMContainerFinishedEvent(containerId, containerStatus, RMContainerEventType.EXPIRE)); drainDispatcher.await(); assertEquals(RMContainerState.RUNNING, rmContainer.getState()); verify(writer, never()).containerFinished(any(RMContainer.class)); verify(publisher, never()).containerFinished(any(RMContainer.class), anyLong()); }
Example 8
Source File: FifoScheduler.java From big-c with Apache License 2.0 | 4 votes |
private int assignContainer(FiCaSchedulerNode node, FiCaSchedulerApp application, Priority priority, int assignableContainers, ResourceRequest request, NodeType type) { LOG.debug("assignContainers:" + " node=" + node.getRMNode().getNodeAddress() + " application=" + application.getApplicationId().getId() + " priority=" + priority.getPriority() + " assignableContainers=" + assignableContainers + " request=" + request + " type=" + type); Resource capability = request.getCapability(); int availableContainers = node.getAvailableResource().getMemory() / capability.getMemory(); // TODO: A buggy // application // with this // zero would // crash the // scheduler. int assignedContainers = Math.min(assignableContainers, availableContainers); if (assignedContainers > 0) { for (int i=0; i < assignedContainers; ++i) { NodeId nodeId = node.getRMNode().getNodeID(); ContainerId containerId = BuilderUtils.newContainerId(application .getApplicationAttemptId(), application.getNewContainerId()); // Create the container Container container = BuilderUtils.newContainer(containerId, nodeId, node.getRMNode() .getHttpAddress(), capability, priority, null); // Allocate! // Inform the application RMContainer rmContainer = application.allocate(type, node, priority, request, container); // Inform the node node.allocateContainer(rmContainer); // Update usage for this container increaseUsedResources(rmContainer); } } return assignedContainers; }
Example 9
Source File: TestRMContainerImpl.java From big-c with Apache License 2.0 | 4 votes |
@Test public void testReleaseWhileRunning() { DrainDispatcher drainDispatcher = new DrainDispatcher(); EventHandler<RMAppAttemptEvent> appAttemptEventHandler = mock(EventHandler.class); EventHandler generic = mock(EventHandler.class); drainDispatcher.register(RMAppAttemptEventType.class, appAttemptEventHandler); drainDispatcher.register(RMNodeEventType.class, generic); drainDispatcher.init(new YarnConfiguration()); drainDispatcher.start(); NodeId nodeId = BuilderUtils.newNodeId("host", 3425); ApplicationId appId = BuilderUtils.newApplicationId(1, 1); ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId( appId, 1); ContainerId containerId = BuilderUtils.newContainerId(appAttemptId, 1); ContainerAllocationExpirer expirer = mock(ContainerAllocationExpirer.class); Resource resource = BuilderUtils.newResource(512, 1); Priority priority = BuilderUtils.newPriority(5); Container container = BuilderUtils.newContainer(containerId, nodeId, "host:3465", resource, priority, null); ConcurrentMap<ApplicationId, RMApp> rmApps = spy(new ConcurrentHashMap<ApplicationId, RMApp>()); RMApp rmApp = mock(RMApp.class); when(rmApp.getRMAppAttempt((ApplicationAttemptId)Matchers.any())).thenReturn(null); Mockito.doReturn(rmApp).when(rmApps).get((ApplicationId)Matchers.any()); RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class); SystemMetricsPublisher publisher = mock(SystemMetricsPublisher.class); RMContext rmContext = mock(RMContext.class); when(rmContext.getDispatcher()).thenReturn(drainDispatcher); when(rmContext.getContainerAllocationExpirer()).thenReturn(expirer); when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer); when(rmContext.getRMApps()).thenReturn(rmApps); when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher); when(rmContext.getYarnConfiguration()).thenReturn(new YarnConfiguration()); RMContainer rmContainer = new RMContainerImpl(container, appAttemptId, nodeId, "user", rmContext); assertEquals(RMContainerState.NEW, rmContainer.getState()); assertEquals(resource, rmContainer.getAllocatedResource()); assertEquals(nodeId, rmContainer.getAllocatedNode()); assertEquals(priority, rmContainer.getAllocatedPriority()); verify(writer).containerStarted(any(RMContainer.class)); verify(publisher).containerCreated(any(RMContainer.class), anyLong()); rmContainer.handle(new RMContainerEvent(containerId, RMContainerEventType.START)); drainDispatcher.await(); assertEquals(RMContainerState.ALLOCATED, rmContainer.getState()); rmContainer.handle(new RMContainerEvent(containerId, RMContainerEventType.ACQUIRED)); drainDispatcher.await(); assertEquals(RMContainerState.ACQUIRED, rmContainer.getState()); rmContainer.handle(new RMContainerEvent(containerId, RMContainerEventType.LAUNCHED)); drainDispatcher.await(); assertEquals(RMContainerState.RUNNING, rmContainer.getState()); assertEquals("http://host:3465/node/containerlogs/container_1_0001_01_000001/user", rmContainer.getLogURL()); // In RUNNING state. Verify RELEASED and associated actions. reset(appAttemptEventHandler); ContainerStatus containerStatus = SchedulerUtils .createAbnormalContainerStatus(containerId, SchedulerUtils.RELEASED_CONTAINER); rmContainer.handle(new RMContainerFinishedEvent(containerId, containerStatus, RMContainerEventType.RELEASED)); drainDispatcher.await(); assertEquals(RMContainerState.RELEASED, rmContainer.getState()); assertEquals(SchedulerUtils.RELEASED_CONTAINER, rmContainer.getDiagnosticsInfo()); assertEquals(ContainerExitStatus.ABORTED, rmContainer.getContainerExitStatus()); assertEquals(ContainerState.COMPLETE, rmContainer.getContainerState()); verify(writer).containerFinished(any(RMContainer.class)); verify(publisher).containerFinished(any(RMContainer.class), anyLong()); ArgumentCaptor<RMAppAttemptContainerFinishedEvent> captor = ArgumentCaptor .forClass(RMAppAttemptContainerFinishedEvent.class); verify(appAttemptEventHandler).handle(captor.capture()); RMAppAttemptContainerFinishedEvent cfEvent = captor.getValue(); assertEquals(appAttemptId, cfEvent.getApplicationAttemptId()); assertEquals(containerStatus, cfEvent.getContainerStatus()); assertEquals(RMAppAttemptEventType.CONTAINER_FINISHED, cfEvent.getType()); // In RELEASED state. A FINIHSED event may come in. rmContainer.handle(new RMContainerFinishedEvent(containerId, SchedulerUtils .createAbnormalContainerStatus(containerId, "FinishedContainer"), RMContainerEventType.FINISHED)); assertEquals(RMContainerState.RELEASED, rmContainer.getState()); }
Example 10
Source File: TestRMContainerImpl.java From big-c with Apache License 2.0 | 4 votes |
@Test public void testExpireWhileRunning() { DrainDispatcher drainDispatcher = new DrainDispatcher(); EventHandler<RMAppAttemptEvent> appAttemptEventHandler = mock(EventHandler.class); EventHandler generic = mock(EventHandler.class); drainDispatcher.register(RMAppAttemptEventType.class, appAttemptEventHandler); drainDispatcher.register(RMNodeEventType.class, generic); drainDispatcher.init(new YarnConfiguration()); drainDispatcher.start(); NodeId nodeId = BuilderUtils.newNodeId("host", 3425); ApplicationId appId = BuilderUtils.newApplicationId(1, 1); ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId( appId, 1); ContainerId containerId = BuilderUtils.newContainerId(appAttemptId, 1); ContainerAllocationExpirer expirer = mock(ContainerAllocationExpirer.class); Resource resource = BuilderUtils.newResource(512, 1); Priority priority = BuilderUtils.newPriority(5); Container container = BuilderUtils.newContainer(containerId, nodeId, "host:3465", resource, priority, null); RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class); SystemMetricsPublisher publisher = mock(SystemMetricsPublisher.class); RMContext rmContext = mock(RMContext.class); when(rmContext.getDispatcher()).thenReturn(drainDispatcher); when(rmContext.getContainerAllocationExpirer()).thenReturn(expirer); when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer); when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher); when(rmContext.getYarnConfiguration()).thenReturn(new YarnConfiguration()); RMContainer rmContainer = new RMContainerImpl(container, appAttemptId, nodeId, "user", rmContext); assertEquals(RMContainerState.NEW, rmContainer.getState()); assertEquals(resource, rmContainer.getAllocatedResource()); assertEquals(nodeId, rmContainer.getAllocatedNode()); assertEquals(priority, rmContainer.getAllocatedPriority()); verify(writer).containerStarted(any(RMContainer.class)); verify(publisher).containerCreated(any(RMContainer.class), anyLong()); rmContainer.handle(new RMContainerEvent(containerId, RMContainerEventType.START)); drainDispatcher.await(); assertEquals(RMContainerState.ALLOCATED, rmContainer.getState()); rmContainer.handle(new RMContainerEvent(containerId, RMContainerEventType.ACQUIRED)); drainDispatcher.await(); assertEquals(RMContainerState.ACQUIRED, rmContainer.getState()); rmContainer.handle(new RMContainerEvent(containerId, RMContainerEventType.LAUNCHED)); drainDispatcher.await(); assertEquals(RMContainerState.RUNNING, rmContainer.getState()); assertEquals("http://host:3465/node/containerlogs/container_1_0001_01_000001/user", rmContainer.getLogURL()); // In RUNNING state. Verify EXPIRE and associated actions. reset(appAttemptEventHandler); ContainerStatus containerStatus = SchedulerUtils .createAbnormalContainerStatus(containerId, SchedulerUtils.EXPIRED_CONTAINER); rmContainer.handle(new RMContainerFinishedEvent(containerId, containerStatus, RMContainerEventType.EXPIRE)); drainDispatcher.await(); assertEquals(RMContainerState.RUNNING, rmContainer.getState()); verify(writer, never()).containerFinished(any(RMContainer.class)); verify(publisher, never()).containerFinished(any(RMContainer.class), anyLong()); }