org.apache.hadoop.yarn.api.records.ResourceRequest Java Examples
The following examples show how to use
org.apache.hadoop.yarn.api.records.ResourceRequest.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: SchedulerUtils.java From big-c with Apache License 2.0 | 6 votes |
private static void normalizeNodeLabelExpressionInRequest( ResourceRequest resReq, QueueInfo queueInfo) { String labelExp = resReq.getNodeLabelExpression(); // if queue has default label expression, and RR doesn't have, use the // default label expression of queue if (labelExp == null && queueInfo != null && ResourceRequest.ANY .equals(resReq.getResourceName())) { labelExp = queueInfo.getDefaultNodeLabelExpression(); } // If labelExp still equals to null, set it to be NO_LABEL if (labelExp == null) { labelExp = RMNodeLabelsManager.NO_LABEL; } resReq.setNodeLabelExpression(labelExp); }
Example #2
Source File: RMAppAttemptBlock.java From hadoop with Apache License 2.0 | 6 votes |
private Resource getTotalResource(List<ResourceRequest> requests) { Resource totalResource = Resource.newInstance(0, 0, 0); if (requests == null) { return totalResource; } for (ResourceRequest request : requests) { if (request.getNumContainers() == 0) { continue; } if (request.getResourceName().equals(ResourceRequest.ANY)) { Resources.addTo( totalResource, Resources.multiply(request.getCapability(), request.getNumContainers())); } } return totalResource; }
Example #3
Source File: MockAM.java From hadoop with Apache License 2.0 | 6 votes |
public List<ResourceRequest> createReq(String[] hosts, int memory, int priority, int containers, String labelExpression) throws Exception { List<ResourceRequest> reqs = new ArrayList<ResourceRequest>(); if (hosts != null) { for (String host : hosts) { // only add host/rack request when asked host isn't ANY if (!host.equals(ResourceRequest.ANY)) { ResourceRequest hostReq = createResourceReq(host, memory, priority, containers, labelExpression); reqs.add(hostReq); ResourceRequest rackReq = createResourceReq("/default-rack", memory, priority, containers, labelExpression); reqs.add(rackReq); } } } ResourceRequest offRackReq = createResourceReq(ResourceRequest.ANY, memory, priority, containers, labelExpression); reqs.add(offRackReq); return reqs; }
Example #4
Source File: FSAppAttempt.java From hadoop with Apache License 2.0 | 6 votes |
@Override public void updateDemand() { demand = Resources.createResource(0); // Demand is current consumption plus outstanding requests Resources.addTo(demand, getCurrentConsumption()); // Add up outstanding resource requests synchronized (this) { for (Priority p : getPriorities()) { for (ResourceRequest r : getResourceRequests(p).values()) { Resource total = Resources.multiply(r.getCapability(), r.getNumContainers()); Resources.addTo(demand, total); } } } }
Example #5
Source File: SchedulerUtils.java From hadoop with Apache License 2.0 | 6 votes |
public static void normalizeAndValidateRequest(ResourceRequest resReq, Resource maximumResource, String queueName, YarnScheduler scheduler, boolean isRecovery, RMContext rmContext, QueueInfo queueInfo) throws InvalidResourceRequestException { if (queueInfo == null) { try { queueInfo = scheduler.getQueueInfo(queueName, false, false); } catch (IOException e) { // it is possible queue cannot get when queue mapping is set, just ignore // the queueInfo here, and move forward } } SchedulerUtils.normalizeNodeLabelExpressionInRequest(resReq, queueInfo); if (!isRecovery) { validateResourceRequest(resReq, maximumResource, queueInfo, rmContext); } }
Example #6
Source File: Task.java From hadoop with Apache License 2.0 | 6 votes |
public Task(Application application, Priority priority, String[] hosts) { this.applicationId = application.getApplicationId(); this.priority = priority; taskId = application.getNextTaskId(); state = State.PENDING; // Special case: Don't care about locality if (!(hosts.length == 1 && hosts[0].equals(ResourceRequest.ANY))) { for (String host : hosts) { this.hosts.add(host); this.racks.add(Application.resolve(host)); } } LOG.info("Task " + taskId + " added to application " + this.applicationId + " with " + this.hosts.size() + " hosts, " + racks.size() + " racks"); }
Example #7
Source File: FSAppAttempt.java From big-c with Apache License 2.0 | 6 votes |
/** * Whether this app has containers requests that could be satisfied on the * given node, if the node had full space. */ public boolean hasContainerForNode(Priority prio, FSSchedulerNode node) { ResourceRequest anyRequest = getResourceRequest(prio, ResourceRequest.ANY); ResourceRequest rackRequest = getResourceRequest(prio, node.getRackName()); ResourceRequest nodeRequest = getResourceRequest(prio, node.getNodeName()); return // There must be outstanding requests at the given priority: anyRequest != null && anyRequest.getNumContainers() > 0 && // If locality relaxation is turned off at *-level, there must be a // non-zero request for the node's rack: (anyRequest.getRelaxLocality() || (rackRequest != null && rackRequest.getNumContainers() > 0)) && // If locality relaxation is turned off at rack-level, there must be a // non-zero request at the node: (rackRequest == null || rackRequest.getRelaxLocality() || (nodeRequest != null && nodeRequest.getNumContainers() > 0)) && // The requested container must be able to fit on the node: Resources.lessThanOrEqual(RESOURCE_CALCULATOR, null, anyRequest.getCapability(), node.getRMNode().getTotalCapability()); }
Example #8
Source File: MockAM.java From hadoop with Apache License 2.0 | 6 votes |
public List<Container> allocateAndWaitForContainers(int nContainer, int memory, MockNM nm) throws Exception { // AM request for containers allocate("ANY", memory, nContainer, null); // kick the scheduler nm.nodeHeartbeat(true); List<Container> conts = allocate(new ArrayList<ResourceRequest>(), null) .getAllocatedContainers(); while (conts.size() < nContainer) { nm.nodeHeartbeat(true); conts.addAll(allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>()).getAllocatedContainers()); Thread.sleep(500); } return conts; }
Example #9
Source File: AllocateRequest.java From big-c with Apache License 2.0 | 6 votes |
@Public @Stable public static AllocateRequest newInstance(int responseID, float appProgress, List<ResourceRequest> resourceAsk, List<ContainerId> containersToBeReleased, ResourceBlacklistRequest resourceBlacklistRequest, List<ContainerResourceIncreaseRequest> increaseRequests) { AllocateRequest allocateRequest = Records.newRecord(AllocateRequest.class); allocateRequest.setResponseId(responseID); allocateRequest.setProgress(appProgress); allocateRequest.setAskList(resourceAsk); allocateRequest.setReleaseList(containersToBeReleased); allocateRequest.setResourceBlacklistRequest(resourceBlacklistRequest); allocateRequest.setIncreaseRequests(increaseRequests); return allocateRequest; }
Example #10
Source File: TestContainerAllocation.java From big-c with Apache License 2.0 | 6 votes |
private LogAggregationContext getLogAggregationContextFromContainerToken( MockRM rm1, MockNM nm1, LogAggregationContext logAggregationContext) throws Exception { RMApp app2 = rm1.submitApp(200, logAggregationContext); MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1); nm1.nodeHeartbeat(true); // request a container. am2.allocate("127.0.0.1", 512, 1, new ArrayList<ContainerId>()); ContainerId containerId = ContainerId.newContainerId(am2.getApplicationAttemptId(), 2); rm1.waitForState(nm1, containerId, RMContainerState.ALLOCATED); // acquire the container. List<Container> containers = am2.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>()).getAllocatedContainers(); Assert.assertEquals(containerId, containers.get(0).getId()); // container token is generated. Assert.assertNotNull(containers.get(0).getContainerToken()); ContainerTokenIdentifier token = BuilderUtils.newContainerTokenIdentifier(containers.get(0) .getContainerToken()); return token.getLogAggregationContext(); }
Example #11
Source File: LeafQueue.java From hadoop with Apache License 2.0 | 5 votes |
private Resource assignNodeLocalContainers(Resource clusterResource, ResourceRequest nodeLocalResourceRequest, FiCaSchedulerNode node, FiCaSchedulerApp application, Priority priority, RMContainer reservedContainer, MutableObject allocatedContainer, ResourceLimits currentResoureLimits) { if (canAssign(application, priority, node, NodeType.NODE_LOCAL, reservedContainer)) { return assignContainer(clusterResource, node, application, priority, nodeLocalResourceRequest, NodeType.NODE_LOCAL, reservedContainer, allocatedContainer, currentResoureLimits); } return Resources.none(); }
Example #12
Source File: SchedulerUtils.java From hadoop with Apache License 2.0 | 5 votes |
public static void normalizeAndValidateRequest(ResourceRequest resReq, Resource maximumResource, String queueName, YarnScheduler scheduler, boolean isRecovery, RMContext rmContext) throws InvalidResourceRequestException { normalizeAndValidateRequest(resReq, maximumResource, queueName, scheduler, isRecovery, rmContext, null); }
Example #13
Source File: AppSchedulingInfo.java From big-c with Apache License 2.0 | 5 votes |
synchronized private void decrementOutstanding( ResourceRequest offSwitchRequest) { int numOffSwitchContainers = offSwitchRequest.getNumContainers() - 1; // Do not remove ANY offSwitchRequest.setNumContainers(numOffSwitchContainers); // Do we have any outstanding requests? // If there is nothing, we need to deactivate this application if (numOffSwitchContainers == 0) { checkForDeactivation(); } }
Example #14
Source File: TestContainerAllocation.java From hadoop with Apache License 2.0 | 5 votes |
@Test public void testContainerTokenGeneratedOnPullRequest() throws Exception { MockRM rm1 = new MockRM(conf); rm1.start(); MockNM nm1 = rm1.registerNode("127.0.0.1:1234", 8000); RMApp app1 = rm1.submitApp(200); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // request a container. am1.allocate("127.0.0.1", 1024, 1, new ArrayList<ContainerId>()); ContainerId containerId2 = ContainerId.newContainerId(am1.getApplicationAttemptId(), 2); rm1.waitForState(nm1, containerId2, RMContainerState.ALLOCATED); RMContainer container = rm1.getResourceScheduler().getRMContainer(containerId2); // no container token is generated. Assert.assertEquals(containerId2, container.getContainerId()); Assert.assertNull(container.getContainer().getContainerToken()); // acquire the container. List<Container> containers = am1.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>()).getAllocatedContainers(); Assert.assertEquals(containerId2, containers.get(0).getId()); // container token is generated. Assert.assertNotNull(containers.get(0).getContainerToken()); rm1.stop(); }
Example #15
Source File: ApplicationSubmissionContextPBImpl.java From hadoop with Apache License 2.0 | 5 votes |
@Override public void setAMContainerResourceRequest(ResourceRequest request) { maybeInitBuilder(); if (request == null) { builder.clearAmContainerResourceRequest(); } this.amResourceRequest = request; }
Example #16
Source File: AllocateRequestPBImpl.java From big-c with Apache License 2.0 | 5 votes |
private void addAsksToProto() { maybeInitBuilder(); builder.clearAsk(); if (ask == null) return; Iterable<ResourceRequestProto> iterable = new Iterable<ResourceRequestProto>() { @Override public Iterator<ResourceRequestProto> iterator() { return new Iterator<ResourceRequestProto>() { Iterator<ResourceRequest> iter = ask.iterator(); @Override public boolean hasNext() { return iter.hasNext(); } @Override public ResourceRequestProto next() { return convertToProtoFormat(iter.next()); } @Override public void remove() { throw new UnsupportedOperationException(); } }; } }; builder.addAllAsk(iterable); }
Example #17
Source File: AppSchedulingInfo.java From big-c with Apache License 2.0 | 5 votes |
/** * The {@link ResourceScheduler} is allocating data-local resources to the * application. * * @param allocatedContainers * resources allocated to the application */ synchronized private void allocateOffSwitch(SchedulerNode node, Priority priority, ResourceRequest offSwitchRequest, Container container, List<ResourceRequest> resourceRequests) { // Update future requirements decrementOutstanding(offSwitchRequest); // Update cloned OffRack requests for recovery resourceRequests.add(cloneResourceRequest(offSwitchRequest)); }
Example #18
Source File: AppSchedulingInfo.java From hadoop with Apache License 2.0 | 5 votes |
synchronized private void checkForDeactivation() { boolean deactivate = true; for (Priority priority : getPriorities()) { ResourceRequest request = getResourceRequest(priority, ResourceRequest.ANY); if (request != null) { if (request.getNumContainers() > 0) { deactivate = false; break; } } } if (deactivate) { activeUsersManager.deactivateApplication(user, applicationId); } }
Example #19
Source File: ResourceSchedulerWrapper.java From big-c with Apache License 2.0 | 5 votes |
@Override public Allocation allocate(ApplicationAttemptId attemptId, List<ResourceRequest> resourceRequests, List<ContainerId> containerIds, List<String> strings, List<String> strings2) { if (metricsON) { final Timer.Context context = schedulerAllocateTimer.time(); Allocation allocation = null; try { allocation = scheduler.allocate(attemptId, resourceRequests, containerIds, strings, strings2); return allocation; } finally { context.stop(); schedulerAllocateCounter.inc(); try { updateQueueWithAllocateRequest(allocation, attemptId, resourceRequests, containerIds); } catch (IOException e) { e.printStackTrace(); } } } else { return scheduler.allocate(attemptId, resourceRequests, containerIds, strings, strings2); } }
Example #20
Source File: TestContinuousScheduling.java From hadoop with Apache License 2.0 | 5 votes |
@Test (timeout = 60000) public void testSchedulingDelay() throws InterruptedException { // Add one node String host = "127.0.0.1"; RMNode node1 = MockNodes.newNodeInfo( 1, Resources.createResource(4096, 4, 4), 1, host); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); scheduler.handle(nodeEvent1); NodeUpdateSchedulerEvent nodeUpdateEvent = new NodeUpdateSchedulerEvent(node1); scheduler.handle(nodeUpdateEvent); // Create one application and submit one each of node-local, rack-local // and ANY requests ApplicationAttemptId appAttemptId = createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++); createMockRMApp(appAttemptId); scheduler.addApplication(appAttemptId.getApplicationId(), "queue11", "user11", false); scheduler.addApplicationAttempt(appAttemptId, false, false); List<ResourceRequest> ask = new ArrayList<>(); ask.add(createResourceRequest(1024, 1, 1, ResourceRequest.ANY, 1, 1, true)); scheduler.allocate( appAttemptId, ask, new ArrayList<ContainerId>(), null, null); FSAppAttempt app = scheduler.getSchedulerApp(appAttemptId); // Advance time and let continuous scheduling kick in mockClock.tick(1); while (1024 != app.getCurrentConsumption().getMemory()) { Thread.sleep(100); } assertEquals(1024, app.getCurrentConsumption().getMemory()); }
Example #21
Source File: DagAwareYarnTaskScheduler.java From tez with Apache License 2.0 | 5 votes |
@GuardedBy("DagAwareYarnTaskScheduler.this") void add(HeldContainer hc) { add(hc, hc.getHost()); add(hc, hc.getRack()); add(hc, ResourceRequest.ANY); ++numContainers; }
Example #22
Source File: TestFifoScheduler.java From big-c with Apache License 2.0 | 5 votes |
private ResourceRequest createResourceRequest(int memory, String host, int priority, int numContainers) { ResourceRequest request = recordFactory .newRecordInstance(ResourceRequest.class); request.setCapability(Resources.createResource(memory)); request.setResourceName(host); request.setNumContainers(numContainers); Priority prio = recordFactory.newRecordInstance(Priority.class); prio.setPriority(priority); request.setPriority(prio); return request; }
Example #23
Source File: AMSimulator.java From big-c with Apache License 2.0 | 5 votes |
protected AllocateRequest createAllocateRequest(List<ResourceRequest> ask, List<ContainerId> toRelease) { AllocateRequest allocateRequest = recordFactory.newRecordInstance(AllocateRequest.class); allocateRequest.setResponseId(RESPONSE_ID ++); allocateRequest.setAskList(ask); allocateRequest.setReleaseList(toRelease); return allocateRequest; }
Example #24
Source File: FifoScheduler.java From hadoop with Apache License 2.0 | 5 votes |
private int getMaxAllocatableContainers(FiCaSchedulerApp application, Priority priority, FiCaSchedulerNode node, NodeType type) { int maxContainers = 0; ResourceRequest offSwitchRequest = application.getResourceRequest(priority, ResourceRequest.ANY); if (offSwitchRequest != null) { maxContainers = offSwitchRequest.getNumContainers(); } if (type == NodeType.OFF_SWITCH) { return maxContainers; } if (type == NodeType.RACK_LOCAL) { ResourceRequest rackLocalRequest = application.getResourceRequest(priority, node.getRMNode().getRackName()); if (rackLocalRequest == null) { return maxContainers; } maxContainers = Math.min(maxContainers, rackLocalRequest.getNumContainers()); } if (type == NodeType.NODE_LOCAL) { ResourceRequest nodeLocalRequest = application.getResourceRequest(priority, node.getRMNode().getNodeAddress()); if (nodeLocalRequest != null) { maxContainers = Math.min(maxContainers, nodeLocalRequest.getNumContainers()); } } return maxContainers; }
Example #25
Source File: DagAwareYarnTaskScheduler.java From tez with Apache License 2.0 | 5 votes |
@GuardedBy("this") @Nullable private TaskRequest tryAssignReuseContainerAppRunning(HeldContainer hc) { if (!hc.isAssignable()) { LOG.debug("Skipping scheduling of container {} because it state is {}", hc.getId(), hc.getState()); return null; } TaskRequest assignedRequest = tryAssignReuseContainerForAffinity(hc); if (assignedRequest != null) { return assignedRequest; } for (Entry<Priority,RequestPriorityStats> entry : requestTracker.getStatsEntries()) { Priority priority = entry.getKey(); RequestPriorityStats stats = entry.getValue(); if (!stats.allowedVertices.intersects(stats.vertices)) { LOG.debug("Skipping requests at priority {} because all requesting vertices are blocked by higher priority requests", priority); continue; } String matchLocation = hc.getMatchingLocation(); if (stats.localityCount <= 0) { LOG.debug("Overriding locality match of container {} to ANY since there are no locality requests at priority {}", hc.getId(), priority); matchLocation = ResourceRequest.ANY; } assignedRequest = tryAssignReuseContainerForPriority(hc, matchLocation, priority, stats.allowedVertices); if (assignedRequest != null) { break; } } return assignedRequest; }
Example #26
Source File: LeafQueue.java From hadoop with Apache License 2.0 | 5 votes |
private boolean checkResourceRequestMatchingNodeLabel(ResourceRequest offswitchResourceRequest, FiCaSchedulerNode node) { String askedNodeLabel = offswitchResourceRequest.getNodeLabelExpression(); if (null == askedNodeLabel) { askedNodeLabel = RMNodeLabelsManager.NO_LABEL; } return askedNodeLabel.equals(node.getPartition()); }
Example #27
Source File: ApplicationSubmissionContextPBImpl.java From big-c with Apache License 2.0 | 5 votes |
@Override public ResourceRequest getAMContainerResourceRequest() { ApplicationSubmissionContextProtoOrBuilder p = viaProto ? proto : builder; if (this.amResourceRequest != null) { return amResourceRequest; } // Else via proto if (!p.hasAmContainerResourceRequest()) { return null; } amResourceRequest = convertFromProtoFormat(p.getAmContainerResourceRequest()); return amResourceRequest; }
Example #28
Source File: AMRMClientImpl.java From big-c with Apache License 2.0 | 5 votes |
ResourceRequestInfo(Priority priority, String resourceName, Resource capability, boolean relaxLocality) { remoteRequest = ResourceRequest.newInstance(priority, resourceName, capability, 0); remoteRequest.setRelaxLocality(relaxLocality); containerRequests = new LinkedHashSet<T>(); }
Example #29
Source File: FifoScheduler.java From big-c with Apache License 2.0 | 5 votes |
private int assignOffSwitchContainers(FiCaSchedulerNode node, FiCaSchedulerApp application, Priority priority) { int assignedContainers = 0; ResourceRequest request = application.getResourceRequest(priority, ResourceRequest.ANY); if (request != null) { assignedContainers = assignContainer(node, application, priority, request.getNumContainers(), request, NodeType.OFF_SWITCH); } return assignedContainers; }
Example #30
Source File: BuilderUtils.java From big-c with Apache License 2.0 | 5 votes |
public static ResourceRequest newResourceRequest(ResourceRequest r) { ResourceRequest request = recordFactory .newRecordInstance(ResourceRequest.class); request.setPriority(r.getPriority()); request.setResourceName(r.getResourceName()); request.setCapability(r.getCapability()); request.setNumContainers(r.getNumContainers()); return request; }