Java Code Examples for org.apache.hadoop.yarn.api.records.ResourceRequest#setNodeLabelExpression()
The following examples show how to use
org.apache.hadoop.yarn.api.records.ResourceRequest#setNodeLabelExpression() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: SchedulerUtils.java From hadoop with Apache License 2.0 | 6 votes |
private static void normalizeNodeLabelExpressionInRequest( ResourceRequest resReq, QueueInfo queueInfo) { String labelExp = resReq.getNodeLabelExpression(); // if queue has default label expression, and RR doesn't have, use the // default label expression of queue if (labelExp == null && queueInfo != null && ResourceRequest.ANY .equals(resReq.getResourceName())) { labelExp = queueInfo.getDefaultNodeLabelExpression(); } // If labelExp still equals to null, set it to be NO_LABEL if (labelExp == null) { labelExp = RMNodeLabelsManager.NO_LABEL; } resReq.setNodeLabelExpression(labelExp); }
Example 2
Source File: MockAM.java From hadoop with Apache License 2.0 | 6 votes |
public ResourceRequest createResourceReq(String resource, int memory, int priority, int containers, String labelExpression) throws Exception { ResourceRequest req = Records.newRecord(ResourceRequest.class); req.setResourceName(resource); req.setNumContainers(containers); Priority pri = Records.newRecord(Priority.class); pri.setPriority(priority); req.setPriority(pri); Resource capability = Records.newRecord(Resource.class); capability.setMemory(memory); req.setCapability(capability); if (labelExpression != null) { req.setNodeLabelExpression(labelExpression); } return req; }
Example 3
Source File: SchedulerUtils.java From big-c with Apache License 2.0 | 6 votes |
private static void normalizeNodeLabelExpressionInRequest( ResourceRequest resReq, QueueInfo queueInfo) { String labelExp = resReq.getNodeLabelExpression(); // if queue has default label expression, and RR doesn't have, use the // default label expression of queue if (labelExp == null && queueInfo != null && ResourceRequest.ANY .equals(resReq.getResourceName())) { labelExp = queueInfo.getDefaultNodeLabelExpression(); } // If labelExp still equals to null, set it to be NO_LABEL if (labelExp == null) { labelExp = RMNodeLabelsManager.NO_LABEL; } resReq.setNodeLabelExpression(labelExp); }
Example 4
Source File: MockAM.java From big-c with Apache License 2.0 | 6 votes |
public ResourceRequest createResourceReq(String resource, int memory, int priority, int containers, String labelExpression) throws Exception { ResourceRequest req = Records.newRecord(ResourceRequest.class); req.setResourceName(resource); req.setNumContainers(containers); Priority pri = Records.newRecord(Priority.class); pri.setPriority(priority); req.setPriority(pri); Resource capability = Records.newRecord(Resource.class); capability.setMemory(memory); req.setCapability(capability); if (labelExpression != null) { req.setNodeLabelExpression(labelExpression); } return req; }
Example 5
Source File: BuilderUtils.java From hadoop with Apache License 2.0 | 5 votes |
public static ResourceRequest newResourceRequest(Priority priority, String hostName, Resource capability, int numContainers, String label) { ResourceRequest request = recordFactory.newRecordInstance(ResourceRequest.class); request.setPriority(priority); request.setResourceName(hostName); request.setCapability(capability); request.setNumContainers(numContainers); request.setNodeLabelExpression(label); return request; }
Example 6
Source File: FairSchedulerTestBase.java From hadoop with Apache License 2.0 | 5 votes |
protected ResourceRequest createResourceRequest( int memory, int vcores, int gcores, String host, int priority, int numContainers, boolean relaxLocality) { ResourceRequest request = recordFactory.newRecordInstance(ResourceRequest.class); request.setCapability(BuilderUtils.newResource(memory, vcores, gcores)); request.setResourceName(host); request.setNumContainers(numContainers); Priority prio = recordFactory.newRecordInstance(Priority.class); prio.setPriority(priority); request.setPriority(prio); request.setRelaxLocality(relaxLocality); request.setNodeLabelExpression(RMNodeLabelsManager.NO_LABEL); return request; }
Example 7
Source File: RMContainerRequestor.java From hadoop with Apache License 2.0 | 5 votes |
private void addResourceRequest(Priority priority, String resourceName, Resource capability, String nodeLabelExpression) { Map<String, Map<Resource, ResourceRequest>> remoteRequests = this.remoteRequestsTable.get(priority); if (remoteRequests == null) { remoteRequests = new HashMap<String, Map<Resource, ResourceRequest>>(); this.remoteRequestsTable.put(priority, remoteRequests); if (LOG.isDebugEnabled()) { LOG.debug("Added priority=" + priority); } } Map<Resource, ResourceRequest> reqMap = remoteRequests.get(resourceName); if (reqMap == null) { reqMap = new HashMap<Resource, ResourceRequest>(); remoteRequests.put(resourceName, reqMap); } ResourceRequest remoteRequest = reqMap.get(capability); if (remoteRequest == null) { remoteRequest = recordFactory.newRecordInstance(ResourceRequest.class); remoteRequest.setPriority(priority); remoteRequest.setResourceName(resourceName); remoteRequest.setCapability(capability); remoteRequest.setNumContainers(0); remoteRequest.setNodeLabelExpression(nodeLabelExpression); reqMap.put(capability, remoteRequest); } remoteRequest.setNumContainers(remoteRequest.getNumContainers() + 1); // Note this down for next interaction with ResourceManager addResourceRequestToAsk(remoteRequest); if (LOG.isDebugEnabled()) { LOG.debug("addResourceRequest:" + " applicationId=" + applicationId.getId() + " priority=" + priority.getPriority() + " resourceName=" + resourceName + " numContainers=" + remoteRequest.getNumContainers() + " #asks=" + ask.size()); } }
Example 8
Source File: RMAppManager.java From hadoop with Apache License 2.0 | 4 votes |
private ResourceRequest validateAndCreateResourceRequest( ApplicationSubmissionContext submissionContext, boolean isRecovery) throws InvalidResourceRequestException { // Validation of the ApplicationSubmissionContext needs to be completed // here. Only those fields that are dependent on RM's configuration are // checked here as they have to be validated whether they are part of new // submission or just being recovered. // Check whether AM resource requirements are within required limits if (!submissionContext.getUnmanagedAM()) { ResourceRequest amReq = submissionContext.getAMContainerResourceRequest(); if (amReq == null) { amReq = BuilderUtils .newResourceRequest(RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY, submissionContext.getResource(), 1); } // set label expression for AM container if (null == amReq.getNodeLabelExpression()) { amReq.setNodeLabelExpression(submissionContext .getNodeLabelExpression()); } try { SchedulerUtils.normalizeAndValidateRequest(amReq, scheduler.getMaximumResourceCapability(), submissionContext.getQueue(), scheduler, isRecovery, rmContext); } catch (InvalidResourceRequestException e) { LOG.warn("RM app submission failed in validating AM resource request" + " for application " + submissionContext.getApplicationId(), e); throw e; } SchedulerUtils.normalizeRequest(amReq, scheduler.getResourceCalculator(), scheduler.getClusterResource(), scheduler.getMinimumResourceCapability(), scheduler.getMaximumResourceCapability(), scheduler.getMinimumResourceCapability()); return amReq; } return null; }
Example 9
Source File: TestSchedulerUtils.java From hadoop with Apache License 2.0 | 4 votes |
@Test (timeout = 30000) public void testNormalizeNodeLabelExpression() throws IOException { // mock queue and scheduler YarnScheduler scheduler = mock(YarnScheduler.class); Set<String> queueAccessibleNodeLabels = Sets.newHashSet(); QueueInfo queueInfo = mock(QueueInfo.class); when(queueInfo.getQueueName()).thenReturn("queue"); when(queueInfo.getAccessibleNodeLabels()).thenReturn(queueAccessibleNodeLabels); when(queueInfo.getDefaultNodeLabelExpression()).thenReturn(" x "); when(scheduler.getQueueInfo(any(String.class), anyBoolean(), anyBoolean())) .thenReturn(queueInfo); Resource maxResource = Resources.createResource( YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_GCORES); // queue has labels, success cases try { // set queue accessible node labels to [x, y] queueAccessibleNodeLabels.clear(); queueAccessibleNodeLabels.addAll(Arrays.asList("x", "y")); rmContext.getNodeLabelManager().addToCluserNodeLabels( ImmutableSet.of("x", "y")); Resource resource = Resources.createResource( 0, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_GCORES); ResourceRequest resReq = BuilderUtils.newResourceRequest( mock(Priority.class), ResourceRequest.ANY, resource, 1); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext); Assert.assertTrue(resReq.getNodeLabelExpression().equals("x")); resReq.setNodeLabelExpression(" y "); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext); Assert.assertTrue(resReq.getNodeLabelExpression().equals("y")); } catch (InvalidResourceRequestException e) { e.printStackTrace(); fail("Should be valid when request labels is a subset of queue labels"); } finally { rmContext.getNodeLabelManager().removeFromClusterNodeLabels( Arrays.asList("x", "y")); } }
Example 10
Source File: RMAppManager.java From big-c with Apache License 2.0 | 4 votes |
private ResourceRequest validateAndCreateResourceRequest( ApplicationSubmissionContext submissionContext, boolean isRecovery) throws InvalidResourceRequestException { // Validation of the ApplicationSubmissionContext needs to be completed // here. Only those fields that are dependent on RM's configuration are // checked here as they have to be validated whether they are part of new // submission or just being recovered. // Check whether AM resource requirements are within required limits if (!submissionContext.getUnmanagedAM()) { ResourceRequest amReq = submissionContext.getAMContainerResourceRequest(); if (amReq == null) { amReq = BuilderUtils .newResourceRequest(RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY, submissionContext.getResource(), 1); } // set label expression for AM container if (null == amReq.getNodeLabelExpression()) { amReq.setNodeLabelExpression(submissionContext .getNodeLabelExpression()); } try { SchedulerUtils.normalizeAndValidateRequest(amReq, scheduler.getMaximumResourceCapability(), submissionContext.getQueue(), scheduler, isRecovery, rmContext); } catch (InvalidResourceRequestException e) { LOG.warn("RM app submission failed in validating AM resource request" + " for application " + submissionContext.getApplicationId(), e); throw e; } SchedulerUtils.normalizeRequest(amReq, scheduler.getResourceCalculator(), scheduler.getClusterResource(), scheduler.getMinimumResourceCapability(), scheduler.getMaximumResourceCapability(), scheduler.getMinimumResourceCapability()); return amReq; } return null; }
Example 11
Source File: TestSchedulerUtils.java From big-c with Apache License 2.0 | 4 votes |
@Test (timeout = 30000) public void testNormalizeNodeLabelExpression() throws IOException { // mock queue and scheduler YarnScheduler scheduler = mock(YarnScheduler.class); Set<String> queueAccessibleNodeLabels = Sets.newHashSet(); QueueInfo queueInfo = mock(QueueInfo.class); when(queueInfo.getQueueName()).thenReturn("queue"); when(queueInfo.getAccessibleNodeLabels()).thenReturn(queueAccessibleNodeLabels); when(queueInfo.getDefaultNodeLabelExpression()).thenReturn(" x "); when(scheduler.getQueueInfo(any(String.class), anyBoolean(), anyBoolean())) .thenReturn(queueInfo); Resource maxResource = Resources.createResource( YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES); // queue has labels, success cases try { // set queue accessible node labels to [x, y] queueAccessibleNodeLabels.clear(); queueAccessibleNodeLabels.addAll(Arrays.asList("x", "y")); rmContext.getNodeLabelManager().addToCluserNodeLabels( ImmutableSet.of("x", "y")); Resource resource = Resources.createResource( 0, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); ResourceRequest resReq = BuilderUtils.newResourceRequest( mock(Priority.class), ResourceRequest.ANY, resource, 1); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext); Assert.assertTrue(resReq.getNodeLabelExpression().equals("x")); resReq.setNodeLabelExpression(" y "); SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext); Assert.assertTrue(resReq.getNodeLabelExpression().equals("y")); } catch (InvalidResourceRequestException e) { e.printStackTrace(); fail("Should be valid when request labels is a subset of queue labels"); } finally { rmContext.getNodeLabelManager().removeFromClusterNodeLabels( Arrays.asList("x", "y")); } }