Java Code Examples for org.apache.hadoop.yarn.server.utils.BuilderUtils#newNodeReport()

The following examples show how to use org.apache.hadoop.yarn.server.utils.BuilderUtils#newNodeReport() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   File: ClientRMService.java    License: Apache License 2.0 6 votes vote down vote up
private NodeReport createNodeReports(RMNode rmNode) {    
  SchedulerNodeReport schedulerNodeReport = 
      scheduler.getNodeReport(rmNode.getNodeID());
  Resource used = BuilderUtils.newResource(0, 0, 0);
  int numContainers = 0;
  if (schedulerNodeReport != null) {
    used = schedulerNodeReport.getUsedResource();
    numContainers = schedulerNodeReport.getNumContainers();
  } 
  
  NodeReport report =
      BuilderUtils.newNodeReport(rmNode.getNodeID(), rmNode.getState(),
          rmNode.getHttpAddress(), rmNode.getRackName(), used,
          rmNode.getTotalCapability(), numContainers,
          rmNode.getHealthReport(), rmNode.getLastHealthReportTime(),
          rmNode.getNodeLabels());

  return report;
}
 
Example 2
Source Project: big-c   File: ClientRMService.java    License: Apache License 2.0 6 votes vote down vote up
private NodeReport createNodeReports(RMNode rmNode) {    
  SchedulerNodeReport schedulerNodeReport = 
      scheduler.getNodeReport(rmNode.getNodeID());
  Resource used = BuilderUtils.newResource(0, 0);
  int numContainers = 0;
  if (schedulerNodeReport != null) {
    used = schedulerNodeReport.getUsedResource();
    numContainers = schedulerNodeReport.getNumContainers();
  } 
  
  NodeReport report =
      BuilderUtils.newNodeReport(rmNode.getNodeID(), rmNode.getState(),
          rmNode.getHttpAddress(), rmNode.getRackName(), used,
          rmNode.getTotalCapability(), numContainers,
          rmNode.getHealthReport(), rmNode.getLastHealthReportTime(),
          rmNode.getNodeLabels());

  return report;
}
 
Example 3
Source Project: attic-apex-core   File: HostLocalTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testNodeLocal()
{
  LogicalPlan dag = new LogicalPlan();
  dag.getAttributes().put(com.datatorrent.api.Context.DAGContext.APPLICATION_PATH, new File("target", HostLocalTest.class.getName()).getAbsolutePath());
  dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());


  GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
  dag.setOperatorAttribute(o1,OperatorContext.MEMORY_MB,256);

  GenericTestOperator partitioned = dag.addOperator("partitioned", GenericTestOperator.class);
  dag.setOperatorAttribute(partitioned,OperatorContext.MEMORY_MB,256);
  dag.getMeta(partitioned).getAttributes().put(OperatorContext.LOCALITY_HOST, "host1");

  dag.addStream("o1_outport1", o1.outport1, partitioned.inport1).setLocality(Locality.NODE_LOCAL);

  StreamingContainerManager scm = new StreamingContainerManager(dag);

  ResourceRequestHandler rr = new ResourceRequestHandler();

  int containerMem = 1000;
  Map<String, NodeReport> nodeReports = Maps.newHashMap();
  NodeReport nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host1", 0),
      NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);
  nodeReports.put(nr.getNodeId().getHost(), nr);
  nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host2", 0),
      NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);
  nodeReports.put(nr.getNodeId().getHost(), nr);

  // set resources
  rr.updateNodeReports(Lists.newArrayList(nodeReports.values()));

  for (ContainerStartRequest csr : scm.containerStartRequests) {
    String host = rr.getHost(csr, true);
    csr.container.host = host;
    Assert.assertEquals("Hosts set to host1", "host1", host);
  }

}
 
Example 4
Source Project: attic-apex-core   File: HostLocalTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testThreadLocal()
{
  LogicalPlan dag = new LogicalPlan();
  dag.getAttributes().put(com.datatorrent.api.Context.DAGContext.APPLICATION_PATH, new File("target", HostLocalTest.class.getName()).getAbsolutePath());
  dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());

  GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
  dag.getMeta(o1).getAttributes().put(OperatorContext.LOCALITY_HOST, "host2");

  GenericTestOperator partitioned = dag.addOperator("partitioned", GenericTestOperator.class);
  dag.addStream("o1_outport1", o1.outport1, partitioned.inport1).setLocality(Locality.THREAD_LOCAL);
  dag.setOperatorAttribute(o1,OperatorContext.MEMORY_MB,256);
  dag.setOperatorAttribute(partitioned,OperatorContext.MEMORY_MB,256);

  StreamingContainerManager scm = new StreamingContainerManager(dag);

  ResourceRequestHandler rr = new ResourceRequestHandler();

  int containerMem = 1000;
  Map<String, NodeReport> nodeReports = Maps.newHashMap();
  NodeReport nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host1", 0),
      NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);
  nodeReports.put(nr.getNodeId().getHost(), nr);
  nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host2", 0),
      NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);
  nodeReports.put(nr.getNodeId().getHost(), nr);

  // set resources
  rr.updateNodeReports(Lists.newArrayList(nodeReports.values()));
  Assert.assertEquals("number of containers is 1", 1, scm.containerStartRequests.size());
  for (ContainerStartRequest csr : scm.containerStartRequests) {
    String host = rr.getHost(csr, true);
    csr.container.host = host;
    Assert.assertEquals("Hosts set to host2", "host2", host);
  }
}
 
Example 5
Source Project: attic-apex-core   File: HostLocalTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testContainerLocal()
{
  LogicalPlan dag = new LogicalPlan();
  dag.getAttributes().put(com.datatorrent.api.Context.DAGContext.APPLICATION_PATH, new File("target", HostLocalTest.class.getName()).getAbsolutePath());
  dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());

  GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
  dag.getMeta(o1).getAttributes().put(OperatorContext.LOCALITY_HOST, "host2");

  GenericTestOperator partitioned = dag.addOperator("partitioned", GenericTestOperator.class);
  dag.addStream("o1_outport1", o1.outport1, partitioned.inport1).setLocality(Locality.CONTAINER_LOCAL);
  dag.setOperatorAttribute(o1, OperatorContext.MEMORY_MB, 256);
  dag.setOperatorAttribute(partitioned,OperatorContext.MEMORY_MB,256);

  StreamingContainerManager scm = new StreamingContainerManager(dag);

  ResourceRequestHandler rr = new ResourceRequestHandler();

  int containerMem = 1000;
  Map<String, NodeReport> nodeReports = Maps.newHashMap();
  NodeReport nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host1", 0), NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);
  nodeReports.put(nr.getNodeId().getHost(), nr);
  nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host2", 0), NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);
  nodeReports.put(nr.getNodeId().getHost(), nr);

  // set resources
  rr.updateNodeReports(Lists.newArrayList(nodeReports.values()));
  Assert.assertEquals("number of containers is 1", 1, scm.containerStartRequests.size());
  for (ContainerStartRequest csr : scm.containerStartRequests) {
    String host = rr.getHost(csr, true);
    csr.container.host = host;
    Assert.assertEquals("Hosts set to host2", "host2", host);
  }
}
 
Example 6
Source Project: attic-apex-core   File: HostLocalTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testContainerLocalWithVCores()
{
  LogicalPlan dag = new LogicalPlan();
  dag.getAttributes().put(com.datatorrent.api.Context.DAGContext.APPLICATION_PATH, new File("target", HostLocalTest.class.getName()).getAbsolutePath());
  dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());

  GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
  dag.getMeta(o1).getAttributes().put(OperatorContext.LOCALITY_HOST, "host2");

  GenericTestOperator partitioned = dag.addOperator("partitioned", GenericTestOperator.class);
  dag.addStream("o1_outport1", o1.outport1, partitioned.inport1).setLocality(Locality.CONTAINER_LOCAL);
  dag.setOperatorAttribute(o1,OperatorContext.MEMORY_MB,256);
  dag.setOperatorAttribute(o1,OperatorContext.VCORES,1);
  dag.setOperatorAttribute(partitioned,OperatorContext.VCORES,1);

  StreamingContainerManager scm = new StreamingContainerManager(dag);

  ResourceRequestHandler rr = new ResourceRequestHandler();

  int containerMem = 1000;
  Map<String, NodeReport> nodeReports = Maps.newHashMap();
  NodeReport nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host1", 0),
      NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);
  nodeReports.put(nr.getNodeId().getHost(), nr);
  nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host2", 0),
      NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);
  nodeReports.put(nr.getNodeId().getHost(), nr);

  // set resources
  rr.updateNodeReports(Lists.newArrayList(nodeReports.values()));
  Assert.assertEquals("number of containers is 1", 1, scm.containerStartRequests.size());
  for (ContainerStartRequest csr : scm.containerStartRequests) {
    String host = rr.getHost(csr, true);
    csr.container.host = host;
    Assert.assertEquals("number of vcores", 2, csr.container.getRequiredVCores());
    Assert.assertEquals("Hosts set to host2", "host2", host);
  }
}
 
Example 7
Source Project: attic-apex-core   File: HostLocalTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testUnavailableResources()
{
  LogicalPlan dag = new LogicalPlan();
  dag.getAttributes().put(com.datatorrent.api.Context.DAGContext.APPLICATION_PATH, new File("target", HostLocalTest.class.getName()).getAbsolutePath());
  dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());

  GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
  dag.getMeta(o1).getAttributes().put(OperatorContext.LOCALITY_HOST, "host2");

  GenericTestOperator partitioned = dag.addOperator("partitioned", GenericTestOperator.class);
  dag.addStream("o1_outport1", o1.outport1, partitioned.inport1).setLocality(Locality.CONTAINER_LOCAL);
  dag.setOperatorAttribute(o1, OperatorContext.MEMORY_MB, 256);
  dag.setOperatorAttribute(o1, OperatorContext.VCORES, 2);
  dag.setOperatorAttribute(partitioned, OperatorContext.VCORES, 1);

  StreamingContainerManager scm = new StreamingContainerManager(dag);

  ResourceRequestHandler rr = new ResourceRequestHandler();

  int containerMem = 1000;
  Map<String, NodeReport> nodeReports = Maps.newHashMap();
  NodeReport nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host1", 0),
      NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);
  nodeReports.put(nr.getNodeId().getHost(), nr);
  nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host2", 0),
      NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);
  nodeReports.put(nr.getNodeId().getHost(), nr);

  // set resources
  rr.updateNodeReports(Lists.newArrayList(nodeReports.values()));
  Assert.assertEquals("number of containers is 1", 1, scm.containerStartRequests.size());
  for (ContainerStartRequest csr : scm.containerStartRequests) {
    String host = rr.getHost(csr, true);
    Assert.assertEquals("number of vcores", 3, csr.container.getRequiredVCores());
    Assert.assertNull("Host is null", host);
  }
}
 
Example 8
@Test
public void testAntiAffinityInOperators()
{
  LogicalPlan dag = new LogicalPlan();
  dag.getAttributes().put(com.datatorrent.api.Context.DAGContext.APPLICATION_PATH, testMeta.getAbsolutePath());
  dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());

  GenericTestOperator o1 = dag.addOperator("O1", GenericTestOperator.class);
  dag.setOperatorAttribute(o1, OperatorContext.MEMORY_MB, 256);

  GenericTestOperator o2 = dag.addOperator("O2", GenericTestOperator.class);
  dag.setOperatorAttribute(o2, OperatorContext.MEMORY_MB, 256);

  dag.getMeta(o1).getAttributes().put(OperatorContext.LOCALITY_HOST, "host1");
  AffinityRulesSet ruleSet = new AffinityRulesSet();

  List<AffinityRule> rules = new ArrayList<>();
  ruleSet.setAffinityRules(rules);
  AffinityRule rule1 = new AffinityRule(Type.ANTI_AFFINITY, Locality.NODE_LOCAL, false, "O1", "O2");
  rules.add(rule1);
  dag.setAttribute(DAGContext.AFFINITY_RULES_SET, ruleSet);

  dag.addStream("o1_outport1", o1.outport1, o2.inport1);// .setLocality(Locality.NODE_LOCAL);

  StreamingContainerManager scm = new StreamingContainerManager(dag);

  ResourceRequestHandler rr = new ResourceRequestHandler();

  int containerMem = 1000;
  Map<String, NodeReport> nodeReports = Maps.newHashMap();
  NodeReport nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host1", 0), NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);
  nodeReports.put(nr.getNodeId().getHost(), nr);
  nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host2", 0), NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);
  nodeReports.put(nr.getNodeId().getHost(), nr);

  // set resources
  rr.updateNodeReports(Lists.newArrayList(nodeReports.values()));

  for (ContainerStartRequest csr : scm.containerStartRequests) {
    String host = rr.getHost(csr, true);
    csr.container.host = host;
    if (csr.container.getOperators().get(0).getName().equals("O1")) {
      Assert.assertEquals("Hosts set to host1 for Operator O1", "host1", host);
    }
    if (csr.container.getOperators().get(0).getName().equals("O2")) {
      Assert.assertEquals("Hosts set to host2 for Operator O2", "host2", host);
    }
  }
}
 
Example 9
Source Project: attic-apex-core   File: HostLocalTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testPartitionLocality()
{
  int partitionCount = 3;
  LogicalPlan dag = new LogicalPlan();
  dag.getAttributes().put(com.datatorrent.api.Context.DAGContext.APPLICATION_PATH, new File("target", HostLocalTest.class.getName()).getAbsolutePath());
  dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());

  GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);

  GenericTestOperator partitioned = dag.addOperator("partitioned", GenericTestOperator.class);
  LocalityPartitioner partitioner = new LocalityPartitioner();
  partitioner.setPartitionCount(partitionCount);
  dag.getMeta(partitioned).getAttributes().put(OperatorContext.PARTITIONER, partitioner);
  dag.addStream("o1_outport1", o1.outport1, partitioned.inport1);

  StreamingContainerManager scm = new StreamingContainerManager(dag);

  ResourceRequestHandler rr = new ResourceRequestHandler();

  int containerMem = 1000;
  Map<String, NodeReport> nodeReports = Maps.newHashMap();
  for (int i = 0; i < partitionCount; i++) {
    NodeReport nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host" + (i + 1), 0),
        NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);
    nodeReports.put(nr.getNodeId().getHost(), nr);
  }

  // set resources
  rr.updateNodeReports(Lists.newArrayList(nodeReports.values()));
  Set<String> expectedHosts = Sets.newHashSet();
  for (int i = 0; i < partitionCount; i++) {
    expectedHosts.add("host" + (i + 1));
  }
  for (ContainerStartRequest csr : scm.containerStartRequests) {
    String host = rr.getHost(csr, true);
    if (host != null) {
      expectedHosts.remove(host);
    }
  }
  Assert.assertTrue("All the allocated hosts removed", expectedHosts.isEmpty());

}
 
Example 10
Source Project: attic-apex-core   File: LocalityTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testNodeLocal()
{

  LogicalPlan dag = new LogicalPlan();
  dag.getAttributes().put(com.datatorrent.api.Context.DAGContext.APPLICATION_PATH, new File("target", LocalityTest.class.getName()).getAbsolutePath());
  dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());

  GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);

  GenericTestOperator partitioned = dag.addOperator("partitioned", GenericTestOperator.class);
  dag.getMeta(partitioned).getAttributes().put(OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(2));

  GenericTestOperator partitionedParallel = dag.addOperator("partitionedParallel", GenericTestOperator.class);

  dag.addStream("o1_outport1", o1.outport1, partitioned.inport1).setLocality(null);

  dag.addStream("partitioned_outport1", partitioned.outport1, partitionedParallel.inport2).setLocality(Locality.NODE_LOCAL);
  dag.setInputPortAttribute(partitionedParallel.inport2, PortContext.PARTITION_PARALLEL, true);

  GenericTestOperator single = dag.addOperator("single", GenericTestOperator.class);
  dag.addStream("partitionedParallel_outport1", partitionedParallel.outport1, single.inport1);

  int maxContainers = 7;
  dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, maxContainers);

  StreamingContainerManager scm = new StreamingContainerManager(dag);
  Assert.assertEquals("number required containers", 6, scm.containerStartRequests.size());

  ResourceRequestHandler rr = new ResourceRequestHandler();

  int containerMem = 2000;
  Map<String, NodeReport> nodeReports = Maps.newHashMap();
  NodeReport nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host1", 0), NodeState.RUNNING, "httpAddress",
      "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);
  nodeReports.put(nr.getNodeId().getHost(), nr);
  nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host2", 0), NodeState.RUNNING, "httpAddress",
      "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);
  nodeReports.put(nr.getNodeId().getHost(), nr);

  // set resources
  rr.updateNodeReports(Lists.newArrayList(nodeReports.values()));

  Map<PTContainer, String> requestedHosts = Maps.newHashMap();
  for (ContainerStartRequest csr : scm.containerStartRequests) {
    String host = rr.getHost(csr, true);
    csr.container.host = host;
    // update the node report
    if (host != null) {
      requestedHosts.put(csr.container, host);
      nr = nodeReports.get(host);
      nr.getUsed().setMemory(nr.getUsed().getMemory() + containerMem);
    }
  }

  Assert.assertEquals("" + requestedHosts, nodeReports.keySet(), Sets.newHashSet(requestedHosts.values()));

  for (Map.Entry<PTContainer, String> e : requestedHosts.entrySet()) {
    for (PTOperator oper : e.getKey().getOperators()) {
      if (oper.getNodeLocalOperators().getOperatorSet().size() > 1) {
        String expHost = null;
        for (PTOperator nodeLocalOper : oper.getNodeLocalOperators().getOperatorSet()) {
          Assert.assertNotNull("host null " + nodeLocalOper.getContainer(), nodeLocalOper.getContainer().host);
          if (expHost == null) {
            expHost = nodeLocalOper.getContainer().host;
          } else {
            Assert.assertEquals("expected same host " + nodeLocalOper, expHost, nodeLocalOper.getContainer().host);
          }
        }
      }
    }
  }

}