Java Code Examples for org.apache.hadoop.net.NetUtils#normalizeHostName()

The following examples show how to use org.apache.hadoop.net.NetUtils#normalizeHostName() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: NodesListManager.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
public boolean isValidNode(String hostName) {
  synchronized (hostsReader) {
    Set<String> hostsList = hostsReader.getHosts();
    Set<String> excludeList = hostsReader.getExcludedHosts();
    String ip = NetUtils.normalizeHostName(hostName);
    return (hostsList.isEmpty() || hostsList.contains(hostName) || hostsList
        .contains(ip))
        && !(excludeList.contains(hostName) || excludeList.contains(ip));
  }
}
Example 2
Source File: NodesListManager.java    From big-c with Apache License 2.0 5 votes vote down vote up 
public boolean isValidNode(String hostName) {
  synchronized (hostsReader) {
    Set<String> hostsList = hostsReader.getHosts();
    Set<String> excludeList = hostsReader.getExcludedHosts();
    String ip = NetUtils.normalizeHostName(hostName);
    return (hostsList.isEmpty() || hostsList.contains(hostName) || hostsList
        .contains(ip))
        && !(excludeList.contains(hostName) || excludeList.contains(ip));
  }
}
Example 3
Source File: HourGlass.java    From RDFS with Apache License 2.0 5 votes vote down vote up 
/**
 * Obtain the cluster information from RPC
 * @throws IOException
 */
void updateClusterStatus() throws IOException {
  taskTrackers.clear();
  for (TaskTrackerStatus status : client.getTaskTrackerStatus()) {
    String host = NetUtils.normalizeHostName(status.getHost());
    taskTrackers.put(host, status);
  }
  runnableMaps = client.getRunnableTasks(TaskType.MAP);
  runnableReduces = client.getRunnableTasks(TaskType.REDUCE);
  LOG.info(String.format("Update cluster status. " +
      "cluster:%s runnableMaps:%s runnableReduces:%s",
      address, runnableMaps, runnableReduces));
}
Example 4
Source File: TestRMRestart.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Test (timeout = 60000)
public void testDecomissionedNMsMetricsOnRMRestart() throws Exception {
  YarnConfiguration conf = new YarnConfiguration();
  conf.set(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH,
    hostFile.getAbsolutePath());
  writeToHostsFile("");
  final DrainDispatcher dispatcher = new DrainDispatcher();
  MockRM rm1 = null, rm2 = null;
  try {
    rm1 = new MockRM(conf) {
      @Override
      protected Dispatcher createDispatcher() {
        return dispatcher;
      }
    };
    rm1.start();
    MockNM nm1 = rm1.registerNode("localhost:1234", 8000);
    MockNM nm2 = rm1.registerNode("host2:1234", 8000);
    Assert
        .assertEquals(0,
            ClusterMetrics.getMetrics().getNumDecommisionedNMs());
    String ip = NetUtils.normalizeHostName("localhost");
    // Add 2 hosts to exclude list.
    writeToHostsFile("host2", ip);

    // refresh nodes
    rm1.getNodesListManager().refreshNodes(conf);
    NodeHeartbeatResponse nodeHeartbeat = nm1.nodeHeartbeat(true);
    Assert
        .assertTrue(
            NodeAction.SHUTDOWN.equals(nodeHeartbeat.getNodeAction()));
    nodeHeartbeat = nm2.nodeHeartbeat(true);
    Assert.assertTrue("The decommisioned metrics are not updated",
        NodeAction.SHUTDOWN.equals(nodeHeartbeat.getNodeAction()));

    dispatcher.await();
    Assert
        .assertEquals(2,
            ClusterMetrics.getMetrics().getNumDecommisionedNMs());
    rm1.stop();
    rm1 = null;
    Assert
        .assertEquals(0,
            ClusterMetrics.getMetrics().getNumDecommisionedNMs());

    // restart RM.
    rm2 = new MockRM(conf);
    rm2.start();
    Assert
        .assertEquals(2,
            ClusterMetrics.getMetrics().getNumDecommisionedNMs());
  } finally {
    if (rm1 != null) {
      rm1.stop();
    }
    if (rm2 != null) {
      rm2.stop();
    }
  }
}
Example 5
Source File: TestResourceTrackerService.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
/**
 * Decommissioning using a pre-configured include hosts file
 */
@Test
public void testDecommissionWithIncludeHosts() throws Exception {

  writeToHostsFile("localhost", "host1", "host2");
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, hostFile
      .getAbsolutePath());

  rm = new MockRM(conf);
  rm.start();

  MockNM nm1 = rm.registerNode("host1:1234", 5120);
  MockNM nm2 = rm.registerNode("host2:5678", 10240);
  MockNM nm3 = rm.registerNode("localhost:4433", 1024);
  
  ClusterMetrics metrics = ClusterMetrics.getMetrics();
  assert(metrics != null);
  int metricCount = metrics.getNumDecommisionedNMs();

  NodeHeartbeatResponse nodeHeartbeat = nm1.nodeHeartbeat(true);
  Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat.getNodeAction()));
  nodeHeartbeat = nm2.nodeHeartbeat(true);
  Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat.getNodeAction()));
  nodeHeartbeat = nm3.nodeHeartbeat(true);
  Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat.getNodeAction()));

  // To test that IPs also work
  String ip = NetUtils.normalizeHostName("localhost");
  writeToHostsFile("host1", ip);

  rm.getNodesListManager().refreshNodes(conf);

  checkDecommissionedNMCount(rm, ++metricCount);

  nodeHeartbeat = nm1.nodeHeartbeat(true);
  Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat.getNodeAction()));
  Assert
      .assertEquals(1, ClusterMetrics.getMetrics().getNumDecommisionedNMs());

  nodeHeartbeat = nm2.nodeHeartbeat(true);
  Assert.assertTrue("Node is not decommisioned.", NodeAction.SHUTDOWN
      .equals(nodeHeartbeat.getNodeAction()));

  nodeHeartbeat = nm3.nodeHeartbeat(true);
  Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat.getNodeAction()));
  Assert.assertEquals(metricCount, ClusterMetrics.getMetrics()
    .getNumDecommisionedNMs());
}
Example 6
Source File: TestResourceTrackerService.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
/**
 * Decommissioning using a pre-configured exclude hosts file
 */
@Test
public void testDecommissionWithExcludeHosts() throws Exception {
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH, hostFile
      .getAbsolutePath());

  writeToHostsFile("");
  final DrainDispatcher dispatcher = new DrainDispatcher();
  rm = new MockRM(conf) {
    @Override
    protected Dispatcher createDispatcher() {
      return dispatcher;
    }
  };
  rm.start();

  MockNM nm1 = rm.registerNode("host1:1234", 5120);
  MockNM nm2 = rm.registerNode("host2:5678", 10240);
  MockNM nm3 = rm.registerNode("localhost:4433", 1024);

  dispatcher.await();

  int metricCount = ClusterMetrics.getMetrics().getNumDecommisionedNMs();
  NodeHeartbeatResponse nodeHeartbeat = nm1.nodeHeartbeat(true);
  Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat.getNodeAction()));
  nodeHeartbeat = nm2.nodeHeartbeat(true);
  Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat.getNodeAction()));
  dispatcher.await();

  // To test that IPs also work
  String ip = NetUtils.normalizeHostName("localhost");
  writeToHostsFile("host2", ip);

  rm.getNodesListManager().refreshNodes(conf);

  checkDecommissionedNMCount(rm, metricCount + 2);

  nodeHeartbeat = nm1.nodeHeartbeat(true);
  Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat.getNodeAction()));
  nodeHeartbeat = nm2.nodeHeartbeat(true);
  Assert.assertTrue("The decommisioned metrics are not updated",
      NodeAction.SHUTDOWN.equals(nodeHeartbeat.getNodeAction()));

  nodeHeartbeat = nm3.nodeHeartbeat(true);
  Assert.assertTrue("The decommisioned metrics are not updated",
      NodeAction.SHUTDOWN.equals(nodeHeartbeat.getNodeAction()));
  dispatcher.await();

  writeToHostsFile("");
  rm.getNodesListManager().refreshNodes(conf);

  nm3 = rm.registerNode("localhost:4433", 1024);
  dispatcher.await();
  nodeHeartbeat = nm3.nodeHeartbeat(true);
  dispatcher.await();
  Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat.getNodeAction()));
  // decommissined node is 1 since 1 node is rejoined after updating exclude
  // file
  checkDecommissionedNMCount(rm, metricCount + 1);
}
Example 7
Source File: TestRMRestart.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Test (timeout = 60000)
public void testDecomissionedNMsMetricsOnRMRestart() throws Exception {
  YarnConfiguration conf = new YarnConfiguration();
  conf.set(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH,
    hostFile.getAbsolutePath());
  writeToHostsFile("");
  final DrainDispatcher dispatcher = new DrainDispatcher();
  MockRM rm1 = null, rm2 = null;
  try {
    rm1 = new MockRM(conf) {
      @Override
      protected Dispatcher createDispatcher() {
        return dispatcher;
      }
    };
    rm1.start();
    MockNM nm1 = rm1.registerNode("localhost:1234", 8000);
    MockNM nm2 = rm1.registerNode("host2:1234", 8000);
    Assert
        .assertEquals(0,
            ClusterMetrics.getMetrics().getNumDecommisionedNMs());
    String ip = NetUtils.normalizeHostName("localhost");
    // Add 2 hosts to exclude list.
    writeToHostsFile("host2", ip);

    // refresh nodes
    rm1.getNodesListManager().refreshNodes(conf);
    NodeHeartbeatResponse nodeHeartbeat = nm1.nodeHeartbeat(true);
    Assert
        .assertTrue(
            NodeAction.SHUTDOWN.equals(nodeHeartbeat.getNodeAction()));
    nodeHeartbeat = nm2.nodeHeartbeat(true);
    Assert.assertTrue("The decommisioned metrics are not updated",
        NodeAction.SHUTDOWN.equals(nodeHeartbeat.getNodeAction()));

    dispatcher.await();
    Assert
        .assertEquals(2,
            ClusterMetrics.getMetrics().getNumDecommisionedNMs());
    rm1.stop();
    rm1 = null;
    Assert
        .assertEquals(0,
            ClusterMetrics.getMetrics().getNumDecommisionedNMs());

    // restart RM.
    rm2 = new MockRM(conf);
    rm2.start();
    Assert
        .assertEquals(2,
            ClusterMetrics.getMetrics().getNumDecommisionedNMs());
  } finally {
    if (rm1 != null) {
      rm1.stop();
    }
    if (rm2 != null) {
      rm2.stop();
    }
  }
}
Example 8
Source File: TestResourceTrackerService.java    From big-c with Apache License 2.0 4 votes vote down vote up 
/**
 * Decommissioning using a pre-configured include hosts file
 */
@Test
public void testDecommissionWithIncludeHosts() throws Exception {

  writeToHostsFile("localhost", "host1", "host2");
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, hostFile
      .getAbsolutePath());

  rm = new MockRM(conf);
  rm.start();

  MockNM nm1 = rm.registerNode("host1:1234", 5120);
  MockNM nm2 = rm.registerNode("host2:5678", 10240);
  MockNM nm3 = rm.registerNode("localhost:4433", 1024);
  
  ClusterMetrics metrics = ClusterMetrics.getMetrics();
  assert(metrics != null);
  int metricCount = metrics.getNumDecommisionedNMs();

  NodeHeartbeatResponse nodeHeartbeat = nm1.nodeHeartbeat(true);
  Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat.getNodeAction()));
  nodeHeartbeat = nm2.nodeHeartbeat(true);
  Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat.getNodeAction()));
  nodeHeartbeat = nm3.nodeHeartbeat(true);
  Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat.getNodeAction()));

  // To test that IPs also work
  String ip = NetUtils.normalizeHostName("localhost");
  writeToHostsFile("host1", ip);

  rm.getNodesListManager().refreshNodes(conf);

  checkDecommissionedNMCount(rm, ++metricCount);

  nodeHeartbeat = nm1.nodeHeartbeat(true);
  Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat.getNodeAction()));
  Assert
      .assertEquals(1, ClusterMetrics.getMetrics().getNumDecommisionedNMs());

  nodeHeartbeat = nm2.nodeHeartbeat(true);
  Assert.assertTrue("Node is not decommisioned.", NodeAction.SHUTDOWN
      .equals(nodeHeartbeat.getNodeAction()));

  nodeHeartbeat = nm3.nodeHeartbeat(true);
  Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat.getNodeAction()));
  Assert.assertEquals(metricCount, ClusterMetrics.getMetrics()
    .getNumDecommisionedNMs());
}
Example 9
Source File: TestResourceTrackerService.java    From big-c with Apache License 2.0 4 votes vote down vote up 
/**
 * Decommissioning using a pre-configured exclude hosts file
 */
@Test
public void testDecommissionWithExcludeHosts() throws Exception {
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH, hostFile
      .getAbsolutePath());

  writeToHostsFile("");
  final DrainDispatcher dispatcher = new DrainDispatcher();
  rm = new MockRM(conf) {
    @Override
    protected Dispatcher createDispatcher() {
      return dispatcher;
    }
  };
  rm.start();

  MockNM nm1 = rm.registerNode("host1:1234", 5120);
  MockNM nm2 = rm.registerNode("host2:5678", 10240);
  MockNM nm3 = rm.registerNode("localhost:4433", 1024);

  dispatcher.await();

  int metricCount = ClusterMetrics.getMetrics().getNumDecommisionedNMs();
  NodeHeartbeatResponse nodeHeartbeat = nm1.nodeHeartbeat(true);
  Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat.getNodeAction()));
  nodeHeartbeat = nm2.nodeHeartbeat(true);
  Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat.getNodeAction()));
  dispatcher.await();

  // To test that IPs also work
  String ip = NetUtils.normalizeHostName("localhost");
  writeToHostsFile("host2", ip);

  rm.getNodesListManager().refreshNodes(conf);

  checkDecommissionedNMCount(rm, metricCount + 2);

  nodeHeartbeat = nm1.nodeHeartbeat(true);
  Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat.getNodeAction()));
  nodeHeartbeat = nm2.nodeHeartbeat(true);
  Assert.assertTrue("The decommisioned metrics are not updated",
      NodeAction.SHUTDOWN.equals(nodeHeartbeat.getNodeAction()));

  nodeHeartbeat = nm3.nodeHeartbeat(true);
  Assert.assertTrue("The decommisioned metrics are not updated",
      NodeAction.SHUTDOWN.equals(nodeHeartbeat.getNodeAction()));
  dispatcher.await();

  writeToHostsFile("");
  rm.getNodesListManager().refreshNodes(conf);

  nm3 = rm.registerNode("localhost:4433", 1024);
  dispatcher.await();
  nodeHeartbeat = nm3.nodeHeartbeat(true);
  dispatcher.await();
  Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat.getNodeAction()));
  // decommissined node is 1 since 1 node is rejoined after updating exclude
  // file
  checkDecommissionedNMCount(rm, metricCount + 1);
}