Java Code Examples for org.apache.hadoop.net.NetUtils#addStaticResolution()

@Parameters({
        "hive.hadoop2.metastoreHost",
        "hive.hadoop2.metastorePort",
        "hive.hadoop2.databaseName",
        "hive.hadoop2.hiveVersionMajor",
        "hive.hadoop2.timeZone",
})
@BeforeClass
public void initialize(String host, int port, String databaseName, int hiveVersionMajor, String timeZone)
{
    String hadoopMasterIp = System.getProperty("hadoop-master-ip");
    if (hadoopMasterIp != null) {
        // Even though Hadoop is accessed by proxy, Hadoop still tries to resolve hadoop-master
        // (e.g: in: NameNodeProxies.createProxy)
        // This adds a static resolution for hadoop-master to docker container internal ip
        NetUtils.addStaticResolution("hadoop-master", hadoopMasterIp);
    }

    setup(host, port, databaseName, timeZone);

    checkArgument(hiveVersionMajor > 0, "Invalid hiveVersionMajor: %s", hiveVersionMajor);
    this.hiveVersionMajor = hiveVersionMajor;
}
 

public void setConf(Configuration conf) {
  if (conf instanceof JobConf) {
    this.conf = (JobConf) conf;
  } else {
    this.conf = new JobConf(conf);
  }
  this.mapOutputFile = ReflectionUtils.newInstance(
      conf.getClass(MRConfig.TASK_LOCAL_OUTPUT_CLASS,
        MROutputFiles.class, MapOutputFile.class), conf);
  this.lDirAlloc = new LocalDirAllocator(MRConfig.LOCAL_DIR);
  // add the static resolutions (this is required for the junit to
  // work on testcases that simulate multiple nodes on a single physical
  // node.
  String hostToResolved[] = conf.getStrings(MRConfig.STATIC_RESOLUTIONS);
  if (hostToResolved != null) {
    for (String str : hostToResolved) {
      String name = str.substring(0, str.indexOf('='));
      String resolvedName = str.substring(str.indexOf('=') + 1);
      NetUtils.addStaticResolution(name, resolvedName);
    }
  }
}
 

public void setConf(Configuration conf) {
  if (conf instanceof JobConf) {
    this.conf = (JobConf) conf;
  } else {
    this.conf = new JobConf(conf);
  }
  this.mapOutputFile.setConf(this.conf);
  this.lDirAlloc = new LocalDirAllocator("mapred.local.dir");
  // add the static resolutions (this is required for the junit to
  // work on testcases that simulate multiple nodes on a single physical
  // node.
  String hostToResolved[] = conf.getStrings("hadoop.net.static.resolutions");
  if (hostToResolved != null) {
    for (String str : hostToResolved) {
      String name = str.substring(0, str.indexOf('='));
      String resolvedName = str.substring(str.indexOf('=') + 1);
      NetUtils.addStaticResolution(name, resolvedName);
    }
  }
}
 

/**
 * Start the tasktracker.
 */
public void startTaskTracker(String host, String rack, int idx, int numDir) 
throws IOException {
  if (rack != null) {
    StaticMapping.addNodeToRack(host, rack);
  }
  if (host != null) {
    NetUtils.addStaticResolution(host, "localhost");
  }
  TaskTrackerRunner taskTracker;
  taskTracker = new TaskTrackerRunner(idx, numDir, host, conf);
  
  Thread taskTrackerThread = new Thread(taskTracker);
  taskTrackerList.add(taskTracker);
  taskTrackerThreadList.add(taskTrackerThread);
  taskTrackerThread.start();
  ++numTaskTrackers;
}
 

@BeforeClass
public static void initializeDummyHostnameResolution() throws Exception {
  String previousIpAddress;
  for (String hostName : dummyHostNames) {
    if (null != (previousIpAddress = NetUtils.getStaticResolution(hostName))) {
      savedStaticResolution.put(hostName, previousIpAddress);
    }
    NetUtils.addStaticResolution(hostName, anyIpAddress);
  }
}
 

public void testSetSocketAddress() throws IOException {
  Configuration conf = new Configuration();
  NetUtils.addStaticResolution("host", "127.0.0.1");
  final String defaultAddr = "host:1";
  
  InetSocketAddress addr = NetUtils.createSocketAddr(defaultAddr);    
  conf.setSocketAddr("myAddress", addr);
  assertEquals(defaultAddr, NetUtils.getHostPortString(addr));
}
 

/**
 * Load the static resolutions from configuration. This is required for junit
 * to work on testcases that simulate multiple nodes on a single physical
 * node.
 * @param conf The configuration.
 */
public static void loadStaticResolutions(Configuration conf) {
  String hostToResolved[] = conf.getStrings("hadoop.net.static.resolutions");
  if (hostToResolved != null) {
    for (String str : hostToResolved) {
      String name = str.substring(0, str.indexOf('='));
      String resolvedName = str.substring(str.indexOf('=') + 1);
      NetUtils.addStaticResolution(name, resolvedName);
    }
  }
}
 

@Override
protected void setUp() throws Exception {
  super.setUp();
  NetUtils.addStaticResolution("host1", "172.0.0.1");
  NetUtils.addStaticResolution("host2", "172.0.0.1");
  NetUtils.addStaticResolution("host3", "172.0.0.1");
  NetUtils.addStaticResolution("host4", "172.0.0.1");
  NetUtils.addStaticResolution("host5", "172.0.0.1");
  NetUtils.addStaticResolution("host6", "172.0.0.1");
}
 

public void testSetSocketAddress() throws IOException {
  Configuration conf = new Configuration();
  NetUtils.addStaticResolution("host", "127.0.0.1");
  final String defaultAddr = "host:1";
  
  InetSocketAddress addr = NetUtils.createSocketAddr(defaultAddr);    
  conf.setSocketAddr("myAddress", addr);
  assertEquals(defaultAddr, NetUtils.getHostPortString(addr));
}
 

/**
 * Start the tasktracker.
 */
public void startTaskTracker(String host, String rack, int idx, int numDir) 
throws IOException {
  if (rack != null) {
    StaticMapping.addNodeToRack(host, rack);
  }
  if (host != null) {
    NetUtils.addStaticResolution(host, "localhost");
  }
  TaskTrackerRunner taskTracker;
  taskTracker = new TaskTrackerRunner(idx, numDir, host, conf);
  
  addTaskTracker(taskTracker);
}
 

@Test
public void testSocketAddrWithNameToStaticName() {
  String staticHost = "host1";
  NetUtils.addStaticResolution(staticHost, "localhost");
  verifyServiceAddr(staticHost, "127.0.0.1");
}
 

@Test
public void testSocketAddrWithName() {
  String staticHost = "my";
  NetUtils.addStaticResolution(staticHost, "localhost");
  verifyServiceAddr("LocalHost", "127.0.0.1");
}
 

private void startDataNodes() throws IOException {
  if (racks != null && numDataNodes > racks.length ) {
    throw new IllegalArgumentException( "The length of racks [" + 
                                        racks.length +
                                        "] is less than the number " +
                                        "of datanodes [" +
                                        numDataNodes + "].");
  }
  if (hosts != null && numDataNodes > hosts.length ) {
    throw new IllegalArgumentException( "The length of hosts [" + 
                                        hosts.length +
                                        "] is less than the number " +
                                        "of datanodes [" +
                                        numDataNodes + "].");
  }

  //Generate some hostnames if required
  if (racks != null && hosts == null) {
    LOG.info("Generating host names for datanodes");
    hosts = new String[numDataNodes];
    for (int i = 0; i < numDataNodes; i++) {
      hosts[i] = "host" + i + ".foo.com";
    }
  }
  
  
  String[] dnArgs = { HdfsConstants.StartupOption.REGULAR.getName() };
  
  for (int i = 0; i < numDataNodes; i++) {
    Configuration dnConf = new Configuration(conf);

    File dir1 = new File(dataDir, "data"+(2*i+1));
    File dir2 = new File(dataDir, "data"+(2*i+2));
    dir1.mkdirs();
    dir2.mkdirs();
    if (!dir1.isDirectory() || !dir2.isDirectory()) { 
      throw new IOException("Mkdirs failed to create directory for DataNode "
                            + i + ": " + dir1 + " or " + dir2);
    }
    dnConf.set("dfs.data.dir", dir1.getPath() + "," + dir2.getPath()); 

    LOG.info("Starting DataNode " + i + " with dfs.data.dir: " 
                       + dnConf.get("dfs.data.dir"));
    
    if (hosts != null) {
      dnConf.set("slave.host.name", hosts[i]);
      LOG.info("Starting DataNode " + i + " with hostname set to: " 
                         + dnConf.get("slave.host.name"));
    }

    if (racks != null) {
      String name = hosts[i];
      LOG.info("Adding node with hostname : " + name + " to rack "+
                         racks[i]);
      StaticMapping.addNodeToRack(name,
                                  racks[i]);
    }
    Configuration newconf = new Configuration(dnConf); // save config
    if (hosts != null) {
      NetUtils.addStaticResolution(hosts[i], "localhost");
    }
    AvatarDataNode dn = AvatarDataNode.instantiateDataNode(dnArgs, dnConf);
    //since the HDFS does things based on IP:port, we need to add the mapping
    //for IP:port to rackId
    
    String ipAddr = dn.getSelfAddr().getAddress().getHostAddress();
    if (racks != null) {
      int port = dn.getSelfAddr().getPort();
      System.out.println("Adding node with IP:port : " + ipAddr + ":" + port+
                          " to rack " + racks[i]);
      StaticMapping.addNodeToRack(ipAddr + ":" + port,
                                racks[i]);
    }
    dn.runDatanodeDaemon();
    dataNodes.add(new DataNodeProperties(dn, newconf, dnArgs));

  }

}
 

@Test
public void testSocketAddrWithIPToStaticIP() {
  String staticHost = "1.2.3.4";
  NetUtils.addStaticResolution(staticHost, "255.255.255.255");
  verifyServiceAddr(staticHost, "255.255.255.255");
}
 

@Test
public void testSocketAddrWithNameToStaticIP() {
  String staticHost = "host3";
  NetUtils.addStaticResolution(staticHost, "255.255.255.255");
  verifyServiceAddr(staticHost, "255.255.255.255");
}
 

@Test
public void testSocketAddrWithIP() {
  String staticHost = "127.0.0.1";
  NetUtils.addStaticResolution(staticHost, "localhost");
  verifyServiceAddr(staticHost, "127.0.0.1");
}
 

@Test
public void testSocketAddrWithName() {
  String staticHost = "my";
  NetUtils.addStaticResolution(staticHost, "localhost");
  verifyServiceAddr("LocalHost", "127.0.0.1");
}
 

/**
 * Add namenodes to the static resolution list to avoid going
 * through DNS which can be really slow in some configurations.
 */
private void setupStaticHostResolution(int nameServiceIdCount) {
  for (int i = 0; i < nameServiceIdCount; i++) {
    NetUtils.addStaticResolution("nn" + i, "localhost");
  }
}
 

@Test
public void testSocketAddrWithIPToStaticIP() {
  String staticHost = "1.2.3.4";
  NetUtils.addStaticResolution(staticHost, "255.255.255.255");
  verifyServiceAddr(staticHost, "255.255.255.255");
}
 

@AfterClass
public static void restoreDummyHostnameResolution() throws Exception {
  for (Map.Entry<String, String> hostnameToIpEntry : savedStaticResolution.entrySet()) {
    NetUtils.addStaticResolution(hostnameToIpEntry.getKey(), hostnameToIpEntry.getValue());
  }
}