Java Code Examples for org.apache.hadoop.net.NetUtils#addStaticResolution()

The following examples show how to use org.apache.hadoop.net.NetUtils#addStaticResolution() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestHive.java    From presto with Apache License 2.0 6 votes vote down vote up
@Parameters({
        "hive.hadoop2.metastoreHost",
        "hive.hadoop2.metastorePort",
        "hive.hadoop2.databaseName",
        "hive.hadoop2.hiveVersionMajor",
        "hive.hadoop2.timeZone",
})
@BeforeClass
public void initialize(String host, int port, String databaseName, int hiveVersionMajor, String timeZone)
{
    String hadoopMasterIp = System.getProperty("hadoop-master-ip");
    if (hadoopMasterIp != null) {
        // Even though Hadoop is accessed by proxy, Hadoop still tries to resolve hadoop-master
        // (e.g: in: NameNodeProxies.createProxy)
        // This adds a static resolution for hadoop-master to docker container internal ip
        NetUtils.addStaticResolution("hadoop-master", hadoopMasterIp);
    }

    setup(host, port, databaseName, timeZone);

    checkArgument(hiveVersionMajor > 0, "Invalid hiveVersionMajor: %s", hiveVersionMajor);
    this.hiveVersionMajor = hiveVersionMajor;
}
 
Example 2
Source File: Task.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public void setConf(Configuration conf) {
  if (conf instanceof JobConf) {
    this.conf = (JobConf) conf;
  } else {
    this.conf = new JobConf(conf);
  }
  this.mapOutputFile = ReflectionUtils.newInstance(
      conf.getClass(MRConfig.TASK_LOCAL_OUTPUT_CLASS,
        MROutputFiles.class, MapOutputFile.class), conf);
  this.lDirAlloc = new LocalDirAllocator(MRConfig.LOCAL_DIR);
  // add the static resolutions (this is required for the junit to
  // work on testcases that simulate multiple nodes on a single physical
  // node.
  String hostToResolved[] = conf.getStrings(MRConfig.STATIC_RESOLUTIONS);
  if (hostToResolved != null) {
    for (String str : hostToResolved) {
      String name = str.substring(0, str.indexOf('='));
      String resolvedName = str.substring(str.indexOf('=') + 1);
      NetUtils.addStaticResolution(name, resolvedName);
    }
  }
}
 
Example 3
Source File: Task.java    From hadoop-gpu with Apache License 2.0 6 votes vote down vote up
public void setConf(Configuration conf) {
  if (conf instanceof JobConf) {
    this.conf = (JobConf) conf;
  } else {
    this.conf = new JobConf(conf);
  }
  this.mapOutputFile.setConf(this.conf);
  this.lDirAlloc = new LocalDirAllocator("mapred.local.dir");
  // add the static resolutions (this is required for the junit to
  // work on testcases that simulate multiple nodes on a single physical
  // node.
  String hostToResolved[] = conf.getStrings("hadoop.net.static.resolutions");
  if (hostToResolved != null) {
    for (String str : hostToResolved) {
      String name = str.substring(0, str.indexOf('='));
      String resolvedName = str.substring(str.indexOf('=') + 1);
      NetUtils.addStaticResolution(name, resolvedName);
    }
  }
}
 
Example 4
Source File: MiniMRCluster.java    From hadoop-gpu with Apache License 2.0 6 votes vote down vote up
/**
 * Start the tasktracker.
 */
public void startTaskTracker(String host, String rack, int idx, int numDir) 
throws IOException {
  if (rack != null) {
    StaticMapping.addNodeToRack(host, rack);
  }
  if (host != null) {
    NetUtils.addStaticResolution(host, "localhost");
  }
  TaskTrackerRunner taskTracker;
  taskTracker = new TaskTrackerRunner(idx, numDir, host, conf);
  
  Thread taskTrackerThread = new Thread(taskTracker);
  taskTrackerList.add(taskTracker);
  taskTrackerThreadList.add(taskTrackerThread);
  taskTrackerThread.start();
  ++numTaskTrackers;
}
 
Example 5
Source File: TestWebAppUtils.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void initializeDummyHostnameResolution() throws Exception {
  String previousIpAddress;
  for (String hostName : dummyHostNames) {
    if (null != (previousIpAddress = NetUtils.getStaticResolution(hostName))) {
      savedStaticResolution.put(hostName, previousIpAddress);
    }
    NetUtils.addStaticResolution(hostName, anyIpAddress);
  }
}
 
Example 6
Source File: TestConfiguration.java    From big-c with Apache License 2.0 5 votes vote down vote up
public void testSetSocketAddress() throws IOException {
  Configuration conf = new Configuration();
  NetUtils.addStaticResolution("host", "127.0.0.1");
  final String defaultAddr = "host:1";
  
  InetSocketAddress addr = NetUtils.createSocketAddr(defaultAddr);    
  conf.setSocketAddr("myAddress", addr);
  assertEquals(defaultAddr, NetUtils.getHostPortString(addr));
}
 
Example 7
Source File: Task.java    From RDFS with Apache License 2.0 5 votes vote down vote up
/**
 * Load the static resolutions from configuration. This is required for junit
 * to work on testcases that simulate multiple nodes on a single physical
 * node.
 * @param conf The configuration.
 */
public static void loadStaticResolutions(Configuration conf) {
  String hostToResolved[] = conf.getStrings("hadoop.net.static.resolutions");
  if (hostToResolved != null) {
    for (String str : hostToResolved) {
      String name = str.substring(0, str.indexOf('='));
      String resolvedName = str.substring(str.indexOf('=') + 1);
      NetUtils.addStaticResolution(name, resolvedName);
    }
  }
}
 
Example 8
Source File: TestAmFilterInitializer.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
protected void setUp() throws Exception {
  super.setUp();
  NetUtils.addStaticResolution("host1", "172.0.0.1");
  NetUtils.addStaticResolution("host2", "172.0.0.1");
  NetUtils.addStaticResolution("host3", "172.0.0.1");
  NetUtils.addStaticResolution("host4", "172.0.0.1");
  NetUtils.addStaticResolution("host5", "172.0.0.1");
  NetUtils.addStaticResolution("host6", "172.0.0.1");
}
 
Example 9
Source File: TestConfiguration.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public void testSetSocketAddress() throws IOException {
  Configuration conf = new Configuration();
  NetUtils.addStaticResolution("host", "127.0.0.1");
  final String defaultAddr = "host:1";
  
  InetSocketAddress addr = NetUtils.createSocketAddr(defaultAddr);    
  conf.setSocketAddr("myAddress", addr);
  assertEquals(defaultAddr, NetUtils.getHostPortString(addr));
}
 
Example 10
Source File: MiniMRCluster.java    From RDFS with Apache License 2.0 5 votes vote down vote up
/**
 * Start the tasktracker.
 */
public void startTaskTracker(String host, String rack, int idx, int numDir) 
throws IOException {
  if (rack != null) {
    StaticMapping.addNodeToRack(host, rack);
  }
  if (host != null) {
    NetUtils.addStaticResolution(host, "localhost");
  }
  TaskTrackerRunner taskTracker;
  taskTracker = new TaskTrackerRunner(idx, numDir, host, conf);
  
  addTaskTracker(taskTracker);
}
 
Example 11
Source File: TestSecurityUtil.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test
public void testSocketAddrWithNameToStaticName() {
  String staticHost = "host1";
  NetUtils.addStaticResolution(staticHost, "localhost");
  verifyServiceAddr(staticHost, "127.0.0.1");
}
 
Example 12
Source File: TestSecurityUtil.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test
public void testSocketAddrWithName() {
  String staticHost = "my";
  NetUtils.addStaticResolution(staticHost, "localhost");
  verifyServiceAddr("LocalHost", "127.0.0.1");
}
 
Example 13
Source File: MiniAvatarCluster.java    From RDFS with Apache License 2.0 4 votes vote down vote up
private void startDataNodes() throws IOException {
  if (racks != null && numDataNodes > racks.length ) {
    throw new IllegalArgumentException( "The length of racks [" + 
                                        racks.length +
                                        "] is less than the number " +
                                        "of datanodes [" +
                                        numDataNodes + "].");
  }
  if (hosts != null && numDataNodes > hosts.length ) {
    throw new IllegalArgumentException( "The length of hosts [" + 
                                        hosts.length +
                                        "] is less than the number " +
                                        "of datanodes [" +
                                        numDataNodes + "].");
  }

  //Generate some hostnames if required
  if (racks != null && hosts == null) {
    LOG.info("Generating host names for datanodes");
    hosts = new String[numDataNodes];
    for (int i = 0; i < numDataNodes; i++) {
      hosts[i] = "host" + i + ".foo.com";
    }
  }
  
  
  String[] dnArgs = { HdfsConstants.StartupOption.REGULAR.getName() };
  
  for (int i = 0; i < numDataNodes; i++) {
    Configuration dnConf = new Configuration(conf);

    File dir1 = new File(dataDir, "data"+(2*i+1));
    File dir2 = new File(dataDir, "data"+(2*i+2));
    dir1.mkdirs();
    dir2.mkdirs();
    if (!dir1.isDirectory() || !dir2.isDirectory()) { 
      throw new IOException("Mkdirs failed to create directory for DataNode "
                            + i + ": " + dir1 + " or " + dir2);
    }
    dnConf.set("dfs.data.dir", dir1.getPath() + "," + dir2.getPath()); 

    LOG.info("Starting DataNode " + i + " with dfs.data.dir: " 
                       + dnConf.get("dfs.data.dir"));
    
    if (hosts != null) {
      dnConf.set("slave.host.name", hosts[i]);
      LOG.info("Starting DataNode " + i + " with hostname set to: " 
                         + dnConf.get("slave.host.name"));
    }

    if (racks != null) {
      String name = hosts[i];
      LOG.info("Adding node with hostname : " + name + " to rack "+
                         racks[i]);
      StaticMapping.addNodeToRack(name,
                                  racks[i]);
    }
    Configuration newconf = new Configuration(dnConf); // save config
    if (hosts != null) {
      NetUtils.addStaticResolution(hosts[i], "localhost");
    }
    AvatarDataNode dn = AvatarDataNode.instantiateDataNode(dnArgs, dnConf);
    //since the HDFS does things based on IP:port, we need to add the mapping
    //for IP:port to rackId
    
    String ipAddr = dn.getSelfAddr().getAddress().getHostAddress();
    if (racks != null) {
      int port = dn.getSelfAddr().getPort();
      System.out.println("Adding node with IP:port : " + ipAddr + ":" + port+
                          " to rack " + racks[i]);
      StaticMapping.addNodeToRack(ipAddr + ":" + port,
                                racks[i]);
    }
    dn.runDatanodeDaemon();
    dataNodes.add(new DataNodeProperties(dn, newconf, dnArgs));

  }

}
 
Example 14
Source File: TestSecurityUtil.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Test
public void testSocketAddrWithIPToStaticIP() {
  String staticHost = "1.2.3.4";
  NetUtils.addStaticResolution(staticHost, "255.255.255.255");
  verifyServiceAddr(staticHost, "255.255.255.255");
}
 
Example 15
Source File: TestSecurityUtil.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Test
public void testSocketAddrWithNameToStaticIP() {
  String staticHost = "host3";
  NetUtils.addStaticResolution(staticHost, "255.255.255.255");
  verifyServiceAddr(staticHost, "255.255.255.255");
}
 
Example 16
Source File: TestSecurityUtil.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Test
public void testSocketAddrWithIP() {
  String staticHost = "127.0.0.1";
  NetUtils.addStaticResolution(staticHost, "localhost");
  verifyServiceAddr(staticHost, "127.0.0.1");
}
 
Example 17
Source File: TestSecurityUtil.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Test
public void testSocketAddrWithName() {
  String staticHost = "my";
  NetUtils.addStaticResolution(staticHost, "localhost");
  verifyServiceAddr("LocalHost", "127.0.0.1");
}
 
Example 18
Source File: TestGetConf.java    From hadoop with Apache License 2.0 4 votes vote down vote up
/**
 * Add namenodes to the static resolution list to avoid going
 * through DNS which can be really slow in some configurations.
 */
private void setupStaticHostResolution(int nameServiceIdCount) {
  for (int i = 0; i < nameServiceIdCount; i++) {
    NetUtils.addStaticResolution("nn" + i, "localhost");
  }
}
 
Example 19
Source File: TestSecurityUtil.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test
public void testSocketAddrWithIPToStaticIP() {
  String staticHost = "1.2.3.4";
  NetUtils.addStaticResolution(staticHost, "255.255.255.255");
  verifyServiceAddr(staticHost, "255.255.255.255");
}
 
Example 20
Source File: TestWebAppUtils.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@AfterClass
public static void restoreDummyHostnameResolution() throws Exception {
  for (Map.Entry<String, String> hostnameToIpEntry : savedStaticResolution.entrySet()) {
    NetUtils.addStaticResolution(hostnameToIpEntry.getKey(), hostnameToIpEntry.getValue());
  }
}