Java Code Examples for org.apache.hadoop.net.NetUtils#addStaticResolution()
The following examples show how to use
org.apache.hadoop.net.NetUtils#addStaticResolution() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestHive.java From presto with Apache License 2.0 | 6 votes |
@Parameters({ "hive.hadoop2.metastoreHost", "hive.hadoop2.metastorePort", "hive.hadoop2.databaseName", "hive.hadoop2.hiveVersionMajor", "hive.hadoop2.timeZone", }) @BeforeClass public void initialize(String host, int port, String databaseName, int hiveVersionMajor, String timeZone) { String hadoopMasterIp = System.getProperty("hadoop-master-ip"); if (hadoopMasterIp != null) { // Even though Hadoop is accessed by proxy, Hadoop still tries to resolve hadoop-master // (e.g: in: NameNodeProxies.createProxy) // This adds a static resolution for hadoop-master to docker container internal ip NetUtils.addStaticResolution("hadoop-master", hadoopMasterIp); } setup(host, port, databaseName, timeZone); checkArgument(hiveVersionMajor > 0, "Invalid hiveVersionMajor: %s", hiveVersionMajor); this.hiveVersionMajor = hiveVersionMajor; }
Example 2
Source File: Task.java From hadoop with Apache License 2.0 | 6 votes |
public void setConf(Configuration conf) { if (conf instanceof JobConf) { this.conf = (JobConf) conf; } else { this.conf = new JobConf(conf); } this.mapOutputFile = ReflectionUtils.newInstance( conf.getClass(MRConfig.TASK_LOCAL_OUTPUT_CLASS, MROutputFiles.class, MapOutputFile.class), conf); this.lDirAlloc = new LocalDirAllocator(MRConfig.LOCAL_DIR); // add the static resolutions (this is required for the junit to // work on testcases that simulate multiple nodes on a single physical // node. String hostToResolved[] = conf.getStrings(MRConfig.STATIC_RESOLUTIONS); if (hostToResolved != null) { for (String str : hostToResolved) { String name = str.substring(0, str.indexOf('=')); String resolvedName = str.substring(str.indexOf('=') + 1); NetUtils.addStaticResolution(name, resolvedName); } } }
Example 3
Source File: Task.java From hadoop-gpu with Apache License 2.0 | 6 votes |
public void setConf(Configuration conf) { if (conf instanceof JobConf) { this.conf = (JobConf) conf; } else { this.conf = new JobConf(conf); } this.mapOutputFile.setConf(this.conf); this.lDirAlloc = new LocalDirAllocator("mapred.local.dir"); // add the static resolutions (this is required for the junit to // work on testcases that simulate multiple nodes on a single physical // node. String hostToResolved[] = conf.getStrings("hadoop.net.static.resolutions"); if (hostToResolved != null) { for (String str : hostToResolved) { String name = str.substring(0, str.indexOf('=')); String resolvedName = str.substring(str.indexOf('=') + 1); NetUtils.addStaticResolution(name, resolvedName); } } }
Example 4
Source File: MiniMRCluster.java From hadoop-gpu with Apache License 2.0 | 6 votes |
/** * Start the tasktracker. */ public void startTaskTracker(String host, String rack, int idx, int numDir) throws IOException { if (rack != null) { StaticMapping.addNodeToRack(host, rack); } if (host != null) { NetUtils.addStaticResolution(host, "localhost"); } TaskTrackerRunner taskTracker; taskTracker = new TaskTrackerRunner(idx, numDir, host, conf); Thread taskTrackerThread = new Thread(taskTracker); taskTrackerList.add(taskTracker); taskTrackerThreadList.add(taskTrackerThread); taskTrackerThread.start(); ++numTaskTrackers; }
Example 5
Source File: TestWebAppUtils.java From hadoop with Apache License 2.0 | 5 votes |
@BeforeClass public static void initializeDummyHostnameResolution() throws Exception { String previousIpAddress; for (String hostName : dummyHostNames) { if (null != (previousIpAddress = NetUtils.getStaticResolution(hostName))) { savedStaticResolution.put(hostName, previousIpAddress); } NetUtils.addStaticResolution(hostName, anyIpAddress); } }
Example 6
Source File: TestConfiguration.java From big-c with Apache License 2.0 | 5 votes |
public void testSetSocketAddress() throws IOException { Configuration conf = new Configuration(); NetUtils.addStaticResolution("host", "127.0.0.1"); final String defaultAddr = "host:1"; InetSocketAddress addr = NetUtils.createSocketAddr(defaultAddr); conf.setSocketAddr("myAddress", addr); assertEquals(defaultAddr, NetUtils.getHostPortString(addr)); }
Example 7
Source File: Task.java From RDFS with Apache License 2.0 | 5 votes |
/** * Load the static resolutions from configuration. This is required for junit * to work on testcases that simulate multiple nodes on a single physical * node. * @param conf The configuration. */ public static void loadStaticResolutions(Configuration conf) { String hostToResolved[] = conf.getStrings("hadoop.net.static.resolutions"); if (hostToResolved != null) { for (String str : hostToResolved) { String name = str.substring(0, str.indexOf('=')); String resolvedName = str.substring(str.indexOf('=') + 1); NetUtils.addStaticResolution(name, resolvedName); } } }
Example 8
Source File: TestAmFilterInitializer.java From big-c with Apache License 2.0 | 5 votes |
@Override protected void setUp() throws Exception { super.setUp(); NetUtils.addStaticResolution("host1", "172.0.0.1"); NetUtils.addStaticResolution("host2", "172.0.0.1"); NetUtils.addStaticResolution("host3", "172.0.0.1"); NetUtils.addStaticResolution("host4", "172.0.0.1"); NetUtils.addStaticResolution("host5", "172.0.0.1"); NetUtils.addStaticResolution("host6", "172.0.0.1"); }
Example 9
Source File: TestConfiguration.java From hadoop with Apache License 2.0 | 5 votes |
public void testSetSocketAddress() throws IOException { Configuration conf = new Configuration(); NetUtils.addStaticResolution("host", "127.0.0.1"); final String defaultAddr = "host:1"; InetSocketAddress addr = NetUtils.createSocketAddr(defaultAddr); conf.setSocketAddr("myAddress", addr); assertEquals(defaultAddr, NetUtils.getHostPortString(addr)); }
Example 10
Source File: MiniMRCluster.java From RDFS with Apache License 2.0 | 5 votes |
/** * Start the tasktracker. */ public void startTaskTracker(String host, String rack, int idx, int numDir) throws IOException { if (rack != null) { StaticMapping.addNodeToRack(host, rack); } if (host != null) { NetUtils.addStaticResolution(host, "localhost"); } TaskTrackerRunner taskTracker; taskTracker = new TaskTrackerRunner(idx, numDir, host, conf); addTaskTracker(taskTracker); }
Example 11
Source File: TestSecurityUtil.java From big-c with Apache License 2.0 | 4 votes |
@Test public void testSocketAddrWithNameToStaticName() { String staticHost = "host1"; NetUtils.addStaticResolution(staticHost, "localhost"); verifyServiceAddr(staticHost, "127.0.0.1"); }
Example 12
Source File: TestSecurityUtil.java From big-c with Apache License 2.0 | 4 votes |
@Test public void testSocketAddrWithName() { String staticHost = "my"; NetUtils.addStaticResolution(staticHost, "localhost"); verifyServiceAddr("LocalHost", "127.0.0.1"); }
Example 13
Source File: MiniAvatarCluster.java From RDFS with Apache License 2.0 | 4 votes |
private void startDataNodes() throws IOException { if (racks != null && numDataNodes > racks.length ) { throw new IllegalArgumentException( "The length of racks [" + racks.length + "] is less than the number " + "of datanodes [" + numDataNodes + "]."); } if (hosts != null && numDataNodes > hosts.length ) { throw new IllegalArgumentException( "The length of hosts [" + hosts.length + "] is less than the number " + "of datanodes [" + numDataNodes + "]."); } //Generate some hostnames if required if (racks != null && hosts == null) { LOG.info("Generating host names for datanodes"); hosts = new String[numDataNodes]; for (int i = 0; i < numDataNodes; i++) { hosts[i] = "host" + i + ".foo.com"; } } String[] dnArgs = { HdfsConstants.StartupOption.REGULAR.getName() }; for (int i = 0; i < numDataNodes; i++) { Configuration dnConf = new Configuration(conf); File dir1 = new File(dataDir, "data"+(2*i+1)); File dir2 = new File(dataDir, "data"+(2*i+2)); dir1.mkdirs(); dir2.mkdirs(); if (!dir1.isDirectory() || !dir2.isDirectory()) { throw new IOException("Mkdirs failed to create directory for DataNode " + i + ": " + dir1 + " or " + dir2); } dnConf.set("dfs.data.dir", dir1.getPath() + "," + dir2.getPath()); LOG.info("Starting DataNode " + i + " with dfs.data.dir: " + dnConf.get("dfs.data.dir")); if (hosts != null) { dnConf.set("slave.host.name", hosts[i]); LOG.info("Starting DataNode " + i + " with hostname set to: " + dnConf.get("slave.host.name")); } if (racks != null) { String name = hosts[i]; LOG.info("Adding node with hostname : " + name + " to rack "+ racks[i]); StaticMapping.addNodeToRack(name, racks[i]); } Configuration newconf = new Configuration(dnConf); // save config if (hosts != null) { NetUtils.addStaticResolution(hosts[i], "localhost"); } AvatarDataNode dn = AvatarDataNode.instantiateDataNode(dnArgs, dnConf); //since the HDFS does things based on IP:port, we need to add the mapping //for IP:port to rackId String ipAddr = dn.getSelfAddr().getAddress().getHostAddress(); if (racks != null) { int port = dn.getSelfAddr().getPort(); System.out.println("Adding node with IP:port : " + ipAddr + ":" + port+ " to rack " + racks[i]); StaticMapping.addNodeToRack(ipAddr + ":" + port, racks[i]); } dn.runDatanodeDaemon(); dataNodes.add(new DataNodeProperties(dn, newconf, dnArgs)); } }
Example 14
Source File: TestSecurityUtil.java From hadoop with Apache License 2.0 | 4 votes |
@Test public void testSocketAddrWithIPToStaticIP() { String staticHost = "1.2.3.4"; NetUtils.addStaticResolution(staticHost, "255.255.255.255"); verifyServiceAddr(staticHost, "255.255.255.255"); }
Example 15
Source File: TestSecurityUtil.java From hadoop with Apache License 2.0 | 4 votes |
@Test public void testSocketAddrWithNameToStaticIP() { String staticHost = "host3"; NetUtils.addStaticResolution(staticHost, "255.255.255.255"); verifyServiceAddr(staticHost, "255.255.255.255"); }
Example 16
Source File: TestSecurityUtil.java From hadoop with Apache License 2.0 | 4 votes |
@Test public void testSocketAddrWithIP() { String staticHost = "127.0.0.1"; NetUtils.addStaticResolution(staticHost, "localhost"); verifyServiceAddr(staticHost, "127.0.0.1"); }
Example 17
Source File: TestSecurityUtil.java From hadoop with Apache License 2.0 | 4 votes |
@Test public void testSocketAddrWithName() { String staticHost = "my"; NetUtils.addStaticResolution(staticHost, "localhost"); verifyServiceAddr("LocalHost", "127.0.0.1"); }
Example 18
Source File: TestGetConf.java From hadoop with Apache License 2.0 | 4 votes |
/** * Add namenodes to the static resolution list to avoid going * through DNS which can be really slow in some configurations. */ private void setupStaticHostResolution(int nameServiceIdCount) { for (int i = 0; i < nameServiceIdCount; i++) { NetUtils.addStaticResolution("nn" + i, "localhost"); } }
Example 19
Source File: TestSecurityUtil.java From big-c with Apache License 2.0 | 4 votes |
@Test public void testSocketAddrWithIPToStaticIP() { String staticHost = "1.2.3.4"; NetUtils.addStaticResolution(staticHost, "255.255.255.255"); verifyServiceAddr(staticHost, "255.255.255.255"); }
Example 20
Source File: TestWebAppUtils.java From hadoop with Apache License 2.0 | 4 votes |
@AfterClass public static void restoreDummyHostnameResolution() throws Exception { for (Map.Entry<String, String> hostnameToIpEntry : savedStaticResolution.entrySet()) { NetUtils.addStaticResolution(hostnameToIpEntry.getKey(), hostnameToIpEntry.getValue()); } }