Java Code Examples for org.apache.hadoop.net.DNS

The following examples show how to use org.apache.hadoop.net.DNS. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: DFSClient.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Return the socket addresses to use with each configured
 * local interface. Local interfaces may be specified by IP
 * address, IP address range using CIDR notation, interface
 * name (e.g. eth0) or sub-interface name (e.g. eth0:0).
 * The socket addresses consist of the IPs for the interfaces
 * and the ephemeral port (port 0). If an IP, IP range, or
 * interface name matches an interface with sub-interfaces
 * only the IP of the interface is used. Sub-interfaces can
 * be used by specifying them explicitly (by IP or name).
 * 
 * @return SocketAddresses for the configured local interfaces,
 *    or an empty array if none are configured
 * @throws UnknownHostException if a given interface name is invalid
 */
private static SocketAddress[] getLocalInterfaceAddrs(
    String interfaceNames[]) throws UnknownHostException {
  List<SocketAddress> localAddrs = new ArrayList<SocketAddress>();
  for (String interfaceName : interfaceNames) {
    if (InetAddresses.isInetAddress(interfaceName)) {
      localAddrs.add(new InetSocketAddress(interfaceName, 0));
    } else if (NetUtils.isValidSubnet(interfaceName)) {
      for (InetAddress addr : NetUtils.getIPs(interfaceName, false)) {
        localAddrs.add(new InetSocketAddress(addr, 0));
      }
    } else {
      for (String ip : DNS.getIPs(interfaceName, false)) {
        localAddrs.add(new InetSocketAddress(ip, 0));
      }
    }
  }
  return localAddrs.toArray(new SocketAddress[localAddrs.size()]);
}
 
Example 2
Source Project: hadoop   Source File: NNThroughputBenchmark.java    License: Apache License 2.0 6 votes vote down vote up
void register() throws IOException {
  // get versions from the namenode
  nsInfo = nameNodeProto.versionRequest();
  dnRegistration = new DatanodeRegistration(
      new DatanodeID(DNS.getDefaultIP("default"),
          DNS.getDefaultHost("default", "default"),
          DataNode.generateUuid(), getNodePort(dnIdx),
          DFSConfigKeys.DFS_DATANODE_HTTP_DEFAULT_PORT,
          DFSConfigKeys.DFS_DATANODE_HTTPS_DEFAULT_PORT,
          DFSConfigKeys.DFS_DATANODE_IPC_DEFAULT_PORT),
      new DataStorage(nsInfo),
      new ExportedBlockKeys(), VersionInfo.getVersion());
  // register datanode
  dnRegistration = nameNodeProto.registerDatanode(dnRegistration);
  //first block reports
  storage = new DatanodeStorage(DatanodeStorage.generateUuid());
  final StorageBlockReport[] reports = {
      new StorageBlockReport(storage, BlockListAsLongs.EMPTY)
  };
  nameNodeProto.blockReport(dnRegistration, 
      nameNode.getNamesystem().getBlockPoolId(), reports,
          new BlockReportContext(1, 0, System.nanoTime()));
}
 
Example 3
Source Project: hadoop   Source File: GangliaContext31.java    License: Apache License 2.0 6 votes vote down vote up
public void init(String contextName, ContextFactory factory) {
  super.init(contextName, factory);

  LOG.debug("Initializing the GangliaContext31 for Ganglia 3.1 metrics.");

  // Take the hostname from the DNS class.

  Configuration conf = new Configuration();

  if (conf.get("slave.host.name") != null) {
    hostName = conf.get("slave.host.name");
  } else {
    try {
      hostName = DNS.getDefaultHost(
        conf.get("dfs.datanode.dns.interface","default"),
        conf.get("dfs.datanode.dns.nameserver","default"));
    } catch (UnknownHostException uhe) {
      LOG.error(uhe);
  	hostName = "UNKNOWN.example.com";
    }
  }
}
 
Example 4
Source Project: big-c   Source File: DFSClient.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Return the socket addresses to use with each configured
 * local interface. Local interfaces may be specified by IP
 * address, IP address range using CIDR notation, interface
 * name (e.g. eth0) or sub-interface name (e.g. eth0:0).
 * The socket addresses consist of the IPs for the interfaces
 * and the ephemeral port (port 0). If an IP, IP range, or
 * interface name matches an interface with sub-interfaces
 * only the IP of the interface is used. Sub-interfaces can
 * be used by specifying them explicitly (by IP or name).
 * 
 * @return SocketAddresses for the configured local interfaces,
 *    or an empty array if none are configured
 * @throws UnknownHostException if a given interface name is invalid
 */
private static SocketAddress[] getLocalInterfaceAddrs(
    String interfaceNames[]) throws UnknownHostException {
  List<SocketAddress> localAddrs = new ArrayList<SocketAddress>();
  for (String interfaceName : interfaceNames) {
    if (InetAddresses.isInetAddress(interfaceName)) {
      localAddrs.add(new InetSocketAddress(interfaceName, 0));
    } else if (NetUtils.isValidSubnet(interfaceName)) {
      for (InetAddress addr : NetUtils.getIPs(interfaceName, false)) {
        localAddrs.add(new InetSocketAddress(addr, 0));
      }
    } else {
      for (String ip : DNS.getIPs(interfaceName, false)) {
        localAddrs.add(new InetSocketAddress(ip, 0));
      }
    }
  }
  return localAddrs.toArray(new SocketAddress[localAddrs.size()]);
}
 
Example 5
Source Project: big-c   Source File: NNThroughputBenchmark.java    License: Apache License 2.0 6 votes vote down vote up
void register() throws IOException {
  // get versions from the namenode
  nsInfo = nameNodeProto.versionRequest();
  dnRegistration = new DatanodeRegistration(
      new DatanodeID(DNS.getDefaultIP("default"),
          DNS.getDefaultHost("default", "default"),
          DataNode.generateUuid(), getNodePort(dnIdx),
          DFSConfigKeys.DFS_DATANODE_HTTP_DEFAULT_PORT,
          DFSConfigKeys.DFS_DATANODE_HTTPS_DEFAULT_PORT,
          DFSConfigKeys.DFS_DATANODE_IPC_DEFAULT_PORT),
      new DataStorage(nsInfo),
      new ExportedBlockKeys(), VersionInfo.getVersion());
  // register datanode
  dnRegistration = nameNodeProto.registerDatanode(dnRegistration);
  //first block reports
  storage = new DatanodeStorage(DatanodeStorage.generateUuid());
  final StorageBlockReport[] reports = {
      new StorageBlockReport(storage, BlockListAsLongs.EMPTY)
  };
  nameNodeProto.blockReport(dnRegistration, 
      nameNode.getNamesystem().getBlockPoolId(), reports,
          new BlockReportContext(1, 0, System.nanoTime()));
}
 
Example 6
Source Project: big-c   Source File: GangliaContext31.java    License: Apache License 2.0 6 votes vote down vote up
public void init(String contextName, ContextFactory factory) {
  super.init(contextName, factory);

  LOG.debug("Initializing the GangliaContext31 for Ganglia 3.1 metrics.");

  // Take the hostname from the DNS class.

  Configuration conf = new Configuration();

  if (conf.get("slave.host.name") != null) {
    hostName = conf.get("slave.host.name");
  } else {
    try {
      hostName = DNS.getDefaultHost(
        conf.get("dfs.datanode.dns.interface","default"),
        conf.get("dfs.datanode.dns.nameserver","default"));
    } catch (UnknownHostException uhe) {
      LOG.error(uhe);
  	hostName = "UNKNOWN.example.com";
    }
  }
}
 
Example 7
Source Project: HiveKudu-Handler   Source File: HiveKuduTableInputFormat.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * This method might seem alien, but we do this in order to resolve the hostnames the same way
 * Hadoop does. This ensures we get locality if Kudu is running along MR/YARN.
 * @param host hostname we got from the master
 * @param port port we got from the master
 * @return reverse DNS'd address
 */
private String reverseDNS(String host, Integer port) {
    LOG.warn("I was called : reverseDNS");
    String location = this.reverseDNSCacheMap.get(host);
    if (location != null) {
        return location;
    }
    // The below InetSocketAddress creation does a name resolution.
    InetSocketAddress isa = new InetSocketAddress(host, port);
    if (isa.isUnresolved()) {
        LOG.warn("Failed address resolve for: " + isa);
    }
    InetAddress tabletInetAddress = isa.getAddress();
    try {
        location = domainNamePointerToHostName(
                DNS.reverseDns(tabletInetAddress, this.nameServer));
        this.reverseDNSCacheMap.put(host, location);
    } catch (NamingException e) {
        LOG.warn("Cannot resolve the host name for " + tabletInetAddress + " because of " + e);
        location = host;
    }
    return location;
}
 
Example 8
Source Project: hbase   Source File: TableInputFormatBase.java    License: Apache License 2.0 6 votes vote down vote up
String reverseDNS(InetAddress ipAddress) throws UnknownHostException {
  String hostName = this.reverseDNSCacheMap.get(ipAddress);
  if (hostName == null) {
    String ipAddressString = null;
    try {
      ipAddressString = DNS.reverseDns(ipAddress, null);
    } catch (Exception e) {
      // We can use InetAddress in case the jndi failed to pull up the reverse DNS entry from the
      // name service. Also, in case of ipv6, we need to use the InetAddress since resolving
      // reverse DNS using jndi doesn't work well with ipv6 addresses.
      ipAddressString = InetAddress.getByName(ipAddress.getHostAddress()).getHostName();
    }
    if (ipAddressString == null) throw new UnknownHostException("No host found for " + ipAddress);
    hostName = Strings.domainNamePointerToHostName(ipAddressString);
    this.reverseDNSCacheMap.put(ipAddress, hostName);
  }
  return hostName;
}
 
Example 9
Source Project: RDFS   Source File: DataNodeCluster.java    License: Apache License 2.0 6 votes vote down vote up
static private String getUniqueRackPrefix() {

  String ip = "unknownIP";
  try {
    ip = DNS.getDefaultIP("default");
  } catch (UnknownHostException ignored) {
    System.out.println("Could not find ip address of \"default\" inteface.");
  }
  
  int rand = 0;
  try {
    rand = SecureRandom.getInstance("SHA1PRNG").nextInt(Integer.MAX_VALUE);
  } catch (NoSuchAlgorithmException e) {
    rand = (new Random()).nextInt(Integer.MAX_VALUE);
  }
  return "/Rack-" + rand + "-"+ ip  + "-" + 
                    System.currentTimeMillis();
}
 
Example 10
Source Project: RDFS   Source File: Standby.java    License: Apache License 2.0 6 votes vote down vote up
Standby(AvatarNode avatarNode, Configuration startupConf, Configuration conf) 
  throws IOException {
  this.running = true;
  this.avatarNode = avatarNode;
  this.confg = conf;
  this.startupConf = startupConf;
  this.fsImage = avatarNode.getFSImage();
  this.fsnamesys = avatarNode.getNamesystem();
  this.sleepBetweenErrors = startupConf.getInt("hdfs.avatarnode.sleep", 5000);
  initSecondary(startupConf); // start webserver for secondary namenode

  this.machineName =
    DNS.getDefaultHost(conf.get("dfs.namenode.dns.interface","default"),
                       conf.get("dfs.namenode.dns.nameserver","default"));
  LOG.info("machineName=" + machineName);
  
  this.editsFile = this.avatarNode.getRemoteEditsFile(conf);
  this.editsFileNew = this.avatarNode.getRemoteEditsFileNew(conf);
  
  InetSocketAddress addr = NameNode.getAddress(conf);
  this.tmpImageFileForValidation = new File("/tmp", 
      "hadoop_image." + addr.getHostName() + ":" + addr.getPort());
  checkpointStatus("No checkpoint initiated");
}
 
Example 11
Source Project: hadoop-gpu   Source File: DataNodeCluster.java    License: Apache License 2.0 6 votes vote down vote up
static private String getUniqueRackPrefix() {

  String ip = "unknownIP";
  try {
    ip = DNS.getDefaultIP("default");
  } catch (UnknownHostException ignored) {
    System.out.println("Could not find ip address of \"default\" inteface.");
  }
  
  int rand = 0;
  try {
    rand = SecureRandom.getInstance("SHA1PRNG").nextInt(Integer.MAX_VALUE);
  } catch (NoSuchAlgorithmException e) {
    rand = (new Random()).nextInt(Integer.MAX_VALUE);
  }
  return "/Rack-" + rand + "-"+ ip  + "-" + 
                    System.currentTimeMillis();
}
 
Example 12
Source Project: hadoop-ozone   Source File: HddsUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Returns the hostname for this datanode. If the hostname is not
 * explicitly configured in the given config, then it is determined
 * via the DNS class.
 *
 * @param conf Configuration
 *
 * @return the hostname (NB: may not be a FQDN)
 * @throws UnknownHostException if the dfs.datanode.dns.interface
 *    option is used and the hostname can not be determined
 */
public static String getHostName(ConfigurationSource conf)
    throws UnknownHostException {
  String name = conf.get(DFS_DATANODE_HOST_NAME_KEY);
  if (name == null) {
    String dnsInterface = conf.get(
        CommonConfigurationKeysPublic.HADOOP_SECURITY_DNS_INTERFACE_KEY);
    String nameServer = conf.get(
        CommonConfigurationKeysPublic.HADOOP_SECURITY_DNS_NAMESERVER_KEY);
    boolean fallbackToHosts = false;

    if (dnsInterface == null) {
      // Try the legacy configuration keys.
      dnsInterface = conf.get(DFS_DATANODE_DNS_INTERFACE_KEY);
      dnsInterface = conf.get(DFS_DATANODE_DNS_INTERFACE_KEY);
      nameServer = conf.get(DFS_DATANODE_DNS_NAMESERVER_KEY);
    } else {
      // If HADOOP_SECURITY_DNS_* is set then also attempt hosts file
      // resolution if DNS fails. We will not use hosts file resolution
      // by default to avoid breaking existing clusters.
      fallbackToHosts = true;
    }

    name = DNS.getDefaultHost(dnsInterface, nameServer, fallbackToHosts);
  }
  return name;
}
 
Example 13
Source Project: DataLink   Source File: HostnameSupplier.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public String get() {
    try {
        final String dnsInterface = hbaseConfig.get("hbase.regionserver.dns.interface", DEFAULT_DNS_INTERFACE);
        String hostname = Strings.domainNamePointerToHostName(DNS.getDefaultHost(dnsInterface));
        return hostname;
    } catch( UnknownHostException ukhe) {
        throw new RuntimeException(ukhe);
    }
}
 
Example 14
Source Project: pxf   Source File: SecureLogin.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Retrieve the name of the current host. The code is copied from org.hadoop.security.SecurityUtil class.
 * @param conf configuration
 * @return name of the host
 * @throws IOException
 */
private String getLocalHostName(@Nullable Configuration conf) throws IOException {
    if (conf != null) {
        String dnsInterface = conf.get("hadoop.security.dns.interface");
        String nameServer = conf.get("hadoop.security.dns.nameserver");
        if (dnsInterface != null) {
            return DNS.getDefaultHost(dnsInterface, nameServer, true);
        }
        if (nameServer != null) {
            throw new IllegalArgumentException("hadoop.security.dns.nameserver requires hadoop.security.dns.interface. Check your configuration.");
        }
    }
    return InetAddress.getLocalHost().getCanonicalHostName();
}
 
Example 15
Source Project: hadoop   Source File: RpcProgramNfs3.java    License: Apache License 2.0 5 votes vote down vote up
public static RpcProgramNfs3 createRpcProgramNfs3(NfsConfiguration config,
    DatagramSocket registrationSocket, boolean allowInsecurePorts)
    throws IOException {
  DefaultMetricsSystem.initialize("Nfs3");
  String displayName = DNS.getDefaultHost("default", "default")
      + config.getInt(NfsConfigKeys.DFS_NFS_SERVER_PORT_KEY,
          NfsConfigKeys.DFS_NFS_SERVER_PORT_DEFAULT);
  metrics = Nfs3Metrics.create(config, displayName);
  return new RpcProgramNfs3(config, registrationSocket, allowInsecurePorts);
}
 
Example 16
Source Project: hadoop   Source File: NNStorage.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Generate new blockpoolID.
 * 
 * @return new blockpoolID
 */ 
static String newBlockPoolID() throws UnknownHostException{
  String ip = "unknownIP";
  try {
    ip = DNS.getDefaultIP("default");
  } catch (UnknownHostException e) {
    LOG.warn("Could not find ip address of \"default\" inteface.");
    throw e;
  }
  
  int rand = DFSUtil.getSecureRandom().nextInt(Integer.MAX_VALUE);
  String bpid = "BP-" + rand + "-"+ ip + "-" + Time.now();
  return bpid;
}
 
Example 17
Source Project: hadoop   Source File: DataNode.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Returns the hostname for this datanode. If the hostname is not
 * explicitly configured in the given config, then it is determined
 * via the DNS class.
 *
 * @param config configuration
 * @return the hostname (NB: may not be a FQDN)
 * @throws UnknownHostException if the dfs.datanode.dns.interface
 *    option is used and the hostname can not be determined
 */
private static String getHostName(Configuration config)
    throws UnknownHostException {
  String name = config.get(DFS_DATANODE_HOST_NAME_KEY);
  if (name == null) {
    name = DNS.getDefaultHost(
        config.get(DFS_DATANODE_DNS_INTERFACE_KEY,
                   DFS_DATANODE_DNS_INTERFACE_DEFAULT),
        config.get(DFS_DATANODE_DNS_NAMESERVER_KEY,
                   DFS_DATANODE_DNS_NAMESERVER_DEFAULT));
  }
  return name;
}
 
Example 18
Source Project: hadoop   Source File: DataNodeCluster.java    License: Apache License 2.0 5 votes vote down vote up
static private String getUniqueRackPrefix() {

  String ip = "unknownIP";
  try {
    ip = DNS.getDefaultIP("default");
  } catch (UnknownHostException ignored) {
    System.out.println("Could not find ip address of \"default\" inteface.");
  }
  
  int rand = DFSUtil.getSecureRandom().nextInt(Integer.MAX_VALUE);
  return "/Rack-" + rand + "-"+ ip  + "-" + Time.now(); 
}
 
Example 19
Source Project: big-c   Source File: RpcProgramNfs3.java    License: Apache License 2.0 5 votes vote down vote up
public static RpcProgramNfs3 createRpcProgramNfs3(NfsConfiguration config,
    DatagramSocket registrationSocket, boolean allowInsecurePorts)
    throws IOException {
  DefaultMetricsSystem.initialize("Nfs3");
  String displayName = DNS.getDefaultHost("default", "default")
      + config.getInt(NfsConfigKeys.DFS_NFS_SERVER_PORT_KEY,
          NfsConfigKeys.DFS_NFS_SERVER_PORT_DEFAULT);
  metrics = Nfs3Metrics.create(config, displayName);
  return new RpcProgramNfs3(config, registrationSocket, allowInsecurePorts);
}
 
Example 20
Source Project: big-c   Source File: NNStorage.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Generate new blockpoolID.
 * 
 * @return new blockpoolID
 */ 
static String newBlockPoolID() throws UnknownHostException{
  String ip = "unknownIP";
  try {
    ip = DNS.getDefaultIP("default");
  } catch (UnknownHostException e) {
    LOG.warn("Could not find ip address of \"default\" inteface.");
    throw e;
  }
  
  int rand = DFSUtil.getSecureRandom().nextInt(Integer.MAX_VALUE);
  String bpid = "BP-" + rand + "-"+ ip + "-" + Time.now();
  return bpid;
}
 
Example 21
Source Project: big-c   Source File: DataNode.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Returns the hostname for this datanode. If the hostname is not
 * explicitly configured in the given config, then it is determined
 * via the DNS class.
 *
 * @param config configuration
 * @return the hostname (NB: may not be a FQDN)
 * @throws UnknownHostException if the dfs.datanode.dns.interface
 *    option is used and the hostname can not be determined
 */
private static String getHostName(Configuration config)
    throws UnknownHostException {
  String name = config.get(DFS_DATANODE_HOST_NAME_KEY);
  if (name == null) {
    name = DNS.getDefaultHost(
        config.get(DFS_DATANODE_DNS_INTERFACE_KEY,
                   DFS_DATANODE_DNS_INTERFACE_DEFAULT),
        config.get(DFS_DATANODE_DNS_NAMESERVER_KEY,
                   DFS_DATANODE_DNS_NAMESERVER_DEFAULT));
  }
  return name;
}
 
Example 22
Source Project: big-c   Source File: DataNodeCluster.java    License: Apache License 2.0 5 votes vote down vote up
static private String getUniqueRackPrefix() {

  String ip = "unknownIP";
  try {
    ip = DNS.getDefaultIP("default");
  } catch (UnknownHostException ignored) {
    System.out.println("Could not find ip address of \"default\" inteface.");
  }
  
  int rand = DFSUtil.getSecureRandom().nextInt(Integer.MAX_VALUE);
  return "/Rack-" + rand + "-"+ ip  + "-" + Time.now(); 
}
 
Example 23
Source Project: hbase   Source File: ConnectionUtils.java    License: Apache License 2.0 5 votes vote down vote up
private static String getMyAddress() {
  try {
    return DNS.getDefaultHost("default", "default");
  } catch (UnknownHostException uhe) {
    LOG.error("cannot determine my address", uhe);
    return null;
  }
}
 
Example 24
Source Project: RDFS   Source File: NNThroughputBenchmark.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Get data-node in the form 
 * <host name> : <port>
 * where port is a 6 digit integer.
 * This is necessary in order to provide lexocographic ordering.
 * Host names are all the same, the ordering goes by port numbers.
 */
private static String getNodeName(int port) throws IOException {
  String machineName = DNS.getDefaultHost("default", "default");
  String sPort = String.valueOf(100000 + port);
  if(sPort.length() > 6)
    throw new IOException("Too many data-nodes.");
  return machineName + ":" + sPort;
}
 
Example 25
Source Project: RDFS   Source File: NNThroughputBenchmark.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Get data-node in the form <host name> : <port> where port is a 6
 * digit integer. This is necessary in order to provide lexocographic
 * ordering. Host names are all the same, the ordering goes by port
 * numbers.
 */
private static String getNodeName(int port) throws IOException {
	String machineName = DNS.getDefaultHost("default", "default");
	String sPort = String.valueOf(100000 + port);
	if (sPort.length() > 6)
		throw new IOException("Too many data-nodes.");
	return machineName + ":" + sPort;
}
 
Example 26
Source Project: RDFS   Source File: DataNode.java    License: Apache License 2.0 5 votes vote down vote up
public static String createNewStorageId(int port) {
  /* Return 
   * "DS-randInt-ipaddr-currentTimeMillis"
   * It is considered extermely rare for all these numbers to match
   * on a different machine accidentally for the following 
   * a) SecureRandom(INT_MAX) is pretty much random (1 in 2 billion), and
   * b) Good chance ip address would be different, and
   * c) Even on the same machine, Datanode is designed to use different ports.
   * d) Good chance that these are started at different times.
   * For a confict to occur all the 4 above have to match!.
   * The format of this string can be changed anytime in future without
   * affecting its functionality.
   */
  String ip = "unknownIP";
  try {
    ip = DNS.getDefaultIP("default");
  } catch (UnknownHostException ignored) {
    LOG.warn("Could not find ip address of \"default\" inteface.");
  }

  int rand = 0;
  try {
    rand = SecureRandom.getInstance("SHA1PRNG").nextInt(Integer.MAX_VALUE);
  } catch (NoSuchAlgorithmException e) {
    LOG.warn("Could not use SecureRandom");
    rand = R.nextInt(Integer.MAX_VALUE);
  }
  return "DS-" + rand + "-"+ ip + "-" + port + "-" + 
                    System.currentTimeMillis();
}
 
Example 27
Source Project: hbase-indexer   Source File: Main.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * @param conf the configuration object containing the hbase-indexer configuration, as well
 *             as the hbase/hadoop settings. Typically created using {@link HBaseIndexerConfiguration}.
 */
public void startServices(Configuration conf) throws Exception {
    String hostname = Strings.domainNamePointerToHostName(DNS.getDefaultHost(
            conf.get("hbase.regionserver.dns.interface", "default"),
            conf.get("hbase.regionserver.dns.nameserver", "default")));

    log.debug("Using hostname " + hostname);

    String zkConnectString = conf.get(ConfKeys.ZK_CONNECT_STRING);
    int zkSessionTimeout = HBaseIndexerConfiguration.getSessionTimeout(conf);
    zk = new StateWatchingZooKeeper(zkConnectString, zkSessionTimeout);

    tablePool = ConnectionFactory.createConnection(conf);

    String zkRoot = conf.get(ConfKeys.ZK_ROOT_NODE);

    indexerModel = new IndexerModelImpl(zk, zkRoot);

    sepModel = new SepModelImpl(zk, conf);

    indexerMaster = new IndexerMaster(zk, indexerModel, conf, conf, zkConnectString,
            sepModel);
    indexerMaster.start();

    IndexerRegistry indexerRegistry = new IndexerRegistry();
    IndexerProcessRegistry indexerProcessRegistry = new IndexerProcessRegistryImpl(zk, conf);
    indexerSupervisor = new IndexerSupervisor(indexerModel, zk, hostname, indexerRegistry,
            indexerProcessRegistry, tablePool, conf);

    indexerSupervisor.init();
    startHttpServer();

}
 
Example 28
Source Project: SpyGlass   Source File: HBaseInputFormatGranular.java    License: Apache License 2.0 5 votes vote down vote up
private String reverseDNS(InetAddress ipAddress) throws NamingException {
	String hostName = this.reverseDNSCacheMap.get(ipAddress);
	if (hostName == null) {
		hostName = Strings.domainNamePointerToHostName(DNS.reverseDns(
				ipAddress, this.nameServer));
		this.reverseDNSCacheMap.put(ipAddress, hostName);
	}
	return hostName;
}
 
Example 29
Source Project: hadoop-gpu   Source File: NNThroughputBenchmark.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Get data-node in the form 
 * <host name> : <port>
 * where port is a 6 digit integer.
 * This is necessary in order to provide lexocographic ordering.
 * Host names are all the same, the ordering goes by port numbers.
 */
private static String getNodeName(int port) throws IOException {
  String machineName = DNS.getDefaultHost("default", "default");
  String sPort = String.valueOf(100000 + port);
  if(sPort.length() > 6)
    throw new IOException("Too many data-nodes.");
  return machineName + ":" + sPort;
}
 
Example 30
Source Project: hadoop-gpu   Source File: DataNode.java    License: Apache License 2.0 5 votes vote down vote up
public static void setNewStorageID(DatanodeRegistration dnReg) {
  /* Return 
   * "DS-randInt-ipaddr-currentTimeMillis"
   * It is considered extermely rare for all these numbers to match
   * on a different machine accidentally for the following 
   * a) SecureRandom(INT_MAX) is pretty much random (1 in 2 billion), and
   * b) Good chance ip address would be different, and
   * c) Even on the same machine, Datanode is designed to use different ports.
   * d) Good chance that these are started at different times.
   * For a confict to occur all the 4 above have to match!.
   * The format of this string can be changed anytime in future without
   * affecting its functionality.
   */
  String ip = "unknownIP";
  try {
    ip = DNS.getDefaultIP("default");
  } catch (UnknownHostException ignored) {
    LOG.warn("Could not find ip address of \"default\" inteface.");
  }
  
  int rand = 0;
  try {
    rand = SecureRandom.getInstance("SHA1PRNG").nextInt(Integer.MAX_VALUE);
  } catch (NoSuchAlgorithmException e) {
    LOG.warn("Could not use SecureRandom");
    rand = R.nextInt(Integer.MAX_VALUE);
  }
  dnReg.storageID = "DS-" + rand + "-"+ ip + "-" + dnReg.getPort() + "-" + 
                    System.currentTimeMillis();
}