Java Code Examples for org.apache.hadoop.net.DNSToSwitchMapping

The following examples show how to use org.apache.hadoop.net.DNSToSwitchMapping. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop-ozone   Source File: SCMNodeManager.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Constructs SCM machine Manager.
 */
public SCMNodeManager(OzoneConfiguration conf,
    SCMStorageConfig scmStorageConfig, EventPublisher eventPublisher,
    NetworkTopology networkTopology) {
  this.nodeStateManager = new NodeStateManager(conf, eventPublisher);
  this.version = VersionInfo.getLatestVersion();
  this.commandQueue = new CommandQueue();
  this.scmStorageConfig = scmStorageConfig;
  LOG.info("Entering startup safe mode.");
  registerMXBean();
  this.metrics = SCMNodeMetrics.create(this);
  this.clusterMap = networkTopology;
  Class<? extends DNSToSwitchMapping> dnsToSwitchMappingClass =
      conf.getClass(
          DFSConfigKeysLegacy.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
          TableMapping.class, DNSToSwitchMapping.class);
  DNSToSwitchMapping newInstance = ReflectionUtils.newInstance(
      dnsToSwitchMappingClass, conf);
  this.dnsToSwitchMapping =
      ((newInstance instanceof CachedDNSToSwitchMapping) ? newInstance
          : new CachedDNSToSwitchMapping(newInstance));
  this.useHostname = conf.getBoolean(
      DFSConfigKeysLegacy.DFS_DATANODE_USE_DN_HOSTNAME,
      DFSConfigKeysLegacy.DFS_DATANODE_USE_DN_HOSTNAME_DEFAULT);
}
 
Example 2
@Before
public void setup() throws Exception {
  OzoneConfiguration conf = new OzoneConfiguration();
  conf.set(ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY,
      "org.apache.hadoop.hdds.scm.container.placement.algorithms." +
          "SCMContainerPlacementRackAware");
  // TODO enable when RATIS-788 is fixed
  conf.setBoolean(OMConfigKeys.OZONE_OM_RATIS_ENABLE_KEY, false);
  conf.setClass(NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
      StaticMapping.class, DNSToSwitchMapping.class);
  StaticMapping.addNodeToRack(NetUtils.normalizeHostNames(
      Collections.singleton(HddsUtils.getHostName(conf))).get(0),
      "/rack1");
  cluster = MiniOzoneCluster.newBuilder(conf)
      .setNumDatanodes(4)
      .setTotalPipelineNumLimit(10)
      .build();
  cluster.waitForClusterToBeReady();
  metrics = getMetrics(SCMContainerPlacementMetrics.class.getSimpleName());
  ozClient = OzoneClientFactory.getRpcClient(conf);
  store = ozClient.getObjectStore();
}
 
Example 3
Source Project: hadoop   Source File: RackResolver.java    License: Apache License 2.0 6 votes vote down vote up
public synchronized static void init(Configuration conf) {
  if (initCalled) {
    return;
  } else {
    initCalled = true;
  }
  Class<? extends DNSToSwitchMapping> dnsToSwitchMappingClass =
    conf.getClass(
      CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, 
      ScriptBasedMapping.class,
      DNSToSwitchMapping.class);
  try {
    DNSToSwitchMapping newInstance = ReflectionUtils.newInstance(
        dnsToSwitchMappingClass, conf);
    // Wrap around the configured class with the Cached implementation so as
    // to save on repetitive lookups.
    // Check if the impl is already caching, to avoid double caching.
    dnsToSwitchMapping =
        ((newInstance instanceof CachedDNSToSwitchMapping) ? newInstance
            : new CachedDNSToSwitchMapping(newInstance));
  } catch (Exception e) {
    throw new RuntimeException(e);
  }
}
 
Example 4
Source Project: hadoop   Source File: TestRackResolver.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCaching() {
  Configuration conf = new Configuration();
  conf.setClass(
    CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
    MyResolver.class, DNSToSwitchMapping.class);
  RackResolver.init(conf);
  try {
    InetAddress iaddr = InetAddress.getByName("host1");
    MyResolver.resolvedHost1 = iaddr.getHostAddress();
  } catch (UnknownHostException e) {
    // Ignore if not found
  }
  Node node = RackResolver.resolve("host1");
  Assert.assertEquals("/rack1", node.getNetworkLocation());
  node = RackResolver.resolve("host1");
  Assert.assertEquals("/rack1", node.getNetworkLocation());
  node = RackResolver.resolve(invalidHost);
  Assert.assertEquals(NetworkTopology.DEFAULT_RACK, node.getNetworkLocation());
}
 
Example 5
Source Project: hadoop   Source File: TestAMRMClientContainerRequest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testFillInRacks() {
  AMRMClientImpl<ContainerRequest> client =
      new AMRMClientImpl<ContainerRequest>();
  
  Configuration conf = new Configuration();
  conf.setClass(
      CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
      MyResolver.class, DNSToSwitchMapping.class);
  client.init(conf);
 
  Resource capability = Resource.newInstance(1024, 1, 1);
  ContainerRequest request =
      new ContainerRequest(capability, new String[] {"host1", "host2"},
          new String[] {"/rack2"}, Priority.newInstance(1));
  client.addContainerRequest(request);
  verifyResourceRequest(client, request, "host1", true);
  verifyResourceRequest(client, request, "host2", true);
  verifyResourceRequest(client, request, "/rack1", true);
  verifyResourceRequest(client, request, "/rack2", true);
  verifyResourceRequest(client, request, ResourceRequest.ANY, true);
}
 
Example 6
Source Project: hadoop   Source File: TestAMRMClientContainerRequest.java    License: Apache License 2.0 6 votes vote down vote up
@Test (expected = InvalidContainerRequestException.class)
public void testDifferentLocalityRelaxationSamePriority() {
  AMRMClientImpl<ContainerRequest> client =
      new AMRMClientImpl<ContainerRequest>();
  Configuration conf = new Configuration();
  conf.setClass(
      CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
      MyResolver.class, DNSToSwitchMapping.class);
  client.init(conf);
  
  Resource capability = Resource.newInstance(1024, 1, 1);
  ContainerRequest request1 =
      new ContainerRequest(capability, new String[] {"host1", "host2"},
          null, Priority.newInstance(1), false);
  client.addContainerRequest(request1);
  ContainerRequest request2 =
      new ContainerRequest(capability, new String[] {"host3"},
          null, Priority.newInstance(1), true);
  client.addContainerRequest(request2);
}
 
Example 7
Source Project: hadoop   Source File: TestAMRMClientContainerRequest.java    License: Apache License 2.0 6 votes vote down vote up
@Test (expected = InvalidContainerRequestException.class)
public void testLocalityRelaxationDifferentLevels() {
  AMRMClientImpl<ContainerRequest> client =
      new AMRMClientImpl<ContainerRequest>();
  Configuration conf = new Configuration();
  conf.setClass(
      CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
      MyResolver.class, DNSToSwitchMapping.class);
  client.init(conf);
  
  Resource capability = Resource.newInstance(1024, 1, 1);
  ContainerRequest request1 =
      new ContainerRequest(capability, new String[] {"host1", "host2"},
          null, Priority.newInstance(1), false);
  client.addContainerRequest(request1);
  ContainerRequest request2 =
      new ContainerRequest(capability, null,
          new String[] {"rack1"}, Priority.newInstance(1), true);
  client.addContainerRequest(request2);
}
 
Example 8
Source Project: big-c   Source File: RackResolver.java    License: Apache License 2.0 6 votes vote down vote up
public synchronized static void init(Configuration conf) {
  if (initCalled) {
    return;
  } else {
    initCalled = true;
  }
  Class<? extends DNSToSwitchMapping> dnsToSwitchMappingClass =
    conf.getClass(
      CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, 
      ScriptBasedMapping.class,
      DNSToSwitchMapping.class);
  try {
    DNSToSwitchMapping newInstance = ReflectionUtils.newInstance(
        dnsToSwitchMappingClass, conf);
    // Wrap around the configured class with the Cached implementation so as
    // to save on repetitive lookups.
    // Check if the impl is already caching, to avoid double caching.
    dnsToSwitchMapping =
        ((newInstance instanceof CachedDNSToSwitchMapping) ? newInstance
            : new CachedDNSToSwitchMapping(newInstance));
  } catch (Exception e) {
    throw new RuntimeException(e);
  }
}
 
Example 9
Source Project: big-c   Source File: TestRackResolver.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCaching() {
  Configuration conf = new Configuration();
  conf.setClass(
    CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
    MyResolver.class, DNSToSwitchMapping.class);
  RackResolver.init(conf);
  try {
    InetAddress iaddr = InetAddress.getByName("host1");
    MyResolver.resolvedHost1 = iaddr.getHostAddress();
  } catch (UnknownHostException e) {
    // Ignore if not found
  }
  Node node = RackResolver.resolve("host1");
  Assert.assertEquals("/rack1", node.getNetworkLocation());
  node = RackResolver.resolve("host1");
  Assert.assertEquals("/rack1", node.getNetworkLocation());
  node = RackResolver.resolve(invalidHost);
  Assert.assertEquals(NetworkTopology.DEFAULT_RACK, node.getNetworkLocation());
}
 
Example 10
Source Project: big-c   Source File: TestAMRMClientContainerRequest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testFillInRacks() {
  AMRMClientImpl<ContainerRequest> client =
      new AMRMClientImpl<ContainerRequest>();
  
  Configuration conf = new Configuration();
  conf.setClass(
      CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
      MyResolver.class, DNSToSwitchMapping.class);
  client.init(conf);
 
  Resource capability = Resource.newInstance(1024, 1);
  ContainerRequest request =
      new ContainerRequest(capability, new String[] {"host1", "host2"},
          new String[] {"/rack2"}, Priority.newInstance(1));
  client.addContainerRequest(request);
  verifyResourceRequest(client, request, "host1", true);
  verifyResourceRequest(client, request, "host2", true);
  verifyResourceRequest(client, request, "/rack1", true);
  verifyResourceRequest(client, request, "/rack2", true);
  verifyResourceRequest(client, request, ResourceRequest.ANY, true);
}
 
Example 11
Source Project: big-c   Source File: TestAMRMClientContainerRequest.java    License: Apache License 2.0 6 votes vote down vote up
@Test (expected = InvalidContainerRequestException.class)
public void testDifferentLocalityRelaxationSamePriority() {
  AMRMClientImpl<ContainerRequest> client =
      new AMRMClientImpl<ContainerRequest>();
  Configuration conf = new Configuration();
  conf.setClass(
      CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
      MyResolver.class, DNSToSwitchMapping.class);
  client.init(conf);
  
  Resource capability = Resource.newInstance(1024, 1);
  ContainerRequest request1 =
      new ContainerRequest(capability, new String[] {"host1", "host2"},
          null, Priority.newInstance(1), false);
  client.addContainerRequest(request1);
  ContainerRequest request2 =
      new ContainerRequest(capability, new String[] {"host3"},
          null, Priority.newInstance(1), true);
  client.addContainerRequest(request2);
}
 
Example 12
Source Project: big-c   Source File: TestAMRMClientContainerRequest.java    License: Apache License 2.0 6 votes vote down vote up
@Test (expected = InvalidContainerRequestException.class)
public void testLocalityRelaxationDifferentLevels() {
  AMRMClientImpl<ContainerRequest> client =
      new AMRMClientImpl<ContainerRequest>();
  Configuration conf = new Configuration();
  conf.setClass(
      CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
      MyResolver.class, DNSToSwitchMapping.class);
  client.init(conf);
  
  Resource capability = Resource.newInstance(1024, 1);
  ContainerRequest request1 =
      new ContainerRequest(capability, new String[] {"host1", "host2"},
          null, Priority.newInstance(1), false);
  client.addContainerRequest(request1);
  ContainerRequest request2 =
      new ContainerRequest(capability, null,
          new String[] {"rack1"}, Priority.newInstance(1), true);
  client.addContainerRequest(request2);
}
 
Example 13
Source Project: hbase   Source File: TestBaseLoadBalancer.java    License: Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void beforeAllTests() throws Exception {
  Configuration conf = HBaseConfiguration.create();
  conf.setClass("hbase.util.ip.to.rack.determiner", MockMapping.class, DNSToSwitchMapping.class);
  loadBalancer = new MockBalancer();
  loadBalancer.setConf(conf);
  MasterServices st = Mockito.mock(MasterServices.class);
  Mockito.when(st.getServerName()).thenReturn(master);
  loadBalancer.setMasterServices(st);

  // Set up the rack topologies (5 machines per rack)
  rackManager = Mockito.mock(RackManager.class);
  for (int i = 0; i < NUM_SERVERS; i++) {
    servers[i] = ServerName.valueOf("foo"+i+":1234",-1);
    if (i < 5) {
      Mockito.when(rackManager.getRack(servers[i])).thenReturn("rack1");
    }
    if (i >= 5 && i < 10) {
      Mockito.when(rackManager.getRack(servers[i])).thenReturn("rack2");
    }
    if (i >= 10) {
      Mockito.when(rackManager.getRack(servers[i])).thenReturn("rack3");
    }
  }
}
 
Example 14
Source Project: RDFS   Source File: MiniMRCluster.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Create the job tracker and run it.
 */
public void run() {
  try {
    jc = (jc == null) ? createJobConf() : createJobConf(jc);
    File f = new File("build/test/mapred/local").getAbsoluteFile();
    jc.set("mapred.local.dir",f.getAbsolutePath());
    jc.setClass("topology.node.switch.mapping.impl", 
        StaticMapping.class, DNSToSwitchMapping.class);
    String id = 
      new SimpleDateFormat("yyyyMMddHHmmssSSS").format(new Date());
    tracker = JobTracker.startTracker(jc, id);
    tracker.offerService();
  } catch (Throwable e) {
    LOG.error("Job tracker crashed", e);
    isActive = false;
  }
}
 
Example 15
Source Project: RDFS   Source File: MiniCoronaCluster.java    License: Apache License 2.0 6 votes vote down vote up
static void configureJobConf(JobConf conf, String namenode,
    int clusterManagerPort, int proxyJobTrackerPort,
    UnixUserGroupInformation ugi) {
  FileSystem.setDefaultUri(conf, namenode);
  conf.set(CoronaConf.CM_ADDRESS,
             "localhost:" + clusterManagerPort);
  conf.set(CoronaConf.PROXY_JOB_TRACKER_ADDRESS,
    "localhost:" + proxyJobTrackerPort);
  conf.set("mapred.job.tracker", "corona");
  conf.set("mapred.job.tracker.http.address",
                      "127.0.0.1:0");
  conf.setClass("topology.node.switch.mapping.impl",
      StaticMapping.class, DNSToSwitchMapping.class);
  conf.set("mapred.job.tracker.class", CoronaJobTracker.class.getName());
  if (ugi != null) {
    conf.set("mapred.system.dir", "/mapred/system");
    UnixUserGroupInformation.saveToConf(conf,
        UnixUserGroupInformation.UGI_PROPERTY_NAME, ugi);
  }
  // for debugging have all task output sent to the test output
  JobClient.setTaskOutputFilter(conf, JobClient.TaskStatusFilter.ALL);
}
 
Example 16
Source Project: RDFS   Source File: BlockPlacementPolicy.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Get an instance of the configured Block Placement Policy based on the
 * value of the configuration paramater dfs.block.replicator.classname.
 * 
 * @param conf the configuration to be used
 * @param stats an object thatis used to retrieve the load on the cluster
 * @param clusterMap the network topology of the cluster
 * @param namesystem the FSNamesystem
 * @return an instance of BlockPlacementPolicy
 */
public static BlockPlacementPolicy getInstance(Configuration conf, 
                                               FSClusterStats stats,
                                               NetworkTopology clusterMap,
                                               HostsFileReader hostsReader,
                                               DNSToSwitchMapping dnsToSwitchMapping,
                                               FSNamesystem namesystem) {
  Class<? extends BlockPlacementPolicy> replicatorClass =
                    conf.getClass("dfs.block.replicator.classname",
                                  BlockPlacementPolicyDefault.class,
                                  BlockPlacementPolicy.class);
  BlockPlacementPolicy replicator = (BlockPlacementPolicy) ReflectionUtils.newInstance(
                                                           replicatorClass, conf);
  replicator.initialize(conf, stats, clusterMap, hostsReader, 
                        dnsToSwitchMapping, namesystem);
  return replicator;
}
 
Example 17
Source Project: presto   Source File: HdfsConfigurationInitializer.java    License: Apache License 2.0 5 votes vote down vote up
public void initializeConfiguration(Configuration config)
{
    copy(resourcesConfiguration, config);

    // this is to prevent dfs client from doing reverse DNS lookups to determine whether nodes are rack local
    config.setClass(NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, NoOpDNSToSwitchMapping.class, DNSToSwitchMapping.class);

    if (socksProxy != null) {
        config.setClass(HADOOP_RPC_SOCKET_FACTORY_CLASS_DEFAULT_KEY, SocksSocketFactory.class, SocketFactory.class);
        config.set(HADOOP_SOCKS_SERVER_KEY, socksProxy.toString());
    }

    if (domainSocketPath != null) {
        config.setStrings(DFS_DOMAIN_SOCKET_PATH_KEY, domainSocketPath);
    }

    // only enable short circuit reads if domain socket path is properly configured
    if (!config.get(DFS_DOMAIN_SOCKET_PATH_KEY, "").trim().isEmpty()) {
        config.setBooleanIfUnset(HdfsClientConfigKeys.Read.ShortCircuit.KEY, true);
    }

    config.setInt(DFS_CLIENT_SOCKET_TIMEOUT_KEY, toIntExact(dfsTimeout.toMillis()));
    config.setInt(IPC_PING_INTERVAL_KEY, toIntExact(ipcPingInterval.toMillis()));
    config.setInt(IPC_CLIENT_CONNECT_TIMEOUT_KEY, toIntExact(dfsConnectTimeout.toMillis()));
    config.setInt(IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, dfsConnectMaxRetries);
    config.setInt(DFS_CLIENT_KEY_PROVIDER_CACHE_EXPIRY_MS, dfsKeyProviderCacheTtlMillis);

    if (wireEncryptionEnabled) {
        config.set(HADOOP_RPC_PROTECTION, "privacy");
        config.setBoolean("dfs.encrypt.data.transfer", true);
    }

    config.setInt("fs.cache.max-size", fileSystemMaxCacheSize);

    configurationInitializers.forEach(configurationInitializer -> configurationInitializer.initializeConfiguration(config));
}
 
Example 18
Source Project: hadoop   Source File: TestRackResolverScriptBasedMapping.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testScriptName() {
  Configuration conf = new Configuration();
  conf
      .setClass(
          CommonConfigurationKeysPublic.
              NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
          ScriptBasedMapping.class, DNSToSwitchMapping.class);
  conf.set(CommonConfigurationKeysPublic.NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY,
      "testScript");
  RackResolver.init(conf);
  Assert.assertEquals(RackResolver.getDnsToSwitchMapping().toString(),
      "script-based mapping with script testScript");
}
 
Example 19
Source Project: hadoop   Source File: TestAMRMClientContainerRequest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testInvalidValidWhenOldRemoved() {
  AMRMClientImpl<ContainerRequest> client =
      new AMRMClientImpl<ContainerRequest>();
  Configuration conf = new Configuration();
  conf.setClass(
      CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
      MyResolver.class, DNSToSwitchMapping.class);
  client.init(conf);
  
  Resource capability = Resource.newInstance(1024, 1, 1);
  ContainerRequest request1 =
      new ContainerRequest(capability, new String[] {"host1", "host2"},
          null, Priority.newInstance(1), false);
  client.addContainerRequest(request1);
  
  client.removeContainerRequest(request1);

  ContainerRequest request2 =
      new ContainerRequest(capability, new String[] {"host3"},
          null, Priority.newInstance(1), true);
  client.addContainerRequest(request2);
  
  client.removeContainerRequest(request2);
  
  ContainerRequest request3 =
      new ContainerRequest(capability, new String[] {"host1", "host2"},
          null, Priority.newInstance(1), false);
  client.addContainerRequest(request3);
  
  client.removeContainerRequest(request3);
  
  ContainerRequest request4 =
      new ContainerRequest(capability, null,
          new String[] {"rack1"}, Priority.newInstance(1), true);
  client.addContainerRequest(request4);

}
 
Example 20
Source Project: hadoop   Source File: TestDatanodeManager.java    License: Apache License 2.0 5 votes vote down vote up
@Test (timeout = 100000)
public void testRejectUnresolvedDatanodes() throws IOException {
  //Create the DatanodeManager which will be tested
  FSNamesystem fsn = Mockito.mock(FSNamesystem.class);
  Mockito.when(fsn.hasWriteLock()).thenReturn(true);
  
  Configuration conf = new Configuration();
  
  //Set configuration property for rejecting unresolved topology mapping
  conf.setBoolean(
      DFSConfigKeys.DFS_REJECT_UNRESOLVED_DN_TOPOLOGY_MAPPING_KEY, true);
  
  //set TestDatanodeManager.MyResolver to be used for topology resolving
  conf.setClass(
      CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
      TestDatanodeManager.MyResolver.class, DNSToSwitchMapping.class);
  
  //create DatanodeManager
  DatanodeManager dm = new DatanodeManager(Mockito.mock(BlockManager.class),
      fsn, conf);
  
  //storageID to register.
  String storageID = "someStorageID-123";
  
  DatanodeRegistration dr = Mockito.mock(DatanodeRegistration.class);
  Mockito.when(dr.getDatanodeUuid()).thenReturn(storageID);
  
  try {
    //Register this node
    dm.registerDatanode(dr);
    Assert.fail("Expected an UnresolvedTopologyException");
  } catch (UnresolvedTopologyException ute) {
    LOG.info("Expected - topology is not resolved and " +
        "registration is rejected.");
  } catch (Exception e) {
    Assert.fail("Expected an UnresolvedTopologyException");
  }
}
 
Example 21
Source Project: big-c   Source File: TestRackResolverScriptBasedMapping.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testScriptName() {
  Configuration conf = new Configuration();
  conf
      .setClass(
          CommonConfigurationKeysPublic.
              NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
          ScriptBasedMapping.class, DNSToSwitchMapping.class);
  conf.set(CommonConfigurationKeysPublic.NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY,
      "testScript");
  RackResolver.init(conf);
  Assert.assertEquals(RackResolver.getDnsToSwitchMapping().toString(),
      "script-based mapping with script testScript");
}
 
Example 22
Source Project: big-c   Source File: TestAMRMClientContainerRequest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testInvalidValidWhenOldRemoved() {
  AMRMClientImpl<ContainerRequest> client =
      new AMRMClientImpl<ContainerRequest>();
  Configuration conf = new Configuration();
  conf.setClass(
      CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
      MyResolver.class, DNSToSwitchMapping.class);
  client.init(conf);
  
  Resource capability = Resource.newInstance(1024, 1);
  ContainerRequest request1 =
      new ContainerRequest(capability, new String[] {"host1", "host2"},
          null, Priority.newInstance(1), false);
  client.addContainerRequest(request1);
  
  client.removeContainerRequest(request1);

  ContainerRequest request2 =
      new ContainerRequest(capability, new String[] {"host3"},
          null, Priority.newInstance(1), true);
  client.addContainerRequest(request2);
  
  client.removeContainerRequest(request2);
  
  ContainerRequest request3 =
      new ContainerRequest(capability, new String[] {"host1", "host2"},
          null, Priority.newInstance(1), false);
  client.addContainerRequest(request3);
  
  client.removeContainerRequest(request3);
  
  ContainerRequest request4 =
      new ContainerRequest(capability, null,
          new String[] {"rack1"}, Priority.newInstance(1), true);
  client.addContainerRequest(request4);

}
 
Example 23
Source Project: big-c   Source File: TestDatanodeManager.java    License: Apache License 2.0 5 votes vote down vote up
@Test (timeout = 100000)
public void testRejectUnresolvedDatanodes() throws IOException {
  //Create the DatanodeManager which will be tested
  FSNamesystem fsn = Mockito.mock(FSNamesystem.class);
  Mockito.when(fsn.hasWriteLock()).thenReturn(true);
  
  Configuration conf = new Configuration();
  
  //Set configuration property for rejecting unresolved topology mapping
  conf.setBoolean(
      DFSConfigKeys.DFS_REJECT_UNRESOLVED_DN_TOPOLOGY_MAPPING_KEY, true);
  
  //set TestDatanodeManager.MyResolver to be used for topology resolving
  conf.setClass(
      CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
      TestDatanodeManager.MyResolver.class, DNSToSwitchMapping.class);
  
  //create DatanodeManager
  DatanodeManager dm = new DatanodeManager(Mockito.mock(BlockManager.class),
      fsn, conf);
  
  //storageID to register.
  String storageID = "someStorageID-123";
  
  DatanodeRegistration dr = Mockito.mock(DatanodeRegistration.class);
  Mockito.when(dr.getDatanodeUuid()).thenReturn(storageID);
  
  try {
    //Register this node
    dm.registerDatanode(dr);
    Assert.fail("Expected an UnresolvedTopologyException");
  } catch (UnresolvedTopologyException ute) {
    LOG.info("Expected - topology is not resolved and " +
        "registration is rejected.");
  } catch (Exception e) {
    Assert.fail("Expected an UnresolvedTopologyException");
  }
}
 
Example 24
Source Project: hbase   Source File: TestStochasticBalancerJmxMetrics.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Setup the environment for the test.
 */
@BeforeClass
public static void setupBeforeClass() throws Exception {

  conf = UTIL.getConfiguration();

  conf.setClass("hbase.util.ip.to.rack.determiner", MockMapping.class, DNSToSwitchMapping.class);
  conf.setFloat("hbase.master.balancer.stochastic.maxMovePercent", 0.75f);
  conf.setFloat("hbase.regions.slop", 0.0f);
  conf.set(CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY, JMXListener.class.getName());
  Random rand = new Random();
  for (int i = 0; i < 10; i++) {
    do {
      int sign = i % 2 == 0 ? 1 : -1;
      connectorPort += sign * rand.nextInt(100);
    } while (!HBaseTestingUtility.available(connectorPort));
    try {
      conf.setInt("regionserver.rmi.registry.port", connectorPort);

      UTIL.startMiniCluster();
      break;
    } catch (Exception e) {
      LOG.debug("Encountered exception when starting cluster. Trying port " + connectorPort, e);
      try {
        // this is to avoid "IllegalStateException: A mini-cluster is already running"
        UTIL.shutdownMiniCluster();
      } catch (Exception ex) {
        LOG.debug("Encountered exception shutting down cluster", ex);
      }
    }
  }
}
 
Example 25
Source Project: hbase   Source File: BalancerTestBase.java    License: Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void beforeAllTests() throws Exception {
  conf = HBaseConfiguration.create();
  conf.setClass("hbase.util.ip.to.rack.determiner", MockMapping.class, DNSToSwitchMapping.class);
  conf.setFloat("hbase.master.balancer.stochastic.maxMovePercent", 0.75f);
  conf.setFloat("hbase.regions.slop", 0.0f);
  conf.setFloat("hbase.master.balancer.stochastic.localityCost", 0);
  loadBalancer = new StochasticLoadBalancer();
  loadBalancer.setConf(conf);
}
 
Example 26
Source Project: hbase   Source File: TestSimpleLoadBalancer.java    License: Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void beforeAllTests() throws Exception {
  Configuration conf = HBaseConfiguration.create();
  conf.setClass("hbase.util.ip.to.rack.determiner", MockMapping.class, DNSToSwitchMapping.class);
  conf.set("hbase.regions.slop", "0");
  loadBalancer = new SimpleLoadBalancer();
  loadBalancer.setConf(conf);
}
 
Example 27
Source Project: RDFS   Source File: BlockPlacementPolicyRaid.java    License: Apache License 2.0 5 votes vote down vote up
/** {@inheritDoc} */
 @Override
 public void initialize(Configuration conf,  FSClusterStats stats,
                        NetworkTopology clusterMap, HostsFileReader hostsReader,
                        DNSToSwitchMapping dnsToSwitchMapping, FSNamesystem namesystem) {
   super.initialize(conf, stats, clusterMap, 
                    hostsReader, dnsToSwitchMapping, namesystem);
   this.conf = conf;
   this.namesystem = namesystem;
this.cachedLocatedBlocks = new CachedLocatedBlocks(this.conf, namesystem);
   this.cachedFullPathNames = new CachedFullPathNames(this.conf, namesystem);
 }
 
Example 28
Source Project: RDFS   Source File: TopologyCache.java    License: Apache License 2.0 5 votes vote down vote up
public TopologyCache(Configuration conf) {
  super(conf);

  dnsToSwitchMapping = ReflectionUtils.newInstance
    (conf.getClass("topology.node.switch.mapping.impl", ScriptBasedMapping.class,
                   DNSToSwitchMapping.class), conf);
  LOG.info("DnsToSwitchMapping class = " + dnsToSwitchMapping.getClass().getName());
}
 
Example 29
Source Project: RDFS   Source File: BlockPlacementPolicyDefault.java    License: Apache License 2.0 5 votes vote down vote up
/** {@inheritDoc} */
public void initialize(Configuration conf, FSClusterStats stats,
    NetworkTopology clusterMap, HostsFileReader hostsReader,
    DNSToSwitchMapping dnsToSwitchMapping, FSNamesystem ns) {
  this.considerLoad = conf.getBoolean("dfs.replication.considerLoad", true);
  this.stats = stats;
  this.clusterMap = clusterMap;
  Configuration newConf = new Configuration();
  this.attemptMultiplier = newConf.getInt("dfs.replication.attemptMultiplier", 200);
}
 
Example 30
Source Project: sahara-extra   Source File: SwiftNativeFileSystemStore.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Initialize the filesystem store -this creates the REST client binding.
 *
 * @param fsURI         URI of the filesystem, which is used to map to the filesystem-specific
 *                      options in the configuration file
 * @param configuration configuration
 * @throws IOException on any failure.
 */
public void initialize(URI fsURI, Configuration configuration) throws IOException {
  this.uri = fsURI;
  dnsToSwitchMapping = ReflectionUtils.newInstance(
      configuration.getClass("topology.node.switch.mapping.impl", ScriptBasedMapping.class,
          DNSToSwitchMapping.class), configuration);

  this.swiftRestClient = SwiftRestClient.getInstance(fsURI, configuration);
}