Java Code Examples for org.apache.hadoop.net.NetworkTopology#DEFAULT_RACK

The following examples show how to use org.apache.hadoop.net.NetworkTopology#DEFAULT_RACK . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: TestRMContainerAllocator.java From big-c with Apache License 2.0

6 votes

private ContainerRequestEvent
    createReq(JobId jobId, int taskAttemptId, int memory, String[] hosts,
        boolean earlierFailedAttempt, boolean reduce) {
  TaskId taskId;
  if (reduce) {
    taskId = MRBuilderUtils.newTaskId(jobId, 0, TaskType.REDUCE);
  } else {
    taskId = MRBuilderUtils.newTaskId(jobId, 0, TaskType.MAP);
  }
  TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId,
      taskAttemptId);
  Resource containerNeed = Resource.newInstance(memory, 1);
  if (earlierFailedAttempt) {
    return ContainerRequestEvent
        .createContainerRequestEventForFailedContainer(attemptId,
            containerNeed);
  }
  return new ContainerRequestEvent(attemptId, containerNeed, hosts,
      new String[] { NetworkTopology.DEFAULT_RACK });
}

Example 2

Source File: FSNamesystem.java From hadoop-gpu with Apache License 2.0

6 votes

private void resolveNetworkLocation (DatanodeDescriptor node) {
  List<String> names = new ArrayList<String>(1);
  if (dnsToSwitchMapping instanceof CachedDNSToSwitchMapping) {
    // get the node's IP address
    names.add(node.getHost());
  } else {
    // get the node's host name
    String hostName = node.getHostName();
    int colon = hostName.indexOf(":");
    hostName = (colon==-1)?hostName:hostName.substring(0,colon);
    names.add(hostName);
  }
  
  // resolve its network location
  List<String> rName = dnsToSwitchMapping.resolve(names);
  String networkLocation;
  if (rName == null) {
    LOG.error("The resolve call returned null! Using " + 
        NetworkTopology.DEFAULT_RACK + " for host " + names);
    networkLocation = NetworkTopology.DEFAULT_RACK;
  } else {
    networkLocation = rName.get(0);
  }
  node.setNetworkLocation(networkLocation);
}

Example 3

Source File: TestRMContainerAllocator.java From hadoop with Apache License 2.0

6 votes

private ContainerRequestEvent
    createReq(JobId jobId, int taskAttemptId, int memory, String[] hosts,
        boolean earlierFailedAttempt, boolean reduce) {
  TaskId taskId;
  if (reduce) {
    taskId = MRBuilderUtils.newTaskId(jobId, 0, TaskType.REDUCE);
  } else {
    taskId = MRBuilderUtils.newTaskId(jobId, 0, TaskType.MAP);
  }
  TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId,
      taskAttemptId);
  Resource containerNeed = Resource.newInstance(memory, 1);
  if (earlierFailedAttempt) {
    return ContainerRequestEvent
        .createContainerRequestEventForFailedContainer(attemptId,
            containerNeed);
  }
  return new ContainerRequestEvent(attemptId, containerNeed, hosts,
      new String[] { NetworkTopology.DEFAULT_RACK });
}

Example 4

Source File: TestRMContainerAllocator.java From hadoop with Apache License 2.0

6 votes

private void validateLabelsRequests(ResourceRequest resourceRequest,
    boolean isReduce) {
  switch (resourceRequest.getResourceName()) {
  case "map":
  case "reduce":
  case NetworkTopology.DEFAULT_RACK:
    Assert.assertNull(resourceRequest.getNodeLabelExpression());
    break;
  case "*":
    Assert.assertEquals(isReduce ? "ReduceNodes" : "MapNodes",
        resourceRequest.getNodeLabelExpression());
    break;
  default:
    Assert.fail("Invalid resource location "
        + resourceRequest.getResourceName());
  }
}

Example 5

Source File: FileInputFormat.java From hadoop with Apache License 2.0

5 votes

private String[] fakeRacks(BlockLocation[] blkLocations, int index) 
throws IOException {
  String[] allHosts = blkLocations[index].getHosts();
  String[] allTopos = new String[allHosts.length];
  for (int i = 0; i < allHosts.length; i++) {
    allTopos[i] = NetworkTopology.DEFAULT_RACK + "/" + allHosts[i];
  }
  return allTopos;
}

Example 6

Source File: RackResolver.java From big-c with Apache License 2.0

5 votes

private static Node coreResolve(String hostName) {
  List <String> tmpList = new ArrayList<String>(1);
  tmpList.add(hostName);
  List <String> rNameList = dnsToSwitchMapping.resolve(tmpList);
  String rName = null;
  if (rNameList == null || rNameList.get(0) == null) {
    rName = NetworkTopology.DEFAULT_RACK;
    LOG.info("Couldn't resolve " + hostName + ". Falling back to "
        + NetworkTopology.DEFAULT_RACK);
  } else {
    rName = rNameList.get(0);
    LOG.info("Resolved " + hostName + " to " + rName);
  }
  return new NodeBase(hostName, rName);
}

Example 7

Source File: RackResolver.java From hadoop with Apache License 2.0

5 votes

private static Node coreResolve(String hostName) {
  List <String> tmpList = new ArrayList<String>(1);
  tmpList.add(hostName);
  List <String> rNameList = dnsToSwitchMapping.resolve(tmpList);
  String rName = null;
  if (rNameList == null || rNameList.get(0) == null) {
    rName = NetworkTopology.DEFAULT_RACK;
    LOG.info("Couldn't resolve " + hostName + ". Falling back to "
        + NetworkTopology.DEFAULT_RACK);
  } else {
    rName = rNameList.get(0);
    LOG.info("Resolved " + hostName + " to " + rName);
  }
  return new NodeBase(hostName, rName);
}

Example 8

Source File: FileInputFormat.java From big-c with Apache License 2.0

5 votes

private String[] fakeRacks(BlockLocation[] blkLocations, int index) 
throws IOException {
  String[] allHosts = blkLocations[index].getHosts();
  String[] allTopos = new String[allHosts.length];
  for (int i = 0; i < allHosts.length; i++) {
    allTopos[i] = NetworkTopology.DEFAULT_RACK + "/" + allHosts[i];
  }
  return allTopos;
}

Example 9

Source File: FileInputFormat.java From RDFS with Apache License 2.0

5 votes

private String[] fakeRacks(BlockLocation[] blkLocations, int index) 
throws IOException {
  String[] allHosts = blkLocations[index].getHosts();
  String[] allTopos = new String[allHosts.length];
  for (int i = 0; i < allHosts.length; i++) {
    allTopos[i] = NetworkTopology.DEFAULT_RACK + "/" + allHosts[i];
  }
  return allTopos;
}

Example 10

Source File: TestStartupDefaultRack.java From RDFS with Apache License 2.0

5 votes

@Test
public void testStartup() throws Exception {
  conf = new Configuration();
  conf.setClass("dfs.block.replicator.classname",
      BlockPlacementPolicyConfigurable.class, BlockPlacementPolicy.class);
  File baseDir = MiniDFSCluster.getBaseDirectory(conf);
  baseDir.mkdirs();
  File hostsFile = new File(baseDir, "hosts");
  FileOutputStream out = new FileOutputStream(hostsFile);
  out.write("h1\n".getBytes());
  out.write("h2\n".getBytes());
  out.write("h3\n".getBytes());
  out.close();
  conf.set("dfs.hosts", hostsFile.getAbsolutePath());
  StaticMapping.addNodeToRack("h1", "/r1");
  StaticMapping.addNodeToRack("h2", "/r2");
  StaticMapping.addNodeToRack("h3", NetworkTopology.DEFAULT_RACK);
  cluster = new MiniDFSCluster(conf, 3, new String[] { "/r1", "/r2",
      NetworkTopology.DEFAULT_RACK }, new String[] { "h1", "h2", "h3" },
      true, false);
  DFSTestUtil util = new DFSTestUtil("/testStartup", 10, 10, 1024);
  util.createFiles(cluster.getFileSystem(), "/");
  util.checkFiles(cluster.getFileSystem(), "/");
  assertEquals(2,
      cluster.getNameNode().getDatanodeReport(DatanodeReportType.LIVE).length);
  cluster.shutdown();
}

Example 11

Source File: MiniCoronaCluster.java From RDFS with Apache License 2.0

5 votes

private MiniCoronaCluster(Builder builder) throws IOException {
  ContextFactory.resetFactory();
  setNoEmitMetricsContext();
  if (builder.racks != null && builder.hosts != null) {
    if (builder.racks.length != builder.hosts.length) {
      throw new IllegalArgumentException(
          "The number of hosts and racks must be the same");
    }
  }
  this.conf = builder.conf != null ? builder.conf : new JobConf();
  this.namenode = builder.namenode;
  this.ugi = builder.ugi;
  this.conf.set(CoronaConf.CM_ADDRESS, "localhost:0");
  this.conf.set(CoronaConf.CPU_TO_RESOURCE_PARTITIONING, TstUtils.std_cpu_to_resource_partitioning);
  this.clusterManagerPort = startClusterManager(this.conf);
  this.conf.set(CoronaConf.PROXY_JOB_TRACKER_ADDRESS, "localhost:0");
  pjt = ProxyJobTracker.startProxyTracker(new CoronaConf(conf));
  this.proxyJobTrackerPort = pjt.getRpcPort();
  configureJobConf(conf, builder.namenode, clusterManagerPort,
    proxyJobTrackerPort, builder.ugi);
  for (int i = 0; i < builder.numTaskTrackers; ++i) {
    String host = builder.hosts == null ?
        "host" + i + ".foo.com" : builder.hosts[i];
    String rack = builder.racks == null ?
        NetworkTopology.DEFAULT_RACK : builder.racks[i];
    startTaskTracker(host, rack, i, builder.numDir);
  }
  waitTaskTrackers();
}

Example 12

Source File: FileInputFormat.java From hadoop-gpu with Apache License 2.0

5 votes

private String[] fakeRacks(BlockLocation[] blkLocations, int index) 
throws IOException {
  String[] allHosts = blkLocations[index].getHosts();
  String[] allTopos = new String[allHosts.length];
  for (int i = 0; i < allHosts.length; i++) {
    allTopos[i] = NetworkTopology.DEFAULT_RACK + "/" + allHosts[i];
  }
  return allTopos;
}

Example 13

Source File: TestAMRMClientOnRMRestart.java From hadoop with Apache License 2.0

4 votes

private ContainerRequest createReq(int priority, int memory, String[] hosts) {
  Resource capability = Resource.newInstance(memory, 1);
  Priority priorityOfContainer = Priority.newInstance(priority);
  return new ContainerRequest(capability, hosts,
      new String[] { NetworkTopology.DEFAULT_RACK }, priorityOfContainer);
}

Example 14

Source File: Application.java From big-c with Apache License 2.0

4 votes

public static String resolve(String hostName) {
  return NetworkTopology.DEFAULT_RACK;
}

Example 15

Source File: TestAMRMClientOnRMRestart.java From big-c with Apache License 2.0

4 votes

private ContainerRequest createReq(int priority, int memory, String[] hosts) {
  Resource capability = Resource.newInstance(memory, 1);
  Priority priorityOfContainer = Priority.newInstance(priority);
  return new ContainerRequest(capability, hosts,
      new String[] { NetworkTopology.DEFAULT_RACK }, priorityOfContainer);
}

Example 16

Source File: MiniMRCluster.java From RDFS with Apache License 2.0

4 votes

public MiniMRCluster(int jobTrackerPort, int taskTrackerPort,
    int numTaskTrackers, String namenode, 
    int numDir, String[] racks, String[] hosts, UnixUserGroupInformation ugi,
    JobConf conf, int numTrackerToExclude) throws IOException {
  if (racks != null && racks.length < numTaskTrackers) {
    LOG.error("Invalid number of racks specified. It should be at least " +
        "equal to the number of tasktrackers");
    shutdown();
  }
  if (hosts != null && numTaskTrackers > hosts.length ) {
    throw new IllegalArgumentException( "The length of hosts [" + hosts.length
        + "] is less than the number of tasktrackers [" + numTaskTrackers + "].");
  }
   
   //Generate rack names if required
   if (racks == null) {
     System.out.println("Generating rack names for tasktrackers");
     racks = new String[numTaskTrackers];
     for (int i=0; i < racks.length; ++i) {
       racks[i] = NetworkTopology.DEFAULT_RACK;
     }
   }
   
  //Generate some hostnames if required
  if (hosts == null) {
    System.out.println("Generating host names for tasktrackers");
    hosts = new String[numTaskTrackers];
    for (int i = 0; i < numTaskTrackers; i++) {
      hosts[i] = "host" + i + ".foo.com";
    }
  }
  this.jobTrackerPort = jobTrackerPort;
  this.taskTrackerPort = taskTrackerPort;
  this.jobTrackerInfoPort = 0;
  this.numTaskTrackers = 0;
  this.namenode = namenode;
  this.ugi = ugi;
  this.conf = conf; // this is the conf the mr starts with
  this.numTrackerToExclude = numTrackerToExclude;

  // start the jobtracker
  startJobTracker();

  // Create the TaskTrackers
  for (int idx = 0; idx < numTaskTrackers; idx++) {
    String rack = null;
    String host = null;
    if (racks != null) {
      rack = racks[idx];
    }
    if (hosts != null) {
      host = hosts[idx];
    }
    
    startTaskTracker(host, rack, idx, numDir);
  }

  this.job = createJobConf(conf);
  waitUntilIdle();
}

Example 17

Source File: MiniMRCluster.java From hadoop-gpu with Apache License 2.0

4 votes

public MiniMRCluster(int jobTrackerPort, int taskTrackerPort,
    int numTaskTrackers, String namenode, 
    int numDir, String[] racks, String[] hosts, UnixUserGroupInformation ugi,
    JobConf conf) throws IOException {
  if (racks != null && racks.length < numTaskTrackers) {
    LOG.error("Invalid number of racks specified. It should be at least " +
        "equal to the number of tasktrackers");
    shutdown();
  }
  if (hosts != null && numTaskTrackers > hosts.length ) {
    throw new IllegalArgumentException( "The length of hosts [" + hosts.length
        + "] is less than the number of tasktrackers [" + numTaskTrackers + "].");
  }
   
   //Generate rack names if required
   if (racks == null) {
     System.out.println("Generating rack names for tasktrackers");
     racks = new String[numTaskTrackers];
     for (int i=0; i < racks.length; ++i) {
       racks[i] = NetworkTopology.DEFAULT_RACK;
     }
   }
   
  //Generate some hostnames if required
  if (hosts == null) {
    System.out.println("Generating host names for tasktrackers");
    hosts = new String[numTaskTrackers];
    for (int i = 0; i < numTaskTrackers; i++) {
      hosts[i] = "host" + i + ".foo.com";
    }
  }
  this.jobTrackerPort = jobTrackerPort;
  this.taskTrackerPort = taskTrackerPort;
  this.jobTrackerInfoPort = 0;
  this.numTaskTrackers = 0;
  this.namenode = namenode;
  this.ugi = ugi;
  this.conf = conf; // this is the conf the mr starts with

  // start the jobtracker
  startJobTracker();

  // Create the TaskTrackers
  for (int idx = 0; idx < numTaskTrackers; idx++) {
    String rack = null;
    String host = null;
    if (racks != null) {
      rack = racks[idx];
    }
    if (hosts != null) {
      host = hosts[idx];
    }
    
    startTaskTracker(host, rack, idx, numDir);
  }

  this.job = createJobConf(conf);
  waitUntilIdle();
}

Example 18

Source File: Application.java From hadoop with Apache License 2.0

4 votes

public static String resolve(String hostName) {
  return NetworkTopology.DEFAULT_RACK;
}