org.apache.hadoop.net.NodeBase Java Examples

The following examples show how to use org.apache.hadoop.net.NodeBase. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: CombineFileInputFormat.java    From big-c with Apache License 2.0 6 votes vote down vote up
OneBlockInfo(Path path, long offset, long len, 
             String[] hosts, String[] topologyPaths) {
  this.onepath = path;
  this.offset = offset;
  this.hosts = hosts;
  this.length = len;
  assert (hosts.length == topologyPaths.length ||
          topologyPaths.length == 0);

  // if the file system does not have any rack information, then
  // use dummy rack location.
  if (topologyPaths.length == 0) {
    topologyPaths = new String[hosts.length];
    for (int i = 0; i < topologyPaths.length; i++) {
      topologyPaths[i] = (new NodeBase(hosts[i], 
                          NetworkTopology.DEFAULT_RACK)).toString();
    }
  }

  // The topology paths have the host name included as the last 
  // component. Strip it.
  this.racks = new String[topologyPaths.length];
  for (int i = 0; i < topologyPaths.length; i++) {
    this.racks[i] = (new NodeBase(topologyPaths[i])).getNetworkLocation();
  }
}
 
Example #2
Source File: CombineFileInputFormat.java    From RDFS with Apache License 2.0 6 votes vote down vote up
OneBlockInfo(Path path, long offset, long len,
             String[] hosts, String[] topologyPaths) {
  this.onepath = path;
  this.offset = offset;
  this.hosts = hosts;
  this.length = len;
  assert (hosts.length == topologyPaths.length ||
          topologyPaths.length == 0);

  // if the file ystem does not have any rack information, then
  // use dummy rack location.
  if (topologyPaths.length == 0) {
    topologyPaths = new String[hosts.length];
    for (int i = 0; i < topologyPaths.length; i++) {
      topologyPaths[i] = (new NodeBase(hosts[i], NetworkTopology.DEFAULT_RACK)).
                                      toString();
    }
  }

  // The topology paths have the host name included as the last
  // component. Strip it.
  this.racks = new String[topologyPaths.length];
  for (int i = 0; i < topologyPaths.length; i++) {
    this.racks[i] = (new NodeBase(topologyPaths[i])).getNetworkLocation();
  }
}
 
Example #3
Source File: HdfsPartitioner.java    From sqoop-on-spark with Apache License 2.0 6 votes vote down vote up
OneBlockInfo(Path path, long offset, long len,
             String[] hosts, String[] topologyPaths) {
  this.onepath = path;
  this.offset = offset;
  this.hosts = hosts;
  this.length = len;
  assert (hosts.length == topologyPaths.length ||
          topologyPaths.length == 0);

  // if the file system does not have any rack information, then
  // use dummy rack location.
  if (topologyPaths.length == 0) {
    topologyPaths = new String[hosts.length];
    for (int i = 0; i < topologyPaths.length; i++) {
      topologyPaths[i] = (new NodeBase(hosts[i],
                          NetworkTopology.DEFAULT_RACK)).toString();
    }
  }

  // The topology paths have the host name included as the last
  // component. Strip it.
  this.racks = new String[topologyPaths.length];
  for (int i = 0; i < topologyPaths.length; i++) {
    this.racks[i] = (new NodeBase(topologyPaths[i])).getNetworkLocation();
  }
}
 
Example #4
Source File: TopologyCache.java    From RDFS with Apache License 2.0 6 votes vote down vote up
private Node resolveAndGetNode(String name) {
  List <String> rNameList = dnsToSwitchMapping.resolve(Arrays.asList(new String [] {name}));
  String networkLoc = NodeBase.normalize(rNameList.get(0));
  Node node = null;

  // we depend on clusterMap to get a canonical node object
  // we synchronize this section to guarantee that two concurrent
  // insertions into the clusterMap don't happen (resulting in
  // multiple copies of the same node being created and returned)
  synchronized (clusterMap) {
    while ((node = clusterMap.getNode(networkLoc+"/"+name)) == null) {
      clusterMap.add(new NodeBase(name, networkLoc));
    }
  }

  return node;
}
 
Example #5
Source File: BlockPlacementPolicyDefault.java    From big-c with Apache License 2.0 6 votes vote down vote up
private DatanodeStorageInfo chooseFromNextRack(Node next,
    Set<Node> excludedNodes,
    long blocksize,
    int maxNodesPerRack,
    List<DatanodeStorageInfo> results,
    boolean avoidStaleNodes,
    EnumMap<StorageType, Integer> storageTypes) throws NotEnoughReplicasException {
  final String nextRack = next.getNetworkLocation();
  try {
    return chooseRandom(nextRack, excludedNodes, blocksize, maxNodesPerRack,
        results, avoidStaleNodes, storageTypes);
  } catch(NotEnoughReplicasException e) {
    if (LOG.isDebugEnabled()) {
      LOG.debug("Failed to choose from the next rack (location = " + nextRack
          + "), retry choosing ramdomly", e);
    }
    //otherwise randomly choose one from the network
    return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize,
        maxNodesPerRack, results, avoidStaleNodes, storageTypes);
  }
}
 
Example #6
Source File: BlockPlacementPolicyDefault.java    From RDFS with Apache License 2.0 6 votes vote down vote up
protected DatanodeDescriptor chooseLocalNode(
                                           DatanodeDescriptor localMachine,
                                           HashMap<Node, Node> excludedNodes,
                                           long blocksize,
                                           int maxNodesPerRack,
                                           List<DatanodeDescriptor> results)
  throws NotEnoughReplicasException {
  // if no local machine, randomly choose one node
  if (localMachine == null)
    return chooseRandom(NodeBase.ROOT, excludedNodes, 
                        blocksize, maxNodesPerRack, results);
    
  // otherwise try local machine first
  Node oldNode = excludedNodes.put(localMachine, localMachine);
  if (oldNode == null) { // was not in the excluded list
    if (isGoodTarget(localMachine, blocksize,
                     maxNodesPerRack, false, results)) {
      results.add(localMachine);
      return localMachine;
    }
  } 
    
  // try a node on local rack
  return chooseLocalRack(localMachine, excludedNodes, 
                         blocksize, maxNodesPerRack, results);
}
 
Example #7
Source File: CombineFileInputFormat.java    From hraven with Apache License 2.0 6 votes vote down vote up
OneBlockInfo(Path path, long offset, long len, 
             String[] hosts, String[] topologyPaths) {
  this.onepath = path;
  this.offset = offset;
  this.hosts = hosts;
  this.length = len;
  assert (hosts.length == topologyPaths.length ||
          topologyPaths.length == 0);

  // if the file system does not have any rack information, then
  // use dummy rack location.
  if (topologyPaths.length == 0) {
    topologyPaths = new String[hosts.length];
    for (int i = 0; i < topologyPaths.length; i++) {
      topologyPaths[i] = (new NodeBase(hosts[i], 
                          NetworkTopology.DEFAULT_RACK)).toString();
    }
  }

  // The topology paths have the host name included as the last 
  // component. Strip it.
  this.racks = new String[topologyPaths.length];
  for (int i = 0; i < topologyPaths.length; i++) {
    this.racks[i] = (new NodeBase(topologyPaths[i])).getNetworkLocation();
  }
}
 
Example #8
Source File: CombineFileInputFormat.java    From hadoop-gpu with Apache License 2.0 6 votes vote down vote up
OneBlockInfo(Path path, long offset, long len, 
             String[] hosts, String[] topologyPaths) {
  this.onepath = path;
  this.offset = offset;
  this.hosts = hosts;
  this.length = len;
  assert (hosts.length == topologyPaths.length ||
          topologyPaths.length == 0);

  // if the file ystem does not have any rack information, then
  // use dummy rack location.
  if (topologyPaths.length == 0) {
    topologyPaths = new String[hosts.length];
    for (int i = 0; i < topologyPaths.length; i++) {
      topologyPaths[i] = (new NodeBase(hosts[i], NetworkTopology.DEFAULT_RACK)).
                                      toString();
    }
  }

  // The topology paths have the host name included as the last 
  // component. Strip it.
  this.racks = new String[topologyPaths.length];
  for (int i = 0; i < topologyPaths.length; i++) {
    this.racks[i] = (new NodeBase(topologyPaths[i])).getNetworkLocation();
  }
}
 
Example #9
Source File: BlockPlacementPolicyDefault.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private DatanodeStorageInfo chooseFromNextRack(Node next,
    Set<Node> excludedNodes,
    long blocksize,
    int maxNodesPerRack,
    List<DatanodeStorageInfo> results,
    boolean avoidStaleNodes,
    EnumMap<StorageType, Integer> storageTypes) throws NotEnoughReplicasException {
  final String nextRack = next.getNetworkLocation();
  try {
    return chooseRandom(nextRack, excludedNodes, blocksize, maxNodesPerRack,
        results, avoidStaleNodes, storageTypes);
  } catch(NotEnoughReplicasException e) {
    if (LOG.isDebugEnabled()) {
      LOG.debug("Failed to choose from the next rack (location = " + nextRack
          + "), retry choosing ramdomly", e);
    }
    //otherwise randomly choose one from the network
    return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize,
        maxNodesPerRack, results, avoidStaleNodes, storageTypes);
  }
}
 
Example #10
Source File: CombineFileInputFormat.java    From hadoop with Apache License 2.0 6 votes vote down vote up
OneBlockInfo(Path path, long offset, long len, 
             String[] hosts, String[] topologyPaths) {
  this.onepath = path;
  this.offset = offset;
  this.hosts = hosts;
  this.length = len;
  assert (hosts.length == topologyPaths.length ||
          topologyPaths.length == 0);

  // if the file system does not have any rack information, then
  // use dummy rack location.
  if (topologyPaths.length == 0) {
    topologyPaths = new String[hosts.length];
    for (int i = 0; i < topologyPaths.length; i++) {
      topologyPaths[i] = (new NodeBase(hosts[i], 
                          NetworkTopology.DEFAULT_RACK)).toString();
    }
  }

  // The topology paths have the host name included as the last 
  // component. Strip it.
  this.racks = new String[topologyPaths.length];
  for (int i = 0; i < topologyPaths.length; i++) {
    this.racks[i] = (new NodeBase(topologyPaths[i])).getNetworkLocation();
  }
}
 
Example #11
Source File: CombineFileInputFormat.java    From aliyun-maxcompute-data-collectors with Apache License 2.0 6 votes vote down vote up
OneBlockInfo(Path path, long offset, long len,
             String[] hosts, String[] topologyPaths) {
  this.onepath = path;
  this.offset = offset;
  this.hosts = hosts;
  this.length = len;
  assert (hosts.length == topologyPaths.length ||
          topologyPaths.length == 0);

  // if the file system does not have any rack information, then
  // use dummy rack location.
  if (topologyPaths.length == 0) {
    topologyPaths = new String[hosts.length];
    for (int i = 0; i < topologyPaths.length; i++) {
      topologyPaths[i] = (new NodeBase(hosts[i],
                          NetworkTopology.DEFAULT_RACK)).toString();
    }
  }

  // The topology paths have the host name included as the last
  // component. Strip it.
  this.racks = new String[topologyPaths.length];
  for (int i = 0; i < topologyPaths.length; i++) {
    this.racks[i] = (new NodeBase(topologyPaths[i])).getNetworkLocation();
  }
}
 
Example #12
Source File: ReplicationTargetChooser.java    From hadoop-gpu with Apache License 2.0 6 votes vote down vote up
private DatanodeDescriptor chooseLocalNode(
                                           DatanodeDescriptor localMachine,
                                           List<Node> excludedNodes,
                                           long blocksize,
                                           int maxNodesPerRack,
                                           List<DatanodeDescriptor> results)
  throws NotEnoughReplicasException {
  // if no local machine, randomly choose one node
  if (localMachine == null)
    return chooseRandom(NodeBase.ROOT, excludedNodes, 
                        blocksize, maxNodesPerRack, results);
    
  // otherwise try local machine first
  if (!excludedNodes.contains(localMachine)) {
    excludedNodes.add(localMachine);
    if (isGoodTarget(localMachine, blocksize,
                     maxNodesPerRack, false, results)) {
      results.add(localMachine);
      return localMachine;
    }
  } 
    
  // try a node on local rack
  return chooseLocalRack(localMachine, excludedNodes, 
                         blocksize, maxNodesPerRack, results);
}
 
Example #13
Source File: RackResolver.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private static Node coreResolve(String hostName) {
  List <String> tmpList = new ArrayList<String>(1);
  tmpList.add(hostName);
  List <String> rNameList = dnsToSwitchMapping.resolve(tmpList);
  String rName = null;
  if (rNameList == null || rNameList.get(0) == null) {
    rName = NetworkTopology.DEFAULT_RACK;
    LOG.info("Couldn't resolve " + hostName + ". Falling back to "
        + NetworkTopology.DEFAULT_RACK);
  } else {
    rName = rNameList.get(0);
    LOG.info("Resolved " + hostName + " to " + rName);
  }
  return new NodeBase(hostName, rName);
}
 
Example #14
Source File: OfferLifeCycleManagerTest.java    From incubator-myriad with Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() throws Exception {
  NodeStore store = new NodeStore();
  NodeIdProto nodeId = NodeIdProto.newBuilder().setHost("localhost").setPort(8000).build();
  RMNode rmNode = new RMNodeImpl(new NodeIdPBImpl(nodeId), new MockRMContext(), "localhost", 8000, 8070, new NodeBase(),
          new ResourcePBImpl(), "1.0");
  SchedulerNode node = new FiCaSchedulerNode(rmNode, false);
  store.add(node);
  manager = new OfferLifecycleManager(store, new MyriadDriver(new MockSchedulerDriver()));
}
 
Example #15
Source File: JobTracker.java    From RDFS with Apache License 2.0 5 votes vote down vote up
public Node resolveAndAddToTopology(String name) {
  List <String> tmpList = new ArrayList<String>(1);
  tmpList.add(name);
  List <String> rNameList = dnsToSwitchMapping.resolve(tmpList);
  String rName = rNameList.get(0);
  String networkLoc = NodeBase.normalize(rName);
  return addHostToNodeMapping(name, networkLoc);
}
 
Example #16
Source File: BlockPlacementPolicyWithNodeGroup.java    From big-c with Apache License 2.0 5 votes vote down vote up
private DatanodeStorageInfo chooseLocalNodeGroup(
    NetworkTopologyWithNodeGroup clusterMap, Node localMachine,
    Set<Node> excludedNodes, long blocksize, int maxNodesPerRack,
    List<DatanodeStorageInfo> results, boolean avoidStaleNodes,
    EnumMap<StorageType, Integer> storageTypes) throws
    NotEnoughReplicasException {
  // no local machine, so choose a random machine
  if (localMachine == null) {
    return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize,
        maxNodesPerRack, results, avoidStaleNodes, storageTypes);
  }

  // choose one from the local node group
  try {
    return chooseRandom(
        clusterMap.getNodeGroup(localMachine.getNetworkLocation()),
        excludedNodes, blocksize, maxNodesPerRack, results, avoidStaleNodes,
        storageTypes);
  } catch (NotEnoughReplicasException e1) {
    final DatanodeDescriptor newLocal = secondNode(localMachine, results);
    if (newLocal != null) {
      try {
        return chooseRandom(
            clusterMap.getNodeGroup(newLocal.getNetworkLocation()),
            excludedNodes, blocksize, maxNodesPerRack, results,
            avoidStaleNodes, storageTypes);
      } catch(NotEnoughReplicasException e2) {
        //otherwise randomly choose one from the network
        return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize,
            maxNodesPerRack, results, avoidStaleNodes, storageTypes);
      }
    } else {
      //otherwise randomly choose one from the network
      return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize,
          maxNodesPerRack, results, avoidStaleNodes, storageTypes);
    }
  }
}
 
Example #17
Source File: BlockPlacementPolicyWithNodeGroup.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
protected DatanodeStorageInfo chooseLocalRack(Node localMachine,
    Set<Node> excludedNodes, long blocksize, int maxNodesPerRack,
    List<DatanodeStorageInfo> results, boolean avoidStaleNodes,
    EnumMap<StorageType, Integer> storageTypes) throws
    NotEnoughReplicasException {
  // no local machine, so choose a random machine
  if (localMachine == null) {
    return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize,
        maxNodesPerRack, results, avoidStaleNodes, storageTypes);
  }

  // choose one from the local rack, but off-nodegroup
  try {
    final String scope = NetworkTopology.getFirstHalf(localMachine.getNetworkLocation());
    return chooseRandom(scope, excludedNodes, blocksize, maxNodesPerRack,
        results, avoidStaleNodes, storageTypes);
  } catch (NotEnoughReplicasException e1) {
    // find the second replica
    final DatanodeDescriptor newLocal = secondNode(localMachine, results);
    if (newLocal != null) {
      try {
        return chooseRandom(
            clusterMap.getRack(newLocal.getNetworkLocation()), excludedNodes,
            blocksize, maxNodesPerRack, results, avoidStaleNodes,
            storageTypes);
      } catch(NotEnoughReplicasException e2) {
        //otherwise randomly choose one from the network
        return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize,
            maxNodesPerRack, results, avoidStaleNodes, storageTypes);
      }
    } else {
      //otherwise randomly choose one from the network
      return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize,
          maxNodesPerRack, results, avoidStaleNodes, storageTypes);
    }
  }
}
 
Example #18
Source File: RackResolver.java    From big-c with Apache License 2.0 5 votes vote down vote up
private static Node coreResolve(String hostName) {
  List <String> tmpList = new ArrayList<String>(1);
  tmpList.add(hostName);
  List <String> rNameList = dnsToSwitchMapping.resolve(tmpList);
  String rName = null;
  if (rNameList == null || rNameList.get(0) == null) {
    rName = NetworkTopology.DEFAULT_RACK;
    LOG.info("Couldn't resolve " + hostName + ". Falling back to "
        + NetworkTopology.DEFAULT_RACK);
  } else {
    rName = rNameList.get(0);
    LOG.info("Resolved " + hostName + " to " + rName);
  }
  return new NodeBase(hostName, rName);
}
 
Example #19
Source File: BlockPlacementPolicyWithNodeGroup.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private DatanodeStorageInfo chooseLocalNodeGroup(
    NetworkTopologyWithNodeGroup clusterMap, Node localMachine,
    Set<Node> excludedNodes, long blocksize, int maxNodesPerRack,
    List<DatanodeStorageInfo> results, boolean avoidStaleNodes,
    EnumMap<StorageType, Integer> storageTypes) throws
    NotEnoughReplicasException {
  // no local machine, so choose a random machine
  if (localMachine == null) {
    return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize,
        maxNodesPerRack, results, avoidStaleNodes, storageTypes);
  }

  // choose one from the local node group
  try {
    return chooseRandom(
        clusterMap.getNodeGroup(localMachine.getNetworkLocation()),
        excludedNodes, blocksize, maxNodesPerRack, results, avoidStaleNodes,
        storageTypes);
  } catch (NotEnoughReplicasException e1) {
    final DatanodeDescriptor newLocal = secondNode(localMachine, results);
    if (newLocal != null) {
      try {
        return chooseRandom(
            clusterMap.getNodeGroup(newLocal.getNetworkLocation()),
            excludedNodes, blocksize, maxNodesPerRack, results,
            avoidStaleNodes, storageTypes);
      } catch(NotEnoughReplicasException e2) {
        //otherwise randomly choose one from the network
        return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize,
            maxNodesPerRack, results, avoidStaleNodes, storageTypes);
      }
    } else {
      //otherwise randomly choose one from the network
      return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize,
          maxNodesPerRack, results, avoidStaleNodes, storageTypes);
    }
  }
}
 
Example #20
Source File: BlockPlacementPolicyWithNodeGroup.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
protected DatanodeStorageInfo chooseLocalRack(Node localMachine,
    Set<Node> excludedNodes, long blocksize, int maxNodesPerRack,
    List<DatanodeStorageInfo> results, boolean avoidStaleNodes,
    EnumMap<StorageType, Integer> storageTypes) throws
    NotEnoughReplicasException {
  // no local machine, so choose a random machine
  if (localMachine == null) {
    return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize,
        maxNodesPerRack, results, avoidStaleNodes, storageTypes);
  }

  // choose one from the local rack, but off-nodegroup
  try {
    final String scope = NetworkTopology.getFirstHalf(localMachine.getNetworkLocation());
    return chooseRandom(scope, excludedNodes, blocksize, maxNodesPerRack,
        results, avoidStaleNodes, storageTypes);
  } catch (NotEnoughReplicasException e1) {
    // find the second replica
    final DatanodeDescriptor newLocal = secondNode(localMachine, results);
    if (newLocal != null) {
      try {
        return chooseRandom(
            clusterMap.getRack(newLocal.getNetworkLocation()), excludedNodes,
            blocksize, maxNodesPerRack, results, avoidStaleNodes,
            storageTypes);
      } catch(NotEnoughReplicasException e2) {
        //otherwise randomly choose one from the network
        return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize,
            maxNodesPerRack, results, avoidStaleNodes, storageTypes);
      }
    } else {
      //otherwise randomly choose one from the network
      return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize,
          maxNodesPerRack, results, avoidStaleNodes, storageTypes);
    }
  }
}
 
Example #21
Source File: JobTracker.java    From hadoop-gpu with Apache License 2.0 5 votes vote down vote up
public Node resolveAndAddToTopology(String name) {
  List <String> tmpList = new ArrayList<String>(1);
  tmpList.add(name);
  List <String> rNameList = dnsToSwitchMapping.resolve(tmpList);
  String rName = rNameList.get(0);
  String networkLoc = NodeBase.normalize(rName);
  return addHostToNodeMapping(name, networkLoc);
}
 
Example #22
Source File: BlockPlacementPolicyDefault.java    From RDFS with Apache License 2.0 4 votes vote down vote up
protected DatanodeDescriptor chooseLocalRack(
                                           DatanodeDescriptor localMachine,
                                           HashMap<Node, Node> excludedNodes,
                                           long blocksize,
                                           int maxNodesPerRack,
                                           List<DatanodeDescriptor> results)
  throws NotEnoughReplicasException {
  // no local machine, so choose a random machine
  if (localMachine == null) {
    return chooseRandom(NodeBase.ROOT, excludedNodes, 
                        blocksize, maxNodesPerRack, results);
  }
    
  // choose one from the local rack
  try {
    return chooseRandom(
                        localMachine.getNetworkLocation(),
                        excludedNodes, blocksize, maxNodesPerRack, results);
  } catch (NotEnoughReplicasException e1) {
    // find the second replica
    DatanodeDescriptor newLocal=null;
    for(Iterator<DatanodeDescriptor> iter=results.iterator();
        iter.hasNext();) {
      DatanodeDescriptor nextNode = iter.next();
      if (nextNode != localMachine) {
        newLocal = nextNode;
        break;
      }
    }
    if (newLocal != null) {
      try {
        return chooseRandom(
                            newLocal.getNetworkLocation(),
                            excludedNodes, blocksize, maxNodesPerRack, results);
      } catch(NotEnoughReplicasException e2) {
        //otherwise randomly choose one from the network
        return chooseRandom(NodeBase.ROOT, excludedNodes,
                            blocksize, maxNodesPerRack, results);
      }
    } else {
      //otherwise randomly choose one from the network
      return chooseRandom(NodeBase.ROOT, excludedNodes,
                          blocksize, maxNodesPerRack, results);
    }
  }
}
 
Example #23
Source File: BlockPlacementPolicyDefault.java    From RDFS with Apache License 2.0 4 votes vote down vote up
protected boolean isGoodTarget(DatanodeDescriptor node,
                             long blockSize, int maxTargetPerLoc,
                             boolean considerLoad,
                             List<DatanodeDescriptor> results) {
  Log logr = FSNamesystem.LOG;
  // check if the node is (being) decommissed
  if (node.isDecommissionInProgress() || node.isDecommissioned()) {
    if (logr.isDebugEnabled()) {
      logr.debug("Node "+ NodeBase.getPath(node) +
              " is not chosen because the node is (being) decommissioned");
    }
    return false;
  }

  long remaining = node.getRemaining() - 
                   (node.getBlocksScheduled() * blockSize); 
  // check the remaining capacity of the target machine
  if (blockSize* FSConstants.MIN_BLOCKS_FOR_WRITE>remaining) {
    if (logr.isDebugEnabled()) {
      logr.debug("Node "+ NodeBase.getPath(node) +
              " is not chosen because the node does not have enough space" +
              " for block size " + blockSize +
              " with Remaining = " + node.getRemaining() + 
              " and Scheduled = " + node.getBlocksScheduled());
    }
    return false;
  }
    
  // check the communication traffic of the target machine
  if (considerLoad) {
    double avgLoad = 0;
    int size = clusterMap.getNumOfLeaves();
    if (size != 0 && stats != null) {
      avgLoad = (double)stats.getTotalLoad()/size;
    }
    if (node.getXceiverCount() > (2.0 * avgLoad)) {
      if (logr.isDebugEnabled()) {
        logr.debug("Node "+NodeBase.getPath(node)+
                " is not chosen because the node is too busy");
      }
      return false;
    }
  }
    
  // check if the target rack has chosen too many nodes
  String rackname = node.getNetworkLocation();
  int counter=1;
  for(Iterator<DatanodeDescriptor> iter = results.iterator();
      iter.hasNext();) {
    Node result = iter.next();
    if (rackname.equals(result.getNetworkLocation())) {
      counter++;
    }
  }
  if (counter>maxTargetPerLoc) {
    if (logr.isDebugEnabled()) {
      logr.debug("Node "+NodeBase.getPath(node)+
              " is not chosen because the rack has too many chosen nodes");
    }
    return false;
  }
  return true;
}
 
Example #24
Source File: BlockPlacementPolicyDefault.java    From RDFS with Apache License 2.0 4 votes vote down vote up
protected DatanodeDescriptor chooseTarget(int numOfReplicas,
                                        DatanodeDescriptor writer,
                                        HashMap<Node, Node> excludedNodes,
                                        long blocksize,
                                        int maxNodesPerRack,
                                        List<DatanodeDescriptor> results,
                                        boolean newBlock) {
    
  if (numOfReplicas == 0 || clusterMap.getNumOfLeaves()==0) {
    return writer;
  }
    
  int numOfResults = results.size();
  if (writer == null && !newBlock) {
    writer = results.get(0);
  }
    
  try {
    if (numOfResults == 0) {
      chooseLocalNode(writer, excludedNodes, 
                      blocksize, maxNodesPerRack, results);
      if (newBlock && writer == null) {
        writer = results.get(0);
      }
      if (--numOfReplicas == 0) {
        return writer;
      }
    }
    if (numOfResults <= 1) {
      choose2ndRack(writer, excludedNodes,
          blocksize, maxNodesPerRack, results);
      if (--numOfReplicas == 0) {
        return writer;
      }
    }
    if (numOfResults <= 2) {
      if (clusterMap.isOnSameRack(results.get(0), results.get(1))) {
        choose2ndRack(writer, excludedNodes,
            blocksize, maxNodesPerRack, results);
      } else if (newBlock){
        chooseLocalRack(results.get(1), excludedNodes, blocksize, 
                        maxNodesPerRack, results);
      } else {
        chooseLocalRack(writer, excludedNodes, blocksize,
                        maxNodesPerRack, results);
      }
      if (--numOfReplicas == 0) {
        return writer;
      }
    }
    chooseRandom(numOfReplicas, NodeBase.ROOT, excludedNodes, 
                 blocksize, maxNodesPerRack, results);
  } catch (NotEnoughReplicasException e) {
    FSNamesystem.LOG.warn("Not able to place enough replicas, still in need of "
             + numOfReplicas);
  }
  return writer;
}
 
Example #25
Source File: ReplicationTargetChooser.java    From hadoop-gpu with Apache License 2.0 4 votes vote down vote up
private DatanodeDescriptor chooseLocalRack(
                                           DatanodeDescriptor localMachine,
                                           List<Node> excludedNodes,
                                           long blocksize,
                                           int maxNodesPerRack,
                                           List<DatanodeDescriptor> results)
  throws NotEnoughReplicasException {
  // no local machine, so choose a random machine
  if (localMachine == null) {
    return chooseRandom(NodeBase.ROOT, excludedNodes, 
                        blocksize, maxNodesPerRack, results);
  }
    
  // choose one from the local rack
  try {
    return chooseRandom(
                        localMachine.getNetworkLocation(),
                        excludedNodes, blocksize, maxNodesPerRack, results);
  } catch (NotEnoughReplicasException e1) {
    // find the second replica
    DatanodeDescriptor newLocal=null;
    for(Iterator<DatanodeDescriptor> iter=results.iterator();
        iter.hasNext();) {
      DatanodeDescriptor nextNode = iter.next();
      if (nextNode != localMachine) {
        newLocal = nextNode;
        break;
      }
    }
    if (newLocal != null) {
      try {
        return chooseRandom(
                            newLocal.getNetworkLocation(),
                            excludedNodes, blocksize, maxNodesPerRack, results);
      } catch(NotEnoughReplicasException e2) {
        //otherwise randomly choose one from the network
        return chooseRandom(NodeBase.ROOT, excludedNodes,
                            blocksize, maxNodesPerRack, results);
      }
    } else {
      //otherwise randomly choose one from the network
      return chooseRandom(NodeBase.ROOT, excludedNodes,
                          blocksize, maxNodesPerRack, results);
    }
  }
}
 
Example #26
Source File: DatanodeInfo.java    From RDFS with Apache License 2.0 4 votes vote down vote up
/** Sets the rack name */
public synchronized void setNetworkLocation(String location) {
  this.location = NodeBase.normalize(location);
}
 
Example #27
Source File: ReplicationTargetChooser.java    From hadoop-gpu with Apache License 2.0 4 votes vote down vote up
private DatanodeDescriptor chooseTarget(int numOfReplicas,
                                        DatanodeDescriptor writer,
                                        List<Node> excludedNodes,
                                        long blocksize,
                                        int maxNodesPerRack,
                                        List<DatanodeDescriptor> results) {
    
  if (numOfReplicas == 0 || clusterMap.getNumOfLeaves()==0) {
    return writer;
  }
    
  int numOfResults = results.size();
  boolean newBlock = (numOfResults==0);
  if (writer == null && !newBlock) {
    writer = (DatanodeDescriptor)results.get(0);
  }
    
  try {
    switch(numOfResults) {
    case 0:
      writer = chooseLocalNode(writer, excludedNodes, 
                               blocksize, maxNodesPerRack, results);
      if (--numOfReplicas == 0) {
        break;
      }
    case 1:
      chooseRemoteRack(1, results.get(0), excludedNodes, 
                       blocksize, maxNodesPerRack, results);
      if (--numOfReplicas == 0) {
        break;
      }
    case 2:
      if (clusterMap.isOnSameRack(results.get(0), results.get(1))) {
        chooseRemoteRack(1, results.get(0), excludedNodes,
                         blocksize, maxNodesPerRack, results);
      } else if (newBlock){
        chooseLocalRack(results.get(1), excludedNodes, blocksize, 
                        maxNodesPerRack, results);
      } else {
        chooseLocalRack(writer, excludedNodes, blocksize,
                        maxNodesPerRack, results);
      }
      if (--numOfReplicas == 0) {
        break;
      }
    default:
      chooseRandom(numOfReplicas, NodeBase.ROOT, excludedNodes, 
                   blocksize, maxNodesPerRack, results);
    }
  } catch (NotEnoughReplicasException e) {
    FSNamesystem.LOG.warn("Not able to place enough replicas, still in need of "
             + numOfReplicas);
  }
  return writer;
}
 
Example #28
Source File: ReplicationTargetChooser.java    From hadoop-gpu with Apache License 2.0 4 votes vote down vote up
private boolean isGoodTarget(DatanodeDescriptor node,
                             long blockSize, int maxTargetPerLoc,
                             boolean considerLoad,
                             List<DatanodeDescriptor> results) {
  Log logr = FSNamesystem.LOG;
  // check if the node is (being) decommissed
  if (node.isDecommissionInProgress() || node.isDecommissioned()) {
    logr.debug("Node "+NodeBase.getPath(node)+
              " is not chosen because the node is (being) decommissioned");
    return false;
  }

  long remaining = node.getRemaining() - 
                   (node.getBlocksScheduled() * blockSize); 
  // check the remaining capacity of the target machine
  if (blockSize* FSConstants.MIN_BLOCKS_FOR_WRITE>remaining) {
    logr.debug("Node "+NodeBase.getPath(node)+
              " is not chosen because the node does not have enough space");
    return false;
  }
    
  // check the communication traffic of the target machine
  if (considerLoad) {
    double avgLoad = 0;
    int size = clusterMap.getNumOfLeaves();
    if (size != 0) {
      avgLoad = (double)fs.getTotalLoad()/size;
    }
    if (node.getXceiverCount() > (2.0 * avgLoad)) {
      logr.debug("Node "+NodeBase.getPath(node)+
                " is not chosen because the node is too busy");
      return false;
    }
  }
    
  // check if the target rack has chosen too many nodes
  String rackname = node.getNetworkLocation();
  int counter=1;
  for(Iterator<DatanodeDescriptor> iter = results.iterator();
      iter.hasNext();) {
    Node result = iter.next();
    if (rackname.equals(result.getNetworkLocation())) {
      counter++;
    }
  }
  if (counter>maxTargetPerLoc) {
    logr.debug("Node "+NodeBase.getPath(node)+
              " is not chosen because the rack has too many chosen nodes");
    return false;
  }
  return true;
}
 
Example #29
Source File: DatanodeInfo.java    From hadoop-gpu with Apache License 2.0 4 votes vote down vote up
/** Sets the rack name */
public synchronized void setNetworkLocation(String location) {
  this.location = NodeBase.normalize(location);
}
 
Example #30
Source File: TestObjectFactory.java    From incubator-myriad with Apache License 2.0 4 votes vote down vote up
public static RMNode getRMNode(String host, int port, Resource resource) {
  NodeId id = NodeId.newInstance(host, port);
  RMContext context = new MockRMContext();
  return new RMNodeImpl(id, context, id.getHost(), id.getPort(), id.getPort(), new NodeBase(host, "/tmp"), resource, "version-one");
}