Java Code Examples for org.apache.hadoop.net.Node#getParent()

Map<Node, List<TaskInProgress>> createCache(JobClient.RawSplit[] splits,
                                            int maxLevel) {
  Map<Node, List<TaskInProgress>> cache =
    new IdentityHashMap<Node, List<TaskInProgress>>(maxLevel);

  for (int i = 0; i < splits.length; i++) {
    String[] splitLocations = splits[i].getLocations();
    if (splitLocations.length == 0) {
      nonLocalMaps.add(maps[i]);
      continue;
    }

    for(String host: splitLocations) {
      Node node = jobtracker.getNode(host);
      if (node == null) {
        node = jobtracker.resolveAndAddToTopology(host);
      }
      LOG.debug("tip:" + maps[i].getTIPId() + " has split on node:" + node);
      for (int j = 0; j < maxLevel; j++) {
        List<TaskInProgress> hostMaps = cache.get(node);
        if (hostMaps == null) {
          hostMaps = new ArrayList<TaskInProgress>();
          cache.put(node, hostMaps);
          hostMaps.add(maps[i]);
        }
        //check whether the hostMaps already contains an entry for a TIP
        //This will be true for nodes that are racks and multiple nodes in
        //the rack contain the input for a tip. Note that if it already
        //exists in the hostMaps, it must be the last element there since
        //we process one TIP at a time sequentially in the split-size order
        if (hostMaps.get(hostMaps.size() - 1) != maps[i]) {
          hostMaps.add(maps[i]);
        }
        node = node.getParent();
      }
    }
  }
  return cache;
}
 

private int getMatchingLevelForNodes(Node n1, Node n2) {
  int count = 0;
  do {
    if (n1.equals(n2)) {
      return count;
    }
    ++count;
    n1 = n1.getParent();
    n2 = n2.getParent();
  } while (n1 != null && n2 != null);
  return this.maxLevel;
}
 

/**
 * Remove a map TIP from the lists for running maps.
 * Called when a map fails/completes (note if a map is killed,
 * it won't be present in the list since it was completed earlier)
 * @param tip the tip that needs to be retired
 */
private synchronized void retireMap(TaskInProgress tip) {
  if (runningMapCache == null) {
    LOG.warn("Running cache for maps missing!! "
             + "Job details are missing.");
    return;
  }

  String[] splitLocations = tip.getSplitLocations();

  // Remove the TIP from the list for running non-local maps
  if (splitLocations.length == 0) {
    nonLocalRunningMaps.remove(tip);
    return;
  }

  // Remove from the running map caches
  for(String host: splitLocations) {
    Node node = jobtracker.getNode(host);

    for (int j = 0; j < maxLevel; ++j) {
      Set<TaskInProgress> hostMaps = runningMapCache.get(node);
      if (hostMaps != null) {
        hostMaps.remove(tip);
        if (hostMaps.size() == 0) {
          runningMapCache.remove(node);
        }
      }
      node = node.getParent();
    }
  }
}
 

/**
 * Adds a map tip to the list of running maps.
 * @param tip the tip that needs to be scheduled as running
 */
protected synchronized void scheduleMap(TaskInProgress tip) {
  runningMapTaskStats.add(0.0f);
  runningTaskMapByteProcessingRateStats.add(0.0f);
  if (runningMapCache == null) {
    LOG.warn("Running cache for maps is missing!! "
             + "Job details are missing.");
    return;
  }
  String[] splitLocations = tip.getSplitLocations();

  // Add the TIP to the list of non-local running TIPs
  if (splitLocations.length == 0) {
    nonLocalRunningMaps.add(tip);
    return;
  }

  for(String host: splitLocations) {
    Node node = jobtracker.getNode(host);

    for (int j = 0; j < maxLevel; ++j) {
      Set<TaskInProgress> hostMaps = runningMapCache.get(node);
      if (hostMaps == null) {
        // create a cache if needed
        hostMaps = new LinkedHashSet<TaskInProgress>();
        runningMapCache.put(node, hostMaps);
      }
      hostMaps.add(tip);
      node = node.getParent();
    }
  }
}
 

/**
 * Adds the failed TIP in the front of the list for non-running maps
 * @param tip the tip that needs to be failed
 */
private synchronized void failMap(TaskInProgress tip) {
  if (nonRunningMapCache == null) {
    LOG.warn("Non-running cache for maps missing!! "
             + "Job details are missing.");
    return;
  }

  // 1. Its added everywhere since other nodes (having this split local)
  //    might have removed this tip from their local cache
  // 2. Give high priority to failed tip - fail early

  String[] splitLocations = tip.getSplitLocations();

  // Add the TIP in the front of the list for non-local non-running maps
  if (splitLocations.length == 0) {
    nonLocalMaps.add(0, tip);
    return;
  }

  for(String host: splitLocations) {
    Node node = jobtracker.getNode(host);

    for (int j = 0; j < maxLevel; ++j) {
      List<TaskInProgress> hostMaps = nonRunningMapCache.get(node);
      if (hostMaps == null) {
        hostMaps = new LinkedList<TaskInProgress>();
        nonRunningMapCache.put(node, hostMaps);
      }
      hostMaps.add(0, tip);
      node = node.getParent();
    }
  }
}
 

public static int getMatchingLevelForNodes(Node n1, Node n2, int maxLevel) {
  int count = 0;
  do {
    if (n1.equals(n2)) {
      return count;
    }
    ++count;
    n1 = n1.getParent();
    n2 = n2.getParent();
  } while (n1 != null && n2 != null);
  return maxLevel;
}
 

private Map<Node, List<TaskInProgress>> createCache(
                       JobClient.RawSplit[] splits, int maxLevel) {
  Map<Node, List<TaskInProgress>> cache = 
    new IdentityHashMap<Node, List<TaskInProgress>>(maxLevel);
  
  for (int i = 0; i < splits.length; i++) {
    String[] splitLocations = splits[i].getLocations();
    if (splitLocations.length == 0) {
      nonLocalMaps.add(maps[i]);
      continue;
    }

    for(String host: splitLocations) {
      Node node = jobtracker.resolveAndAddToTopology(host);
      LOG.info("tip:" + maps[i].getTIPId() + " has split on node:" + node);
      for (int j = 0; j < maxLevel; j++) {
        List<TaskInProgress> hostMaps = cache.get(node);
        if (hostMaps == null) {
          hostMaps = new ArrayList<TaskInProgress>();
          cache.put(node, hostMaps);
          hostMaps.add(maps[i]);
        }
        //check whether the hostMaps already contains an entry for a TIP
        //This will be true for nodes that are racks and multiple nodes in
        //the rack contain the input for a tip. Note that if it already
        //exists in the hostMaps, it must be the last element there since
        //we process one TIP at a time sequentially in the split-size order
        if (hostMaps.get(hostMaps.size() - 1) != maps[i]) {
          hostMaps.add(maps[i]);
        }
        node = node.getParent();
      }
    }
  }
  return cache;
}
 

private int getMatchingLevelForNodes(Node n1, Node n2) {
  int count = 0;
  do {
    if (n1.equals(n2)) {
      return count;
    }
    ++count;
    n1 = n1.getParent();
    n2 = n2.getParent();
  } while (n1 != null);
  return this.maxLevel;
}
 

/**
 * Remove a map TIP from the lists for running maps.
 * Called when a map fails/completes (note if a map is killed,
 * it won't be present in the list since it was completed earlier)
 * @param tip the tip that needs to be retired
 */
private synchronized void retireMap(TaskInProgress tip) {
  if (runningMapCache == null) {
    LOG.warn("Running cache for maps missing!! "
             + "Job details are missing.");
    return;
  }
  
  String[] splitLocations = tip.getSplitLocations();

  // Remove the TIP from the list for running non-local maps
  if (splitLocations.length == 0) {
    nonLocalRunningMaps.remove(tip);
    return;
  }

  // Remove from the running map caches
  for(String host: splitLocations) {
    Node node = jobtracker.getNode(host);

    for (int j = 0; j < maxLevel; ++j) {
      Set<TaskInProgress> hostMaps = runningMapCache.get(node);
      if (hostMaps != null) {
        hostMaps.remove(tip);
        if (hostMaps.size() == 0) {
          runningMapCache.remove(node);
        }
      }
      node = node.getParent();
    }
  }
}
 

/**
 * Adds a map tip to the list of running maps.
 * @param tip the tip that needs to be scheduled as running
 */
private synchronized void scheduleMap(TaskInProgress tip) {
  
  if (runningMapCache == null) {
    LOG.warn("Running cache for maps is missing!! " 
             + "Job details are missing.");
    return;
  }
  String[] splitLocations = tip.getSplitLocations();

  // Add the TIP to the list of non-local running TIPs
  if (splitLocations.length == 0) {
    nonLocalRunningMaps.add(tip);
    return;
  }

  for(String host: splitLocations) {
    Node node = jobtracker.getNode(host);

    for (int j = 0; j < maxLevel; ++j) {
      Set<TaskInProgress> hostMaps = runningMapCache.get(node);
      if (hostMaps == null) {
        // create a cache if needed
        hostMaps = new LinkedHashSet<TaskInProgress>();
        runningMapCache.put(node, hostMaps);
      }
      hostMaps.add(tip);
      node = node.getParent();
    }
  }
}
 

/**
 * Adds the failed TIP in the front of the list for non-running maps
 * @param tip the tip that needs to be failed
 */
private synchronized void failMap(TaskInProgress tip) {
  if (nonRunningMapCache == null) {
    LOG.warn("Non-running cache for maps missing!! "
             + "Job details are missing.");
    return;
  }

  // 1. Its added everywhere since other nodes (having this split local)
  //    might have removed this tip from their local cache
  // 2. Give high priority to failed tip - fail early

  String[] splitLocations = tip.getSplitLocations();

  // Add the TIP in the front of the list for non-local non-running maps
  if (splitLocations.length == 0) {
    nonLocalMaps.add(0, tip);
    return;
  }

  for(String host: splitLocations) {
    Node node = jobtracker.getNode(host);
    
    for (int j = 0; j < maxLevel; ++j) {
      List<TaskInProgress> hostMaps = nonRunningMapCache.get(node);
      if (hostMaps == null) {
        hostMaps = new LinkedList<TaskInProgress>();
        nonRunningMapCache.put(node, hostMaps);
      }
      hostMaps.add(0, tip);
      node = node.getParent();
    }
  }
}
 

public static Node getParentNode(Node node, int level) {
  for (int i = 0; i < level; ++i) {
    node = node.getParent();
  }
  return node;
}
 

public static Node getParentNode(Node node, int level) {
  for (int i = 0; i < level; ++i) {
    node = node.getParent();
  }
  return node;
}
 

public static Node getParentNode(Node node, int level) {
  for (int i = 0; i < level; ++i) {
    node = node.getParent();
  }
  return node;
}