org.apache.hadoop.net.NetworkTopology Java Examples

The following examples show how to use org.apache.hadoop.net.NetworkTopology. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: BlockPlacementPolicyWithNodeGroup.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * {@inheritDoc}
 */
@Override
protected void chooseRemoteRack(int numOfReplicas,
    DatanodeDescriptor localMachine, Set<Node> excludedNodes,
    long blocksize, int maxReplicasPerRack, List<DatanodeStorageInfo> results,
    boolean avoidStaleNodes, EnumMap<StorageType, Integer> storageTypes)
    throws NotEnoughReplicasException {
  int oldNumOfReplicas = results.size();

  final String rackLocation = NetworkTopology.getFirstHalf(
      localMachine.getNetworkLocation());
  try {
    // randomly choose from remote racks
    chooseRandom(numOfReplicas, "~" + rackLocation, excludedNodes, blocksize,
        maxReplicasPerRack, results, avoidStaleNodes, storageTypes);
  } catch (NotEnoughReplicasException e) {
    // fall back to the local rack
    chooseRandom(numOfReplicas - (results.size() - oldNumOfReplicas),
        rackLocation, excludedNodes, blocksize,
        maxReplicasPerRack, results, avoidStaleNodes, storageTypes);
  }
}
 
Example #2
Source File: DatanodeInfo.java    From hadoop-gpu with Apache License 2.0 6 votes vote down vote up
/** A formatted string for printing the status of the DataNode. */
public String dumpDatanode() {
  StringBuffer buffer = new StringBuffer();
  long c = getCapacity();
  long r = getRemaining();
  long u = getDfsUsed();
  buffer.append(name);
  if (!NetworkTopology.DEFAULT_RACK.equals(location)) {
    buffer.append(" "+location);
  }
  if (isDecommissioned()) {
    buffer.append(" DD");
  } else if (isDecommissionInProgress()) {
    buffer.append(" DP");
  } else {
    buffer.append(" IN");
  }
  buffer.append(" " + c + "(" + StringUtils.byteDesc(c)+")");
  buffer.append(" " + u + "(" + StringUtils.byteDesc(u)+")");
  buffer.append(" " + StringUtils.limitDecimalTo2(((1.0*u)/c)*100)+"%");
  buffer.append(" " + r + "(" + StringUtils.byteDesc(r)+")");
  buffer.append(" " + new Date(lastUpdate));
  return buffer.toString();
}
 
Example #3
Source File: BlockPlacementPolicyDefault.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
public void initialize(Configuration conf,  FSClusterStats stats,
                       NetworkTopology clusterMap, 
                       Host2NodesMap host2datanodeMap) {
  this.considerLoad = conf.getBoolean(
      DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY, true);
  this.stats = stats;
  this.clusterMap = clusterMap;
  this.host2datanodeMap = host2datanodeMap;
  this.heartbeatInterval = conf.getLong(
      DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,
      DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT) * 1000;
  this.tolerateHeartbeatMultiplier = conf.getInt(
      DFSConfigKeys.DFS_NAMENODE_TOLERATE_HEARTBEAT_MULTIPLIER_KEY,
      DFSConfigKeys.DFS_NAMENODE_TOLERATE_HEARTBEAT_MULTIPLIER_DEFAULT);
  this.staleInterval = conf.getLong(
      DFSConfigKeys.DFS_NAMENODE_STALE_DATANODE_INTERVAL_KEY, 
      DFSConfigKeys.DFS_NAMENODE_STALE_DATANODE_INTERVAL_DEFAULT);
}
 
Example #4
Source File: BlockPlacementPolicyWithNodeGroup.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * {@inheritDoc}
 */
@Override
protected void chooseRemoteRack(int numOfReplicas,
    DatanodeDescriptor localMachine, Set<Node> excludedNodes,
    long blocksize, int maxReplicasPerRack, List<DatanodeStorageInfo> results,
    boolean avoidStaleNodes, EnumMap<StorageType, Integer> storageTypes)
    throws NotEnoughReplicasException {
  int oldNumOfReplicas = results.size();

  final String rackLocation = NetworkTopology.getFirstHalf(
      localMachine.getNetworkLocation());
  try {
    // randomly choose from remote racks
    chooseRandom(numOfReplicas, "~" + rackLocation, excludedNodes, blocksize,
        maxReplicasPerRack, results, avoidStaleNodes, storageTypes);
  } catch (NotEnoughReplicasException e) {
    // fall back to the local rack
    chooseRandom(numOfReplicas - (results.size() - oldNumOfReplicas),
        rackLocation, excludedNodes, blocksize,
        maxReplicasPerRack, results, avoidStaleNodes, storageTypes);
  }
}
 
Example #5
Source File: TestRMContainerAllocator.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private void validateLabelsRequests(ResourceRequest resourceRequest,
    boolean isReduce) {
  switch (resourceRequest.getResourceName()) {
  case "map":
  case "reduce":
  case NetworkTopology.DEFAULT_RACK:
    Assert.assertNull(resourceRequest.getNodeLabelExpression());
    break;
  case "*":
    Assert.assertEquals(isReduce ? "ReduceNodes" : "MapNodes",
        resourceRequest.getNodeLabelExpression());
    break;
  default:
    Assert.fail("Invalid resource location "
        + resourceRequest.getResourceName());
  }
}
 
Example #6
Source File: TestRMContainerAllocator.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private ContainerRequestEvent
    createReq(JobId jobId, int taskAttemptId, int memory, String[] hosts,
        boolean earlierFailedAttempt, boolean reduce) {
  TaskId taskId;
  if (reduce) {
    taskId = MRBuilderUtils.newTaskId(jobId, 0, TaskType.REDUCE);
  } else {
    taskId = MRBuilderUtils.newTaskId(jobId, 0, TaskType.MAP);
  }
  TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId,
      taskAttemptId);
  Resource containerNeed = Resource.newInstance(memory, 1);
  if (earlierFailedAttempt) {
    return ContainerRequestEvent
        .createContainerRequestEventForFailedContainer(attemptId,
            containerNeed);
  }
  return new ContainerRequestEvent(attemptId, containerNeed, hosts,
      new String[] { NetworkTopology.DEFAULT_RACK });
}
 
Example #7
Source File: HdfsPartitioner.java    From sqoop-on-spark with Apache License 2.0 6 votes vote down vote up
OneBlockInfo(Path path, long offset, long len,
             String[] hosts, String[] topologyPaths) {
  this.onepath = path;
  this.offset = offset;
  this.hosts = hosts;
  this.length = len;
  assert (hosts.length == topologyPaths.length ||
          topologyPaths.length == 0);

  // if the file system does not have any rack information, then
  // use dummy rack location.
  if (topologyPaths.length == 0) {
    topologyPaths = new String[hosts.length];
    for (int i = 0; i < topologyPaths.length; i++) {
      topologyPaths[i] = (new NodeBase(hosts[i],
                          NetworkTopology.DEFAULT_RACK)).toString();
    }
  }

  // The topology paths have the host name included as the last
  // component. Strip it.
  this.racks = new String[topologyPaths.length];
  for (int i = 0; i < topologyPaths.length; i++) {
    this.racks[i] = (new NodeBase(topologyPaths[i])).getNetworkLocation();
  }
}
 
Example #8
Source File: TopologyCache.java    From RDFS with Apache License 2.0 6 votes vote down vote up
/**
 * Return the Node in the network topology that corresponds to the hostname
 */
public Node getNode(String name) {
  Node n = hostnameToNodeMap.get(name);

  // it's ok if multiple threads try to resolve the same host at the same time
  // the assumption is that resolve() will return a canonical node object and
  // the put operation is therefore idempotent
  if (n == null) {
    n = resolveAndGetNode(name);
    hostnameToNodeMap.put(name, n);
    // Make an entry for the node at the max level in the cache
    nodesAtMaxLevel.add(
      getParentNode(n, NetworkTopology.DEFAULT_HOST_LEVEL - 1));
  }

  return n;
}
 
Example #9
Source File: BlockReconstructor.java    From RDFS with Apache License 2.0 6 votes vote down vote up
/**
 * Return the distances among an array of nodes
 * @param nodes
 *        the nodes array
 * @return
 *        an distance matrix, distance of nodes[i][j] is put in
 *        element indexed by [i][j]
 */
static public int[][] getRealDistances(DatanodeInfo[] nodes) {
	// TODO Auto-generated method stub
	int[][] distances = new int[nodes.length][nodes.length];
	NetworkTopology netTopology = new NetworkTopology();
	for(int i = 0; i < distances.length; i++)
		netTopology.add(nodes[i]);
	for (int i = 0; i < distances.length; i++) {
		for (int j = 0; j < i; j++) {
			distances[i][j] = netTopology.getDistance(nodes[i], nodes[j]);
			distances[j][i] = distances[i][j];
		}
		distances[i][i] = 0;
	}

	return distances;
}
 
Example #10
Source File: Mover.java    From big-c with Apache License 2.0 6 votes vote down vote up
boolean chooseTarget(DBlock db, Source source,
    List<StorageType> targetTypes, Matcher matcher) {
  final NetworkTopology cluster = dispatcher.getCluster(); 
  for (StorageType t : targetTypes) {
    for(StorageGroup target : storages.getTargetStorages(t)) {
      if (matcher.match(cluster, source.getDatanodeInfo(),
          target.getDatanodeInfo())) {
        final PendingMove pm = source.addPendingMove(db, target);
        if (pm != null) {
          dispatcher.executePendingMove(pm);
          return true;
        }
      }
    }
  }
  return false;
}
 
Example #11
Source File: CombineFileInputFormat.java    From hadoop with Apache License 2.0 6 votes vote down vote up
OneBlockInfo(Path path, long offset, long len, 
             String[] hosts, String[] topologyPaths) {
  this.onepath = path;
  this.offset = offset;
  this.hosts = hosts;
  this.length = len;
  assert (hosts.length == topologyPaths.length ||
          topologyPaths.length == 0);

  // if the file system does not have any rack information, then
  // use dummy rack location.
  if (topologyPaths.length == 0) {
    topologyPaths = new String[hosts.length];
    for (int i = 0; i < topologyPaths.length; i++) {
      topologyPaths[i] = (new NodeBase(hosts[i], 
                          NetworkTopology.DEFAULT_RACK)).toString();
    }
  }

  // The topology paths have the host name included as the last 
  // component. Strip it.
  this.racks = new String[topologyPaths.length];
  for (int i = 0; i < topologyPaths.length; i++) {
    this.racks[i] = (new NodeBase(topologyPaths[i])).getNetworkLocation();
  }
}
 
Example #12
Source File: TestReplicationPolicyWithNodeGroup.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Scan the targets list: all targets should be on different NodeGroups.
 * Return false if two targets are found on the same NodeGroup.
 */
private static boolean checkTargetsOnDifferentNodeGroup(
    DatanodeStorageInfo[] targets) {
  if(targets.length == 0)
    return true;
  Set<String> targetSet = new HashSet<String>();
  for(DatanodeStorageInfo storage:targets) {
    final DatanodeDescriptor node = storage.getDatanodeDescriptor();
    String nodeGroup = NetworkTopology.getLastHalf(node.getNetworkLocation());
    if(targetSet.contains(nodeGroup)) {
      return false;
    } else {
      targetSet.add(nodeGroup);
    }
  }
  return true;
}
 
Example #13
Source File: CombineFileInputFormat.java    From big-c with Apache License 2.0 6 votes vote down vote up
OneBlockInfo(Path path, long offset, long len, 
             String[] hosts, String[] topologyPaths) {
  this.onepath = path;
  this.offset = offset;
  this.hosts = hosts;
  this.length = len;
  assert (hosts.length == topologyPaths.length ||
          topologyPaths.length == 0);

  // if the file system does not have any rack information, then
  // use dummy rack location.
  if (topologyPaths.length == 0) {
    topologyPaths = new String[hosts.length];
    for (int i = 0; i < topologyPaths.length; i++) {
      topologyPaths[i] = (new NodeBase(hosts[i], 
                          NetworkTopology.DEFAULT_RACK)).toString();
    }
  }

  // The topology paths have the host name included as the last 
  // component. Strip it.
  this.racks = new String[topologyPaths.length];
  for (int i = 0; i < topologyPaths.length; i++) {
    this.racks[i] = (new NodeBase(topologyPaths[i])).getNetworkLocation();
  }
}
 
Example #14
Source File: CombineFileInputFormat.java    From hraven with Apache License 2.0 6 votes vote down vote up
OneBlockInfo(Path path, long offset, long len, 
             String[] hosts, String[] topologyPaths) {
  this.onepath = path;
  this.offset = offset;
  this.hosts = hosts;
  this.length = len;
  assert (hosts.length == topologyPaths.length ||
          topologyPaths.length == 0);

  // if the file system does not have any rack information, then
  // use dummy rack location.
  if (topologyPaths.length == 0) {
    topologyPaths = new String[hosts.length];
    for (int i = 0; i < topologyPaths.length; i++) {
      topologyPaths[i] = (new NodeBase(hosts[i], 
                          NetworkTopology.DEFAULT_RACK)).toString();
    }
  }

  // The topology paths have the host name included as the last 
  // component. Strip it.
  this.racks = new String[topologyPaths.length];
  for (int i = 0; i < topologyPaths.length; i++) {
    this.racks[i] = (new NodeBase(topologyPaths[i])).getNetworkLocation();
  }
}
 
Example #15
Source File: TestConfigurableBlockPlacement.java    From RDFS with Apache License 2.0 6 votes vote down vote up
private VerifiablePolicy initTest() throws Exception {
  VerifiablePolicy policy = new VerifiablePolicy();
  Configuration conf = new Configuration();
  TestClusterStats stats = new TestClusterStats();
  NetworkTopology clusterMap = new NetworkTopology();
  TestHostsReader hostsReader = new TestHostsReader();
  TestMapping dnsToSwitchMapping = new TestMapping();

  for (DatanodeDescriptor d: dataNodes) {
    clusterMap.add(d);
  }

  conf.setInt("dfs.replication.rackwindow", 2);
  conf.setInt("dfs.replication.machineWindow", 2);

  policy.initialize(conf, stats, clusterMap, hostsReader, dnsToSwitchMapping, null);
  return policy;
}
 
Example #16
Source File: DatanodeInfo.java    From RDFS with Apache License 2.0 6 votes vote down vote up
/** A formatted string for printing the status of the DataNode. */
public String dumpDatanode() {
  StringBuffer buffer = new StringBuffer();
  long c = getCapacity();
  long r = getRemaining();
  long u = getDfsUsed();
  buffer.append(name);
  if (!NetworkTopology.DEFAULT_RACK.equals(location)) {
    buffer.append(" "+location);
  }
  if (isDecommissioned()) {
    buffer.append(" DD");
  } else if (isDecommissionInProgress()) {
    buffer.append(" DP");
  } else {
    buffer.append(" IN");
  }
  buffer.append(" " + c + "(" + StringUtils.byteDesc(c)+")");
  buffer.append(" " + u + "(" + StringUtils.byteDesc(u)+")");
  buffer.append(" " + StringUtils.limitDecimalTo2(((1.0*u)/c)*100)+"%");
  buffer.append(" " + r + "(" + StringUtils.byteDesc(r)+")");
  buffer.append(" " + new Date(lastUpdate));
  return buffer.toString();
}
 
Example #17
Source File: TestConfigurableBlockPlacement.java    From RDFS with Apache License 2.0 6 votes vote down vote up
public void testChooseTargetWithDefaultRack() throws Exception {
  VerifiablePolicy policy = initTest();
  TestMapping mapping = (TestMapping) policy.dnsToSwitchMapping;
  mapping.assignDefaultRack = true;
  try {
    policy.hostsUpdated();
    fail("Did not throw : " + DefaultRackException.class);
  } catch (DefaultRackException e) {}

  // Verify there is no default rack.
  RackRingInfo info = policy.racksMap.get(NetworkTopology.DEFAULT_RACK);
  assertNull(info);

  HashMap<Node, Node> emptyMap = new HashMap<Node, Node>();
  List<DatanodeDescriptor> results = new ArrayList<DatanodeDescriptor>();
  for (int i = 0; i < dataNodes.length; i++) {
    policy.chooseTarget(3, dataNodes[i], emptyMap, 512, 4, results, true);
  }
}
 
Example #18
Source File: CombineFileInputFormat.java    From aliyun-maxcompute-data-collectors with Apache License 2.0 6 votes vote down vote up
OneBlockInfo(Path path, long offset, long len,
             String[] hosts, String[] topologyPaths) {
  this.onepath = path;
  this.offset = offset;
  this.hosts = hosts;
  this.length = len;
  assert (hosts.length == topologyPaths.length ||
          topologyPaths.length == 0);

  // if the file system does not have any rack information, then
  // use dummy rack location.
  if (topologyPaths.length == 0) {
    topologyPaths = new String[hosts.length];
    for (int i = 0; i < topologyPaths.length; i++) {
      topologyPaths[i] = (new NodeBase(hosts[i],
                          NetworkTopology.DEFAULT_RACK)).toString();
    }
  }

  // The topology paths have the host name included as the last
  // component. Strip it.
  this.racks = new String[topologyPaths.length];
  for (int i = 0; i < topologyPaths.length; i++) {
    this.racks[i] = (new NodeBase(topologyPaths[i])).getNetworkLocation();
  }
}
 
Example #19
Source File: TestReplicationPolicyWithNodeGroup.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Scan the targets list: all targets should be on different NodeGroups.
 * Return false if two targets are found on the same NodeGroup.
 */
private static boolean checkTargetsOnDifferentNodeGroup(
    DatanodeStorageInfo[] targets) {
  if(targets.length == 0)
    return true;
  Set<String> targetSet = new HashSet<String>();
  for(DatanodeStorageInfo storage:targets) {
    final DatanodeDescriptor node = storage.getDatanodeDescriptor();
    String nodeGroup = NetworkTopology.getLastHalf(node.getNetworkLocation());
    if(targetSet.contains(nodeGroup)) {
      return false;
    } else {
      targetSet.add(nodeGroup);
    }
  }
  return true;
}
 
Example #20
Source File: TestRackResolver.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testCaching() {
  Configuration conf = new Configuration();
  conf.setClass(
    CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
    MyResolver.class, DNSToSwitchMapping.class);
  RackResolver.init(conf);
  try {
    InetAddress iaddr = InetAddress.getByName("host1");
    MyResolver.resolvedHost1 = iaddr.getHostAddress();
  } catch (UnknownHostException e) {
    // Ignore if not found
  }
  Node node = RackResolver.resolve("host1");
  Assert.assertEquals("/rack1", node.getNetworkLocation());
  node = RackResolver.resolve("host1");
  Assert.assertEquals("/rack1", node.getNetworkLocation());
  node = RackResolver.resolve(invalidHost);
  Assert.assertEquals(NetworkTopology.DEFAULT_RACK, node.getNetworkLocation());
}
 
Example #21
Source File: Mover.java    From hadoop with Apache License 2.0 6 votes vote down vote up
boolean chooseTarget(DBlock db, Source source,
    List<StorageType> targetTypes, Matcher matcher) {
  final NetworkTopology cluster = dispatcher.getCluster(); 
  for (StorageType t : targetTypes) {
    for(StorageGroup target : storages.getTargetStorages(t)) {
      if (matcher.match(cluster, source.getDatanodeInfo(),
          target.getDatanodeInfo())) {
        final PendingMove pm = source.addPendingMove(db, target);
        if (pm != null) {
          dispatcher.executePendingMove(pm);
          return true;
        }
      }
    }
  }
  return false;
}
 
Example #22
Source File: FileInputFormat.java    From RDFS with Apache License 2.0 5 votes vote down vote up
private String[] fakeRacks(BlockLocation[] blkLocations, int index) 
throws IOException {
  String[] allHosts = blkLocations[index].getHosts();
  String[] allTopos = new String[allHosts.length];
  for (int i = 0; i < allHosts.length; i++) {
    allTopos[i] = NetworkTopology.DEFAULT_RACK + "/" + allHosts[i];
  }
  return allTopos;
}
 
Example #23
Source File: BlockPlacementPolicy.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Get an instance of the configured Block Placement Policy based on the
 * the configuration property
 * {@link  DFSConfigKeys#DFS_BLOCK_REPLICATOR_CLASSNAME_KEY}.
 * 
 * @param conf the configuration to be used
 * @param stats an object that is used to retrieve the load on the cluster
 * @param clusterMap the network topology of the cluster
 * @return an instance of BlockPlacementPolicy
 */
public static BlockPlacementPolicy getInstance(Configuration conf, 
                                               FSClusterStats stats,
                                               NetworkTopology clusterMap,
                                               Host2NodesMap host2datanodeMap) {
  final Class<? extends BlockPlacementPolicy> replicatorClass = conf.getClass(
      DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_KEY,
      DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_DEFAULT,
      BlockPlacementPolicy.class);
  final BlockPlacementPolicy replicator = ReflectionUtils.newInstance(
      replicatorClass, conf);
  replicator.initialize(conf, stats, clusterMap, host2datanodeMap);
  return replicator;
}
 
Example #24
Source File: NNThroughputBenchmark.java    From big-c with Apache License 2.0 5 votes vote down vote up
static void setNameNodeLoggingLevel(Level logLevel) {
  LOG.fatal("Log level = " + logLevel.toString());
  // change log level to NameNode logs
  DFSTestUtil.setNameNodeLogLevel(logLevel);
  GenericTestUtils.setLogLevel(LogManager.getLogger(
          NetworkTopology.class.getName()), logLevel);
  GenericTestUtils.setLogLevel(LogManager.getLogger(
          Groups.class.getName()), logLevel);
}
 
Example #25
Source File: TestBalancerWithNodeGroup.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private Set<ExtendedBlock> getBlocksOnRack(List<LocatedBlock> blks, String rack) {
  Set<ExtendedBlock> ret = new HashSet<ExtendedBlock>();
  for (LocatedBlock blk : blks) {
    for (DatanodeInfo di : blk.getLocations()) {
      if (rack.equals(NetworkTopology.getFirstHalf(di.getNetworkLocation()))) {
        ret.add(blk.getBlock());
        break;
      }
    }
  }
  return ret;
}
 
Example #26
Source File: FileInputFormat.java    From big-c with Apache License 2.0 5 votes vote down vote up
private String[] fakeRacks(BlockLocation[] blkLocations, int index) 
throws IOException {
  String[] allHosts = blkLocations[index].getHosts();
  String[] allTopos = new String[allHosts.length];
  for (int i = 0; i < allHosts.length; i++) {
    allTopos[i] = NetworkTopology.DEFAULT_RACK + "/" + allHosts[i];
  }
  return allTopos;
}
 
Example #27
Source File: BlockPlacementPolicyFakeData.java    From RDFS with Apache License 2.0 5 votes vote down vote up
public BlockPlacementPolicyFakeData(Configuration conf,  FSClusterStats stats,
                         NetworkTopology clusterMap) {
  initialize(conf, stats, clusterMap, null, null, null);
  
  FSNamesystem.LOG.info("new BlockPlacementPolicyFakeData: " + this);
  lastInstance = this;
}
 
Example #28
Source File: DatanodeInfo.java    From big-c with Apache License 2.0 5 votes vote down vote up
/** A formatted string for printing the status of the DataNode. */
public String dumpDatanode() {
  StringBuilder buffer = new StringBuilder();
  long c = getCapacity();
  long r = getRemaining();
  long u = getDfsUsed();
  long cc = getCacheCapacity();
  long cr = getCacheRemaining();
  long cu = getCacheUsed();
  buffer.append(getName());
  if (!NetworkTopology.DEFAULT_RACK.equals(location)) {
    buffer.append(" "+location);
  }
  if (isDecommissioned()) {
    buffer.append(" DD");
  } else if (isDecommissionInProgress()) {
    buffer.append(" DP");
  } else {
    buffer.append(" IN");
  }
  buffer.append(" " + c + "(" + StringUtils.byteDesc(c)+")");
  buffer.append(" " + u + "(" + StringUtils.byteDesc(u)+")");
  buffer.append(" " + percent2String(u/(double)c));
  buffer.append(" " + r + "(" + StringUtils.byteDesc(r)+")");
  buffer.append(" " + cc + "(" + StringUtils.byteDesc(cc)+")");
  buffer.append(" " + cu + "(" + StringUtils.byteDesc(cu)+")");
  buffer.append(" " + percent2String(cu/(double)cc));
  buffer.append(" " + cr + "(" + StringUtils.byteDesc(cr)+")");
  buffer.append(" " + new Date(lastUpdate));
  return buffer.toString();
}
 
Example #29
Source File: TestBalancerWithNodeGroup.java    From big-c with Apache License 2.0 5 votes vote down vote up
private Set<ExtendedBlock> getBlocksOnRack(List<LocatedBlock> blks, String rack) {
  Set<ExtendedBlock> ret = new HashSet<ExtendedBlock>();
  for (LocatedBlock blk : blks) {
    for (DatanodeInfo di : blk.getLocations()) {
      if (rack.equals(NetworkTopology.getFirstHalf(di.getNetworkLocation()))) {
        ret.add(blk.getBlock());
        break;
      }
    }
  }
  return ret;
}
 
Example #30
Source File: DatanodeInfo.java    From hadoop-gpu with Apache License 2.0 5 votes vote down vote up
/** A formatted string for reporting the status of the DataNode. */
public String getDatanodeReport() {
  StringBuffer buffer = new StringBuffer();
  long c = getCapacity();
  long r = getRemaining();
  long u = getDfsUsed();
  long nonDFSUsed = getNonDfsUsed();
  float usedPercent = getDfsUsedPercent();
  float remainingPercent = getRemainingPercent();

  buffer.append("Name: "+name+"\n");
  if (!NetworkTopology.DEFAULT_RACK.equals(location)) {
    buffer.append("Rack: "+location+"\n");
  }
  buffer.append("Decommission Status : ");
  if (isDecommissioned()) {
    buffer.append("Decommissioned\n");
  } else if (isDecommissionInProgress()) {
    buffer.append("Decommission in progress\n");
  } else {
    buffer.append("Normal\n");
  }
  buffer.append("Configured Capacity: "+c+" ("+StringUtils.byteDesc(c)+")"+"\n");
  buffer.append("DFS Used: "+u+" ("+StringUtils.byteDesc(u)+")"+"\n");
  buffer.append("Non DFS Used: "+nonDFSUsed+" ("+StringUtils.byteDesc(nonDFSUsed)+")"+"\n");
  buffer.append("DFS Remaining: " +r+ "("+StringUtils.byteDesc(r)+")"+"\n");
  buffer.append("DFS Used%: "+StringUtils.limitDecimalTo2(usedPercent)+"%\n");
  buffer.append("DFS Remaining%: "+StringUtils.limitDecimalTo2(remainingPercent)+"%\n");
  buffer.append("Last contact: "+new Date(lastUpdate)+"\n");
  return buffer.toString();
}