org.apache.hadoop.net.NodeBase Java Examples
The following examples show how to use
org.apache.hadoop.net.NodeBase.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: CombineFileInputFormat.java From big-c with Apache License 2.0 | 6 votes |
OneBlockInfo(Path path, long offset, long len, String[] hosts, String[] topologyPaths) { this.onepath = path; this.offset = offset; this.hosts = hosts; this.length = len; assert (hosts.length == topologyPaths.length || topologyPaths.length == 0); // if the file system does not have any rack information, then // use dummy rack location. if (topologyPaths.length == 0) { topologyPaths = new String[hosts.length]; for (int i = 0; i < topologyPaths.length; i++) { topologyPaths[i] = (new NodeBase(hosts[i], NetworkTopology.DEFAULT_RACK)).toString(); } } // The topology paths have the host name included as the last // component. Strip it. this.racks = new String[topologyPaths.length]; for (int i = 0; i < topologyPaths.length; i++) { this.racks[i] = (new NodeBase(topologyPaths[i])).getNetworkLocation(); } }
Example #2
Source File: CombineFileInputFormat.java From RDFS with Apache License 2.0 | 6 votes |
OneBlockInfo(Path path, long offset, long len, String[] hosts, String[] topologyPaths) { this.onepath = path; this.offset = offset; this.hosts = hosts; this.length = len; assert (hosts.length == topologyPaths.length || topologyPaths.length == 0); // if the file ystem does not have any rack information, then // use dummy rack location. if (topologyPaths.length == 0) { topologyPaths = new String[hosts.length]; for (int i = 0; i < topologyPaths.length; i++) { topologyPaths[i] = (new NodeBase(hosts[i], NetworkTopology.DEFAULT_RACK)). toString(); } } // The topology paths have the host name included as the last // component. Strip it. this.racks = new String[topologyPaths.length]; for (int i = 0; i < topologyPaths.length; i++) { this.racks[i] = (new NodeBase(topologyPaths[i])).getNetworkLocation(); } }
Example #3
Source File: HdfsPartitioner.java From sqoop-on-spark with Apache License 2.0 | 6 votes |
OneBlockInfo(Path path, long offset, long len, String[] hosts, String[] topologyPaths) { this.onepath = path; this.offset = offset; this.hosts = hosts; this.length = len; assert (hosts.length == topologyPaths.length || topologyPaths.length == 0); // if the file system does not have any rack information, then // use dummy rack location. if (topologyPaths.length == 0) { topologyPaths = new String[hosts.length]; for (int i = 0; i < topologyPaths.length; i++) { topologyPaths[i] = (new NodeBase(hosts[i], NetworkTopology.DEFAULT_RACK)).toString(); } } // The topology paths have the host name included as the last // component. Strip it. this.racks = new String[topologyPaths.length]; for (int i = 0; i < topologyPaths.length; i++) { this.racks[i] = (new NodeBase(topologyPaths[i])).getNetworkLocation(); } }
Example #4
Source File: TopologyCache.java From RDFS with Apache License 2.0 | 6 votes |
private Node resolveAndGetNode(String name) { List <String> rNameList = dnsToSwitchMapping.resolve(Arrays.asList(new String [] {name})); String networkLoc = NodeBase.normalize(rNameList.get(0)); Node node = null; // we depend on clusterMap to get a canonical node object // we synchronize this section to guarantee that two concurrent // insertions into the clusterMap don't happen (resulting in // multiple copies of the same node being created and returned) synchronized (clusterMap) { while ((node = clusterMap.getNode(networkLoc+"/"+name)) == null) { clusterMap.add(new NodeBase(name, networkLoc)); } } return node; }
Example #5
Source File: BlockPlacementPolicyDefault.java From big-c with Apache License 2.0 | 6 votes |
private DatanodeStorageInfo chooseFromNextRack(Node next, Set<Node> excludedNodes, long blocksize, int maxNodesPerRack, List<DatanodeStorageInfo> results, boolean avoidStaleNodes, EnumMap<StorageType, Integer> storageTypes) throws NotEnoughReplicasException { final String nextRack = next.getNetworkLocation(); try { return chooseRandom(nextRack, excludedNodes, blocksize, maxNodesPerRack, results, avoidStaleNodes, storageTypes); } catch(NotEnoughReplicasException e) { if (LOG.isDebugEnabled()) { LOG.debug("Failed to choose from the next rack (location = " + nextRack + "), retry choosing ramdomly", e); } //otherwise randomly choose one from the network return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize, maxNodesPerRack, results, avoidStaleNodes, storageTypes); } }
Example #6
Source File: BlockPlacementPolicyDefault.java From RDFS with Apache License 2.0 | 6 votes |
protected DatanodeDescriptor chooseLocalNode( DatanodeDescriptor localMachine, HashMap<Node, Node> excludedNodes, long blocksize, int maxNodesPerRack, List<DatanodeDescriptor> results) throws NotEnoughReplicasException { // if no local machine, randomly choose one node if (localMachine == null) return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize, maxNodesPerRack, results); // otherwise try local machine first Node oldNode = excludedNodes.put(localMachine, localMachine); if (oldNode == null) { // was not in the excluded list if (isGoodTarget(localMachine, blocksize, maxNodesPerRack, false, results)) { results.add(localMachine); return localMachine; } } // try a node on local rack return chooseLocalRack(localMachine, excludedNodes, blocksize, maxNodesPerRack, results); }
Example #7
Source File: CombineFileInputFormat.java From hraven with Apache License 2.0 | 6 votes |
OneBlockInfo(Path path, long offset, long len, String[] hosts, String[] topologyPaths) { this.onepath = path; this.offset = offset; this.hosts = hosts; this.length = len; assert (hosts.length == topologyPaths.length || topologyPaths.length == 0); // if the file system does not have any rack information, then // use dummy rack location. if (topologyPaths.length == 0) { topologyPaths = new String[hosts.length]; for (int i = 0; i < topologyPaths.length; i++) { topologyPaths[i] = (new NodeBase(hosts[i], NetworkTopology.DEFAULT_RACK)).toString(); } } // The topology paths have the host name included as the last // component. Strip it. this.racks = new String[topologyPaths.length]; for (int i = 0; i < topologyPaths.length; i++) { this.racks[i] = (new NodeBase(topologyPaths[i])).getNetworkLocation(); } }
Example #8
Source File: CombineFileInputFormat.java From hadoop-gpu with Apache License 2.0 | 6 votes |
OneBlockInfo(Path path, long offset, long len, String[] hosts, String[] topologyPaths) { this.onepath = path; this.offset = offset; this.hosts = hosts; this.length = len; assert (hosts.length == topologyPaths.length || topologyPaths.length == 0); // if the file ystem does not have any rack information, then // use dummy rack location. if (topologyPaths.length == 0) { topologyPaths = new String[hosts.length]; for (int i = 0; i < topologyPaths.length; i++) { topologyPaths[i] = (new NodeBase(hosts[i], NetworkTopology.DEFAULT_RACK)). toString(); } } // The topology paths have the host name included as the last // component. Strip it. this.racks = new String[topologyPaths.length]; for (int i = 0; i < topologyPaths.length; i++) { this.racks[i] = (new NodeBase(topologyPaths[i])).getNetworkLocation(); } }
Example #9
Source File: BlockPlacementPolicyDefault.java From hadoop with Apache License 2.0 | 6 votes |
private DatanodeStorageInfo chooseFromNextRack(Node next, Set<Node> excludedNodes, long blocksize, int maxNodesPerRack, List<DatanodeStorageInfo> results, boolean avoidStaleNodes, EnumMap<StorageType, Integer> storageTypes) throws NotEnoughReplicasException { final String nextRack = next.getNetworkLocation(); try { return chooseRandom(nextRack, excludedNodes, blocksize, maxNodesPerRack, results, avoidStaleNodes, storageTypes); } catch(NotEnoughReplicasException e) { if (LOG.isDebugEnabled()) { LOG.debug("Failed to choose from the next rack (location = " + nextRack + "), retry choosing ramdomly", e); } //otherwise randomly choose one from the network return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize, maxNodesPerRack, results, avoidStaleNodes, storageTypes); } }
Example #10
Source File: CombineFileInputFormat.java From hadoop with Apache License 2.0 | 6 votes |
OneBlockInfo(Path path, long offset, long len, String[] hosts, String[] topologyPaths) { this.onepath = path; this.offset = offset; this.hosts = hosts; this.length = len; assert (hosts.length == topologyPaths.length || topologyPaths.length == 0); // if the file system does not have any rack information, then // use dummy rack location. if (topologyPaths.length == 0) { topologyPaths = new String[hosts.length]; for (int i = 0; i < topologyPaths.length; i++) { topologyPaths[i] = (new NodeBase(hosts[i], NetworkTopology.DEFAULT_RACK)).toString(); } } // The topology paths have the host name included as the last // component. Strip it. this.racks = new String[topologyPaths.length]; for (int i = 0; i < topologyPaths.length; i++) { this.racks[i] = (new NodeBase(topologyPaths[i])).getNetworkLocation(); } }
Example #11
Source File: CombineFileInputFormat.java From aliyun-maxcompute-data-collectors with Apache License 2.0 | 6 votes |
OneBlockInfo(Path path, long offset, long len, String[] hosts, String[] topologyPaths) { this.onepath = path; this.offset = offset; this.hosts = hosts; this.length = len; assert (hosts.length == topologyPaths.length || topologyPaths.length == 0); // if the file system does not have any rack information, then // use dummy rack location. if (topologyPaths.length == 0) { topologyPaths = new String[hosts.length]; for (int i = 0; i < topologyPaths.length; i++) { topologyPaths[i] = (new NodeBase(hosts[i], NetworkTopology.DEFAULT_RACK)).toString(); } } // The topology paths have the host name included as the last // component. Strip it. this.racks = new String[topologyPaths.length]; for (int i = 0; i < topologyPaths.length; i++) { this.racks[i] = (new NodeBase(topologyPaths[i])).getNetworkLocation(); } }
Example #12
Source File: ReplicationTargetChooser.java From hadoop-gpu with Apache License 2.0 | 6 votes |
private DatanodeDescriptor chooseLocalNode( DatanodeDescriptor localMachine, List<Node> excludedNodes, long blocksize, int maxNodesPerRack, List<DatanodeDescriptor> results) throws NotEnoughReplicasException { // if no local machine, randomly choose one node if (localMachine == null) return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize, maxNodesPerRack, results); // otherwise try local machine first if (!excludedNodes.contains(localMachine)) { excludedNodes.add(localMachine); if (isGoodTarget(localMachine, blocksize, maxNodesPerRack, false, results)) { results.add(localMachine); return localMachine; } } // try a node on local rack return chooseLocalRack(localMachine, excludedNodes, blocksize, maxNodesPerRack, results); }
Example #13
Source File: RackResolver.java From hadoop with Apache License 2.0 | 5 votes |
private static Node coreResolve(String hostName) { List <String> tmpList = new ArrayList<String>(1); tmpList.add(hostName); List <String> rNameList = dnsToSwitchMapping.resolve(tmpList); String rName = null; if (rNameList == null || rNameList.get(0) == null) { rName = NetworkTopology.DEFAULT_RACK; LOG.info("Couldn't resolve " + hostName + ". Falling back to " + NetworkTopology.DEFAULT_RACK); } else { rName = rNameList.get(0); LOG.info("Resolved " + hostName + " to " + rName); } return new NodeBase(hostName, rName); }
Example #14
Source File: OfferLifeCycleManagerTest.java From incubator-myriad with Apache License 2.0 | 5 votes |
@Before public void setUp() throws Exception { NodeStore store = new NodeStore(); NodeIdProto nodeId = NodeIdProto.newBuilder().setHost("localhost").setPort(8000).build(); RMNode rmNode = new RMNodeImpl(new NodeIdPBImpl(nodeId), new MockRMContext(), "localhost", 8000, 8070, new NodeBase(), new ResourcePBImpl(), "1.0"); SchedulerNode node = new FiCaSchedulerNode(rmNode, false); store.add(node); manager = new OfferLifecycleManager(store, new MyriadDriver(new MockSchedulerDriver())); }
Example #15
Source File: JobTracker.java From RDFS with Apache License 2.0 | 5 votes |
public Node resolveAndAddToTopology(String name) { List <String> tmpList = new ArrayList<String>(1); tmpList.add(name); List <String> rNameList = dnsToSwitchMapping.resolve(tmpList); String rName = rNameList.get(0); String networkLoc = NodeBase.normalize(rName); return addHostToNodeMapping(name, networkLoc); }
Example #16
Source File: BlockPlacementPolicyWithNodeGroup.java From big-c with Apache License 2.0 | 5 votes |
private DatanodeStorageInfo chooseLocalNodeGroup( NetworkTopologyWithNodeGroup clusterMap, Node localMachine, Set<Node> excludedNodes, long blocksize, int maxNodesPerRack, List<DatanodeStorageInfo> results, boolean avoidStaleNodes, EnumMap<StorageType, Integer> storageTypes) throws NotEnoughReplicasException { // no local machine, so choose a random machine if (localMachine == null) { return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize, maxNodesPerRack, results, avoidStaleNodes, storageTypes); } // choose one from the local node group try { return chooseRandom( clusterMap.getNodeGroup(localMachine.getNetworkLocation()), excludedNodes, blocksize, maxNodesPerRack, results, avoidStaleNodes, storageTypes); } catch (NotEnoughReplicasException e1) { final DatanodeDescriptor newLocal = secondNode(localMachine, results); if (newLocal != null) { try { return chooseRandom( clusterMap.getNodeGroup(newLocal.getNetworkLocation()), excludedNodes, blocksize, maxNodesPerRack, results, avoidStaleNodes, storageTypes); } catch(NotEnoughReplicasException e2) { //otherwise randomly choose one from the network return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize, maxNodesPerRack, results, avoidStaleNodes, storageTypes); } } else { //otherwise randomly choose one from the network return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize, maxNodesPerRack, results, avoidStaleNodes, storageTypes); } } }
Example #17
Source File: BlockPlacementPolicyWithNodeGroup.java From big-c with Apache License 2.0 | 5 votes |
@Override protected DatanodeStorageInfo chooseLocalRack(Node localMachine, Set<Node> excludedNodes, long blocksize, int maxNodesPerRack, List<DatanodeStorageInfo> results, boolean avoidStaleNodes, EnumMap<StorageType, Integer> storageTypes) throws NotEnoughReplicasException { // no local machine, so choose a random machine if (localMachine == null) { return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize, maxNodesPerRack, results, avoidStaleNodes, storageTypes); } // choose one from the local rack, but off-nodegroup try { final String scope = NetworkTopology.getFirstHalf(localMachine.getNetworkLocation()); return chooseRandom(scope, excludedNodes, blocksize, maxNodesPerRack, results, avoidStaleNodes, storageTypes); } catch (NotEnoughReplicasException e1) { // find the second replica final DatanodeDescriptor newLocal = secondNode(localMachine, results); if (newLocal != null) { try { return chooseRandom( clusterMap.getRack(newLocal.getNetworkLocation()), excludedNodes, blocksize, maxNodesPerRack, results, avoidStaleNodes, storageTypes); } catch(NotEnoughReplicasException e2) { //otherwise randomly choose one from the network return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize, maxNodesPerRack, results, avoidStaleNodes, storageTypes); } } else { //otherwise randomly choose one from the network return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize, maxNodesPerRack, results, avoidStaleNodes, storageTypes); } } }
Example #18
Source File: RackResolver.java From big-c with Apache License 2.0 | 5 votes |
private static Node coreResolve(String hostName) { List <String> tmpList = new ArrayList<String>(1); tmpList.add(hostName); List <String> rNameList = dnsToSwitchMapping.resolve(tmpList); String rName = null; if (rNameList == null || rNameList.get(0) == null) { rName = NetworkTopology.DEFAULT_RACK; LOG.info("Couldn't resolve " + hostName + ". Falling back to " + NetworkTopology.DEFAULT_RACK); } else { rName = rNameList.get(0); LOG.info("Resolved " + hostName + " to " + rName); } return new NodeBase(hostName, rName); }
Example #19
Source File: BlockPlacementPolicyWithNodeGroup.java From hadoop with Apache License 2.0 | 5 votes |
private DatanodeStorageInfo chooseLocalNodeGroup( NetworkTopologyWithNodeGroup clusterMap, Node localMachine, Set<Node> excludedNodes, long blocksize, int maxNodesPerRack, List<DatanodeStorageInfo> results, boolean avoidStaleNodes, EnumMap<StorageType, Integer> storageTypes) throws NotEnoughReplicasException { // no local machine, so choose a random machine if (localMachine == null) { return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize, maxNodesPerRack, results, avoidStaleNodes, storageTypes); } // choose one from the local node group try { return chooseRandom( clusterMap.getNodeGroup(localMachine.getNetworkLocation()), excludedNodes, blocksize, maxNodesPerRack, results, avoidStaleNodes, storageTypes); } catch (NotEnoughReplicasException e1) { final DatanodeDescriptor newLocal = secondNode(localMachine, results); if (newLocal != null) { try { return chooseRandom( clusterMap.getNodeGroup(newLocal.getNetworkLocation()), excludedNodes, blocksize, maxNodesPerRack, results, avoidStaleNodes, storageTypes); } catch(NotEnoughReplicasException e2) { //otherwise randomly choose one from the network return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize, maxNodesPerRack, results, avoidStaleNodes, storageTypes); } } else { //otherwise randomly choose one from the network return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize, maxNodesPerRack, results, avoidStaleNodes, storageTypes); } } }
Example #20
Source File: BlockPlacementPolicyWithNodeGroup.java From hadoop with Apache License 2.0 | 5 votes |
@Override protected DatanodeStorageInfo chooseLocalRack(Node localMachine, Set<Node> excludedNodes, long blocksize, int maxNodesPerRack, List<DatanodeStorageInfo> results, boolean avoidStaleNodes, EnumMap<StorageType, Integer> storageTypes) throws NotEnoughReplicasException { // no local machine, so choose a random machine if (localMachine == null) { return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize, maxNodesPerRack, results, avoidStaleNodes, storageTypes); } // choose one from the local rack, but off-nodegroup try { final String scope = NetworkTopology.getFirstHalf(localMachine.getNetworkLocation()); return chooseRandom(scope, excludedNodes, blocksize, maxNodesPerRack, results, avoidStaleNodes, storageTypes); } catch (NotEnoughReplicasException e1) { // find the second replica final DatanodeDescriptor newLocal = secondNode(localMachine, results); if (newLocal != null) { try { return chooseRandom( clusterMap.getRack(newLocal.getNetworkLocation()), excludedNodes, blocksize, maxNodesPerRack, results, avoidStaleNodes, storageTypes); } catch(NotEnoughReplicasException e2) { //otherwise randomly choose one from the network return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize, maxNodesPerRack, results, avoidStaleNodes, storageTypes); } } else { //otherwise randomly choose one from the network return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize, maxNodesPerRack, results, avoidStaleNodes, storageTypes); } } }
Example #21
Source File: JobTracker.java From hadoop-gpu with Apache License 2.0 | 5 votes |
public Node resolveAndAddToTopology(String name) { List <String> tmpList = new ArrayList<String>(1); tmpList.add(name); List <String> rNameList = dnsToSwitchMapping.resolve(tmpList); String rName = rNameList.get(0); String networkLoc = NodeBase.normalize(rName); return addHostToNodeMapping(name, networkLoc); }
Example #22
Source File: BlockPlacementPolicyDefault.java From RDFS with Apache License 2.0 | 4 votes |
protected DatanodeDescriptor chooseLocalRack( DatanodeDescriptor localMachine, HashMap<Node, Node> excludedNodes, long blocksize, int maxNodesPerRack, List<DatanodeDescriptor> results) throws NotEnoughReplicasException { // no local machine, so choose a random machine if (localMachine == null) { return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize, maxNodesPerRack, results); } // choose one from the local rack try { return chooseRandom( localMachine.getNetworkLocation(), excludedNodes, blocksize, maxNodesPerRack, results); } catch (NotEnoughReplicasException e1) { // find the second replica DatanodeDescriptor newLocal=null; for(Iterator<DatanodeDescriptor> iter=results.iterator(); iter.hasNext();) { DatanodeDescriptor nextNode = iter.next(); if (nextNode != localMachine) { newLocal = nextNode; break; } } if (newLocal != null) { try { return chooseRandom( newLocal.getNetworkLocation(), excludedNodes, blocksize, maxNodesPerRack, results); } catch(NotEnoughReplicasException e2) { //otherwise randomly choose one from the network return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize, maxNodesPerRack, results); } } else { //otherwise randomly choose one from the network return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize, maxNodesPerRack, results); } } }
Example #23
Source File: BlockPlacementPolicyDefault.java From RDFS with Apache License 2.0 | 4 votes |
protected boolean isGoodTarget(DatanodeDescriptor node, long blockSize, int maxTargetPerLoc, boolean considerLoad, List<DatanodeDescriptor> results) { Log logr = FSNamesystem.LOG; // check if the node is (being) decommissed if (node.isDecommissionInProgress() || node.isDecommissioned()) { if (logr.isDebugEnabled()) { logr.debug("Node "+ NodeBase.getPath(node) + " is not chosen because the node is (being) decommissioned"); } return false; } long remaining = node.getRemaining() - (node.getBlocksScheduled() * blockSize); // check the remaining capacity of the target machine if (blockSize* FSConstants.MIN_BLOCKS_FOR_WRITE>remaining) { if (logr.isDebugEnabled()) { logr.debug("Node "+ NodeBase.getPath(node) + " is not chosen because the node does not have enough space" + " for block size " + blockSize + " with Remaining = " + node.getRemaining() + " and Scheduled = " + node.getBlocksScheduled()); } return false; } // check the communication traffic of the target machine if (considerLoad) { double avgLoad = 0; int size = clusterMap.getNumOfLeaves(); if (size != 0 && stats != null) { avgLoad = (double)stats.getTotalLoad()/size; } if (node.getXceiverCount() > (2.0 * avgLoad)) { if (logr.isDebugEnabled()) { logr.debug("Node "+NodeBase.getPath(node)+ " is not chosen because the node is too busy"); } return false; } } // check if the target rack has chosen too many nodes String rackname = node.getNetworkLocation(); int counter=1; for(Iterator<DatanodeDescriptor> iter = results.iterator(); iter.hasNext();) { Node result = iter.next(); if (rackname.equals(result.getNetworkLocation())) { counter++; } } if (counter>maxTargetPerLoc) { if (logr.isDebugEnabled()) { logr.debug("Node "+NodeBase.getPath(node)+ " is not chosen because the rack has too many chosen nodes"); } return false; } return true; }
Example #24
Source File: BlockPlacementPolicyDefault.java From RDFS with Apache License 2.0 | 4 votes |
protected DatanodeDescriptor chooseTarget(int numOfReplicas, DatanodeDescriptor writer, HashMap<Node, Node> excludedNodes, long blocksize, int maxNodesPerRack, List<DatanodeDescriptor> results, boolean newBlock) { if (numOfReplicas == 0 || clusterMap.getNumOfLeaves()==0) { return writer; } int numOfResults = results.size(); if (writer == null && !newBlock) { writer = results.get(0); } try { if (numOfResults == 0) { chooseLocalNode(writer, excludedNodes, blocksize, maxNodesPerRack, results); if (newBlock && writer == null) { writer = results.get(0); } if (--numOfReplicas == 0) { return writer; } } if (numOfResults <= 1) { choose2ndRack(writer, excludedNodes, blocksize, maxNodesPerRack, results); if (--numOfReplicas == 0) { return writer; } } if (numOfResults <= 2) { if (clusterMap.isOnSameRack(results.get(0), results.get(1))) { choose2ndRack(writer, excludedNodes, blocksize, maxNodesPerRack, results); } else if (newBlock){ chooseLocalRack(results.get(1), excludedNodes, blocksize, maxNodesPerRack, results); } else { chooseLocalRack(writer, excludedNodes, blocksize, maxNodesPerRack, results); } if (--numOfReplicas == 0) { return writer; } } chooseRandom(numOfReplicas, NodeBase.ROOT, excludedNodes, blocksize, maxNodesPerRack, results); } catch (NotEnoughReplicasException e) { FSNamesystem.LOG.warn("Not able to place enough replicas, still in need of " + numOfReplicas); } return writer; }
Example #25
Source File: ReplicationTargetChooser.java From hadoop-gpu with Apache License 2.0 | 4 votes |
private DatanodeDescriptor chooseLocalRack( DatanodeDescriptor localMachine, List<Node> excludedNodes, long blocksize, int maxNodesPerRack, List<DatanodeDescriptor> results) throws NotEnoughReplicasException { // no local machine, so choose a random machine if (localMachine == null) { return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize, maxNodesPerRack, results); } // choose one from the local rack try { return chooseRandom( localMachine.getNetworkLocation(), excludedNodes, blocksize, maxNodesPerRack, results); } catch (NotEnoughReplicasException e1) { // find the second replica DatanodeDescriptor newLocal=null; for(Iterator<DatanodeDescriptor> iter=results.iterator(); iter.hasNext();) { DatanodeDescriptor nextNode = iter.next(); if (nextNode != localMachine) { newLocal = nextNode; break; } } if (newLocal != null) { try { return chooseRandom( newLocal.getNetworkLocation(), excludedNodes, blocksize, maxNodesPerRack, results); } catch(NotEnoughReplicasException e2) { //otherwise randomly choose one from the network return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize, maxNodesPerRack, results); } } else { //otherwise randomly choose one from the network return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize, maxNodesPerRack, results); } } }
Example #26
Source File: DatanodeInfo.java From RDFS with Apache License 2.0 | 4 votes |
/** Sets the rack name */ public synchronized void setNetworkLocation(String location) { this.location = NodeBase.normalize(location); }
Example #27
Source File: ReplicationTargetChooser.java From hadoop-gpu with Apache License 2.0 | 4 votes |
private DatanodeDescriptor chooseTarget(int numOfReplicas, DatanodeDescriptor writer, List<Node> excludedNodes, long blocksize, int maxNodesPerRack, List<DatanodeDescriptor> results) { if (numOfReplicas == 0 || clusterMap.getNumOfLeaves()==0) { return writer; } int numOfResults = results.size(); boolean newBlock = (numOfResults==0); if (writer == null && !newBlock) { writer = (DatanodeDescriptor)results.get(0); } try { switch(numOfResults) { case 0: writer = chooseLocalNode(writer, excludedNodes, blocksize, maxNodesPerRack, results); if (--numOfReplicas == 0) { break; } case 1: chooseRemoteRack(1, results.get(0), excludedNodes, blocksize, maxNodesPerRack, results); if (--numOfReplicas == 0) { break; } case 2: if (clusterMap.isOnSameRack(results.get(0), results.get(1))) { chooseRemoteRack(1, results.get(0), excludedNodes, blocksize, maxNodesPerRack, results); } else if (newBlock){ chooseLocalRack(results.get(1), excludedNodes, blocksize, maxNodesPerRack, results); } else { chooseLocalRack(writer, excludedNodes, blocksize, maxNodesPerRack, results); } if (--numOfReplicas == 0) { break; } default: chooseRandom(numOfReplicas, NodeBase.ROOT, excludedNodes, blocksize, maxNodesPerRack, results); } } catch (NotEnoughReplicasException e) { FSNamesystem.LOG.warn("Not able to place enough replicas, still in need of " + numOfReplicas); } return writer; }
Example #28
Source File: ReplicationTargetChooser.java From hadoop-gpu with Apache License 2.0 | 4 votes |
private boolean isGoodTarget(DatanodeDescriptor node, long blockSize, int maxTargetPerLoc, boolean considerLoad, List<DatanodeDescriptor> results) { Log logr = FSNamesystem.LOG; // check if the node is (being) decommissed if (node.isDecommissionInProgress() || node.isDecommissioned()) { logr.debug("Node "+NodeBase.getPath(node)+ " is not chosen because the node is (being) decommissioned"); return false; } long remaining = node.getRemaining() - (node.getBlocksScheduled() * blockSize); // check the remaining capacity of the target machine if (blockSize* FSConstants.MIN_BLOCKS_FOR_WRITE>remaining) { logr.debug("Node "+NodeBase.getPath(node)+ " is not chosen because the node does not have enough space"); return false; } // check the communication traffic of the target machine if (considerLoad) { double avgLoad = 0; int size = clusterMap.getNumOfLeaves(); if (size != 0) { avgLoad = (double)fs.getTotalLoad()/size; } if (node.getXceiverCount() > (2.0 * avgLoad)) { logr.debug("Node "+NodeBase.getPath(node)+ " is not chosen because the node is too busy"); return false; } } // check if the target rack has chosen too many nodes String rackname = node.getNetworkLocation(); int counter=1; for(Iterator<DatanodeDescriptor> iter = results.iterator(); iter.hasNext();) { Node result = iter.next(); if (rackname.equals(result.getNetworkLocation())) { counter++; } } if (counter>maxTargetPerLoc) { logr.debug("Node "+NodeBase.getPath(node)+ " is not chosen because the rack has too many chosen nodes"); return false; } return true; }
Example #29
Source File: DatanodeInfo.java From hadoop-gpu with Apache License 2.0 | 4 votes |
/** Sets the rack name */ public synchronized void setNetworkLocation(String location) { this.location = NodeBase.normalize(location); }
Example #30
Source File: TestObjectFactory.java From incubator-myriad with Apache License 2.0 | 4 votes |
public static RMNode getRMNode(String host, int port, Resource resource) { NodeId id = NodeId.newInstance(host, port); RMContext context = new MockRMContext(); return new RMNodeImpl(id, context, id.getHost(), id.getPort(), id.getPort(), new NodeBase(host, "/tmp"), resource, "version-one"); }