org.apache.hadoop.mapreduce.ClusterMetrics Java Examples

The following examples show how to use org.apache.hadoop.mapreduce.ClusterMetrics. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: JobClient.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Get status information about the Map-Reduce cluster.
 *  
 * @return the status information about the Map-Reduce cluster as an object
 *         of {@link ClusterStatus}.
 * @throws IOException
 */
public ClusterStatus getClusterStatus() throws IOException {
  try {
    return clientUgi.doAs(new PrivilegedExceptionAction<ClusterStatus>() {
      public ClusterStatus run() throws IOException, InterruptedException {
        ClusterMetrics metrics = cluster.getClusterStatus();
        return new ClusterStatus(metrics.getTaskTrackerCount(), metrics
          .getBlackListedTaskTrackerCount(), cluster
          .getTaskTrackerExpiryInterval(), metrics.getOccupiedMapSlots(),
          metrics.getOccupiedReduceSlots(), metrics.getMapSlotCapacity(),
          metrics.getReduceSlotCapacity(), cluster.getJobTrackerStatus(),
          metrics.getDecommissionedTaskTrackerCount(), metrics
            .getGrayListedTaskTrackerCount());
      }
    });
  } catch (InterruptedException ie) {
    throw new IOException(ie);
  }
}
 
Example #2
Source File: JobClient.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Get status information about the Map-Reduce cluster.
 *  
 * @param  detailed if true then get a detailed status including the
 *         tracker names
 * @return the status information about the Map-Reduce cluster as an object
 *         of {@link ClusterStatus}.
 * @throws IOException
 */
public ClusterStatus getClusterStatus(boolean detailed) throws IOException {
  try {
    return clientUgi.doAs(new PrivilegedExceptionAction<ClusterStatus>() {
      public ClusterStatus run() throws IOException, InterruptedException {
      ClusterMetrics metrics = cluster.getClusterStatus();
      return new ClusterStatus(arrayToStringList(cluster.getActiveTaskTrackers()),
        arrayToBlackListInfo(cluster.getBlackListedTaskTrackers()),
        cluster.getTaskTrackerExpiryInterval(), metrics.getOccupiedMapSlots(),
        metrics.getOccupiedReduceSlots(), metrics.getMapSlotCapacity(),
        metrics.getReduceSlotCapacity(), 
        cluster.getJobTrackerStatus());
      }
    });
  } catch (InterruptedException ie) {
    throw new IOException(ie);
  }
}
 
Example #3
Source File: DistSum.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Choose a Machine in runtime according to the cluster status.
 */
private Machine chooseMachine(Configuration conf) throws IOException {
  final int parts = conf.getInt(N_PARTS, Integer.MAX_VALUE);
  try {
    for(;; Thread.sleep(2000)) {
      //get cluster status
      final ClusterMetrics status = cluster.getClusterStatus();
      final int m = 
        status.getMapSlotCapacity() - status.getOccupiedMapSlots();
      final int r = 
        status.getReduceSlotCapacity() - status.getOccupiedReduceSlots();
      if (m >= parts || r >= parts) {
        //favor ReduceSide machine
        final Machine value = r >= parts?
            ReduceSide.INSTANCE: MapSide.INSTANCE;
        Util.out.println("  " + this + " is " + value + " (m=" + m + ", r=" + r + ")");
        return value;
      }
    }
  } catch (InterruptedException e) {
    throw new IOException(e);
  }    
}
 
Example #4
Source File: JobClient.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Get status information about the Map-Reduce cluster.
 *  
 * @return the status information about the Map-Reduce cluster as an object
 *         of {@link ClusterStatus}.
 * @throws IOException
 */
public ClusterStatus getClusterStatus() throws IOException {
  try {
    return clientUgi.doAs(new PrivilegedExceptionAction<ClusterStatus>() {
      public ClusterStatus run() throws IOException, InterruptedException {
        ClusterMetrics metrics = cluster.getClusterStatus();
        return new ClusterStatus(metrics.getTaskTrackerCount(), metrics
          .getBlackListedTaskTrackerCount(), cluster
          .getTaskTrackerExpiryInterval(), metrics.getOccupiedMapSlots(),
          metrics.getOccupiedReduceSlots(), metrics.getMapSlotCapacity(),
          metrics.getReduceSlotCapacity(), cluster.getJobTrackerStatus(),
          metrics.getDecommissionedTaskTrackerCount(), metrics
            .getGrayListedTaskTrackerCount());
      }
    });
  } catch (InterruptedException ie) {
    throw new IOException(ie);
  }
}
 
Example #5
Source File: JobClient.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Get status information about the Map-Reduce cluster.
 *  
 * @param  detailed if true then get a detailed status including the
 *         tracker names
 * @return the status information about the Map-Reduce cluster as an object
 *         of {@link ClusterStatus}.
 * @throws IOException
 */
public ClusterStatus getClusterStatus(boolean detailed) throws IOException {
  try {
    return clientUgi.doAs(new PrivilegedExceptionAction<ClusterStatus>() {
      public ClusterStatus run() throws IOException, InterruptedException {
      ClusterMetrics metrics = cluster.getClusterStatus();
      return new ClusterStatus(arrayToStringList(cluster.getActiveTaskTrackers()),
        arrayToBlackListInfo(cluster.getBlackListedTaskTrackers()),
        cluster.getTaskTrackerExpiryInterval(), metrics.getOccupiedMapSlots(),
        metrics.getOccupiedReduceSlots(), metrics.getMapSlotCapacity(),
        metrics.getReduceSlotCapacity(), 
        cluster.getJobTrackerStatus());
      }
    });
  } catch (InterruptedException ie) {
    throw new IOException(ie);
  }
}
 
Example #6
Source File: DistSum.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Choose a Machine in runtime according to the cluster status.
 */
private Machine chooseMachine(Configuration conf) throws IOException {
  final int parts = conf.getInt(N_PARTS, Integer.MAX_VALUE);
  try {
    for(;; Thread.sleep(2000)) {
      //get cluster status
      final ClusterMetrics status = cluster.getClusterStatus();
      final int m = 
        status.getMapSlotCapacity() - status.getOccupiedMapSlots();
      final int r = 
        status.getReduceSlotCapacity() - status.getOccupiedReduceSlots();
      if (m >= parts || r >= parts) {
        //favor ReduceSide machine
        final Machine value = r >= parts?
            ReduceSide.INSTANCE: MapSide.INSTANCE;
        Util.out.println("  " + this + " is " + value + " (m=" + m + ", r=" + r + ")");
        return value;
      }
    }
  } catch (InterruptedException e) {
    throw new IOException(e);
  }    
}
 
Example #7
Source File: ResourceMgrDelegate.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public ClusterMetrics getClusterMetrics() throws IOException,
    InterruptedException {
  try {
    YarnClusterMetrics metrics = client.getYarnClusterMetrics();
    ClusterMetrics oldMetrics =
        new ClusterMetrics(1, 1, 1, 1, 1, 1,
            metrics.getNumNodeManagers() * 10,
            metrics.getNumNodeManagers() * 2, 1,
            metrics.getNumNodeManagers(), 0, 0);
    return oldMetrics;
  } catch (YarnException e) {
    throw new IOException(e);
  }
}
 
Example #8
Source File: ResourceMgrDelegate.java    From tez with Apache License 2.0 5 votes vote down vote up
public ClusterMetrics getClusterMetrics() throws IOException,
    InterruptedException {
  YarnClusterMetrics metrics;
  try {
    metrics = client.getYarnClusterMetrics();
  } catch (YarnException e) {
    throw new IOException(e);
  }
  ClusterMetrics oldMetrics = new ClusterMetrics(1, 1, 1, 1, 1, 1, 
      metrics.getNumNodeManagers() * 10, metrics.getNumNodeManagers() * 2, 1,
      metrics.getNumNodeManagers(), 0, 0);
  return oldMetrics;
}
 
Example #9
Source File: ResourceMgrDelegate.java    From big-c with Apache License 2.0 5 votes vote down vote up
public ClusterMetrics getClusterMetrics() throws IOException,
    InterruptedException {
  try {
    YarnClusterMetrics metrics = client.getYarnClusterMetrics();
    ClusterMetrics oldMetrics =
        new ClusterMetrics(1, 1, 1, 1, 1, 1,
            metrics.getNumNodeManagers() * 10,
            metrics.getNumNodeManagers() * 2, 1,
            metrics.getNumNodeManagers(), 0, 0);
    return oldMetrics;
  } catch (YarnException e) {
    throw new IOException(e);
  }
}
 
Example #10
Source File: ResourceMgrDelegate.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
public ClusterMetrics getClusterMetrics() throws IOException,
    InterruptedException {
  YarnClusterMetrics metrics;
  try {
    metrics = client.getYarnClusterMetrics();
  } catch (YarnException e) {
    throw new IOException(e);
  }
  ClusterMetrics oldMetrics = new ClusterMetrics(1, 1, 1, 1, 1, 1, 
      metrics.getNumNodeManagers() * 10, metrics.getNumNodeManagers() * 2, 1,
      metrics.getNumNodeManagers(), 0, 0);
  return oldMetrics;
}
 
Example #11
Source File: JobTracker.java    From RDFS with Apache License 2.0 5 votes vote down vote up
public synchronized ClusterMetrics getClusterMetrics() {
  return new ClusterMetrics(totalMaps,
    totalReduces, occupiedMapSlots, occupiedReduceSlots,
    reservedMapSlots, reservedReduceSlots,
    totalMapTaskCapacity, totalReduceTaskCapacity,
    totalSubmissions,
    taskTrackers.size() - getBlacklistedTrackerCount(),
    getBlacklistedTrackerCount(), getExcludedNodes().size()) ;
}
 
Example #12
Source File: YARNRunner.java    From tez with Apache License 2.0 4 votes vote down vote up
@Override
public ClusterMetrics getClusterMetrics() throws IOException,
    InterruptedException {
  return resMgrDelegate.getClusterMetrics();
}
 
Example #13
Source File: YARNRunner.java    From incubator-tez with Apache License 2.0 4 votes vote down vote up
@Override
public ClusterMetrics getClusterMetrics() throws IOException,
    InterruptedException {
  return resMgrDelegate.getClusterMetrics();
}
 
Example #14
Source File: MockSimulatorJobTracker.java    From RDFS with Apache License 2.0 4 votes vote down vote up
@Override
public ClusterMetrics getClusterMetrics() throws IOException,
    InterruptedException {
  throw new UnsupportedOperationException();
}
 
Example #15
Source File: TestClusterStatus.java    From RDFS with Apache License 2.0 4 votes vote down vote up
public void testReservedSlots() throws IOException {
  JobConf conf = mr.createJobConf();

  conf.setNumReduceTasks(1);
  conf.setSpeculativeExecution(false);
  
  //Set task tracker objects for reservation.
  TaskTracker tt1 = jobTracker.getTaskTracker(trackers[0]);
  TaskTracker tt2 = jobTracker.getTaskTracker(trackers[1]);
  TaskTrackerStatus status1 = new TaskTrackerStatus(
      trackers[0],JobInProgress.convertTrackerNameToHostName(
          trackers[0]),0,new ArrayList<TaskStatus>(), 0, 2, 2);
  TaskTrackerStatus status2 = new TaskTrackerStatus(
      trackers[1],JobInProgress.convertTrackerNameToHostName(
          trackers[1]),0,new ArrayList<TaskStatus>(), 0, 2, 2);
  tt1.setStatus(status1);
  tt2.setStatus(status2);
  
  fakeJob = new FakeJobInProgress(new JobID("jt", 1), new JobConf(conf),
                  jobTracker);
  
  sendHeartBeat(jobTracker, status1, false, true, trackers[0], responseId);
  sendHeartBeat(jobTracker, status2, false, true, trackers[1], responseId);
  responseId++; 
  ClusterMetrics metrics = jobTracker.getClusterMetrics();
  assertEquals("reserved map slots do not match", 
    2, metrics.getReservedMapSlots());
  assertEquals("reserved reduce slots do not match", 
    2, metrics.getReservedReduceSlots());

  // redo to test re-reservations.
  sendHeartBeat(jobTracker, status1, false, true, trackers[0], responseId);
  sendHeartBeat(jobTracker, status2, false, true, trackers[1], responseId);
  responseId++; 
  metrics = jobTracker.getClusterMetrics();
  assertEquals("reserved map slots do not match", 
      4, metrics.getReservedMapSlots());
  assertEquals("reserved reduce slots do not match", 
      4, metrics.getReservedReduceSlots());

  // undo reservations now.
  scheduler.setUnreserveSlots(true);
  sendHeartBeat(jobTracker, status1, false, true, trackers[0], responseId);
  sendHeartBeat(jobTracker, status2, false, true, trackers[1], responseId);
  responseId++;
  metrics = jobTracker.getClusterMetrics();
  assertEquals("map slots should have been unreserved",
      0, metrics.getReservedMapSlots());
  assertEquals("reduce slots should have been unreserved",
      0, metrics.getReservedReduceSlots());
}
 
Example #16
Source File: TestClusterStatus.java    From RDFS with Apache License 2.0 4 votes vote down vote up
public void testClusterMetrics() throws IOException, InterruptedException {
  assertEquals("tasktracker count doesn't match", trackers.length,
    client.getClusterStatus().getTaskTrackers());
  
  List<TaskStatus> list = new ArrayList<TaskStatus>();

  // create a map task status, which uses 2 slots. 
  int mapSlotsPerTask = 2;
  addMapTaskAttemptToList(list, mapSlotsPerTask, TaskStatus.State.RUNNING);
  
  // create a reduce task status, which uses 1 slot.
  int reduceSlotsPerTask = 1;
  addReduceTaskAttemptToList(list, 
      reduceSlotsPerTask, TaskStatus.State.RUNNING);
  
  // create TaskTrackerStatus and send heartbeats
  sendHeartbeats(list);

  // assert ClusterMetrics
  ClusterMetrics metrics = jobTracker.getClusterMetrics();
  assertEquals("occupied map slots do not match", mapSlotsPerTask,
    metrics.getOccupiedMapSlots());
  assertEquals("occupied reduce slots do not match", reduceSlotsPerTask,
    metrics.getOccupiedReduceSlots());
  assertEquals("map slot capacities do not match",
    mapSlotsPerTracker * trackers.length,
    metrics.getMapSlotCapacity());
  assertEquals("reduce slot capacities do not match",
    reduceSlotsPerTracker * trackers.length,
    metrics.getReduceSlotCapacity());
  assertEquals("running map tasks do not match", 1,
    metrics.getRunningMaps());
  assertEquals("running reduce tasks do not match", 1,
    metrics.getRunningReduces());
  
  // assert the values in ClusterStatus also
  ClusterStatus stat = client.getClusterStatus();
  assertEquals("running map tasks do not match", 1,
    stat.getMapTasks());
  assertEquals("running reduce tasks do not match", 1,
    stat.getReduceTasks());
  assertEquals("map slot capacities do not match",
    mapSlotsPerTracker * trackers.length,
    stat.getMaxMapTasks());
  assertEquals("reduce slot capacities do not match",
    reduceSlotsPerTracker * trackers.length,
    stat.getMaxReduceTasks());
  
  // send a heartbeat finishing only a map and check
  // counts are updated.
  list.clear();
  addMapTaskAttemptToList(list, mapSlotsPerTask, TaskStatus.State.SUCCEEDED);
  addReduceTaskAttemptToList(list, 
      reduceSlotsPerTask, TaskStatus.State.RUNNING);
  sendHeartbeats(list);
  metrics = jobTracker.getClusterMetrics();
  assertEquals(0, metrics.getOccupiedMapSlots());
  assertEquals(reduceSlotsPerTask, metrics.getOccupiedReduceSlots());
  
  // send a heartbeat finishing the reduce task also.
  list.clear();
  addReduceTaskAttemptToList(list, 
      reduceSlotsPerTask, TaskStatus.State.SUCCEEDED);
  sendHeartbeats(list);
  metrics = jobTracker.getClusterMetrics();
  assertEquals(0, metrics.getOccupiedReduceSlots());
}
 
Example #17
Source File: HadoopClientProtocol.java    From ignite with Apache License 2.0 4 votes vote down vote up
/** {@inheritDoc} */
@Override public ClusterMetrics getClusterMetrics() throws IOException, InterruptedException {
    return new ClusterMetrics(0, 0, 0, 0, 0, 0, 1000, 1000, 1, 100, 0, 0);
}
 
Example #18
Source File: LocalJobRunner.java    From big-c with Apache License 2.0 4 votes vote down vote up
public ClusterMetrics getClusterMetrics() {
  int numMapTasks = map_tasks.get();
  int numReduceTasks = reduce_tasks.get();
  return new ClusterMetrics(numMapTasks, numReduceTasks, numMapTasks,
      numReduceTasks, 0, 0, 1, 1, jobs.size(), 1, 0, 0);
}
 
Example #19
Source File: YARNRunner.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Override
public ClusterMetrics getClusterMetrics() throws IOException,
    InterruptedException {
  return resMgrDelegate.getClusterMetrics();
}
 
Example #20
Source File: LocalJobRunner.java    From hadoop with Apache License 2.0 4 votes vote down vote up
public ClusterMetrics getClusterMetrics() {
  int numMapTasks = map_tasks.get();
  int numReduceTasks = reduce_tasks.get();
  return new ClusterMetrics(numMapTasks, numReduceTasks, numMapTasks,
      numReduceTasks, 0, 0, 1, 1, jobs.size(), 1, 0, 0);
}
 
Example #21
Source File: YARNRunner.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Override
public ClusterMetrics getClusterMetrics() throws IOException,
    InterruptedException {
  return resMgrDelegate.getClusterMetrics();
}
 
Example #22
Source File: ClientProtocol.java    From big-c with Apache License 2.0 2 votes vote down vote up
/**
 * Get the current status of the cluster
 * 
 * @return summary of the state of the cluster
 */
public ClusterMetrics getClusterMetrics() 
throws IOException, InterruptedException;
 
Example #23
Source File: ClientProtocol.java    From hadoop with Apache License 2.0 2 votes vote down vote up
/**
 * Get the current status of the cluster
 * 
 * @return summary of the state of the cluster
 */
public ClusterMetrics getClusterMetrics() 
throws IOException, InterruptedException;