Java Code Examples for org.apache.hadoop.mapreduce.ClusterMetrics

The following examples show how to use org.apache.hadoop.mapreduce.ClusterMetrics. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: JobClient.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Get status information about the Map-Reduce cluster.
 *  
 * @return the status information about the Map-Reduce cluster as an object
 *         of {@link ClusterStatus}.
 * @throws IOException
 */
public ClusterStatus getClusterStatus() throws IOException {
  try {
    return clientUgi.doAs(new PrivilegedExceptionAction<ClusterStatus>() {
      public ClusterStatus run() throws IOException, InterruptedException {
        ClusterMetrics metrics = cluster.getClusterStatus();
        return new ClusterStatus(metrics.getTaskTrackerCount(), metrics
          .getBlackListedTaskTrackerCount(), cluster
          .getTaskTrackerExpiryInterval(), metrics.getOccupiedMapSlots(),
          metrics.getOccupiedReduceSlots(), metrics.getMapSlotCapacity(),
          metrics.getReduceSlotCapacity(), cluster.getJobTrackerStatus(),
          metrics.getDecommissionedTaskTrackerCount(), metrics
            .getGrayListedTaskTrackerCount());
      }
    });
  } catch (InterruptedException ie) {
    throw new IOException(ie);
  }
}
 
Example 2
Source Project: hadoop   Source File: JobClient.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Get status information about the Map-Reduce cluster.
 *  
 * @param  detailed if true then get a detailed status including the
 *         tracker names
 * @return the status information about the Map-Reduce cluster as an object
 *         of {@link ClusterStatus}.
 * @throws IOException
 */
public ClusterStatus getClusterStatus(boolean detailed) throws IOException {
  try {
    return clientUgi.doAs(new PrivilegedExceptionAction<ClusterStatus>() {
      public ClusterStatus run() throws IOException, InterruptedException {
      ClusterMetrics metrics = cluster.getClusterStatus();
      return new ClusterStatus(arrayToStringList(cluster.getActiveTaskTrackers()),
        arrayToBlackListInfo(cluster.getBlackListedTaskTrackers()),
        cluster.getTaskTrackerExpiryInterval(), metrics.getOccupiedMapSlots(),
        metrics.getOccupiedReduceSlots(), metrics.getMapSlotCapacity(),
        metrics.getReduceSlotCapacity(), 
        cluster.getJobTrackerStatus());
      }
    });
  } catch (InterruptedException ie) {
    throw new IOException(ie);
  }
}
 
Example 3
Source Project: hadoop   Source File: DistSum.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Choose a Machine in runtime according to the cluster status.
 */
private Machine chooseMachine(Configuration conf) throws IOException {
  final int parts = conf.getInt(N_PARTS, Integer.MAX_VALUE);
  try {
    for(;; Thread.sleep(2000)) {
      //get cluster status
      final ClusterMetrics status = cluster.getClusterStatus();
      final int m = 
        status.getMapSlotCapacity() - status.getOccupiedMapSlots();
      final int r = 
        status.getReduceSlotCapacity() - status.getOccupiedReduceSlots();
      if (m >= parts || r >= parts) {
        //favor ReduceSide machine
        final Machine value = r >= parts?
            ReduceSide.INSTANCE: MapSide.INSTANCE;
        Util.out.println("  " + this + " is " + value + " (m=" + m + ", r=" + r + ")");
        return value;
      }
    }
  } catch (InterruptedException e) {
    throw new IOException(e);
  }    
}
 
Example 4
Source Project: big-c   Source File: JobClient.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Get status information about the Map-Reduce cluster.
 *  
 * @return the status information about the Map-Reduce cluster as an object
 *         of {@link ClusterStatus}.
 * @throws IOException
 */
public ClusterStatus getClusterStatus() throws IOException {
  try {
    return clientUgi.doAs(new PrivilegedExceptionAction<ClusterStatus>() {
      public ClusterStatus run() throws IOException, InterruptedException {
        ClusterMetrics metrics = cluster.getClusterStatus();
        return new ClusterStatus(metrics.getTaskTrackerCount(), metrics
          .getBlackListedTaskTrackerCount(), cluster
          .getTaskTrackerExpiryInterval(), metrics.getOccupiedMapSlots(),
          metrics.getOccupiedReduceSlots(), metrics.getMapSlotCapacity(),
          metrics.getReduceSlotCapacity(), cluster.getJobTrackerStatus(),
          metrics.getDecommissionedTaskTrackerCount(), metrics
            .getGrayListedTaskTrackerCount());
      }
    });
  } catch (InterruptedException ie) {
    throw new IOException(ie);
  }
}
 
Example 5
Source Project: big-c   Source File: JobClient.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Get status information about the Map-Reduce cluster.
 *  
 * @param  detailed if true then get a detailed status including the
 *         tracker names
 * @return the status information about the Map-Reduce cluster as an object
 *         of {@link ClusterStatus}.
 * @throws IOException
 */
public ClusterStatus getClusterStatus(boolean detailed) throws IOException {
  try {
    return clientUgi.doAs(new PrivilegedExceptionAction<ClusterStatus>() {
      public ClusterStatus run() throws IOException, InterruptedException {
      ClusterMetrics metrics = cluster.getClusterStatus();
      return new ClusterStatus(arrayToStringList(cluster.getActiveTaskTrackers()),
        arrayToBlackListInfo(cluster.getBlackListedTaskTrackers()),
        cluster.getTaskTrackerExpiryInterval(), metrics.getOccupiedMapSlots(),
        metrics.getOccupiedReduceSlots(), metrics.getMapSlotCapacity(),
        metrics.getReduceSlotCapacity(), 
        cluster.getJobTrackerStatus());
      }
    });
  } catch (InterruptedException ie) {
    throw new IOException(ie);
  }
}
 
Example 6
Source Project: big-c   Source File: DistSum.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Choose a Machine in runtime according to the cluster status.
 */
private Machine chooseMachine(Configuration conf) throws IOException {
  final int parts = conf.getInt(N_PARTS, Integer.MAX_VALUE);
  try {
    for(;; Thread.sleep(2000)) {
      //get cluster status
      final ClusterMetrics status = cluster.getClusterStatus();
      final int m = 
        status.getMapSlotCapacity() - status.getOccupiedMapSlots();
      final int r = 
        status.getReduceSlotCapacity() - status.getOccupiedReduceSlots();
      if (m >= parts || r >= parts) {
        //favor ReduceSide machine
        final Machine value = r >= parts?
            ReduceSide.INSTANCE: MapSide.INSTANCE;
        Util.out.println("  " + this + " is " + value + " (m=" + m + ", r=" + r + ")");
        return value;
      }
    }
  } catch (InterruptedException e) {
    throw new IOException(e);
  }    
}
 
Example 7
Source Project: hadoop   Source File: ResourceMgrDelegate.java    License: Apache License 2.0 5 votes vote down vote up
public ClusterMetrics getClusterMetrics() throws IOException,
    InterruptedException {
  try {
    YarnClusterMetrics metrics = client.getYarnClusterMetrics();
    ClusterMetrics oldMetrics =
        new ClusterMetrics(1, 1, 1, 1, 1, 1,
            metrics.getNumNodeManagers() * 10,
            metrics.getNumNodeManagers() * 2, 1,
            metrics.getNumNodeManagers(), 0, 0);
    return oldMetrics;
  } catch (YarnException e) {
    throw new IOException(e);
  }
}
 
Example 8
Source Project: big-c   Source File: ResourceMgrDelegate.java    License: Apache License 2.0 5 votes vote down vote up
public ClusterMetrics getClusterMetrics() throws IOException,
    InterruptedException {
  try {
    YarnClusterMetrics metrics = client.getYarnClusterMetrics();
    ClusterMetrics oldMetrics =
        new ClusterMetrics(1, 1, 1, 1, 1, 1,
            metrics.getNumNodeManagers() * 10,
            metrics.getNumNodeManagers() * 2, 1,
            metrics.getNumNodeManagers(), 0, 0);
    return oldMetrics;
  } catch (YarnException e) {
    throw new IOException(e);
  }
}
 
Example 9
Source Project: RDFS   Source File: JobTracker.java    License: Apache License 2.0 5 votes vote down vote up
public synchronized ClusterMetrics getClusterMetrics() {
  return new ClusterMetrics(totalMaps,
    totalReduces, occupiedMapSlots, occupiedReduceSlots,
    reservedMapSlots, reservedReduceSlots,
    totalMapTaskCapacity, totalReduceTaskCapacity,
    totalSubmissions,
    taskTrackers.size() - getBlacklistedTrackerCount(),
    getBlacklistedTrackerCount(), getExcludedNodes().size()) ;
}
 
Example 10
Source Project: incubator-tez   Source File: ResourceMgrDelegate.java    License: Apache License 2.0 5 votes vote down vote up
public ClusterMetrics getClusterMetrics() throws IOException,
    InterruptedException {
  YarnClusterMetrics metrics;
  try {
    metrics = client.getYarnClusterMetrics();
  } catch (YarnException e) {
    throw new IOException(e);
  }
  ClusterMetrics oldMetrics = new ClusterMetrics(1, 1, 1, 1, 1, 1, 
      metrics.getNumNodeManagers() * 10, metrics.getNumNodeManagers() * 2, 1,
      metrics.getNumNodeManagers(), 0, 0);
  return oldMetrics;
}
 
Example 11
Source Project: tez   Source File: ResourceMgrDelegate.java    License: Apache License 2.0 5 votes vote down vote up
public ClusterMetrics getClusterMetrics() throws IOException,
    InterruptedException {
  YarnClusterMetrics metrics;
  try {
    metrics = client.getYarnClusterMetrics();
  } catch (YarnException e) {
    throw new IOException(e);
  }
  ClusterMetrics oldMetrics = new ClusterMetrics(1, 1, 1, 1, 1, 1, 
      metrics.getNumNodeManagers() * 10, metrics.getNumNodeManagers() * 2, 1,
      metrics.getNumNodeManagers(), 0, 0);
  return oldMetrics;
}
 
Example 12
Source Project: hadoop   Source File: YARNRunner.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public ClusterMetrics getClusterMetrics() throws IOException,
    InterruptedException {
  return resMgrDelegate.getClusterMetrics();
}
 
Example 13
Source Project: hadoop   Source File: LocalJobRunner.java    License: Apache License 2.0 4 votes vote down vote up
public ClusterMetrics getClusterMetrics() {
  int numMapTasks = map_tasks.get();
  int numReduceTasks = reduce_tasks.get();
  return new ClusterMetrics(numMapTasks, numReduceTasks, numMapTasks,
      numReduceTasks, 0, 0, 1, 1, jobs.size(), 1, 0, 0);
}
 
Example 14
Source Project: big-c   Source File: YARNRunner.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public ClusterMetrics getClusterMetrics() throws IOException,
    InterruptedException {
  return resMgrDelegate.getClusterMetrics();
}
 
Example 15
Source Project: big-c   Source File: LocalJobRunner.java    License: Apache License 2.0 4 votes vote down vote up
public ClusterMetrics getClusterMetrics() {
  int numMapTasks = map_tasks.get();
  int numReduceTasks = reduce_tasks.get();
  return new ClusterMetrics(numMapTasks, numReduceTasks, numMapTasks,
      numReduceTasks, 0, 0, 1, 1, jobs.size(), 1, 0, 0);
}
 
Example 16
Source Project: ignite   Source File: HadoopClientProtocol.java    License: Apache License 2.0 4 votes vote down vote up
/** {@inheritDoc} */
@Override public ClusterMetrics getClusterMetrics() throws IOException, InterruptedException {
    return new ClusterMetrics(0, 0, 0, 0, 0, 0, 1000, 1000, 1, 100, 0, 0);
}
 
Example 17
Source Project: RDFS   Source File: TestClusterStatus.java    License: Apache License 2.0 4 votes vote down vote up
public void testClusterMetrics() throws IOException, InterruptedException {
  assertEquals("tasktracker count doesn't match", trackers.length,
    client.getClusterStatus().getTaskTrackers());
  
  List<TaskStatus> list = new ArrayList<TaskStatus>();

  // create a map task status, which uses 2 slots. 
  int mapSlotsPerTask = 2;
  addMapTaskAttemptToList(list, mapSlotsPerTask, TaskStatus.State.RUNNING);
  
  // create a reduce task status, which uses 1 slot.
  int reduceSlotsPerTask = 1;
  addReduceTaskAttemptToList(list, 
      reduceSlotsPerTask, TaskStatus.State.RUNNING);
  
  // create TaskTrackerStatus and send heartbeats
  sendHeartbeats(list);

  // assert ClusterMetrics
  ClusterMetrics metrics = jobTracker.getClusterMetrics();
  assertEquals("occupied map slots do not match", mapSlotsPerTask,
    metrics.getOccupiedMapSlots());
  assertEquals("occupied reduce slots do not match", reduceSlotsPerTask,
    metrics.getOccupiedReduceSlots());
  assertEquals("map slot capacities do not match",
    mapSlotsPerTracker * trackers.length,
    metrics.getMapSlotCapacity());
  assertEquals("reduce slot capacities do not match",
    reduceSlotsPerTracker * trackers.length,
    metrics.getReduceSlotCapacity());
  assertEquals("running map tasks do not match", 1,
    metrics.getRunningMaps());
  assertEquals("running reduce tasks do not match", 1,
    metrics.getRunningReduces());
  
  // assert the values in ClusterStatus also
  ClusterStatus stat = client.getClusterStatus();
  assertEquals("running map tasks do not match", 1,
    stat.getMapTasks());
  assertEquals("running reduce tasks do not match", 1,
    stat.getReduceTasks());
  assertEquals("map slot capacities do not match",
    mapSlotsPerTracker * trackers.length,
    stat.getMaxMapTasks());
  assertEquals("reduce slot capacities do not match",
    reduceSlotsPerTracker * trackers.length,
    stat.getMaxReduceTasks());
  
  // send a heartbeat finishing only a map and check
  // counts are updated.
  list.clear();
  addMapTaskAttemptToList(list, mapSlotsPerTask, TaskStatus.State.SUCCEEDED);
  addReduceTaskAttemptToList(list, 
      reduceSlotsPerTask, TaskStatus.State.RUNNING);
  sendHeartbeats(list);
  metrics = jobTracker.getClusterMetrics();
  assertEquals(0, metrics.getOccupiedMapSlots());
  assertEquals(reduceSlotsPerTask, metrics.getOccupiedReduceSlots());
  
  // send a heartbeat finishing the reduce task also.
  list.clear();
  addReduceTaskAttemptToList(list, 
      reduceSlotsPerTask, TaskStatus.State.SUCCEEDED);
  sendHeartbeats(list);
  metrics = jobTracker.getClusterMetrics();
  assertEquals(0, metrics.getOccupiedReduceSlots());
}
 
Example 18
Source Project: RDFS   Source File: TestClusterStatus.java    License: Apache License 2.0 4 votes vote down vote up
public void testReservedSlots() throws IOException {
  JobConf conf = mr.createJobConf();

  conf.setNumReduceTasks(1);
  conf.setSpeculativeExecution(false);
  
  //Set task tracker objects for reservation.
  TaskTracker tt1 = jobTracker.getTaskTracker(trackers[0]);
  TaskTracker tt2 = jobTracker.getTaskTracker(trackers[1]);
  TaskTrackerStatus status1 = new TaskTrackerStatus(
      trackers[0],JobInProgress.convertTrackerNameToHostName(
          trackers[0]),0,new ArrayList<TaskStatus>(), 0, 2, 2);
  TaskTrackerStatus status2 = new TaskTrackerStatus(
      trackers[1],JobInProgress.convertTrackerNameToHostName(
          trackers[1]),0,new ArrayList<TaskStatus>(), 0, 2, 2);
  tt1.setStatus(status1);
  tt2.setStatus(status2);
  
  fakeJob = new FakeJobInProgress(new JobID("jt", 1), new JobConf(conf),
                  jobTracker);
  
  sendHeartBeat(jobTracker, status1, false, true, trackers[0], responseId);
  sendHeartBeat(jobTracker, status2, false, true, trackers[1], responseId);
  responseId++; 
  ClusterMetrics metrics = jobTracker.getClusterMetrics();
  assertEquals("reserved map slots do not match", 
    2, metrics.getReservedMapSlots());
  assertEquals("reserved reduce slots do not match", 
    2, metrics.getReservedReduceSlots());

  // redo to test re-reservations.
  sendHeartBeat(jobTracker, status1, false, true, trackers[0], responseId);
  sendHeartBeat(jobTracker, status2, false, true, trackers[1], responseId);
  responseId++; 
  metrics = jobTracker.getClusterMetrics();
  assertEquals("reserved map slots do not match", 
      4, metrics.getReservedMapSlots());
  assertEquals("reserved reduce slots do not match", 
      4, metrics.getReservedReduceSlots());

  // undo reservations now.
  scheduler.setUnreserveSlots(true);
  sendHeartBeat(jobTracker, status1, false, true, trackers[0], responseId);
  sendHeartBeat(jobTracker, status2, false, true, trackers[1], responseId);
  responseId++;
  metrics = jobTracker.getClusterMetrics();
  assertEquals("map slots should have been unreserved",
      0, metrics.getReservedMapSlots());
  assertEquals("reduce slots should have been unreserved",
      0, metrics.getReservedReduceSlots());
}
 
Example 19
Source Project: RDFS   Source File: MockSimulatorJobTracker.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public ClusterMetrics getClusterMetrics() throws IOException,
    InterruptedException {
  throw new UnsupportedOperationException();
}
 
Example 20
Source Project: incubator-tez   Source File: YARNRunner.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public ClusterMetrics getClusterMetrics() throws IOException,
    InterruptedException {
  return resMgrDelegate.getClusterMetrics();
}
 
Example 21
Source Project: tez   Source File: YARNRunner.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public ClusterMetrics getClusterMetrics() throws IOException,
    InterruptedException {
  return resMgrDelegate.getClusterMetrics();
}
 
Example 22
Source Project: hadoop   Source File: ClientProtocol.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Get the current status of the cluster
 * 
 * @return summary of the state of the cluster
 */
public ClusterMetrics getClusterMetrics() 
throws IOException, InterruptedException;
 
Example 23
Source Project: big-c   Source File: ClientProtocol.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Get the current status of the cluster
 * 
 * @return summary of the state of the cluster
 */
public ClusterMetrics getClusterMetrics() 
throws IOException, InterruptedException;