Java Code Examples for org.apache.hadoop.mapreduce.TaskReport

The following examples show how to use org.apache.hadoop.mapreduce.TaskReport. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: CLI.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Display the information about a job's tasks, of a particular type and
 * in a particular state
 * 
 * @param job the job
 * @param type the type of the task (map/reduce/setup/cleanup)
 * @param state the state of the task 
 * (pending/running/completed/failed/killed)
 */
protected void displayTasks(Job job, String type, String state) 
throws IOException, InterruptedException {
  TaskReport[] reports = job.getTaskReports(TaskType.valueOf(
      org.apache.hadoop.util.StringUtils.toUpperCase(type)));
  for (TaskReport report : reports) {
    TIPStatus status = report.getCurrentStatus();
    if ((state.equalsIgnoreCase("pending") && status ==TIPStatus.PENDING) ||
        (state.equalsIgnoreCase("running") && status ==TIPStatus.RUNNING) ||
        (state.equalsIgnoreCase("completed") && status == TIPStatus.COMPLETE) ||
        (state.equalsIgnoreCase("failed") && status == TIPStatus.FAILED) ||
        (state.equalsIgnoreCase("killed") && status == TIPStatus.KILLED)) {
      printTaskAttempts(report);
    }
  }
}
 
Example 2
Source Project: big-c   Source File: CLI.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Display the information about a job's tasks, of a particular type and
 * in a particular state
 * 
 * @param job the job
 * @param type the type of the task (map/reduce/setup/cleanup)
 * @param state the state of the task 
 * (pending/running/completed/failed/killed)
 */
protected void displayTasks(Job job, String type, String state) 
throws IOException, InterruptedException {
  TaskReport[] reports = job.getTaskReports(TaskType.valueOf(
      org.apache.hadoop.util.StringUtils.toUpperCase(type)));
  for (TaskReport report : reports) {
    TIPStatus status = report.getCurrentStatus();
    if ((state.equalsIgnoreCase("pending") && status ==TIPStatus.PENDING) ||
        (state.equalsIgnoreCase("running") && status ==TIPStatus.RUNNING) ||
        (state.equalsIgnoreCase("completed") && status == TIPStatus.COMPLETE) ||
        (state.equalsIgnoreCase("failed") && status == TIPStatus.FAILED) ||
        (state.equalsIgnoreCase("killed") && status == TIPStatus.KILLED)) {
      printTaskAttempts(report);
    }
  }
}
 
Example 3
Source Project: sequenceiq-samples   Source File: MRJobStatus.java    License: Apache License 2.0 6 votes vote down vote up
public JobStatus printJobStatus(YARNRunner yarnRunner, JobID jobID) throws IOException, InterruptedException {
	JobStatus jobStatus;
	jobStatus = yarnRunner.getJobStatus(jobID);
	
	// print overall job M/R progresses
	LOGGER.info("\nJob " + jobStatus.getJobName() + "in queue (" + jobStatus.getQueue() + ")" + " progress M/R: " + jobStatus.getMapProgress() + "/" + jobStatus.getReduceProgress());
	LOGGER.info("Tracking URL : " + jobStatus.getTrackingUrl());
	LOGGER.info("Reserved memory : " + jobStatus.getReservedMem() + ", used memory : "+ jobStatus.getUsedMem() + " and used slots : "+ jobStatus.getNumUsedSlots());
	
	// list map & reduce tasks statuses and progress		
	TaskReport[] reports = yarnRunner.getTaskReports(jobID, TaskType.MAP);
	for (int i = 0; i < reports.length; i++) {
		LOGGER.info("MAP: Status " + reports[i].getCurrentStatus() + " with task ID " + reports[i].getTaskID() + ", and progress " + reports[i].getProgress()); 
	}
	reports = yarnRunner.getTaskReports(jobID, TaskType.REDUCE);
	for (int i = 0; i < reports.length; i++) {
		LOGGER.info("REDUCE: " + reports[i].getCurrentStatus() + " with task ID " + reports[i].getTaskID() + ", and progress " + reports[i].getProgress()); 
	}
	return jobStatus;
}
 
Example 4
Source Project: incubator-retired-blur   Source File: IndexerJobDriver.java    License: Apache License 2.0 6 votes vote down vote up
PartitionedInputResult(Path partitionedInputData, Counters counters, int shards, TaskReport[] taskReports) {
  _partitionedInputData = partitionedInputData;
  _counters = counters;
  _rowIdsFromNewData = new long[shards];
  _rowIdsToUpdateFromNewData = new long[shards];
  _rowIdsFromIndex = new long[shards];
  for (TaskReport tr : taskReports) {
    int id = tr.getTaskID().getId();
    Counters taskCounters = tr.getTaskCounters();
    Counter total = taskCounters.findCounter(BlurIndexCounter.ROW_IDS_FROM_NEW_DATA);
    _rowIdsFromNewData[id] = total.getValue();
    Counter update = taskCounters.findCounter(BlurIndexCounter.ROW_IDS_TO_UPDATE_FROM_NEW_DATA);
    _rowIdsToUpdateFromNewData[id] = update.getValue();
    Counter index = taskCounters.findCounter(BlurIndexCounter.ROW_IDS_FROM_INDEX);
    _rowIdsFromIndex[id] = index.getValue();
  }
}
 
Example 5
Source Project: hadoop   Source File: JobClientUnitTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testMapTaskReportsWithNullJob() throws Exception {
  TestJobClient client = new TestJobClient(new JobConf());
  Cluster mockCluster = mock(Cluster.class);
  client.setCluster(mockCluster);
  JobID id = new JobID("test",0);
  
  when(mockCluster.getJob(id)).thenReturn(null);
  
  TaskReport[] result = client.getMapTaskReports(id);
  assertEquals(0, result.length);
  
  verify(mockCluster).getJob(id);
}
 
Example 6
Source Project: hadoop   Source File: JobClientUnitTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testReduceTaskReportsWithNullJob() throws Exception {
  TestJobClient client = new TestJobClient(new JobConf());
  Cluster mockCluster = mock(Cluster.class);
  client.setCluster(mockCluster);
  JobID id = new JobID("test",0);
  
  when(mockCluster.getJob(id)).thenReturn(null);
  
  TaskReport[] result = client.getReduceTaskReports(id);
  assertEquals(0, result.length);
  
  verify(mockCluster).getJob(id);
}
 
Example 7
Source Project: hadoop   Source File: JobClientUnitTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSetupTaskReportsWithNullJob() throws Exception {
  TestJobClient client = new TestJobClient(new JobConf());
  Cluster mockCluster = mock(Cluster.class);
  client.setCluster(mockCluster);
  JobID id = new JobID("test",0);
  
  when(mockCluster.getJob(id)).thenReturn(null);
  
  TaskReport[] result = client.getSetupTaskReports(id);
  assertEquals(0, result.length);
  
  verify(mockCluster).getJob(id);
}
 
Example 8
Source Project: hadoop   Source File: JobClientUnitTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCleanupTaskReportsWithNullJob() throws Exception {
  TestJobClient client = new TestJobClient(new JobConf());
  Cluster mockCluster = mock(Cluster.class);
  client.setCluster(mockCluster);
  JobID id = new JobID("test",0);
  
  when(mockCluster.getJob(id)).thenReturn(null);
  
  TaskReport[] result = client.getCleanupTaskReports(id);
  assertEquals(0, result.length);
  
  verify(mockCluster).getJob(id);
}
 
Example 9
Source Project: hadoop   Source File: CLI.java    License: Apache License 2.0 5 votes vote down vote up
private void printTaskAttempts(TaskReport report) {
  if (report.getCurrentStatus() == TIPStatus.COMPLETE) {
    System.out.println(report.getSuccessfulTaskAttemptId());
  } else if (report.getCurrentStatus() == TIPStatus.RUNNING) {
    for (TaskAttemptID t : 
      report.getRunningTaskAttemptIds()) {
      System.out.println(t);
    }
  }
}
 
Example 10
Source Project: big-c   Source File: JobClientUnitTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testMapTaskReportsWithNullJob() throws Exception {
  TestJobClient client = new TestJobClient(new JobConf());
  Cluster mockCluster = mock(Cluster.class);
  client.setCluster(mockCluster);
  JobID id = new JobID("test",0);
  
  when(mockCluster.getJob(id)).thenReturn(null);
  
  TaskReport[] result = client.getMapTaskReports(id);
  assertEquals(0, result.length);
  
  verify(mockCluster).getJob(id);
}
 
Example 11
Source Project: big-c   Source File: JobClientUnitTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testReduceTaskReportsWithNullJob() throws Exception {
  TestJobClient client = new TestJobClient(new JobConf());
  Cluster mockCluster = mock(Cluster.class);
  client.setCluster(mockCluster);
  JobID id = new JobID("test",0);
  
  when(mockCluster.getJob(id)).thenReturn(null);
  
  TaskReport[] result = client.getReduceTaskReports(id);
  assertEquals(0, result.length);
  
  verify(mockCluster).getJob(id);
}
 
Example 12
Source Project: big-c   Source File: JobClientUnitTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSetupTaskReportsWithNullJob() throws Exception {
  TestJobClient client = new TestJobClient(new JobConf());
  Cluster mockCluster = mock(Cluster.class);
  client.setCluster(mockCluster);
  JobID id = new JobID("test",0);
  
  when(mockCluster.getJob(id)).thenReturn(null);
  
  TaskReport[] result = client.getSetupTaskReports(id);
  assertEquals(0, result.length);
  
  verify(mockCluster).getJob(id);
}
 
Example 13
Source Project: big-c   Source File: JobClientUnitTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCleanupTaskReportsWithNullJob() throws Exception {
  TestJobClient client = new TestJobClient(new JobConf());
  Cluster mockCluster = mock(Cluster.class);
  client.setCluster(mockCluster);
  JobID id = new JobID("test",0);
  
  when(mockCluster.getJob(id)).thenReturn(null);
  
  TaskReport[] result = client.getCleanupTaskReports(id);
  assertEquals(0, result.length);
  
  verify(mockCluster).getJob(id);
}
 
Example 14
Source Project: big-c   Source File: CLI.java    License: Apache License 2.0 5 votes vote down vote up
private void printTaskAttempts(TaskReport report) {
  if (report.getCurrentStatus() == TIPStatus.COMPLETE) {
    System.out.println(report.getSuccessfulTaskAttemptId());
  } else if (report.getCurrentStatus() == TIPStatus.RUNNING) {
    for (TaskAttemptID t : 
      report.getRunningTaskAttemptIds()) {
      System.out.println(t);
    }
  }
}
 
Example 15
Source Project: indexr   Source File: UpdateColumnJob.java    License: Apache License 2.0 5 votes vote down vote up
public boolean doRun(Config upcolConfig) throws Exception {
    JobConf jobConf = new JobConf(getConf(), UpdateColumnJob.class);
    jobConf.setKeepFailedTaskFiles(false);
    jobConf.setNumReduceTasks(0);
    String jobName = String.format("indexr-upcol-%s-%s-%s",
            upcolConfig.table,
            LocalDateTime.now().format(timeFormatter),
            RandomStringUtils.randomAlphabetic(5));
    jobConf.setJobName(jobName);
    jobConf.set(CONFKEY, JsonUtil.toJson(upcolConfig));
    Path workDir = new Path(jobConf.getWorkingDirectory(), jobName);
    jobConf.setWorkingDirectory(workDir);

    Job job = Job.getInstance(jobConf);
    job.setInputFormatClass(SegmentInputFormat.class);
    job.setMapperClass(UpColSegmentMapper.class);
    job.setJarByClass(UpdateColumnJob.class);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(Text.class);
    job.setMapSpeculativeExecution(false);
    job.setOutputFormatClass(UpColSegmentOutputFormat.class);

    job.submit();
    boolean ok = job.waitForCompletion(true);
    if (!ok) {
        TaskReport[] reports = job.getTaskReports(TaskType.MAP);
        if (reports != null) {
            for (TaskReport report : reports) {
                log.error("Error in task [%s] : %s", report.getTaskId(), Arrays.toString(report.getDiagnostics()));
            }
        }
    }
    return ok;
}
 
Example 16
Source Project: hadoop   Source File: YARNRunner.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public TaskReport[] getTaskReports(JobID jobID, TaskType taskType)
throws IOException, InterruptedException {
  return clientCache.getClient(jobID)
      .getTaskReports(jobID, taskType);
}
 
Example 17
Source Project: hadoop   Source File: JobClientUnitTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testShowJob() throws Exception {
  TestJobClient client = new TestJobClient(new JobConf());

  long startTime = System.currentTimeMillis();

  JobID jobID = new JobID(String.valueOf(startTime), 12345);

  JobStatus mockJobStatus = mock(JobStatus.class);
  when(mockJobStatus.getJobID()).thenReturn(jobID);
  when(mockJobStatus.getJobName()).thenReturn(jobID.toString());
  when(mockJobStatus.getState()).thenReturn(JobStatus.State.RUNNING);
  when(mockJobStatus.getStartTime()).thenReturn(startTime);
  when(mockJobStatus.getUsername()).thenReturn("mockuser");
  when(mockJobStatus.getQueue()).thenReturn("mockqueue");
  when(mockJobStatus.getPriority()).thenReturn(JobPriority.NORMAL);
  when(mockJobStatus.getNumUsedSlots()).thenReturn(1);
  when(mockJobStatus.getNumReservedSlots()).thenReturn(1);
  when(mockJobStatus.getUsedMem()).thenReturn(1024);
  when(mockJobStatus.getReservedMem()).thenReturn(512);
  when(mockJobStatus.getNeededMem()).thenReturn(2048);
  when(mockJobStatus.getSchedulingInfo()).thenReturn("NA");

  Job mockJob = mock(Job.class);
  when(mockJob.getTaskReports(isA(TaskType.class))).thenReturn(
    new TaskReport[5]);

  Cluster mockCluster = mock(Cluster.class);
  when(mockCluster.getJob(jobID)).thenReturn(mockJob);

  client.setCluster(mockCluster);
  
  ByteArrayOutputStream out = new ByteArrayOutputStream();
  client.displayJobList(new JobStatus[] {mockJobStatus}, new PrintWriter(out));
  String commandLineOutput = out.toString();
  System.out.println(commandLineOutput);
  Assert.assertTrue(commandLineOutput.contains("Total jobs:1"));

  verify(mockJobStatus, atLeastOnce()).getJobID();
  verify(mockJobStatus).getState();
  verify(mockJobStatus).getStartTime();
  verify(mockJobStatus).getUsername();
  verify(mockJobStatus).getQueue();
  verify(mockJobStatus).getPriority();
  verify(mockJobStatus).getNumUsedSlots();
  verify(mockJobStatus).getNumReservedSlots();
  verify(mockJobStatus).getUsedMem();
  verify(mockJobStatus).getReservedMem();
  verify(mockJobStatus).getNeededMem();
  verify(mockJobStatus).getSchedulingInfo();

  // This call should not go to each AM.
  verify(mockCluster, never()).getJob(jobID);
  verify(mockJob, never()).getTaskReports(isA(TaskType.class));
}
 
Example 18
Source Project: hadoop   Source File: TestCLI.java    License: Apache License 2.0 4 votes vote down vote up
private TaskReport[] getTaskReports(JobID jobId, TaskType type) {
  return new TaskReport[] { new TaskReport(), new TaskReport() };
}
 
Example 19
Source Project: big-c   Source File: YARNRunner.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public TaskReport[] getTaskReports(JobID jobID, TaskType taskType)
throws IOException, InterruptedException {
  return clientCache.getClient(jobID)
      .getTaskReports(jobID, taskType);
}
 
Example 20
Source Project: big-c   Source File: JobClientUnitTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testShowJob() throws Exception {
  TestJobClient client = new TestJobClient(new JobConf());

  long startTime = System.currentTimeMillis();

  JobID jobID = new JobID(String.valueOf(startTime), 12345);

  JobStatus mockJobStatus = mock(JobStatus.class);
  when(mockJobStatus.getJobID()).thenReturn(jobID);
  when(mockJobStatus.getJobName()).thenReturn(jobID.toString());
  when(mockJobStatus.getState()).thenReturn(JobStatus.State.RUNNING);
  when(mockJobStatus.getStartTime()).thenReturn(startTime);
  when(mockJobStatus.getUsername()).thenReturn("mockuser");
  when(mockJobStatus.getQueue()).thenReturn("mockqueue");
  when(mockJobStatus.getPriority()).thenReturn(JobPriority.NORMAL);
  when(mockJobStatus.getNumUsedSlots()).thenReturn(1);
  when(mockJobStatus.getNumReservedSlots()).thenReturn(1);
  when(mockJobStatus.getUsedMem()).thenReturn(1024);
  when(mockJobStatus.getReservedMem()).thenReturn(512);
  when(mockJobStatus.getNeededMem()).thenReturn(2048);
  when(mockJobStatus.getSchedulingInfo()).thenReturn("NA");

  Job mockJob = mock(Job.class);
  when(mockJob.getTaskReports(isA(TaskType.class))).thenReturn(
    new TaskReport[5]);

  Cluster mockCluster = mock(Cluster.class);
  when(mockCluster.getJob(jobID)).thenReturn(mockJob);

  client.setCluster(mockCluster);
  
  ByteArrayOutputStream out = new ByteArrayOutputStream();
  client.displayJobList(new JobStatus[] {mockJobStatus}, new PrintWriter(out));
  String commandLineOutput = out.toString();
  System.out.println(commandLineOutput);
  Assert.assertTrue(commandLineOutput.contains("Total jobs:1"));

  verify(mockJobStatus, atLeastOnce()).getJobID();
  verify(mockJobStatus).getState();
  verify(mockJobStatus).getStartTime();
  verify(mockJobStatus).getUsername();
  verify(mockJobStatus).getQueue();
  verify(mockJobStatus).getPriority();
  verify(mockJobStatus).getNumUsedSlots();
  verify(mockJobStatus).getNumReservedSlots();
  verify(mockJobStatus).getUsedMem();
  verify(mockJobStatus).getReservedMem();
  verify(mockJobStatus).getNeededMem();
  verify(mockJobStatus).getSchedulingInfo();

  // This call should not go to each AM.
  verify(mockCluster, never()).getJob(jobID);
  verify(mockJob, never()).getTaskReports(isA(TaskType.class));
}
 
Example 21
Source Project: big-c   Source File: TestCLI.java    License: Apache License 2.0 4 votes vote down vote up
private TaskReport[] getTaskReports(JobID jobId, TaskType type) {
  return new TaskReport[] { new TaskReport(), new TaskReport() };
}
 
Example 22
Source Project: ignite   Source File: HadoopClientProtocol.java    License: Apache License 2.0 4 votes vote down vote up
/** {@inheritDoc} */
@Override public TaskReport[] getTaskReports(JobID jobid, TaskType type) throws IOException, InterruptedException {
    return new TaskReport[0];
}
 
Example 23
Source Project: RDFS   Source File: MockSimulatorJobTracker.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public TaskReport[] getTaskReports(JobID jobid, TaskType type)
    throws IOException, InterruptedException {
  throw new UnsupportedOperationException();
}
 
Example 24
Source Project: incubator-tez   Source File: ClientServiceDelegate.java    License: Apache License 2.0 4 votes vote down vote up
public TaskReport[] getTaskReports(
    JobID oldJobID, TaskType taskType)
     throws IOException{
  // TEZ-146: need to return real task reports
  return new TaskReport[0];
}
 
Example 25
Source Project: incubator-tez   Source File: YARNRunner.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public TaskReport[] getTaskReports(JobID jobID, TaskType taskType)
throws IOException, InterruptedException {
  return clientCache.getClient(jobID)
      .getTaskReports(jobID, taskType);
}
 
Example 26
Source Project: tez   Source File: ClientServiceDelegate.java    License: Apache License 2.0 4 votes vote down vote up
public TaskReport[] getTaskReports(
    JobID oldJobID, TaskType taskType)
     throws IOException{
  // TEZ-146: need to return real task reports
  return new TaskReport[0];
}
 
Example 27
Source Project: tez   Source File: YARNRunner.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public TaskReport[] getTaskReports(JobID jobID, TaskType taskType)
throws IOException, InterruptedException {
  return clientCache.getClient(jobID)
      .getTaskReports(jobID, taskType);
}
 
Example 28
Source Project: hadoop   Source File: ClientProtocol.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Grab a bunch of info on the tasks that make up the job
 */
public TaskReport[] getTaskReports(JobID jobid, TaskType type)
throws IOException, InterruptedException;
 
Example 29
Source Project: big-c   Source File: ClientProtocol.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Grab a bunch of info on the tasks that make up the job
 */
public TaskReport[] getTaskReports(JobID jobid, TaskType type)
throws IOException, InterruptedException;