org.apache.hadoop.mapreduce.TypeConverter Java Examples

The following examples show how to use org.apache.hadoop.mapreduce.TypeConverter. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source Project: big-c   Author: yncxcw   File: TestClientServiceDelegate.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testRemoteExceptionFromHistoryServer() throws Exception {

  MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
  when(historyServerProxy.getJobReport(getJobReportRequest())).thenThrow(
      new IOException("Job ID doesnot Exist"));

  ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
  when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId()))
      .thenReturn(null);

  ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(
      historyServerProxy, rm);

  try {
    clientServiceDelegate.getJobStatus(oldJobId);
    Assert.fail("Invoke should throw exception after retries.");
  } catch (IOException e) {
    Assert.assertTrue(e.getMessage().contains(
        "Job ID doesnot Exist"));
  }
}
 
Example #2
Source Project: incubator-tez   Author: apache   File: YARNRunner.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public JobStatus getJobStatus(JobID jobID) throws IOException,
    InterruptedException {
  String user = UserGroupInformation.getCurrentUser().getShortUserName();
  String jobFile = MRApps.getJobFile(conf, user, jobID);
  DAGStatus dagStatus;
  try {
    if(dagClient == null) {
      dagClient = TezClient.getDAGClient(TypeConverter.toYarn(jobID).getAppId(), tezConf);
    }
    dagStatus = dagClient.getDAGStatus(null);
    return new DAGJobStatus(dagClient.getApplicationReport(), dagStatus, jobFile);
  } catch (TezException e) {
    throw new IOException(e);
  }
}
 
Example #3
Source Project: big-c   Author: yncxcw   File: TestJobImpl.java    License: Apache License 2.0 6 votes vote down vote up
private static StubbedJob createStubbedJob(Configuration conf,
    Dispatcher dispatcher, int numSplits, AppContext appContext) {
  JobID jobID = JobID.forName("job_1234567890000_0001");
  JobId jobId = TypeConverter.toYarn(jobID);
  if (appContext == null) {
    appContext = mock(AppContext.class);
    when(appContext.hasSuccessfullyUnregistered()).thenReturn(true);
  }
  StubbedJob job = new StubbedJob(jobId,
      ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 0), 0),
      conf,dispatcher.getEventHandler(), true, "somebody", numSplits, appContext);
  dispatcher.register(JobEventType.class, job);
  EventHandler mockHandler = mock(EventHandler.class);
  dispatcher.register(TaskEventType.class, mockHandler);
  dispatcher.register(org.apache.hadoop.mapreduce.jobhistory.EventType.class,
      mockHandler);
  dispatcher.register(JobFinishEvent.Type.class, mockHandler);
  return job;
}
 
Example #4
Source Project: hadoop   Author: naver   File: CommitterEventHandler.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void serviceInit(Configuration conf) throws Exception {
  super.serviceInit(conf);
  commitThreadCancelTimeoutMs = conf.getInt(
      MRJobConfig.MR_AM_COMMITTER_CANCEL_TIMEOUT_MS,
      MRJobConfig.DEFAULT_MR_AM_COMMITTER_CANCEL_TIMEOUT_MS);
  commitWindowMs = conf.getLong(MRJobConfig.MR_AM_COMMIT_WINDOW_MS,
      MRJobConfig.DEFAULT_MR_AM_COMMIT_WINDOW_MS);
  try {
    fs = FileSystem.get(conf);
    JobID id = TypeConverter.fromYarn(context.getApplicationID());
    JobId jobId = TypeConverter.toYarn(id);
    String user = UserGroupInformation.getCurrentUser().getShortUserName();
    startCommitFile = MRApps.getStartJobCommitFile(conf, user, jobId);
    endCommitSuccessFile = MRApps.getEndJobCommitSuccessFile(conf, user, jobId);
    endCommitFailureFile = MRApps.getEndJobCommitFailureFile(conf, user, jobId);
  } catch (IOException e) {
    throw new YarnRuntimeException(e);
  }
}
 
Example #5
Source Project: hadoop   Author: naver   File: TaskAttemptImpl.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
private void sendLaunchedEvents() {
  JobCounterUpdateEvent jce = new JobCounterUpdateEvent(attemptId.getTaskId()
      .getJobId());
  jce.addCounterUpdate(attemptId.getTaskId().getTaskType() == TaskType.MAP ?
      JobCounter.TOTAL_LAUNCHED_MAPS : JobCounter.TOTAL_LAUNCHED_REDUCES, 1);
  eventHandler.handle(jce);

  LOG.info("TaskAttempt: [" + attemptId
      + "] using containerId: [" + container.getId() + " on NM: ["
      + StringInterner.weakIntern(container.getNodeId().toString()) + "]");
  TaskAttemptStartedEvent tase =
    new TaskAttemptStartedEvent(TypeConverter.fromYarn(attemptId),
        TypeConverter.fromYarn(attemptId.getTaskId().getTaskType()),
        launchTime, trackerName, httpPort, shufflePort, container.getId(),
        locality.toString(), avataar.toString());
  eventHandler.handle(
      new JobHistoryEvent(attemptId.getTaskId().getJobId(), tase));
}
 
Example #6
Source Project: tez   Author: apache   File: YARNRunner.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public JobStatus getJobStatus(JobID jobID) throws IOException,
    InterruptedException {
  String user = UserGroupInformation.getCurrentUser().getShortUserName();
  String jobFile = MRApps.getJobFile(conf, user, jobID);
  DAGStatus dagStatus;
  try {
    if(dagClient == null) {
      dagClient = MRTezClient.getDAGClient(TypeConverter.toYarn(jobID).getAppId(), tezConf, null);
    }
    dagStatus = dagClient.getDAGStatus(null);
    return new DAGJobStatus(dagClient.getApplicationReport(), dagStatus, jobFile);
  } catch (TezException e) {
    throw new IOException(e);
  }
}
 
Example #7
Source Project: big-c   Author: yncxcw   File: TaskAttemptListenerImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Override
 public void reportDiagnosticInfo(TaskAttemptID taskAttemptID, String diagnosticInfo)
throws IOException {
   diagnosticInfo = StringInterner.weakIntern(diagnosticInfo);
   LOG.info("Diagnostics report from " + taskAttemptID.toString() + ": "
       + diagnosticInfo);

   org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId attemptID =
     TypeConverter.toYarn(taskAttemptID);
   taskHeartbeatHandler.progressing(attemptID);

   // This is mainly used for cases where we want to propagate exception traces
   // of tasks that fail.

   // This call exists as a hadoop mapreduce legacy wherein all changes in
   // counters/progress/phase/output-size are reported through statusUpdate()
   // call but not diagnosticInformation.
   context.getEventHandler().handle(
       new TaskAttemptDiagnosticsUpdateEvent(attemptID, diagnosticInfo));
 }
 
Example #8
Source Project: hadoop   Author: naver   File: TestClientServiceDelegate.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testJobReportFromHistoryServer() throws Exception {                                 
  MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);                           
  when(historyServerProxy.getJobReport(getJobReportRequest())).thenReturn(                      
      getJobReportResponseFromHistoryServer());                                                 
  ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);                                     
  when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId()))                      
  .thenReturn(null);                                                                        
  ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(                       
      historyServerProxy, rm);

  JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
  Assert.assertNotNull(jobStatus);
  Assert.assertEquals("TestJobFilePath", jobStatus.getJobFile());                               
  Assert.assertEquals("http://TestTrackingUrl", jobStatus.getTrackingUrl());                    
  Assert.assertEquals(1.0f, jobStatus.getMapProgress(), 0.0f);
  Assert.assertEquals(1.0f, jobStatus.getReduceProgress(), 0.0f);
}
 
Example #9
Source Project: hadoop   Author: naver   File: TestJobImpl.java    License: Apache License 2.0 6 votes vote down vote up
private static StubbedJob createStubbedJob(Configuration conf,
    Dispatcher dispatcher, int numSplits, AppContext appContext) {
  JobID jobID = JobID.forName("job_1234567890000_0001");
  JobId jobId = TypeConverter.toYarn(jobID);
  if (appContext == null) {
    appContext = mock(AppContext.class);
    when(appContext.hasSuccessfullyUnregistered()).thenReturn(true);
  }
  StubbedJob job = new StubbedJob(jobId,
      ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 0), 0),
      conf,dispatcher.getEventHandler(), true, "somebody", numSplits, appContext);
  dispatcher.register(JobEventType.class, job);
  EventHandler mockHandler = mock(EventHandler.class);
  dispatcher.register(TaskEventType.class, mockHandler);
  dispatcher.register(org.apache.hadoop.mapreduce.jobhistory.EventType.class,
      mockHandler);
  dispatcher.register(JobFinishEvent.Type.class, mockHandler);
  return job;
}
 
Example #10
Source Project: big-c   Author: yncxcw   File: CommitterEventHandler.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void serviceInit(Configuration conf) throws Exception {
  super.serviceInit(conf);
  commitThreadCancelTimeoutMs = conf.getInt(
      MRJobConfig.MR_AM_COMMITTER_CANCEL_TIMEOUT_MS,
      MRJobConfig.DEFAULT_MR_AM_COMMITTER_CANCEL_TIMEOUT_MS);
  commitWindowMs = conf.getLong(MRJobConfig.MR_AM_COMMIT_WINDOW_MS,
      MRJobConfig.DEFAULT_MR_AM_COMMIT_WINDOW_MS);
  try {
    fs = FileSystem.get(conf);
    JobID id = TypeConverter.fromYarn(context.getApplicationID());
    JobId jobId = TypeConverter.toYarn(id);
    String user = UserGroupInformation.getCurrentUser().getShortUserName();
    startCommitFile = MRApps.getStartJobCommitFile(conf, user, jobId);
    endCommitSuccessFile = MRApps.getEndJobCommitSuccessFile(conf, user, jobId);
    endCommitFailureFile = MRApps.getEndJobCommitFailureFile(conf, user, jobId);
  } catch (IOException e) {
    throw new YarnRuntimeException(e);
  }
}
 
Example #11
Source Project: big-c   Author: yncxcw   File: CompletedJob.java    License: Apache License 2.0 6 votes vote down vote up
private void loadAllTasks() {
  if (tasksLoaded.get()) {
    return;
  }
  tasksLock.lock();
  try {
    if (tasksLoaded.get()) {
      return;
    }
    for (Map.Entry<TaskID, TaskInfo> entry : jobInfo.getAllTasks().entrySet()) {
      TaskId yarnTaskID = TypeConverter.toYarn(entry.getKey());
      TaskInfo taskInfo = entry.getValue();
      Task task = new CompletedTask(yarnTaskID, taskInfo);
      tasks.put(yarnTaskID, task);
      if (task.getType() == TaskType.MAP) {
        mapTasks.put(task.getID(), task);
      } else if (task.getType() == TaskType.REDUCE) {
        reduceTasks.put(task.getID(), task);
      }
    }
    tasksLoaded.set(true);
  } finally {
    tasksLock.unlock();
  }
}
 
Example #12
Source Project: hadoop   Author: naver   File: ClientServiceDelegate.java    License: Apache License 2.0 6 votes vote down vote up
public JobStatus getJobStatus(JobID oldJobID) throws IOException {
  org.apache.hadoop.mapreduce.v2.api.records.JobId jobId =
    TypeConverter.toYarn(oldJobID);
  GetJobReportRequest request =
      recordFactory.newRecordInstance(GetJobReportRequest.class);
  request.setJobId(jobId);
  JobReport report = ((GetJobReportResponse) invoke("getJobReport",
      GetJobReportRequest.class, request)).getJobReport();
  JobStatus jobStatus = null;
  if (report != null) {
    if (StringUtils.isEmpty(report.getJobFile())) {
      String jobFile = MRApps.getJobFile(conf, report.getUser(), oldJobID);
      report.setJobFile(jobFile);
    }
    String historyTrackingUrl = report.getTrackingUrl();
    String url = StringUtils.isNotEmpty(historyTrackingUrl)
        ? historyTrackingUrl : trackingUrl;
    jobStatus = TypeConverter.fromYarn(report, url);
  }
  return jobStatus;
}
 
Example #13
Source Project: hadoop   Author: naver   File: ClientServiceDelegate.java    License: Apache License 2.0 6 votes vote down vote up
public org.apache.hadoop.mapreduce.TaskReport[] getTaskReports(JobID oldJobID, TaskType taskType)
     throws IOException{
  org.apache.hadoop.mapreduce.v2.api.records.JobId jobId =
    TypeConverter.toYarn(oldJobID);
  GetTaskReportsRequest request =
      recordFactory.newRecordInstance(GetTaskReportsRequest.class);
  request.setJobId(jobId);
  request.setTaskType(TypeConverter.toYarn(taskType));

  List<org.apache.hadoop.mapreduce.v2.api.records.TaskReport> taskReports =
    ((GetTaskReportsResponse) invoke("getTaskReports", GetTaskReportsRequest.class,
        request)).getTaskReportList();

  return TypeConverter.fromYarn
  (taskReports).toArray(new org.apache.hadoop.mapreduce.TaskReport[0]);
}
 
Example #14
Source Project: hadoop   Author: naver   File: TestFileNameIndexUtils.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testQueueNamePercentEncoding() throws IOException {
  JobIndexInfo info = new JobIndexInfo();
  JobID oldJobId = JobID.forName(JOB_ID);
  JobId jobId = TypeConverter.toYarn(oldJobId);
  info.setJobId(jobId);
  info.setSubmitTime(Long.parseLong(SUBMIT_TIME));
  info.setUser(USER_NAME);
  info.setJobName(JOB_NAME);
  info.setFinishTime(Long.parseLong(FINISH_TIME));
  info.setNumMaps(Integer.parseInt(NUM_MAPS));
  info.setNumReduces(Integer.parseInt(NUM_REDUCES));
  info.setJobStatus(JOB_STATUS);
  info.setQueueName(QUEUE_NAME_WITH_DELIMITER);
  info.setJobStartTime(Long.parseLong(JOB_START_TIME));

  String jobHistoryFile = FileNameIndexUtils.getDoneFileName(info);
  Assert.assertTrue("Queue name not encoded correctly into job history file",
      jobHistoryFile.contains(QUEUE_NAME_WITH_DELIMITER_ESCAPE));
}
 
Example #15
Source Project: big-c   Author: yncxcw   File: TestClientServiceDelegate.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testJobReportFromHistoryServer() throws Exception {                                 
  MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);                           
  when(historyServerProxy.getJobReport(getJobReportRequest())).thenReturn(                      
      getJobReportResponseFromHistoryServer());                                                 
  ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);                                     
  when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId()))                      
  .thenReturn(null);                                                                        
  ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(                       
      historyServerProxy, rm);

  JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
  Assert.assertNotNull(jobStatus);
  Assert.assertEquals("TestJobFilePath", jobStatus.getJobFile());                               
  Assert.assertEquals("http://TestTrackingUrl", jobStatus.getTrackingUrl());                    
  Assert.assertEquals(1.0f, jobStatus.getMapProgress(), 0.0f);
  Assert.assertEquals(1.0f, jobStatus.getReduceProgress(), 0.0f);
}
 
Example #16
Source Project: big-c   Author: yncxcw   File: TaskAttemptImpl.java    License: Apache License 2.0 6 votes vote down vote up
private static
    TaskAttemptUnsuccessfulCompletionEvent
    createTaskAttemptUnsuccessfulCompletionEvent(TaskAttemptImpl taskAttempt,
        TaskAttemptStateInternal attemptState) {
  TaskAttemptUnsuccessfulCompletionEvent tauce =
      new TaskAttemptUnsuccessfulCompletionEvent(
          TypeConverter.fromYarn(taskAttempt.attemptId),
          TypeConverter.fromYarn(taskAttempt.attemptId.getTaskId()
              .getTaskType()), attemptState.toString(),
          taskAttempt.finishTime,
          taskAttempt.container == null ? "UNKNOWN"
              : taskAttempt.container.getNodeId().getHost(),
          taskAttempt.container == null ? -1 
              : taskAttempt.container.getNodeId().getPort(),    
          taskAttempt.nodeRackName == null ? "UNKNOWN" 
              : taskAttempt.nodeRackName,
          StringUtils.join(
              LINE_SEPARATOR, taskAttempt.getDiagnostics()),
              taskAttempt.getCounters(), taskAttempt
              .getProgressSplitBlock().burst());
  return tauce;
}
 
Example #17
Source Project: hadoop   Author: naver   File: MRWebAppUtil.java    License: Apache License 2.0 5 votes vote down vote up
public static String getApplicationWebURLOnJHSWithoutScheme(Configuration conf,
    ApplicationId appId)
    throws UnknownHostException {
  //construct the history url for job
  String addr = getJHSWebappURLWithoutScheme(conf);
  Iterator<String> it = ADDR_SPLITTER.split(addr).iterator();
  it.next(); // ignore the bind host
  String port = it.next();
  // Use hs address to figure out the host for webapp
  addr = conf.get(JHAdminConfig.MR_HISTORY_ADDRESS,
      JHAdminConfig.DEFAULT_MR_HISTORY_ADDRESS);
  String host = ADDR_SPLITTER.split(addr).iterator().next();
  String hsAddress = JOINER.join(host, ":", port);
  InetSocketAddress address = NetUtils.createSocketAddr(
    hsAddress, getDefaultJHSWebappPort(),
    getDefaultJHSWebappURLWithoutScheme());
  StringBuffer sb = new StringBuffer();
  if (address.getAddress().isAnyLocalAddress() || 
      address.getAddress().isLoopbackAddress()) {
    sb.append(InetAddress.getLocalHost().getCanonicalHostName());
  } else {
    sb.append(address.getHostName());
  }
  sb.append(":").append(address.getPort());
  sb.append("/jobhistory/job/");
  JobID jobId = TypeConverter.fromYarn(appId);
  sb.append(jobId.toString());
  return sb.toString();
}
 
Example #18
Source Project: big-c   Author: yncxcw   File: TestRecovery.java    License: Apache License 2.0 5 votes vote down vote up
private void writeBadOutput(TaskAttempt attempt, Configuration conf)
  throws Exception {
  TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, 
      TypeConverter.fromYarn(attempt.getID()));
 
  TextOutputFormat<?, ?> theOutputFormat = new TextOutputFormat();
  RecordWriter theRecordWriter = theOutputFormat
      .getRecordWriter(tContext);
  
  NullWritable nullWritable = NullWritable.get();
  try {
    theRecordWriter.write(key2, val2);
    theRecordWriter.write(null, nullWritable);
    theRecordWriter.write(null, val2);
    theRecordWriter.write(nullWritable, val1);
    theRecordWriter.write(key1, nullWritable);
    theRecordWriter.write(key2, null);
    theRecordWriter.write(null, null);
    theRecordWriter.write(key1, val1);
  } finally {
    theRecordWriter.close(tContext);
  }
  
  OutputFormat outputFormat = ReflectionUtils.newInstance(
      tContext.getOutputFormatClass(), conf);
  OutputCommitter committer = outputFormat.getOutputCommitter(tContext);
  committer.commitTask(tContext);
}
 
Example #19
Source Project: big-c   Author: yncxcw   File: TaskImpl.java    License: Apache License 2.0 5 votes vote down vote up
private void sendTaskStartedEvent() {
  TaskStartedEvent tse = new TaskStartedEvent(
      TypeConverter.fromYarn(taskId), getLaunchTime(),
      TypeConverter.fromYarn(taskId.getTaskType()),
      getSplitsAsString());
  eventHandler
      .handle(new JobHistoryEvent(taskId.getJobId(), tse));
  historyTaskStartGenerated = true;
}
 
Example #20
Source Project: big-c   Author: yncxcw   File: ResourceMgrDelegate.java    License: Apache License 2.0 5 votes vote down vote up
public QueueInfo getQueue(String queueName) throws IOException,
InterruptedException {
  try {
    org.apache.hadoop.yarn.api.records.QueueInfo queueInfo =
        client.getQueueInfo(queueName);
    return (queueInfo == null) ? null : TypeConverter.fromYarn(queueInfo,
        conf);
  } catch (YarnException e) {
    throw new IOException(e);
  }
}
 
Example #21
Source Project: big-c   Author: yncxcw   File: TestYARNRunner.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() throws Exception {
  resourceMgrDelegate = mock(ResourceMgrDelegate.class);
  conf = new YarnConfiguration();
  conf.set(YarnConfiguration.RM_PRINCIPAL, "mapred/[email protected]");
  clientCache = new ClientCache(conf, resourceMgrDelegate);
  clientCache = spy(clientCache);
  yarnRunner = new YARNRunner(conf, resourceMgrDelegate, clientCache);
  yarnRunner = spy(yarnRunner);
  submissionContext = mock(ApplicationSubmissionContext.class);
  doAnswer(
      new Answer<ApplicationSubmissionContext>() {
        @Override
        public ApplicationSubmissionContext answer(InvocationOnMock invocation)
            throws Throwable {
          return submissionContext;
        }
      }
      ).when(yarnRunner).createApplicationSubmissionContext(any(Configuration.class),
          any(String.class), any(Credentials.class));

  appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
  jobId = TypeConverter.fromYarn(appId);
  if (testWorkDir.exists()) {
    FileContext.getLocalFSFileContext().delete(new Path(testWorkDir.toString()), true);
  }
  testWorkDir.mkdirs();
}
 
Example #22
Source Project: hadoop   Author: naver   File: TaskAttemptListenerImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void fatalError(TaskAttemptID taskAttemptID, String msg)
    throws IOException {
  // This happens only in Child and in the Task.
  LOG.fatal("Task: " + taskAttemptID + " - exited : " + msg);
  reportDiagnosticInfo(taskAttemptID, "Error: " + msg);

  org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId attemptID =
      TypeConverter.toYarn(taskAttemptID);
  context.getEventHandler().handle(
      new TaskAttemptEvent(attemptID, TaskAttemptEventType.TA_FAILMSG));
}
 
Example #23
Source Project: hadoop   Author: naver   File: TaskAttemptListenerImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void fsError(TaskAttemptID taskAttemptID, String message)
    throws IOException {
  // This happens only in Child.
  LOG.fatal("Task: " + taskAttemptID + " - failed due to FSError: "
      + message);
  reportDiagnosticInfo(taskAttemptID, "FSError: " + message);

  org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId attemptID =
      TypeConverter.toYarn(taskAttemptID);
  context.getEventHandler().handle(
      new TaskAttemptEvent(attemptID, TaskAttemptEventType.TA_FAILMSG));
}
 
Example #24
Source Project: incubator-tez   Author: apache   File: ResourceMgrDelegate.java    License: Apache License 2.0 5 votes vote down vote up
public JobID getNewJobID() throws IOException, InterruptedException {
  try {
    this.application = 
        client.createApplication().getNewApplicationResponse();
  } catch (YarnException e) {
    throw new IOException(e);
  }
  this.applicationId = this.application.getApplicationId();
  return TypeConverter.fromYarn(applicationId);
}
 
Example #25
Source Project: hadoop   Author: naver   File: TaskAttemptImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public TaskAttemptReport getReport() {
  TaskAttemptReport result = recordFactory.newRecordInstance(TaskAttemptReport.class);
  readLock.lock();
  try {
    result.setTaskAttemptId(attemptId);
    //take the LOCAL state of attempt
    //DO NOT take from reportedStatus
    
    result.setTaskAttemptState(getState());
    result.setProgress(reportedStatus.progress);
    result.setStartTime(launchTime);
    result.setFinishTime(finishTime);
    result.setShuffleFinishTime(this.reportedStatus.shuffleFinishTime);
    result.setDiagnosticInfo(StringUtils.join(LINE_SEPARATOR, getDiagnostics()));
    result.setPhase(reportedStatus.phase);
    result.setStateString(reportedStatus.stateString);
    result.setCounters(TypeConverter.toYarn(getCounters()));
    result.setContainerId(this.getAssignedContainerID());
    result.setNodeManagerHost(trackerName);
    result.setNodeManagerHttpPort(httpPort);
    if (this.container != null) {
      result.setNodeManagerPort(this.container.getNodeId().getPort());
    }
    return result;
  } finally {
    readLock.unlock();
  }
}
 
Example #26
Source Project: hadoop   Author: naver   File: TaskAttemptImpl.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
public void transition(TaskAttemptImpl taskAttempt, 
    TaskAttemptEvent event) {
  TaskAttemptContext taskContext =
    new TaskAttemptContextImpl(taskAttempt.conf,
        TypeConverter.fromYarn(taskAttempt.attemptId));
  taskAttempt.eventHandler.handle(new CommitterTaskAbortEvent(
      taskAttempt.attemptId, taskContext));
}
 
Example #27
Source Project: hadoop   Author: naver   File: HistoryClientService.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public GetCountersResponse getCounters(GetCountersRequest request)
    throws IOException {
  JobId jobId = request.getJobId();
  Job job = verifyAndGetJob(jobId, true);
  GetCountersResponse response = recordFactory.newRecordInstance(GetCountersResponse.class);
  response.setCounters(TypeConverter.toYarn(job.getAllCounters()));
  return response;
}
 
Example #28
Source Project: big-c   Author: yncxcw   File: ResourceMgrDelegate.java    License: Apache License 2.0 5 votes vote down vote up
public JobID getNewJobID() throws IOException, InterruptedException {
  try {
    this.application = client.createApplication().getApplicationSubmissionContext();
    this.applicationId = this.application.getApplicationId();
    return TypeConverter.fromYarn(applicationId);
  } catch (YarnException e) {
    throw new IOException(e);
  }
}
 
Example #29
Source Project: hadoop   Author: naver   File: MRClientService.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public GetCountersResponse getCounters(GetCountersRequest request) 
  throws IOException {
  JobId jobId = request.getJobId();
  Job job = verifyAndGetJob(jobId, JobACL.VIEW_JOB, true);
  GetCountersResponse response =
    recordFactory.newRecordInstance(GetCountersResponse.class);
  response.setCounters(TypeConverter.toYarn(job.getAllCounters()));
  return response;
}
 
Example #30
Source Project: incubator-tez   Author: apache   File: ResourceMgrDelegate.java    License: Apache License 2.0 5 votes vote down vote up
public QueueInfo[] getQueues() throws IOException, InterruptedException {
  try {
    return TypeConverter.fromYarnQueueInfo(client.getAllQueues(), this.conf);
  } catch (YarnException e) {
    throw new IOException(e);
  }
}