org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEvent Java Examples

The following examples show how to use org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEvent. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: TaskImpl.java From big-c with Apache License 2.0

6 votes

private void handleTaskAttemptCompletion(TaskAttemptId attemptId,
    TaskAttemptCompletionEventStatus status) {
  TaskAttempt attempt = attempts.get(attemptId);
  //raise the completion event only if the container is assigned
  // to nextAttemptNumber
  if (attempt.getNodeHttpAddress() != null) {
    TaskAttemptCompletionEvent tce = recordFactory
        .newRecordInstance(TaskAttemptCompletionEvent.class);
    tce.setEventId(-1);
    String scheme = (encryptedShuffle) ? "https://" : "http://";
    tce.setMapOutputServerAddress(StringInterner.weakIntern(scheme
       + attempt.getNodeHttpAddress().split(":")[0] + ":"
       + attempt.getShufflePort()));
    tce.setStatus(status);
    tce.setAttemptId(attempt.getID());
    int runTime = 0;
    if (attempt.getFinishTime() != 0 && attempt.getLaunchTime() !=0)
      runTime = (int)(attempt.getFinishTime() - attempt.getLaunchTime());
    tce.setAttemptRunTime(runTime);
    
    //raise the event to job so that it adds the completion event to its
    //data structures
    eventHandler.handle(new JobTaskAttemptCompletedEvent(tce));
  }
}

Example #2

Source File: JobImpl.java From big-c with Apache License 2.0

6 votes

@Override
public TaskAttemptCompletionEvent[] getTaskAttemptCompletionEvents(
    int fromEventId, int maxEvents) {
  TaskAttemptCompletionEvent[] events = EMPTY_TASK_ATTEMPT_COMPLETION_EVENTS;
  readLock.lock();
  try {
    if (taskAttemptCompletionEvents.size() > fromEventId) {
      int actualMax = Math.min(maxEvents,
          (taskAttemptCompletionEvents.size() - fromEventId));
      events = taskAttemptCompletionEvents.subList(fromEventId,
          actualMax + fromEventId).toArray(events);
    }
    return events;
  } finally {
    readLock.unlock();
  }
}

Example #3

Source File: JobImpl.java From hadoop with Apache License 2.0

6 votes

@Override
public TaskAttemptCompletionEvent[] getTaskAttemptCompletionEvents(
    int fromEventId, int maxEvents) {
  TaskAttemptCompletionEvent[] events = EMPTY_TASK_ATTEMPT_COMPLETION_EVENTS;
  readLock.lock();
  try {
    if (taskAttemptCompletionEvents.size() > fromEventId) {
      int actualMax = Math.min(maxEvents,
          (taskAttemptCompletionEvents.size() - fromEventId));
      events = taskAttemptCompletionEvents.subList(fromEventId,
          actualMax + fromEventId).toArray(events);
    }
    return events;
  } finally {
    readLock.unlock();
  }
}

Example #4

Source File: TaskImpl.java From hadoop with Apache License 2.0

6 votes

private void handleTaskAttemptCompletion(TaskAttemptId attemptId,
    TaskAttemptCompletionEventStatus status) {
  TaskAttempt attempt = attempts.get(attemptId);
  //raise the completion event only if the container is assigned
  // to nextAttemptNumber
  if (attempt.getNodeHttpAddress() != null) {
    TaskAttemptCompletionEvent tce = recordFactory
        .newRecordInstance(TaskAttemptCompletionEvent.class);
    tce.setEventId(-1);
    String scheme = (encryptedShuffle) ? "https://" : "http://";
    tce.setMapOutputServerAddress(StringInterner.weakIntern(scheme
       + attempt.getNodeHttpAddress().split(":")[0] + ":"
       + attempt.getShufflePort()));
    tce.setStatus(status);
    tce.setAttemptId(attempt.getID());
    int runTime = 0;
    if (attempt.getFinishTime() != 0 && attempt.getLaunchTime() !=0)
      runTime = (int)(attempt.getFinishTime() - attempt.getLaunchTime());
    tce.setAttemptRunTime(runTime);
    
    //raise the event to job so that it adds the completion event to its
    //data structures
    eventHandler.handle(new JobTaskAttemptCompletedEvent(tce));
  }
}

Example #5

Source File: GetTaskAttemptCompletionEventsResponsePBImpl.java From big-c with Apache License 2.0

5 votes

private void addCompletionEventsToProto() {
  maybeInitBuilder();
  builder.clearCompletionEvents();
  if (completionEvents == null)
    return;
  Iterable<TaskAttemptCompletionEventProto> iterable = new Iterable<TaskAttemptCompletionEventProto>() {
    @Override
    public Iterator<TaskAttemptCompletionEventProto> iterator() {
      return new Iterator<TaskAttemptCompletionEventProto>() {

        Iterator<TaskAttemptCompletionEvent> iter = completionEvents.iterator();

        @Override
        public boolean hasNext() {
          return iter.hasNext();
        }

        @Override
        public TaskAttemptCompletionEventProto next() {
          return convertToProtoFormat(iter.next());
        }

        @Override
        public void remove() {
          throw new UnsupportedOperationException();

        }
      };

    }
  };
  builder.addAllCompletionEvents(iterable);
}

Example #6

Source File: TestTaskAttemptListenerImpl.java From big-c with Apache License 2.0

5 votes

private static TaskAttemptCompletionEvent createTce(int eventId,
    boolean isMap, TaskAttemptCompletionEventStatus status) {
  JobId jid = MRBuilderUtils.newJobId(12345, 1, 1);
  TaskId tid = MRBuilderUtils.newTaskId(jid, 0,
      isMap ? org.apache.hadoop.mapreduce.v2.api.records.TaskType.MAP
          : org.apache.hadoop.mapreduce.v2.api.records.TaskType.REDUCE);
  TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(tid, 0);
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  TaskAttemptCompletionEvent tce = recordFactory
      .newRecordInstance(TaskAttemptCompletionEvent.class);
  tce.setEventId(eventId);
  tce.setAttemptId(attemptId);
  tce.setStatus(status);
  return tce;
}

Example #7

Source File: NotRunningJob.java From big-c with Apache License 2.0

5 votes

@Override
public GetTaskAttemptCompletionEventsResponse getTaskAttemptCompletionEvents(
    GetTaskAttemptCompletionEventsRequest request)
    throws IOException {
  GetTaskAttemptCompletionEventsResponse resp =
    recordFactory.newRecordInstance(GetTaskAttemptCompletionEventsResponse.class);
  resp.addAllCompletionEvents(new ArrayList<TaskAttemptCompletionEvent>());
  return resp;
}

Example #8

Source File: TypeConverter.java From big-c with Apache License 2.0

5 votes

public static TaskCompletionEvent[] fromYarn(
    TaskAttemptCompletionEvent[] newEvents) {
  TaskCompletionEvent[] oldEvents =
      new TaskCompletionEvent[newEvents.length];
  int i = 0;
  for (TaskAttemptCompletionEvent newEvent
      : newEvents) {
    oldEvents[i++] = fromYarn(newEvent);
  }
  return oldEvents;
}

Example #9

Source File: TypeConverter.java From big-c with Apache License 2.0

5 votes

public static TaskCompletionEvent fromYarn(
    TaskAttemptCompletionEvent newEvent) {
  return new TaskCompletionEvent(newEvent.getEventId(),
            fromYarn(newEvent.getAttemptId()), newEvent.getAttemptId().getId(),
            newEvent.getAttemptId().getTaskId().getTaskType().equals(TaskType.MAP),
            fromYarn(newEvent.getStatus()),
            newEvent.getMapOutputServerAddress());
}

Example #10

Source File: CompletedJob.java From hadoop with Apache License 2.0

5 votes

private static TaskAttemptCompletionEvent[] getAttemptCompletionEvents(
    List<TaskAttemptCompletionEvent> eventList,
    int startIndex, int maxEvents) {
  TaskAttemptCompletionEvent[] events = new TaskAttemptCompletionEvent[0];
  if (eventList.size() > startIndex) {
    int actualMax = Math.min(maxEvents,
        (eventList.size() - startIndex));
    events = eventList.subList(startIndex, actualMax + startIndex)
        .toArray(events);
  }
  return events;
}

Example #11

Source File: CompletedJob.java From hadoop with Apache License 2.0

5 votes

@Override
public synchronized TaskAttemptCompletionEvent[] getTaskAttemptCompletionEvents(
    int fromEventId, int maxEvents) {
  if (completionEvents == null) {
    constructTaskAttemptCompletionEvents();
  }
  return getAttemptCompletionEvents(completionEvents,
      fromEventId, maxEvents);
}

Example #12

Source File: GetTaskAttemptCompletionEventsResponsePBImpl.java From big-c with Apache License 2.0

5 votes

private void initCompletionEvents() {
  if (this.completionEvents != null) {
    return;
  }
  GetTaskAttemptCompletionEventsResponseProtoOrBuilder p = viaProto ? proto : builder;
  List<TaskAttemptCompletionEventProto> list = p.getCompletionEventsList();
  this.completionEvents = new ArrayList<TaskAttemptCompletionEvent>();

  for (TaskAttemptCompletionEventProto c : list) {
    this.completionEvents.add(convertFromProtoFormat(c));
  }
}

Example #13

Source File: GetTaskAttemptCompletionEventsResponsePBImpl.java From hadoop with Apache License 2.0

5 votes

private void addCompletionEventsToProto() {
  maybeInitBuilder();
  builder.clearCompletionEvents();
  if (completionEvents == null)
    return;
  Iterable<TaskAttemptCompletionEventProto> iterable = new Iterable<TaskAttemptCompletionEventProto>() {
    @Override
    public Iterator<TaskAttemptCompletionEventProto> iterator() {
      return new Iterator<TaskAttemptCompletionEventProto>() {

        Iterator<TaskAttemptCompletionEvent> iter = completionEvents.iterator();

        @Override
        public boolean hasNext() {
          return iter.hasNext();
        }

        @Override
        public TaskAttemptCompletionEventProto next() {
          return convertToProtoFormat(iter.next());
        }

        @Override
        public void remove() {
          throw new UnsupportedOperationException();

        }
      };

    }
  };
  builder.addAllCompletionEvents(iterable);
}

Example #14

Source File: GetTaskAttemptCompletionEventsResponsePBImpl.java From hadoop with Apache License 2.0

5 votes

@Override
public void addAllCompletionEvents(final List<TaskAttemptCompletionEvent> completionEvents) {
  if (completionEvents == null)
    return;
  initCompletionEvents();
  this.completionEvents.addAll(completionEvents);
}

Example #15

Source File: GetTaskAttemptCompletionEventsResponsePBImpl.java From hadoop with Apache License 2.0

5 votes

private void initCompletionEvents() {
  if (this.completionEvents != null) {
    return;
  }
  GetTaskAttemptCompletionEventsResponseProtoOrBuilder p = viaProto ? proto : builder;
  List<TaskAttemptCompletionEventProto> list = p.getCompletionEventsList();
  this.completionEvents = new ArrayList<TaskAttemptCompletionEvent>();

  for (TaskAttemptCompletionEventProto c : list) {
    this.completionEvents.add(convertFromProtoFormat(c));
  }
}

Example #16

Source File: GetTaskAttemptCompletionEventsResponsePBImpl.java From big-c with Apache License 2.0

5 votes

@Override
public void addAllCompletionEvents(final List<TaskAttemptCompletionEvent> completionEvents) {
  if (completionEvents == null)
    return;
  initCompletionEvents();
  this.completionEvents.addAll(completionEvents);
}

Example #17

Source File: TypeConverter.java From hadoop with Apache License 2.0

5 votes

public static TaskCompletionEvent[] fromYarn(
    TaskAttemptCompletionEvent[] newEvents) {
  TaskCompletionEvent[] oldEvents =
      new TaskCompletionEvent[newEvents.length];
  int i = 0;
  for (TaskAttemptCompletionEvent newEvent
      : newEvents) {
    oldEvents[i++] = fromYarn(newEvent);
  }
  return oldEvents;
}

Example #18

Source File: NotRunningJob.java From tez with Apache License 2.0

5 votes

@Override
public GetTaskAttemptCompletionEventsResponse getTaskAttemptCompletionEvents(
    GetTaskAttemptCompletionEventsRequest request)
    throws IOException {
  GetTaskAttemptCompletionEventsResponse resp =
    recordFactory.newRecordInstance(GetTaskAttemptCompletionEventsResponse.class);
  resp.addAllCompletionEvents(new ArrayList<TaskAttemptCompletionEvent>());
  return resp;
}

Example #19

Source File: NotRunningJob.java From incubator-tez with Apache License 2.0

5 votes

@Override
public GetTaskAttemptCompletionEventsResponse getTaskAttemptCompletionEvents(
    GetTaskAttemptCompletionEventsRequest request)
    throws IOException {
  GetTaskAttemptCompletionEventsResponse resp =
    recordFactory.newRecordInstance(GetTaskAttemptCompletionEventsResponse.class);
  resp.addAllCompletionEvents(new ArrayList<TaskAttemptCompletionEvent>());
  return resp;
}

Example #20

Source File: TestTaskAttemptListenerImpl.java From hadoop with Apache License 2.0

5 votes

private static TaskAttemptCompletionEvent createTce(int eventId,
    boolean isMap, TaskAttemptCompletionEventStatus status) {
  JobId jid = MRBuilderUtils.newJobId(12345, 1, 1);
  TaskId tid = MRBuilderUtils.newTaskId(jid, 0,
      isMap ? org.apache.hadoop.mapreduce.v2.api.records.TaskType.MAP
          : org.apache.hadoop.mapreduce.v2.api.records.TaskType.REDUCE);
  TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(tid, 0);
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  TaskAttemptCompletionEvent tce = recordFactory
      .newRecordInstance(TaskAttemptCompletionEvent.class);
  tce.setEventId(eventId);
  tce.setAttemptId(attemptId);
  tce.setStatus(status);
  return tce;
}

Example #21

Source File: CompletedJob.java From big-c with Apache License 2.0

5 votes

private static TaskAttemptCompletionEvent[] getAttemptCompletionEvents(
    List<TaskAttemptCompletionEvent> eventList,
    int startIndex, int maxEvents) {
  TaskAttemptCompletionEvent[] events = new TaskAttemptCompletionEvent[0];
  if (eventList.size() > startIndex) {
    int actualMax = Math.min(maxEvents,
        (eventList.size() - startIndex));
    events = eventList.subList(startIndex, actualMax + startIndex)
        .toArray(events);
  }
  return events;
}

Example #22

Source File: NotRunningJob.java From hadoop with Apache License 2.0

5 votes

@Override
public GetTaskAttemptCompletionEventsResponse getTaskAttemptCompletionEvents(
    GetTaskAttemptCompletionEventsRequest request)
    throws IOException {
  GetTaskAttemptCompletionEventsResponse resp =
    recordFactory.newRecordInstance(GetTaskAttemptCompletionEventsResponse.class);
  resp.addAllCompletionEvents(new ArrayList<TaskAttemptCompletionEvent>());
  return resp;
}

Example #23

Source File: TypeConverter.java From hadoop with Apache License 2.0

5 votes

public static TaskCompletionEvent fromYarn(
    TaskAttemptCompletionEvent newEvent) {
  return new TaskCompletionEvent(newEvent.getEventId(),
            fromYarn(newEvent.getAttemptId()), newEvent.getAttemptId().getId(),
            newEvent.getAttemptId().getTaskId().getTaskType().equals(TaskType.MAP),
            fromYarn(newEvent.getStatus()),
            newEvent.getMapOutputServerAddress());
}

Example #24

Source File: CompletedJob.java From big-c with Apache License 2.0

5 votes

@Override
public synchronized TaskAttemptCompletionEvent[] getTaskAttemptCompletionEvents(
    int fromEventId, int maxEvents) {
  if (completionEvents == null) {
    constructTaskAttemptCompletionEvents();
  }
  return getAttemptCompletionEvents(completionEvents,
      fromEventId, maxEvents);
}

Example #25

Source File: GetTaskAttemptCompletionEventsResponsePBImpl.java From big-c with Apache License 2.0

4 votes

@Override
public TaskAttemptCompletionEvent getCompletionEvent(int index) {
  initCompletionEvents();
  return this.completionEvents.get(index);
}

Example #26

Source File: GetTaskAttemptCompletionEventsResponsePBImpl.java From big-c with Apache License 2.0

4 votes

@Override
public List<TaskAttemptCompletionEvent> getCompletionEventList() {
  initCompletionEvents();
  return this.completionEvents;
}

Example #27

Source File: JobTaskAttemptCompletedEvent.java From big-c with Apache License 2.0

4 votes

public JobTaskAttemptCompletedEvent(TaskAttemptCompletionEvent completionEvent) {
  super(completionEvent.getAttemptId().getTaskId().getJobId(), 
      JobEventType.JOB_TASK_ATTEMPT_COMPLETED);
  this.completionEvent = completionEvent;
}

Example #28

Source File: PartialJob.java From big-c with Apache License 2.0

4 votes

@Override
public TaskAttemptCompletionEvent[] getTaskAttemptCompletionEvents(
    int fromEventId, int maxEvents) {
  return null;
}

Example #29

Source File: TestHsWebServicesAcls.java From big-c with Apache License 2.0

4 votes

@Override
public TaskAttemptCompletionEvent[] getTaskAttemptCompletionEvents(
    int fromEventId, int maxEvents) {
  return mockJob.getTaskAttemptCompletionEvents(fromEventId, maxEvents);
}

Example #30

Source File: TestRuntimeEstimators.java From big-c with Apache License 2.0

4 votes

@Override
public TaskAttemptCompletionEvent[]
        getTaskAttemptCompletionEvents(int fromEventId, int maxEvents) {
  throw new UnsupportedOperationException("Not supported yet.");
}