Java Code Examples for org.apache.hadoop.yarn.api.records.timeline.TimelineEntity#addEvent()

private void publishApplicationCreatedEvent(ApplicationCreatedEvent event) {
  TimelineEntity entity =
      createApplicationEntity(event.getApplicationId());
  Map<String, Object> entityInfo = new HashMap<String, Object>();
  entityInfo.put(ApplicationMetricsConstants.NAME_ENTITY_INFO,
      event.getApplicationName());
  entityInfo.put(ApplicationMetricsConstants.TYPE_ENTITY_INFO,
      event.getApplicationType());
  entityInfo.put(ApplicationMetricsConstants.USER_ENTITY_INFO,
      event.getUser());
  entityInfo.put(ApplicationMetricsConstants.QUEUE_ENTITY_INFO,
      event.getQueue());
  entityInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO,
      event.getSubmittedTime());
  entity.setOtherInfo(entityInfo);
  TimelineEvent tEvent = new TimelineEvent();
  tEvent.setEventType(
      ApplicationMetricsConstants.CREATED_EVENT_TYPE);
  tEvent.setTimestamp(event.getTimestamp());
  entity.addEvent(tEvent);
  putEntity(entity);
}
 

private static TimelineEntity convertTaskFinishedEvent(TaskFinishedEvent event) {
  TimelineEntity atsEntity = new TimelineEntity();
  atsEntity.setEntityId(event.getTaskID().toString());
  atsEntity.setEntityType(EntityTypes.TEZ_TASK_ID.name());

  atsEntity.addPrimaryFilter(EntityTypes.TEZ_DAG_ID.name(),
      event.getTaskID().getVertexID().getDAGId().toString());
  atsEntity.addPrimaryFilter(EntityTypes.TEZ_VERTEX_ID.name(),
      event.getTaskID().getVertexID().toString());

  TimelineEvent finishEvt = new TimelineEvent();
  finishEvt.setEventType(HistoryEventType.TASK_FINISHED.name());
  finishEvt.setTimestamp(event.getFinishTime());
  atsEntity.addEvent(finishEvt);

  atsEntity.addOtherInfo(ATSConstants.FINISH_TIME, event.getFinishTime());
  atsEntity.addOtherInfo(ATSConstants.TIME_TAKEN, (event.getFinishTime() - event.getStartTime()));
  atsEntity.addOtherInfo(ATSConstants.STATUS, event.getState().name());

  atsEntity.addOtherInfo(ATSConstants.DIAGNOSTICS, event.getDiagnostics());
  atsEntity.addOtherInfo(ATSConstants.COUNTERS,
      DAGUtils.convertCountersToATSMap(event.getTezCounters()));

  return atsEntity;
}
 

private void publishAppAttemptFinishedEvent(AppAttemptFinishedEvent event) {
  TimelineEntity entity =
      createAppAttemptEntity(event.getApplicationAttemptId());
  TimelineEvent tEvent = new TimelineEvent();
  tEvent.setEventType(AppAttemptMetricsConstants.FINISHED_EVENT_TYPE);
  tEvent.setTimestamp(event.getTimestamp());
  Map<String, Object> eventInfo = new HashMap<String, Object>();
  eventInfo.put(
      AppAttemptMetricsConstants.TRACKING_URL_EVENT_INFO,
      event.getTrackingUrl());
  eventInfo.put(
      AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_EVENT_INFO,
      event.getOriginalTrackingURL());
  eventInfo.put(AppAttemptMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO,
      event.getDiagnosticsInfo());
  eventInfo.put(AppAttemptMetricsConstants.FINAL_STATUS_EVENT_INFO,
      event.getFinalApplicationStatus().toString());
  eventInfo.put(AppAttemptMetricsConstants.STATE_EVENT_INFO,
      event.getYarnApplicationAttemptState().toString());
  tEvent.setEventInfo(eventInfo);
  entity.addEvent(tEvent);
  putEntity(entity);
}
 

private void
    publishAppAttemptRegisteredEvent(AppAttemptRegisteredEvent event) {
  TimelineEntity entity =
      createAppAttemptEntity(event.getApplicationAttemptId());
  TimelineEvent tEvent = new TimelineEvent();
  tEvent.setEventType(
      AppAttemptMetricsConstants.REGISTERED_EVENT_TYPE);
  tEvent.setTimestamp(event.getTimestamp());
  Map<String, Object> eventInfo = new HashMap<String, Object>();
  eventInfo.put(
      AppAttemptMetricsConstants.TRACKING_URL_EVENT_INFO,
      event.getTrackingUrl());
  eventInfo.put(
      AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_EVENT_INFO,
      event.getOriginalTrackingURL());
  eventInfo.put(AppAttemptMetricsConstants.HOST_EVENT_INFO,
      event.getHost());
  eventInfo.put(AppAttemptMetricsConstants.RPC_PORT_EVENT_INFO,
      event.getRpcPort());
  eventInfo.put(
      AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO,
      event.getMasterContainerId().toString());
  tEvent.setEventInfo(eventInfo);
  entity.addEvent(tEvent);
  putEntity(entity);
}
 

private static TimelineEntity generateEntity() {
  TimelineEntity entity = new TimelineEntity();
  entity.setEntityId("entity id");
  entity.setEntityType("entity type");
  entity.setStartTime(System.currentTimeMillis());
  for (int i = 0; i < 2; ++i) {
    TimelineEvent event = new TimelineEvent();
    event.setTimestamp(System.currentTimeMillis());
    event.setEventType("test event type " + i);
    event.addEventInfo("key1", "val1");
    event.addEventInfo("key2", "val2");
    entity.addEvent(event);
  }
  entity.addRelatedEntity("test ref type 1", "test ref id 1");
  entity.addRelatedEntity("test ref type 2", "test ref id 2");
  entity.addPrimaryFilter("pkey1", "pval1");
  entity.addPrimaryFilter("pkey2", "pval2");
  entity.addOtherInfo("okey1", "oval1");
  entity.addOtherInfo("okey2", "oval2");
  entity.setDomainId("domain id 1");
  return entity;
}
 

private static void publishApplicationAttemptEvent(
    final TimelineClient timelineClient, String appAttemptId,
    DSEvent appEvent, String domainId, UserGroupInformation ugi) {
  final TimelineEntity entity = new TimelineEntity();
  entity.setEntityId(appAttemptId);
  entity.setEntityType(DSEntity.DS_APP_ATTEMPT.toString());
  entity.setDomainId(domainId);
  entity.addPrimaryFilter("user", ugi.getShortUserName());
  TimelineEvent event = new TimelineEvent();
  event.setEventType(appEvent.toString());
  event.setTimestamp(System.currentTimeMillis());
  entity.addEvent(event);
  try {
    timelineClient.putEntities(entity);
  } catch (YarnException | IOException e) {
    LOG.error("App Attempt "
        + (appEvent.equals(DSEvent.DS_APP_ATTEMPT_START) ? "start" : "end")
        + " event could not be published for "
        + appAttemptId.toString(), e);
  }
}
 

private static TimelineEntity convertDAGSubmittedToDAGExtraInfoEntity(DAGSubmittedEvent event) {
  TimelineEntity atsEntity = new TimelineEntity();
  atsEntity.setEntityId(event.getDagID().toString());
  atsEntity.setEntityType(EntityTypes.TEZ_DAG_EXTRA_INFO.name());

  atsEntity.addRelatedEntity(EntityTypes.TEZ_DAG_ID.name(), event.getDagID().toString());

  TimelineEvent submitEvt = new TimelineEvent();
  submitEvt.setEventType(HistoryEventType.DAG_SUBMITTED.name());
  submitEvt.setTimestamp(event.getSubmitTime());
  atsEntity.addEvent(submitEvt);

  atsEntity.setStartTime(event.getSubmitTime());

  try {
    atsEntity.addOtherInfo(ATSConstants.DAG_PLAN,
        DAGUtils.convertDAGPlanToATSMap(event.getDAGPlan()));
  } catch (IOException e) {
    throw new TezUncheckedException(e);
  }
  return atsEntity;
}
 

private static void publishContainerEndEvent(
    final TimelineClient timelineClient, ContainerStatus container,
    String domainId, UserGroupInformation ugi) {
  final TimelineEntity entity = new TimelineEntity();
  entity.setEntityId(container.getContainerId().toString());
  entity.setEntityType(DSEntity.DS_CONTAINER.toString());
  entity.setDomainId(domainId);
  entity.addPrimaryFilter("user", ugi.getShortUserName());
  TimelineEvent event = new TimelineEvent();
  event.setTimestamp(System.currentTimeMillis());
  event.setEventType(DSEvent.DS_CONTAINER_END.toString());
  event.addEventInfo("State", container.getState().name());
  event.addEventInfo("Exit Status", container.getExitStatus());
  entity.addEvent(event);
  try {
    timelineClient.putEntities(entity);
  } catch (YarnException | IOException e) {
    LOG.error("Container end event could not be published for "
        + container.getContainerId().toString(), e);
  }
}
 

private void publishContainerFinishedEvent(ContainerFinishedEvent event) {
  TimelineEntity entity = createContainerEntity(event.getContainerId());
  TimelineEvent tEvent = new TimelineEvent();
  tEvent.setEventType(ContainerMetricsConstants.FINISHED_EVENT_TYPE);
  tEvent.setTimestamp(event.getTimestamp());
  Map<String, Object> eventInfo = new HashMap<String, Object>();
  eventInfo.put(ContainerMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO,
      event.getDiagnosticsInfo());
  eventInfo.put(ContainerMetricsConstants.EXIT_STATUS_EVENT_INFO,
      event.getContainerExitStatus());
  eventInfo.put(ContainerMetricsConstants.STATE_EVENT_INFO,
      event.getContainerState().toString());
  tEvent.setEventInfo(eventInfo);
  entity.addEvent(tEvent);
  putEntity(entity);
}
 

private static TimelineEntity convertContainerLaunchedEvent(ContainerLaunchedEvent event) {
  TimelineEntity atsEntity = new TimelineEntity();
  atsEntity.setEntityId("tez_"
      + event.getContainerId().toString());
  atsEntity.setEntityType(EntityTypes.TEZ_CONTAINER_ID.name());

  atsEntity.addRelatedEntity(EntityTypes.TEZ_APPLICATION_ATTEMPT.name(),
      "tez_" + event.getApplicationAttemptId().toString());

  atsEntity.addPrimaryFilter(ATSConstants.APPLICATION_ID,
      event.getApplicationAttemptId().getApplicationId().toString());

  atsEntity.addOtherInfo(ATSConstants.CONTAINER_ID,
      event.getContainerId().toString());
  atsEntity.setStartTime(event.getLaunchTime());

  TimelineEvent launchEvt = new TimelineEvent();
  launchEvt.setEventType(HistoryEventType.CONTAINER_LAUNCHED.name());
  launchEvt.setTimestamp(event.getLaunchTime());
  atsEntity.addEvent(launchEvt);

  return atsEntity;
}
 

private static TimelineEntity convertDAGInitializedEvent(DAGInitializedEvent event) {
  TimelineEntity atsEntity = new TimelineEntity();
  atsEntity.setEntityId(event.getDagID().toString());
  atsEntity.setEntityType(EntityTypes.TEZ_DAG_ID.name());

  TimelineEvent finishEvt = new TimelineEvent();
  finishEvt.setEventType(HistoryEventType.DAG_INITIALIZED.name());
  finishEvt.setTimestamp(event.getInitTime());
  atsEntity.addEvent(finishEvt);

  atsEntity.addPrimaryFilter(ATSConstants.USER, event.getUser());
  atsEntity.addPrimaryFilter(ATSConstants.DAG_NAME, event.getDagName());

  atsEntity.addOtherInfo(ATSConstants.INIT_TIME, event.getInitTime());

  return atsEntity;
}
 

private static TimelineEntity convertContainerStoppedEvent(ContainerStoppedEvent event) {
  TimelineEntity atsEntity = new TimelineEntity();
  atsEntity.setEntityId("tez_"
      + event.getContainerId().toString());
  atsEntity.setEntityType(EntityTypes.TEZ_CONTAINER_ID.name());

  // In case, a container is stopped in a different attempt
  atsEntity.addRelatedEntity(EntityTypes.TEZ_APPLICATION_ATTEMPT.name(),
      "tez_" + event.getApplicationAttemptId().toString());

  TimelineEvent stoppedEvt = new TimelineEvent();
  stoppedEvt.setEventType(HistoryEventType.CONTAINER_STOPPED.name());
  stoppedEvt.setTimestamp(event.getStoppedTime());
  atsEntity.addEvent(stoppedEvt);

  atsEntity.addOtherInfo(ATSConstants.EXIT_STATUS, event.getExitStatus());
  atsEntity.addOtherInfo(ATSConstants.FINISH_TIME, event.getStoppedTime());

  return atsEntity;
}
 

private static TimelineEntity convertDAGStartedEvent(DAGStartedEvent event) {
  TimelineEntity atsEntity = new TimelineEntity();
  atsEntity.setEntityId(event.getDagID().toString());
  atsEntity.setEntityType(EntityTypes.TEZ_DAG_ID.name());

  TimelineEvent startEvt = new TimelineEvent();
  startEvt.setEventType(HistoryEventType.DAG_STARTED.name());
  startEvt.setTimestamp(event.getStartTime());
  atsEntity.addEvent(startEvt);

  atsEntity.addPrimaryFilter(ATSConstants.USER, event.getUser());
  atsEntity.addPrimaryFilter(ATSConstants.DAG_NAME, event.getDagName());

  atsEntity.addOtherInfo(ATSConstants.START_TIME, event.getStartTime());

  return atsEntity;
}
 

private static TimelineEntity convertContainerStoppedEvent(ContainerStoppedEvent event) {
  TimelineEntity atsEntity = new TimelineEntity();
  atsEntity.setEntityId("tez_"
      + event.getContainerId().toString());
  atsEntity.setEntityType(EntityTypes.TEZ_CONTAINER_ID.name());

  // In case, a container is stopped in a different attempt
  atsEntity.addRelatedEntity(EntityTypes.TEZ_APPLICATION_ATTEMPT.name(),
      "tez_" + event.getApplicationAttemptId().toString());

  TimelineEvent stoppedEvt = new TimelineEvent();
  stoppedEvt.setEventType(HistoryEventType.CONTAINER_STOPPED.name());
  stoppedEvt.setTimestamp(event.getStoppedTime());
  atsEntity.addEvent(stoppedEvt);

  atsEntity.addPrimaryFilter(ATSConstants.APPLICATION_ID,
      event.getApplicationAttemptId().getApplicationId().toString());
  atsEntity.addPrimaryFilter(ATSConstants.EXIT_STATUS, event.getExitStatus());

  atsEntity.addOtherInfo(ATSConstants.EXIT_STATUS, event.getExitStatus());
  atsEntity.addOtherInfo(ATSConstants.FINISH_TIME, event.getStoppedTime());

  return atsEntity;
}
 

private static void publishContainerStartEvent(
    final TimelineClient timelineClient, Container container, String domainId,
    UserGroupInformation ugi) {
  final TimelineEntity entity = new TimelineEntity();
  entity.setEntityId(container.getId().toString());
  entity.setEntityType(DSEntity.DS_CONTAINER.toString());
  entity.setDomainId(domainId);
  entity.addPrimaryFilter("user", ugi.getShortUserName());
  TimelineEvent event = new TimelineEvent();
  event.setTimestamp(System.currentTimeMillis());
  event.setEventType(DSEvent.DS_CONTAINER_START.toString());
  event.addEventInfo("Node", container.getNodeId().toString());
  event.addEventInfo("Resources", container.getResource().toString());
  entity.addEvent(event);

  try {
    ugi.doAs(new PrivilegedExceptionAction<TimelinePutResponse>() {
      @Override
      public TimelinePutResponse run() throws Exception {
        return timelineClient.putEntities(entity);
      }
    });
  } catch (Exception e) {
    LOG.error("Container start event could not be published for "
        + container.getId().toString(),
        e instanceof UndeclaredThrowableException ? e.getCause() : e);
  }
}
 

private static TimelineEntity convertVertexInitializedEvent(VertexInitializedEvent event) {
  TimelineEntity atsEntity = new TimelineEntity();
  atsEntity.setEntityId(event.getVertexID().toString());
  atsEntity.setEntityType(EntityTypes.TEZ_VERTEX_ID.name());

  atsEntity.addRelatedEntity(EntityTypes.TEZ_DAG_ID.name(),
      event.getVertexID().getDAGId().toString());

  atsEntity.addPrimaryFilter(ATSConstants.APPLICATION_ID,
      event.getVertexID().getDAGId().getApplicationId().toString());
  atsEntity.addPrimaryFilter(EntityTypes.TEZ_DAG_ID.name(),
      event.getVertexID().getDAGId().toString());

  TimelineEvent initEvt = new TimelineEvent();
  initEvt.setEventType(HistoryEventType.VERTEX_INITIALIZED.name());
  initEvt.setTimestamp(event.getInitedTime());
  atsEntity.addEvent(initEvt);

  atsEntity.setStartTime(event.getInitedTime());

  atsEntity.addOtherInfo(ATSConstants.VERTEX_NAME, event.getVertexName());
  atsEntity.addOtherInfo(ATSConstants.INIT_REQUESTED_TIME, event.getInitRequestedTime());
  atsEntity.addOtherInfo(ATSConstants.INIT_TIME, event.getInitedTime());
  atsEntity.addOtherInfo(ATSConstants.NUM_TASKS, event.getNumTasks());
  atsEntity.addOtherInfo(ATSConstants.PROCESSOR_CLASS_NAME, event.getProcessorName());
  if (event.getServicePluginInfo() != null) {
    atsEntity.addOtherInfo(ATSConstants.SERVICE_PLUGIN,
        DAGUtils.convertServicePluginToATSMap(event.getServicePluginInfo()));
  }

  return atsEntity;
}
 

private static TimelineEntity convertDAGRecoveredEvent(DAGRecoveredEvent event) {
  TimelineEntity atsEntity = new TimelineEntity();
  atsEntity.setEntityId(event.getDagID().toString());
  atsEntity.setEntityType(EntityTypes.TEZ_DAG_ID.name());

  TimelineEvent recoverEvt = new TimelineEvent();
  recoverEvt.setEventType(HistoryEventType.DAG_RECOVERED.name());
  recoverEvt.setTimestamp(event.getRecoveredTime());
  recoverEvt.addEventInfo(ATSConstants.APPLICATION_ATTEMPT_ID,
      event.getApplicationAttemptId().toString());
  if (event.getRecoveredDagState() != null) {
    recoverEvt.addEventInfo(ATSConstants.DAG_STATE, event.getRecoveredDagState().name());
  }
  if (event.getRecoveryFailureReason() != null) {
    recoverEvt.addEventInfo(ATSConstants.RECOVERY_FAILURE_REASON,
        event.getRecoveryFailureReason());
  }

  atsEntity.addEvent(recoverEvt);

  atsEntity.addPrimaryFilter(ATSConstants.USER, event.getUser());
  atsEntity.addPrimaryFilter(ATSConstants.APPLICATION_ID,
      event.getApplicationAttemptId().getApplicationId().toString());
  atsEntity.addPrimaryFilter(ATSConstants.DAG_NAME, event.getDagName());
  atsEntity.addOtherInfo(ATSConstants.IN_PROGRESS_LOGS_URL + "_"
      + event.getApplicationAttemptId().getAttemptId(), event.getContainerLogs());

  return atsEntity;
}
 

private static TimelineEntity convertTaskStartedEvent(TaskStartedEvent event) {
  TimelineEntity atsEntity = new TimelineEntity();
  atsEntity.setEntityId(event.getTaskID().toString());
  atsEntity.setEntityType(EntityTypes.TEZ_TASK_ID.name());

  atsEntity.addRelatedEntity(EntityTypes.TEZ_VERTEX_ID.name(),
      event.getTaskID().getVertexID().toString());

  atsEntity.addPrimaryFilter(ATSConstants.APPLICATION_ID,
      event.getTaskID().getVertexID().getDAGId().getApplicationId().toString());
  atsEntity.addPrimaryFilter(EntityTypes.TEZ_DAG_ID.name(),
      event.getTaskID().getVertexID().getDAGId().toString());
  atsEntity.addPrimaryFilter(EntityTypes.TEZ_VERTEX_ID.name(),
      event.getTaskID().getVertexID().toString());

  TimelineEvent startEvt = new TimelineEvent();
  startEvt.setEventType(HistoryEventType.TASK_STARTED.name());
  startEvt.setTimestamp(event.getStartTime());
  atsEntity.addEvent(startEvt);

  atsEntity.setStartTime(event.getStartTime());

  atsEntity.addOtherInfo(ATSConstants.START_TIME, event.getStartTime());
  atsEntity.addOtherInfo(ATSConstants.SCHEDULED_TIME, event.getScheduledTime());
  atsEntity.addOtherInfo(ATSConstants.STATUS, event.getState().name());

  return atsEntity;
}
 

private static TimelineEntity convertDAGSubmittedEvent(DAGSubmittedEvent event) {
  TimelineEntity atsEntity = new TimelineEntity();
  atsEntity.setEntityId(event.getDagID().toString());
  atsEntity.setEntityType(EntityTypes.TEZ_DAG_ID.name());

  atsEntity.addRelatedEntity(EntityTypes.TEZ_APPLICATION_ATTEMPT.name(),
      "tez_" + event.getApplicationAttemptId().toString());
  atsEntity.addRelatedEntity(ATSConstants.APPLICATION_ID,
      event.getApplicationAttemptId().getApplicationId().toString());
  atsEntity.addRelatedEntity(ATSConstants.APPLICATION_ATTEMPT_ID,
      event.getApplicationAttemptId().toString());
  atsEntity.addRelatedEntity(ATSConstants.USER, event.getUser());

  TimelineEvent submitEvt = new TimelineEvent();
  submitEvt.setEventType(HistoryEventType.DAG_SUBMITTED.name());
  submitEvt.setTimestamp(event.getSubmitTime());
  atsEntity.addEvent(submitEvt);

  atsEntity.setStartTime(event.getSubmitTime());

  atsEntity.addPrimaryFilter(ATSConstants.USER, event.getUser());
  atsEntity.addPrimaryFilter(ATSConstants.DAG_NAME, event.getDAGName());

  try {
    atsEntity.addOtherInfo(ATSConstants.DAG_PLAN,
        DAGUtils.convertDAGPlanToATSMap(event.getDAGPlan()));
  } catch (IOException e) {
    throw new TezUncheckedException(e);
  }
  atsEntity.addOtherInfo(ATSConstants.APPLICATION_ID,
      event.getApplicationAttemptId().getApplicationId().toString());

  return atsEntity;
}
 

private void publishApplicationACLsUpdatedEvent(
    ApplicationACLsUpdatedEvent event) {
  TimelineEntity entity =
      createApplicationEntity(event.getApplicationId());
  TimelineEvent tEvent = new TimelineEvent();
  Map<String, Object> entityInfo = new HashMap<String, Object>();
  entityInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO,
      event.getViewAppACLs());
  entity.setOtherInfo(entityInfo);
  tEvent.setEventType(
      ApplicationMetricsConstants.ACLS_UPDATED_EVENT_TYPE);
  tEvent.setTimestamp(event.getTimestamp());
  entity.addEvent(tEvent);
  putEntity(entity);
}