Java Code Examples for org.apache.hadoop.yarn.api.records.timeline.TimelineEntity#setEntityId()

private static TimelineEntity convertDAGInitializedEvent(DAGInitializedEvent event) {
  TimelineEntity atsEntity = new TimelineEntity();
  atsEntity.setEntityId(event.getDagID().toString());
  atsEntity.setEntityType(EntityTypes.TEZ_DAG_ID.name());

  TimelineEvent finishEvt = new TimelineEvent();
  finishEvt.setEventType(HistoryEventType.DAG_INITIALIZED.name());
  finishEvt.setTimestamp(event.getInitTime());
  atsEntity.addEvent(finishEvt);

  atsEntity.addPrimaryFilter(ATSConstants.USER, event.getUser());
  atsEntity.addPrimaryFilter(ATSConstants.DAG_NAME, event.getDagName());

  atsEntity.addOtherInfo(ATSConstants.INIT_TIME, event.getInitTime());

  return atsEntity;
}
 

private static TimelineEntity convertTaskStartedEvent(TaskStartedEvent event) {
  TimelineEntity atsEntity = new TimelineEntity();
  atsEntity.setEntityId(event.getTaskID().toString());
  atsEntity.setEntityType(EntityTypes.TEZ_TASK_ID.name());

  atsEntity.addRelatedEntity(EntityTypes.TEZ_VERTEX_ID.name(),
      event.getTaskID().getVertexID().toString());
  atsEntity.addPrimaryFilter(EntityTypes.TEZ_DAG_ID.name(),
      event.getTaskID().getVertexID().getDAGId().toString());
  atsEntity.addPrimaryFilter(EntityTypes.TEZ_VERTEX_ID.name(),
      event.getTaskID().getVertexID().toString());

  TimelineEvent startEvt = new TimelineEvent();
  startEvt.setEventType(HistoryEventType.TASK_STARTED.name());
  startEvt.setTimestamp(event.getStartTime());
  atsEntity.addEvent(startEvt);

  atsEntity.addOtherInfo(ATSConstants.START_TIME, event.getStartTime());
  atsEntity.addOtherInfo(ATSConstants.SCHEDULED_TIME, event.getScheduledTime());

  return atsEntity;
}
 

private static TimelineEntity convertDAGFinishedToDAGExtraInfoEntity(DAGFinishedEvent event) {
  TimelineEntity atsEntity = new TimelineEntity();
  atsEntity.setEntityId(event.getDagID().toString());
  atsEntity.setEntityType(EntityTypes.TEZ_DAG_EXTRA_INFO.name());

  atsEntity.addRelatedEntity(EntityTypes.TEZ_DAG_ID.name(), event.getDagID().toString());

  TimelineEvent submitEvt = new TimelineEvent();
  submitEvt.setEventType(HistoryEventType.DAG_FINISHED.name());
  submitEvt.setTimestamp(event.getFinishTime());
  atsEntity.addEvent(submitEvt);

  atsEntity.addOtherInfo(ATSConstants.COUNTERS,
      DAGUtils.convertCountersToATSMap(event.getTezCounters()));
  return atsEntity;
}
 

@Test
public void testPostEntitiesWithPrimaryFilter() throws Exception {
  TimelineEntities entities = new TimelineEntities();
  TimelineEntity entity = new TimelineEntity();
  Map<String, Set<Object>> filters = new HashMap<String, Set<Object>>();
  filters.put(TimelineStore.SystemFilter.ENTITY_OWNER.toString(),
      new HashSet<Object>());
  entity.setPrimaryFilters(filters);
  entity.setEntityId("test id 6");
  entity.setEntityType("test type 6");
  entity.setStartTime(System.currentTimeMillis());
  entities.addEntity(entity);
  WebResource r = resource();
  ClientResponse response = r.path("ws").path("v1").path("timeline")
      .queryParam("user.name", "tester")
      .accept(MediaType.APPLICATION_JSON)
      .type(MediaType.APPLICATION_JSON)
      .post(ClientResponse.class, entities);
  TimelinePutResponse putResposne =
      response.getEntity(TimelinePutResponse.class);
  Assert.assertEquals(0, putResposne.getErrors().size());
}
 

private static TimelineEntity convertAMLaunchedEvent(AMLaunchedEvent event) {
  TimelineEntity atsEntity = new TimelineEntity();
  atsEntity.setEntityId("tez_"
      + event.getApplicationAttemptId().toString());
  atsEntity.setEntityType(EntityTypes.TEZ_APPLICATION_ATTEMPT.name());

  atsEntity.addPrimaryFilter(ATSConstants.USER, event.getUser());
  atsEntity.addPrimaryFilter(ATSConstants.APPLICATION_ID,
      event.getApplicationAttemptId().getApplicationId().toString());

  atsEntity.setStartTime(event.getLaunchTime());

  TimelineEvent launchEvt = new TimelineEvent();
  launchEvt.setEventType(HistoryEventType.AM_LAUNCHED.name());
  launchEvt.setTimestamp(event.getLaunchTime());
  atsEntity.addEvent(launchEvt);

  atsEntity.addOtherInfo(ATSConstants.APP_SUBMIT_TIME, event.getAppSubmitTime());
  atsEntity.addOtherInfo(ATSConstants.APPLICATION_ID,
      event.getApplicationAttemptId().getApplicationId().toString());
  atsEntity.addOtherInfo(ATSConstants.APPLICATION_ATTEMPT_ID,
      event.getApplicationAttemptId().toString());
  atsEntity.addOtherInfo(ATSConstants.USER, event.getUser());

  return atsEntity;
}
 

private static TimelineEntity convertContainerLaunchedEvent(ContainerLaunchedEvent event) {
  TimelineEntity atsEntity = new TimelineEntity();
  atsEntity.setEntityId("tez_"
      + event.getContainerId().toString());
  atsEntity.setEntityType(EntityTypes.TEZ_CONTAINER_ID.name());

  atsEntity.addRelatedEntity(EntityTypes.TEZ_APPLICATION_ATTEMPT.name(),
      "tez_" + event.getApplicationAttemptId().toString());

  atsEntity.addPrimaryFilter(ATSConstants.APPLICATION_ID,
      event.getApplicationAttemptId().getApplicationId().toString());

  atsEntity.addOtherInfo(ATSConstants.CONTAINER_ID,
      event.getContainerId().toString());
  atsEntity.setStartTime(event.getLaunchTime());

  TimelineEvent launchEvt = new TimelineEvent();
  launchEvt.setEventType(HistoryEventType.CONTAINER_LAUNCHED.name());
  launchEvt.setTimestamp(event.getLaunchTime());
  atsEntity.addEvent(launchEvt);

  return atsEntity;
}
 

@Test
public void testRelatingToNonExistingEntity() throws IOException {
  TimelineEntity entityToStore = new TimelineEntity();
  entityToStore.setEntityType("TEST_ENTITY_TYPE_1");
  entityToStore.setEntityId("TEST_ENTITY_ID_1");
  entityToStore.setDomainId(TimelineDataManager.DEFAULT_DOMAIN_ID);
  entityToStore.addRelatedEntity("TEST_ENTITY_TYPE_2", "TEST_ENTITY_ID_2");
  TimelineEntities entities = new TimelineEntities();
  entities.addEntity(entityToStore);
  store.put(entities);
  TimelineEntity entityToGet =
      store.getEntity("TEST_ENTITY_ID_2", "TEST_ENTITY_TYPE_2", null);
  Assert.assertNotNull(entityToGet);
  Assert.assertEquals("DEFAULT", entityToGet.getDomainId());
  Assert.assertEquals("TEST_ENTITY_TYPE_1",
      entityToGet.getRelatedEntities().keySet().iterator().next());
  Assert.assertEquals("TEST_ENTITY_ID_1",
      entityToGet.getRelatedEntities().values().iterator().next()
          .iterator().next());
}
 

private static void publishApplicationAttemptEvent(
    final TimelineClient timelineClient, String appAttemptId,
    DSEvent appEvent, String domainId, UserGroupInformation ugi) {
  final TimelineEntity entity = new TimelineEntity();
  entity.setEntityId(appAttemptId);
  entity.setEntityType(DSEntity.DS_APP_ATTEMPT.toString());
  entity.setDomainId(domainId);
  entity.addPrimaryFilter("user", ugi.getShortUserName());
  TimelineEvent event = new TimelineEvent();
  event.setEventType(appEvent.toString());
  event.setTimestamp(System.currentTimeMillis());
  entity.addEvent(event);
  try {
    timelineClient.putEntities(entity);
  } catch (YarnException | IOException e) {
    LOG.error("App Attempt "
        + (appEvent.equals(DSEvent.DS_APP_ATTEMPT_START) ? "start" : "end")
        + " event could not be published for "
        + appAttemptId.toString(), e);
  }
}
 

private static void publishContainerEndEvent(
    final TimelineClient timelineClient, ContainerStatus container,
    String domainId, UserGroupInformation ugi) {
  final TimelineEntity entity = new TimelineEntity();
  entity.setEntityId(container.getContainerId().toString());
  entity.setEntityType(DSEntity.DS_CONTAINER.toString());
  entity.setDomainId(domainId);
  entity.addPrimaryFilter("user", ugi.getShortUserName());
  TimelineEvent event = new TimelineEvent();
  event.setTimestamp(System.currentTimeMillis());
  event.setEventType(DSEvent.DS_CONTAINER_END.toString());
  event.addEventInfo("State", container.getState().name());
  event.addEventInfo("Exit Status", container.getExitStatus());
  entity.addEvent(event);
  try {
    timelineClient.putEntities(entity);
  } catch (YarnException | IOException e) {
    LOG.error("Container end event could not be published for "
        + container.getContainerId().toString(), e);
  }
}
 

private static TimelineEntity maskFields(
    TimelineEntity entity, EnumSet<Field> fields) {
  // Conceal the fields that are not going to be exposed
  TimelineEntity entityToReturn = new TimelineEntity();
  entityToReturn.setEntityId(entity.getEntityId());
  entityToReturn.setEntityType(entity.getEntityType());
  entityToReturn.setStartTime(entity.getStartTime());
  entityToReturn.setDomainId(entity.getDomainId());
  // Deep copy
  if (fields.contains(Field.EVENTS)) {
    entityToReturn.addEvents(entity.getEvents());
  } else if (fields.contains(Field.LAST_EVENT_ONLY)) {
    entityToReturn.addEvent(entity.getEvents().get(0));
  } else {
    entityToReturn.setEvents(null);
  }
  if (fields.contains(Field.RELATED_ENTITIES)) {
    entityToReturn.addRelatedEntities(entity.getRelatedEntities());
  } else {
    entityToReturn.setRelatedEntities(null);
  }
  if (fields.contains(Field.PRIMARY_FILTERS)) {
    entityToReturn.addPrimaryFilters(entity.getPrimaryFilters());
  } else {
    entityToReturn.setPrimaryFilters(null);
  }
  if (fields.contains(Field.OTHER_INFO)) {
    entityToReturn.addOtherInfo(entity.getOtherInfo());
  } else {
    entityToReturn.setOtherInfo(null);
  }
  return entityToReturn;
}
 

private static TimelineEntity createContainerEntity(
    ContainerId containerId) {
  TimelineEntity entity = new TimelineEntity();
  entity.setEntityType(
      ContainerMetricsConstants.ENTITY_TYPE);
  entity.setEntityId(containerId.toString());
  entity.addPrimaryFilter(ContainerMetricsConstants.PARENT_PRIMARIY_FILTER,
      containerId.getApplicationAttemptId().toString());
  return entity;
}
 

private static void publishContainerStartEvent(
    final TimelineClient timelineClient, Container container, String domainId,
    UserGroupInformation ugi) {
  final TimelineEntity entity = new TimelineEntity();
  entity.setEntityId(container.getId().toString());
  entity.setEntityType(DSEntity.DS_CONTAINER.toString());
  entity.setDomainId(domainId);
  entity.addPrimaryFilter("user", ugi.getShortUserName());
  TimelineEvent event = new TimelineEvent();
  event.setTimestamp(System.currentTimeMillis());
  event.setEventType(DSEvent.DS_CONTAINER_START.toString());
  event.addEventInfo("Node", container.getNodeId().toString());
  event.addEventInfo("Resources", container.getResource().toString());
  entity.addEvent(event);

  try {
    ugi.doAs(new PrivilegedExceptionAction<TimelinePutResponse>() {
      @Override
      public TimelinePutResponse run() throws Exception {
        return timelineClient.putEntities(entity);
      }
    });
  } catch (Exception e) {
    LOG.error("Container start event could not be published for "
        + container.getId().toString(),
        e instanceof UndeclaredThrowableException ? e.getCause() : e);
  }
}
 

private static TimelineEntity convertVertexReconfigureDoneEvent(
    VertexConfigurationDoneEvent event) {
  TimelineEntity atsEntity = new TimelineEntity();
  atsEntity.setEntityId(event.getVertexID().toString());
  atsEntity.setEntityType(EntityTypes.TEZ_VERTEX_ID.name());

  atsEntity.addPrimaryFilter(ATSConstants.APPLICATION_ID,
      event.getVertexID().getDAGId().getApplicationId().toString());
  atsEntity.addPrimaryFilter(EntityTypes.TEZ_DAG_ID.name(),
      event.getVertexID().getDAGId().toString());

  TimelineEvent updateEvt = new TimelineEvent();
  updateEvt.setEventType(HistoryEventType.VERTEX_CONFIGURE_DONE.name());
  updateEvt.setTimestamp(event.getReconfigureDoneTime());

  Map<String,Object> eventInfo = new HashMap<String, Object>();
  if (event.getSourceEdgeProperties() != null && !event.getSourceEdgeProperties().isEmpty()) {
    Map<String, Object> updatedEdgeManagers = new HashMap<String, Object>();
    for (Entry<String, EdgeProperty> entry :
        event.getSourceEdgeProperties().entrySet()) {
      updatedEdgeManagers.put(entry.getKey(),
          DAGUtils.convertEdgeProperty(entry.getValue()));
    }
    eventInfo.put(ATSConstants.UPDATED_EDGE_MANAGERS, updatedEdgeManagers);
  }
  eventInfo.put(ATSConstants.NUM_TASKS, event.getNumTasks());
  updateEvt.setEventInfo(eventInfo);
  atsEntity.addEvent(updateEvt);

  atsEntity.addOtherInfo(ATSConstants.NUM_TASKS, event.getNumTasks());

  return atsEntity;
}
 

private static void publishContainerStartEvent(
    final TimelineClient timelineClient, Container container, String domainId,
    UserGroupInformation ugi) {
  final TimelineEntity entity = new TimelineEntity();
  entity.setEntityId(container.getId().toString());
  entity.setEntityType(DSEntity.DS_CONTAINER.toString());
  entity.setDomainId(domainId);
  entity.addPrimaryFilter("user", ugi.getShortUserName());
  TimelineEvent event = new TimelineEvent();
  event.setTimestamp(System.currentTimeMillis());
  event.setEventType(DSEvent.DS_CONTAINER_START.toString());
  event.addEventInfo("Node", container.getNodeId().toString());
  event.addEventInfo("Resources", container.getResource().toString());
  entity.addEvent(event);

  try {
    ugi.doAs(new PrivilegedExceptionAction<TimelinePutResponse>() {
      @Override
      public TimelinePutResponse run() throws Exception {
        return timelineClient.putEntities(entity);
      }
    });
  } catch (Exception e) {
    LOG.error("Container start event could not be published for "
        + container.getId().toString(),
        e instanceof UndeclaredThrowableException ? e.getCause() : e);
  }
}
 

private static TimelineEntity convertTaskStartedEvent(TaskStartedEvent event) {
  TimelineEntity atsEntity = new TimelineEntity();
  atsEntity.setEntityId(event.getTaskID().toString());
  atsEntity.setEntityType(EntityTypes.TEZ_TASK_ID.name());

  atsEntity.addRelatedEntity(EntityTypes.TEZ_VERTEX_ID.name(),
      event.getTaskID().getVertexID().toString());

  atsEntity.addPrimaryFilter(ATSConstants.APPLICATION_ID,
      event.getTaskID().getVertexID().getDAGId().getApplicationId().toString());
  atsEntity.addPrimaryFilter(EntityTypes.TEZ_DAG_ID.name(),
      event.getTaskID().getVertexID().getDAGId().toString());
  atsEntity.addPrimaryFilter(EntityTypes.TEZ_VERTEX_ID.name(),
      event.getTaskID().getVertexID().toString());

  TimelineEvent startEvt = new TimelineEvent();
  startEvt.setEventType(HistoryEventType.TASK_STARTED.name());
  startEvt.setTimestamp(event.getStartTime());
  atsEntity.addEvent(startEvt);

  atsEntity.setStartTime(event.getStartTime());

  atsEntity.addOtherInfo(ATSConstants.START_TIME, event.getStartTime());
  atsEntity.addOtherInfo(ATSConstants.SCHEDULED_TIME, event.getScheduledTime());
  atsEntity.addOtherInfo(ATSConstants.STATUS, event.getState().name());

  return atsEntity;
}
 

@Test
public void testPostEntitiesToDefaultDomain() throws Exception {
  AdminACLsManager oldAdminACLsManager =
      timelineACLsManager.setAdminACLsManager(adminACLsManager);
  try {
    TimelineEntities entities = new TimelineEntities();
    TimelineEntity entity = new TimelineEntity();
    entity.setEntityId("test id 7");
    entity.setEntityType("test type 7");
    entity.setStartTime(System.currentTimeMillis());
    entities.addEntity(entity);
    WebResource r = resource();
    ClientResponse response = r.path("ws").path("v1").path("timeline")
        .queryParam("user.name", "anybody_1")
        .accept(MediaType.APPLICATION_JSON)
        .type(MediaType.APPLICATION_JSON)
        .post(ClientResponse.class, entities);
    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
    TimelinePutResponse putResposne =
        response.getEntity(TimelinePutResponse.class);
    Assert.assertNotNull(putResposne);
    Assert.assertEquals(0, putResposne.getErrors().size());
    // verify the entity exists in the store
    response = r.path("ws").path("v1").path("timeline")
        .path("test type 7").path("test id 7")
        .queryParam("user.name", "any_body_2")
        .accept(MediaType.APPLICATION_JSON)
        .get(ClientResponse.class);
    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
    entity = response.getEntity(TimelineEntity.class);
    Assert.assertNotNull(entity);
    Assert.assertEquals("test id 7", entity.getEntityId());
    Assert.assertEquals("test type 7", entity.getEntityType());
    Assert.assertEquals(TimelineDataManager.DEFAULT_DOMAIN_ID,
        entity.getDomainId());
  } finally {
    timelineACLsManager.setAdminACLsManager(oldAdminACLsManager);
  }
}
 

@Test
public void testPutEntities() throws Exception {
  TestTimelineClient client = new TestTimelineClient();
  try {
    client.init(conf);
    client.start();
    TimelineEntity expectedEntity = new TimelineEntity();
    expectedEntity.setEntityType("test entity type");
    expectedEntity.setEntityId("test entity id");
    expectedEntity.setDomainId("test domain id");
    TimelineEvent event = new TimelineEvent();
    event.setEventType("test event type");
    event.setTimestamp(0L);
    expectedEntity.addEvent(event);

    TimelinePutResponse response = client.putEntities(expectedEntity);
    Assert.assertEquals(0, response.getErrors().size());
    Assert.assertTrue(client.resp.toString().contains("https"));

    TimelineEntity actualEntity = store.getEntity(
        expectedEntity.getEntityId(), expectedEntity.getEntityType(),
        EnumSet.allOf(Field.class));
    Assert.assertNotNull(actualEntity);
    Assert.assertEquals(
        expectedEntity.getEntityId(), actualEntity.getEntityId());
    Assert.assertEquals(
        expectedEntity.getEntityType(), actualEntity.getEntityType());
  } finally {
    client.stop();
    client.close();
  }
}
 

private static TimelineEntity createContainerEntity(
    ContainerId containerId) {
  TimelineEntity entity = new TimelineEntity();
  entity.setEntityType(
      ContainerMetricsConstants.ENTITY_TYPE);
  entity.setEntityId(containerId.toString());
  entity.addPrimaryFilter(ContainerMetricsConstants.PARENT_PRIMARIY_FILTER,
      containerId.getApplicationAttemptId().toString());
  return entity;
}
 

private static TimelineEntity createAppAttemptEntity(
    ApplicationAttemptId appAttemptId) {
  TimelineEntity entity = new TimelineEntity();
  entity.setEntityType(
      AppAttemptMetricsConstants.ENTITY_TYPE);
  entity.setEntityId(appAttemptId.toString());
  entity.addPrimaryFilter(AppAttemptMetricsConstants.PARENT_PRIMARY_FILTER,
      appAttemptId.getApplicationId().toString());
  return entity;
}
 

private static TimelineEntity createApplicationEntity(
    ApplicationId applicationId) {
  TimelineEntity entity = new TimelineEntity();
  entity.setEntityType(ApplicationMetricsConstants.ENTITY_TYPE);
  entity.setEntityId(applicationId.toString());
  return entity;
}