org.apache.hadoop.yarn.api.records.timeline.TimelineEntity Java Examples

The following examples show how to use org.apache.hadoop.yarn.api.records.timeline.TimelineEntity. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestTimelineACLsManager.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testCorruptedOwnerInfoForEntity() throws Exception {
  Configuration conf = new YarnConfiguration();
  conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
  conf.set(YarnConfiguration.YARN_ADMIN_ACL, "owner");
  TimelineACLsManager timelineACLsManager =
      new TimelineACLsManager(conf);
  timelineACLsManager.setTimelineStore(new TestTimelineStore());
  TimelineEntity entity = new TimelineEntity();
  try {
    timelineACLsManager.checkAccess(
        UserGroupInformation.createRemoteUser("owner"),
        ApplicationAccessType.VIEW_APP, entity);
    Assert.fail("Exception is expected");
  } catch (YarnException e) {
    Assert.assertTrue("It's not the exact expected exception", e.getMessage()
        .contains("doesn't exist."));
  }
}
 
Example #2
Source File: HistoryEventTimelineConversion.java    From tez with Apache License 2.0 6 votes vote down vote up
private static TimelineEntity convertDAGSubmittedToDAGExtraInfoEntity(DAGSubmittedEvent event) {
  TimelineEntity atsEntity = new TimelineEntity();
  atsEntity.setEntityId(event.getDagID().toString());
  atsEntity.setEntityType(EntityTypes.TEZ_DAG_EXTRA_INFO.name());

  atsEntity.addRelatedEntity(EntityTypes.TEZ_DAG_ID.name(), event.getDagID().toString());

  TimelineEvent submitEvt = new TimelineEvent();
  submitEvt.setEventType(HistoryEventType.DAG_SUBMITTED.name());
  submitEvt.setTimestamp(event.getSubmitTime());
  atsEntity.addEvent(submitEvt);

  atsEntity.setStartTime(event.getSubmitTime());

  try {
    atsEntity.addOtherInfo(ATSConstants.DAG_PLAN,
        DAGUtils.convertDAGPlanToATSMap(event.getDAGPlan()));
  } catch (IOException e) {
    throw new TezUncheckedException(e);
  }
  return atsEntity;
}
 
Example #3
Source File: TestTimelineWebServices.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testGetEntity() throws Exception {
  WebResource r = resource();
  ClientResponse response = r.path("ws").path("v1").path("timeline")
      .path("type_1").path("id_1")
      .accept(MediaType.APPLICATION_JSON)
      .get(ClientResponse.class);
  assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
  TimelineEntity entity = response.getEntity(TimelineEntity.class);
  Assert.assertNotNull(entity);
  Assert.assertEquals("id_1", entity.getEntityId());
  Assert.assertEquals("type_1", entity.getEntityType());
  Assert.assertEquals(123l, entity.getStartTime().longValue());
  Assert.assertEquals(2, entity.getEvents().size());
  Assert.assertEquals(4, entity.getPrimaryFilters().size());
  Assert.assertEquals(4, entity.getOtherInfo().size());
}
 
Example #4
Source File: HistoryEventTimelineConversion.java    From tez with Apache License 2.0 6 votes vote down vote up
private static TimelineEntity convertAMLaunchedEvent(AMLaunchedEvent event) {
  TimelineEntity atsEntity = new TimelineEntity();
  atsEntity.setEntityId("tez_"
      + event.getApplicationAttemptId().toString());
  atsEntity.setEntityType(EntityTypes.TEZ_APPLICATION_ATTEMPT.name());

  atsEntity.addPrimaryFilter(ATSConstants.USER, event.getUser());
  atsEntity.addPrimaryFilter(ATSConstants.APPLICATION_ID,
      event.getApplicationAttemptId().getApplicationId().toString());

  atsEntity.setStartTime(event.getLaunchTime());

  TimelineEvent launchEvt = new TimelineEvent();
  launchEvt.setEventType(HistoryEventType.AM_LAUNCHED.name());
  launchEvt.setTimestamp(event.getLaunchTime());
  atsEntity.addEvent(launchEvt);

  atsEntity.addOtherInfo(ATSConstants.APP_SUBMIT_TIME, event.getAppSubmitTime());
  atsEntity.addOtherInfo(ATSConstants.APPLICATION_ID,
      event.getApplicationAttemptId().getApplicationId().toString());
  atsEntity.addOtherInfo(ATSConstants.APPLICATION_ATTEMPT_ID,
      event.getApplicationAttemptId().toString());
  atsEntity.addOtherInfo(ATSConstants.USER, event.getUser());

  return atsEntity;
}
 
Example #5
Source File: ApplicationMaster.java    From big-c with Apache License 2.0 6 votes vote down vote up
private static void publishApplicationAttemptEvent(
    final TimelineClient timelineClient, String appAttemptId,
    DSEvent appEvent, String domainId, UserGroupInformation ugi) {
  final TimelineEntity entity = new TimelineEntity();
  entity.setEntityId(appAttemptId);
  entity.setEntityType(DSEntity.DS_APP_ATTEMPT.toString());
  entity.setDomainId(domainId);
  entity.addPrimaryFilter("user", ugi.getShortUserName());
  TimelineEvent event = new TimelineEvent();
  event.setEventType(appEvent.toString());
  event.setTimestamp(System.currentTimeMillis());
  entity.addEvent(event);
  try {
    timelineClient.putEntities(entity);
  } catch (YarnException | IOException e) {
    LOG.error("App Attempt "
        + (appEvent.equals(DSEvent.DS_APP_ATTEMPT_START) ? "start" : "end")
        + " event could not be published for "
        + appAttemptId.toString(), e);
  }
}
 
Example #6
Source File: TimelineStoreTestUtils.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Create a test entity
 */
protected static TimelineEntity createEntity(String entityId, String entityType,
    Long startTime, List<TimelineEvent> events,
    Map<String, Set<String>> relatedEntities,
    Map<String, Set<Object>> primaryFilters,
    Map<String, Object> otherInfo, String domainId) {
  TimelineEntity entity = new TimelineEntity();
  entity.setEntityId(entityId);
  entity.setEntityType(entityType);
  entity.setStartTime(startTime);
  entity.setEvents(events);
  if (relatedEntities != null) {
    for (Entry<String, Set<String>> e : relatedEntities.entrySet()) {
      for (String v : e.getValue()) {
        entity.addRelatedEntity(e.getKey(), v);
      }
    }
  } else {
    entity.setRelatedEntities(null);
  }
  entity.setPrimaryFilters(primaryFilters);
  entity.setOtherInfo(otherInfo);
  entity.setDomainId(domainId);
  return entity;
}
 
Example #7
Source File: TestTimelineWebServices.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testGetEntityFields1() throws Exception {
  WebResource r = resource();
  ClientResponse response = r.path("ws").path("v1").path("timeline")
      .path("type_1").path("id_1").queryParam("fields", "events,otherinfo")
      .accept(MediaType.APPLICATION_JSON)
      .get(ClientResponse.class);
  assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
  TimelineEntity entity = response.getEntity(TimelineEntity.class);
  Assert.assertNotNull(entity);
  Assert.assertEquals("id_1", entity.getEntityId());
  Assert.assertEquals("type_1", entity.getEntityType());
  Assert.assertEquals(123l, entity.getStartTime().longValue());
  Assert.assertEquals(2, entity.getEvents().size());
  Assert.assertEquals(0, entity.getPrimaryFilters().size());
  Assert.assertEquals(4, entity.getOtherInfo().size());
}
 
Example #8
Source File: HistoryEventTimelineConversion.java    From tez with Apache License 2.0 6 votes vote down vote up
private static TimelineEntity convertDAGFinishedToDAGExtraInfoEntity(DAGFinishedEvent event) {
  TimelineEntity atsEntity = new TimelineEntity();
  atsEntity.setEntityId(event.getDagID().toString());
  atsEntity.setEntityType(EntityTypes.TEZ_DAG_EXTRA_INFO.name());

  atsEntity.addRelatedEntity(EntityTypes.TEZ_DAG_ID.name(), event.getDagID().toString());

  TimelineEvent submitEvt = new TimelineEvent();
  submitEvt.setEventType(HistoryEventType.DAG_FINISHED.name());
  submitEvt.setTimestamp(event.getFinishTime());
  atsEntity.addEvent(submitEvt);

  atsEntity.addOtherInfo(ATSConstants.COUNTERS,
      DAGUtils.convertCountersToATSMap(event.getTezCounters()));
  return atsEntity;
}
 
Example #9
Source File: HistoryEventTimelineConversion.java    From incubator-tez with Apache License 2.0 6 votes vote down vote up
private static TimelineEntity convertContainerLaunchedEvent(ContainerLaunchedEvent event) {
  TimelineEntity atsEntity = new TimelineEntity();
  atsEntity.setEntityId("tez_"
      + event.getContainerId().toString());
  atsEntity.setEntityType(EntityTypes.TEZ_CONTAINER_ID.name());

  atsEntity.addRelatedEntity(EntityTypes.TEZ_APPLICATION_ATTEMPT.name(),
      "tez_" + event.getApplicationAttemptId().toString());
  atsEntity.addRelatedEntity(ATSConstants.CONTAINER_ID,
      event.getContainerId().toString());

  atsEntity.setStartTime(event.getLaunchTime());

  TimelineEvent launchEvt = new TimelineEvent();
  launchEvt.setEventType(HistoryEventType.CONTAINER_LAUNCHED.name());
  launchEvt.setTimestamp(event.getLaunchTime());
  atsEntity.addEvent(launchEvt);

  return atsEntity;
}
 
Example #10
Source File: ApplicationHistoryManagerOnTimelineStore.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
public Map<ContainerId, ContainerReport> getContainers(
    ApplicationAttemptId appAttemptId) throws YarnException, IOException {
  ApplicationReportExt app = getApplication(
      appAttemptId.getApplicationId(), ApplicationReportField.USER_AND_ACLS);
  checkAccess(app);
  TimelineEntities entities = timelineDataManager.getEntities(
      ContainerMetricsConstants.ENTITY_TYPE,
      new NameValuePair(
          ContainerMetricsConstants.PARENT_PRIMARIY_FILTER,
          appAttemptId.toString()), null, null, null,
      null, null, Long.MAX_VALUE, EnumSet.allOf(Field.class),
      UserGroupInformation.getLoginUser());
  Map<ContainerId, ContainerReport> containers =
      new LinkedHashMap<ContainerId, ContainerReport>();
  if (entities != null && entities.getEntities() != null) {
    for (TimelineEntity entity : entities.getEntities()) {
      ContainerReport container = convertToContainerReport(
          entity, serverHttpAddress, app.appReport.getUser());
      containers.put(container.getContainerId(), container);
    }
  }
  return containers;
}
 
Example #11
Source File: TestTimelineWebServices.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testGetEntityFields2() throws Exception {
  WebResource r = resource();
  ClientResponse response = r.path("ws").path("v1").path("timeline")
      .path("type_1").path("id_1").queryParam("fields", "lasteventonly," +
          "primaryfilters,relatedentities")
      .accept(MediaType.APPLICATION_JSON)
      .get(ClientResponse.class);
  assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
  TimelineEntity entity = response.getEntity(TimelineEntity.class);
  Assert.assertNotNull(entity);
  Assert.assertEquals("id_1", entity.getEntityId());
  Assert.assertEquals("type_1", entity.getEntityType());
  Assert.assertEquals(123l, entity.getStartTime().longValue());
  Assert.assertEquals(1, entity.getEvents().size());
  Assert.assertEquals(4, entity.getPrimaryFilters().size());
  Assert.assertEquals(0, entity.getOtherInfo().size());
}
 
Example #12
Source File: TestHistoryEventTimelineConversion.java    From tez with Apache License 2.0 6 votes vote down vote up
private void assertDagFinishedExtraInfoEntity(long finishTime, TimelineEntity timelineEntity) {
  Assert.assertEquals(EntityTypes.TEZ_DAG_EXTRA_INFO.name(), timelineEntity.getEntityType());
  Assert.assertEquals(tezDAGID.toString(), timelineEntity.getEntityId());

  Assert.assertEquals(1, timelineEntity.getRelatedEntities().size());
  Assert.assertTrue(
      timelineEntity.getRelatedEntities().get(ATSConstants.TEZ_DAG_ID).contains(
          tezDAGID.toString()));

  Assert.assertEquals(1, timelineEntity.getEvents().size());
  TimelineEvent timelineEvent = timelineEntity.getEvents().get(0);
  Assert.assertEquals(HistoryEventType.DAG_FINISHED.name(), timelineEvent.getEventType());
  Assert.assertEquals(finishTime, timelineEvent.getTimestamp());

  Assert.assertTrue(timelineEntity.getOtherInfo().containsKey(ATSConstants.COUNTERS));
}
 
Example #13
Source File: HistoryEventTimelineConversion.java    From tez with Apache License 2.0 6 votes vote down vote up
private static TimelineEntity convertAppLaunchedEvent(AppLaunchedEvent event) {
  TimelineEntity atsEntity = new TimelineEntity();
  atsEntity.setEntityId("tez_"
      + event.getApplicationId().toString());
  atsEntity.setEntityType(EntityTypes.TEZ_APPLICATION.name());

  atsEntity.addPrimaryFilter(ATSConstants.USER, event.getUser());

  atsEntity.addOtherInfo(ATSConstants.CONFIG,
      DAGUtils.convertConfigurationToATSMap(event.getConf()));
  atsEntity.addOtherInfo(ATSConstants.APPLICATION_ID,
          event.getApplicationId().toString());
  atsEntity.addOtherInfo(ATSConstants.USER, event.getUser());

  atsEntity.setStartTime(event.getLaunchTime());

  if (event.getVersion() != null) {
    atsEntity.addOtherInfo(ATSConstants.TEZ_VERSION,
        DAGUtils.convertTezVersionToATSMap(event.getVersion()));
  }
  atsEntity.addOtherInfo(ATSConstants.DAG_AM_WEB_SERVICE_VERSION, AMWebController.VERSION);

  return atsEntity;
}
 
Example #14
Source File: TestTimelineWebServices.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testGetEntityFields1() throws Exception {
  WebResource r = resource();
  ClientResponse response = r.path("ws").path("v1").path("timeline")
      .path("type_1").path("id_1").queryParam("fields", "events,otherinfo")
      .accept(MediaType.APPLICATION_JSON)
      .get(ClientResponse.class);
  assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
  TimelineEntity entity = response.getEntity(TimelineEntity.class);
  Assert.assertNotNull(entity);
  Assert.assertEquals("id_1", entity.getEntityId());
  Assert.assertEquals("type_1", entity.getEntityType());
  Assert.assertEquals(123l, entity.getStartTime().longValue());
  Assert.assertEquals(2, entity.getEvents().size());
  Assert.assertEquals(0, entity.getPrimaryFilters().size());
  Assert.assertEquals(4, entity.getOtherInfo().size());
}
 
Example #15
Source File: HistoryEventTimelineConversion.java    From incubator-tez with Apache License 2.0 6 votes vote down vote up
private static TimelineEntity convertTaskAttemptFinishedEvent(TaskAttemptFinishedEvent event) {
  TimelineEntity atsEntity = new TimelineEntity();
  atsEntity.setEntityId(event.getTaskAttemptID().toString());
  atsEntity.setEntityType(EntityTypes.TEZ_TASK_ATTEMPT_ID.name());

  atsEntity.addPrimaryFilter(EntityTypes.TEZ_DAG_ID.name(),
      event.getTaskAttemptID().getTaskID().getVertexID().getDAGId().toString());
  atsEntity.addPrimaryFilter(EntityTypes.TEZ_VERTEX_ID.name(),
      event.getTaskAttemptID().getTaskID().getVertexID().toString());
  atsEntity.addPrimaryFilter(EntityTypes.TEZ_TASK_ID.name(),
      event.getTaskAttemptID().getTaskID().toString());

  TimelineEvent finishEvt = new TimelineEvent();
  finishEvt.setEventType(HistoryEventType.TASK_ATTEMPT_FINISHED.name());
  finishEvt.setTimestamp(event.getFinishTime());
  atsEntity.addEvent(finishEvt);

  atsEntity.addOtherInfo(ATSConstants.FINISH_TIME, event.getFinishTime());
  atsEntity.addOtherInfo(ATSConstants.TIME_TAKEN, (event.getFinishTime() - event.getStartTime()));
  atsEntity.addOtherInfo(ATSConstants.STATUS, event.getState().name());
  atsEntity.addOtherInfo(ATSConstants.DIAGNOSTICS, event.getDiagnostics());
  atsEntity.addOtherInfo(ATSConstants.COUNTERS,
      DAGUtils.convertCountersToATSMap(event.getCounters()));

  return atsEntity;
}
 
Example #16
Source File: HistoryEventTimelineConversion.java    From incubator-tez with Apache License 2.0 6 votes vote down vote up
private static TimelineEntity convertAMLaunchedEvent(AMLaunchedEvent event) {
  TimelineEntity atsEntity = new TimelineEntity();
  atsEntity.setEntityId("tez_"
      + event.getApplicationAttemptId().toString());
  atsEntity.setEntityType(EntityTypes.TEZ_APPLICATION_ATTEMPT.name());

  atsEntity.addRelatedEntity(ATSConstants.APPLICATION_ID,
      event.getApplicationAttemptId().getApplicationId().toString());
  atsEntity.addRelatedEntity(ATSConstants.APPLICATION_ATTEMPT_ID,
      event.getApplicationAttemptId().toString());
  atsEntity.addRelatedEntity(ATSConstants.USER, event.getUser());

  atsEntity.addPrimaryFilter(ATSConstants.USER, event.getUser());

  atsEntity.setStartTime(event.getLaunchTime());

  TimelineEvent launchEvt = new TimelineEvent();
  launchEvt.setEventType(HistoryEventType.AM_LAUNCHED.name());
  launchEvt.setTimestamp(event.getLaunchTime());
  atsEntity.addEvent(launchEvt);

  atsEntity.addOtherInfo(ATSConstants.APP_SUBMIT_TIME, event.getAppSubmitTime());

  return atsEntity;
}
 
Example #17
Source File: TestTimelineACLsManager.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testYarnACLsNotEnabledForEntity() throws Exception {
  Configuration conf = new YarnConfiguration();
  conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, false);
  TimelineACLsManager timelineACLsManager =
      new TimelineACLsManager(conf);
  timelineACLsManager.setTimelineStore(new TestTimelineStore());
  TimelineEntity entity = new TimelineEntity();
  entity.addPrimaryFilter(
      TimelineStore.SystemFilter.ENTITY_OWNER
          .toString(), "owner");
  entity.setDomainId("domain_id_1");
  Assert.assertTrue(
      "Always true when ACLs are not enabled",
      timelineACLsManager.checkAccess(
          UserGroupInformation.createRemoteUser("user"),
          ApplicationAccessType.VIEW_APP, entity));
  Assert.assertTrue(
      "Always true when ACLs are not enabled",
      timelineACLsManager.checkAccess(
          UserGroupInformation.createRemoteUser("user"),
          ApplicationAccessType.MODIFY_APP, entity));
}
 
Example #18
Source File: TestTimelineClient.java    From big-c with Apache License 2.0 6 votes vote down vote up
private static TimelineEntity generateEntity() {
  TimelineEntity entity = new TimelineEntity();
  entity.setEntityId("entity id");
  entity.setEntityType("entity type");
  entity.setStartTime(System.currentTimeMillis());
  for (int i = 0; i < 2; ++i) {
    TimelineEvent event = new TimelineEvent();
    event.setTimestamp(System.currentTimeMillis());
    event.setEventType("test event type " + i);
    event.addEventInfo("key1", "val1");
    event.addEventInfo("key2", "val2");
    entity.addEvent(event);
  }
  entity.addRelatedEntity("test ref type 1", "test ref id 1");
  entity.addRelatedEntity("test ref type 2", "test ref id 2");
  entity.addPrimaryFilter("pkey1", "pval1");
  entity.addPrimaryFilter("pkey2", "pval2");
  entity.addOtherInfo("okey1", "oval1");
  entity.addOtherInfo("okey2", "oval2");
  entity.setDomainId("domain id 1");
  return entity;
}
 
Example #19
Source File: HistoryEventTimelineConversion.java    From incubator-tez with Apache License 2.0 6 votes vote down vote up
private static TimelineEntity convertVertexStartedEvent(VertexStartedEvent event) {
  TimelineEntity atsEntity = new TimelineEntity();
  atsEntity.setEntityId(event.getVertexID().toString());
  atsEntity.setEntityType(EntityTypes.TEZ_VERTEX_ID.name());

  atsEntity.addPrimaryFilter(EntityTypes.TEZ_DAG_ID.name(),
      event.getVertexID().getDAGId().toString());

  TimelineEvent startEvt = new TimelineEvent();
  startEvt.setEventType(HistoryEventType.VERTEX_STARTED.name());
  startEvt.setTimestamp(event.getStartTime());
  atsEntity.addEvent(startEvt);

  atsEntity.addOtherInfo(ATSConstants.START_REQUESTED_TIME, event.getStartRequestedTime());
  atsEntity.addOtherInfo(ATSConstants.START_TIME, event.getStartTime());

  return atsEntity;
}
 
Example #20
Source File: ApplicationMaster.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private static void publishApplicationAttemptEvent(
    final TimelineClient timelineClient, String appAttemptId,
    DSEvent appEvent, String domainId, UserGroupInformation ugi) {
  final TimelineEntity entity = new TimelineEntity();
  entity.setEntityId(appAttemptId);
  entity.setEntityType(DSEntity.DS_APP_ATTEMPT.toString());
  entity.setDomainId(domainId);
  entity.addPrimaryFilter("user", ugi.getShortUserName());
  TimelineEvent event = new TimelineEvent();
  event.setEventType(appEvent.toString());
  event.setTimestamp(System.currentTimeMillis());
  entity.addEvent(event);
  try {
    timelineClient.putEntities(entity);
  } catch (YarnException | IOException e) {
    LOG.error("App Attempt "
        + (appEvent.equals(DSEvent.DS_APP_ATTEMPT_START) ? "start" : "end")
        + " event could not be published for "
        + appAttemptId.toString(), e);
  }
}
 
Example #21
Source File: ApplicationMaster.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private static void publishContainerEndEvent(
    final TimelineClient timelineClient, ContainerStatus container,
    String domainId, UserGroupInformation ugi) {
  final TimelineEntity entity = new TimelineEntity();
  entity.setEntityId(container.getContainerId().toString());
  entity.setEntityType(DSEntity.DS_CONTAINER.toString());
  entity.setDomainId(domainId);
  entity.addPrimaryFilter("user", ugi.getShortUserName());
  TimelineEvent event = new TimelineEvent();
  event.setTimestamp(System.currentTimeMillis());
  event.setEventType(DSEvent.DS_CONTAINER_END.toString());
  event.addEventInfo("State", container.getState().name());
  event.addEventInfo("Exit Status", container.getExitStatus());
  entity.addEvent(event);
  try {
    timelineClient.putEntities(entity);
  } catch (YarnException | IOException e) {
    LOG.error("Container end event could not be published for "
        + container.getContainerId().toString(), e);
  }
}
 
Example #22
Source File: TestTimelineWebServices.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testGetEntity() throws Exception {
  WebResource r = resource();
  ClientResponse response = r.path("ws").path("v1").path("timeline")
      .path("type_1").path("id_1")
      .accept(MediaType.APPLICATION_JSON)
      .get(ClientResponse.class);
  assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
  TimelineEntity entity = response.getEntity(TimelineEntity.class);
  Assert.assertNotNull(entity);
  Assert.assertEquals("id_1", entity.getEntityId());
  Assert.assertEquals("type_1", entity.getEntityType());
  Assert.assertEquals(123l, entity.getStartTime().longValue());
  Assert.assertEquals(2, entity.getEvents().size());
  Assert.assertEquals(4, entity.getPrimaryFilters().size());
  Assert.assertEquals(4, entity.getOtherInfo().size());
}
 
Example #23
Source File: SystemMetricsPublisher.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private void publishContainerFinishedEvent(ContainerFinishedEvent event) {
  TimelineEntity entity = createContainerEntity(event.getContainerId());
  TimelineEvent tEvent = new TimelineEvent();
  tEvent.setEventType(ContainerMetricsConstants.FINISHED_EVENT_TYPE);
  tEvent.setTimestamp(event.getTimestamp());
  Map<String, Object> eventInfo = new HashMap<String, Object>();
  eventInfo.put(ContainerMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO,
      event.getDiagnosticsInfo());
  eventInfo.put(ContainerMetricsConstants.EXIT_STATUS_EVENT_INFO,
      event.getContainerExitStatus());
  eventInfo.put(ContainerMetricsConstants.STATE_EVENT_INFO,
      event.getContainerState().toString());
  tEvent.setEventInfo(eventInfo);
  entity.addEvent(tEvent);
  putEntity(entity);
}
 
Example #24
Source File: TestLeveldbTimelineStore.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testDeleteEntities() throws IOException, InterruptedException {
  assertEquals(3, getEntities("type_1").size());
  assertEquals(1, getEntities("type_2").size());

  assertEquals(false, deleteNextEntity(entityType1,
      writeReverseOrderedLong(60l)));
  assertEquals(3, getEntities("type_1").size());
  assertEquals(1, getEntities("type_2").size());

  assertEquals(true, deleteNextEntity(entityType1,
      writeReverseOrderedLong(123l)));
  List<TimelineEntity> entities = getEntities("type_2");
  assertEquals(1, entities.size());
  verifyEntityInfo(entityId2, entityType2, events2, Collections.singletonMap(
      entityType1, Collections.singleton(entityId1b)), EMPTY_PRIMARY_FILTERS,
      EMPTY_MAP, entities.get(0), domainId1);
  entities = getEntitiesWithPrimaryFilter("type_1", userFilter);
  assertEquals(2, entities.size());
  verifyEntityInfo(entityId1b, entityType1, events1, EMPTY_REL_ENTITIES,
      primaryFilters, otherInfo, entities.get(0), domainId1);
  // can retrieve entities across domains
  verifyEntityInfo(entityId6, entityType1, EMPTY_EVENTS, EMPTY_REL_ENTITIES,
      primaryFilters, otherInfo, entities.get(1), domainId2);

  ((LeveldbTimelineStore)store).discardOldEntities(-123l);
  assertEquals(2, getEntities("type_1").size());
  assertEquals(0, getEntities("type_2").size());
  assertEquals(6, ((LeveldbTimelineStore)store).getEntityTypes().size());

  ((LeveldbTimelineStore)store).discardOldEntities(123l);
  assertEquals(0, getEntities("type_1").size());
  assertEquals(0, getEntities("type_2").size());
  assertEquals(0, ((LeveldbTimelineStore)store).getEntityTypes().size());
  assertEquals(0, getEntitiesWithPrimaryFilter("type_1", userFilter).size());
}
 
Example #25
Source File: TestHistoryEventTimelineConversion.java    From tez with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 5000)
public void testConvertDAGRecoveredEvent() {
  long recoverTime = random.nextLong();

  DAGRecoveredEvent event = new DAGRecoveredEvent(applicationAttemptId, tezDAGID,
      dagPlan.getName(), user, recoverTime, containerLogs);

  List<TimelineEntity> entities = HistoryEventTimelineConversion.convertToTimelineEntities(event);
  Assert.assertEquals(1, entities.size());
  TimelineEntity timelineEntity = entities.get(0);

  Assert.assertEquals(EntityTypes.TEZ_DAG_ID.name(), timelineEntity.getEntityType());
  Assert.assertEquals(tezDAGID.toString(), timelineEntity.getEntityId());

  Assert.assertEquals(0, timelineEntity.getRelatedEntities().size());

  Assert.assertEquals(1, timelineEntity.getEvents().size());
  TimelineEvent timelineEvent = timelineEntity.getEvents().get(0);
  Assert.assertEquals(HistoryEventType.DAG_RECOVERED.name(), timelineEvent.getEventType());
  Assert.assertEquals(recoverTime, timelineEvent.getTimestamp());

  Assert.assertTrue(timelineEvent.getEventInfo().containsKey(ATSConstants.APPLICATION_ATTEMPT_ID));
  Assert.assertEquals(applicationAttemptId.toString(),
      timelineEvent.getEventInfo().get(ATSConstants.APPLICATION_ATTEMPT_ID));

  Assert.assertEquals(3, timelineEntity.getPrimaryFilters().size());
  Assert.assertTrue(
      timelineEntity.getPrimaryFilters().get(ATSConstants.APPLICATION_ID).contains(
          applicationId.toString()));
  Assert.assertTrue(
      timelineEntity.getPrimaryFilters().get(ATSConstants.DAG_NAME).contains("DAGPlanMock"));
  Assert.assertTrue(
      timelineEntity.getPrimaryFilters().get(ATSConstants.USER).contains(user));
  Assert.assertEquals(containerLogs,
      timelineEntity.getOtherInfo().get(ATSConstants.IN_PROGRESS_LOGS_URL + "_"
          + applicationAttemptId.getAttemptId()));
}
 
Example #26
Source File: TimelineStoreTestUtils.java    From big-c with Apache License 2.0 5 votes vote down vote up
public void testGetEntitiesWithSecondaryFilters() throws IOException {
  // test using secondary filter
  List<TimelineEntity> entities = getEntitiesWithFilters("type_1", null,
      goodTestingFilters);
  assertEquals(3, entities.size());
  verifyEntityInfo(entityId1, entityType1, events1, EMPTY_REL_ENTITIES,
      primaryFilters, otherInfo, entities.get(0), domainId1);
  verifyEntityInfo(entityId1b, entityType1, events1, EMPTY_REL_ENTITIES,
      primaryFilters, otherInfo, entities.get(1), domainId1);
  verifyEntityInfo(entityId6, entityType1, EMPTY_EVENTS, EMPTY_REL_ENTITIES,
      primaryFilters, otherInfo, entities.get(2), domainId2);

  entities = getEntitiesWithFilters("type_1", userFilter, goodTestingFilters);
  assertEquals(3, entities.size());
  verifyEntityInfo(entityId1, entityType1, events1, EMPTY_REL_ENTITIES,
      primaryFilters, otherInfo, entities.get(0), domainId1);
  verifyEntityInfo(entityId1b, entityType1, events1, EMPTY_REL_ENTITIES,
      primaryFilters, otherInfo, entities.get(1), domainId1);
  verifyEntityInfo(entityId6, entityType1, EMPTY_EVENTS, EMPTY_REL_ENTITIES,
      primaryFilters, otherInfo, entities.get(2), domainId2);

  entities = getEntitiesWithFilters("type_1", null,
      Collections.singleton(new NameValuePair("user", "none")));
  assertEquals(0, entities.size());

  entities = getEntitiesWithFilters("type_1", null, badTestingFilters);
  assertEquals(0, entities.size());

  entities = getEntitiesWithFilters("type_1", userFilter, badTestingFilters);
  assertEquals(0, entities.size());

  entities = getEntitiesWithFilters("type_5", null, badTestingFilters);
  assertEquals(0, entities.size());
}
 
Example #27
Source File: TestTimelineWebServices.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testPostEntitiesToDefaultDomain() throws Exception {
  AdminACLsManager oldAdminACLsManager =
      timelineACLsManager.setAdminACLsManager(adminACLsManager);
  try {
    TimelineEntities entities = new TimelineEntities();
    TimelineEntity entity = new TimelineEntity();
    entity.setEntityId("test id 7");
    entity.setEntityType("test type 7");
    entity.setStartTime(System.currentTimeMillis());
    entities.addEntity(entity);
    WebResource r = resource();
    ClientResponse response = r.path("ws").path("v1").path("timeline")
        .queryParam("user.name", "anybody_1")
        .accept(MediaType.APPLICATION_JSON)
        .type(MediaType.APPLICATION_JSON)
        .post(ClientResponse.class, entities);
    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
    TimelinePutResponse putResposne =
        response.getEntity(TimelinePutResponse.class);
    Assert.assertNotNull(putResposne);
    Assert.assertEquals(0, putResposne.getErrors().size());
    // verify the entity exists in the store
    response = r.path("ws").path("v1").path("timeline")
        .path("test type 7").path("test id 7")
        .queryParam("user.name", "any_body_2")
        .accept(MediaType.APPLICATION_JSON)
        .get(ClientResponse.class);
    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
    entity = response.getEntity(TimelineEntity.class);
    Assert.assertNotNull(entity);
    Assert.assertEquals("test id 7", entity.getEntityId());
    Assert.assertEquals("test type 7", entity.getEntityType());
    Assert.assertEquals(TimelineDataManager.DEFAULT_DOMAIN_ID,
        entity.getDomainId());
  } finally {
    timelineACLsManager.setAdminACLsManager(oldAdminACLsManager);
  }
}
 
Example #28
Source File: SystemMetricsPublisher.java    From big-c with Apache License 2.0 5 votes vote down vote up
private void publishApplicationACLsUpdatedEvent(
    ApplicationACLsUpdatedEvent event) {
  TimelineEntity entity =
      createApplicationEntity(event.getApplicationId());
  TimelineEvent tEvent = new TimelineEvent();
  Map<String, Object> entityInfo = new HashMap<String, Object>();
  entityInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO,
      event.getViewAppACLs());
  entity.setOtherInfo(entityInfo);
  tEvent.setEventType(
      ApplicationMetricsConstants.ACLS_UPDATED_EVENT_TYPE);
  tEvent.setTimestamp(event.getTimestamp());
  entity.addEvent(tEvent);
  putEntity(entity);
}
 
Example #29
Source File: TestATSV15HistoryLoggingService.java    From tez with Apache License 2.0 5 votes vote down vote up
private TimelinePutResponse putEntityHelper(TimelineEntityGroupId groupId, Object[] args, int firstEntityIdx) {
  List<TimelineEntity> groupEntities = entityLog.get(groupId);
  if (groupEntities == null) {
    groupEntities = new ArrayList<>();
    entityLog.put(groupId, groupEntities);
  }
  for (int i = firstEntityIdx; i < args.length; i++) {
    groupEntities.add((TimelineEntity) args[i]);
  }
  return null;
}
 
Example #30
Source File: HistoryEventTimelineConversion.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
private static TimelineEntity convertAMStartedEvent(AMStartedEvent event) {
  TimelineEntity atsEntity = new TimelineEntity();
  atsEntity.setEntityId("tez_"
      + event.getApplicationAttemptId().toString());
  atsEntity.setEntityType(EntityTypes.TEZ_APPLICATION_ATTEMPT.name());

  atsEntity.addPrimaryFilter(ATSConstants.USER, event.getUser());

  TimelineEvent startEvt = new TimelineEvent();
  startEvt.setEventType(HistoryEventType.AM_STARTED.name());
  startEvt.setTimestamp(event.getStartTime());
  atsEntity.addEvent(startEvt);

  return atsEntity;
}