org.apache.hadoop.yarn.client.api.TimelineClient Java Examples

The following examples show how to use org.apache.hadoop.yarn.client.api.TimelineClient. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TimelineDelegationTokenIdentifier.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
public long renew(Token<?> token, Configuration conf) throws IOException,
    InterruptedException {
  TimelineClient client = TimelineClient.createTimelineClient();
  try {
    client.init(conf);
    client.start();
    return client.renewDelegationToken(
        (Token<TimelineDelegationTokenIdentifier>) token);
  } catch (YarnException e) {
    throw new IOException(e);
  } finally {
    client.stop();
  }
}
 
Example #2
Source File: JstormMaster.java    From jstorm with Apache License 2.0 6 votes vote down vote up
void startTimelineClient(final Configuration conf)
        throws YarnException, IOException, InterruptedException {
    try {
        appSubmitterUgi.doAs(new PrivilegedExceptionAction<Void>() {
            @Override
            public Void run() throws Exception {
                if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
                        YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
                    // Creating the Timeline Client
                    timelineClient = TimelineClient.createTimelineClient();
                    timelineClient.init(conf);
                    timelineClient.start();
                } else {
                    timelineClient = null;
                    LOG.warn("Timeline service is not enabled");
                }
                return null;
            }
        });
    } catch (UndeclaredThrowableException e) {
        throw new YarnException(e.getCause());
    }
}
 
Example #3
Source File: ApplicationMaster.java    From metron with Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
void startTimelineClient(final Configuration conf)
        throws YarnException, IOException, InterruptedException {
  try {
    appSubmitterUgi.doAs(new PrivilegedExceptionAction<Void>() {
      @Override
      public Void run() throws Exception {
        if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
                YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
          // Creating the Timeline Client
          timelineClient = TimelineClient.createTimelineClient();
          timelineClient.init(conf);
          timelineClient.start();
        } else {
          timelineClient = null;
          LOG.warn("Timeline service is not enabled");
        }
        return null;
      }
    });
  } catch (UndeclaredThrowableException e) {
    throw new YarnException(e.getCause());
  }
}
 
Example #4
Source File: ApplicationMaster.java    From big-c with Apache License 2.0 6 votes vote down vote up
private static void publishApplicationAttemptEvent(
    final TimelineClient timelineClient, String appAttemptId,
    DSEvent appEvent, String domainId, UserGroupInformation ugi) {
  final TimelineEntity entity = new TimelineEntity();
  entity.setEntityId(appAttemptId);
  entity.setEntityType(DSEntity.DS_APP_ATTEMPT.toString());
  entity.setDomainId(domainId);
  entity.addPrimaryFilter("user", ugi.getShortUserName());
  TimelineEvent event = new TimelineEvent();
  event.setEventType(appEvent.toString());
  event.setTimestamp(System.currentTimeMillis());
  entity.addEvent(event);
  try {
    timelineClient.putEntities(entity);
  } catch (YarnException | IOException e) {
    LOG.error("App Attempt "
        + (appEvent.equals(DSEvent.DS_APP_ATTEMPT_START) ? "start" : "end")
        + " event could not be published for "
        + appAttemptId.toString(), e);
  }
}
 
Example #5
Source File: ApplicationMaster.java    From big-c with Apache License 2.0 6 votes vote down vote up
private static void publishContainerEndEvent(
    final TimelineClient timelineClient, ContainerStatus container,
    String domainId, UserGroupInformation ugi) {
  final TimelineEntity entity = new TimelineEntity();
  entity.setEntityId(container.getContainerId().toString());
  entity.setEntityType(DSEntity.DS_CONTAINER.toString());
  entity.setDomainId(domainId);
  entity.addPrimaryFilter("user", ugi.getShortUserName());
  TimelineEvent event = new TimelineEvent();
  event.setTimestamp(System.currentTimeMillis());
  event.setEventType(DSEvent.DS_CONTAINER_END.toString());
  event.addEventInfo("State", container.getState().name());
  event.addEventInfo("Exit Status", container.getExitStatus());
  entity.addEvent(event);
  try {
    timelineClient.putEntities(entity);
  } catch (YarnException | IOException e) {
    LOG.error("Container end event could not be published for "
        + container.getContainerId().toString(), e);
  }
}
 
Example #6
Source File: ApplicationMaster.java    From big-c with Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
void startTimelineClient(final Configuration conf)
    throws YarnException, IOException, InterruptedException {
  try {
    appSubmitterUgi.doAs(new PrivilegedExceptionAction<Void>() {
      @Override
      public Void run() throws Exception {
        if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
            YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
          // Creating the Timeline Client
          timelineClient = TimelineClient.createTimelineClient();
          timelineClient.init(conf);
          timelineClient.start();
        } else {
          timelineClient = null;
          LOG.warn("Timeline service is not enabled");
        }
        return null;
      }
    });
  } catch (UndeclaredThrowableException e) {
    throw new YarnException(e.getCause());
  }
}
 
Example #7
Source File: TestTimelineAuthenticationFilter.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testPutDomains() throws Exception {
  KerberosTestUtils.doAs(HTTP_USER + "/localhost", new Callable<Void>() {
    @Override
    public Void call() throws Exception {
      TimelineClient client = createTimelineClientForUGI();
      TimelineDomain domainToStore = new TimelineDomain();
      domainToStore.setId(TestTimelineAuthenticationFilter.class.getName());
      domainToStore.setReaders("*");
      domainToStore.setWriters("*");
      client.putDomain(domainToStore);
      TimelineDomain domainToRead =
          testTimelineServer.getTimelineStore().getDomain(
              TestTimelineAuthenticationFilter.class.getName());
      Assert.assertNotNull(domainToRead);
      return null;
    }
  });
}
 
Example #8
Source File: TestTimelineAuthenticationFilter.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testPutTimelineEntities() throws Exception {
  KerberosTestUtils.doAs(HTTP_USER + "/localhost", new Callable<Void>() {
    @Override
    public Void call() throws Exception {
      TimelineClient client = createTimelineClientForUGI();
      TimelineEntity entityToStore = new TimelineEntity();
      entityToStore.setEntityType(
          TestTimelineAuthenticationFilter.class.getName());
      entityToStore.setEntityId("entity1");
      entityToStore.setStartTime(0L);
      TimelinePutResponse putResponse = client.putEntities(entityToStore);
      Assert.assertEquals(0, putResponse.getErrors().size());
      TimelineEntity entityToRead =
          testTimelineServer.getTimelineStore().getEntity(
              "entity1", TestTimelineAuthenticationFilter.class.getName(), null);
      Assert.assertNotNull(entityToRead);
      return null;
    }
  });
}
 
Example #9
Source File: SystemMetricsPublisher.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
protected void serviceInit(Configuration conf) throws Exception {
  publishSystemMetrics =
      conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
          YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED) &&
      conf.getBoolean(YarnConfiguration.RM_SYSTEM_METRICS_PUBLISHER_ENABLED,
          YarnConfiguration.DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_ENABLED);

  if (publishSystemMetrics) {
    client = TimelineClient.createTimelineClient();
    addIfService(client);

    dispatcher = createDispatcher(conf);
    dispatcher.register(SystemMetricsEventType.class,
        new ForwardingEventHandler());
    addIfService(dispatcher);
    LOG.info("YARN system metrics publishing service is enabled");
  } else {
    LOG.info("YARN system metrics publishing service is not enabled");
  }
  super.serviceInit(conf);
}
 
Example #10
Source File: TimelineDelegationTokenIdentifier.java    From big-c with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
public void cancel(Token<?> token, Configuration conf) throws IOException,
    InterruptedException {
  TimelineClient client = TimelineClient.createTimelineClient();
  try {
    client.init(conf);
    client.start();
    client.cancelDelegationToken(
        (Token<TimelineDelegationTokenIdentifier>) token);
  } catch (YarnException e) {
    throw new IOException(e);
  } finally {
    client.stop();
  }
}
 
Example #11
Source File: TimelineDelegationTokenIdentifier.java    From big-c with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
public long renew(Token<?> token, Configuration conf) throws IOException,
    InterruptedException {
  TimelineClient client = TimelineClient.createTimelineClient();
  try {
    client.init(conf);
    client.start();
    return client.renewDelegationToken(
        (Token<TimelineDelegationTokenIdentifier>) token);
  } catch (YarnException e) {
    throw new IOException(e);
  } finally {
    client.stop();
  }
}
 
Example #12
Source File: ApplicationMaster.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private static void publishApplicationAttemptEvent(
    final TimelineClient timelineClient, String appAttemptId,
    DSEvent appEvent, String domainId, UserGroupInformation ugi) {
  final TimelineEntity entity = new TimelineEntity();
  entity.setEntityId(appAttemptId);
  entity.setEntityType(DSEntity.DS_APP_ATTEMPT.toString());
  entity.setDomainId(domainId);
  entity.addPrimaryFilter("user", ugi.getShortUserName());
  TimelineEvent event = new TimelineEvent();
  event.setEventType(appEvent.toString());
  event.setTimestamp(System.currentTimeMillis());
  entity.addEvent(event);
  try {
    timelineClient.putEntities(entity);
  } catch (YarnException | IOException e) {
    LOG.error("App Attempt "
        + (appEvent.equals(DSEvent.DS_APP_ATTEMPT_START) ? "start" : "end")
        + " event could not be published for "
        + appAttemptId.toString(), e);
  }
}
 
Example #13
Source File: ApplicationMaster.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
void startTimelineClient(final Configuration conf)
    throws YarnException, IOException, InterruptedException {
  try {
    appSubmitterUgi.doAs(new PrivilegedExceptionAction<Void>() {
      @Override
      public Void run() throws Exception {
        if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
            YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
          // Creating the Timeline Client
          timelineClient = TimelineClient.createTimelineClient();
          timelineClient.init(conf);
          timelineClient.start();
        } else {
          timelineClient = null;
          LOG.warn("Timeline service is not enabled");
        }
        return null;
      }
    });
  } catch (UndeclaredThrowableException e) {
    throw new YarnException(e.getCause());
  }
}
 
Example #14
Source File: YarnModule.java    From sylph with Apache License 2.0 6 votes vote down vote up
@Override
public YarnClient get()
{
    YarnClient client = YarnClient.createYarnClient();
    if (yarnConfiguration.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false)) {
        try {
            TimelineClient.createTimelineClient();
        }
        catch (NoClassDefFoundError e) {
            logger.warn("createTimelineClient() error with {}", TimelineClient.class.getResource(TimelineClient.class.getSimpleName() + ".class"), e);
            yarnConfiguration.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false);
        }
    }
    client.init(yarnConfiguration);
    client.start();
    return client;
}
 
Example #15
Source File: TestTimelineAuthenticationFilter.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testPutDomains() throws Exception {
  KerberosTestUtils.doAs(HTTP_USER + "/localhost", new Callable<Void>() {
    @Override
    public Void call() throws Exception {
      TimelineClient client = createTimelineClientForUGI();
      TimelineDomain domainToStore = new TimelineDomain();
      domainToStore.setId(TestTimelineAuthenticationFilter.class.getName());
      domainToStore.setReaders("*");
      domainToStore.setWriters("*");
      client.putDomain(domainToStore);
      TimelineDomain domainToRead =
          testTimelineServer.getTimelineStore().getDomain(
              TestTimelineAuthenticationFilter.class.getName());
      Assert.assertNotNull(domainToRead);
      return null;
    }
  });
}
 
Example #16
Source File: TestTimelineAuthenticationFilter.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testPutTimelineEntities() throws Exception {
  KerberosTestUtils.doAs(HTTP_USER + "/localhost", new Callable<Void>() {
    @Override
    public Void call() throws Exception {
      TimelineClient client = createTimelineClientForUGI();
      TimelineEntity entityToStore = new TimelineEntity();
      entityToStore.setEntityType(
          TestTimelineAuthenticationFilter.class.getName());
      entityToStore.setEntityId("entity1");
      entityToStore.setStartTime(0L);
      TimelinePutResponse putResponse = client.putEntities(entityToStore);
      Assert.assertEquals(0, putResponse.getErrors().size());
      TimelineEntity entityToRead =
          testTimelineServer.getTimelineStore().getEntity(
              "entity1", TestTimelineAuthenticationFilter.class.getName(), null);
      Assert.assertNotNull(entityToRead);
      return null;
    }
  });
}
 
Example #17
Source File: SystemMetricsPublisher.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
protected void serviceInit(Configuration conf) throws Exception {
  publishSystemMetrics =
      conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
          YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED) &&
      conf.getBoolean(YarnConfiguration.RM_SYSTEM_METRICS_PUBLISHER_ENABLED,
          YarnConfiguration.DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_ENABLED);

  if (publishSystemMetrics) {
    client = TimelineClient.createTimelineClient();
    addIfService(client);

    dispatcher = createDispatcher(conf);
    dispatcher.register(SystemMetricsEventType.class,
        new ForwardingEventHandler());
    addIfService(dispatcher);
    LOG.info("YARN system metrics publishing service is enabled");
  } else {
    LOG.info("YARN system metrics publishing service is not enabled");
  }
  super.serviceInit(conf);
}
 
Example #18
Source File: ApplicationMaster.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private static void publishContainerEndEvent(
    final TimelineClient timelineClient, ContainerStatus container,
    String domainId, UserGroupInformation ugi) {
  final TimelineEntity entity = new TimelineEntity();
  entity.setEntityId(container.getContainerId().toString());
  entity.setEntityType(DSEntity.DS_CONTAINER.toString());
  entity.setDomainId(domainId);
  entity.addPrimaryFilter("user", ugi.getShortUserName());
  TimelineEvent event = new TimelineEvent();
  event.setTimestamp(System.currentTimeMillis());
  event.setEventType(DSEvent.DS_CONTAINER_END.toString());
  event.addEventInfo("State", container.getState().name());
  event.addEventInfo("Exit Status", container.getExitStatus());
  entity.addEvent(event);
  try {
    timelineClient.putEntities(entity);
  } catch (YarnException | IOException e) {
    LOG.error("Container end event could not be published for "
        + container.getContainerId().toString(), e);
  }
}
 
Example #19
Source File: TimelineDelegationTokenIdentifier.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
public void cancel(Token<?> token, Configuration conf) throws IOException,
    InterruptedException {
  TimelineClient client = TimelineClient.createTimelineClient();
  try {
    client.init(conf);
    client.start();
    client.cancelDelegationToken(
        (Token<TimelineDelegationTokenIdentifier>) token);
  } catch (YarnException e) {
    throw new IOException(e);
  } finally {
    client.stop();
  }
}
 
Example #20
Source File: TestTimelineClient.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private static TimelineClientImpl createTimelineClient(
    YarnConfiguration conf) {
  TimelineClientImpl client =
      spy((TimelineClientImpl) TimelineClient.createTimelineClient());
  client.init(conf);
  client.start();
  return client;
}
 
Example #21
Source File: JstormMaster.java    From jstorm with Apache License 2.0 5 votes vote down vote up
private static void publishContainerStartEvent(
        final TimelineClient timelineClient, Container container, String domainId,
        UserGroupInformation ugi) {
    final TimelineEntity entity = new TimelineEntity();
    entity.setEntityId(container.getId().toString());
    entity.setEntityType(DSEntity.DS_CONTAINER.toString());
    entity.setDomainId(domainId);
    entity.addPrimaryFilter(JOYConstants.USER, ugi.getShortUserName());
    TimelineEvent event = new TimelineEvent();
    event.setTimestamp(System.currentTimeMillis());
    event.setEventType(DSEvent.DS_CONTAINER_START.toString());
    event.addEventInfo(JOYConstants.NODE, container.getNodeId().toString());
    event.addEventInfo(JOYConstants.RESOURCES, container.getResource().toString());
    entity.addEvent(event);

    try {
        ugi.doAs(new PrivilegedExceptionAction<TimelinePutResponse>() {
            @Override
            public TimelinePutResponse run() throws Exception {
                return timelineClient.putEntities(entity);
            }
        });
    } catch (Exception e) {
        LOG.error("Container start event could not be published for "
                        + container.getId().toString(),
                e instanceof UndeclaredThrowableException ? e.getCause() : e);
    }
}
 
Example #22
Source File: JstormOnYarn.java    From jstorm with Apache License 2.0 5 votes vote down vote up
private void prepareTimelineDomain() {
    TimelineClient timelineClient = null;
    if (jstormClientContext.conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
            YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
        timelineClient = TimelineClient.createTimelineClient();
        timelineClient.init(jstormClientContext.conf);
        timelineClient.start();
    } else {
        LOG.warn("Cannot put the domain " + jstormClientContext.domainId +
                " because the timeline service is not enabled");
        return;
    }
    try {
        TimelineDomain domain = new TimelineDomain();
        domain.setId(jstormClientContext.domainId);
        domain.setReaders(
                jstormClientContext.viewACLs != null && jstormClientContext.viewACLs.length() > 0 ? jstormClientContext.viewACLs : JOYConstants.BLANK);
        domain.setWriters(
                jstormClientContext.modifyACLs != null && jstormClientContext.modifyACLs.length() > 0 ? jstormClientContext.modifyACLs : JOYConstants.BLANK);
        timelineClient.putDomain(domain);
        LOG.info("Put the timeline domain: " +
                TimelineUtils.dumpTimelineRecordtoJSON(domain));
    } catch (Exception e) {
        LOG.error("Error when putting the timeline domain", e);
    } finally {
        timelineClient.stop();
    }
}
 
Example #23
Source File: ATSHistoryACLPolicyManager.java    From tez with Apache License 2.0 5 votes vote down vote up
private void initializeTimelineClient() {
  if (this.conf == null) {
    throw new TezUncheckedException("ATSACLManager not configured");
  }
  if (timelineClient != null) {
    this.timelineClient.stop();
    this.timelineClient = null;
  }
  if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
    YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
    this.timelineClient = TimelineClient.createTimelineClient();
    this.timelineClient.init(this.conf);
    this.timelineClient.start();
  } else {
    this.timelineClient = null;
    if (conf.get(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, "")
       .equals(atsHistoryLoggingServiceClassName)) {
      LOG.warn(atsHistoryLoggingServiceClassName
          + " is disabled due to Timeline Service being disabled, "
          + YarnConfiguration.TIMELINE_SERVICE_ENABLED + " set to false");
    }
  }
  try {
    this.user = UserGroupInformation.getCurrentUser().getShortUserName();
  } catch (IOException e) {
    throw new TezUncheckedException("Unable to get Current User UGI", e);
  }
}
 
Example #24
Source File: TestATSHistoryLoggingService.java    From tez with Apache License 2.0 5 votes vote down vote up
@Before
public void setup() throws Exception {
  appContext = mock(AppContext.class);
  historyACLPolicyManager = mock(HistoryACLPolicyManager.class);
  atsHistoryLoggingService = new ATSHistoryLoggingService();
  atsHistoryLoggingService.setAppContext(appContext);
  conf = new Configuration(false);
  conf.setLong(TezConfiguration.YARN_ATS_EVENT_FLUSH_TIMEOUT_MILLIS,
      1000l);
  conf.setInt(TezConfiguration.YARN_ATS_MAX_EVENTS_PER_BATCH, 2);
  conf.setBoolean(TezConfiguration.TEZ_AM_ALLOW_DISABLED_TIMELINE_DOMAINS, true);
  conf.set(TezConfiguration.YARN_ATS_ACL_SESSION_DOMAIN_ID, "test-domain");
  atsInvokeCounter = 0;
  atsEntitiesCounter = 0;
  atsHistoryLoggingService.init(conf);
  atsHistoryLoggingService.historyACLPolicyManager = historyACLPolicyManager;
  atsHistoryLoggingService.timelineClient = mock(TimelineClient.class);

  when(appContext.getClock()).thenReturn(clock);
  when(appContext.getCurrentDAGID()).thenReturn(null);
  when(appContext.getApplicationID()).thenReturn(appId);
  when(atsHistoryLoggingService.timelineClient.putEntities(
      Matchers.<TimelineEntity[]>anyVararg())).thenAnswer(
      new Answer<Object>() {
        @Override
        public Object answer(InvocationOnMock invocation) throws Throwable {
          ++atsInvokeCounter;
          atsEntitiesCounter += invocation.getArguments().length;
          try {
            Thread.sleep(500l);
          } catch (InterruptedException e) {
            // do nothing
          }
          return null;
        }
      }
  );
}
 
Example #25
Source File: ATSV15HistoryACLPolicyManager.java    From tez with Apache License 2.0 5 votes vote down vote up
private void initializeTimelineClient() {
  if (this.conf == null) {
    throw new TezUncheckedException("ATSACLManager not configured");
  }
  if (timelineClient != null) {
    this.timelineClient.stop();
    this.timelineClient = null;
  }
  if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
    YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
    this.timelineClient = TimelineClient.createTimelineClient();
    this.timelineClient.init(this.conf);
    this.timelineClient.start();
  } else {
    this.timelineClient = null;
    if (conf.get(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, "")
       .equals(atsHistoryLoggingServiceClassName)) {
      LOG.warn(atsHistoryLoggingServiceClassName
          + " is disabled due to Timeline Service being disabled, "
          + YarnConfiguration.TIMELINE_SERVICE_ENABLED + " set to false");
    }
  }
  try {
    this.user = UserGroupInformation.getCurrentUser().getShortUserName();
  } catch (IOException e) {
    throw new TezUncheckedException("Unable to get Current User UGI", e);
  }
}
 
Example #26
Source File: TestATSHistoryLoggingService.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
@Before
public void setup() throws Exception {
  appContext = mock(AppContext.class);
  atsHistoryLoggingService = new ATSHistoryLoggingService();
  atsHistoryLoggingService.setAppContext(appContext);
  conf = new Configuration(false);
  conf.setLong(TezConfiguration.YARN_ATS_EVENT_FLUSH_TIMEOUT_MILLIS,
      1000l);
  atsInvokeCounter = 0;
  atsHistoryLoggingService.init(conf);
  atsHistoryLoggingService.timelineClient = mock(TimelineClient.class);
  atsHistoryLoggingService.start();
  when(appContext.getClock()).thenReturn(clock);
  when(atsHistoryLoggingService.timelineClient.putEntities(any(TimelineEntity.class))).thenAnswer(
      new Answer<Object>() {
        @Override
        public Object answer(InvocationOnMock invocation) throws Throwable {
          ++atsInvokeCounter;
          try {
            Thread.sleep(500);
          } catch (InterruptedException e) {
            // do nothing
          }
          return null;
        }
      }
  );
}
 
Example #27
Source File: ATSHistoryLoggingService.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
@Override
public void serviceInit(Configuration conf) throws Exception {
  LOG.info("Initializing ATSService");
  timelineClient = TimelineClient.createTimelineClient();
  timelineClient.init(conf);
  maxTimeToWaitOnShutdown = conf.getLong(
      TezConfiguration.YARN_ATS_EVENT_FLUSH_TIMEOUT_MILLIS,
      TezConfiguration.YARN_ATS_EVENT_FLUSH_TIMEOUT_MILLIS_DEFAULT);
}
 
Example #28
Source File: ContainerRequestListener.java    From metron with Apache License 2.0 5 votes vote down vote up
public ContainerRequestListener( TimelineClient timelineClient
                               , UserGroupInformation appSubmitterUgi
                               , String domainId
                               , int minMemorySize
                               )
{
  this.domainId = domainId;
  this.appSubmitterUgi = appSubmitterUgi;
  this.timelineClient = timelineClient;
  state = new ContainerTracker(minMemorySize);
}
 
Example #29
Source File: Client.java    From metron with Apache License 2.0 5 votes vote down vote up
private void prepareTimelineDomain() {
  TimelineClient timelineClient = null;
  if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
          YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
    timelineClient = TimelineClient.createTimelineClient();
    timelineClient.init(conf);
    timelineClient.start();
  } else {
    LOG.warn("Cannot put the domain " + domainId +
            " because the timeline service is not enabled");
    return;
  }
  try {
    TimelineDomain domain = new TimelineDomain();
    domain.setId(domainId);
    domain.setReaders(
            viewACLs != null && viewACLs.length() > 0 ? viewACLs : " ");
    domain.setWriters(
            modifyACLs != null && modifyACLs.length() > 0 ? modifyACLs : " ");
    timelineClient.putDomain(domain);
    LOG.info("Put the timeline domain: " +
            TimelineUtils.dumpTimelineRecordtoJSON(domain));
  } catch (Exception e) {
    LOG.error("Error when putting the timeline domain", e);
  } finally {
    timelineClient.stop();
  }
}
 
Example #30
Source File: ApplicationMaster.java    From big-c with Apache License 2.0 5 votes vote down vote up
private static void publishContainerStartEvent(
    final TimelineClient timelineClient, Container container, String domainId,
    UserGroupInformation ugi) {
  final TimelineEntity entity = new TimelineEntity();
  entity.setEntityId(container.getId().toString());
  entity.setEntityType(DSEntity.DS_CONTAINER.toString());
  entity.setDomainId(domainId);
  entity.addPrimaryFilter("user", ugi.getShortUserName());
  TimelineEvent event = new TimelineEvent();
  event.setTimestamp(System.currentTimeMillis());
  event.setEventType(DSEvent.DS_CONTAINER_START.toString());
  event.addEventInfo("Node", container.getNodeId().toString());
  event.addEventInfo("Resources", container.getResource().toString());
  entity.addEvent(event);

  try {
    ugi.doAs(new PrivilegedExceptionAction<TimelinePutResponse>() {
      @Override
      public TimelinePutResponse run() throws Exception {
        return timelineClient.putEntities(entity);
      }
    });
  } catch (Exception e) {
    LOG.error("Container start event could not be published for "
        + container.getId().toString(),
        e instanceof UndeclaredThrowableException ? e.getCause() : e);
  }
}