Java Code Examples for org.apache.hadoop.yarn.client.api.TimelineClient#createTimelineClient()

The following examples show how to use org.apache.hadoop.yarn.client.api.TimelineClient#createTimelineClient() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: TimelineDelegationTokenIdentifier.java From hadoop with Apache License 2.0

6 votes

@SuppressWarnings("unchecked")
@Override
public long renew(Token<?> token, Configuration conf) throws IOException,
    InterruptedException {
  TimelineClient client = TimelineClient.createTimelineClient();
  try {
    client.init(conf);
    client.start();
    return client.renewDelegationToken(
        (Token<TimelineDelegationTokenIdentifier>) token);
  } catch (YarnException e) {
    throw new IOException(e);
  } finally {
    client.stop();
  }
}

Example 2

Source File: TimelineDelegationTokenIdentifier.java From hadoop with Apache License 2.0

6 votes

@SuppressWarnings("unchecked")
@Override
public void cancel(Token<?> token, Configuration conf) throws IOException,
    InterruptedException {
  TimelineClient client = TimelineClient.createTimelineClient();
  try {
    client.init(conf);
    client.start();
    client.cancelDelegationToken(
        (Token<TimelineDelegationTokenIdentifier>) token);
  } catch (YarnException e) {
    throw new IOException(e);
  } finally {
    client.stop();
  }
}

Example 3

Source File: SystemMetricsPublisher.java From hadoop with Apache License 2.0

6 votes

@Override
protected void serviceInit(Configuration conf) throws Exception {
  publishSystemMetrics =
      conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
          YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED) &&
      conf.getBoolean(YarnConfiguration.RM_SYSTEM_METRICS_PUBLISHER_ENABLED,
          YarnConfiguration.DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_ENABLED);

  if (publishSystemMetrics) {
    client = TimelineClient.createTimelineClient();
    addIfService(client);

    dispatcher = createDispatcher(conf);
    dispatcher.register(SystemMetricsEventType.class,
        new ForwardingEventHandler());
    addIfService(dispatcher);
    LOG.info("YARN system metrics publishing service is enabled");
  } else {
    LOG.info("YARN system metrics publishing service is not enabled");
  }
  super.serviceInit(conf);
}

Example 4

Source File: YarnModule.java From sylph with Apache License 2.0

6 votes

@Override
public YarnClient get()
{
    YarnClient client = YarnClient.createYarnClient();
    if (yarnConfiguration.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false)) {
        try {
            TimelineClient.createTimelineClient();
        }
        catch (NoClassDefFoundError e) {
            logger.warn("createTimelineClient() error with {}", TimelineClient.class.getResource(TimelineClient.class.getSimpleName() + ".class"), e);
            yarnConfiguration.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false);
        }
    }
    client.init(yarnConfiguration);
    client.start();
    return client;
}

Example 5

Source File: TimelineDelegationTokenIdentifier.java From big-c with Apache License 2.0

6 votes

@SuppressWarnings("unchecked")
@Override
public long renew(Token<?> token, Configuration conf) throws IOException,
    InterruptedException {
  TimelineClient client = TimelineClient.createTimelineClient();
  try {
    client.init(conf);
    client.start();
    return client.renewDelegationToken(
        (Token<TimelineDelegationTokenIdentifier>) token);
  } catch (YarnException e) {
    throw new IOException(e);
  } finally {
    client.stop();
  }
}

Example 6

Source File: TimelineDelegationTokenIdentifier.java From big-c with Apache License 2.0

6 votes

@SuppressWarnings("unchecked")
@Override
public void cancel(Token<?> token, Configuration conf) throws IOException,
    InterruptedException {
  TimelineClient client = TimelineClient.createTimelineClient();
  try {
    client.init(conf);
    client.start();
    client.cancelDelegationToken(
        (Token<TimelineDelegationTokenIdentifier>) token);
  } catch (YarnException e) {
    throw new IOException(e);
  } finally {
    client.stop();
  }
}

Example 7

Source File: SystemMetricsPublisher.java From big-c with Apache License 2.0

6 votes

@Override
protected void serviceInit(Configuration conf) throws Exception {
  publishSystemMetrics =
      conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
          YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED) &&
      conf.getBoolean(YarnConfiguration.RM_SYSTEM_METRICS_PUBLISHER_ENABLED,
          YarnConfiguration.DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_ENABLED);

  if (publishSystemMetrics) {
    client = TimelineClient.createTimelineClient();
    addIfService(client);

    dispatcher = createDispatcher(conf);
    dispatcher.register(SystemMetricsEventType.class,
        new ForwardingEventHandler());
    addIfService(dispatcher);
    LOG.info("YARN system metrics publishing service is enabled");
  } else {
    LOG.info("YARN system metrics publishing service is not enabled");
  }
  super.serviceInit(conf);
}

Example 8

Source File: JstormOnYarn.java From jstorm with Apache License 2.0

5 votes

private void prepareTimelineDomain() {
    TimelineClient timelineClient = null;
    if (jstormClientContext.conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
            YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
        timelineClient = TimelineClient.createTimelineClient();
        timelineClient.init(jstormClientContext.conf);
        timelineClient.start();
    } else {
        LOG.warn("Cannot put the domain " + jstormClientContext.domainId +
                " because the timeline service is not enabled");
        return;
    }
    try {
        TimelineDomain domain = new TimelineDomain();
        domain.setId(jstormClientContext.domainId);
        domain.setReaders(
                jstormClientContext.viewACLs != null && jstormClientContext.viewACLs.length() > 0 ? jstormClientContext.viewACLs : JOYConstants.BLANK);
        domain.setWriters(
                jstormClientContext.modifyACLs != null && jstormClientContext.modifyACLs.length() > 0 ? jstormClientContext.modifyACLs : JOYConstants.BLANK);
        timelineClient.putDomain(domain);
        LOG.info("Put the timeline domain: " +
                TimelineUtils.dumpTimelineRecordtoJSON(domain));
    } catch (Exception e) {
        LOG.error("Error when putting the timeline domain", e);
    } finally {
        timelineClient.stop();
    }
}

Example 9

Source File: ATSHistoryACLPolicyManager.java From tez with Apache License 2.0

5 votes

private void initializeTimelineClient() {
  if (this.conf == null) {
    throw new TezUncheckedException("ATSACLManager not configured");
  }
  if (timelineClient != null) {
    this.timelineClient.stop();
    this.timelineClient = null;
  }
  if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
    YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
    this.timelineClient = TimelineClient.createTimelineClient();
    this.timelineClient.init(this.conf);
    this.timelineClient.start();
  } else {
    this.timelineClient = null;
    if (conf.get(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, "")
       .equals(atsHistoryLoggingServiceClassName)) {
      LOG.warn(atsHistoryLoggingServiceClassName
          + " is disabled due to Timeline Service being disabled, "
          + YarnConfiguration.TIMELINE_SERVICE_ENABLED + " set to false");
    }
  }
  try {
    this.user = UserGroupInformation.getCurrentUser().getShortUserName();
  } catch (IOException e) {
    throw new TezUncheckedException("Unable to get Current User UGI", e);
  }
}

Example 10

Source File: ATSV15HistoryACLPolicyManager.java From tez with Apache License 2.0

5 votes

private void initializeTimelineClient() {
  if (this.conf == null) {
    throw new TezUncheckedException("ATSACLManager not configured");
  }
  if (timelineClient != null) {
    this.timelineClient.stop();
    this.timelineClient = null;
  }
  if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
    YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
    this.timelineClient = TimelineClient.createTimelineClient();
    this.timelineClient.init(this.conf);
    this.timelineClient.start();
  } else {
    this.timelineClient = null;
    if (conf.get(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, "")
       .equals(atsHistoryLoggingServiceClassName)) {
      LOG.warn(atsHistoryLoggingServiceClassName
          + " is disabled due to Timeline Service being disabled, "
          + YarnConfiguration.TIMELINE_SERVICE_ENABLED + " set to false");
    }
  }
  try {
    this.user = UserGroupInformation.getCurrentUser().getShortUserName();
  } catch (IOException e) {
    throw new TezUncheckedException("Unable to get Current User UGI", e);
  }
}

Example 11

Source File: ATSHistoryLoggingService.java From incubator-tez with Apache License 2.0

5 votes

@Override
public void serviceInit(Configuration conf) throws Exception {
  LOG.info("Initializing ATSService");
  timelineClient = TimelineClient.createTimelineClient();
  timelineClient.init(conf);
  maxTimeToWaitOnShutdown = conf.getLong(
      TezConfiguration.YARN_ATS_EVENT_FLUSH_TIMEOUT_MILLIS,
      TezConfiguration.YARN_ATS_EVENT_FLUSH_TIMEOUT_MILLIS_DEFAULT);
}

Example 12

Source File: Client.java From metron with Apache License 2.0

5 votes

private void prepareTimelineDomain() {
  TimelineClient timelineClient = null;
  if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
          YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
    timelineClient = TimelineClient.createTimelineClient();
    timelineClient.init(conf);
    timelineClient.start();
  } else {
    LOG.warn("Cannot put the domain " + domainId +
            " because the timeline service is not enabled");
    return;
  }
  try {
    TimelineDomain domain = new TimelineDomain();
    domain.setId(domainId);
    domain.setReaders(
            viewACLs != null && viewACLs.length() > 0 ? viewACLs : " ");
    domain.setWriters(
            modifyACLs != null && modifyACLs.length() > 0 ? modifyACLs : " ");
    timelineClient.putDomain(domain);
    LOG.info("Put the timeline domain: " +
            TimelineUtils.dumpTimelineRecordtoJSON(domain));
  } catch (Exception e) {
    LOG.error("Error when putting the timeline domain", e);
  } finally {
    timelineClient.stop();
  }
}

Example 13

Source File: Client.java From big-c with Apache License 2.0

5 votes

private void prepareTimelineDomain() {
  TimelineClient timelineClient = null;
  if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
      YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
    timelineClient = TimelineClient.createTimelineClient();
    timelineClient.init(conf);
    timelineClient.start();
  } else {
    LOG.warn("Cannot put the domain " + domainId +
        " because the timeline service is not enabled");
    return;
  }
  try {
    //TODO: we need to check and combine the existing timeline domain ACLs,
    //but let's do it once we have client java library to query domains.
    TimelineDomain domain = new TimelineDomain();
    domain.setId(domainId);
    domain.setReaders(
        viewACLs != null && viewACLs.length() > 0 ? viewACLs : " ");
    domain.setWriters(
        modifyACLs != null && modifyACLs.length() > 0 ? modifyACLs : " ");
    timelineClient.putDomain(domain);
    LOG.info("Put the timeline domain: " +
        TimelineUtils.dumpTimelineRecordtoJSON(domain));
  } catch (Exception e) {
    LOG.error("Error when putting the timeline domain", e);
  } finally {
    timelineClient.stop();
  }
}

Example 14

Source File: Client.java From hadoop with Apache License 2.0

5 votes

private void prepareTimelineDomain() {
  TimelineClient timelineClient = null;
  if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
      YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
    timelineClient = TimelineClient.createTimelineClient();
    timelineClient.init(conf);
    timelineClient.start();
  } else {
    LOG.warn("Cannot put the domain " + domainId +
        " because the timeline service is not enabled");
    return;
  }
  try {
    //TODO: we need to check and combine the existing timeline domain ACLs,
    //but let's do it once we have client java library to query domains.
    TimelineDomain domain = new TimelineDomain();
    domain.setId(domainId);
    domain.setReaders(
        viewACLs != null && viewACLs.length() > 0 ? viewACLs : " ");
    domain.setWriters(
        modifyACLs != null && modifyACLs.length() > 0 ? modifyACLs : " ");
    timelineClient.putDomain(domain);
    LOG.info("Put the timeline domain: " +
        TimelineUtils.dumpTimelineRecordtoJSON(domain));
  } catch (Exception e) {
    LOG.error("Error when putting the timeline domain", e);
  } finally {
    timelineClient.stop();
  }
}

Example 15

Source File: YarnClientImpl.java From big-c with Apache License 2.0

4 votes

TimelineClient createTimelineClient() throws IOException, YarnException {
  return TimelineClient.createTimelineClient();
}

Example 16

Source File: TestTimelineAuthenticationFilter.java From big-c with Apache License 2.0

4 votes

private TimelineClient createTimelineClientForUGI() {
  TimelineClient client = TimelineClient.createTimelineClient();
  client.init(conf);
  client.start();
  return client;
}

Example 17

Source File: ATSV15HistoryLoggingService.java From tez with Apache License 2.0

4 votes

@Override
public void serviceInit(Configuration serviceConf) throws Exception {
  Configuration conf = new Configuration(serviceConf);

  String summaryEntityTypesStr = EntityTypes.TEZ_APPLICATION
      + "," + EntityTypes.TEZ_APPLICATION_ATTEMPT
      + "," + EntityTypes.TEZ_DAG_ID;

  // Ensure that summary entity types are defined properly for Tez.
  if (conf.getBoolean(TezConfiguration.TEZ_AM_ATS_V15_OVERRIDE_SUMMARY_TYPES,
      TezConfiguration.TEZ_AM_ATS_V15_OVERRIDE_SUMMARY_TYPES_DEFAULT)) {
    conf.set(YarnConfiguration.TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_SUMMARY_ENTITY_TYPES,
        summaryEntityTypesStr);
  }

  historyLoggingEnabled = conf.getBoolean(TezConfiguration.TEZ_AM_HISTORY_LOGGING_ENABLED,
      TezConfiguration.TEZ_AM_HISTORY_LOGGING_ENABLED_DEFAULT);
  if (!historyLoggingEnabled) {
    LOG.info("ATSService: History Logging disabled. "
        + TezConfiguration.TEZ_AM_HISTORY_LOGGING_ENABLED + " set to false");
    return;
  }

  if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
    YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
    timelineClient = TimelineClient.createTimelineClient();
    timelineClient.init(conf);
  } else {
    this.timelineClient = null;
    if (conf.get(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, "")
      .equals(atsHistoryLoggingServiceClassName)) {
      LOG.warn(atsHistoryLoggingServiceClassName
          + " is disabled due to Timeline Service being disabled, "
          + YarnConfiguration.TIMELINE_SERVICE_ENABLED + " set to false");
    }
  }
  maxTimeToWaitOnShutdown = conf.getLong(
      TezConfiguration.YARN_ATS_EVENT_FLUSH_TIMEOUT_MILLIS,
      TezConfiguration.YARN_ATS_EVENT_FLUSH_TIMEOUT_MILLIS_DEFAULT);
  maxPollingTimeMillis = conf.getInt(
      TezConfiguration.YARN_ATS_MAX_POLLING_TIME_PER_EVENT,
      TezConfiguration.YARN_ATS_MAX_POLLING_TIME_PER_EVENT_DEFAULT);
  if (maxTimeToWaitOnShutdown < 0) {
    waitForeverOnShutdown = true;
  }

  LOG.info("Initializing " + ATSV15HistoryLoggingService.class.getSimpleName() + " with "
      + ", maxPollingTime(ms)=" + maxPollingTimeMillis
      + ", waitTimeForShutdown(ms)=" + maxTimeToWaitOnShutdown
      + ", TimelineACLManagerClass=" + atsHistoryACLManagerClassName);

  try {
    historyACLPolicyManager = ReflectionUtils.createClazzInstance(
        atsHistoryACLManagerClassName);
    historyACLPolicyManager.setConf(conf);
  } catch (TezReflectionException e) {
    LOG.warn("Could not instantiate object for " + atsHistoryACLManagerClassName
        + ". ACLs cannot be enforced correctly for history data in Timeline", e);
    if (!conf.getBoolean(TezConfiguration.TEZ_AM_ALLOW_DISABLED_TIMELINE_DOMAINS,
        TezConfiguration.TEZ_AM_ALLOW_DISABLED_TIMELINE_DOMAINS_DEFAULT)) {
      throw e;
    }
    historyACLPolicyManager = null;
  }

  numDagsPerGroup = conf.getInt(TezConfiguration.TEZ_HISTORY_LOGGING_TIMELINE_NUM_DAGS_PER_GROUP,
      TezConfiguration.TEZ_HISTORY_LOGGING_TIMELINE_NUM_DAGS_PER_GROUP_DEFAULT);
}

Example 18

Source File: ATSHistoryLoggingService.java From tez with Apache License 2.0

4 votes

@Override
public void serviceInit(Configuration conf) throws Exception {
  historyLoggingEnabled = conf.getBoolean(TezConfiguration.TEZ_AM_HISTORY_LOGGING_ENABLED,
      TezConfiguration.TEZ_AM_HISTORY_LOGGING_ENABLED_DEFAULT);
  if (!historyLoggingEnabled) {
    LOG.info("ATSService: History Logging disabled. "
        + TezConfiguration.TEZ_AM_HISTORY_LOGGING_ENABLED + " set to false");
    return;
  }

  if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
    YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
    timelineClient = TimelineClient.createTimelineClient();
    timelineClient.init(conf);
  } else {
    this.timelineClient = null;
    if (conf.get(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, "")
      .equals(atsHistoryLoggingServiceClassName)) {
      LOG.warn(atsHistoryLoggingServiceClassName
          + " is disabled due to Timeline Service being disabled, "
          + YarnConfiguration.TIMELINE_SERVICE_ENABLED + " set to false");
    }
  }
  maxTimeToWaitOnShutdown = conf.getLong(
      TezConfiguration.YARN_ATS_EVENT_FLUSH_TIMEOUT_MILLIS,
      TezConfiguration.YARN_ATS_EVENT_FLUSH_TIMEOUT_MILLIS_DEFAULT);
  maxEventsPerBatch = conf.getInt(
      TezConfiguration.YARN_ATS_MAX_EVENTS_PER_BATCH,
      TezConfiguration.YARN_ATS_MAX_EVENTS_PER_BATCH_DEFAULT);
  maxPollingTimeMillis = conf.getInt(
      TezConfiguration.YARN_ATS_MAX_POLLING_TIME_PER_EVENT,
      TezConfiguration.YARN_ATS_MAX_POLLING_TIME_PER_EVENT_DEFAULT);
  if (maxTimeToWaitOnShutdown < 0) {
    waitForeverOnShutdown = true;
  }

  LOG.info("Initializing " + ATSHistoryLoggingService.class.getSimpleName() + " with "
    + "maxEventsPerBatch=" + maxEventsPerBatch
    + ", maxPollingTime(ms)=" + maxPollingTimeMillis
    + ", waitTimeForShutdown(ms)=" + maxTimeToWaitOnShutdown
    + ", TimelineACLManagerClass=" + atsHistoryACLManagerClassName);

  try {
    historyACLPolicyManager = ReflectionUtils.createClazzInstance(
        atsHistoryACLManagerClassName);
    historyACLPolicyManager.setConf(conf);
  } catch (TezReflectionException e) {
    LOG.warn("Could not instantiate object for " + atsHistoryACLManagerClassName
        + ". ACLs cannot be enforced correctly for history data in Timeline", e);
    if (!conf.getBoolean(TezConfiguration.TEZ_AM_ALLOW_DISABLED_TIMELINE_DOMAINS,
        TezConfiguration.TEZ_AM_ALLOW_DISABLED_TIMELINE_DOMAINS_DEFAULT)) {
      throw e;
    }
    historyACLPolicyManager = null;
  }

}

Example 19

Source File: YarnClientImpl.java From hadoop with Apache License 2.0

4 votes

TimelineClient createTimelineClient() throws IOException, YarnException {
  return TimelineClient.createTimelineClient();
}

Example 20

Source File: TestTimelineAuthenticationFilter.java From hadoop with Apache License 2.0

4 votes

private TimelineClient createTimelineClientForUGI() {
  TimelineClient client = TimelineClient.createTimelineClient();
  client.init(conf);
  client.start();
  return client;
}