Java Code Examples for org.apache.samza.system.SystemAdmin#start()

The following examples show how to use org.apache.samza.system.SystemAdmin#start() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: DiagnosticsUtil.java    From samza with Apache License 2.0 5 votes vote down vote up 
public static void createDiagnosticsStream(Config config) {
  if (!new JobConfig(config).getDiagnosticsEnabled()) {
    return;
  }
  // if diagnostics is enabled, create diagnostics stream if it doesnt exist

  SystemAdmins systemAdmins = new SystemAdmins(config);
  String diagnosticsSystemStreamName = new MetricsConfig(config)
      .getMetricsSnapshotReporterStream(MetricsConfig.METRICS_SNAPSHOT_REPORTER_NAME_FOR_DIAGNOSTICS)
      .orElseThrow(() -> new ConfigException("Missing required config: " +
          String.format(MetricsConfig.METRICS_SNAPSHOT_REPORTER_STREAM,
              MetricsConfig.METRICS_SNAPSHOT_REPORTER_NAME_FOR_DIAGNOSTICS)));

  SystemStream diagnosticsSystemStream = StreamUtil.getSystemStreamFromNames(diagnosticsSystemStreamName);
  SystemAdmin diagnosticsSysAdmin = systemAdmins.getSystemAdmin(diagnosticsSystemStream.getSystem());
  StreamSpec diagnosticsStreamSpec = new StreamSpec(DIAGNOSTICS_STREAM_ID, diagnosticsSystemStream.getStream(),
      diagnosticsSystemStream.getSystem(), new StreamConfig(config).getStreamProperties(DIAGNOSTICS_STREAM_ID));

  log.info("Creating diagnostics stream {}", diagnosticsSystemStream.getStream());
  diagnosticsSysAdmin.start();

  if (diagnosticsSysAdmin.createStream(diagnosticsStreamSpec)) {
    log.info("Created diagnostics stream {}", diagnosticsSystemStream.getStream());
  } else {
    log.info("Diagnostics stream {} already exists", diagnosticsSystemStream.getStream());
  }

  diagnosticsSysAdmin.stop();
}
Example 2
Source File: TestStartpoint.java    From samza with Apache License 2.0 5 votes vote down vote up 
private static CoordinatorStreamStore createCoordinatorStreamStore(Config applicationConfig) {
  SystemStream coordinatorSystemStream = CoordinatorStreamUtil.getCoordinatorSystemStream(applicationConfig);
  SystemAdmins systemAdmins = new SystemAdmins(applicationConfig);
  SystemAdmin coordinatorSystemAdmin = systemAdmins.getSystemAdmin(coordinatorSystemStream.getSystem());
  coordinatorSystemAdmin.start();
  CoordinatorStreamUtil.createCoordinatorStream(coordinatorSystemStream, coordinatorSystemAdmin);
  coordinatorSystemAdmin.stop();
  return new CoordinatorStreamStore(applicationConfig, new NoOpMetricsRegistry());
}
Example 3
Source File: StreamAppender.java    From samza with Apache License 2.0 4 votes vote down vote up 
protected void setupSystem() {
  config = getConfig();
  Log4jSystemConfig log4jSystemConfig = new Log4jSystemConfig(config);

  if (streamName == null) {
    streamName = getStreamName(log4jSystemConfig.getJobName(), log4jSystemConfig.getJobId());
  }

  // TODO we need the ACTUAL metrics registry, or the metrics won't get reported by the metric reporters!
  MetricsRegistry metricsRegistry = new MetricsRegistryMap();
  metrics = new StreamAppenderMetrics("stream-appender", metricsRegistry);

  String systemName = log4jSystemConfig.getSystemName();
  String systemFactoryName = log4jSystemConfig.getSystemFactory(systemName)
      .orElseThrow(() -> new SamzaException(
          "Could not figure out \"" + systemName + "\" system factory for log4j StreamAppender to use"));
  SystemFactory systemFactory = ReflectionUtil.getObj(systemFactoryName, SystemFactory.class);

  setSerde(log4jSystemConfig, systemName, streamName);

  if (config.getBoolean(CREATE_STREAM_ENABLED, false)) {
    // Explicitly create stream appender stream with the partition count the same as the number of containers.
    System.out.println("[StreamAppender] creating stream " + streamName + " with partition count " + getPartitionCount());
    StreamSpec streamSpec =
        StreamSpec.createStreamAppenderStreamSpec(streamName, systemName, getPartitionCount());

    // SystemAdmin only needed for stream creation here.
    SystemAdmin systemAdmin = systemFactory.getAdmin(systemName, config);
    systemAdmin.start();
    systemAdmin.createStream(streamSpec);
    systemAdmin.stop();
  }

  systemProducer = systemFactory.getProducer(systemName, config, metricsRegistry);
  systemStream = new SystemStream(systemName, streamName);
  systemProducer.register(SOURCE);
  systemProducer.start();

  log.info(SOURCE + " has been registered in " + systemName + ". So all the logs will be sent to " + streamName
      + " in " + systemName + ". Logs are partitioned by " + key);

  startTransferThread();
}
Example 4
Source File: StreamAppender.java    From samza with Apache License 2.0 4 votes vote down vote up 
protected void setupSystem() {
  config = getConfig();
  Log4jSystemConfig log4jSystemConfig = new Log4jSystemConfig(config);

  if (streamName == null) {
    streamName = getStreamName(log4jSystemConfig.getJobName(), log4jSystemConfig.getJobId());
  }

  // TODO we need the ACTUAL metrics registry, or the metrics won't get reported by the metric reporters!
  MetricsRegistry metricsRegistry = new MetricsRegistryMap();
  metrics = new StreamAppenderMetrics("stream-appender", metricsRegistry);

  String systemName = log4jSystemConfig.getSystemName();
  String systemFactoryName = log4jSystemConfig.getSystemFactory(systemName)
      .orElseThrow(() -> new SamzaException(
          "Could not figure out \"" + systemName + "\" system factory for log4j StreamAppender to use"));
  SystemFactory systemFactory = ReflectionUtil.getObj(systemFactoryName, SystemFactory.class);

  setSerde(log4jSystemConfig, systemName, streamName);

  if (config.getBoolean(CREATE_STREAM_ENABLED, false)) {
    // Explicitly create stream appender stream with the partition count the same as the number of containers.
    System.out.println("[StreamAppender] creating stream " + streamName + " with partition count " + getPartitionCount());
    StreamSpec streamSpec = StreamSpec.createStreamAppenderStreamSpec(streamName, systemName, getPartitionCount());

    // SystemAdmin only needed for stream creation here.
    SystemAdmin systemAdmin = systemFactory.getAdmin(systemName, config);
    systemAdmin.start();
    systemAdmin.createStream(streamSpec);
    systemAdmin.stop();
  }

  systemProducer = systemFactory.getProducer(systemName, config, metricsRegistry);
  systemStream = new SystemStream(systemName, streamName);
  systemProducer.register(SOURCE);
  systemProducer.start();

  log.info(SOURCE + " has been registered in " + systemName + ". So all the logs will be sent to " + streamName
      + " in " + systemName + ". Logs are partitioned by " + key);

  startTransferThread();
}