Java Code Examples for org.apache.samza.config.Config#getLong()

@Override
public void describe(StreamApplicationDescriptor appDescriptor) {
  Config config = appDescriptor.getConfig();
  KafkaSystemDescriptor kafkaSystemDescriptor = new KafkaSystemDescriptor(TEST_SYSTEM);
  KafkaOutputDescriptor<PageView>
      outputDescriptor = kafkaSystemDescriptor.getOutputDescriptor(NON_GUEST_PAGE_VIEW_STREAM, new NoOpSerde<>());
  OutputStream<PageView> nonGuestPageViewStream = appDescriptor.getOutputStream(outputDescriptor);

  Predicate<PageView> failProcess = (Predicate<PageView> & Serializable) (ignored) -> config.getBoolean(FAIL_PROCESS, false);
  Predicate<PageView> failDownstreamOperator = (Predicate<PageView> & Serializable) (ignored) -> config.getBoolean(FAIL_DOWNSTREAM_OPERATOR, false);
  Supplier<Long> processJitter = (Supplier<Long> & Serializable) () -> config.getLong(PROCESS_JITTER, 100);

  appDescriptor.getInputStream(kafkaSystemDescriptor.getInputDescriptor(PAGE_VIEW_STREAM, new NoOpSerde<PageView>()))
      .flatMapAsync(pageView -> filterGuestPageViews(pageView, failProcess, processJitter))
      .filter(pageView -> filterLoginPageViews(pageView, failDownstreamOperator))
      .sendTo(nonGuestPageViewStream);
}
 

public SamzaPipelineResult(
    StreamApplication app,
    ApplicationRunner runner,
    SamzaExecutionContext executionContext,
    SamzaPipelineLifeCycleListener listener,
    Config config) {
  this.executionContext = executionContext;
  this.runner = runner;
  this.app = app;
  this.listener = listener;
  this.shutdownTiemoutMs =
      config.getLong(TASK_SHUTDOWN_MS, DEFAULT_TASK_SHUTDOWN_MS) + SHUTDOWN_TIMEOUT_BUFFER;
}
 

public TestAvroSystemConsumer(String systemName, Config config) {
  numMessages = config.getInt(String.format("systems.%s.%s", systemName, CFG_NUM_MESSAGES), DEFAULT_NUM_EVENTS);
  includeNullForeignKeys = config.getBoolean(String.format("systems.%s.%s", systemName,
      CFG_INCLUDE_NULL_FOREIGN_KEYS), false);
  includeNullSimpleRecords = config.getBoolean(String.format("systems.%s.%s", systemName,
      CFG_INCLUDE_NULL_SIMPLE_RECORDS), false);
  sleepBetweenPollsMs = config.getLong(String.format("systems.%s.%s", systemName, CFG_SLEEP_BETWEEN_POLLS_MS), 0);
}
 

public static Long getBlockCacheSize(Config storeConfig, int numTasksForContainer) {
  long cacheSize = storeConfig.getLong("container.cache.size.bytes", 100 * 1024 * 1024L);
  return cacheSize / numTasksForContainer;
}
 

public SamzaSqlApplicationConfig(Config staticConfig, List<String> inputSystemStreams,
    List<String> outputSystemStreams) {

  ioResolver = createIOResolver(staticConfig);

  this.outputSystemStreams = new LinkedList<>(outputSystemStreams);

  // There could be duplicate streams across different queries. Let's dedupe them.
  Set<String> inputSystemStreamSet = new HashSet<>(inputSystemStreams);
  Set<String> outputSystemStreamSet = new HashSet<>(outputSystemStreams);

  // Let's get the output system stream configs before input system stream configs. This is to account for
  // table descriptor that could be both input and output. Please note that there could be only one
  // instance of table descriptor and writable table is a readable table but vice versa is not true.
  outputSystemStreamConfigsBySource = outputSystemStreamSet.stream()
       .collect(Collectors.toMap(Function.identity(), x -> ioResolver.fetchSinkInfo(x)));

  inputSystemStreamConfigBySource = inputSystemStreamSet.stream()
      .collect(Collectors.toMap(Function.identity(), src -> ioResolver.fetchSourceInfo(src)));

  Map<String, SqlIOConfig> systemStreamConfigsBySource = new HashMap<>(inputSystemStreamConfigBySource);
  systemStreamConfigsBySource.putAll(outputSystemStreamConfigsBySource);

  Set<SqlIOConfig> systemStreamConfigs = new HashSet<>(systemStreamConfigsBySource.values());

  relSchemaProvidersBySource = systemStreamConfigs.stream()
      .collect(Collectors.toMap(SqlIOConfig::getSource,
        x -> initializePlugin("RelSchemaProvider", x.getRelSchemaProviderName(), staticConfig,
          CFG_FMT_REL_SCHEMA_PROVIDER_DOMAIN,
          (o, c) -> ((RelSchemaProviderFactory) o).create(x.getSystemStream(), c))));

  samzaRelConvertersBySource = systemStreamConfigs.stream()
      .collect(Collectors.toMap(SqlIOConfig::getSource,
        x -> initializePlugin("SamzaRelConverter", x.getSamzaRelConverterName(), staticConfig,
          CFG_FMT_SAMZA_REL_CONVERTER_DOMAIN, (o, c) -> ((SamzaRelConverterFactory) o).create(x.getSystemStream(),
            relSchemaProvidersBySource.get(x.getSource()), c))));

  samzaRelTableKeyConvertersBySource = systemStreamConfigs.stream()
      .filter(SqlIOConfig::isRemoteTable)
      .collect(Collectors.toMap(SqlIOConfig::getSource,
        x -> initializePlugin("SamzaRelTableKeyConverter", x.getSamzaRelTableKeyConverterName(),
          staticConfig, CFG_FMT_SAMZA_REL_TABLE_KEY_CONVERTER_DOMAIN,
          (o, c) -> ((SamzaRelTableKeyConverterFactory) o).create(x.getSystemStream(), c))));

  udfResolver = createUdfResolver(staticConfig);
  udfMetadata = udfResolver.getUdfs();

  metadataTopicPrefix =
      staticConfig.get(CFG_METADATA_TOPIC_PREFIX, DEFAULT_METADATA_TOPIC_PREFIX);

  processSystemEvents = staticConfig.getBoolean(CFG_SQL_PROCESS_SYSTEM_EVENTS, true);
  windowDurationMs = staticConfig.getLong(CFG_GROUPBY_WINDOW_DURATION_MS, DEFAULT_GROUPBY_WINDOW_DURATION_MS);
}