org.apache.samza.runtime.LocalApplicationRunner Java Examples

@VisibleForTesting
public static Map<String, String> localRunConfig() {
  // Default Samza config using local deployment of a single JVM
  return ImmutableMap.<String, String>builder()
      .put(APP_RUNNER_CLASS, LocalApplicationRunner.class.getName())
      .put(
          JobCoordinatorConfig.JOB_COORDINATOR_FACTORY,
          PassthroughJobCoordinatorFactory.class.getName())
      .put(GROUPER_FACTORY, SingleContainerGrouperFactory.class.getName())
      .put(COMMIT_MS, "-1")
      .put("processor.id", "1")
      .put(
          // TODO: remove after SAMZA-1531 is resolved
          ApplicationConfig.APP_RUN_ID,
          String.valueOf(System.currentTimeMillis())
              + "-"
              // use the most significant bits in UUID (8 digits) to avoid collision
              + UUID.randomUUID().toString().substring(0, 8))
      .build();
}
 

@Test
public void testComputeSamzaConfigs() {
  Map<String, String> configs = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(10);
  String sql1 = "Insert into testavro.outputTopic(id,long_value) select id, MyTest(id) as long_value from testavro.SIMPLE1";
  configs.put(SamzaSqlApplicationConfig.CFG_SQL_STMT, sql1);
  configs.put(SamzaSqlApplicationRunner.RUNNER_CONFIG, SamzaSqlApplicationRunner.class.getName());
  MapConfig samzaConfig = new MapConfig(configs);
  Config newConfigs = SamzaSqlApplicationRunner.computeSamzaConfigs(true, samzaConfig);
  Assert.assertEquals(newConfigs.get(SamzaSqlApplicationRunner.RUNNER_CONFIG), LocalApplicationRunner.class.getName());
  // Check whether five new configs added.
  Assert.assertEquals(newConfigs.size(), configs.size() + 5);

  newConfigs = SamzaSqlApplicationRunner.computeSamzaConfigs(false, samzaConfig);
  Assert.assertEquals(newConfigs.get(SamzaSqlApplicationRunner.RUNNER_CONFIG), RemoteApplicationRunner.class.getName());

  // Check whether five new configs added.
  Assert.assertEquals(newConfigs.size(), configs.size() + 5);
}
 

/**
 * Run the application with the specified timeout
 *
 * @param timeout time to wait for the application to finish. This timeout does not include
 *                input stream initialization time or the assertion time over output streams. This timeout just accounts
 *                for time that samza job takes run. Timeout must be greater than 0.
 * @throws SamzaException if Samza job fails with exception and returns UnsuccessfulFinish as the statuscode
 */
public void run(Duration timeout) {
  Preconditions.checkNotNull(app);
  Preconditions.checkState(!timeout.isZero() || !timeout.isNegative(), "Timeouts should be positive");
  // Cleaning store directories to ensure current run does not pick up state from previous run
  deleteStoreDirectories();
  Config config = new MapConfig(JobPlanner.generateSingleJobConfig(configs));
  final LocalApplicationRunner runner = new LocalApplicationRunner(app, config, new InMemoryMetadataStoreFactory());
  runner.run(externalContext);
  if (!runner.waitForFinish(timeout)) {
    throw new SamzaException("Timed out waiting for application to finish");
  }
  ApplicationStatus status = runner.status();
  deleteStoreDirectories();
  if (status.getStatusCode() == ApplicationStatus.StatusCode.UnsuccessfulFinish) {
    throw new SamzaException("Application could not finish successfully", status.getThrowable());
  }
}
 

private static void validateZKStandAloneRun(Map<String, String> config) {
  checkArgument(
      config.containsKey(APP_RUNNER_CLASS),
      "Config %s not found for %s Deployment",
      APP_RUNNER_CLASS,
      SamzaExecutionEnvironment.STANDALONE);
  checkArgument(
      config.get(APP_RUNNER_CLASS).equals(LocalApplicationRunner.class.getName()),
      "Config %s must be set to %s for %s Deployment",
      APP_RUNNER_CLASS,
      LocalApplicationRunner.class.getName(),
      SamzaExecutionEnvironment.STANDALONE);
  checkArgument(
      config.containsKey(JobCoordinatorConfig.JOB_COORDINATOR_FACTORY),
      "Config %s not found for %s Deployment",
      JobCoordinatorConfig.JOB_COORDINATOR_FACTORY,
      SamzaExecutionEnvironment.STANDALONE);
  checkArgument(
      config
          .get(JobCoordinatorConfig.JOB_COORDINATOR_FACTORY)
          .equals(ZkJobCoordinatorFactory.class.getName()),
      "Config %s must be set to %s for %s Deployment",
      JobCoordinatorConfig.JOB_COORDINATOR_FACTORY,
      ZkJobCoordinatorFactory.class.getName(),
      SamzaExecutionEnvironment.STANDALONE);
  checkArgument(
      config.containsKey(ZkConfig.ZK_CONNECT),
      "Config %s not found for %s Deployment",
      ZkConfig.ZK_CONNECT,
      SamzaExecutionEnvironment.STANDALONE);
}
 

public static void main(String[] args) {
  CommandLine cmdLine = new CommandLine();
  OptionSet options = cmdLine.parser().parse(args);
  Config config = cmdLine.loadConfig(options);

  AzureApplication app = new AzureApplication();
  LocalApplicationRunner runner = new LocalApplicationRunner(app, config);
  runner.run();

  runner.waitForFinish();
}
 

/**
 * Executes the application using the local application runner.
 * It takes two required command line arguments
 *  config-factory: a fully {@link org.apache.samza.config.factories.PropertiesConfigFactory} class name
 *  config-path: path to application properties
 *
 * @param args command line arguments
 */
public static void main(String[] args) {
  CommandLine cmdLine = new CommandLine();
  OptionSet options = cmdLine.parser().parse(args);
  Config config = cmdLine.loadConfig(options);

  WikipediaApplication app = new WikipediaApplication();
  LocalApplicationRunner runner = new LocalApplicationRunner(app, config);
  runner.run();
  runner.waitForFinish();
}
 

@Test
public void testWithConfigRewriter() throws Exception {
  Map<String, String> configs = getBaseJobConfig(bootstrapUrl(), zkConnect());
  configs.put("streams.PageView.samza.system", "test");
  configs.put("streams.PageView.source", Base64Serializer.serialize(TestTableData.generatePageViews(10)));
  configs.put("streams.PageView.partitionCount", String.valueOf(4));
  configs.put("task.inputs", "test.PageView");
  configs.put("job.config.rewriter.my-rewriter.class", MyConfigRewriter.class.getName());
  configs.put("job.config.rewriters", "my-rewriter");

  Config config = new MapConfig(configs);
  final LocalApplicationRunner runner = new LocalApplicationRunner(new MyTaskApplication(), config);
  executeRun(runner, config);
  runner.waitForFinish();
}
 

@Test
public void testStreamTableJoin() throws Exception {

  int count = 10;
  PageView[] pageViews = TestTableData.generatePageViews(count);
  Profile[] profiles = TestTableData.generateProfiles(count);

  int partitionCount = 4;
  Map<String, String> configs = getBaseJobConfig(bootstrapUrl(), zkConnect());

  configs.put("streams.PageView.samza.system", "test");
  configs.put("streams.PageView.source", Base64Serializer.serialize(pageViews));
  configs.put("streams.PageView.partitionCount", String.valueOf(partitionCount));

  configs.put("streams.Profile.samza.system", "test");
  configs.put("streams.Profile.samza.bootstrap", "true");
  configs.put("streams.Profile.source", Base64Serializer.serialize(profiles));
  configs.put("streams.Profile.partitionCount", String.valueOf(partitionCount));

  Config config = new MapConfig(configs);
  final LocalApplicationRunner runner = new LocalApplicationRunner(new StreamTableJoinApp(), config);
  executeRun(runner, config);
  runner.waitForFinish();

  assertEquals(count * partitionCount, StreamTableJoinApp.received.size());
  assertEquals(count * partitionCount, StreamTableJoinApp.joined.size());
  assertTrue(StreamTableJoinApp.joined.get(0) instanceof EnrichedPageView);
}
 

@Test
public void testSendTo() throws Exception {

  int count = 10;
  Profile[] profiles = TestTableData.generateProfiles(count);

  int partitionCount = 4;
  Map<String, String> configs = getBaseJobConfig(bootstrapUrl(), zkConnect());

  configs.put("streams.Profile.samza.system", "test");
  configs.put("streams.Profile.source", Base64Serializer.serialize(profiles));
  configs.put("streams.Profile.partitionCount", String.valueOf(partitionCount));

  MyMapFunction mapFn = new MyMapFunction();

  final StreamApplication app = appDesc -> {

    Table<KV<Integer, Profile>> table = appDesc.getTable(new InMemoryTableDescriptor("t1",
        KVSerde.of(new IntegerSerde(), new ProfileJsonSerde())));
    DelegatingSystemDescriptor ksd = new DelegatingSystemDescriptor("test");
    GenericInputDescriptor<Profile> isd = ksd.getInputDescriptor("Profile", new NoOpSerde<>());

    appDesc.getInputStream(isd)
        .map(mapFn)
        .sendTo(table);
  };

  Config config = new MapConfig(configs);
  final LocalApplicationRunner runner = new LocalApplicationRunner(app, config);
  executeRun(runner, config);
  runner.waitForFinish();

  for (int i = 0; i < partitionCount; i++) {
    MyMapFunction mapFnCopy = MyMapFunction.getMapFunctionByTask(String.format("Partition %d", i));
    assertEquals(count, mapFnCopy.received.size());
    mapFnCopy.received.forEach(p -> Assert.assertTrue(mapFnCopy.table.get(p.getMemberId()) != null));
  }
}
 

@Test
public void testTableWithLowLevelApi() throws Exception {
  Map<String, String> configs = getBaseJobConfig(bootstrapUrl(), zkConnect());
  configs.put("streams.PageView.samza.system", "test");
  configs.put("streams.PageView.source", Base64Serializer.serialize(TestTableData.generatePageViews(10)));
  configs.put("streams.PageView.partitionCount", String.valueOf(4));
  configs.put("task.inputs", "test.PageView");

  Config config = new MapConfig(configs);
  final LocalApplicationRunner runner = new LocalApplicationRunner(new MyTaskApplication(), config);
  executeRun(runner, config);
  runner.waitForFinish();
}
 

@Override
public void addMoreSystemConfigs(Properties props) {
  props.put("app.runner.class", LocalApplicationRunner.class.getName());
  List<Integer> partitions = IntStream.range(startPartition, endPartition).boxed().collect(Collectors.toList());
  props.put(ApplicationConfig.APP_NAME, "SamzaBench");
  props.put(JobConfig.PROCESSOR_ID, "1");
  props.put(JobCoordinatorConfig.JOB_COORDINATOR_FACTORY, PassthroughJobCoordinatorFactory.class.getName());
  props.put(String.format(ConfigBasedSspGrouperFactory.CONFIG_STREAM_PARTITIONS, streamId),
      Joiner.on(",").join(partitions));
  props.put(TaskConfig.GROUPER_FACTORY, ConfigBasedSspGrouperFactory.class.getName());
}
 

@Test
public void testSamzaStandAloneExecutionEnvironmentConfig() {
  SamzaPipelineOptions options = PipelineOptionsFactory.create().as(SamzaPipelineOptions.class);
  options.setJobName("TestEnvConfig");
  options.setRunner(SamzaRunner.class);
  options.setSamzaExecutionEnvironment(SamzaExecutionEnvironment.STANDALONE);
  options.setConfigOverride(
      ImmutableMap.<String, String>builder().put(ZkConfig.ZK_CONNECT, "localhost:2181").build());

  Pipeline pipeline = Pipeline.create(options);
  pipeline.apply(Create.of(1, 2, 3)).apply(Sum.integersGlobally());

  pipeline.replaceAll(SamzaTransformOverrides.getDefaultOverrides());

  final Map<PValue, String> idMap = PViewToIdMapper.buildIdMap(pipeline);
  final ConfigBuilder configBuilder = new ConfigBuilder(options);
  SamzaPipelineTranslator.createConfig(pipeline, options, idMap, configBuilder);
  try {
    Config config = configBuilder.build();
    assertEquals(config.get(APP_RUNNER_CLASS), LocalApplicationRunner.class.getName());
    assertEquals(
        config.get(JobCoordinatorConfig.JOB_COORDINATOR_FACTORY),
        ZkJobCoordinatorFactory.class.getName());
  } catch (IllegalArgumentException e) {
    throw new AssertionError(
        String.format(
            "Failed to validate correct configs for %s samza execution environment",
            SamzaExecutionEnvironment.STANDALONE),
        e);
  }
}
 

public static Map<String, String> standAloneRunConfig() {
  // Default Samza config using stand alone deployment
  return ImmutableMap.<String, String>builder()
      .put(APP_RUNNER_CLASS, LocalApplicationRunner.class.getName())
      .put(JobCoordinatorConfig.JOB_COORDINATOR_FACTORY, ZkJobCoordinatorFactory.class.getName())
      .build();
}
 

public static void main(String[] args) {
    String[] configArgs = {"--config-factory=org.apache.samza.config.factories.PropertiesConfigFactory"
            , "--config-path=samza-connector/src/main/Properties/config.properties"};
    CommandLine cmdLine = new CommandLine();
    OptionSet options = cmdLine.parser().parse(configArgs);
    Config config = cmdLine.loadConfig(options);
    LocalApplicationRunner runner = new LocalApplicationRunner(new SamzaSumDemo(), config);
    runner.run();
    runner.waitForFinish();
}
 

/**
 * Test if two processors coming up at the same time agree on a single runid
 * 1. bring up two processors
 * 2. wait till they start consuimg messages
 * 3. check if first processor run.id matches that of second processor
 */
@Test
public void testAgreeingOnSameRunIdForBatch() throws InterruptedException {
  publishKafkaEvents(inputKafkaTopic, 0, NUM_KAFKA_EVENTS, PROCESSOR_IDS[0]);

  Map<String, String> configMap =
      buildStreamApplicationConfigMap(testStreamAppName, testStreamAppId, true, Optional.empty());
  configMap.put(JobConfig.PROCESSOR_ID, PROCESSOR_IDS[0]);
  applicationConfig1 = new ApplicationConfig(new MapConfig(configMap));
  configMap.put(JobConfig.PROCESSOR_ID, PROCESSOR_IDS[1]);
  applicationConfig2 = new ApplicationConfig(new MapConfig(configMap));


  // Create StreamApplication from configuration.
  CountDownLatch processedMessagesLatch1 = new CountDownLatch(1);
  CountDownLatch processedMessagesLatch2 = new CountDownLatch(1);

  ApplicationRunner appRunner1 = ApplicationRunners.getApplicationRunner(TestStreamApplication.getInstance(
      TEST_SYSTEM, ImmutableList.of(inputKafkaTopic), outputKafkaTopic, processedMessagesLatch1, null, null,
      applicationConfig1), applicationConfig1);
  ApplicationRunner appRunner2 = ApplicationRunners.getApplicationRunner(TestStreamApplication.getInstance(
      TEST_SYSTEM, ImmutableList.of(inputKafkaTopic), outputKafkaTopic, processedMessagesLatch2, null, null,
      applicationConfig2), applicationConfig2);

  executeRun(appRunner1, applicationConfig1);
  executeRun(appRunner2, applicationConfig2);

  processedMessagesLatch1.await();
  processedMessagesLatch2.await();

  // At this stage, both the processors are running.
  // check if their runId matches

  LocalApplicationRunner localApplicationRunner1 = (LocalApplicationRunner) appRunner1;
  LocalApplicationRunner localApplicationRunner2 = (LocalApplicationRunner) appRunner2;

  assertEquals("RunId of the two processors does not match", localApplicationRunner2.getRunId(), localApplicationRunner1.getRunId());

  appRunner1.kill();
  appRunner1.waitForFinish();
  appRunner2.kill();
  appRunner2.waitForFinish();
}
 

/**
 * Test if a new processors joining an existing qurorum get the same runid
 * 1. bring up two processors
 * 2. wait till they start consuming messages
 * 3. bring up a third processor
 * 4. wait till third processor starts consuming messsages
 * 5. check if third processor run.id matches that of first twp
 */
@Test
public void testNewProcessorGetsSameRunIdForBatch() throws InterruptedException {
  publishKafkaEvents(inputKafkaTopic, 0, NUM_KAFKA_EVENTS, PROCESSOR_IDS[0]);

  Map<String, String> configMap =
      buildStreamApplicationConfigMap(testStreamAppName, testStreamAppId, true, Optional.empty());
  configMap.put(JobConfig.PROCESSOR_ID, PROCESSOR_IDS[0]);
  applicationConfig1 = new ApplicationConfig(new MapConfig(configMap));
  configMap.put(JobConfig.PROCESSOR_ID, PROCESSOR_IDS[1]);
  applicationConfig2 = new ApplicationConfig(new MapConfig(configMap));
  configMap.put(JobConfig.PROCESSOR_ID, PROCESSOR_IDS[2]);
  applicationConfig3 = new ApplicationConfig(new MapConfig(configMap));

  // Create StreamApplication from configuration.
  CountDownLatch processedMessagesLatch1 = new CountDownLatch(1);
  CountDownLatch processedMessagesLatch2 = new CountDownLatch(1);

  ApplicationRunner appRunner1 = ApplicationRunners.getApplicationRunner(TestStreamApplication.getInstance(
      TEST_SYSTEM, ImmutableList.of(inputKafkaTopic), outputKafkaTopic, processedMessagesLatch1, null, null,
      applicationConfig1), applicationConfig1);
  ApplicationRunner appRunner2 = ApplicationRunners.getApplicationRunner(TestStreamApplication.getInstance(
      TEST_SYSTEM, ImmutableList.of(inputKafkaTopic), outputKafkaTopic, processedMessagesLatch2, null, null,
      applicationConfig2), applicationConfig2);

  executeRun(appRunner1, applicationConfig1);
  executeRun(appRunner2, applicationConfig2);

  processedMessagesLatch1.await();
  processedMessagesLatch2.await();

  // At this stage, both the processors are running.
  // check if their runId matches

  LocalApplicationRunner localApplicationRunner1 = (LocalApplicationRunner) appRunner1;
  LocalApplicationRunner localApplicationRunner2 = (LocalApplicationRunner) appRunner2;

  assertEquals("RunId of the two processors does not match", localApplicationRunner2.getRunId(), localApplicationRunner1.getRunId());

  //Bring up a new processsor
  CountDownLatch processedMessagesLatch3 = new CountDownLatch(1);
  ApplicationRunner appRunner3 = ApplicationRunners.getApplicationRunner(TestStreamApplication.getInstance(
      TEST_SYSTEM, ImmutableList.of(inputKafkaTopic), outputKafkaTopic, processedMessagesLatch3, null, null,
      applicationConfig3), applicationConfig3);
  executeRun(appRunner3, applicationConfig3);
  processedMessagesLatch3.await();

  // At this stage, the new processor is running.
  // check if new processor's runId matches that of the older processors
  LocalApplicationRunner localApplicationRunner3 = (LocalApplicationRunner) appRunner3;
  assertEquals("RunId of the new processor does not match that of old processor", localApplicationRunner3.getRunId(), localApplicationRunner1.getRunId());


  appRunner1.kill();
  appRunner1.waitForFinish();
  appRunner2.kill();
  appRunner2.waitForFinish();
  appRunner3.kill();
  appRunner3.waitForFinish();
}
 

/**
 * Test one group of processors dying and a new processor coming up generates new run.id
 * 1. bring up two processors
 * 2. wait till they start consuimg messages
 * 3. kill and shutdown neatly both the processors
 * 4. bring up a new processor
 * 5. wait till new processor starts consuming messages
 * 6. check if new processor has new runid different from shutdown processors
 */
@Test
public void testAllProcesssorDieNewProcessorGetsNewRunIdForBatch() throws InterruptedException {
  publishKafkaEvents(inputKafkaTopic, 0, NUM_KAFKA_EVENTS, PROCESSOR_IDS[0]);

  Map<String, String> configMap =
      buildStreamApplicationConfigMap(testStreamAppName, testStreamAppId, true, Optional.empty());
  configMap.put(JobConfig.PROCESSOR_ID, PROCESSOR_IDS[0]);
  applicationConfig1 = new ApplicationConfig(new MapConfig(configMap));
  configMap.put(JobConfig.PROCESSOR_ID, PROCESSOR_IDS[1]);
  applicationConfig2 = new ApplicationConfig(new MapConfig(configMap));
  configMap.put(JobConfig.PROCESSOR_ID, PROCESSOR_IDS[2]);
  applicationConfig3 = new ApplicationConfig(new MapConfig(configMap));

  // Create StreamApplication from configuration.
  CountDownLatch processedMessagesLatch1 = new CountDownLatch(1);
  CountDownLatch processedMessagesLatch2 = new CountDownLatch(1);

  ApplicationRunner appRunner1 = ApplicationRunners.getApplicationRunner(TestStreamApplication.getInstance(
      TEST_SYSTEM, ImmutableList.of(inputKafkaTopic), outputKafkaTopic, processedMessagesLatch1, null, null,
      applicationConfig1), applicationConfig1);
  ApplicationRunner appRunner2 = ApplicationRunners.getApplicationRunner(TestStreamApplication.getInstance(
      TEST_SYSTEM, ImmutableList.of(inputKafkaTopic), outputKafkaTopic, processedMessagesLatch2, null, null,
      applicationConfig2), applicationConfig2);

  executeRun(appRunner1, applicationConfig1);
  executeRun(appRunner2, applicationConfig2);

  processedMessagesLatch1.await();
  processedMessagesLatch2.await();

  // At this stage, both the processors are running.
  // check if their runId matches

  LocalApplicationRunner localApplicationRunner1 = (LocalApplicationRunner) appRunner1;
  LocalApplicationRunner localApplicationRunner2 = (LocalApplicationRunner) appRunner2;

  assertEquals("RunId of the two processors does not match", localApplicationRunner2.getRunId(), localApplicationRunner1.getRunId());

  String oldRunId = localApplicationRunner1.getRunId().get();

  // shut down both the processors
  appRunner1.kill();
  appRunner1.waitForFinish();
  appRunner2.kill();
  appRunner2.waitForFinish();

  //Bring up a new processsor
  CountDownLatch processedMessagesLatch3 = new CountDownLatch(1);
  ApplicationRunner appRunner3 = ApplicationRunners.getApplicationRunner(TestStreamApplication.getInstance(
      TEST_SYSTEM, ImmutableList.of(inputKafkaTopic), outputKafkaTopic, processedMessagesLatch3, null, null,
      applicationConfig3), applicationConfig3);
  executeRun(appRunner3, applicationConfig3);
  processedMessagesLatch3.await();

  // At this stage, the new processor is running.
  // check if new processor's runId matches that of the older processors
  LocalApplicationRunner localApplicationRunner3 = (LocalApplicationRunner) appRunner3;

  assertNotEquals("RunId of the new processor same as that of old stopped processors", oldRunId, localApplicationRunner3.getRunId());

  appRunner3.kill();
  appRunner3.waitForFinish();
}
 

/**
 * Test if first processor dying changes the runid for new processors joining
 * 1. bring up two processors
 * 2. wait till they start consuimg messages
 * 3. kill and shutdown first processor
 * 4. bring up a new processor
 * 5. wait till new processor starts consuming messages
 * 6. check if new processor gets same run.id
 */
@Test
public void testFirstProcessorDiesButSameRunIdForBatch() throws InterruptedException {
  publishKafkaEvents(inputKafkaTopic, 0, NUM_KAFKA_EVENTS, PROCESSOR_IDS[0]);

  Map<String, String> configMap =
      buildStreamApplicationConfigMap(testStreamAppName, testStreamAppId, true, Optional.empty());
  configMap.put(JobConfig.PROCESSOR_ID, PROCESSOR_IDS[0]);
  applicationConfig1 = new ApplicationConfig(new MapConfig(configMap));
  configMap.put(JobConfig.PROCESSOR_ID, PROCESSOR_IDS[1]);
  applicationConfig2 = new ApplicationConfig(new MapConfig(configMap));
  configMap.put(JobConfig.PROCESSOR_ID, PROCESSOR_IDS[2]);
  applicationConfig3 = new ApplicationConfig(new MapConfig(configMap));

  CountDownLatch processedMessagesLatch1 = new CountDownLatch(1);
  ApplicationRunner appRunner1 = ApplicationRunners.getApplicationRunner(TestStreamApplication.getInstance(
      TEST_SYSTEM, ImmutableList.of(inputKafkaTopic), outputKafkaTopic, processedMessagesLatch1, null, null,
      applicationConfig1), applicationConfig1);
  executeRun(appRunner1, applicationConfig1);

  // firt processor is up and running
  processedMessagesLatch1.await();

  LocalApplicationRunner localApplicationRunner1 = (LocalApplicationRunner) appRunner1;

  // bring up second processor
  CountDownLatch processedMessagesLatch2 = new CountDownLatch(1);
  ApplicationRunner appRunner2 = ApplicationRunners.getApplicationRunner(TestStreamApplication.getInstance(
      TEST_SYSTEM, ImmutableList.of(inputKafkaTopic), outputKafkaTopic, processedMessagesLatch2, null, null,
      applicationConfig2), applicationConfig2);
  executeRun(appRunner2, applicationConfig2);

  // second processor is up and running
  processedMessagesLatch2.await();
  LocalApplicationRunner localApplicationRunner2 = (LocalApplicationRunner) appRunner2;

  assertEquals("RunId of the two processors does not match", localApplicationRunner2.getRunId(), localApplicationRunner1.getRunId());

  // shut down first processor
  appRunner1.kill();
  appRunner1.waitForFinish();

  //Bring up a new processsor
  CountDownLatch processedMessagesLatch3 = new CountDownLatch(1);
  ApplicationRunner appRunner3 = ApplicationRunners.getApplicationRunner(TestStreamApplication.getInstance(
      TEST_SYSTEM, ImmutableList.of(inputKafkaTopic), outputKafkaTopic, processedMessagesLatch3, null, null,
      applicationConfig3), applicationConfig3);
  executeRun(appRunner3, applicationConfig3);
  processedMessagesLatch3.await();

  // At this stage, the new processor is running.
  // check if new processor runid matches the old ones
  LocalApplicationRunner localApplicationRunner3 = (LocalApplicationRunner) appRunner3;
  assertEquals("RunId of the new processor is not the same as that of earlier processors", localApplicationRunner2.getRunId(), localApplicationRunner3.getRunId());


  appRunner2.kill();
  appRunner2.waitForFinish();
  appRunner3.kill();
  appRunner3.waitForFinish();
}
 

private void doTestStreamTableJoinRemoteTable(String testName, boolean batchRead, boolean batchWrite) throws Exception {
  final InMemoryWriteFunction writer = new InMemoryWriteFunction(testName);

  batchReads.put(testName, new AtomicInteger());
  batchWrites.put(testName, new AtomicInteger());
  writtenRecords.put(testName, new CopyOnWriteArrayList<>());

  final int count = 16;
  final int batchSize = 4;
  PageView[] pageViews = generatePageViewsWithDistinctKeys(count);
  String profiles = Base64Serializer.serialize(generateProfiles(count));

  int partitionCount = 1;
  Map<String, String> configs = TestLocalTableEndToEnd.getBaseJobConfig(bootstrapUrl(), zkConnect());

  configs.put("streams.PageView.samza.system", "test");
  configs.put("streams.PageView.source", Base64Serializer.serialize(pageViews));
  configs.put("streams.PageView.partitionCount", String.valueOf(partitionCount));
  configs.put("task.max.concurrency", String.valueOf(count));
  configs.put("task.async.commit", String.valueOf(true));

  final RateLimiter readRateLimiter = mock(RateLimiter.class, withSettings().serializable());
  final RateLimiter writeRateLimiter = mock(RateLimiter.class, withSettings().serializable());
  final TableRateLimiter.CreditFunction creditFunction = (k, v, args)->1;
  final StreamApplication app = appDesc -> {
    RemoteTableDescriptor<Integer, Profile> inputTableDesc = new RemoteTableDescriptor<>("profile-table-1");
    inputTableDesc
        .withReadFunction(InMemoryReadFunction.getInMemoryReadFunction(testName, profiles))
        .withRateLimiter(readRateLimiter, creditFunction, null);
    if (batchRead) {
      inputTableDesc.withBatchProvider(new CompactBatchProvider().withMaxBatchSize(batchSize).withMaxBatchDelay(Duration.ofHours(1)));
    }

    // dummy reader
    TableReadFunction readFn = new MyReadFunction();

    RemoteTableDescriptor<Integer, EnrichedPageView> outputTableDesc = new RemoteTableDescriptor<>("enriched-page-view-table-1");
    outputTableDesc
        .withReadFunction(readFn)
        .withWriteFunction(writer)
        .withRateLimiter(writeRateLimiter, creditFunction, creditFunction);
    if (batchWrite) {
      outputTableDesc.withBatchProvider(new CompactBatchProvider().withMaxBatchSize(batchSize).withMaxBatchDelay(Duration.ofHours(1)));
    }

    Table<KV<Integer, EnrichedPageView>> outputTable = appDesc.getTable(outputTableDesc);

    Table<KV<Integer, Profile>> inputTable = appDesc.getTable(inputTableDesc);

    DelegatingSystemDescriptor ksd = new DelegatingSystemDescriptor("test");
    GenericInputDescriptor<PageView> isd = ksd.getInputDescriptor("PageView", new NoOpSerde<>());
    appDesc.getInputStream(isd)
        .map(pv -> new KV<>(pv.getMemberId(), pv))
        .join(inputTable, new PageViewToProfileJoinFunction())
        .map(m -> new KV(m.getMemberId(), m))
        .sendTo(outputTable);
  };

  Config config = new MapConfig(configs);
  final LocalApplicationRunner runner = new LocalApplicationRunner(app, config);
  executeRun(runner, config);

  runner.waitForFinish();
  int numExpected = count * partitionCount;
  Assert.assertEquals(numExpected, writtenRecords.get(testName).size());
  Assert.assertTrue(writtenRecords.get(testName).get(0) instanceof EnrichedPageView);

  if (batchRead) {
    Assert.assertEquals(numExpected / batchSize, batchReads.get(testName).get());
  }
  if (batchWrite) {
    Assert.assertEquals(numExpected / batchSize, batchWrites.get(testName).get());
  }
}
 

private void doTestStreamTableJoinRemoteTable(boolean withCache, boolean defaultCache, boolean withArgs, String testName)
    throws Exception {

  writtenRecords.put(testName, new ArrayList<>());

  int count = 10;
  final PageView[] pageViews = generatePageViews(count);
  final String profiles = Base64Serializer.serialize(generateProfiles(count));

  final int partitionCount = 4;
  final Map<String, String> configs = TestLocalTableEndToEnd.getBaseJobConfig(bootstrapUrl(), zkConnect());
  configs.put("streams.PageView.samza.system", "test");
  configs.put("streams.PageView.source", Base64Serializer.serialize(pageViews));
  configs.put("streams.PageView.partitionCount", String.valueOf(partitionCount));

  final RateLimiter readRateLimiter = mock(RateLimiter.class, withSettings().serializable());
  final TableRateLimiter.CreditFunction creditFunction = (k, v, args) -> 1;
  final StreamApplication app = appDesc -> {

    final RemoteTableDescriptor joinTableDesc =
            new RemoteTableDescriptor<Integer, TestTableData.Profile>("profile-table-1")
        .withReadFunction(InMemoryProfileReadFunction.getInMemoryReadFunction(testName, profiles))
        .withRateLimiter(readRateLimiter, creditFunction, null);

    final RemoteTableDescriptor outputTableDesc =
            new RemoteTableDescriptor<Integer, EnrichedPageView>("enriched-page-view-table-1")
        .withReadFunction(new NoOpTableReadFunction<>())
        .withReadRateLimiterDisabled()
        .withWriteFunction(new InMemoryEnrichedPageViewWriteFunction(testName))
        .withWriteRateLimit(1000);

    final Table<KV<Integer, EnrichedPageView>> outputTable = withCache
        ? getCachingTable(outputTableDesc, defaultCache, appDesc)
        : appDesc.getTable(outputTableDesc);

    final Table<KV<Integer, Profile>> joinTable = withCache
        ? getCachingTable(joinTableDesc, defaultCache, appDesc)
        : appDesc.getTable(joinTableDesc);

    final DelegatingSystemDescriptor ksd = new DelegatingSystemDescriptor("test");
    final GenericInputDescriptor<PageView> isd = ksd.getInputDescriptor("PageView", new NoOpSerde<>());

    if (!withArgs) {
      appDesc.getInputStream(isd)
          .map(pv -> new KV<>(pv.getMemberId(), pv))
          .join(joinTable, new PageViewToProfileJoinFunction())
          .map(m -> new KV(m.getMemberId(), m))
          .sendTo(outputTable);

    } else {
      counters.put(testName, new AtomicInteger());

      final RemoteTableDescriptor counterTableDesc =
          new RemoteTableDescriptor("counter-table-1")
              .withReadFunction(new InMemoryCounterReadFunction(testName))
              .withWriteFunction(new InMemoryCounterWriteFunction(testName))
              .withRateLimiterDisabled();

      final Table counterTable = withCache
          ? getCachingTable(counterTableDesc, defaultCache, appDesc)
          : appDesc.getTable(counterTableDesc);

      final String counterTableName = ((TableImpl) counterTable).getTableId();
      appDesc.getInputStream(isd)
          .map(new TestReadWriteMapFunction(counterTableName))
          .map(pv -> new KV<>(pv.getMemberId(), pv))
          .join(joinTable, new PageViewToProfileJoinFunction(), true)
          .map(m -> new KV(m.getMemberId(), m))
          .sendTo(outputTable, true);
    }
  };

  final Config config = new MapConfig(configs);
  final LocalApplicationRunner runner = new LocalApplicationRunner(app, config);
  executeRun(runner, config);
  runner.waitForFinish();

  final int numExpected = count * partitionCount;
  Assert.assertEquals(numExpected, writtenRecords.get(testName).size());
  Assert.assertTrue(writtenRecords.get(testName).get(0) instanceof EnrichedPageView);
  if (!withArgs) {
    writtenRecords.get(testName).forEach(epv -> Assert.assertFalse(epv.company.contains("-")));
  } else {
    writtenRecords.get(testName).forEach(epv -> Assert.assertTrue(epv.company.endsWith("-r-w")));
    Assert.assertEquals(numExpected, counters.get(testName).get());
  }
}
 

public void runAndWaitForFinish(ExternalContext externalContext) {
  Validate.isTrue(runner instanceof LocalApplicationRunner, "This method can be called only in standalone mode.");
  run(externalContext);
  waitForFinish();
}
 

public static Config computeSamzaConfigs(Boolean localRunner, Config config) {
  Map<String, String> newConfig = new HashMap<>();

  // TODO: Introduce an API to return a dsl string containing one or more sql statements
  List<String> dslStmts = SamzaSqlDslConverter.fetchSqlFromConfig(config);

  // This is needed because the SQL file may not be available in all the node managers.
  String sqlJson = SamzaSqlApplicationConfig.serializeSqlStmts(dslStmts);
  newConfig.put(SamzaSqlApplicationConfig.CFG_SQL_STMTS_JSON, sqlJson);

  List<String> inputSystemStreams = new LinkedList<>();
  List<String> outputSystemStreams = new LinkedList<>();

  SamzaSqlApplicationConfig.populateSystemStreamsAndGetRelRoots(dslStmts, config, inputSystemStreams,
      outputSystemStreams);

  SqlIOResolver ioResolver = SamzaSqlApplicationConfig.createIOResolver(config);

  // Populate stream to system mapping config for input and output system streams
  for (String source : inputSystemStreams) {
    SqlIOConfig inputSystemStreamConfig = ioResolver.fetchSourceInfo(source);
    newConfig.put(String.format(CFG_FMT_SAMZA_STREAM_SYSTEM, inputSystemStreamConfig.getStreamId()),
        inputSystemStreamConfig.getSystemName());
    newConfig.putAll(inputSystemStreamConfig.getConfig());
  }

  for (String sink : outputSystemStreams) {
    SqlIOConfig outputSystemStreamConfig = ioResolver.fetchSinkInfo(sink);
    newConfig.put(String.format(CFG_FMT_SAMZA_STREAM_SYSTEM, outputSystemStreamConfig.getStreamId()),
        outputSystemStreamConfig.getSystemName());
    newConfig.putAll(outputSystemStreamConfig.getConfig());
  }

  newConfig.putAll(config);

  if (localRunner) {
    newConfig.put(RUNNER_CONFIG, LocalApplicationRunner.class.getName());
  } else {
    newConfig.put(RUNNER_CONFIG, RemoteApplicationRunner.class.getName());
  }

  LOG.info("New Samza configs: " + newConfig);
  return new MapConfig(newConfig);
}
 

@Test
public void testDualStreamTableJoin() throws Exception {

  int count = 10;
  PageView[] pageViews = TestTableData.generatePageViews(count);
  Profile[] profiles = TestTableData.generateProfiles(count);

  int partitionCount = 4;
  Map<String, String> configs = getBaseJobConfig(bootstrapUrl(), zkConnect());

  configs.put("streams.Profile1.samza.system", "test");
  configs.put("streams.Profile1.source", Base64Serializer.serialize(profiles));
  configs.put("streams.Profile1.samza.bootstrap", "true");
  configs.put("streams.Profile1.partitionCount", String.valueOf(partitionCount));

  configs.put("streams.Profile2.samza.system", "test");
  configs.put("streams.Profile2.source", Base64Serializer.serialize(profiles));
  configs.put("streams.Profile2.samza.bootstrap", "true");
  configs.put("streams.Profile2.partitionCount", String.valueOf(partitionCount));

  configs.put("streams.PageView1.samza.system", "test");
  configs.put("streams.PageView1.source", Base64Serializer.serialize(pageViews));
  configs.put("streams.PageView1.partitionCount", String.valueOf(partitionCount));

  configs.put("streams.PageView2.samza.system", "test");
  configs.put("streams.PageView2.source", Base64Serializer.serialize(pageViews));
  configs.put("streams.PageView2.partitionCount", String.valueOf(partitionCount));

  Config config = new MapConfig(configs);
  final LocalApplicationRunner runner = new LocalApplicationRunner(new DualStreamTableJoinApp(), config);
  executeRun(runner, config);
  runner.waitForFinish();

  assertEquals(count * partitionCount, DualStreamTableJoinApp.sentToProfileTable1.size());
  assertEquals(count * partitionCount, DualStreamTableJoinApp.sentToProfileTable2.size());

  assertEquals(count * partitionCount, DualStreamTableJoinApp.joinedPageViews1.size());
  assertEquals(count * partitionCount, DualStreamTableJoinApp.joinedPageViews2.size());
  assertTrue(DualStreamTableJoinApp.joinedPageViews1.get(0) instanceof EnrichedPageView);
  assertTrue(DualStreamTableJoinApp.joinedPageViews2.get(0) instanceof EnrichedPageView);
}
 

@Test
public void testEndToEnd() throws Exception {

  Bucket inputBucket = cluster.openBucket(inputBucketName);
  inputBucket.upsert(ByteArrayDocument.create("Alice", "20".getBytes()));
  inputBucket.upsert(ByteArrayDocument.create("Bob", "30".getBytes()));
  inputBucket.upsert(ByteArrayDocument.create("Chris", "40".getBytes()));
  inputBucket.upsert(ByteArrayDocument.create("David", "50".getBytes()));
  inputBucket.close();

  String[] users = new String[]{"Alice", "Bob", "Chris", "David"};

  int partitionCount = 1;
  Map<String, String> configs = TestLocalTableEndToEnd.getBaseJobConfig(bootstrapUrl(), zkConnect());

  configs.put("streams.User.samza.system", "test");
  configs.put("streams.User.source", Base64Serializer.serialize(users));
  configs.put("streams.User.partitionCount", String.valueOf(partitionCount));
  Config config = new MapConfig(configs);

  final StreamApplication app = appDesc -> {
    DelegatingSystemDescriptor inputSystemDescriptor = new DelegatingSystemDescriptor("test");
    GenericInputDescriptor<String> inputDescriptor =
        inputSystemDescriptor.getInputDescriptor("User", new NoOpSerde<>());

    CouchbaseTableReadFunction<String> readFunction = new CouchbaseTableReadFunction<>(inputBucketName,
            String.class, "couchbase://127.0.0.1")
        .withBootstrapCarrierDirectPort(couchbaseMock.getCarrierPort(inputBucketName))
        .withBootstrapHttpDirectPort(couchbaseMock.getHttpPort())
        .withSerde(new StringSerde());

    CouchbaseTableWriteFunction<JsonObject> writeFunction = new CouchbaseTableWriteFunction<>(outputBucketName,
            JsonObject.class, "couchbase://127.0.0.1")
        .withBootstrapCarrierDirectPort(couchbaseMock.getCarrierPort(outputBucketName))
        .withBootstrapHttpDirectPort(couchbaseMock.getHttpPort());

    RemoteTableDescriptor inputTableDesc = new RemoteTableDescriptor<String, String>("input-table")
        .withReadFunction(readFunction)
        .withRateLimiterDisabled();
    Table<KV<String, String>> inputTable = appDesc.getTable(inputTableDesc);

    RemoteTableDescriptor outputTableDesc = new RemoteTableDescriptor<String, JsonObject>("output-table")
        .withReadFunction(new NoOpTableReadFunction<>())
        .withWriteFunction(writeFunction)
        .withRateLimiterDisabled();
    Table<KV<String, JsonObject>> outputTable = appDesc.getTable(outputTableDesc);

    appDesc.getInputStream(inputDescriptor)
        .map(k -> KV.of(k, k))
        .join(inputTable, new JoinFunction())
        .sendTo(outputTable);
  };

  final LocalApplicationRunner runner = new LocalApplicationRunner(app, config);
  executeRun(runner, config);
  runner.waitForFinish();

  Bucket outputBucket = cluster.openBucket(outputBucketName);
  Assert.assertEquals("{\"name\":\"Alice\",\"age\":\"20\"}", outputBucket.get("Alice").content().toString());
  Assert.assertEquals("{\"name\":\"Bob\",\"age\":\"30\"}", outputBucket.get("Bob").content().toString());
  Assert.assertEquals("{\"name\":\"Chris\",\"age\":\"40\"}", outputBucket.get("Chris").content().toString());
  Assert.assertEquals("{\"name\":\"David\",\"age\":\"50\"}", outputBucket.get("David").content().toString());
  outputBucket.close();
}