org.apache.samza.application.StreamApplication Java Examples

private StreamApplicationDescriptorImpl getKeyedTumblingWindowStreamGraph(AccumulationMode mode,
    Duration duration, Trigger<KV<Integer, Integer>> earlyTrigger) throws IOException {

  StreamApplication userApp = appDesc -> {
    KVSerde<Integer, Integer> kvSerde = KVSerde.of(new IntegerSerde(), new IntegerSerde());
    GenericSystemDescriptor sd = new GenericSystemDescriptor("kafka", "mockFactoryClass");
    GenericInputDescriptor<KV<Integer, Integer>> inputDescriptor = sd.getInputDescriptor("integers", kvSerde);
    appDesc.getInputStream(inputDescriptor)
        .window(Windows.keyedTumblingWindow(KV::getKey, duration, new IntegerSerde(), kvSerde)
            .setEarlyTrigger(earlyTrigger).setAccumulationMode(mode), "w1")
        .sink((message, messageCollector, taskCoordinator) -> {
          SystemStream outputSystemStream = new SystemStream("outputSystem", "outputStream");
          messageCollector.send(new OutgoingMessageEnvelope(outputSystemStream, message));
        });
  };

  return new StreamApplicationDescriptorImpl(userApp, config);
}
 

@Test
public void testImmediateTimer() {
  final InMemorySystemDescriptor isd = new InMemorySystemDescriptor("test");
  final InMemoryInputDescriptor<Integer> imid = isd.getInputDescriptor("test-input", new IntegerSerde());

  StreamApplication app = new StreamApplication() {
    @Override
    public void describe(StreamApplicationDescriptor appDescriptor) {

      appDescriptor.getInputStream(imid)
          .map(new TestFunction());
    }
  };

  TestRunner
      .of(app)
      .addInputStream(imid, Arrays.asList(1, 2, 3, 4, 5))
      .run(Duration.ofSeconds(1));

  assertTrue(timerFired.get());
}
 

@Test
public void testGetStatus() {
  Map<String, String> m = new HashMap<>();
  m.put(JobConfig.JOB_NAME, "jobName");
  m.put(JobConfig.STREAM_JOB_FACTORY_CLASS, MockStreamJobFactory.class.getName());

  m.put(JobConfig.JOB_ID, "newJob");

  StreamApplication userApp = appDesc -> { };
  runner = spy(new RemoteApplicationRunner(userApp, new MapConfig(m)));

  Assert.assertEquals(ApplicationStatus.New, runner.getApplicationStatus(new JobConfig(new MapConfig(m))));

  m.put(JobConfig.JOB_ID, "runningJob");
  runner = spy(new RemoteApplicationRunner(userApp, new MapConfig(m)));
  Assert.assertEquals(ApplicationStatus.Running, runner.getApplicationStatus(new JobConfig(new MapConfig(m))));
}
 

private StreamApplicationDescriptorImpl getAggregateTumblingWindowStreamGraph(AccumulationMode mode, Duration timeDuration,
      Trigger<IntegerEnvelope> earlyTrigger) throws IOException {
  StreamApplication userApp = appDesc -> {
    KVSerde<Integer, Integer> kvSerde = KVSerde.of(new IntegerSerde(), new IntegerSerde());
    GenericSystemDescriptor sd = new GenericSystemDescriptor("kafka", "mockFactoryClass");
    GenericInputDescriptor<KV<Integer, Integer>> inputDescriptor = sd.getInputDescriptor("integers", kvSerde);
    MessageStream<KV<Integer, Integer>> integers = appDesc.getInputStream(inputDescriptor);

    integers
        .map(new KVMapFunction())
        .window(Windows.<IntegerEnvelope, Integer>tumblingWindow(timeDuration, () -> 0, (m, c) -> c + 1, new IntegerSerde())
            .setEarlyTrigger(earlyTrigger)
            .setAccumulationMode(mode), "w1")
        .sink((message, messageCollector, taskCoordinator) -> {
          SystemStream outputSystemStream = new SystemStream("outputSystem", "outputStream");
          messageCollector.send(new OutgoingMessageEnvelope(outputSystemStream, message));
        });
  };

  return new StreamApplicationDescriptorImpl(userApp, config);
}
 

private StreamApplicationDescriptorImpl getKeyedSessionWindowStreamGraph(AccumulationMode mode, Duration duration) throws IOException {
  StreamApplication userApp = appDesc -> {
    KVSerde<Integer, Integer> kvSerde = KVSerde.of(new IntegerSerde(), new IntegerSerde());
    GenericSystemDescriptor sd = new GenericSystemDescriptor("kafka", "mockFactoryClass");
    GenericInputDescriptor<KV<Integer, Integer>> inputDescriptor = sd.getInputDescriptor("integers", kvSerde);
    appDesc.getInputStream(inputDescriptor)
        .window(Windows.keyedSessionWindow(KV::getKey, duration, new IntegerSerde(), kvSerde)
            .setAccumulationMode(mode), "w1")
        .sink((message, messageCollector, taskCoordinator) -> {
          SystemStream outputSystemStream = new SystemStream("outputSystem", "outputStream");
          messageCollector.send(new OutgoingMessageEnvelope(outputSystemStream, message));
        });
  };

  return new StreamApplicationDescriptorImpl(userApp, config);
}
 

private StreamApplicationDescriptorImpl getTumblingWindowStreamGraph(AccumulationMode mode,
    Duration duration, Trigger<KV<Integer, Integer>> earlyTrigger) throws IOException {
  StreamApplication userApp = appDesc -> {
    KVSerde<Integer, Integer> kvSerde = KVSerde.of(new IntegerSerde(), new IntegerSerde());
    GenericSystemDescriptor sd = new GenericSystemDescriptor("kafka", "mockFactoryClass");
    GenericInputDescriptor<KV<Integer, Integer>> inputDescriptor = sd.getInputDescriptor("integers", kvSerde);
    appDesc.getInputStream(inputDescriptor)
        .window(Windows.tumblingWindow(duration, kvSerde).setEarlyTrigger(earlyTrigger)
            .setAccumulationMode(mode), "w1")
        .sink((message, messageCollector, taskCoordinator) -> {
          SystemStream outputSystemStream = new SystemStream("outputSystem", "outputStream");
          messageCollector.send(new OutgoingMessageEnvelope(outputSystemStream, message));
        });
  };

  return new StreamApplicationDescriptorImpl(userApp, config);
}
 

private ApplicationRunner runSamzaApp(StreamApplication app, Config config) {

    final ApplicationRunner runner = ApplicationRunners.getApplicationRunner(app, config);

    ExternalContext externalContext = null;
    if (listener != null) {
      externalContext = listener.onStart();
    }

    runner.run(externalContext);

    if (listener != null
        && options.getSamzaExecutionEnvironment() == SamzaExecutionEnvironment.YARN) {
      listener.onSubmit();
    }

    return runner;
  }
 

public PortablePipelineResult runPortablePipeline(RunnerApi.Pipeline pipeline) {
  final String dotGraph = PipelineDotRenderer.toDotString(pipeline);
  LOG.info("Portable pipeline to run:\n{}", dotGraph);

  final ConfigBuilder configBuilder = new ConfigBuilder(options);
  SamzaPortablePipelineTranslator.createConfig(pipeline, configBuilder, options);
  configBuilder.put(BEAM_DOT_GRAPH, dotGraph);

  final Config config = configBuilder.build();
  options.setConfigOverride(config);

  if (listener != null) {
    listener.onInit(config, options);
  }

  final SamzaExecutionContext executionContext = new SamzaExecutionContext(options);
  final Map<String, MetricsReporterFactory> reporterFactories = getMetricsReporters();
  final StreamApplication app =
      appDescriptor -> {
        appDescriptor
            .withApplicationContainerContextFactory(executionContext.new Factory())
            .withMetricsReporterFactories(reporterFactories);
        SamzaPortablePipelineTranslator.translate(
            pipeline, new PortableTranslationContext(appDescriptor, options));
      };

  ApplicationRunner runner = runSamzaApp(app, config);
  return new SamzaPortablePipelineResult(app, runner, executionContext, listener, config);
}
 

public static StreamApplication getInstance(
    String systemName,
    List<String> inputTopics,
    String outputTopic,
    CountDownLatch processedMessageLatch,
    StreamApplicationCallback callback,
    CountDownLatch kafkaEventsConsumedLatch,
    Config config) {
  String appName = new ApplicationConfig(config).getGlobalAppId();
  String processorName = config.get(JobConfig.PROCESSOR_ID);
  registerLatches(processedMessageLatch, kafkaEventsConsumedLatch, callback, appName, processorName);

  StreamApplication app = new TestStreamApplication(systemName, inputTopics, outputTopic, appName, processorName);
  return app;
}
 

@Test
public void testRunFailure() throws Exception {
  Map<String, String> cfgs = new HashMap<>();
  cfgs.put(ApplicationConfig.PROCESSOR_ID, "0");
  config = new MapConfig(cfgs);
  ProcessorLifecycleListenerFactory mockFactory = (pContext, cfg) -> mock(ProcessorLifecycleListener.class);
  mockApp = (StreamApplication) appDesc -> appDesc.withProcessorLifecycleListenerFactory(mockFactory);
  prepareTest();

  // return the jobConfigs from the planner
  doReturn(Collections.singletonList(new JobConfig(new MapConfig(config)))).when(localPlanner).prepareJobs();

  StreamProcessor sp = mock(StreamProcessor.class);
  CoordinatorStreamStore coordinatorStreamStore = mock(CoordinatorStreamStore.class);
  ArgumentCaptor<StreamProcessor.StreamProcessorLifecycleListenerFactory> captor =
      ArgumentCaptor.forClass(StreamProcessor.StreamProcessorLifecycleListenerFactory.class);

  doAnswer(i -> {
    throw new Exception("test failure");
  }).when(sp).start();

  ExternalContext externalContext = mock(ExternalContext.class);
  doReturn(sp).when(runner)
      .createStreamProcessor(anyObject(), anyObject(), captor.capture(), eq(Optional.of(externalContext)), any(
          CoordinatorStreamStore.class));
  doReturn(coordinatorStreamStore).when(runner).createCoordinatorStreamStore(any(Config.class));

  try {
    runner.run(externalContext);
    runner.waitForFinish();
  } catch (Throwable th) {
    assertNotNull(th);
  }

  verify(coordinatorStreamStore).init();
  verify(coordinatorStreamStore, never()).close();

  assertEquals(runner.status(), ApplicationStatus.UnsuccessfulFinish);
}
 

@Test
public void testRunCompleteWithouCoordinatorStreamStore() throws Exception {
  Map<String, String> cfgs = new HashMap<>();
  cfgs.put(ApplicationConfig.APP_PROCESSOR_ID_GENERATOR_CLASS, UUIDGenerator.class.getName());
  config = new MapConfig(cfgs);
  ProcessorLifecycleListenerFactory mockFactory = (pContext, cfg) -> mock(ProcessorLifecycleListener.class);
  mockApp = (StreamApplication) appDesc -> appDesc.withProcessorLifecycleListenerFactory(mockFactory);
  prepareTest();

  // return the jobConfigs from the planner
  doReturn(Collections.singletonList(new JobConfig(new MapConfig(config)))).when(localPlanner).prepareJobs();

  StreamProcessor sp = mock(StreamProcessor.class);
  ArgumentCaptor<StreamProcessor.StreamProcessorLifecycleListenerFactory> captor =
      ArgumentCaptor.forClass(StreamProcessor.StreamProcessorLifecycleListenerFactory.class);

  doAnswer(i -> {
    ProcessorLifecycleListener listener = captor.getValue().createInstance(sp);
    listener.afterStart();
    listener.afterStop();
    return null;
  }).when(sp).start();

  ExternalContext externalContext = mock(ExternalContext.class);
  doReturn(sp).when(runner)
      .createStreamProcessor(anyObject(), anyObject(), captor.capture(), eq(Optional.of(externalContext)), eq(null));
  doReturn(null).when(runner).createCoordinatorStreamStore(any(Config.class));

  runner.run(externalContext);
  runner.waitForFinish();

  assertEquals(runner.status(), ApplicationStatus.SuccessfulFinish);
}
 

@Test
public void testRunComplete() throws Exception {
  Map<String, String> cfgs = new HashMap<>();
  cfgs.put(ApplicationConfig.APP_PROCESSOR_ID_GENERATOR_CLASS, UUIDGenerator.class.getName());
  config = new MapConfig(cfgs);
  ProcessorLifecycleListenerFactory mockFactory = (pContext, cfg) -> mock(ProcessorLifecycleListener.class);
  mockApp = (StreamApplication) appDesc -> appDesc.withProcessorLifecycleListenerFactory(mockFactory);
  prepareTest();

  // return the jobConfigs from the planner
  doReturn(Collections.singletonList(new JobConfig(new MapConfig(config)))).when(localPlanner).prepareJobs();

  StreamProcessor sp = mock(StreamProcessor.class);
  CoordinatorStreamStore coordinatorStreamStore = mock(CoordinatorStreamStore.class);
  ArgumentCaptor<StreamProcessor.StreamProcessorLifecycleListenerFactory> captor =
      ArgumentCaptor.forClass(StreamProcessor.StreamProcessorLifecycleListenerFactory.class);

  doAnswer(i -> {
    ProcessorLifecycleListener listener = captor.getValue().createInstance(sp);
    listener.afterStart();
    listener.afterStop();
    return null;
  }).when(sp).start();

  ExternalContext externalContext = mock(ExternalContext.class);
  doReturn(sp).when(runner)
      .createStreamProcessor(anyObject(), anyObject(), captor.capture(), eq(Optional.of(externalContext)), any(CoordinatorStreamStore.class));
  doReturn(coordinatorStreamStore).when(runner).createCoordinatorStreamStore(any(Config.class));

  runner.run(externalContext);
  runner.waitForFinish();

  verify(coordinatorStreamStore).init();
  verify(coordinatorStreamStore, never()).close();

  assertEquals(runner.status(), ApplicationStatus.SuccessfulFinish);
}
 

@Test
public void testSendTo() throws Exception {

  int count = 10;
  Profile[] profiles = TestTableData.generateProfiles(count);

  int partitionCount = 4;
  Map<String, String> configs = getBaseJobConfig(bootstrapUrl(), zkConnect());

  configs.put("streams.Profile.samza.system", "test");
  configs.put("streams.Profile.source", Base64Serializer.serialize(profiles));
  configs.put("streams.Profile.partitionCount", String.valueOf(partitionCount));

  MyMapFunction mapFn = new MyMapFunction();

  final StreamApplication app = appDesc -> {

    Table<KV<Integer, Profile>> table = appDesc.getTable(new InMemoryTableDescriptor("t1",
        KVSerde.of(new IntegerSerde(), new ProfileJsonSerde())));
    DelegatingSystemDescriptor ksd = new DelegatingSystemDescriptor("test");
    GenericInputDescriptor<Profile> isd = ksd.getInputDescriptor("Profile", new NoOpSerde<>());

    appDesc.getInputStream(isd)
        .map(mapFn)
        .sendTo(table);
  };

  Config config = new MapConfig(configs);
  final LocalApplicationRunner runner = new LocalApplicationRunner(app, config);
  executeRun(runner, config);
  runner.waitForFinish();

  for (int i = 0; i < partitionCount; i++) {
    MyMapFunction mapFnCopy = MyMapFunction.getMapFunctionByTask(String.format("Partition %d", i));
    assertEquals(count, mapFnCopy.received.size());
    mapFnCopy.received.forEach(p -> Assert.assertTrue(mapFnCopy.table.get(p.getMemberId()) != null));
  }
}
 

@Before
public void setUp() {
  Map<String, String> config = new HashMap<>();
  config.put(ApplicationConfig.APP_NAME, "test-app");
  StreamApplication userApp = appDesc -> { };
  runner = spy(new RemoteApplicationRunner(userApp, new MapConfig(config)));
}
 

SamzaPortablePipelineResult(
    StreamApplication app,
    ApplicationRunner runner,
    SamzaExecutionContext executionContext,
    SamzaPipelineLifeCycleListener listener,
    Config config) {
  super(app, runner, executionContext, listener, config);
}
 

public SamzaPipelineResult(
    StreamApplication app,
    ApplicationRunner runner,
    SamzaExecutionContext executionContext,
    SamzaPipelineLifeCycleListener listener,
    Config config) {
  this.executionContext = executionContext;
  this.runner = runner;
  this.app = app;
  this.listener = listener;
  this.shutdownTiemoutMs =
      config.getLong(TASK_SHUTDOWN_MS, DEFAULT_TASK_SHUTDOWN_MS) + SHUTDOWN_TIMEOUT_BUFFER;
}
 

public void start() throws IOException, InterruptedException {
  super.start();
  MessageConsumer consumeFn = new MessageConsumer();
  StreamApplication app = appDesc -> {
    String systemFactoryName = new SystemConfig(config).getSystemFactory(systemName).get();
    GenericSystemDescriptor sd = new GenericSystemDescriptor(systemName, systemFactoryName);
    GenericInputDescriptor<Object> isd = sd.getInputDescriptor(streamId, new NoOpSerde<>());
    MessageStream<Object> stream = appDesc.getInputStream(isd);
    stream.map(consumeFn);
  };
  ApplicationRunner runner = ApplicationRunners.getApplicationRunner(app, new MapConfig());

  runner.run();

  while (consumeFn.getEventsConsumed() < totalEvents) {
    Thread.sleep(10);
  }

  Instant endTime = Instant.now();

  runner.kill();

  System.out.println("\n*******************");
  System.out.println(String.format("Started at %s Ending at %s ", consumeFn.startTime, endTime));
  System.out.println(String.format("Event Rate is %s Messages/Sec ",
      consumeFn.getEventsConsumed() * 1000 / Duration.between(consumeFn.startTime, Instant.now()).toMillis()));

  System.out.println(
      "Event Rate is " + consumeFn.getEventsConsumed() * 1000 / Duration.between(consumeFn.startTime, endTime).toMillis());
  System.out.println("*******************\n");

  System.exit(0);
}
 

StreamApplication getRepartitionJoinStreamApplication() {
  return appDesc -> {
    MessageStream<KV<String, Object>> input1 = appDesc.getInputStream(input1Descriptor);
    MessageStream<KV<String, Object>> input2 = appDesc.getInputStream(input2Descriptor);
    OutputStream<KV<String, Object>> output = appDesc.getOutputStream(outputDescriptor);
    JoinFunction<String, Object, Object, KV<String, Object>> mockJoinFn = mock(JoinFunction.class);
    input1
        .partitionBy(KV::getKey, KV::getValue, defaultSerde, "p1")
        .map(kv -> kv.value)
        .join(input2.map(kv -> kv.value), mockJoinFn,
            new StringSerde(), new JsonSerdeV2<>(Object.class), new JsonSerdeV2<>(Object.class),
            Duration.ofHours(1), "j1")
        .sendTo(output);
  };
}
 

@Test
public void testProcessorLifecycleListenerFactory() {
  ProcessorLifecycleListenerFactory mockFactory = mock(ProcessorLifecycleListenerFactory.class);
  StreamApplication testApp = appSpec -> appSpec.withProcessorLifecycleListenerFactory(mockFactory);
  StreamApplicationDescriptorImpl appDesc = new StreamApplicationDescriptorImpl(testApp, getConfig());
  assertEquals(appDesc.getProcessorLifecycleListenerFactory(), mockFactory);
}
 

@Test
public void testNoApplicationTaskContextFactory() {
  StreamApplication testApp = appDesc -> {
  };
  StreamApplicationDescriptorImpl appSpec = new StreamApplicationDescriptorImpl(testApp, getConfig());
  assertEquals(appSpec.getApplicationTaskContextFactory(), Optional.empty());
}
 

@Test
public void testApplicationTaskContextFactory() {
  ApplicationTaskContextFactory factory = mock(ApplicationTaskContextFactory.class);
  StreamApplication testApp = appDesc -> appDesc.withApplicationTaskContextFactory(factory);
  StreamApplicationDescriptorImpl appSpec = new StreamApplicationDescriptorImpl(testApp, getConfig());
  assertEquals(appSpec.getApplicationTaskContextFactory(), Optional.of(factory));
}
 

@Test
public void testNoApplicationContainerContextFactory() {
  StreamApplication testApp = appDesc -> {
  };
  StreamApplicationDescriptorImpl appSpec = new StreamApplicationDescriptorImpl(testApp, getConfig());
  assertEquals(appSpec.getApplicationContainerContextFactory(), Optional.empty());
}
 

@Test
public void testApplicationContainerContextFactory() {
  ApplicationContainerContextFactory factory = mock(ApplicationContainerContextFactory.class);
  StreamApplication testApp = appDesc -> appDesc.withApplicationContainerContextFactory(factory);
  StreamApplicationDescriptorImpl appSpec = new StreamApplicationDescriptorImpl(testApp, getConfig());
  assertEquals(appSpec.getApplicationContainerContextFactory(), Optional.of(factory));
}
 

@Test
public void testConstructor() {
  StreamApplication mockApp = mock(StreamApplication.class);
  Config mockConfig = getConfig();
  StreamApplicationDescriptorImpl appDesc = new StreamApplicationDescriptorImpl(mockApp, mockConfig);
  verify(mockApp).describe(appDesc);
  assertEquals(mockConfig, appDesc.getConfig());
}
 

/**
 * Create a new instance of {@link ApplicationDescriptorImpl} based on {@link SamzaApplication} and {@link Config}
 *
 * @param app an implementation of {@link SamzaApplication}. The {@code app} has to have a proper fully-qualified class name.
 * @param config the {@link Config} for the application
 * @return the {@link ApplicationDescriptorImpl} instance containing the processing logic and the config
 */
public static ApplicationDescriptorImpl<? extends ApplicationDescriptor> getAppDescriptor(SamzaApplication app, Config config) {
  if (app instanceof StreamApplication) {
    return new StreamApplicationDescriptorImpl((StreamApplication) app, config);
  }
  if (app instanceof TaskApplication) {
    return new TaskApplicationDescriptorImpl((TaskApplication) app, config);
  }
  throw new IllegalArgumentException(String.format("User application class %s is not supported. Only StreamApplication "
      + "and TaskApplication are supported.", app.getClass().getName()));
}
 

@Test
public void testGetAppRunner() {
  Map<String, String> configMap = new HashMap<>();
  configMap.put("app.runner.class", MockApplicationRunner.class.getName());
  Config config = new MapConfig(configMap);
  StreamApplication app = mock(StreamApplication.class);
  ApplicationRunner appRunner = ApplicationRunners.getApplicationRunner(app, config);
  assertTrue(appRunner instanceof MockApplicationRunner);
}
 

@Test
public void testEndToEnd() throws Exception {

  Bucket inputBucket = cluster.openBucket(inputBucketName);
  inputBucket.upsert(ByteArrayDocument.create("Alice", "20".getBytes()));
  inputBucket.upsert(ByteArrayDocument.create("Bob", "30".getBytes()));
  inputBucket.upsert(ByteArrayDocument.create("Chris", "40".getBytes()));
  inputBucket.upsert(ByteArrayDocument.create("David", "50".getBytes()));
  inputBucket.close();

  String[] users = new String[]{"Alice", "Bob", "Chris", "David"};

  int partitionCount = 1;
  Map<String, String> configs = TestLocalTableEndToEnd.getBaseJobConfig(bootstrapUrl(), zkConnect());

  configs.put("streams.User.samza.system", "test");
  configs.put("streams.User.source", Base64Serializer.serialize(users));
  configs.put("streams.User.partitionCount", String.valueOf(partitionCount));
  Config config = new MapConfig(configs);

  final StreamApplication app = appDesc -> {
    DelegatingSystemDescriptor inputSystemDescriptor = new DelegatingSystemDescriptor("test");
    GenericInputDescriptor<String> inputDescriptor =
        inputSystemDescriptor.getInputDescriptor("User", new NoOpSerde<>());

    CouchbaseTableReadFunction<String> readFunction = new CouchbaseTableReadFunction<>(inputBucketName,
            String.class, "couchbase://127.0.0.1")
        .withBootstrapCarrierDirectPort(couchbaseMock.getCarrierPort(inputBucketName))
        .withBootstrapHttpDirectPort(couchbaseMock.getHttpPort())
        .withSerde(new StringSerde());

    CouchbaseTableWriteFunction<JsonObject> writeFunction = new CouchbaseTableWriteFunction<>(outputBucketName,
            JsonObject.class, "couchbase://127.0.0.1")
        .withBootstrapCarrierDirectPort(couchbaseMock.getCarrierPort(outputBucketName))
        .withBootstrapHttpDirectPort(couchbaseMock.getHttpPort());

    RemoteTableDescriptor inputTableDesc = new RemoteTableDescriptor<String, String>("input-table")
        .withReadFunction(readFunction)
        .withRateLimiterDisabled();
    Table<KV<String, String>> inputTable = appDesc.getTable(inputTableDesc);

    RemoteTableDescriptor outputTableDesc = new RemoteTableDescriptor<String, JsonObject>("output-table")
        .withReadFunction(new NoOpTableReadFunction<>())
        .withWriteFunction(writeFunction)
        .withRateLimiterDisabled();
    Table<KV<String, JsonObject>> outputTable = appDesc.getTable(outputTableDesc);

    appDesc.getInputStream(inputDescriptor)
        .map(k -> KV.of(k, k))
        .join(inputTable, new JoinFunction())
        .sendTo(outputTable);
  };

  final LocalApplicationRunner runner = new LocalApplicationRunner(app, config);
  executeRun(runner, config);
  runner.waitForFinish();

  Bucket outputBucket = cluster.openBucket(outputBucketName);
  Assert.assertEquals("{\"name\":\"Alice\",\"age\":\"20\"}", outputBucket.get("Alice").content().toString());
  Assert.assertEquals("{\"name\":\"Bob\",\"age\":\"30\"}", outputBucket.get("Bob").content().toString());
  Assert.assertEquals("{\"name\":\"Chris\",\"age\":\"40\"}", outputBucket.get("Chris").content().toString());
  Assert.assertEquals("{\"name\":\"David\",\"age\":\"50\"}", outputBucket.get("David").content().toString());
  outputBucket.close();
}
 

@Override
public SamzaPipelineResult run(Pipeline pipeline) {
  MetricsEnvironment.setMetricsSupported(true);

  if (LOG.isDebugEnabled()) {
    LOG.debug("Pre-processed Beam pipeline:\n{}", PipelineDotRenderer.toDotString(pipeline));
  }

  pipeline.replaceAll(SamzaTransformOverrides.getDefaultOverrides());

  final String dotGraph = PipelineDotRenderer.toDotString(pipeline);
  LOG.info("Beam pipeline DOT graph:\n{}", dotGraph);

  final Map<PValue, String> idMap = PViewToIdMapper.buildIdMap(pipeline);
  final ConfigBuilder configBuilder = new ConfigBuilder(options);

  SamzaPipelineTranslator.createConfig(pipeline, options, idMap, configBuilder);
  configBuilder.put(BEAM_DOT_GRAPH, dotGraph);

  final Config config = configBuilder.build();
  options.setConfigOverride(config);

  if (listener != null) {
    listener.onInit(config, options);
  }

  final SamzaExecutionContext executionContext = new SamzaExecutionContext(options);
  final Map<String, MetricsReporterFactory> reporterFactories = getMetricsReporters();

  final StreamApplication app =
      appDescriptor -> {
        appDescriptor.withApplicationContainerContextFactory(executionContext.new Factory());
        appDescriptor.withMetricsReporterFactories(reporterFactories);

        SamzaPipelineTranslator.translate(
            pipeline, new TranslationContext(appDescriptor, idMap, options));
      };

  // perform a final round of validation for the pipeline options now that all configs are
  // generated
  SamzaPipelineOptionsValidator.validate(options);
  ApplicationRunner runner = runSamzaApp(app, config);
  return new SamzaPipelineResult(app, runner, executionContext, listener, config);
}
 

private void runTest(String systemName, StreamApplication app, List<PageView> pageViews,
    List<Profile> profiles) {
  Map<String, String> configs = new HashMap<>();
  configs.put(String.format(StreamConfig.SYSTEM_FOR_STREAM_ID, PAGEVIEW_STREAM), systemName);
  configs.put(String.format(StreamConfig.SYSTEM_FOR_STREAM_ID, PROFILE_STREAM), systemName);
  configs.put(String.format(StreamConfig.SYSTEM_FOR_STREAM_ID, ENRICHED_PAGEVIEW_STREAM), systemName);

  InMemorySystemDescriptor isd = new InMemorySystemDescriptor(systemName);

  InMemoryInputDescriptor<PageView> pageViewStreamDesc = isd
      .getInputDescriptor(PAGEVIEW_STREAM, new NoOpSerde<PageView>());

  InMemoryInputDescriptor<Profile> profileStreamDesc = isd
      .getInputDescriptor(PROFILE_STREAM, new NoOpSerde<Profile>());

  InMemoryOutputDescriptor<EnrichedPageView> outputStreamDesc = isd
      .getOutputDescriptor(ENRICHED_PAGEVIEW_STREAM, new NoOpSerde<EnrichedPageView>());

  TestRunner
      .of(app)
      .addInputStream(pageViewStreamDesc, pageViews)
      .addInputStream(profileStreamDesc, profiles)
      .addOutputStream(outputStreamDesc, 1)
      .addConfig(new MapConfig(configs))
      .run(Duration.ofMillis(100000));

  try {
    Map<Integer, List<EnrichedPageView>> result = TestRunner.consumeStream(outputStreamDesc, Duration.ofMillis(1000));
    List<EnrichedPageView> results = result.values().stream()
        .flatMap(List::stream)
        .collect(Collectors.toList());

    List<EnrichedPageView> expectedEnrichedPageviews = pageViews.stream()
        .flatMap(pv -> profiles.stream()
            .filter(profile -> pv.memberId == profile.memberId)
            .map(profile -> new EnrichedPageView(pv.pageKey, profile.memberId, profile.company)))
        .collect(Collectors.toList());

    boolean successfulJoin = results.stream().allMatch(expectedEnrichedPageviews::contains);
    assertEquals("Mismatch between the expected and actual join count", expectedEnrichedPageviews.size(), results.size());
    assertTrue("Pageview profile join did not succeed for all inputs", successfulJoin);
  } catch (SamzaException e) {
    e.printStackTrace();
  }
}
 

private void doTestStreamTableJoinRemoteTable(boolean withCache, boolean defaultCache, boolean withArgs, String testName)
    throws Exception {

  writtenRecords.put(testName, new ArrayList<>());

  int count = 10;
  final PageView[] pageViews = generatePageViews(count);
  final String profiles = Base64Serializer.serialize(generateProfiles(count));

  final int partitionCount = 4;
  final Map<String, String> configs = TestLocalTableEndToEnd.getBaseJobConfig(bootstrapUrl(), zkConnect());
  configs.put("streams.PageView.samza.system", "test");
  configs.put("streams.PageView.source", Base64Serializer.serialize(pageViews));
  configs.put("streams.PageView.partitionCount", String.valueOf(partitionCount));

  final RateLimiter readRateLimiter = mock(RateLimiter.class, withSettings().serializable());
  final TableRateLimiter.CreditFunction creditFunction = (k, v, args) -> 1;
  final StreamApplication app = appDesc -> {

    final RemoteTableDescriptor joinTableDesc =
            new RemoteTableDescriptor<Integer, TestTableData.Profile>("profile-table-1")
        .withReadFunction(InMemoryProfileReadFunction.getInMemoryReadFunction(testName, profiles))
        .withRateLimiter(readRateLimiter, creditFunction, null);

    final RemoteTableDescriptor outputTableDesc =
            new RemoteTableDescriptor<Integer, EnrichedPageView>("enriched-page-view-table-1")
        .withReadFunction(new NoOpTableReadFunction<>())
        .withReadRateLimiterDisabled()
        .withWriteFunction(new InMemoryEnrichedPageViewWriteFunction(testName))
        .withWriteRateLimit(1000);

    final Table<KV<Integer, EnrichedPageView>> outputTable = withCache
        ? getCachingTable(outputTableDesc, defaultCache, appDesc)
        : appDesc.getTable(outputTableDesc);

    final Table<KV<Integer, Profile>> joinTable = withCache
        ? getCachingTable(joinTableDesc, defaultCache, appDesc)
        : appDesc.getTable(joinTableDesc);

    final DelegatingSystemDescriptor ksd = new DelegatingSystemDescriptor("test");
    final GenericInputDescriptor<PageView> isd = ksd.getInputDescriptor("PageView", new NoOpSerde<>());

    if (!withArgs) {
      appDesc.getInputStream(isd)
          .map(pv -> new KV<>(pv.getMemberId(), pv))
          .join(joinTable, new PageViewToProfileJoinFunction())
          .map(m -> new KV(m.getMemberId(), m))
          .sendTo(outputTable);

    } else {
      counters.put(testName, new AtomicInteger());

      final RemoteTableDescriptor counterTableDesc =
          new RemoteTableDescriptor("counter-table-1")
              .withReadFunction(new InMemoryCounterReadFunction(testName))
              .withWriteFunction(new InMemoryCounterWriteFunction(testName))
              .withRateLimiterDisabled();

      final Table counterTable = withCache
          ? getCachingTable(counterTableDesc, defaultCache, appDesc)
          : appDesc.getTable(counterTableDesc);

      final String counterTableName = ((TableImpl) counterTable).getTableId();
      appDesc.getInputStream(isd)
          .map(new TestReadWriteMapFunction(counterTableName))
          .map(pv -> new KV<>(pv.getMemberId(), pv))
          .join(joinTable, new PageViewToProfileJoinFunction(), true)
          .map(m -> new KV(m.getMemberId(), m))
          .sendTo(outputTable, true);
    }
  };

  final Config config = new MapConfig(configs);
  final LocalApplicationRunner runner = new LocalApplicationRunner(app, config);
  executeRun(runner, config);
  runner.waitForFinish();

  final int numExpected = count * partitionCount;
  Assert.assertEquals(numExpected, writtenRecords.get(testName).size());
  Assert.assertTrue(writtenRecords.get(testName).get(0) instanceof EnrichedPageView);
  if (!withArgs) {
    writtenRecords.get(testName).forEach(epv -> Assert.assertFalse(epv.company.contains("-")));
  } else {
    writtenRecords.get(testName).forEach(epv -> Assert.assertTrue(epv.company.endsWith("-r-w")));
    Assert.assertEquals(numExpected, counters.get(testName).get());
  }
}