org.apache.samza.application.StreamApplication Java Examples

The following examples show how to use org.apache.samza.application.StreamApplication. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestWindowOperator.java    From samza with Apache License 2.0 6 votes vote down vote up
private StreamApplicationDescriptorImpl getKeyedTumblingWindowStreamGraph(AccumulationMode mode,
    Duration duration, Trigger<KV<Integer, Integer>> earlyTrigger) throws IOException {

  StreamApplication userApp = appDesc -> {
    KVSerde<Integer, Integer> kvSerde = KVSerde.of(new IntegerSerde(), new IntegerSerde());
    GenericSystemDescriptor sd = new GenericSystemDescriptor("kafka", "mockFactoryClass");
    GenericInputDescriptor<KV<Integer, Integer>> inputDescriptor = sd.getInputDescriptor("integers", kvSerde);
    appDesc.getInputStream(inputDescriptor)
        .window(Windows.keyedTumblingWindow(KV::getKey, duration, new IntegerSerde(), kvSerde)
            .setEarlyTrigger(earlyTrigger).setAccumulationMode(mode), "w1")
        .sink((message, messageCollector, taskCoordinator) -> {
          SystemStream outputSystemStream = new SystemStream("outputSystem", "outputStream");
          messageCollector.send(new OutgoingMessageEnvelope(outputSystemStream, message));
        });
  };

  return new StreamApplicationDescriptorImpl(userApp, config);
}
 
Example #2
Source File: TestSchedulerFunction.java    From samza with Apache License 2.0 6 votes vote down vote up
@Test
public void testImmediateTimer() {
  final InMemorySystemDescriptor isd = new InMemorySystemDescriptor("test");
  final InMemoryInputDescriptor<Integer> imid = isd.getInputDescriptor("test-input", new IntegerSerde());

  StreamApplication app = new StreamApplication() {
    @Override
    public void describe(StreamApplicationDescriptor appDescriptor) {

      appDescriptor.getInputStream(imid)
          .map(new TestFunction());
    }
  };

  TestRunner
      .of(app)
      .addInputStream(imid, Arrays.asList(1, 2, 3, 4, 5))
      .run(Duration.ofSeconds(1));

  assertTrue(timerFired.get());
}
 
Example #3
Source File: TestRemoteApplicationRunner.java    From samza with Apache License 2.0 6 votes vote down vote up
@Test
public void testGetStatus() {
  Map<String, String> m = new HashMap<>();
  m.put(JobConfig.JOB_NAME, "jobName");
  m.put(JobConfig.STREAM_JOB_FACTORY_CLASS, MockStreamJobFactory.class.getName());

  m.put(JobConfig.JOB_ID, "newJob");

  StreamApplication userApp = appDesc -> { };
  runner = spy(new RemoteApplicationRunner(userApp, new MapConfig(m)));

  Assert.assertEquals(ApplicationStatus.New, runner.getApplicationStatus(new JobConfig(new MapConfig(m))));

  m.put(JobConfig.JOB_ID, "runningJob");
  runner = spy(new RemoteApplicationRunner(userApp, new MapConfig(m)));
  Assert.assertEquals(ApplicationStatus.Running, runner.getApplicationStatus(new JobConfig(new MapConfig(m))));
}
 
Example #4
Source File: TestWindowOperator.java    From samza with Apache License 2.0 6 votes vote down vote up
private StreamApplicationDescriptorImpl getAggregateTumblingWindowStreamGraph(AccumulationMode mode, Duration timeDuration,
      Trigger<IntegerEnvelope> earlyTrigger) throws IOException {
  StreamApplication userApp = appDesc -> {
    KVSerde<Integer, Integer> kvSerde = KVSerde.of(new IntegerSerde(), new IntegerSerde());
    GenericSystemDescriptor sd = new GenericSystemDescriptor("kafka", "mockFactoryClass");
    GenericInputDescriptor<KV<Integer, Integer>> inputDescriptor = sd.getInputDescriptor("integers", kvSerde);
    MessageStream<KV<Integer, Integer>> integers = appDesc.getInputStream(inputDescriptor);

    integers
        .map(new KVMapFunction())
        .window(Windows.<IntegerEnvelope, Integer>tumblingWindow(timeDuration, () -> 0, (m, c) -> c + 1, new IntegerSerde())
            .setEarlyTrigger(earlyTrigger)
            .setAccumulationMode(mode), "w1")
        .sink((message, messageCollector, taskCoordinator) -> {
          SystemStream outputSystemStream = new SystemStream("outputSystem", "outputStream");
          messageCollector.send(new OutgoingMessageEnvelope(outputSystemStream, message));
        });
  };

  return new StreamApplicationDescriptorImpl(userApp, config);
}
 
Example #5
Source File: TestWindowOperator.java    From samza with Apache License 2.0 6 votes vote down vote up
private StreamApplicationDescriptorImpl getKeyedSessionWindowStreamGraph(AccumulationMode mode, Duration duration) throws IOException {
  StreamApplication userApp = appDesc -> {
    KVSerde<Integer, Integer> kvSerde = KVSerde.of(new IntegerSerde(), new IntegerSerde());
    GenericSystemDescriptor sd = new GenericSystemDescriptor("kafka", "mockFactoryClass");
    GenericInputDescriptor<KV<Integer, Integer>> inputDescriptor = sd.getInputDescriptor("integers", kvSerde);
    appDesc.getInputStream(inputDescriptor)
        .window(Windows.keyedSessionWindow(KV::getKey, duration, new IntegerSerde(), kvSerde)
            .setAccumulationMode(mode), "w1")
        .sink((message, messageCollector, taskCoordinator) -> {
          SystemStream outputSystemStream = new SystemStream("outputSystem", "outputStream");
          messageCollector.send(new OutgoingMessageEnvelope(outputSystemStream, message));
        });
  };

  return new StreamApplicationDescriptorImpl(userApp, config);
}
 
Example #6
Source File: TestWindowOperator.java    From samza with Apache License 2.0 6 votes vote down vote up
private StreamApplicationDescriptorImpl getTumblingWindowStreamGraph(AccumulationMode mode,
    Duration duration, Trigger<KV<Integer, Integer>> earlyTrigger) throws IOException {
  StreamApplication userApp = appDesc -> {
    KVSerde<Integer, Integer> kvSerde = KVSerde.of(new IntegerSerde(), new IntegerSerde());
    GenericSystemDescriptor sd = new GenericSystemDescriptor("kafka", "mockFactoryClass");
    GenericInputDescriptor<KV<Integer, Integer>> inputDescriptor = sd.getInputDescriptor("integers", kvSerde);
    appDesc.getInputStream(inputDescriptor)
        .window(Windows.tumblingWindow(duration, kvSerde).setEarlyTrigger(earlyTrigger)
            .setAccumulationMode(mode), "w1")
        .sink((message, messageCollector, taskCoordinator) -> {
          SystemStream outputSystemStream = new SystemStream("outputSystem", "outputStream");
          messageCollector.send(new OutgoingMessageEnvelope(outputSystemStream, message));
        });
  };

  return new StreamApplicationDescriptorImpl(userApp, config);
}
 
Example #7
Source File: SamzaRunner.java    From beam with Apache License 2.0 6 votes vote down vote up
private ApplicationRunner runSamzaApp(StreamApplication app, Config config) {

    final ApplicationRunner runner = ApplicationRunners.getApplicationRunner(app, config);

    ExternalContext externalContext = null;
    if (listener != null) {
      externalContext = listener.onStart();
    }

    runner.run(externalContext);

    if (listener != null
        && options.getSamzaExecutionEnvironment() == SamzaExecutionEnvironment.YARN) {
      listener.onSubmit();
    }

    return runner;
  }
 
Example #8
Source File: SamzaRunner.java    From beam with Apache License 2.0 5 votes vote down vote up
public PortablePipelineResult runPortablePipeline(RunnerApi.Pipeline pipeline) {
  final String dotGraph = PipelineDotRenderer.toDotString(pipeline);
  LOG.info("Portable pipeline to run:\n{}", dotGraph);

  final ConfigBuilder configBuilder = new ConfigBuilder(options);
  SamzaPortablePipelineTranslator.createConfig(pipeline, configBuilder, options);
  configBuilder.put(BEAM_DOT_GRAPH, dotGraph);

  final Config config = configBuilder.build();
  options.setConfigOverride(config);

  if (listener != null) {
    listener.onInit(config, options);
  }

  final SamzaExecutionContext executionContext = new SamzaExecutionContext(options);
  final Map<String, MetricsReporterFactory> reporterFactories = getMetricsReporters();
  final StreamApplication app =
      appDescriptor -> {
        appDescriptor
            .withApplicationContainerContextFactory(executionContext.new Factory())
            .withMetricsReporterFactories(reporterFactories);
        SamzaPortablePipelineTranslator.translate(
            pipeline, new PortableTranslationContext(appDescriptor, options));
      };

  ApplicationRunner runner = runSamzaApp(app, config);
  return new SamzaPortablePipelineResult(app, runner, executionContext, listener, config);
}
 
Example #9
Source File: TestStreamApplication.java    From samza with Apache License 2.0 5 votes vote down vote up
public static StreamApplication getInstance(
    String systemName,
    List<String> inputTopics,
    String outputTopic,
    CountDownLatch processedMessageLatch,
    StreamApplicationCallback callback,
    CountDownLatch kafkaEventsConsumedLatch,
    Config config) {
  String appName = new ApplicationConfig(config).getGlobalAppId();
  String processorName = config.get(JobConfig.PROCESSOR_ID);
  registerLatches(processedMessageLatch, kafkaEventsConsumedLatch, callback, appName, processorName);

  StreamApplication app = new TestStreamApplication(systemName, inputTopics, outputTopic, appName, processorName);
  return app;
}
 
Example #10
Source File: TestLocalApplicationRunner.java    From samza with Apache License 2.0 5 votes vote down vote up
@Test
public void testRunFailure() throws Exception {
  Map<String, String> cfgs = new HashMap<>();
  cfgs.put(ApplicationConfig.PROCESSOR_ID, "0");
  config = new MapConfig(cfgs);
  ProcessorLifecycleListenerFactory mockFactory = (pContext, cfg) -> mock(ProcessorLifecycleListener.class);
  mockApp = (StreamApplication) appDesc -> appDesc.withProcessorLifecycleListenerFactory(mockFactory);
  prepareTest();

  // return the jobConfigs from the planner
  doReturn(Collections.singletonList(new JobConfig(new MapConfig(config)))).when(localPlanner).prepareJobs();

  StreamProcessor sp = mock(StreamProcessor.class);
  CoordinatorStreamStore coordinatorStreamStore = mock(CoordinatorStreamStore.class);
  ArgumentCaptor<StreamProcessor.StreamProcessorLifecycleListenerFactory> captor =
      ArgumentCaptor.forClass(StreamProcessor.StreamProcessorLifecycleListenerFactory.class);

  doAnswer(i -> {
    throw new Exception("test failure");
  }).when(sp).start();

  ExternalContext externalContext = mock(ExternalContext.class);
  doReturn(sp).when(runner)
      .createStreamProcessor(anyObject(), anyObject(), captor.capture(), eq(Optional.of(externalContext)), any(
          CoordinatorStreamStore.class));
  doReturn(coordinatorStreamStore).when(runner).createCoordinatorStreamStore(any(Config.class));

  try {
    runner.run(externalContext);
    runner.waitForFinish();
  } catch (Throwable th) {
    assertNotNull(th);
  }

  verify(coordinatorStreamStore).init();
  verify(coordinatorStreamStore, never()).close();

  assertEquals(runner.status(), ApplicationStatus.UnsuccessfulFinish);
}
 
Example #11
Source File: TestLocalApplicationRunner.java    From samza with Apache License 2.0 5 votes vote down vote up
@Test
public void testRunCompleteWithouCoordinatorStreamStore() throws Exception {
  Map<String, String> cfgs = new HashMap<>();
  cfgs.put(ApplicationConfig.APP_PROCESSOR_ID_GENERATOR_CLASS, UUIDGenerator.class.getName());
  config = new MapConfig(cfgs);
  ProcessorLifecycleListenerFactory mockFactory = (pContext, cfg) -> mock(ProcessorLifecycleListener.class);
  mockApp = (StreamApplication) appDesc -> appDesc.withProcessorLifecycleListenerFactory(mockFactory);
  prepareTest();

  // return the jobConfigs from the planner
  doReturn(Collections.singletonList(new JobConfig(new MapConfig(config)))).when(localPlanner).prepareJobs();

  StreamProcessor sp = mock(StreamProcessor.class);
  ArgumentCaptor<StreamProcessor.StreamProcessorLifecycleListenerFactory> captor =
      ArgumentCaptor.forClass(StreamProcessor.StreamProcessorLifecycleListenerFactory.class);

  doAnswer(i -> {
    ProcessorLifecycleListener listener = captor.getValue().createInstance(sp);
    listener.afterStart();
    listener.afterStop();
    return null;
  }).when(sp).start();

  ExternalContext externalContext = mock(ExternalContext.class);
  doReturn(sp).when(runner)
      .createStreamProcessor(anyObject(), anyObject(), captor.capture(), eq(Optional.of(externalContext)), eq(null));
  doReturn(null).when(runner).createCoordinatorStreamStore(any(Config.class));

  runner.run(externalContext);
  runner.waitForFinish();

  assertEquals(runner.status(), ApplicationStatus.SuccessfulFinish);
}
 
Example #12
Source File: TestLocalApplicationRunner.java    From samza with Apache License 2.0 5 votes vote down vote up
@Test
public void testRunComplete() throws Exception {
  Map<String, String> cfgs = new HashMap<>();
  cfgs.put(ApplicationConfig.APP_PROCESSOR_ID_GENERATOR_CLASS, UUIDGenerator.class.getName());
  config = new MapConfig(cfgs);
  ProcessorLifecycleListenerFactory mockFactory = (pContext, cfg) -> mock(ProcessorLifecycleListener.class);
  mockApp = (StreamApplication) appDesc -> appDesc.withProcessorLifecycleListenerFactory(mockFactory);
  prepareTest();

  // return the jobConfigs from the planner
  doReturn(Collections.singletonList(new JobConfig(new MapConfig(config)))).when(localPlanner).prepareJobs();

  StreamProcessor sp = mock(StreamProcessor.class);
  CoordinatorStreamStore coordinatorStreamStore = mock(CoordinatorStreamStore.class);
  ArgumentCaptor<StreamProcessor.StreamProcessorLifecycleListenerFactory> captor =
      ArgumentCaptor.forClass(StreamProcessor.StreamProcessorLifecycleListenerFactory.class);

  doAnswer(i -> {
    ProcessorLifecycleListener listener = captor.getValue().createInstance(sp);
    listener.afterStart();
    listener.afterStop();
    return null;
  }).when(sp).start();

  ExternalContext externalContext = mock(ExternalContext.class);
  doReturn(sp).when(runner)
      .createStreamProcessor(anyObject(), anyObject(), captor.capture(), eq(Optional.of(externalContext)), any(CoordinatorStreamStore.class));
  doReturn(coordinatorStreamStore).when(runner).createCoordinatorStreamStore(any(Config.class));

  runner.run(externalContext);
  runner.waitForFinish();

  verify(coordinatorStreamStore).init();
  verify(coordinatorStreamStore, never()).close();

  assertEquals(runner.status(), ApplicationStatus.SuccessfulFinish);
}
 
Example #13
Source File: TestLocalTableEndToEnd.java    From samza with Apache License 2.0 5 votes vote down vote up
@Test
public void testSendTo() throws Exception {

  int count = 10;
  Profile[] profiles = TestTableData.generateProfiles(count);

  int partitionCount = 4;
  Map<String, String> configs = getBaseJobConfig(bootstrapUrl(), zkConnect());

  configs.put("streams.Profile.samza.system", "test");
  configs.put("streams.Profile.source", Base64Serializer.serialize(profiles));
  configs.put("streams.Profile.partitionCount", String.valueOf(partitionCount));

  MyMapFunction mapFn = new MyMapFunction();

  final StreamApplication app = appDesc -> {

    Table<KV<Integer, Profile>> table = appDesc.getTable(new InMemoryTableDescriptor("t1",
        KVSerde.of(new IntegerSerde(), new ProfileJsonSerde())));
    DelegatingSystemDescriptor ksd = new DelegatingSystemDescriptor("test");
    GenericInputDescriptor<Profile> isd = ksd.getInputDescriptor("Profile", new NoOpSerde<>());

    appDesc.getInputStream(isd)
        .map(mapFn)
        .sendTo(table);
  };

  Config config = new MapConfig(configs);
  final LocalApplicationRunner runner = new LocalApplicationRunner(app, config);
  executeRun(runner, config);
  runner.waitForFinish();

  for (int i = 0; i < partitionCount; i++) {
    MyMapFunction mapFnCopy = MyMapFunction.getMapFunctionByTask(String.format("Partition %d", i));
    assertEquals(count, mapFnCopy.received.size());
    mapFnCopy.received.forEach(p -> Assert.assertTrue(mapFnCopy.table.get(p.getMemberId()) != null));
  }
}
 
Example #14
Source File: TestRemoteApplicationRunner.java    From samza with Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() {
  Map<String, String> config = new HashMap<>();
  config.put(ApplicationConfig.APP_NAME, "test-app");
  StreamApplication userApp = appDesc -> { };
  runner = spy(new RemoteApplicationRunner(userApp, new MapConfig(config)));
}
 
Example #15
Source File: SamzaPortablePipelineResult.java    From beam with Apache License 2.0 5 votes vote down vote up
SamzaPortablePipelineResult(
    StreamApplication app,
    ApplicationRunner runner,
    SamzaExecutionContext executionContext,
    SamzaPipelineLifeCycleListener listener,
    Config config) {
  super(app, runner, executionContext, listener, config);
}
 
Example #16
Source File: SamzaPipelineResult.java    From beam with Apache License 2.0 5 votes vote down vote up
public SamzaPipelineResult(
    StreamApplication app,
    ApplicationRunner runner,
    SamzaExecutionContext executionContext,
    SamzaPipelineLifeCycleListener listener,
    Config config) {
  this.executionContext = executionContext;
  this.runner = runner;
  this.app = app;
  this.listener = listener;
  this.shutdownTiemoutMs =
      config.getLong(TASK_SHUTDOWN_MS, DEFAULT_TASK_SHUTDOWN_MS) + SHUTDOWN_TIMEOUT_BUFFER;
}
 
Example #17
Source File: SystemConsumerWithSamzaBench.java    From samza with Apache License 2.0 5 votes vote down vote up
public void start() throws IOException, InterruptedException {
  super.start();
  MessageConsumer consumeFn = new MessageConsumer();
  StreamApplication app = appDesc -> {
    String systemFactoryName = new SystemConfig(config).getSystemFactory(systemName).get();
    GenericSystemDescriptor sd = new GenericSystemDescriptor(systemName, systemFactoryName);
    GenericInputDescriptor<Object> isd = sd.getInputDescriptor(streamId, new NoOpSerde<>());
    MessageStream<Object> stream = appDesc.getInputStream(isd);
    stream.map(consumeFn);
  };
  ApplicationRunner runner = ApplicationRunners.getApplicationRunner(app, new MapConfig());

  runner.run();

  while (consumeFn.getEventsConsumed() < totalEvents) {
    Thread.sleep(10);
  }

  Instant endTime = Instant.now();

  runner.kill();

  System.out.println("\n*******************");
  System.out.println(String.format("Started at %s Ending at %s ", consumeFn.startTime, endTime));
  System.out.println(String.format("Event Rate is %s Messages/Sec ",
      consumeFn.getEventsConsumed() * 1000 / Duration.between(consumeFn.startTime, Instant.now()).toMillis()));

  System.out.println(
      "Event Rate is " + consumeFn.getEventsConsumed() * 1000 / Duration.between(consumeFn.startTime, endTime).toMillis());
  System.out.println("*******************\n");

  System.exit(0);
}
 
Example #18
Source File: ExecutionPlannerTestBase.java    From samza with Apache License 2.0 5 votes vote down vote up
StreamApplication getRepartitionJoinStreamApplication() {
  return appDesc -> {
    MessageStream<KV<String, Object>> input1 = appDesc.getInputStream(input1Descriptor);
    MessageStream<KV<String, Object>> input2 = appDesc.getInputStream(input2Descriptor);
    OutputStream<KV<String, Object>> output = appDesc.getOutputStream(outputDescriptor);
    JoinFunction<String, Object, Object, KV<String, Object>> mockJoinFn = mock(JoinFunction.class);
    input1
        .partitionBy(KV::getKey, KV::getValue, defaultSerde, "p1")
        .map(kv -> kv.value)
        .join(input2.map(kv -> kv.value), mockJoinFn,
            new StringSerde(), new JsonSerdeV2<>(Object.class), new JsonSerdeV2<>(Object.class),
            Duration.ofHours(1), "j1")
        .sendTo(output);
  };
}
 
Example #19
Source File: TestStreamApplicationDescriptorImpl.java    From samza with Apache License 2.0 5 votes vote down vote up
@Test
public void testProcessorLifecycleListenerFactory() {
  ProcessorLifecycleListenerFactory mockFactory = mock(ProcessorLifecycleListenerFactory.class);
  StreamApplication testApp = appSpec -> appSpec.withProcessorLifecycleListenerFactory(mockFactory);
  StreamApplicationDescriptorImpl appDesc = new StreamApplicationDescriptorImpl(testApp, getConfig());
  assertEquals(appDesc.getProcessorLifecycleListenerFactory(), mockFactory);
}
 
Example #20
Source File: TestStreamApplicationDescriptorImpl.java    From samza with Apache License 2.0 5 votes vote down vote up
@Test
public void testNoApplicationTaskContextFactory() {
  StreamApplication testApp = appDesc -> {
  };
  StreamApplicationDescriptorImpl appSpec = new StreamApplicationDescriptorImpl(testApp, getConfig());
  assertEquals(appSpec.getApplicationTaskContextFactory(), Optional.empty());
}
 
Example #21
Source File: TestStreamApplicationDescriptorImpl.java    From samza with Apache License 2.0 5 votes vote down vote up
@Test
public void testApplicationTaskContextFactory() {
  ApplicationTaskContextFactory factory = mock(ApplicationTaskContextFactory.class);
  StreamApplication testApp = appDesc -> appDesc.withApplicationTaskContextFactory(factory);
  StreamApplicationDescriptorImpl appSpec = new StreamApplicationDescriptorImpl(testApp, getConfig());
  assertEquals(appSpec.getApplicationTaskContextFactory(), Optional.of(factory));
}
 
Example #22
Source File: TestStreamApplicationDescriptorImpl.java    From samza with Apache License 2.0 5 votes vote down vote up
@Test
public void testNoApplicationContainerContextFactory() {
  StreamApplication testApp = appDesc -> {
  };
  StreamApplicationDescriptorImpl appSpec = new StreamApplicationDescriptorImpl(testApp, getConfig());
  assertEquals(appSpec.getApplicationContainerContextFactory(), Optional.empty());
}
 
Example #23
Source File: TestStreamApplicationDescriptorImpl.java    From samza with Apache License 2.0 5 votes vote down vote up
@Test
public void testApplicationContainerContextFactory() {
  ApplicationContainerContextFactory factory = mock(ApplicationContainerContextFactory.class);
  StreamApplication testApp = appDesc -> appDesc.withApplicationContainerContextFactory(factory);
  StreamApplicationDescriptorImpl appSpec = new StreamApplicationDescriptorImpl(testApp, getConfig());
  assertEquals(appSpec.getApplicationContainerContextFactory(), Optional.of(factory));
}
 
Example #24
Source File: TestStreamApplicationDescriptorImpl.java    From samza with Apache License 2.0 5 votes vote down vote up
@Test
public void testConstructor() {
  StreamApplication mockApp = mock(StreamApplication.class);
  Config mockConfig = getConfig();
  StreamApplicationDescriptorImpl appDesc = new StreamApplicationDescriptorImpl(mockApp, mockConfig);
  verify(mockApp).describe(appDesc);
  assertEquals(mockConfig, appDesc.getConfig());
}
 
Example #25
Source File: ApplicationDescriptorUtil.java    From samza with Apache License 2.0 5 votes vote down vote up
/**
 * Create a new instance of {@link ApplicationDescriptorImpl} based on {@link SamzaApplication} and {@link Config}
 *
 * @param app an implementation of {@link SamzaApplication}. The {@code app} has to have a proper fully-qualified class name.
 * @param config the {@link Config} for the application
 * @return the {@link ApplicationDescriptorImpl} instance containing the processing logic and the config
 */
public static ApplicationDescriptorImpl<? extends ApplicationDescriptor> getAppDescriptor(SamzaApplication app, Config config) {
  if (app instanceof StreamApplication) {
    return new StreamApplicationDescriptorImpl((StreamApplication) app, config);
  }
  if (app instanceof TaskApplication) {
    return new TaskApplicationDescriptorImpl((TaskApplication) app, config);
  }
  throw new IllegalArgumentException(String.format("User application class %s is not supported. Only StreamApplication "
      + "and TaskApplication are supported.", app.getClass().getName()));
}
 
Example #26
Source File: TestApplicationRunners.java    From samza with Apache License 2.0 5 votes vote down vote up
@Test
public void testGetAppRunner() {
  Map<String, String> configMap = new HashMap<>();
  configMap.put("app.runner.class", MockApplicationRunner.class.getName());
  Config config = new MapConfig(configMap);
  StreamApplication app = mock(StreamApplication.class);
  ApplicationRunner appRunner = ApplicationRunners.getApplicationRunner(app, config);
  assertTrue(appRunner instanceof MockApplicationRunner);
}
 
Example #27
Source File: TestCouchbaseRemoteTableEndToEnd.java    From samza with Apache License 2.0 4 votes vote down vote up
@Test
public void testEndToEnd() throws Exception {

  Bucket inputBucket = cluster.openBucket(inputBucketName);
  inputBucket.upsert(ByteArrayDocument.create("Alice", "20".getBytes()));
  inputBucket.upsert(ByteArrayDocument.create("Bob", "30".getBytes()));
  inputBucket.upsert(ByteArrayDocument.create("Chris", "40".getBytes()));
  inputBucket.upsert(ByteArrayDocument.create("David", "50".getBytes()));
  inputBucket.close();

  String[] users = new String[]{"Alice", "Bob", "Chris", "David"};

  int partitionCount = 1;
  Map<String, String> configs = TestLocalTableEndToEnd.getBaseJobConfig(bootstrapUrl(), zkConnect());

  configs.put("streams.User.samza.system", "test");
  configs.put("streams.User.source", Base64Serializer.serialize(users));
  configs.put("streams.User.partitionCount", String.valueOf(partitionCount));
  Config config = new MapConfig(configs);

  final StreamApplication app = appDesc -> {
    DelegatingSystemDescriptor inputSystemDescriptor = new DelegatingSystemDescriptor("test");
    GenericInputDescriptor<String> inputDescriptor =
        inputSystemDescriptor.getInputDescriptor("User", new NoOpSerde<>());

    CouchbaseTableReadFunction<String> readFunction = new CouchbaseTableReadFunction<>(inputBucketName,
            String.class, "couchbase://127.0.0.1")
        .withBootstrapCarrierDirectPort(couchbaseMock.getCarrierPort(inputBucketName))
        .withBootstrapHttpDirectPort(couchbaseMock.getHttpPort())
        .withSerde(new StringSerde());

    CouchbaseTableWriteFunction<JsonObject> writeFunction = new CouchbaseTableWriteFunction<>(outputBucketName,
            JsonObject.class, "couchbase://127.0.0.1")
        .withBootstrapCarrierDirectPort(couchbaseMock.getCarrierPort(outputBucketName))
        .withBootstrapHttpDirectPort(couchbaseMock.getHttpPort());

    RemoteTableDescriptor inputTableDesc = new RemoteTableDescriptor<String, String>("input-table")
        .withReadFunction(readFunction)
        .withRateLimiterDisabled();
    Table<KV<String, String>> inputTable = appDesc.getTable(inputTableDesc);

    RemoteTableDescriptor outputTableDesc = new RemoteTableDescriptor<String, JsonObject>("output-table")
        .withReadFunction(new NoOpTableReadFunction<>())
        .withWriteFunction(writeFunction)
        .withRateLimiterDisabled();
    Table<KV<String, JsonObject>> outputTable = appDesc.getTable(outputTableDesc);

    appDesc.getInputStream(inputDescriptor)
        .map(k -> KV.of(k, k))
        .join(inputTable, new JoinFunction())
        .sendTo(outputTable);
  };

  final LocalApplicationRunner runner = new LocalApplicationRunner(app, config);
  executeRun(runner, config);
  runner.waitForFinish();

  Bucket outputBucket = cluster.openBucket(outputBucketName);
  Assert.assertEquals("{\"name\":\"Alice\",\"age\":\"20\"}", outputBucket.get("Alice").content().toString());
  Assert.assertEquals("{\"name\":\"Bob\",\"age\":\"30\"}", outputBucket.get("Bob").content().toString());
  Assert.assertEquals("{\"name\":\"Chris\",\"age\":\"40\"}", outputBucket.get("Chris").content().toString());
  Assert.assertEquals("{\"name\":\"David\",\"age\":\"50\"}", outputBucket.get("David").content().toString());
  outputBucket.close();
}
 
Example #28
Source File: SamzaRunner.java    From beam with Apache License 2.0 4 votes vote down vote up
@Override
public SamzaPipelineResult run(Pipeline pipeline) {
  MetricsEnvironment.setMetricsSupported(true);

  if (LOG.isDebugEnabled()) {
    LOG.debug("Pre-processed Beam pipeline:\n{}", PipelineDotRenderer.toDotString(pipeline));
  }

  pipeline.replaceAll(SamzaTransformOverrides.getDefaultOverrides());

  final String dotGraph = PipelineDotRenderer.toDotString(pipeline);
  LOG.info("Beam pipeline DOT graph:\n{}", dotGraph);

  final Map<PValue, String> idMap = PViewToIdMapper.buildIdMap(pipeline);
  final ConfigBuilder configBuilder = new ConfigBuilder(options);

  SamzaPipelineTranslator.createConfig(pipeline, options, idMap, configBuilder);
  configBuilder.put(BEAM_DOT_GRAPH, dotGraph);

  final Config config = configBuilder.build();
  options.setConfigOverride(config);

  if (listener != null) {
    listener.onInit(config, options);
  }

  final SamzaExecutionContext executionContext = new SamzaExecutionContext(options);
  final Map<String, MetricsReporterFactory> reporterFactories = getMetricsReporters();

  final StreamApplication app =
      appDescriptor -> {
        appDescriptor.withApplicationContainerContextFactory(executionContext.new Factory());
        appDescriptor.withMetricsReporterFactories(reporterFactories);

        SamzaPipelineTranslator.translate(
            pipeline, new TranslationContext(appDescriptor, idMap, options));
      };

  // perform a final round of validation for the pipeline options now that all configs are
  // generated
  SamzaPipelineOptionsValidator.validate(options);
  ApplicationRunner runner = runSamzaApp(app, config);
  return new SamzaPipelineResult(app, runner, executionContext, listener, config);
}
 
Example #29
Source File: TestLocalTableWithSideInputsEndToEnd.java    From samza with Apache License 2.0 4 votes vote down vote up
private void runTest(String systemName, StreamApplication app, List<PageView> pageViews,
    List<Profile> profiles) {
  Map<String, String> configs = new HashMap<>();
  configs.put(String.format(StreamConfig.SYSTEM_FOR_STREAM_ID, PAGEVIEW_STREAM), systemName);
  configs.put(String.format(StreamConfig.SYSTEM_FOR_STREAM_ID, PROFILE_STREAM), systemName);
  configs.put(String.format(StreamConfig.SYSTEM_FOR_STREAM_ID, ENRICHED_PAGEVIEW_STREAM), systemName);

  InMemorySystemDescriptor isd = new InMemorySystemDescriptor(systemName);

  InMemoryInputDescriptor<PageView> pageViewStreamDesc = isd
      .getInputDescriptor(PAGEVIEW_STREAM, new NoOpSerde<PageView>());

  InMemoryInputDescriptor<Profile> profileStreamDesc = isd
      .getInputDescriptor(PROFILE_STREAM, new NoOpSerde<Profile>());

  InMemoryOutputDescriptor<EnrichedPageView> outputStreamDesc = isd
      .getOutputDescriptor(ENRICHED_PAGEVIEW_STREAM, new NoOpSerde<EnrichedPageView>());

  TestRunner
      .of(app)
      .addInputStream(pageViewStreamDesc, pageViews)
      .addInputStream(profileStreamDesc, profiles)
      .addOutputStream(outputStreamDesc, 1)
      .addConfig(new MapConfig(configs))
      .run(Duration.ofMillis(100000));

  try {
    Map<Integer, List<EnrichedPageView>> result = TestRunner.consumeStream(outputStreamDesc, Duration.ofMillis(1000));
    List<EnrichedPageView> results = result.values().stream()
        .flatMap(List::stream)
        .collect(Collectors.toList());

    List<EnrichedPageView> expectedEnrichedPageviews = pageViews.stream()
        .flatMap(pv -> profiles.stream()
            .filter(profile -> pv.memberId == profile.memberId)
            .map(profile -> new EnrichedPageView(pv.pageKey, profile.memberId, profile.company)))
        .collect(Collectors.toList());

    boolean successfulJoin = results.stream().allMatch(expectedEnrichedPageviews::contains);
    assertEquals("Mismatch between the expected and actual join count", expectedEnrichedPageviews.size(), results.size());
    assertTrue("Pageview profile join did not succeed for all inputs", successfulJoin);
  } catch (SamzaException e) {
    e.printStackTrace();
  }
}
 
Example #30
Source File: TestRemoteTableEndToEnd.java    From samza with Apache License 2.0 4 votes vote down vote up
private void doTestStreamTableJoinRemoteTable(boolean withCache, boolean defaultCache, boolean withArgs, String testName)
    throws Exception {

  writtenRecords.put(testName, new ArrayList<>());

  int count = 10;
  final PageView[] pageViews = generatePageViews(count);
  final String profiles = Base64Serializer.serialize(generateProfiles(count));

  final int partitionCount = 4;
  final Map<String, String> configs = TestLocalTableEndToEnd.getBaseJobConfig(bootstrapUrl(), zkConnect());
  configs.put("streams.PageView.samza.system", "test");
  configs.put("streams.PageView.source", Base64Serializer.serialize(pageViews));
  configs.put("streams.PageView.partitionCount", String.valueOf(partitionCount));

  final RateLimiter readRateLimiter = mock(RateLimiter.class, withSettings().serializable());
  final TableRateLimiter.CreditFunction creditFunction = (k, v, args) -> 1;
  final StreamApplication app = appDesc -> {

    final RemoteTableDescriptor joinTableDesc =
            new RemoteTableDescriptor<Integer, TestTableData.Profile>("profile-table-1")
        .withReadFunction(InMemoryProfileReadFunction.getInMemoryReadFunction(testName, profiles))
        .withRateLimiter(readRateLimiter, creditFunction, null);

    final RemoteTableDescriptor outputTableDesc =
            new RemoteTableDescriptor<Integer, EnrichedPageView>("enriched-page-view-table-1")
        .withReadFunction(new NoOpTableReadFunction<>())
        .withReadRateLimiterDisabled()
        .withWriteFunction(new InMemoryEnrichedPageViewWriteFunction(testName))
        .withWriteRateLimit(1000);

    final Table<KV<Integer, EnrichedPageView>> outputTable = withCache
        ? getCachingTable(outputTableDesc, defaultCache, appDesc)
        : appDesc.getTable(outputTableDesc);

    final Table<KV<Integer, Profile>> joinTable = withCache
        ? getCachingTable(joinTableDesc, defaultCache, appDesc)
        : appDesc.getTable(joinTableDesc);

    final DelegatingSystemDescriptor ksd = new DelegatingSystemDescriptor("test");
    final GenericInputDescriptor<PageView> isd = ksd.getInputDescriptor("PageView", new NoOpSerde<>());

    if (!withArgs) {
      appDesc.getInputStream(isd)
          .map(pv -> new KV<>(pv.getMemberId(), pv))
          .join(joinTable, new PageViewToProfileJoinFunction())
          .map(m -> new KV(m.getMemberId(), m))
          .sendTo(outputTable);

    } else {
      counters.put(testName, new AtomicInteger());

      final RemoteTableDescriptor counterTableDesc =
          new RemoteTableDescriptor("counter-table-1")
              .withReadFunction(new InMemoryCounterReadFunction(testName))
              .withWriteFunction(new InMemoryCounterWriteFunction(testName))
              .withRateLimiterDisabled();

      final Table counterTable = withCache
          ? getCachingTable(counterTableDesc, defaultCache, appDesc)
          : appDesc.getTable(counterTableDesc);

      final String counterTableName = ((TableImpl) counterTable).getTableId();
      appDesc.getInputStream(isd)
          .map(new TestReadWriteMapFunction(counterTableName))
          .map(pv -> new KV<>(pv.getMemberId(), pv))
          .join(joinTable, new PageViewToProfileJoinFunction(), true)
          .map(m -> new KV(m.getMemberId(), m))
          .sendTo(outputTable, true);
    }
  };

  final Config config = new MapConfig(configs);
  final LocalApplicationRunner runner = new LocalApplicationRunner(app, config);
  executeRun(runner, config);
  runner.waitForFinish();

  final int numExpected = count * partitionCount;
  Assert.assertEquals(numExpected, writtenRecords.get(testName).size());
  Assert.assertTrue(writtenRecords.get(testName).get(0) instanceof EnrichedPageView);
  if (!withArgs) {
    writtenRecords.get(testName).forEach(epv -> Assert.assertFalse(epv.company.contains("-")));
  } else {
    writtenRecords.get(testName).forEach(epv -> Assert.assertTrue(epv.company.endsWith("-r-w")));
    Assert.assertEquals(numExpected, counters.get(testName).get());
  }
}