org.apache.samza.application.StreamApplication Java Examples
The following examples show how to use
org.apache.samza.application.StreamApplication.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestWindowOperator.java From samza with Apache License 2.0 | 6 votes |
private StreamApplicationDescriptorImpl getKeyedTumblingWindowStreamGraph(AccumulationMode mode, Duration duration, Trigger<KV<Integer, Integer>> earlyTrigger) throws IOException { StreamApplication userApp = appDesc -> { KVSerde<Integer, Integer> kvSerde = KVSerde.of(new IntegerSerde(), new IntegerSerde()); GenericSystemDescriptor sd = new GenericSystemDescriptor("kafka", "mockFactoryClass"); GenericInputDescriptor<KV<Integer, Integer>> inputDescriptor = sd.getInputDescriptor("integers", kvSerde); appDesc.getInputStream(inputDescriptor) .window(Windows.keyedTumblingWindow(KV::getKey, duration, new IntegerSerde(), kvSerde) .setEarlyTrigger(earlyTrigger).setAccumulationMode(mode), "w1") .sink((message, messageCollector, taskCoordinator) -> { SystemStream outputSystemStream = new SystemStream("outputSystem", "outputStream"); messageCollector.send(new OutgoingMessageEnvelope(outputSystemStream, message)); }); }; return new StreamApplicationDescriptorImpl(userApp, config); }
Example #2
Source File: TestSchedulerFunction.java From samza with Apache License 2.0 | 6 votes |
@Test public void testImmediateTimer() { final InMemorySystemDescriptor isd = new InMemorySystemDescriptor("test"); final InMemoryInputDescriptor<Integer> imid = isd.getInputDescriptor("test-input", new IntegerSerde()); StreamApplication app = new StreamApplication() { @Override public void describe(StreamApplicationDescriptor appDescriptor) { appDescriptor.getInputStream(imid) .map(new TestFunction()); } }; TestRunner .of(app) .addInputStream(imid, Arrays.asList(1, 2, 3, 4, 5)) .run(Duration.ofSeconds(1)); assertTrue(timerFired.get()); }
Example #3
Source File: TestRemoteApplicationRunner.java From samza with Apache License 2.0 | 6 votes |
@Test public void testGetStatus() { Map<String, String> m = new HashMap<>(); m.put(JobConfig.JOB_NAME, "jobName"); m.put(JobConfig.STREAM_JOB_FACTORY_CLASS, MockStreamJobFactory.class.getName()); m.put(JobConfig.JOB_ID, "newJob"); StreamApplication userApp = appDesc -> { }; runner = spy(new RemoteApplicationRunner(userApp, new MapConfig(m))); Assert.assertEquals(ApplicationStatus.New, runner.getApplicationStatus(new JobConfig(new MapConfig(m)))); m.put(JobConfig.JOB_ID, "runningJob"); runner = spy(new RemoteApplicationRunner(userApp, new MapConfig(m))); Assert.assertEquals(ApplicationStatus.Running, runner.getApplicationStatus(new JobConfig(new MapConfig(m)))); }
Example #4
Source File: TestWindowOperator.java From samza with Apache License 2.0 | 6 votes |
private StreamApplicationDescriptorImpl getAggregateTumblingWindowStreamGraph(AccumulationMode mode, Duration timeDuration, Trigger<IntegerEnvelope> earlyTrigger) throws IOException { StreamApplication userApp = appDesc -> { KVSerde<Integer, Integer> kvSerde = KVSerde.of(new IntegerSerde(), new IntegerSerde()); GenericSystemDescriptor sd = new GenericSystemDescriptor("kafka", "mockFactoryClass"); GenericInputDescriptor<KV<Integer, Integer>> inputDescriptor = sd.getInputDescriptor("integers", kvSerde); MessageStream<KV<Integer, Integer>> integers = appDesc.getInputStream(inputDescriptor); integers .map(new KVMapFunction()) .window(Windows.<IntegerEnvelope, Integer>tumblingWindow(timeDuration, () -> 0, (m, c) -> c + 1, new IntegerSerde()) .setEarlyTrigger(earlyTrigger) .setAccumulationMode(mode), "w1") .sink((message, messageCollector, taskCoordinator) -> { SystemStream outputSystemStream = new SystemStream("outputSystem", "outputStream"); messageCollector.send(new OutgoingMessageEnvelope(outputSystemStream, message)); }); }; return new StreamApplicationDescriptorImpl(userApp, config); }
Example #5
Source File: TestWindowOperator.java From samza with Apache License 2.0 | 6 votes |
private StreamApplicationDescriptorImpl getKeyedSessionWindowStreamGraph(AccumulationMode mode, Duration duration) throws IOException { StreamApplication userApp = appDesc -> { KVSerde<Integer, Integer> kvSerde = KVSerde.of(new IntegerSerde(), new IntegerSerde()); GenericSystemDescriptor sd = new GenericSystemDescriptor("kafka", "mockFactoryClass"); GenericInputDescriptor<KV<Integer, Integer>> inputDescriptor = sd.getInputDescriptor("integers", kvSerde); appDesc.getInputStream(inputDescriptor) .window(Windows.keyedSessionWindow(KV::getKey, duration, new IntegerSerde(), kvSerde) .setAccumulationMode(mode), "w1") .sink((message, messageCollector, taskCoordinator) -> { SystemStream outputSystemStream = new SystemStream("outputSystem", "outputStream"); messageCollector.send(new OutgoingMessageEnvelope(outputSystemStream, message)); }); }; return new StreamApplicationDescriptorImpl(userApp, config); }
Example #6
Source File: TestWindowOperator.java From samza with Apache License 2.0 | 6 votes |
private StreamApplicationDescriptorImpl getTumblingWindowStreamGraph(AccumulationMode mode, Duration duration, Trigger<KV<Integer, Integer>> earlyTrigger) throws IOException { StreamApplication userApp = appDesc -> { KVSerde<Integer, Integer> kvSerde = KVSerde.of(new IntegerSerde(), new IntegerSerde()); GenericSystemDescriptor sd = new GenericSystemDescriptor("kafka", "mockFactoryClass"); GenericInputDescriptor<KV<Integer, Integer>> inputDescriptor = sd.getInputDescriptor("integers", kvSerde); appDesc.getInputStream(inputDescriptor) .window(Windows.tumblingWindow(duration, kvSerde).setEarlyTrigger(earlyTrigger) .setAccumulationMode(mode), "w1") .sink((message, messageCollector, taskCoordinator) -> { SystemStream outputSystemStream = new SystemStream("outputSystem", "outputStream"); messageCollector.send(new OutgoingMessageEnvelope(outputSystemStream, message)); }); }; return new StreamApplicationDescriptorImpl(userApp, config); }
Example #7
Source File: SamzaRunner.java From beam with Apache License 2.0 | 6 votes |
private ApplicationRunner runSamzaApp(StreamApplication app, Config config) { final ApplicationRunner runner = ApplicationRunners.getApplicationRunner(app, config); ExternalContext externalContext = null; if (listener != null) { externalContext = listener.onStart(); } runner.run(externalContext); if (listener != null && options.getSamzaExecutionEnvironment() == SamzaExecutionEnvironment.YARN) { listener.onSubmit(); } return runner; }
Example #8
Source File: SamzaRunner.java From beam with Apache License 2.0 | 5 votes |
public PortablePipelineResult runPortablePipeline(RunnerApi.Pipeline pipeline) { final String dotGraph = PipelineDotRenderer.toDotString(pipeline); LOG.info("Portable pipeline to run:\n{}", dotGraph); final ConfigBuilder configBuilder = new ConfigBuilder(options); SamzaPortablePipelineTranslator.createConfig(pipeline, configBuilder, options); configBuilder.put(BEAM_DOT_GRAPH, dotGraph); final Config config = configBuilder.build(); options.setConfigOverride(config); if (listener != null) { listener.onInit(config, options); } final SamzaExecutionContext executionContext = new SamzaExecutionContext(options); final Map<String, MetricsReporterFactory> reporterFactories = getMetricsReporters(); final StreamApplication app = appDescriptor -> { appDescriptor .withApplicationContainerContextFactory(executionContext.new Factory()) .withMetricsReporterFactories(reporterFactories); SamzaPortablePipelineTranslator.translate( pipeline, new PortableTranslationContext(appDescriptor, options)); }; ApplicationRunner runner = runSamzaApp(app, config); return new SamzaPortablePipelineResult(app, runner, executionContext, listener, config); }
Example #9
Source File: TestStreamApplication.java From samza with Apache License 2.0 | 5 votes |
public static StreamApplication getInstance( String systemName, List<String> inputTopics, String outputTopic, CountDownLatch processedMessageLatch, StreamApplicationCallback callback, CountDownLatch kafkaEventsConsumedLatch, Config config) { String appName = new ApplicationConfig(config).getGlobalAppId(); String processorName = config.get(JobConfig.PROCESSOR_ID); registerLatches(processedMessageLatch, kafkaEventsConsumedLatch, callback, appName, processorName); StreamApplication app = new TestStreamApplication(systemName, inputTopics, outputTopic, appName, processorName); return app; }
Example #10
Source File: TestLocalApplicationRunner.java From samza with Apache License 2.0 | 5 votes |
@Test public void testRunFailure() throws Exception { Map<String, String> cfgs = new HashMap<>(); cfgs.put(ApplicationConfig.PROCESSOR_ID, "0"); config = new MapConfig(cfgs); ProcessorLifecycleListenerFactory mockFactory = (pContext, cfg) -> mock(ProcessorLifecycleListener.class); mockApp = (StreamApplication) appDesc -> appDesc.withProcessorLifecycleListenerFactory(mockFactory); prepareTest(); // return the jobConfigs from the planner doReturn(Collections.singletonList(new JobConfig(new MapConfig(config)))).when(localPlanner).prepareJobs(); StreamProcessor sp = mock(StreamProcessor.class); CoordinatorStreamStore coordinatorStreamStore = mock(CoordinatorStreamStore.class); ArgumentCaptor<StreamProcessor.StreamProcessorLifecycleListenerFactory> captor = ArgumentCaptor.forClass(StreamProcessor.StreamProcessorLifecycleListenerFactory.class); doAnswer(i -> { throw new Exception("test failure"); }).when(sp).start(); ExternalContext externalContext = mock(ExternalContext.class); doReturn(sp).when(runner) .createStreamProcessor(anyObject(), anyObject(), captor.capture(), eq(Optional.of(externalContext)), any( CoordinatorStreamStore.class)); doReturn(coordinatorStreamStore).when(runner).createCoordinatorStreamStore(any(Config.class)); try { runner.run(externalContext); runner.waitForFinish(); } catch (Throwable th) { assertNotNull(th); } verify(coordinatorStreamStore).init(); verify(coordinatorStreamStore, never()).close(); assertEquals(runner.status(), ApplicationStatus.UnsuccessfulFinish); }
Example #11
Source File: TestLocalApplicationRunner.java From samza with Apache License 2.0 | 5 votes |
@Test public void testRunCompleteWithouCoordinatorStreamStore() throws Exception { Map<String, String> cfgs = new HashMap<>(); cfgs.put(ApplicationConfig.APP_PROCESSOR_ID_GENERATOR_CLASS, UUIDGenerator.class.getName()); config = new MapConfig(cfgs); ProcessorLifecycleListenerFactory mockFactory = (pContext, cfg) -> mock(ProcessorLifecycleListener.class); mockApp = (StreamApplication) appDesc -> appDesc.withProcessorLifecycleListenerFactory(mockFactory); prepareTest(); // return the jobConfigs from the planner doReturn(Collections.singletonList(new JobConfig(new MapConfig(config)))).when(localPlanner).prepareJobs(); StreamProcessor sp = mock(StreamProcessor.class); ArgumentCaptor<StreamProcessor.StreamProcessorLifecycleListenerFactory> captor = ArgumentCaptor.forClass(StreamProcessor.StreamProcessorLifecycleListenerFactory.class); doAnswer(i -> { ProcessorLifecycleListener listener = captor.getValue().createInstance(sp); listener.afterStart(); listener.afterStop(); return null; }).when(sp).start(); ExternalContext externalContext = mock(ExternalContext.class); doReturn(sp).when(runner) .createStreamProcessor(anyObject(), anyObject(), captor.capture(), eq(Optional.of(externalContext)), eq(null)); doReturn(null).when(runner).createCoordinatorStreamStore(any(Config.class)); runner.run(externalContext); runner.waitForFinish(); assertEquals(runner.status(), ApplicationStatus.SuccessfulFinish); }
Example #12
Source File: TestLocalApplicationRunner.java From samza with Apache License 2.0 | 5 votes |
@Test public void testRunComplete() throws Exception { Map<String, String> cfgs = new HashMap<>(); cfgs.put(ApplicationConfig.APP_PROCESSOR_ID_GENERATOR_CLASS, UUIDGenerator.class.getName()); config = new MapConfig(cfgs); ProcessorLifecycleListenerFactory mockFactory = (pContext, cfg) -> mock(ProcessorLifecycleListener.class); mockApp = (StreamApplication) appDesc -> appDesc.withProcessorLifecycleListenerFactory(mockFactory); prepareTest(); // return the jobConfigs from the planner doReturn(Collections.singletonList(new JobConfig(new MapConfig(config)))).when(localPlanner).prepareJobs(); StreamProcessor sp = mock(StreamProcessor.class); CoordinatorStreamStore coordinatorStreamStore = mock(CoordinatorStreamStore.class); ArgumentCaptor<StreamProcessor.StreamProcessorLifecycleListenerFactory> captor = ArgumentCaptor.forClass(StreamProcessor.StreamProcessorLifecycleListenerFactory.class); doAnswer(i -> { ProcessorLifecycleListener listener = captor.getValue().createInstance(sp); listener.afterStart(); listener.afterStop(); return null; }).when(sp).start(); ExternalContext externalContext = mock(ExternalContext.class); doReturn(sp).when(runner) .createStreamProcessor(anyObject(), anyObject(), captor.capture(), eq(Optional.of(externalContext)), any(CoordinatorStreamStore.class)); doReturn(coordinatorStreamStore).when(runner).createCoordinatorStreamStore(any(Config.class)); runner.run(externalContext); runner.waitForFinish(); verify(coordinatorStreamStore).init(); verify(coordinatorStreamStore, never()).close(); assertEquals(runner.status(), ApplicationStatus.SuccessfulFinish); }
Example #13
Source File: TestLocalTableEndToEnd.java From samza with Apache License 2.0 | 5 votes |
@Test public void testSendTo() throws Exception { int count = 10; Profile[] profiles = TestTableData.generateProfiles(count); int partitionCount = 4; Map<String, String> configs = getBaseJobConfig(bootstrapUrl(), zkConnect()); configs.put("streams.Profile.samza.system", "test"); configs.put("streams.Profile.source", Base64Serializer.serialize(profiles)); configs.put("streams.Profile.partitionCount", String.valueOf(partitionCount)); MyMapFunction mapFn = new MyMapFunction(); final StreamApplication app = appDesc -> { Table<KV<Integer, Profile>> table = appDesc.getTable(new InMemoryTableDescriptor("t1", KVSerde.of(new IntegerSerde(), new ProfileJsonSerde()))); DelegatingSystemDescriptor ksd = new DelegatingSystemDescriptor("test"); GenericInputDescriptor<Profile> isd = ksd.getInputDescriptor("Profile", new NoOpSerde<>()); appDesc.getInputStream(isd) .map(mapFn) .sendTo(table); }; Config config = new MapConfig(configs); final LocalApplicationRunner runner = new LocalApplicationRunner(app, config); executeRun(runner, config); runner.waitForFinish(); for (int i = 0; i < partitionCount; i++) { MyMapFunction mapFnCopy = MyMapFunction.getMapFunctionByTask(String.format("Partition %d", i)); assertEquals(count, mapFnCopy.received.size()); mapFnCopy.received.forEach(p -> Assert.assertTrue(mapFnCopy.table.get(p.getMemberId()) != null)); } }
Example #14
Source File: TestRemoteApplicationRunner.java From samza with Apache License 2.0 | 5 votes |
@Before public void setUp() { Map<String, String> config = new HashMap<>(); config.put(ApplicationConfig.APP_NAME, "test-app"); StreamApplication userApp = appDesc -> { }; runner = spy(new RemoteApplicationRunner(userApp, new MapConfig(config))); }
Example #15
Source File: SamzaPortablePipelineResult.java From beam with Apache License 2.0 | 5 votes |
SamzaPortablePipelineResult( StreamApplication app, ApplicationRunner runner, SamzaExecutionContext executionContext, SamzaPipelineLifeCycleListener listener, Config config) { super(app, runner, executionContext, listener, config); }
Example #16
Source File: SamzaPipelineResult.java From beam with Apache License 2.0 | 5 votes |
public SamzaPipelineResult( StreamApplication app, ApplicationRunner runner, SamzaExecutionContext executionContext, SamzaPipelineLifeCycleListener listener, Config config) { this.executionContext = executionContext; this.runner = runner; this.app = app; this.listener = listener; this.shutdownTiemoutMs = config.getLong(TASK_SHUTDOWN_MS, DEFAULT_TASK_SHUTDOWN_MS) + SHUTDOWN_TIMEOUT_BUFFER; }
Example #17
Source File: SystemConsumerWithSamzaBench.java From samza with Apache License 2.0 | 5 votes |
public void start() throws IOException, InterruptedException { super.start(); MessageConsumer consumeFn = new MessageConsumer(); StreamApplication app = appDesc -> { String systemFactoryName = new SystemConfig(config).getSystemFactory(systemName).get(); GenericSystemDescriptor sd = new GenericSystemDescriptor(systemName, systemFactoryName); GenericInputDescriptor<Object> isd = sd.getInputDescriptor(streamId, new NoOpSerde<>()); MessageStream<Object> stream = appDesc.getInputStream(isd); stream.map(consumeFn); }; ApplicationRunner runner = ApplicationRunners.getApplicationRunner(app, new MapConfig()); runner.run(); while (consumeFn.getEventsConsumed() < totalEvents) { Thread.sleep(10); } Instant endTime = Instant.now(); runner.kill(); System.out.println("\n*******************"); System.out.println(String.format("Started at %s Ending at %s ", consumeFn.startTime, endTime)); System.out.println(String.format("Event Rate is %s Messages/Sec ", consumeFn.getEventsConsumed() * 1000 / Duration.between(consumeFn.startTime, Instant.now()).toMillis())); System.out.println( "Event Rate is " + consumeFn.getEventsConsumed() * 1000 / Duration.between(consumeFn.startTime, endTime).toMillis()); System.out.println("*******************\n"); System.exit(0); }
Example #18
Source File: ExecutionPlannerTestBase.java From samza with Apache License 2.0 | 5 votes |
StreamApplication getRepartitionJoinStreamApplication() { return appDesc -> { MessageStream<KV<String, Object>> input1 = appDesc.getInputStream(input1Descriptor); MessageStream<KV<String, Object>> input2 = appDesc.getInputStream(input2Descriptor); OutputStream<KV<String, Object>> output = appDesc.getOutputStream(outputDescriptor); JoinFunction<String, Object, Object, KV<String, Object>> mockJoinFn = mock(JoinFunction.class); input1 .partitionBy(KV::getKey, KV::getValue, defaultSerde, "p1") .map(kv -> kv.value) .join(input2.map(kv -> kv.value), mockJoinFn, new StringSerde(), new JsonSerdeV2<>(Object.class), new JsonSerdeV2<>(Object.class), Duration.ofHours(1), "j1") .sendTo(output); }; }
Example #19
Source File: TestStreamApplicationDescriptorImpl.java From samza with Apache License 2.0 | 5 votes |
@Test public void testProcessorLifecycleListenerFactory() { ProcessorLifecycleListenerFactory mockFactory = mock(ProcessorLifecycleListenerFactory.class); StreamApplication testApp = appSpec -> appSpec.withProcessorLifecycleListenerFactory(mockFactory); StreamApplicationDescriptorImpl appDesc = new StreamApplicationDescriptorImpl(testApp, getConfig()); assertEquals(appDesc.getProcessorLifecycleListenerFactory(), mockFactory); }
Example #20
Source File: TestStreamApplicationDescriptorImpl.java From samza with Apache License 2.0 | 5 votes |
@Test public void testNoApplicationTaskContextFactory() { StreamApplication testApp = appDesc -> { }; StreamApplicationDescriptorImpl appSpec = new StreamApplicationDescriptorImpl(testApp, getConfig()); assertEquals(appSpec.getApplicationTaskContextFactory(), Optional.empty()); }
Example #21
Source File: TestStreamApplicationDescriptorImpl.java From samza with Apache License 2.0 | 5 votes |
@Test public void testApplicationTaskContextFactory() { ApplicationTaskContextFactory factory = mock(ApplicationTaskContextFactory.class); StreamApplication testApp = appDesc -> appDesc.withApplicationTaskContextFactory(factory); StreamApplicationDescriptorImpl appSpec = new StreamApplicationDescriptorImpl(testApp, getConfig()); assertEquals(appSpec.getApplicationTaskContextFactory(), Optional.of(factory)); }
Example #22
Source File: TestStreamApplicationDescriptorImpl.java From samza with Apache License 2.0 | 5 votes |
@Test public void testNoApplicationContainerContextFactory() { StreamApplication testApp = appDesc -> { }; StreamApplicationDescriptorImpl appSpec = new StreamApplicationDescriptorImpl(testApp, getConfig()); assertEquals(appSpec.getApplicationContainerContextFactory(), Optional.empty()); }
Example #23
Source File: TestStreamApplicationDescriptorImpl.java From samza with Apache License 2.0 | 5 votes |
@Test public void testApplicationContainerContextFactory() { ApplicationContainerContextFactory factory = mock(ApplicationContainerContextFactory.class); StreamApplication testApp = appDesc -> appDesc.withApplicationContainerContextFactory(factory); StreamApplicationDescriptorImpl appSpec = new StreamApplicationDescriptorImpl(testApp, getConfig()); assertEquals(appSpec.getApplicationContainerContextFactory(), Optional.of(factory)); }
Example #24
Source File: TestStreamApplicationDescriptorImpl.java From samza with Apache License 2.0 | 5 votes |
@Test public void testConstructor() { StreamApplication mockApp = mock(StreamApplication.class); Config mockConfig = getConfig(); StreamApplicationDescriptorImpl appDesc = new StreamApplicationDescriptorImpl(mockApp, mockConfig); verify(mockApp).describe(appDesc); assertEquals(mockConfig, appDesc.getConfig()); }
Example #25
Source File: ApplicationDescriptorUtil.java From samza with Apache License 2.0 | 5 votes |
/** * Create a new instance of {@link ApplicationDescriptorImpl} based on {@link SamzaApplication} and {@link Config} * * @param app an implementation of {@link SamzaApplication}. The {@code app} has to have a proper fully-qualified class name. * @param config the {@link Config} for the application * @return the {@link ApplicationDescriptorImpl} instance containing the processing logic and the config */ public static ApplicationDescriptorImpl<? extends ApplicationDescriptor> getAppDescriptor(SamzaApplication app, Config config) { if (app instanceof StreamApplication) { return new StreamApplicationDescriptorImpl((StreamApplication) app, config); } if (app instanceof TaskApplication) { return new TaskApplicationDescriptorImpl((TaskApplication) app, config); } throw new IllegalArgumentException(String.format("User application class %s is not supported. Only StreamApplication " + "and TaskApplication are supported.", app.getClass().getName())); }
Example #26
Source File: TestApplicationRunners.java From samza with Apache License 2.0 | 5 votes |
@Test public void testGetAppRunner() { Map<String, String> configMap = new HashMap<>(); configMap.put("app.runner.class", MockApplicationRunner.class.getName()); Config config = new MapConfig(configMap); StreamApplication app = mock(StreamApplication.class); ApplicationRunner appRunner = ApplicationRunners.getApplicationRunner(app, config); assertTrue(appRunner instanceof MockApplicationRunner); }
Example #27
Source File: TestCouchbaseRemoteTableEndToEnd.java From samza with Apache License 2.0 | 4 votes |
@Test public void testEndToEnd() throws Exception { Bucket inputBucket = cluster.openBucket(inputBucketName); inputBucket.upsert(ByteArrayDocument.create("Alice", "20".getBytes())); inputBucket.upsert(ByteArrayDocument.create("Bob", "30".getBytes())); inputBucket.upsert(ByteArrayDocument.create("Chris", "40".getBytes())); inputBucket.upsert(ByteArrayDocument.create("David", "50".getBytes())); inputBucket.close(); String[] users = new String[]{"Alice", "Bob", "Chris", "David"}; int partitionCount = 1; Map<String, String> configs = TestLocalTableEndToEnd.getBaseJobConfig(bootstrapUrl(), zkConnect()); configs.put("streams.User.samza.system", "test"); configs.put("streams.User.source", Base64Serializer.serialize(users)); configs.put("streams.User.partitionCount", String.valueOf(partitionCount)); Config config = new MapConfig(configs); final StreamApplication app = appDesc -> { DelegatingSystemDescriptor inputSystemDescriptor = new DelegatingSystemDescriptor("test"); GenericInputDescriptor<String> inputDescriptor = inputSystemDescriptor.getInputDescriptor("User", new NoOpSerde<>()); CouchbaseTableReadFunction<String> readFunction = new CouchbaseTableReadFunction<>(inputBucketName, String.class, "couchbase://127.0.0.1") .withBootstrapCarrierDirectPort(couchbaseMock.getCarrierPort(inputBucketName)) .withBootstrapHttpDirectPort(couchbaseMock.getHttpPort()) .withSerde(new StringSerde()); CouchbaseTableWriteFunction<JsonObject> writeFunction = new CouchbaseTableWriteFunction<>(outputBucketName, JsonObject.class, "couchbase://127.0.0.1") .withBootstrapCarrierDirectPort(couchbaseMock.getCarrierPort(outputBucketName)) .withBootstrapHttpDirectPort(couchbaseMock.getHttpPort()); RemoteTableDescriptor inputTableDesc = new RemoteTableDescriptor<String, String>("input-table") .withReadFunction(readFunction) .withRateLimiterDisabled(); Table<KV<String, String>> inputTable = appDesc.getTable(inputTableDesc); RemoteTableDescriptor outputTableDesc = new RemoteTableDescriptor<String, JsonObject>("output-table") .withReadFunction(new NoOpTableReadFunction<>()) .withWriteFunction(writeFunction) .withRateLimiterDisabled(); Table<KV<String, JsonObject>> outputTable = appDesc.getTable(outputTableDesc); appDesc.getInputStream(inputDescriptor) .map(k -> KV.of(k, k)) .join(inputTable, new JoinFunction()) .sendTo(outputTable); }; final LocalApplicationRunner runner = new LocalApplicationRunner(app, config); executeRun(runner, config); runner.waitForFinish(); Bucket outputBucket = cluster.openBucket(outputBucketName); Assert.assertEquals("{\"name\":\"Alice\",\"age\":\"20\"}", outputBucket.get("Alice").content().toString()); Assert.assertEquals("{\"name\":\"Bob\",\"age\":\"30\"}", outputBucket.get("Bob").content().toString()); Assert.assertEquals("{\"name\":\"Chris\",\"age\":\"40\"}", outputBucket.get("Chris").content().toString()); Assert.assertEquals("{\"name\":\"David\",\"age\":\"50\"}", outputBucket.get("David").content().toString()); outputBucket.close(); }
Example #28
Source File: SamzaRunner.java From beam with Apache License 2.0 | 4 votes |
@Override public SamzaPipelineResult run(Pipeline pipeline) { MetricsEnvironment.setMetricsSupported(true); if (LOG.isDebugEnabled()) { LOG.debug("Pre-processed Beam pipeline:\n{}", PipelineDotRenderer.toDotString(pipeline)); } pipeline.replaceAll(SamzaTransformOverrides.getDefaultOverrides()); final String dotGraph = PipelineDotRenderer.toDotString(pipeline); LOG.info("Beam pipeline DOT graph:\n{}", dotGraph); final Map<PValue, String> idMap = PViewToIdMapper.buildIdMap(pipeline); final ConfigBuilder configBuilder = new ConfigBuilder(options); SamzaPipelineTranslator.createConfig(pipeline, options, idMap, configBuilder); configBuilder.put(BEAM_DOT_GRAPH, dotGraph); final Config config = configBuilder.build(); options.setConfigOverride(config); if (listener != null) { listener.onInit(config, options); } final SamzaExecutionContext executionContext = new SamzaExecutionContext(options); final Map<String, MetricsReporterFactory> reporterFactories = getMetricsReporters(); final StreamApplication app = appDescriptor -> { appDescriptor.withApplicationContainerContextFactory(executionContext.new Factory()); appDescriptor.withMetricsReporterFactories(reporterFactories); SamzaPipelineTranslator.translate( pipeline, new TranslationContext(appDescriptor, idMap, options)); }; // perform a final round of validation for the pipeline options now that all configs are // generated SamzaPipelineOptionsValidator.validate(options); ApplicationRunner runner = runSamzaApp(app, config); return new SamzaPipelineResult(app, runner, executionContext, listener, config); }
Example #29
Source File: TestLocalTableWithSideInputsEndToEnd.java From samza with Apache License 2.0 | 4 votes |
private void runTest(String systemName, StreamApplication app, List<PageView> pageViews, List<Profile> profiles) { Map<String, String> configs = new HashMap<>(); configs.put(String.format(StreamConfig.SYSTEM_FOR_STREAM_ID, PAGEVIEW_STREAM), systemName); configs.put(String.format(StreamConfig.SYSTEM_FOR_STREAM_ID, PROFILE_STREAM), systemName); configs.put(String.format(StreamConfig.SYSTEM_FOR_STREAM_ID, ENRICHED_PAGEVIEW_STREAM), systemName); InMemorySystemDescriptor isd = new InMemorySystemDescriptor(systemName); InMemoryInputDescriptor<PageView> pageViewStreamDesc = isd .getInputDescriptor(PAGEVIEW_STREAM, new NoOpSerde<PageView>()); InMemoryInputDescriptor<Profile> profileStreamDesc = isd .getInputDescriptor(PROFILE_STREAM, new NoOpSerde<Profile>()); InMemoryOutputDescriptor<EnrichedPageView> outputStreamDesc = isd .getOutputDescriptor(ENRICHED_PAGEVIEW_STREAM, new NoOpSerde<EnrichedPageView>()); TestRunner .of(app) .addInputStream(pageViewStreamDesc, pageViews) .addInputStream(profileStreamDesc, profiles) .addOutputStream(outputStreamDesc, 1) .addConfig(new MapConfig(configs)) .run(Duration.ofMillis(100000)); try { Map<Integer, List<EnrichedPageView>> result = TestRunner.consumeStream(outputStreamDesc, Duration.ofMillis(1000)); List<EnrichedPageView> results = result.values().stream() .flatMap(List::stream) .collect(Collectors.toList()); List<EnrichedPageView> expectedEnrichedPageviews = pageViews.stream() .flatMap(pv -> profiles.stream() .filter(profile -> pv.memberId == profile.memberId) .map(profile -> new EnrichedPageView(pv.pageKey, profile.memberId, profile.company))) .collect(Collectors.toList()); boolean successfulJoin = results.stream().allMatch(expectedEnrichedPageviews::contains); assertEquals("Mismatch between the expected and actual join count", expectedEnrichedPageviews.size(), results.size()); assertTrue("Pageview profile join did not succeed for all inputs", successfulJoin); } catch (SamzaException e) { e.printStackTrace(); } }
Example #30
Source File: TestRemoteTableEndToEnd.java From samza with Apache License 2.0 | 4 votes |
private void doTestStreamTableJoinRemoteTable(boolean withCache, boolean defaultCache, boolean withArgs, String testName) throws Exception { writtenRecords.put(testName, new ArrayList<>()); int count = 10; final PageView[] pageViews = generatePageViews(count); final String profiles = Base64Serializer.serialize(generateProfiles(count)); final int partitionCount = 4; final Map<String, String> configs = TestLocalTableEndToEnd.getBaseJobConfig(bootstrapUrl(), zkConnect()); configs.put("streams.PageView.samza.system", "test"); configs.put("streams.PageView.source", Base64Serializer.serialize(pageViews)); configs.put("streams.PageView.partitionCount", String.valueOf(partitionCount)); final RateLimiter readRateLimiter = mock(RateLimiter.class, withSettings().serializable()); final TableRateLimiter.CreditFunction creditFunction = (k, v, args) -> 1; final StreamApplication app = appDesc -> { final RemoteTableDescriptor joinTableDesc = new RemoteTableDescriptor<Integer, TestTableData.Profile>("profile-table-1") .withReadFunction(InMemoryProfileReadFunction.getInMemoryReadFunction(testName, profiles)) .withRateLimiter(readRateLimiter, creditFunction, null); final RemoteTableDescriptor outputTableDesc = new RemoteTableDescriptor<Integer, EnrichedPageView>("enriched-page-view-table-1") .withReadFunction(new NoOpTableReadFunction<>()) .withReadRateLimiterDisabled() .withWriteFunction(new InMemoryEnrichedPageViewWriteFunction(testName)) .withWriteRateLimit(1000); final Table<KV<Integer, EnrichedPageView>> outputTable = withCache ? getCachingTable(outputTableDesc, defaultCache, appDesc) : appDesc.getTable(outputTableDesc); final Table<KV<Integer, Profile>> joinTable = withCache ? getCachingTable(joinTableDesc, defaultCache, appDesc) : appDesc.getTable(joinTableDesc); final DelegatingSystemDescriptor ksd = new DelegatingSystemDescriptor("test"); final GenericInputDescriptor<PageView> isd = ksd.getInputDescriptor("PageView", new NoOpSerde<>()); if (!withArgs) { appDesc.getInputStream(isd) .map(pv -> new KV<>(pv.getMemberId(), pv)) .join(joinTable, new PageViewToProfileJoinFunction()) .map(m -> new KV(m.getMemberId(), m)) .sendTo(outputTable); } else { counters.put(testName, new AtomicInteger()); final RemoteTableDescriptor counterTableDesc = new RemoteTableDescriptor("counter-table-1") .withReadFunction(new InMemoryCounterReadFunction(testName)) .withWriteFunction(new InMemoryCounterWriteFunction(testName)) .withRateLimiterDisabled(); final Table counterTable = withCache ? getCachingTable(counterTableDesc, defaultCache, appDesc) : appDesc.getTable(counterTableDesc); final String counterTableName = ((TableImpl) counterTable).getTableId(); appDesc.getInputStream(isd) .map(new TestReadWriteMapFunction(counterTableName)) .map(pv -> new KV<>(pv.getMemberId(), pv)) .join(joinTable, new PageViewToProfileJoinFunction(), true) .map(m -> new KV(m.getMemberId(), m)) .sendTo(outputTable, true); } }; final Config config = new MapConfig(configs); final LocalApplicationRunner runner = new LocalApplicationRunner(app, config); executeRun(runner, config); runner.waitForFinish(); final int numExpected = count * partitionCount; Assert.assertEquals(numExpected, writtenRecords.get(testName).size()); Assert.assertTrue(writtenRecords.get(testName).get(0) instanceof EnrichedPageView); if (!withArgs) { writtenRecords.get(testName).forEach(epv -> Assert.assertFalse(epv.company.contains("-"))); } else { writtenRecords.get(testName).forEach(epv -> Assert.assertTrue(epv.company.endsWith("-r-w"))); Assert.assertEquals(numExpected, counters.get(testName).get()); } }