Java Code Examples for org.apache.beam.sdk.coders.StringUtf8Coder
The following examples show how to use
org.apache.beam.sdk.coders.StringUtf8Coder. These examples are extracted from open source projects.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: deployment-examples Source File: StatefulTeamScoreTest.java License: MIT License | 6 votes |
/** * Tests that {@link UpdateTeamScoreFn} {@link org.apache.beam.sdk.transforms.DoFn} outputs * correctly for one team. */ @Test public void testScoreUpdatesOneTeam() { TestStream<KV<String, GameActionInfo>> createEvents = TestStream.create(KvCoder.of(StringUtf8Coder.of(), AvroCoder.of(GameActionInfo.class))) .advanceWatermarkTo(baseTime) .addElements( event(TestUser.RED_TWO, 99, Duration.standardSeconds(10)), event(TestUser.RED_ONE, 1, Duration.standardSeconds(20)), event(TestUser.RED_ONE, 0, Duration.standardSeconds(30)), event(TestUser.RED_TWO, 100, Duration.standardSeconds(40)), event(TestUser.RED_TWO, 201, Duration.standardSeconds(50))) .advanceWatermarkToInfinity(); PCollection<KV<String, Integer>> teamScores = p.apply(createEvents).apply(ParDo.of(new UpdateTeamScoreFn(100))); String redTeam = TestUser.RED_ONE.getTeam(); PAssert.that(teamScores) .inWindow(GlobalWindow.INSTANCE) .containsInAnyOrder(KV.of(redTeam, 100), KV.of(redTeam, 200), KV.of(redTeam, 401)); p.run().waitUntilFinish(); }
Example 2
Source Project: beam Source File: ToStringTest.java License: Apache License 2.0 | 6 votes |
@Test @Category(NeedsRunner.class) public void testToStringIterableWithDelimiter() { ArrayList<Iterable<String>> iterables = new ArrayList<>(); iterables.add(Arrays.asList(new String[] {"one", "two", "three"})); iterables.add(Arrays.asList(new String[] {"four", "five", "six"})); ArrayList<String> expected = new ArrayList<>(); expected.add("one\ttwo\tthree"); expected.add("four\tfive\tsix"); PCollection<Iterable<String>> input = p.apply(Create.of(iterables).withCoder(IterableCoder.of(StringUtf8Coder.of()))); PCollection<String> output = input.apply(ToString.iterables("\t")); PAssert.that(output).containsInAnyOrder(expected); p.run(); }
Example 3
Source Project: DataflowTemplates Source File: WriteToGCSAvroTest.java License: Apache License 2.0 | 6 votes |
/** * Test whether {@link WriteToGCSAvro} throws an exception if temporary directory is not provided. */ @Test public void testWriteWithoutTempLocation() { expectedException.expect(IllegalArgumentException.class); expectedException.expectMessage("withTempLocation(tempLocation) called with null input. "); pipeline .apply( "CreateInput", Create.of(message).withCoder(KvCoder.of(StringUtf8Coder.of(), StringUtf8Coder.of()))) .apply( "WriteTextFile(s)", WriteToGCSAvro.newBuilder() .withOutputDirectory(FAKE_DIR) .withOutputFilenamePrefix(AVRO_FILENAME_PREFIX) .setNumShards(NUM_SHARDS) .withTempLocation(null) .build()); pipeline.run(); }
Example 4
Source Project: beam Source File: ParDoTranslationTest.java License: Apache License 2.0 | 6 votes |
@Test public void testProcessContext() throws Exception { Pipeline p = Pipeline.create(); SdkComponents sdkComponents = SdkComponents.create(); sdkComponents.registerEnvironment(Environments.createDockerEnvironment("java")); ParDoPayload payload = ParDoTranslation.translateParDo( ParDo.of(new ProcessContextDoFn()) .withOutputTags(new TupleTag<>(), TupleTagList.empty()), PCollection.createPrimitiveOutputInternal( p, WindowingStrategy.globalDefault(), IsBounded.BOUNDED, StringUtf8Coder.of()), DoFnSchemaInformation.create(), TestPipeline.create(), sdkComponents); assertTrue(payload.getRequestsFinalization()); }
Example 5
Source Project: beam Source File: StreamingDataflowWorkerTest.java License: Apache License 2.0 | 6 votes |
@Test public void testHugeCommits() throws Exception { List<ParallelInstruction> instructions = Arrays.asList( makeSourceInstruction(StringUtf8Coder.of()), makeDoFnInstruction(new FanoutFn(), 0, StringUtf8Coder.of()), makeSinkInstruction(StringUtf8Coder.of(), 0)); FakeWindmillServer server = new FakeWindmillServer(errorCollector); StreamingDataflowWorkerOptions options = createTestingPipelineOptions(server); StreamingDataflowWorker worker = makeWorker(instructions, options, true /* publishCounters */); worker.start(); server.addWorkToOffer(makeInput(0, TimeUnit.MILLISECONDS.toMicros(0))); server.waitForAndGetCommits(0); worker.stop(); }
Example 6
Source Project: beam Source File: ProvidedSparkContextTest.java License: Apache License 2.0 | 6 votes |
private void testWithValidProvidedContext(JavaSparkContext jsc) throws Exception { SparkContextOptions options = getSparkContextOptions(jsc); Pipeline p = Pipeline.create(options); PCollection<String> inputWords = p.apply(Create.of(WORDS).withCoder(StringUtf8Coder.of())); PCollection<String> output = inputWords .apply(new WordCount.CountWords()) .apply(MapElements.via(new WordCount.FormatAsTextFn())); PAssert.that(output).containsInAnyOrder(EXPECTED_COUNT_SET); // Run test from pipeline PipelineResult result = p.run(); TestPipeline.verifyPAssertsSucceeded(p, result); }
Example 7
Source Project: beam Source File: EvaluationContextTest.java License: Apache License 2.0 | 6 votes |
@Test public void getExecutionContextDifferentKeysIndependentState() { DirectExecutionContext fooContext = context.getExecutionContext(createdProducer, StructuralKey.of("foo", StringUtf8Coder.of())); StateTag<BagState<Integer>> intBag = StateTags.bag("myBag", VarIntCoder.of()); fooContext.getStepContext("s1").stateInternals().state(StateNamespaces.global(), intBag).add(1); DirectExecutionContext barContext = context.getExecutionContext(createdProducer, StructuralKey.of("bar", StringUtf8Coder.of())); assertThat(barContext, not(equalTo(fooContext))); assertThat( barContext .getStepContext("s1") .stateInternals() .state(StateNamespaces.global(), intBag) .read(), emptyIterable()); }
Example 8
Source Project: beam Source File: WriteFiles.java License: Apache License 2.0 | 6 votes |
@Override public WriteFilesResult<DestinationT> expand( PCollection<List<FileResult<DestinationT>>> input) { List<PCollectionView<?>> finalizeSideInputs = Lists.newArrayList(getSideInputs()); if (numShardsView != null) { finalizeSideInputs.add(numShardsView); } PCollection<KV<DestinationT, String>> outputFilenames = input .apply("Finalize", ParDo.of(new FinalizeFn()).withSideInputs(finalizeSideInputs)) .setCoder(KvCoder.of(destinationCoder, StringUtf8Coder.of())) // Reshuffle the filenames to make sure they are observable downstream // only after each one is done finalizing. .apply(Reshuffle.viaRandomKey()); TupleTag<KV<DestinationT, String>> perDestinationOutputFilenamesTag = new TupleTag<>("perDestinationOutputFilenames"); return WriteFilesResult.in( input.getPipeline(), perDestinationOutputFilenamesTag, outputFilenames); }
Example 9
Source Project: beam Source File: WatchTest.java License: Apache License 2.0 | 6 votes |
@Test public void testCoder() throws Exception { GrowthState pollingState = PollingGrowthState.of( ImmutableMap.of( HashCode.fromString("0123456789abcdef0123456789abcdef"), Instant.now(), HashCode.fromString("01230123012301230123012301230123"), Instant.now()), Instant.now(), "STATE"); GrowthState nonPollingState = NonPollingGrowthState.of( Growth.PollResult.incomplete(Instant.now(), Arrays.asList("A", "B"))); Coder<GrowthState> coder = Watch.GrowthStateCoder.of(StringUtf8Coder.of(), StringUtf8Coder.of()); CoderProperties.coderDecodeEncodeEqual(coder, pollingState); CoderProperties.coderDecodeEncodeEqual(coder, nonPollingState); }
Example 10
Source Project: beam Source File: ReaderFactoryTest.java License: Apache License 2.0 | 6 votes |
@Test public void testCreateUnknownReader() throws Exception { CloudObject spec = CloudObject.forClassName("UnknownSource"); Source cloudSource = new Source(); cloudSource.setSpec(spec); cloudSource.setCodec(CloudObjects.asCloudObject(StringUtf8Coder.of(), /*sdkComponents=*/ null)); try { PipelineOptions options = PipelineOptionsFactory.create(); ReaderRegistry.defaultRegistry() .create( cloudSource, options, BatchModeExecutionContext.forTesting(options, "testStage"), null); Assert.fail("should have thrown an exception"); } catch (Exception exn) { Assert.assertThat(exn.toString(), CoreMatchers.containsString("Unable to create a Reader")); } }
Example 11
Source Project: beam Source File: CombineTest.java License: Apache License 2.0 | 6 votes |
@Test @Category({ValidatesRunner.class, DataflowPortabilityApiUnsupported.class}) public void testFixedWindowsCombine() { PCollection<KV<String, Integer>> input = pipeline .apply( Create.timestamped( TimestampedValue.of(KV.of("a", 1), new Instant(0L)), TimestampedValue.of(KV.of("a", 1), new Instant(1L)), TimestampedValue.of(KV.of("a", 4), new Instant(6L)), TimestampedValue.of(KV.of("b", 1), new Instant(7L)), TimestampedValue.of(KV.of("b", 13), new Instant(8L))) .withCoder(KvCoder.of(StringUtf8Coder.of(), BigEndianIntegerCoder.of()))) .apply(Window.into(FixedWindows.of(Duration.millis(2)))); PCollection<Integer> sum = input.apply(Values.create()).apply(Combine.globally(new SumInts()).withoutDefaults()); PCollection<KV<String, String>> sumPerKey = input.apply(Combine.perKey(new TestCombineFn())); PAssert.that(sum).containsInAnyOrder(2, 5, 13); PAssert.that(sumPerKey) .containsInAnyOrder( Arrays.asList(KV.of("a", "11"), KV.of("a", "4"), KV.of("b", "1"), KV.of("b", "13"))); pipeline.run(); }
Example 12
Source Project: beam Source File: FlinkStreamingTransformTranslatorsTest.java License: Apache License 2.0 | 6 votes |
private Object applyReadSourceTransform( PTransform<?, ?> transform, PCollection.IsBounded isBounded, StreamExecutionEnvironment env) { FlinkStreamingPipelineTranslator.StreamTransformTranslator<PTransform<?, ?>> translator = getReadSourceTranslator(); FlinkStreamingTranslationContext ctx = new FlinkStreamingTranslationContext(env, PipelineOptionsFactory.create()); Pipeline pipeline = Pipeline.create(); PCollection<String> pc = PCollection.createPrimitiveOutputInternal( pipeline, WindowingStrategy.globalDefault(), isBounded, StringUtf8Coder.of()); pc.setName("output"); Map<TupleTag<?>, PValue> outputs = new HashMap<>(); outputs.put(new TupleTag<>(), pc); AppliedPTransform<?, ?, ?> appliedTransform = AppliedPTransform.of( "test-transform", Collections.emptyMap(), outputs, transform, Pipeline.create()); ctx.setCurrentTransform(appliedTransform); translator.translateNode(transform, ctx); return ctx.getInputDataStream(pc).getTransformation(); }
Example 13
Source Project: beam Source File: ParDoTest.java License: Apache License 2.0 | 6 votes |
@Test @Category({ ValidatesRunner.class, UsesTimersInParDo.class, UsesTestStream.class, UsesStatefulParDo.class, UsesStrictTimerOrdering.class }) public void testEventTimeTimerOrdering() throws Exception { final int numTestElements = 100; final Instant now = new Instant(1500000000000L); TestStream.Builder<KV<String, String>> builder = TestStream.create(KvCoder.of(StringUtf8Coder.of(), StringUtf8Coder.of())) .advanceWatermarkTo(new Instant(0)); for (int i = 0; i < numTestElements; i++) { builder = builder.addElements(TimestampedValue.of(KV.of("dummy", "" + i), now.plus(i * 1000))); if ((i + 1) % 10 == 0) { builder = builder.advanceWatermarkTo(now.plus((i + 1) * 1000)); } } testEventTimeTimerOrderingWithInputPTransform( now, numTestElements, builder.advanceWatermarkToInfinity()); }
Example 14
Source Project: beam Source File: CopyOnAccessInMemoryStateInternalsTest.java License: Apache License 2.0 | 6 votes |
@Test public void testCommitWithoutUnderlying() { CopyOnAccessInMemoryStateInternals<String> internals = CopyOnAccessInMemoryStateInternals.withUnderlying(key, null); StateNamespace namespace = new StateNamespaceForTest("foo"); StateTag<BagState<String>> bagTag = StateTags.bag("foo", StringUtf8Coder.of()); BagState<String> stringBag = internals.state(namespace, bagTag); assertThat(stringBag.read(), emptyIterable()); stringBag.add("bar"); stringBag.add("baz"); assertThat(stringBag.read(), containsInAnyOrder("baz", "bar")); internals.commit(); BagState<String> reReadStringBag = internals.state(namespace, bagTag); assertThat(reReadStringBag.read(), containsInAnyOrder("baz", "bar")); assertThat(internals.isEmpty(), is(false)); }
Example 15
Source Project: beam Source File: CombineValuesFnFactoryTest.java License: Apache License 2.0 | 5 votes |
@Test public void testCombineValuesFnMerge() throws Exception { TestReceiver receiver = new TestReceiver(); MeanInts mean = new MeanInts(); Combine.CombineFn<Integer, CountSum, String> combiner = mean; ParDoFn combineParDoFn = createCombineValuesFn( CombinePhase.MERGE, combiner, StringUtf8Coder.of(), BigEndianIntegerCoder.of(), new CountSumCoder(), WindowingStrategy.globalDefault()); combineParDoFn.startBundle(receiver); combineParDoFn.processElement( WindowedValue.valueInGlobalWindow( KV.of( "a", Arrays.asList(new CountSum(3, 6), new CountSum(2, 9), new CountSum(1, 12))))); combineParDoFn.processElement( WindowedValue.valueInGlobalWindow( KV.of("b", Arrays.asList(new CountSum(2, 20), new CountSum(1, 1))))); combineParDoFn.finishBundle(); Object[] expectedReceivedElems = { WindowedValue.valueInGlobalWindow(KV.of("a", new CountSum(6, 27))), WindowedValue.valueInGlobalWindow(KV.of("b", new CountSum(3, 21))), }; assertArrayEquals(expectedReceivedElems, receiver.receivedElems.toArray()); }
Example 16
Source Project: beam Source File: Timer.java License: Apache License 2.0 | 5 votes |
@Override public void encode(Timer<K> timer, OutputStream outStream) throws IOException { keyCoder.encode(timer.getUserKey(), outStream); StringUtf8Coder.of().encode(timer.getDynamicTimerTag(), outStream); windowsCoder.encode(timer.getWindows(), outStream); BooleanCoder.of().encode(timer.getClearBit(), outStream); if (!timer.getClearBit()) { InstantCoder.of().encode(timer.getFireTimestamp(), outStream); InstantCoder.of().encode(timer.getHoldTimestamp(), outStream); PaneInfoCoder.INSTANCE.encode(timer.getPane(), outStream); } }
Example 17
Source Project: beam Source File: CreateTest.java License: Apache License 2.0 | 5 votes |
@Test @Category(NeedsRunner.class) public void testCreateTimestampedEmpty() { PCollection<String> output = p.apply( Create.timestamped(new ArrayList<TimestampedValue<String>>()) .withCoder(StringUtf8Coder.of())); PAssert.that(output).empty(); p.run(); }
Example 18
Source Project: beam Source File: OuterRightJoinTest.java License: Apache License 2.0 | 5 votes |
@Test(expected = NullPointerException.class) public void testJoinNullValueIsNull() { p.enableAbandonedNodeEnforcement(false); Join.rightOuterJoin( p.apply("CreateLeft", Create.empty(KvCoder.of(StringUtf8Coder.of(), VarLongCoder.of()))), p.apply( "CreateRight", Create.empty(KvCoder.of(StringUtf8Coder.of(), StringUtf8Coder.of()))), null); }
Example 19
Source Project: beam Source File: SnsResponseCoderTest.java License: Apache License 2.0 | 5 votes |
@Test public void verifyResponseWithNoStatusCodeAndText() throws IOException { SnsResponse<String> expected = SnsResponse.create("test-3", OptionalInt.empty(), Optional.empty()); SnsResponseCoder<String> coder = SnsResponseCoder.of(StringUtf8Coder.of()); ByteArrayOutputStream output = new ByteArrayOutputStream(); coder.encode(expected, output); ByteArrayInputStream in = new ByteArrayInputStream(output.toByteArray()); SnsResponse<String> actual = coder.decode(in); Assert.assertEquals(expected, actual); }
Example 20
Source Project: beam Source File: SparkPipelineStateTest.java License: Apache License 2.0 | 5 votes |
private void testFailedPipeline(final SparkPipelineOptions options) throws Exception { SparkPipelineResult result = null; try { final Pipeline pipeline = Pipeline.create(options); pipeline .apply(getValues(options)) .setCoder(StringUtf8Coder.of()) .apply( MapElements.via( new SimpleFunction<String, String>() { @Override public String apply(final String input) { throw new MyCustomException(FAILED_THE_BATCH_INTENTIONALLY); } })); result = (SparkPipelineResult) pipeline.run(); result.waitUntilFinish(); } catch (final Exception e) { assertThat(e, instanceOf(Pipeline.PipelineExecutionException.class)); assertThat(e.getCause(), instanceOf(MyCustomException.class)); assertThat(e.getCause().getMessage(), is(FAILED_THE_BATCH_INTENTIONALLY)); assertThat(result.getState(), is(PipelineResult.State.FAILED)); result.cancel(); return; } fail("An injected failure did not affect the pipeline as expected."); }
Example 21
Source Project: beam Source File: ValuesTest.java License: Apache License 2.0 | 5 votes |
@Test @Category(NeedsRunner.class) public void testValues() { PCollection<KV<String, Integer>> input = p.apply( Create.of(Arrays.asList(TABLE)) .withCoder(KvCoder.of(StringUtf8Coder.of(), BigEndianIntegerCoder.of()))); PCollection<Integer> output = input.apply(Values.create()); PAssert.that(output).containsInAnyOrder(1, 2, 3, 4, 4); p.run(); }
Example 22
Source Project: beam Source File: WindowingTest.java License: Apache License 2.0 | 5 votes |
@Test @Category(NeedsRunner.class) public void testEmptyInput() { PCollection<String> input = p.apply(Create.empty(StringUtf8Coder.of())); PCollection<String> output = input.apply(new WindowedCount(FixedWindows.of(new Duration(10)))); PAssert.that(output).empty(); p.run(); }
Example 23
Source Project: beam Source File: JacksonTransformsTest.java License: Apache License 2.0 | 5 votes |
@Test public void testWritingInvalidJsonsWithFailuresSimpleFunction() { WithFailures.Result<PCollection<String>, KV<MyPojo, String>> result = pipeline .apply( Create.of(Iterables.concat(POJOS, INVALID_POJOS)) .withCoder(SerializableCoder.of(MyPojo.class))) .apply( AsJsons.of(MyPojo.class) .exceptionsVia( new SimpleFunction< WithFailures.ExceptionElement<MyPojo>, KV<MyPojo, String>>() { @Override public KV<MyPojo, String> apply( WithFailures.ExceptionElement<MyPojo> failure) { return KV.of( failure.element(), failure.exception().getClass().getCanonicalName()); } })); result.output().setCoder(StringUtf8Coder.of()); PAssert.that(result.output()).containsInAnyOrder(VALID_JSONS); assertWritingWithErrorFunctionHandler(result); pipeline.run(); }
Example 24
Source Project: beam Source File: StreamingDataflowWorkerTest.java License: Apache License 2.0 | 5 votes |
private List<ParallelInstruction> makeUnboundedSourcePipeline( int numMessagesPerShard, // Total number of messages in each split of the unbounded source. DoFn<ValueWithRecordId<KV<Integer, Integer>>, String> doFn) throws Exception { DataflowPipelineOptions options = PipelineOptionsFactory.create().as(DataflowPipelineOptions.class); options.setNumWorkers(1); CloudObject codec = CloudObjects.asCloudObject( WindowedValue.getFullCoder( ValueWithRecordId.ValueWithRecordIdCoder.of( KvCoder.of(VarIntCoder.of(), VarIntCoder.of())), GlobalWindow.Coder.INSTANCE), /*sdkComponents=*/ null); return Arrays.asList( new ParallelInstruction() .setSystemName("Read") .setOriginalName("OriginalReadName") .setRead( new ReadInstruction() .setSource( CustomSources.serializeToCloudSource( new TestCountingSource(numMessagesPerShard), options) .setCodec(codec))) .setOutputs( Arrays.asList( new InstructionOutput() .setName("read_output") .setOriginalName(DEFAULT_OUTPUT_ORIGINAL_NAME) .setSystemName(DEFAULT_OUTPUT_SYSTEM_NAME) .setCodec(codec))), makeDoFnInstruction(doFn, 0, StringUtf8Coder.of(), WindowingStrategy.globalDefault()), makeSinkInstruction(StringUtf8Coder.of(), 1, GlobalWindow.Coder.INSTANCE)); }
Example 25
Source Project: beam Source File: DistinctTest.java License: Apache License 2.0 | 5 votes |
@Test @Category(NeedsRunner.class) public void testDistinct() { List<String> strings = Arrays.asList("k1", "k5", "k5", "k2", "k1", "k2", "k3"); PCollection<String> input = p.apply(Create.of(strings).withCoder(StringUtf8Coder.of())); PCollection<String> output = input.apply(Distinct.create()); PAssert.that(output).containsInAnyOrder("k1", "k5", "k2", "k3"); p.run(); }
Example 26
Source Project: DataflowTemplates Source File: KeyValueToGenericRecordFnTest.java License: Apache License 2.0 | 5 votes |
/** Test whether {@link KeyValueToGenericRecordFn} correctly maps the message. */ @Test @Category(NeedsRunner.class) public void testKeyValueToGenericRecordFn() throws Exception { // Create the test input. final String key = "Name"; final String value = "Generic"; final KV<String, String> message = KV.of(key, value); // Apply the ParDo. PCollection<GenericRecord> results = pipeline .apply( "CreateInput", Create.of(message) .withCoder(KvCoder.of(StringUtf8Coder.of(), StringUtf8Coder.of()))) .apply("GenericRecordCreation", ParDo.of(new KeyValueToGenericRecordFn())) .setCoder(AvroCoder.of(GenericRecord.class, KeyValueToGenericRecordFn.SCHEMA)); // Assert on the results. PAssert.that(results) .satisfies( collection -> { GenericRecord result = collection.iterator().next(); assertThat(result.get("message").toString(), is(equalTo(value))); assertThat(result.get("attributes").toString(), is(equalTo("{Name=Generic}"))); return null; }); // Run the pipeline. pipeline.run(); }
Example 27
Source Project: beam Source File: FlinkKeyUtilsTest.java License: Apache License 2.0 | 5 votes |
@Test public void testEncodeDecode() { String key = "key"; StringUtf8Coder coder = StringUtf8Coder.of(); ByteBuffer byteBuffer = FlinkKeyUtils.encodeKey(key, coder); assertThat(FlinkKeyUtils.decodeKey(byteBuffer, coder), is(key)); }
Example 28
Source Project: beam Source File: InnerJoinTest.java License: Apache License 2.0 | 5 votes |
@Test(expected = NullPointerException.class) public void testJoinRightCollectionNull() { p.enableAbandonedNodeEnforcement(false); Join.innerJoin( p.apply( Create.of(leftListOfKv).withCoder(KvCoder.of(StringUtf8Coder.of(), VarLongCoder.of()))), null); }
Example 29
Source Project: beam Source File: WindowingTest.java License: Apache License 2.0 | 5 votes |
@Override public PCollection<String> expand(PCollection<String> in) { return in.apply( "Window", Window.<String>into(windowFn).withTimestampCombiner(TimestampCombiner.EARLIEST)) .apply(Count.perElement()) .apply("FormatCounts", ParDo.of(new FormatCountsDoFn())) .setCoder(StringUtf8Coder.of()); }
Example 30
Source Project: DataflowTemplates Source File: CsvConvertersTest.java License: Apache License 2.0 | 5 votes |
/** * Tests {@link CsvConverters.LineToFailsafeJson} converts a line to a {@link FailsafeElement} * correctly using a JSON schema. */ @Test public void testLineToFailsafeJsonNoHeadersJsonSchema() { FailsafeElementCoder<String, String> coder = FAILSAFE_ELEMENT_CODER; CoderRegistry coderRegistry = pipeline.getCoderRegistry(); coderRegistry.registerCoderForType(coder.getEncodedTypeDescriptor(), coder); PCollection<String> lines = pipeline.apply(Create.of(RECORD_STRING).withCoder(StringUtf8Coder.of())); PCollectionTuple linesTuple = PCollectionTuple.of(CSV_LINES, lines); PCollectionTuple failsafe = linesTuple.apply( "TestLineToFailsafeJson", CsvConverters.LineToFailsafeJson.newBuilder() .setDelimiter(",") .setUdfFileSystemPath(null) .setUdfFunctionName(null) .setJsonSchemaPath(TEST_JSON_SCHEMA__PATH) .setHeaderTag(CSV_HEADERS) .setLineTag(CSV_LINES) .setUdfOutputTag(PROCESSING_OUT) .setUdfDeadletterTag(PROCESSING_DEADLETTER_OUT) .build()); PAssert.that(failsafe.get(PROCESSING_OUT)) .satisfies( collection -> { FailsafeElement<String, String> result = collection.iterator().next(); assertThat(result.getPayload(), is(equalTo(JSON_STRING_RECORD))); return null; }); pipeline.run(); }