Java Code Examples for org.apache.beam.sdk.coders.StringUtf8Coder

The following examples show how to use org.apache.beam.sdk.coders.StringUtf8Coder. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: deployment-examples   Source File: StatefulTeamScoreTest.java    License: MIT License 6 votes vote down vote up
/**
 * Tests that {@link UpdateTeamScoreFn} {@link org.apache.beam.sdk.transforms.DoFn} outputs
 * correctly for one team.
 */
@Test
public void testScoreUpdatesOneTeam() {

  TestStream<KV<String, GameActionInfo>> createEvents =
      TestStream.create(KvCoder.of(StringUtf8Coder.of(), AvroCoder.of(GameActionInfo.class)))
          .advanceWatermarkTo(baseTime)
          .addElements(
              event(TestUser.RED_TWO, 99, Duration.standardSeconds(10)),
              event(TestUser.RED_ONE, 1, Duration.standardSeconds(20)),
              event(TestUser.RED_ONE, 0, Duration.standardSeconds(30)),
              event(TestUser.RED_TWO, 100, Duration.standardSeconds(40)),
              event(TestUser.RED_TWO, 201, Duration.standardSeconds(50)))
          .advanceWatermarkToInfinity();

  PCollection<KV<String, Integer>> teamScores =
      p.apply(createEvents).apply(ParDo.of(new UpdateTeamScoreFn(100)));

  String redTeam = TestUser.RED_ONE.getTeam();

  PAssert.that(teamScores)
      .inWindow(GlobalWindow.INSTANCE)
      .containsInAnyOrder(KV.of(redTeam, 100), KV.of(redTeam, 200), KV.of(redTeam, 401));

  p.run().waitUntilFinish();
}
 
Example 2
Source Project: beam   Source File: ToStringTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
@Category(NeedsRunner.class)
public void testToStringIterableWithDelimiter() {
  ArrayList<Iterable<String>> iterables = new ArrayList<>();
  iterables.add(Arrays.asList(new String[] {"one", "two", "three"}));
  iterables.add(Arrays.asList(new String[] {"four", "five", "six"}));

  ArrayList<String> expected = new ArrayList<>();
  expected.add("one\ttwo\tthree");
  expected.add("four\tfive\tsix");

  PCollection<Iterable<String>> input =
      p.apply(Create.of(iterables).withCoder(IterableCoder.of(StringUtf8Coder.of())));
  PCollection<String> output = input.apply(ToString.iterables("\t"));
  PAssert.that(output).containsInAnyOrder(expected);
  p.run();
}
 
Example 3
Source Project: DataflowTemplates   Source File: WriteToGCSAvroTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Test whether {@link WriteToGCSAvro} throws an exception if temporary directory is not provided.
 */
@Test
public void testWriteWithoutTempLocation() {
  expectedException.expect(IllegalArgumentException.class);
  expectedException.expectMessage("withTempLocation(tempLocation) called with null input. ");

  pipeline
      .apply(
          "CreateInput",
          Create.of(message).withCoder(KvCoder.of(StringUtf8Coder.of(), StringUtf8Coder.of())))
      .apply(
          "WriteTextFile(s)",
          WriteToGCSAvro.newBuilder()
              .withOutputDirectory(FAKE_DIR)
              .withOutputFilenamePrefix(AVRO_FILENAME_PREFIX)
              .setNumShards(NUM_SHARDS)
              .withTempLocation(null)
              .build());
  pipeline.run();
}
 
Example 4
Source Project: beam   Source File: ParDoTranslationTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testProcessContext() throws Exception {
  Pipeline p = Pipeline.create();
  SdkComponents sdkComponents = SdkComponents.create();
  sdkComponents.registerEnvironment(Environments.createDockerEnvironment("java"));
  ParDoPayload payload =
      ParDoTranslation.translateParDo(
          ParDo.of(new ProcessContextDoFn())
              .withOutputTags(new TupleTag<>(), TupleTagList.empty()),
          PCollection.createPrimitiveOutputInternal(
              p, WindowingStrategy.globalDefault(), IsBounded.BOUNDED, StringUtf8Coder.of()),
          DoFnSchemaInformation.create(),
          TestPipeline.create(),
          sdkComponents);

  assertTrue(payload.getRequestsFinalization());
}
 
Example 5
Source Project: beam   Source File: StreamingDataflowWorkerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testHugeCommits() throws Exception {
  List<ParallelInstruction> instructions =
      Arrays.asList(
          makeSourceInstruction(StringUtf8Coder.of()),
          makeDoFnInstruction(new FanoutFn(), 0, StringUtf8Coder.of()),
          makeSinkInstruction(StringUtf8Coder.of(), 0));

  FakeWindmillServer server = new FakeWindmillServer(errorCollector);
  StreamingDataflowWorkerOptions options = createTestingPipelineOptions(server);
  StreamingDataflowWorker worker = makeWorker(instructions, options, true /* publishCounters */);
  worker.start();

  server.addWorkToOffer(makeInput(0, TimeUnit.MILLISECONDS.toMicros(0)));

  server.waitForAndGetCommits(0);
  worker.stop();
}
 
Example 6
Source Project: beam   Source File: ProvidedSparkContextTest.java    License: Apache License 2.0 6 votes vote down vote up
private void testWithValidProvidedContext(JavaSparkContext jsc) throws Exception {
  SparkContextOptions options = getSparkContextOptions(jsc);

  Pipeline p = Pipeline.create(options);
  PCollection<String> inputWords = p.apply(Create.of(WORDS).withCoder(StringUtf8Coder.of()));
  PCollection<String> output =
      inputWords
          .apply(new WordCount.CountWords())
          .apply(MapElements.via(new WordCount.FormatAsTextFn()));

  PAssert.that(output).containsInAnyOrder(EXPECTED_COUNT_SET);

  // Run test from pipeline
  PipelineResult result = p.run();

  TestPipeline.verifyPAssertsSucceeded(p, result);
}
 
Example 7
Source Project: beam   Source File: EvaluationContextTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void getExecutionContextDifferentKeysIndependentState() {
  DirectExecutionContext fooContext =
      context.getExecutionContext(createdProducer, StructuralKey.of("foo", StringUtf8Coder.of()));

  StateTag<BagState<Integer>> intBag = StateTags.bag("myBag", VarIntCoder.of());

  fooContext.getStepContext("s1").stateInternals().state(StateNamespaces.global(), intBag).add(1);

  DirectExecutionContext barContext =
      context.getExecutionContext(createdProducer, StructuralKey.of("bar", StringUtf8Coder.of()));
  assertThat(barContext, not(equalTo(fooContext)));
  assertThat(
      barContext
          .getStepContext("s1")
          .stateInternals()
          .state(StateNamespaces.global(), intBag)
          .read(),
      emptyIterable());
}
 
Example 8
Source Project: beam   Source File: WriteFiles.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public WriteFilesResult<DestinationT> expand(
    PCollection<List<FileResult<DestinationT>>> input) {

  List<PCollectionView<?>> finalizeSideInputs = Lists.newArrayList(getSideInputs());
  if (numShardsView != null) {
    finalizeSideInputs.add(numShardsView);
  }
  PCollection<KV<DestinationT, String>> outputFilenames =
      input
          .apply("Finalize", ParDo.of(new FinalizeFn()).withSideInputs(finalizeSideInputs))
          .setCoder(KvCoder.of(destinationCoder, StringUtf8Coder.of()))
          // Reshuffle the filenames to make sure they are observable downstream
          // only after each one is done finalizing.
          .apply(Reshuffle.viaRandomKey());

  TupleTag<KV<DestinationT, String>> perDestinationOutputFilenamesTag =
      new TupleTag<>("perDestinationOutputFilenames");
  return WriteFilesResult.in(
      input.getPipeline(), perDestinationOutputFilenamesTag, outputFilenames);
}
 
Example 9
Source Project: beam   Source File: WatchTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCoder() throws Exception {
  GrowthState pollingState =
      PollingGrowthState.of(
          ImmutableMap.of(
              HashCode.fromString("0123456789abcdef0123456789abcdef"), Instant.now(),
              HashCode.fromString("01230123012301230123012301230123"), Instant.now()),
          Instant.now(),
          "STATE");
  GrowthState nonPollingState =
      NonPollingGrowthState.of(
          Growth.PollResult.incomplete(Instant.now(), Arrays.asList("A", "B")));
  Coder<GrowthState> coder =
      Watch.GrowthStateCoder.of(StringUtf8Coder.of(), StringUtf8Coder.of());

  CoderProperties.coderDecodeEncodeEqual(coder, pollingState);
  CoderProperties.coderDecodeEncodeEqual(coder, nonPollingState);
}
 
Example 10
Source Project: beam   Source File: ReaderFactoryTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCreateUnknownReader() throws Exception {
  CloudObject spec = CloudObject.forClassName("UnknownSource");
  Source cloudSource = new Source();
  cloudSource.setSpec(spec);
  cloudSource.setCodec(CloudObjects.asCloudObject(StringUtf8Coder.of(), /*sdkComponents=*/ null));
  try {
    PipelineOptions options = PipelineOptionsFactory.create();
    ReaderRegistry.defaultRegistry()
        .create(
            cloudSource,
            options,
            BatchModeExecutionContext.forTesting(options, "testStage"),
            null);
    Assert.fail("should have thrown an exception");
  } catch (Exception exn) {
    Assert.assertThat(exn.toString(), CoreMatchers.containsString("Unable to create a Reader"));
  }
}
 
Example 11
Source Project: beam   Source File: CombineTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
@Category({ValidatesRunner.class, DataflowPortabilityApiUnsupported.class})
public void testFixedWindowsCombine() {
  PCollection<KV<String, Integer>> input =
      pipeline
          .apply(
              Create.timestamped(
                      TimestampedValue.of(KV.of("a", 1), new Instant(0L)),
                      TimestampedValue.of(KV.of("a", 1), new Instant(1L)),
                      TimestampedValue.of(KV.of("a", 4), new Instant(6L)),
                      TimestampedValue.of(KV.of("b", 1), new Instant(7L)),
                      TimestampedValue.of(KV.of("b", 13), new Instant(8L)))
                  .withCoder(KvCoder.of(StringUtf8Coder.of(), BigEndianIntegerCoder.of())))
          .apply(Window.into(FixedWindows.of(Duration.millis(2))));

  PCollection<Integer> sum =
      input.apply(Values.create()).apply(Combine.globally(new SumInts()).withoutDefaults());

  PCollection<KV<String, String>> sumPerKey = input.apply(Combine.perKey(new TestCombineFn()));

  PAssert.that(sum).containsInAnyOrder(2, 5, 13);
  PAssert.that(sumPerKey)
      .containsInAnyOrder(
          Arrays.asList(KV.of("a", "11"), KV.of("a", "4"), KV.of("b", "1"), KV.of("b", "13")));
  pipeline.run();
}
 
Example 12
private Object applyReadSourceTransform(
    PTransform<?, ?> transform, PCollection.IsBounded isBounded, StreamExecutionEnvironment env) {

  FlinkStreamingPipelineTranslator.StreamTransformTranslator<PTransform<?, ?>> translator =
      getReadSourceTranslator();
  FlinkStreamingTranslationContext ctx =
      new FlinkStreamingTranslationContext(env, PipelineOptionsFactory.create());

  Pipeline pipeline = Pipeline.create();
  PCollection<String> pc =
      PCollection.createPrimitiveOutputInternal(
          pipeline, WindowingStrategy.globalDefault(), isBounded, StringUtf8Coder.of());
  pc.setName("output");

  Map<TupleTag<?>, PValue> outputs = new HashMap<>();
  outputs.put(new TupleTag<>(), pc);
  AppliedPTransform<?, ?, ?> appliedTransform =
      AppliedPTransform.of(
          "test-transform", Collections.emptyMap(), outputs, transform, Pipeline.create());

  ctx.setCurrentTransform(appliedTransform);
  translator.translateNode(transform, ctx);

  return ctx.getInputDataStream(pc).getTransformation();
}
 
Example 13
Source Project: beam   Source File: ParDoTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
@Category({
  ValidatesRunner.class,
  UsesTimersInParDo.class,
  UsesTestStream.class,
  UsesStatefulParDo.class,
  UsesStrictTimerOrdering.class
})
public void testEventTimeTimerOrdering() throws Exception {
  final int numTestElements = 100;
  final Instant now = new Instant(1500000000000L);
  TestStream.Builder<KV<String, String>> builder =
      TestStream.create(KvCoder.of(StringUtf8Coder.of(), StringUtf8Coder.of()))
          .advanceWatermarkTo(new Instant(0));

  for (int i = 0; i < numTestElements; i++) {
    builder =
        builder.addElements(TimestampedValue.of(KV.of("dummy", "" + i), now.plus(i * 1000)));
    if ((i + 1) % 10 == 0) {
      builder = builder.advanceWatermarkTo(now.plus((i + 1) * 1000));
    }
  }

  testEventTimeTimerOrderingWithInputPTransform(
      now, numTestElements, builder.advanceWatermarkToInfinity());
}
 
Example 14
@Test
public void testCommitWithoutUnderlying() {
  CopyOnAccessInMemoryStateInternals<String> internals =
      CopyOnAccessInMemoryStateInternals.withUnderlying(key, null);
  StateNamespace namespace = new StateNamespaceForTest("foo");
  StateTag<BagState<String>> bagTag = StateTags.bag("foo", StringUtf8Coder.of());
  BagState<String> stringBag = internals.state(namespace, bagTag);
  assertThat(stringBag.read(), emptyIterable());

  stringBag.add("bar");
  stringBag.add("baz");
  assertThat(stringBag.read(), containsInAnyOrder("baz", "bar"));

  internals.commit();

  BagState<String> reReadStringBag = internals.state(namespace, bagTag);
  assertThat(reReadStringBag.read(), containsInAnyOrder("baz", "bar"));
  assertThat(internals.isEmpty(), is(false));
}
 
Example 15
Source Project: beam   Source File: CombineValuesFnFactoryTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCombineValuesFnMerge() throws Exception {
  TestReceiver receiver = new TestReceiver();
  MeanInts mean = new MeanInts();

  Combine.CombineFn<Integer, CountSum, String> combiner = mean;

  ParDoFn combineParDoFn =
      createCombineValuesFn(
          CombinePhase.MERGE,
          combiner,
          StringUtf8Coder.of(),
          BigEndianIntegerCoder.of(),
          new CountSumCoder(),
          WindowingStrategy.globalDefault());

  combineParDoFn.startBundle(receiver);
  combineParDoFn.processElement(
      WindowedValue.valueInGlobalWindow(
          KV.of(
              "a", Arrays.asList(new CountSum(3, 6), new CountSum(2, 9), new CountSum(1, 12)))));
  combineParDoFn.processElement(
      WindowedValue.valueInGlobalWindow(
          KV.of("b", Arrays.asList(new CountSum(2, 20), new CountSum(1, 1)))));
  combineParDoFn.finishBundle();

  Object[] expectedReceivedElems = {
    WindowedValue.valueInGlobalWindow(KV.of("a", new CountSum(6, 27))),
    WindowedValue.valueInGlobalWindow(KV.of("b", new CountSum(3, 21))),
  };
  assertArrayEquals(expectedReceivedElems, receiver.receivedElems.toArray());
}
 
Example 16
Source Project: beam   Source File: Timer.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void encode(Timer<K> timer, OutputStream outStream) throws IOException {
  keyCoder.encode(timer.getUserKey(), outStream);
  StringUtf8Coder.of().encode(timer.getDynamicTimerTag(), outStream);
  windowsCoder.encode(timer.getWindows(), outStream);
  BooleanCoder.of().encode(timer.getClearBit(), outStream);
  if (!timer.getClearBit()) {
    InstantCoder.of().encode(timer.getFireTimestamp(), outStream);
    InstantCoder.of().encode(timer.getHoldTimestamp(), outStream);
    PaneInfoCoder.INSTANCE.encode(timer.getPane(), outStream);
  }
}
 
Example 17
Source Project: beam   Source File: CreateTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
@Category(NeedsRunner.class)
public void testCreateTimestampedEmpty() {
  PCollection<String> output =
      p.apply(
          Create.timestamped(new ArrayList<TimestampedValue<String>>())
              .withCoder(StringUtf8Coder.of()));

  PAssert.that(output).empty();
  p.run();
}
 
Example 18
Source Project: beam   Source File: OuterRightJoinTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test(expected = NullPointerException.class)
public void testJoinNullValueIsNull() {
  p.enableAbandonedNodeEnforcement(false);
  Join.rightOuterJoin(
      p.apply("CreateLeft", Create.empty(KvCoder.of(StringUtf8Coder.of(), VarLongCoder.of()))),
      p.apply(
          "CreateRight", Create.empty(KvCoder.of(StringUtf8Coder.of(), StringUtf8Coder.of()))),
      null);
}
 
Example 19
Source Project: beam   Source File: SnsResponseCoderTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void verifyResponseWithNoStatusCodeAndText() throws IOException {

  SnsResponse<String> expected =
      SnsResponse.create("test-3", OptionalInt.empty(), Optional.empty());

  SnsResponseCoder<String> coder = SnsResponseCoder.of(StringUtf8Coder.of());
  ByteArrayOutputStream output = new ByteArrayOutputStream();
  coder.encode(expected, output);

  ByteArrayInputStream in = new ByteArrayInputStream(output.toByteArray());
  SnsResponse<String> actual = coder.decode(in);

  Assert.assertEquals(expected, actual);
}
 
Example 20
Source Project: beam   Source File: SparkPipelineStateTest.java    License: Apache License 2.0 5 votes vote down vote up
private void testFailedPipeline(final SparkPipelineOptions options) throws Exception {

    SparkPipelineResult result = null;

    try {
      final Pipeline pipeline = Pipeline.create(options);
      pipeline
          .apply(getValues(options))
          .setCoder(StringUtf8Coder.of())
          .apply(
              MapElements.via(
                  new SimpleFunction<String, String>() {

                    @Override
                    public String apply(final String input) {
                      throw new MyCustomException(FAILED_THE_BATCH_INTENTIONALLY);
                    }
                  }));

      result = (SparkPipelineResult) pipeline.run();
      result.waitUntilFinish();
    } catch (final Exception e) {
      assertThat(e, instanceOf(Pipeline.PipelineExecutionException.class));
      assertThat(e.getCause(), instanceOf(MyCustomException.class));
      assertThat(e.getCause().getMessage(), is(FAILED_THE_BATCH_INTENTIONALLY));
      assertThat(result.getState(), is(PipelineResult.State.FAILED));
      result.cancel();
      return;
    }

    fail("An injected failure did not affect the pipeline as expected.");
  }
 
Example 21
Source Project: beam   Source File: ValuesTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
@Category(NeedsRunner.class)
public void testValues() {

  PCollection<KV<String, Integer>> input =
      p.apply(
          Create.of(Arrays.asList(TABLE))
              .withCoder(KvCoder.of(StringUtf8Coder.of(), BigEndianIntegerCoder.of())));

  PCollection<Integer> output = input.apply(Values.create());

  PAssert.that(output).containsInAnyOrder(1, 2, 3, 4, 4);

  p.run();
}
 
Example 22
Source Project: beam   Source File: WindowingTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
@Category(NeedsRunner.class)
public void testEmptyInput() {
  PCollection<String> input = p.apply(Create.empty(StringUtf8Coder.of()));

  PCollection<String> output = input.apply(new WindowedCount(FixedWindows.of(new Duration(10))));

  PAssert.that(output).empty();

  p.run();
}
 
Example 23
Source Project: beam   Source File: JacksonTransformsTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testWritingInvalidJsonsWithFailuresSimpleFunction() {
  WithFailures.Result<PCollection<String>, KV<MyPojo, String>> result =
      pipeline
          .apply(
              Create.of(Iterables.concat(POJOS, INVALID_POJOS))
                  .withCoder(SerializableCoder.of(MyPojo.class)))
          .apply(
              AsJsons.of(MyPojo.class)
                  .exceptionsVia(
                      new SimpleFunction<
                          WithFailures.ExceptionElement<MyPojo>, KV<MyPojo, String>>() {
                        @Override
                        public KV<MyPojo, String> apply(
                            WithFailures.ExceptionElement<MyPojo> failure) {
                          return KV.of(
                              failure.element(),
                              failure.exception().getClass().getCanonicalName());
                        }
                      }));
  result.output().setCoder(StringUtf8Coder.of());

  PAssert.that(result.output()).containsInAnyOrder(VALID_JSONS);
  assertWritingWithErrorFunctionHandler(result);

  pipeline.run();
}
 
Example 24
Source Project: beam   Source File: StreamingDataflowWorkerTest.java    License: Apache License 2.0 5 votes vote down vote up
private List<ParallelInstruction> makeUnboundedSourcePipeline(
    int numMessagesPerShard, // Total number of messages in each split of the unbounded source.
    DoFn<ValueWithRecordId<KV<Integer, Integer>>, String> doFn)
    throws Exception {
  DataflowPipelineOptions options =
      PipelineOptionsFactory.create().as(DataflowPipelineOptions.class);
  options.setNumWorkers(1);
  CloudObject codec =
      CloudObjects.asCloudObject(
          WindowedValue.getFullCoder(
              ValueWithRecordId.ValueWithRecordIdCoder.of(
                  KvCoder.of(VarIntCoder.of(), VarIntCoder.of())),
              GlobalWindow.Coder.INSTANCE),
          /*sdkComponents=*/ null);

  return Arrays.asList(
      new ParallelInstruction()
          .setSystemName("Read")
          .setOriginalName("OriginalReadName")
          .setRead(
              new ReadInstruction()
                  .setSource(
                      CustomSources.serializeToCloudSource(
                              new TestCountingSource(numMessagesPerShard), options)
                          .setCodec(codec)))
          .setOutputs(
              Arrays.asList(
                  new InstructionOutput()
                      .setName("read_output")
                      .setOriginalName(DEFAULT_OUTPUT_ORIGINAL_NAME)
                      .setSystemName(DEFAULT_OUTPUT_SYSTEM_NAME)
                      .setCodec(codec))),
      makeDoFnInstruction(doFn, 0, StringUtf8Coder.of(), WindowingStrategy.globalDefault()),
      makeSinkInstruction(StringUtf8Coder.of(), 1, GlobalWindow.Coder.INSTANCE));
}
 
Example 25
Source Project: beam   Source File: DistinctTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
@Category(NeedsRunner.class)
public void testDistinct() {
  List<String> strings = Arrays.asList("k1", "k5", "k5", "k2", "k1", "k2", "k3");

  PCollection<String> input = p.apply(Create.of(strings).withCoder(StringUtf8Coder.of()));

  PCollection<String> output = input.apply(Distinct.create());

  PAssert.that(output).containsInAnyOrder("k1", "k5", "k2", "k3");
  p.run();
}
 
Example 26
/** Test whether {@link KeyValueToGenericRecordFn} correctly maps the message. */
@Test
@Category(NeedsRunner.class)
public void testKeyValueToGenericRecordFn() throws Exception {

  // Create the test input.
  final String key = "Name";
  final String value = "Generic";
  final KV<String, String> message = KV.of(key, value);

  // Apply the ParDo.
  PCollection<GenericRecord> results =
      pipeline
          .apply(
              "CreateInput",
              Create.of(message)
                  .withCoder(KvCoder.of(StringUtf8Coder.of(), StringUtf8Coder.of())))
          .apply("GenericRecordCreation", ParDo.of(new KeyValueToGenericRecordFn()))
          .setCoder(AvroCoder.of(GenericRecord.class, KeyValueToGenericRecordFn.SCHEMA));

  // Assert on the results.
  PAssert.that(results)
      .satisfies(
          collection -> {
            GenericRecord result = collection.iterator().next();
            assertThat(result.get("message").toString(), is(equalTo(value)));
            assertThat(result.get("attributes").toString(), is(equalTo("{Name=Generic}")));
            return null;
          });
  // Run the pipeline.
  pipeline.run();
}
 
Example 27
Source Project: beam   Source File: FlinkKeyUtilsTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testEncodeDecode() {
  String key = "key";
  StringUtf8Coder coder = StringUtf8Coder.of();

  ByteBuffer byteBuffer = FlinkKeyUtils.encodeKey(key, coder);
  assertThat(FlinkKeyUtils.decodeKey(byteBuffer, coder), is(key));
}
 
Example 28
Source Project: beam   Source File: InnerJoinTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test(expected = NullPointerException.class)
public void testJoinRightCollectionNull() {
  p.enableAbandonedNodeEnforcement(false);
  Join.innerJoin(
      p.apply(
          Create.of(leftListOfKv).withCoder(KvCoder.of(StringUtf8Coder.of(), VarLongCoder.of()))),
      null);
}
 
Example 29
Source Project: beam   Source File: WindowingTest.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public PCollection<String> expand(PCollection<String> in) {
  return in.apply(
          "Window",
          Window.<String>into(windowFn).withTimestampCombiner(TimestampCombiner.EARLIEST))
      .apply(Count.perElement())
      .apply("FormatCounts", ParDo.of(new FormatCountsDoFn()))
      .setCoder(StringUtf8Coder.of());
}
 
Example 30
Source Project: DataflowTemplates   Source File: CsvConvertersTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests {@link CsvConverters.LineToFailsafeJson} converts a line to a {@link FailsafeElement}
 * correctly using a JSON schema.
 */
@Test
public void testLineToFailsafeJsonNoHeadersJsonSchema() {

  FailsafeElementCoder<String, String> coder = FAILSAFE_ELEMENT_CODER;

  CoderRegistry coderRegistry = pipeline.getCoderRegistry();
  coderRegistry.registerCoderForType(coder.getEncodedTypeDescriptor(), coder);

  PCollection<String> lines =
      pipeline.apply(Create.of(RECORD_STRING).withCoder(StringUtf8Coder.of()));

  PCollectionTuple linesTuple = PCollectionTuple.of(CSV_LINES, lines);

  PCollectionTuple failsafe =
      linesTuple.apply(
          "TestLineToFailsafeJson",
          CsvConverters.LineToFailsafeJson.newBuilder()
              .setDelimiter(",")
              .setUdfFileSystemPath(null)
              .setUdfFunctionName(null)
              .setJsonSchemaPath(TEST_JSON_SCHEMA__PATH)
              .setHeaderTag(CSV_HEADERS)
              .setLineTag(CSV_LINES)
              .setUdfOutputTag(PROCESSING_OUT)
              .setUdfDeadletterTag(PROCESSING_DEADLETTER_OUT)
              .build());

  PAssert.that(failsafe.get(PROCESSING_OUT))
      .satisfies(
          collection -> {
            FailsafeElement<String, String> result = collection.iterator().next();
            assertThat(result.getPayload(), is(equalTo(JSON_STRING_RECORD)));
            return null;
          });

  pipeline.run();
}