Java Code Examples for org.apache.beam.sdk.values.Row#getRow()

The following examples show how to use org.apache.beam.sdk.values.Row#getRow() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: JavaBeanSchemaTest.java From beam with Apache License 2.0

6 votes

@Test
public void testRecursiveGetters() throws NoSuchSchemaException {
  SchemaRegistry registry = SchemaRegistry.createDefault();
  SchemaTestUtils.assertSchemaEquivalent(
      NESTED_BEAN_SCHEMA, registry.getSchema(NestedBean.class));

  NestedBean bean = new NestedBean(createSimple("string"));
  Row row = registry.getToRowFunction(NestedBean.class).apply(bean);

  Row nestedRow = row.getRow("nested");
  assertEquals("string", nestedRow.getString("str"));
  assertEquals((byte) 1, (Object) nestedRow.getByte("aByte"));
  assertEquals((short) 2, (Object) nestedRow.getInt16("aShort"));
  assertEquals((int) 3, (Object) nestedRow.getInt32("anInt"));
  assertEquals((long) 4, (Object) nestedRow.getInt64("aLong"));
  assertTrue(nestedRow.getBoolean("aBoolean"));
  assertEquals(DATE.toInstant(), nestedRow.getDateTime("dateTime"));
  assertEquals(DATE.toInstant(), nestedRow.getDateTime("instant"));
  assertArrayEquals("not equal", BYTE_ARRAY, nestedRow.getBytes("bytes"));
  assertArrayEquals("not equal", BYTE_ARRAY, nestedRow.getBytes("byteBuffer"));
  assertEquals(BigDecimal.ONE, nestedRow.getDecimal("bigDecimal"));
  assertEquals("stringbuilder", nestedRow.getString("stringBuilder"));
}

Example 2

Source File: JavaFieldSchemaTest.java From beam with Apache License 2.0

6 votes

@Test
public void testRecursiveGetters() throws NoSuchSchemaException {
  SchemaRegistry registry = SchemaRegistry.createDefault();
  SchemaTestUtils.assertSchemaEquivalent(
      NESTED_POJO_SCHEMA, registry.getSchema(NestedPOJO.class));

  NestedPOJO pojo = new NestedPOJO(createSimple("string"));
  Row row = registry.getToRowFunction(NestedPOJO.class).apply(pojo);

  Row nestedRow = row.getRow("nested");
  assertEquals("string", nestedRow.getString("str"));
  assertEquals((byte) 1, (Object) nestedRow.getByte("aByte"));
  assertEquals((short) 2, (Object) nestedRow.getInt16("aShort"));
  assertEquals((int) 3, (Object) nestedRow.getInt32("anInt"));
  assertEquals((long) 4, (Object) nestedRow.getInt64("aLong"));
  assertTrue(nestedRow.getBoolean("aBoolean"));
  assertEquals(DATE.toInstant(), nestedRow.getDateTime("dateTime"));
  assertEquals(INSTANT, nestedRow.getDateTime("instant").toInstant());
  assertArrayEquals("not equal", BYTE_ARRAY, nestedRow.getBytes("bytes"));
  assertArrayEquals("not equal", BYTE_BUFFER.array(), nestedRow.getBytes("byteBuffer"));
  assertEquals(BigDecimal.ONE, nestedRow.getDecimal("bigDecimal"));
  assertEquals("stringbuilder", nestedRow.getString("stringBuilder"));
}

Example 3

Source File: DoFnSchemaInformation.java From beam with Apache License 2.0

5 votes

@Override
public OutputT apply(InputT input) {
  Row row = toRowFunction.apply(input);
  Row selected = rowSelector.select(row);
  if (unbox) {
    selected = selected.getRow(0);
  }
  return fromRowFunction.apply(selected);
}

Example 4

Source File: SchemaTestUtils.java From beam with Apache License 2.0

5 votes

@Override
public boolean matches(Object item) {
  if (!(item instanceof Row)) {
    return false;
  }
  Row row = (Row) item;
  switch (fieldType.getTypeName()) {
    case ROW:
      if (!row.getSchema().getField(fieldIndex).getType().getTypeName().isCompositeType()) {
        return false;
      }
      Row actualRow = row.getRow(fieldIndex);
      return new RowEquivalent((Row) expected).matches(actualRow);
    case ARRAY:
      Row[] expectedArray = ((List<Row>) expected).toArray(new Row[0]);

      return containsInAnyOrder(expectedArray).matches(row.getArray(fieldIndex));
    case ITERABLE:
      Row[] expectedIterable = Iterables.toArray((Iterable<Row>) expected, Row.class);
      List<Row> actualIterable = Lists.newArrayList(row.getIterable(fieldIndex));
      return containsInAnyOrder(expectedIterable).matches(actualIterable);
    case MAP:
      throw new RuntimeException("Not yet implemented for maps");
    default:
      return equalTo(expected).matches(row.getValue(fieldIndex));
  }
}

Example 5

Source File: DebeziumSourceRecordToDataflowCdcFormatTranslatorTest.java From DataflowTemplates with Apache License 2.0

4 votes

@Test
public void testNoPkIsTranslated() {
  Schema keySchema = SchemaBuilder.struct().build();
  Struct key = null;

  Schema internalStructSchema = SchemaBuilder.struct()
      .field("astring", Schema.STRING_SCHEMA).build();

  Schema valueAfterSchema = SchemaBuilder.struct()
      .field("team", Schema.STRING_SCHEMA)
      .field("year_founded", Schema.INT32_SCHEMA)
      .field("some_timestamp", Schema.INT64_SCHEMA)
      .field("float_field", Schema.FLOAT32_SCHEMA)
      .field("double_field", Schema.FLOAT64_SCHEMA)
      .field("struct_field", internalStructSchema)
      .build();

  Schema valueSchema = SchemaBuilder.struct()
      .field("after", valueAfterSchema)
      .field("op", Schema.STRING_SCHEMA)
      .field("ts_ms", Schema.INT64_SCHEMA)
      .build();

  Struct value = new Struct(valueSchema)
      .put("op", "c")
      .put("ts_ms", 1569287580660L)
      .put("after", new Struct(valueAfterSchema)
          .put("team", "team_PXHU")
          .put("year_founded", 1916)
          .put("some_timestamp", 123456579L)
          .put("float_field", new Float(123.456))
          .put("double_field", 123456579.98654321)
          .put("struct_field",
              new Struct(internalStructSchema).put("astring", "mastring")));

  String topicName = "mainstance.cdcForDataflow.team_metadata";

  SourceRecord input = new SourceRecord(
      ImmutableMap.of("server", "mainstance"),
      ImmutableMap.of(
          "file", "mysql-bin.000023",
          "pos", 110489,
          "gtids", "36797132-a366-11e9-ac33-42010a800456:1-6407169",
          "row", 1, "snapshot", true),
      topicName,
      keySchema,
      key,
      valueSchema,
      value);

  DebeziumSourceRecordToDataflowCdcFormatTranslator translator =
      new DebeziumSourceRecordToDataflowCdcFormatTranslator();
  Row translatedRecord = translator.translate(input);
  Row fullRecord = translatedRecord.getRow(DataflowCdcRowFormat.FULL_RECORD);

  assertThat(
      translatedRecord.getSchema().hasField(DataflowCdcRowFormat.PRIMARY_KEY), is(false));
  assertThat(fullRecord.getString("team"),
      is(value.getStruct("after").getString("team")));
  assertThat(fullRecord.getInt32("year_founded"),
      is(value.getStruct("after").getInt32("year_founded")));
  assertThat(fullRecord.getInt64("some_timestamp"),
      is(value.getStruct("after").getInt64("some_timestamp")));
  assertThat(fullRecord.getFloat("float_field"),
      is(value.getStruct("after").getFloat32("float_field")));
  assertThat(fullRecord.getDouble("double_field"),
      is(value.getStruct("after").getFloat64("double_field")));
  assertThat(fullRecord.getRow("struct_field").getString("astring"),
      is(value.getStruct("after").getStruct("struct_field").getString("astring")));
}

Example 6

Source File: DebeziumSourceRecordToDataflowCdcFormatTranslatorTest.java From DataflowTemplates with Apache License 2.0

4 votes

@Test
public void testFullSourceRecordTranslation() {

  Schema keySchema = SchemaBuilder.struct()
      .field("team", Schema.STRING_SCHEMA).build();
  Struct key = new Struct(keySchema).put("team", "team_PXHU");

  Schema internalStructSchema = SchemaBuilder.struct()
      .field("astring", Schema.STRING_SCHEMA).build();

  Schema valueAfterSchema = SchemaBuilder.struct()
      .field("team", Schema.STRING_SCHEMA)
      .field("year_founded", Schema.INT32_SCHEMA)
      .field("some_timestamp", Schema.INT64_SCHEMA)
      .field("float_field", Schema.FLOAT32_SCHEMA)
      .field("double_field", Schema.FLOAT64_SCHEMA)
      .field("struct_field", internalStructSchema)
      .build();

  Schema valueSchema = SchemaBuilder.struct()
      .field("after", valueAfterSchema)
      .field("op", Schema.STRING_SCHEMA)
      .field("ts_ms", Schema.INT64_SCHEMA)
      .build();

  Struct value = new Struct(valueSchema)
      .put("op", "c")
      .put("ts_ms", 1569287580660L)
      .put("after", new Struct(valueAfterSchema)
          .put("team", "team_PXHU")
          .put("year_founded", 1916)
          .put("some_timestamp", 123456579L)
          .put("float_field", new Float(123.456))
          .put("double_field", 123456579.98654321)
          .put("struct_field",
              new Struct(internalStructSchema).put("astring", "mastring")));

  String topicName = "mainstance.cdcForDataflow.team_metadata";

  SourceRecord input = new SourceRecord(
      ImmutableMap.of("server", "mainstance"),
      ImmutableMap.of(
          "file", "mysql-bin.000023",
          "pos", 110489,
          "gtids", "36797132-a366-11e9-ac33-42010a800456:1-6407169",
          "row", 1, "snapshot", true),
      topicName,
      keySchema,
      key,
      valueSchema,
      value);

  DebeziumSourceRecordToDataflowCdcFormatTranslator translator =
      new DebeziumSourceRecordToDataflowCdcFormatTranslator();
  Row translatedRecord = translator.translate(input);

  Row fullRecord = translatedRecord.getRow("fullRecord");

  assertThat(fullRecord.getString("team"),
      is(value.getStruct("after").getString("team")));
  assertThat(fullRecord.getInt32("year_founded"),
      is(value.getStruct("after").getInt32("year_founded")));
  assertThat(fullRecord.getInt64("some_timestamp"),
      is(value.getStruct("after").getInt64("some_timestamp")));
  assertThat(fullRecord.getFloat("float_field"),
      is(value.getStruct("after").getFloat32("float_field")));
  assertThat(fullRecord.getDouble("double_field"),
      is(value.getStruct("after").getFloat64("double_field")));
  assertThat(fullRecord.getRow("struct_field").getString("astring"),
      is(value.getStruct("after").getStruct("struct_field").getString("astring")));
}

Example 7

Source File: BeamSetOperatorsTransforms.java From beam with Apache License 2.0

4 votes

@ProcessElement
public void processElement(@Element Row element, OutputReceiver<Row> o) {
  Row key = element.getRow("key");
  long numLeftRows = 0;
  long numRightRows = 0;
  if (!Iterables.isEmpty(element.<Row>getIterable(leftTag))) {
    numLeftRows = Iterables.size(element.<Row>getIterable(leftTag));
  }
  if (!Iterables.isEmpty(element.<Row>getIterable(rightTag))) {
    numRightRows = Iterables.size(element.<Row>getIterable(rightTag));
  }

  switch (opType) {
    case UNION:
      if (all) {
        for (int i = 0; i < numLeftRows + numRightRows; i++) {
          o.output(key);
        }
      } else {
        // only output the key
        o.output(key);
      }
      break;
    case INTERSECT:
      if (numLeftRows > 0 && numRightRows > 0) {
        if (all) {
          // Say for Row R, there are m instances on left and n instances on right,
          // INTERSECT ALL outputs MIN(m, n) instances of R.
          for (int i = 0; i < Math.min(numLeftRows, numRightRows); i++) {
            o.output(key);
          }
        } else {
          o.output(key);
        }
      }
      break;
    case MINUS:
      // Say for Row R, there are m instances on left and n instances on right:
      // - EXCEPT ALL outputs MAX(m - n, 0) instances of R.
      // - EXCEPT [DISTINCT] outputs a single instance of R if m > 0 and n == 0, else
      //   they output 0 instances.
      if (numLeftRows > 0 && numRightRows == 0) {
        if (all) {
          // output all
          for (int i = 0; i < numLeftRows; i++) {
            o.output(key);
          }
        } else {
          // only output one
          o.output(key);
        }
      } else if (numLeftRows > 0 && numRightRows > 0) {
        long outputCount = numLeftRows - numRightRows;
        if (outputCount > 0) {
          if (all) {
            while (outputCount > 0) {
              outputCount--;
              o.output(key);
            }
          }
          // Dont output any in DISTINCT (if (!all)) case
        }
      }
  }
}