org.apache.kafka.connect.data.Schema.Type Java Examples

void assertFoo(int count, Struct struct) {
  assertEquals(SCHEMA_NAME, struct.schema().name(), "struct.schema().name() does not match.");
  assertNotNull(struct, "struct should not be null");

  assertEquals(count, struct.schema().fields().size(), "struct.schema().fields().size() does not match.");
  for (int i = 1; i <= count; i++) {
    final String fieldName = String.format("f%s", i);
    Field field = struct.schema().field(fieldName);
    assertNotNull(field, "schema should have field " + fieldName);
    assertEquals(Type.INT32, field.schema().type(), "schema().type() for " + fieldName + " does not match.");

    final Integer expectedValue = i;
    final Integer actualValue = struct.getInt32(fieldName);
    assertEquals(
        expectedValue,
        actualValue,
        String.format("value for field %s does not match", fieldName)
    );
  }

}
 

@Test
public void struct10() {
  assertFoo(10,
      struct(SCHEMA_NAME,
          "f1", Type.INT32, false, 1,
          "f2", Type.INT32, false, 2,
          "f3", Type.INT32, false, 3,
          "f4", Type.INT32, false, 4,
          "f5", Type.INT32, false, 5,
          "f6", Type.INT32, false, 6,
          "f7", Type.INT32, false, 7,
          "f8", Type.INT32, false, 8,
          "f9", Type.INT32, false, 9,
          "f10", Type.INT32, false, 10
      )
  );
}
 

@Test
public void struct9() {
  assertFoo(9,
      struct(SCHEMA_NAME,
          "f1", Type.INT32, false, 1,
          "f2", Type.INT32, false, 2,
          "f3", Type.INT32, false, 3,
          "f4", Type.INT32, false, 4,
          "f5", Type.INT32, false, 5,
          "f6", Type.INT32, false, 6,
          "f7", Type.INT32, false, 7,
          "f8", Type.INT32, false, 8,
          "f9", Type.INT32, false, 9
      )
  );
}
 

@Test
public void struct4() {
  assertFoo(4,
      struct(SCHEMA_NAME,
          "f1", Type.INT32, false, 1,
          "f2", Type.INT32, false, 2,
          "f3", Type.INT32, false, 3,
          "f4", Type.INT32, false, 4
      )
  );
}
 

@Override
public String encodePartition(SinkRecord sinkRecord) {
  Object value = sinkRecord.value();
  Schema valueSchema = sinkRecord.valueSchema();
  if (value instanceof Struct) {
    Struct struct = (Struct) value;
    Object partitionKey = struct.get(fieldName);
    Type type = valueSchema.field(fieldName).schema().type();
    switch (type) {
      case INT8:
      case INT16:
      case INT32:
      case INT64:
        Number record = (Number) partitionKey;
        return fieldName + "=" + record.toString();
      case STRING:
        return fieldName + "=" + (String) partitionKey;
      case BOOLEAN:
        boolean booleanRecord = (boolean) partitionKey;
        return fieldName + "=" + Boolean.toString(booleanRecord);
      default:
        log.error("Type {} is not supported as a partition key.", type.getName());
        throw new PartitionException("Error encoding partition.");
    }
  } else {
    log.error("Value is not Struct type.");
    throw new PartitionException("Error encoding partition.");
  }
}
 

public static List<FieldSchema> convertSchema(Schema schema) {
  List<FieldSchema> columns = new ArrayList<>();
  if (Schema.Type.STRUCT.equals(schema.type())) {
    for (Field field: schema.fields()) {
      columns.add(new FieldSchema(
          field.name(), convert(field.schema()).getTypeName(), field.schema().doc()));
    }
  }
  return columns;
}
 

@Test
public void struct8() {
  assertFoo(8,
      struct(SCHEMA_NAME,
          "f1", Type.INT32, false, 1,
          "f2", Type.INT32, false, 2,
          "f3", Type.INT32, false, 3,
          "f4", Type.INT32, false, 4,
          "f5", Type.INT32, false, 5,
          "f6", Type.INT32, false, 6,
          "f7", Type.INT32, false, 7,
          "f8", Type.INT32, false, 8
      )
  );
}
 

@Test
public void struct7() {
  assertFoo(7,
      struct(SCHEMA_NAME,
          "f1", Type.INT32, false, 1,
          "f2", Type.INT32, false, 2,
          "f3", Type.INT32, false, 3,
          "f4", Type.INT32, false, 4,
          "f5", Type.INT32, false, 5,
          "f6", Type.INT32, false, 6,
          "f7", Type.INT32, false, 7
      )
  );
}
 

@Test
public void struct6() {
  assertFoo(6,
      struct(SCHEMA_NAME,
          "f1", Type.INT32, false, 1,
          "f2", Type.INT32, false, 2,
          "f3", Type.INT32, false, 3,
          "f4", Type.INT32, false, 4,
          "f5", Type.INT32, false, 5,
          "f6", Type.INT32, false, 6
      )
  );
}
 

@Test
public void struct5() {
  assertFoo(5,
      struct(SCHEMA_NAME,
          "f1", Type.INT32, false, 1,
          "f2", Type.INT32, false, 2,
          "f3", Type.INT32, false, 3,
          "f4", Type.INT32, false, 4,
          "f5", Type.INT32, false, 5
      )
  );
}
 

@Test
public void struct3() {
  assertFoo(3,
      struct(SCHEMA_NAME,
          "f1", Type.INT32, false, 1,
          "f2", Type.INT32, false, 2,
          "f3", Type.INT32, false, 3
      )
  );
}
 

@Test
public void struct2() {
  assertFoo(2,
      struct(SCHEMA_NAME,
          "f1", Type.INT32, false, 1,
          "f2", Type.INT32, false, 2
      )
  );
}
 

@Test
public void innerStruct() {
  final Struct actual = struct(
      "test",
      "f1", Type.INT32, false, 1,
      "f2", Type.STRUCT, false, struct("test2", "c1", Type.INT32, false, 11)
  );
  assertEquals("test", actual.schema().name());
  assertEquals(Type.INT32, actual.schema().field("f1").schema().type());
  assertEquals(Type.STRUCT, actual.schema().field("f2").schema().type());
  assertEquals("test2", actual.schema().field("f2").schema().name());
}
 

@Test
public void asMap() {
  final Struct input = struct(
      "test",
      "f1", Type.INT32, false, 1,
      "f2", Type.STRUCT, false, struct("test2", "c1", Type.INT32, false, 11)
  );
  final Map<String, Object> expected = ImmutableMap.of(
      "f1", 1,
      "f2", ImmutableMap.of("c1", 11)
  );
  final Map<String, Object> actual = StructHelper.asMap(input);
  assertNotNull(actual, "actual should not be null");
  assertEquals(expected, actual);
}
 

/**
 * Parses Kafka Values
 * 
 * @param schema
 *            - Schema of passed message as per
 *            https://kafka.apache.org/0100/javadoc/org/apache/kafka/connect/data/Schema.html
 * @param value
 *            - Value of the message
 * @return Parsed bytebuffer as per schema type
 */
public static ByteBuffer parseValue(Schema schema, Object value) {
	Schema.Type t = schema.type();
	switch (t) {
	case INT8:
		ByteBuffer byteBuffer = ByteBuffer.allocate(1);
		byteBuffer.put((Byte) value);
		return byteBuffer;
	case INT16:
		ByteBuffer shortBuf = ByteBuffer.allocate(2);
		shortBuf.putShort((Short) value);
		return shortBuf;
	case INT32:
		ByteBuffer intBuf = ByteBuffer.allocate(4);
		intBuf.putInt((Integer) value);
		return intBuf;
	case INT64:
		ByteBuffer longBuf = ByteBuffer.allocate(8);
		longBuf.putLong((Long) value);
		return longBuf;
	case FLOAT32:
		ByteBuffer floatBuf = ByteBuffer.allocate(4);
		floatBuf.putFloat((Float) value);
		return floatBuf;
	case FLOAT64:
		ByteBuffer doubleBuf = ByteBuffer.allocate(8);
		doubleBuf.putDouble((Double) value);
		return doubleBuf;
	case BOOLEAN:
		ByteBuffer boolBuffer = ByteBuffer.allocate(1);
		boolBuffer.put((byte) ((Boolean) value ? 1 : 0));
		return boolBuffer;
	case STRING:
		try {
			return ByteBuffer.wrap(((String) value).getBytes("UTF-8"));
		} catch (UnsupportedEncodingException e) {
			// TODO Auto-generated catch block
			System.out.println("Message cannot be translated:" + e.getLocalizedMessage());
		}
	case ARRAY:
		Schema sch = schema.valueSchema();
		if (sch.type() == Type.MAP || sch.type() == Type.STRUCT) {
			throw new DataException("Invalid schema type.");
		}
		Object[] objs = (Object[]) value;
		ByteBuffer[] byteBuffers = new ByteBuffer[objs.length];
		int noOfByteBuffer = 0;

		for (Object obj : objs) {
			byteBuffers[noOfByteBuffer++] = parseValue(sch, obj);
		}

		ByteBuffer result = ByteBuffer.allocate(Arrays.stream(byteBuffers).mapToInt(Buffer::remaining).sum());
		Arrays.stream(byteBuffers).forEach(bb -> result.put(bb.duplicate()));
		return result;
	case BYTES:
		if (value instanceof byte[])
			return ByteBuffer.wrap((byte[]) value);
		else if (value instanceof ByteBuffer)
			return (ByteBuffer) value;
	case MAP:
		// TO BE IMPLEMENTED
		return ByteBuffer.wrap(null);
	case STRUCT:

		List<ByteBuffer> fieldList = new LinkedList<ByteBuffer>();
		// Parsing each field of structure
		schema.fields().forEach(field -> fieldList.add(parseValue(field.schema(), ((Struct) value).get(field))));
		// Initialize ByteBuffer
		ByteBuffer processedValue = ByteBuffer.allocate(fieldList.stream().mapToInt(Buffer::remaining).sum());
		// Combine bytebuffer of all fields
		fieldList.forEach(buffer -> processedValue.put(buffer.duplicate()));

		return processedValue;

	}
	return null;
}
 

@Test
public void struct1() {
  assertFoo(1,
      struct(SCHEMA_NAME, "f1", Type.INT32, false, 1)
  );
}
 

@TestFactory
public Stream<DynamicTest> apply() {
  List<SchemaAndValue> inputs = Arrays.asList(
      new SchemaAndValue(Schema.FLOAT32_SCHEMA, Float.MAX_VALUE),
      new SchemaAndValue(Schema.FLOAT64_SCHEMA, Double.MAX_VALUE),
      new SchemaAndValue(Schema.INT8_SCHEMA, Byte.MAX_VALUE),
      new SchemaAndValue(Schema.INT16_SCHEMA, Short.MAX_VALUE),
      new SchemaAndValue(Schema.INT32_SCHEMA, Integer.MAX_VALUE),
      new SchemaAndValue(Schema.INT64_SCHEMA, Long.MAX_VALUE),
      new SchemaAndValue(Decimal.schema(2), BigDecimal.valueOf(1234231, 2))
  );
  return inputs.stream().map(i -> dynamicTest(SchemaKey.of(i.schema()).toString(), () -> {

    final Schema valueSchema = SchemaBuilder.struct()
        .name("value")
        .field("name", Schema.STRING_SCHEMA)
        .field("value", i.schema())
        .build();
    final Struct value = new Struct(valueSchema)
        .put("name", "testing")
        .put("value", i.value());

    final SinkRecord input = write(
        "test",
        struct("key",
            "id", Type.INT64, false, 1234L
        ),
        value
    );
    final Struct expectedStruct = struct("value",
        "name", Type.STRING, false, "testing",
        "value", Type.INT64, false, ((Number) i.value()).longValue()
    );

    SinkRecord output = this.transformation.apply(input);
    assertNotNull(output, "output cannot be null.");
    assertNotNull(output.value(), "output.value() cannot be null.");
    assertTrue(output.value() instanceof Struct, "output.value() should be a struct.");
    assertNotNull(output.valueSchema(), "output.valueSchema() cannot be null.");
    assertStruct(expectedStruct, (Struct) output.value());
  }));
}