org.apache.kafka.connect.data.Decimal Java Examples

The following examples show how to use org.apache.kafka.connect.data.Decimal. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: StringParserTest.java    From connect-utils with Apache License 2.0 7 votes vote down vote up
@Test
public void nullableTests() {
  final Schema[] schemas = new Schema[]{
      Schema.OPTIONAL_BOOLEAN_SCHEMA,
      Schema.OPTIONAL_FLOAT32_SCHEMA,
      Schema.OPTIONAL_FLOAT64_SCHEMA,
      Schema.OPTIONAL_INT8_SCHEMA,
      Schema.OPTIONAL_INT16_SCHEMA,
      Schema.OPTIONAL_INT32_SCHEMA,
      Schema.OPTIONAL_INT64_SCHEMA,
      Schema.OPTIONAL_STRING_SCHEMA,
      Decimal.builder(1).optional().build(),
      Timestamp.builder().optional().build(),
      Date.builder().optional().build(),
      Time.builder().optional().build(),
  };

  for (Schema schema : schemas) {
    Object actual = this.parser.parseString(schema, null);
    assertNull(actual);
  }

}
 
Example #2
Source File: SchemaHelperTest.java    From connect-utils with Apache License 2.0 7 votes vote down vote up
@TestFactory
public Stream<DynamicTest> schema() {
  return Arrays.asList(
      of("", Schema.OPTIONAL_STRING_SCHEMA),
      of(Byte.MAX_VALUE, Schema.OPTIONAL_INT8_SCHEMA),
      of(Short.MAX_VALUE, Schema.OPTIONAL_INT16_SCHEMA),
      of(Integer.MAX_VALUE, Schema.OPTIONAL_INT32_SCHEMA),
      of(Long.MAX_VALUE, Schema.OPTIONAL_INT64_SCHEMA),
      of(Float.MAX_VALUE, Schema.OPTIONAL_FLOAT32_SCHEMA),
      of(Double.MAX_VALUE, Schema.OPTIONAL_FLOAT64_SCHEMA),
      of(new Date(), Timestamp.builder().optional().build()),
      of(BigDecimal.ONE, Decimal.builder(0).optional().build())
  )
      .stream()
      .map(t -> dynamicTest(t.toString(), () -> {
        final Schema actual = SchemaHelper.schema(t.input);
        assertSchema(t.expectedSchema, actual);
      }));
}
 
Example #3
Source File: DecimalTypeParser.java    From connect-utils with Apache License 2.0 6 votes vote down vote up
private static int scaleInternal(Schema schema) {
  if (null == schema.parameters()) {
    throw new DataException(NOT_FOUND_MESSAGE);
  }

  String scaleString = schema.parameters().get(Decimal.SCALE_FIELD);
  if (scaleString == null) {
    throw new DataException(NOT_FOUND_MESSAGE);
  } else {
    try {
      return Integer.parseInt(scaleString);
    } catch (NumberFormatException var3) {
      throw new DataException(NOT_PARSABLE_MESSAGE, var3);
    }
  }
}
 
Example #4
Source File: Parser.java    From connect-utils with Apache License 2.0 6 votes vote down vote up
public Parser() {
  this.typeParsers = new HashMap<>();
  registerTypeParser(Schema.BOOLEAN_SCHEMA, new BooleanParser());
  registerTypeParser(Schema.BOOLEAN_SCHEMA, new BooleanParser());
  registerTypeParser(Schema.FLOAT32_SCHEMA, new Float32TypeParser());
  registerTypeParser(Schema.FLOAT64_SCHEMA, new Float64TypeParser());
  registerTypeParser(Schema.INT8_SCHEMA, new Int8TypeParser());
  registerTypeParser(Schema.INT16_SCHEMA, new Int16TypeParser());
  registerTypeParser(Schema.INT32_SCHEMA, new Int32TypeParser());
  registerTypeParser(Schema.INT64_SCHEMA, new Int64TypeParser());
  registerTypeParser(Schema.STRING_SCHEMA, new StringTypeParser());
  registerTypeParser(Decimal.schema(1), new DecimalTypeParser());
  registerTypeParser(Date.SCHEMA, new DateTypeParser());
  registerTypeParser(Time.SCHEMA, new TimeTypeParser());
  registerTypeParser(Timestamp.SCHEMA, new TimestampTypeParser());
}
 
Example #5
Source File: HeaderToFieldConfig.java    From kafka-connect-transform-common with Apache License 2.0 6 votes vote down vote up
public static String toString(Schema schema, String schemaNameOverride) {
  StringBuilder builder = new StringBuilder();
  builder.append(schema.type());
  if (!Strings.isNullOrEmpty(schema.name())) {
    String schemaName = Strings.isNullOrEmpty(schemaNameOverride) ? schema.name() : schemaNameOverride;
    builder.append("(");
    builder.append(schemaName);

    if (Decimal.LOGICAL_NAME.equals(schema.name())) {
      builder.append("[scale=");
      builder.append(schema.parameters().get(Decimal.SCALE_FIELD));
      builder.append("]");
    }

    builder.append(")");
  }
  return builder.toString();
}
 
Example #6
Source File: AssertSchemaTest.java    From connect-utils with Apache License 2.0 6 votes vote down vote up
@TestFactory
public Stream<DynamicTest> assertSchema() {
  List<TestCase> tests = new ArrayList<>();
  of(tests, Schema.STRING_SCHEMA, Schema.STRING_SCHEMA, true);
  of(tests, Schema.STRING_SCHEMA, Schema.OPTIONAL_STRING_SCHEMA, false);
  of(tests, Schema.BYTES_SCHEMA, Decimal.schema(4), false);
  of(tests, null, null, true);
  of(tests, Schema.STRING_SCHEMA, null, false);
  of(tests, null, Schema.STRING_SCHEMA, false);

  return tests.stream().map(testCase -> dynamicTest(testCase.toString(), () -> {
    if (testCase.isEqual) {
      AssertSchema.assertSchema(testCase.expected, testCase.actual);
    } else {
      assertThrows(AssertionFailedError.class, () -> {
        AssertSchema.assertSchema(testCase.expected, testCase.actual);
      });
    }
  }));
}
 
Example #7
Source File: AvroData.java    From apicurio-registry with Apache License 2.0 5 votes vote down vote up
@Override
public Object convert(Schema schema, Object value) {
    if (!(value instanceof BigDecimal)) {
        throw new DataException(
            "Invalid type for Decimal, expected BigDecimal but was " + value.getClass());
    }
    return Decimal.fromLogical(schema, (BigDecimal) value);
}
 
Example #8
Source File: SchemaHelper.java    From camel-kafka-connector with Apache License 2.0 5 votes vote down vote up
/**
 * Try to build a {@link SchemaBuilder} for a value of type {@link Object}
 * However, this will only build the schema only for known types, in case it can not return the precise SchemaBuilder type
 * it will return an optional {@link SchemaBuilder.Type}
 * @param value to return the SchemaBuilder for
 *
 * @return {@link SchemaBuilder} instance
 */
public static SchemaBuilder buildSchemaBuilderForType(final Object value) {
    Objects.requireNonNull(value);

    // gracefully try to infer the schema
    final Schema knownSchema = Values.inferSchema(value);

    if (knownSchema == null) {
        // let's now check for other types
        if (value instanceof Date) {
            return org.apache.kafka.connect.data.Date.builder();
        }
        if (value instanceof BigDecimal) {
            return Decimal.builder(((BigDecimal) value).scale());
        }
        // we re-check map and list since inferSchema function is not tolerant against map and list
        // for now we rely on inferSchema, however it makes some point to build a specific inferSchema method only for this connector
        if (value instanceof Map) {
            return new SchemaBuilder(Schema.Type.MAP);
        }
        if (value instanceof List) {
            return new SchemaBuilder(Schema.Type.ARRAY);
        }
        // if we do not fine any of schema out of the above, we just return an an optional byte schema
        return SchemaBuilder.bytes().optional();
    }
    return SchemaUtil.copySchemaBasics(knownSchema);
}
 
Example #9
Source File: AvroData.java    From apicurio-registry with Apache License 2.0 5 votes vote down vote up
@Override
public Object convert(Schema schema, Object value) {
    if (value instanceof byte[]) {
        return Decimal.toLogical(schema, (byte[]) value);
    } else if (value instanceof ByteBuffer) {
        return Decimal.toLogical(schema, ((ByteBuffer) value).array());
    }
    throw new DataException(
        "Invalid type for Decimal, underlying representation should be bytes but was "
        + value.getClass());
}
 
Example #10
Source File: StringParserTest.java    From connect-utils with Apache License 2.0 5 votes vote down vote up
@TestFactory
Stream<DynamicTest> parseString() {
  List<TestCase> tests = new ArrayList<>();
  of(tests, Schema.FLOAT64_SCHEMA, new Double(Double.MAX_VALUE).toString(), new Double(Double.MAX_VALUE));
  of(tests, Schema.FLOAT64_SCHEMA, new Double(Double.MIN_VALUE).toString(), new Double(Double.MIN_VALUE));

  of(tests, Schema.INT8_SCHEMA, new Byte(Byte.MAX_VALUE).toString(), new Byte(Byte.MAX_VALUE));
  of(tests, Schema.INT8_SCHEMA, new Byte(Byte.MIN_VALUE).toString(), new Byte(Byte.MIN_VALUE));

  of(tests, Schema.INT16_SCHEMA, new Short(Short.MAX_VALUE).toString(), new Short(Short.MAX_VALUE));
  of(tests, Schema.INT16_SCHEMA, new Short(Short.MIN_VALUE).toString(), new Short(Short.MIN_VALUE));

  of(tests, Schema.INT32_SCHEMA, new Integer(Integer.MAX_VALUE).toString(), new Integer(Integer.MAX_VALUE));
  of(tests, Schema.INT32_SCHEMA, new Integer(Integer.MIN_VALUE).toString(), new Integer(Integer.MIN_VALUE));

  of(tests, Schema.INT64_SCHEMA, new Long(Long.MAX_VALUE).toString(), new Long(Long.MAX_VALUE));
  of(tests, Schema.INT64_SCHEMA, new Long(Long.MIN_VALUE).toString(), new Long(Long.MIN_VALUE));

  of(tests, Schema.STRING_SCHEMA, "", "");
  of(tests, Schema.STRING_SCHEMA, "mirror", "mirror");

  for (int SCALE = 3; SCALE < 30; SCALE++) {
    Schema schema = Decimal.schema(SCALE);
    of(tests, schema, "12345", new BigDecimal("12345").setScale(SCALE));
    of(tests, schema, "0", new BigDecimal("0").setScale(SCALE));
    of(tests, schema, "-12345.001", new BigDecimal("-12345.001").setScale(SCALE));
  }

  return tests.stream().map(testCase -> dynamicTest(testCase.toString(), () -> {
    final Object actual = parser.parseString(testCase.schema, testCase.input);
    assertEquals(testCase.expected, actual);
  }));
}
 
Example #11
Source File: JsonNodeTest.java    From connect-utils with Apache License 2.0 5 votes vote down vote up
@Test
public void nullableTests() throws IOException {
  final Schema[] schemas = new Schema[]{
      Schema.OPTIONAL_BOOLEAN_SCHEMA,
      Schema.OPTIONAL_FLOAT32_SCHEMA,
      Schema.OPTIONAL_FLOAT64_SCHEMA,
      Schema.OPTIONAL_INT8_SCHEMA,
      Schema.OPTIONAL_INT16_SCHEMA,
      Schema.OPTIONAL_INT32_SCHEMA,
      Schema.OPTIONAL_INT64_SCHEMA,
      Schema.OPTIONAL_STRING_SCHEMA,
      Decimal.builder(1).optional().build(),
      Timestamp.builder().optional().build(),
      Date.builder().optional().build(),
      Time.builder().optional().build(),
  };

  for (Schema schema : schemas) {
    JsonNode inputNode = null;
    Object actual = this.parser.parseJsonNode(schema, inputNode);
    assertNull(actual);
    inputNode = objectMapper.readTree("{\"foo\": null}");
    inputNode = inputNode.findValue("foo");
    actual = this.parser.parseJsonNode(schema, inputNode);
    assertNull(actual);
  }

}
 
Example #12
Source File: ValueHelperTest.java    From connect-utils with Apache License 2.0 4 votes vote down vote up
@TestFactory
Stream<DynamicTest> value() {
  List<TestCase> tests = new ArrayList<>();
  of(tests, Schema.FLOAT64_SCHEMA, new Double(Double.MAX_VALUE), new Double(Double.MAX_VALUE));
  of(tests, Schema.FLOAT64_SCHEMA, new Double(Double.MIN_VALUE), new Double(Double.MIN_VALUE));

  of(tests, Schema.INT8_SCHEMA, new Byte(Byte.MAX_VALUE), new Byte(Byte.MAX_VALUE));
  of(tests, Schema.INT8_SCHEMA, new Byte(Byte.MIN_VALUE), new Byte(Byte.MIN_VALUE));

  of(tests, Schema.INT16_SCHEMA, new Short(Short.MAX_VALUE), new Short(Short.MAX_VALUE));
  of(tests, Schema.INT16_SCHEMA, new Short(Short.MIN_VALUE), new Short(Short.MIN_VALUE));

  of(tests, Schema.INT32_SCHEMA, new Integer(Integer.MAX_VALUE), new Integer(Integer.MAX_VALUE));
  of(tests, Schema.INT32_SCHEMA, new Integer(Integer.MIN_VALUE), new Integer(Integer.MIN_VALUE));

  of(tests, Schema.INT64_SCHEMA, new Long(Long.MAX_VALUE), new Long(Long.MAX_VALUE));
  of(tests, Schema.INT64_SCHEMA, new Long(Long.MIN_VALUE), new Long(Long.MIN_VALUE));

  of(tests, Schema.STRING_SCHEMA, "", "");
  of(tests, Schema.STRING_SCHEMA, "mirror", "mirror");

  for (int SCALE = 3; SCALE < 30; SCALE++) {
    Schema schema = Decimal.schema(SCALE);
    of(tests, schema, new BigDecimal("12345").setScale(SCALE), new BigDecimal("12345").setScale(SCALE));
    of(tests, schema, new BigDecimal("0").setScale(SCALE), new BigDecimal("0").setScale(SCALE));
    of(tests, schema, new BigDecimal("-12345.001").setScale(SCALE), new BigDecimal("-12345.001").setScale(SCALE));
  }

  Schema structSchema = SchemaBuilder.struct()
      .field("firstName", Schema.STRING_SCHEMA)
      .field("lastName", Schema.STRING_SCHEMA)
      .build();

  of(
      tests,
      structSchema,
      ImmutableMap.of("firstName", "example", "lastName", "user"),
      new Struct(structSchema)
          .put("firstName", "example")
          .put("lastName", "user")
  );

  of(
      tests,
      structSchema,
      new Struct(structSchema)
          .put("firstName", "example")
          .put("lastName", "user"),
      new Struct(structSchema)
          .put("firstName", "example")
          .put("lastName", "user")
  );

  return tests.stream().map(testCase -> dynamicTest(testCase.toString(), () -> {
    final Object actual = ValueHelper.value(testCase.schema, testCase.input);
    assertEquals(testCase.expected, actual);
  }));
}
 
Example #13
Source File: AssertStruct.java    From connect-utils with Apache License 2.0 4 votes vote down vote up
public static void assertStruct(final Struct expected, final Struct actual, String message) {
  String prefix = Strings.isNullOrEmpty(message) ? "" : message + ": ";

  if (null == expected) {
    assertNull(actual, prefix + "actual should be null.");
    return;
  }

  assertSchema(expected.schema(), actual.schema(), "schema does not match.");

  final Set<String> logicalTypes = ImmutableSet.of(
      Timestamp.LOGICAL_NAME,
      Date.LOGICAL_NAME,
      Time.LOGICAL_NAME,
      Decimal.LOGICAL_NAME
  );

  for (Field field : expected.schema().fields()) {
    log.trace("assertStruct() - testing field '{}'", field.name());
    final Object expectedValue = expected.get(field.name());
    final Object actualValue = actual.get(field.name());

    if (!Strings.isNullOrEmpty(field.schema().name()) && logicalTypes.contains(field.schema().name())) {
      assertEquals(expectedValue, actualValue, prefix + field.name() + " does not match.");
    } else {
      switch (field.schema().type()) {
        case ARRAY:
          assertTrue(null == expectedValue || expectedValue instanceof List);
          assertTrue(null == actualValue || actualValue instanceof List);
          List<Object> expectedArray = (List<Object>) expectedValue;
          List<Object> actualArray = (List<Object>) actualValue;
          assertEquals(expectedArray, actualArray, prefix + field.name() + " does not match.");
          break;
        case MAP:
          assertTrue(null == expectedValue || expectedValue instanceof Map);
          assertTrue(null == actualValue || actualValue instanceof Map);
          Map<Object, Object> expectedMap = (Map<Object, Object>) expectedValue;
          Map<Object, Object> actualMap = (Map<Object, Object>) actualValue;
          assertEquals(expectedMap, actualMap, prefix + field.name() + " does not match.");
          break;
        case STRUCT:
          assertTrue(null == expectedValue || expectedValue instanceof Struct);
          assertTrue(null == actualValue || actualValue instanceof Struct);
          Struct expectedStruct = (Struct) expectedValue;
          Struct actualStruct = (Struct) actualValue;
          assertStruct(expectedStruct, actualStruct, prefix + field.name() + " does not match.");
          break;
        case BYTES:
          assertTrue(null == expectedValue || expectedValue instanceof byte[]);
          assertTrue(null == actualValue || actualValue instanceof byte[]);
          byte[] expectedByteArray = (byte[]) expectedValue;
          byte[] actualByteArray = (byte[]) actualValue;
          assertEquals(
              null == expectedByteArray ? "" : BaseEncoding.base32Hex().encode(expectedByteArray).toString(),
              null == actualByteArray ? "" : BaseEncoding.base32Hex().encode(actualByteArray).toString(),
              prefix + field.name() + " does not match."
          );
          break;
        default:
          assertEquals(expectedValue, actualValue, prefix + field.name() + " does not match.");
          break;
      }
    }
  }
}
 
Example #14
Source File: AbstractConverter.java    From connect-utils with Apache License 2.0 4 votes vote down vote up
void convertStruct(final T result, Struct struct) {
  final Schema schema = struct.schema();

  for (final Field field : schema.fields()) {
    final String fieldName = field.name();
    log.trace("convertStruct() - Processing '{}'", field.name());
    final Object fieldValue = struct.get(field);

    try {
      if (null == fieldValue) {
        log.trace("convertStruct() - Setting '{}' to null.", fieldName);
        setNullField(result, fieldName);
        continue;
      }

      log.trace("convertStruct() - Field '{}'.field().schema().type() = '{}'", fieldName, field.schema().type());
      switch (field.schema().type()) {
        case STRING:
          log.trace("convertStruct() - Processing '{}' as string.", fieldName);
          setStringField(result, fieldName, (String) fieldValue);
          break;
        case INT8:
          log.trace("convertStruct() - Processing '{}' as int8.", fieldName);
          setInt8Field(result, fieldName, (Byte) fieldValue);
          break;
        case INT16:
          log.trace("convertStruct() - Processing '{}' as int16.", fieldName);
          setInt16Field(result, fieldName, (Short) fieldValue);
          break;
        case INT32:
          if (org.apache.kafka.connect.data.Date.LOGICAL_NAME.equals(field.schema().name())) {
            log.trace("convertStruct() - Processing '{}' as date.", fieldName);
            setDateField(result, fieldName, (Date) fieldValue);
          } else if (org.apache.kafka.connect.data.Time.LOGICAL_NAME.equals(field.schema().name())) {
            log.trace("convertStruct() - Processing '{}' as time.", fieldName);
            setTimeField(result, fieldName, (Date) fieldValue);
          } else {
            Integer int32Value = (Integer) fieldValue;
            log.trace("convertStruct() - Processing '{}' as int32.", fieldName);
            setInt32Field(result, fieldName, int32Value);
          }
          break;
        case INT64:

          if (Timestamp.LOGICAL_NAME.equals(field.schema().name())) {
            log.trace("convertStruct() - Processing '{}' as timestamp.", fieldName);
            setTimestampField(result, fieldName, (Date) fieldValue);
          } else {
            Long int64Value = (Long) fieldValue;
            log.trace("convertStruct() - Processing '{}' as int64.", fieldName);
            setInt64Field(result, fieldName, int64Value);
          }
          break;
        case BYTES:

          if (Decimal.LOGICAL_NAME.equals(field.schema().name())) {
            log.trace("convertStruct() - Processing '{}' as decimal.", fieldName);
            setDecimalField(result, fieldName, (BigDecimal) fieldValue);
          } else {
            byte[] bytes = (byte[]) fieldValue;
            log.trace("convertStruct() - Processing '{}' as bytes.", fieldName);
            setBytesField(result, fieldName, bytes);
          }
          break;
        case FLOAT32:
          log.trace("convertStruct() - Processing '{}' as float32.", fieldName);
          setFloat32Field(result, fieldName, (Float) fieldValue);
          break;
        case FLOAT64:
          log.trace("convertStruct() - Processing '{}' as float64.", fieldName);
          setFloat64Field(result, fieldName, (Double) fieldValue);
          break;
        case BOOLEAN:
          log.trace("convertStruct() - Processing '{}' as boolean.", fieldName);
          setBooleanField(result, fieldName, (Boolean) fieldValue);
          break;
        case STRUCT:
          log.trace("convertStruct() - Processing '{}' as struct.", fieldName);
          setStructField(result, fieldName, (Struct) fieldValue);
          break;
        case ARRAY:
          log.trace("convertStruct() - Processing '{}' as array.", fieldName);
          setArray(result, fieldName, schema, (List) fieldValue);
          break;
        case MAP:
          log.trace("convertStruct() - Processing '{}' as map.", fieldName);
          setMap(result, fieldName, schema, (Map) fieldValue);
          break;
        default:
          throw new DataException("Unsupported schema.type(): " + schema.type());
      }
    } catch (Exception ex) {
      throw new DataException(
          String.format("Exception thrown while processing field '%s'", fieldName),
          ex
      );
    }
  }
}
 
Example #15
Source File: AssertStructTest.java    From connect-utils with Apache License 2.0 4 votes vote down vote up
@BeforeEach
public void createSchemas() {
  this.decimalSchema = SchemaBuilder.struct()
      .name("com.github.jcustenborder.kafka.connect.utils.Decimal")
      .field("decimal", Decimal.builder(1).optional().build())
      .build();
  this.timestampSchema = SchemaBuilder.struct()
      .name("com.github.jcustenborder.kafka.connect.utils.Timestamp")
      .field("timestamp", Timestamp.builder().optional().build())
      .build();
  this.dateSchema = SchemaBuilder.struct()
      .name("com.github.jcustenborder.kafka.connect.utils.Date")
      .field("date", Date.builder().optional().build())
      .build();
  this.timeSchema = SchemaBuilder.struct()
      .name("com.github.jcustenborder.kafka.connect.utils.Time")
      .field("time", Time.builder().optional().build())
      .build();

  this.simple = SchemaBuilder.struct()
      .name("com.github.jcustenborder.kafka.connect.utils.Simple")
      .optional()
      .field("text", Schema.OPTIONAL_STRING_SCHEMA)
      .build();

  this.pointSchema = SchemaBuilder.struct()
      .name("com.github.jcustenborder.kafka.connect.utils.Point")
      .optional()
      .field("latitude", Schema.FLOAT32_SCHEMA)
      .field("longitude", Schema.FLOAT32_SCHEMA)
      .build();

  this.nestedSchema = SchemaBuilder.struct()
      .name("com.github.jcustenborder.kafka.connect.utils.Nested")
      .field("point", this.pointSchema)
      .build();

  this.arraySchema = SchemaBuilder.struct()
      .name("com.github.jcustenborder.kafka.connect.utils.ArrayOfPoint")
      .field("array", SchemaBuilder.array(this.pointSchema).optional().build())
      .build();

  this.mapSchema = SchemaBuilder.struct()
      .name("com.github.jcustenborder.kafka.connect.utils.MapOfPoint")
      .field("map", SchemaBuilder.map(Schema.STRING_SCHEMA, this.pointSchema).optional().build())
      .build();

  this.structBytesSchema = SchemaBuilder.struct()
      .name("com.github.jcustenborder.kafka.connect.utils.Bytes")
      .field("bytes", Schema.OPTIONAL_BYTES_SCHEMA)
      .build();
  this.buffer = new byte[10];
  new Random().nextBytes(this.buffer);
}
 
Example #16
Source File: AssertStruct.java    From connect-utils with Apache License 2.0 4 votes vote down vote up
public static void assertStruct(final Struct expected, final Struct actual, String message) {
  String prefix = Strings.isNullOrEmpty(message) ? "" : message + ": ";

  if (null == expected) {
    assertNull(actual, prefix + "actual should be null.");
    return;
  }

  assertSchema(expected.schema(), actual.schema(), "schema does not match.");
  for (Field expectedField : expected.schema().fields()) {
    log.trace("assertStruct() - testing field '{}'", expectedField.name());
    final Object expectedValue = expected.get(expectedField.name());
    final Object actualValue = actual.get(expectedField.name());

    if (Decimal.LOGICAL_NAME.equals(expectedField.schema().name())) {
      final BigDecimal expectedDecimal = castAndVerify(BigDecimal.class, expected, expectedField, true);
      final BigDecimal actualDecimal = castAndVerify(BigDecimal.class, actual, expectedField, false);
      assertEquals(expectedDecimal, actualDecimal, prefix + expectedField.name() + " does not match.");
    } else if (Timestamp.LOGICAL_NAME.equals(expectedField.schema().name())
        || Date.LOGICAL_NAME.equals(expectedField.schema().name())
        || Time.LOGICAL_NAME.equals(expectedField.schema().name())) {
      final java.util.Date expectedDate = castAndVerify(java.util.Date.class, expected, expectedField, true);
      final java.util.Date actualDate = castAndVerify(java.util.Date.class, actual, expectedField, false);
      assertEquals(expectedDate, actualDate, prefix + expectedField.name() + " does not match.");
    } else {
      switch (expectedField.schema().type()) {
        case ARRAY:
          final List<Object> expectedArray = castAndVerify(List.class, expected, expectedField, true);
          final List<Object> actualArray = castAndVerify(List.class, actual, expectedField, false);
          assertEquals(expectedArray, actualArray, prefix + expectedField.name() + " does not match.");
          break;
        case MAP:
          final Map<Object, Object> expectedMap = castAndVerify(Map.class, expected, expectedField, true);
          final Map<Object, Object> actualMap = castAndVerify(Map.class, actual, expectedField, false);
          assertEquals(expectedMap, actualMap, prefix + expectedField.name() + " does not match.");
          break;
        case STRUCT:
          final Struct expectedStruct = castAndVerify(Struct.class, expected, expectedField, true);
          final Struct actualStruct = castAndVerify(Struct.class, actual, expectedField, false);
          assertStruct(expectedStruct, actualStruct, prefix + expectedField.name() + " does not match.");
          break;
        case BYTES:
          final byte[] expectedByteArray = castAndVerify(byte[].class, expected, expectedField, true);
          final byte[] actualByteArray = castAndVerify(byte[].class, actual, expectedField, false);
          assertEquals(
              null == expectedByteArray ? "" : BaseEncoding.base32Hex().encode(expectedByteArray).toString(),
              null == actualByteArray ? "" : BaseEncoding.base32Hex().encode(actualByteArray).toString(),
              prefix + expectedField.name() + " does not match."
          );
          break;
        default:
          assertEquals(expectedValue, actualValue, prefix + expectedField.name() + " does not match.");
          break;
      }
    }
  }
}
 
Example #17
Source File: ToLongTest.java    From kafka-connect-transform-common with Apache License 2.0 4 votes vote down vote up
@TestFactory
public Stream<DynamicTest> apply() {
  List<SchemaAndValue> inputs = Arrays.asList(
      new SchemaAndValue(Schema.FLOAT32_SCHEMA, Float.MAX_VALUE),
      new SchemaAndValue(Schema.FLOAT64_SCHEMA, Double.MAX_VALUE),
      new SchemaAndValue(Schema.INT8_SCHEMA, Byte.MAX_VALUE),
      new SchemaAndValue(Schema.INT16_SCHEMA, Short.MAX_VALUE),
      new SchemaAndValue(Schema.INT32_SCHEMA, Integer.MAX_VALUE),
      new SchemaAndValue(Schema.INT64_SCHEMA, Long.MAX_VALUE),
      new SchemaAndValue(Decimal.schema(2), BigDecimal.valueOf(1234231, 2))
  );
  return inputs.stream().map(i -> dynamicTest(SchemaKey.of(i.schema()).toString(), () -> {

    final Schema valueSchema = SchemaBuilder.struct()
        .name("value")
        .field("name", Schema.STRING_SCHEMA)
        .field("value", i.schema())
        .build();
    final Struct value = new Struct(valueSchema)
        .put("name", "testing")
        .put("value", i.value());

    final SinkRecord input = write(
        "test",
        struct("key",
            "id", Type.INT64, false, 1234L
        ),
        value
    );
    final Struct expectedStruct = struct("value",
        "name", Type.STRING, false, "testing",
        "value", Type.INT64, false, ((Number) i.value()).longValue()
    );

    SinkRecord output = this.transformation.apply(input);
    assertNotNull(output, "output cannot be null.");
    assertNotNull(output.value(), "output.value() cannot be null.");
    assertTrue(output.value() instanceof Struct, "output.value() should be a struct.");
    assertNotNull(output.valueSchema(), "output.valueSchema() cannot be null.");
    assertStruct(expectedStruct, (Struct) output.value());
  }));
}
 
Example #18
Source File: SetMaximumPrecisionTest.java    From kafka-connect-transform-common with Apache License 2.0 4 votes vote down vote up
@Test
public void convert() {
  final Schema inputSchema = SchemaBuilder.struct()
      .field("first", Decimal.schema(5))
      .field(
          "second",
          Decimal.builder(5)
              .parameter(SetMaximumPrecision.CONNECT_AVRO_DECIMAL_PRECISION_PROP, "16")
              .optional()
              .build()
      )
      .field(
          "third",
          Decimal.builder(5)
              .parameter(SetMaximumPrecision.CONNECT_AVRO_DECIMAL_PRECISION_PROP, "48")
              .optional()
              .build()
      )
      .build();
  final Struct inputStruct = new Struct(inputSchema)
      .put("first", BigDecimal.ONE)
      .put("second", null)
      .put("third", BigDecimal.ONE);
  final Schema expectedSchema = SchemaBuilder.struct()
      .field(
          "first",
          Decimal.builder(5)
              .parameter(SetMaximumPrecision.CONNECT_AVRO_DECIMAL_PRECISION_PROP, "32")
              .build()
      )
      .field(
          "second",
          Decimal.builder(5)
              .parameter(SetMaximumPrecision.CONNECT_AVRO_DECIMAL_PRECISION_PROP, "16")
              .optional()
              .build()
      )
      .field(
          "third",
          Decimal.builder(5)
              .parameter(SetMaximumPrecision.CONNECT_AVRO_DECIMAL_PRECISION_PROP, "32")
              .optional()
              .build()
      )
      .build();
  final Struct expectedStruct = new Struct(expectedSchema)
      .put("first", BigDecimal.ONE)
      .put("second", null)
      .put("third", BigDecimal.ONE);


  SinkRecord record = record(inputStruct);
  SetMaximumPrecision.Value<SinkRecord> transform = new SetMaximumPrecision.Value<>();
  transform.configure(
      ImmutableMap.of(SetMaximumPrecisionConfig.MAX_PRECISION_CONFIG, 32)
  );


  SinkRecord actual = transform.apply(record);
  assertNotNull(actual);
  assertStruct(expectedStruct, (Struct) actual.value());
}
 
Example #19
Source File: ConversionHandler.java    From kafka-connect-transform-common with Apache License 2.0 4 votes vote down vote up
public DecimalConversionHandler(Schema headerSchema, String header, String field) {
  super(headerSchema, header, field);
  String scaleText = null != headerSchema.parameters() ? headerSchema.parameters().get(Decimal.SCALE_FIELD) : null;
  Preconditions.checkNotNull(scaleText, "schema parameters must contain a '%s' parameter.", Decimal.SCALE_FIELD);
  scale = Integer.parseInt(scaleText);
}
 
Example #20
Source File: BaseTransformation.java    From kafka-connect-transform-common with Apache License 2.0 4 votes vote down vote up
protected SchemaAndValue process(R record, Schema inputSchema, Object input) {
  final SchemaAndValue result;

  if (null == inputSchema && null == input) {
    return new SchemaAndValue(
        null,
        null
    );
  }

  if (input instanceof Map) {
    log.trace("process() - Processing as map");
    result = processMap(record, (Map<String, Object>) input);
    return result;
  }

  if (null == inputSchema) {
    log.trace("process() - Determining schema");
    inputSchema = SchemaHelper.schema(input);
  }

  log.trace("process() - Input has as schema. schema = {}", inputSchema);
  if (Schema.Type.STRUCT == inputSchema.type()) {
    result = processStruct(record, inputSchema, (Struct) input);
  } else if (Timestamp.LOGICAL_NAME.equals(inputSchema.name())) {
    result = processTimestamp(record, inputSchema, (Date) input);
  } else if (org.apache.kafka.connect.data.Date.LOGICAL_NAME.equals(inputSchema.name())) {
    result = processDate(record, inputSchema, (Date) input);
  } else if (Time.LOGICAL_NAME.equals(inputSchema.name())) {
    result = processTime(record, inputSchema, (Date) input);
  } else if (Decimal.LOGICAL_NAME.equals(inputSchema.name())) {
    result = processDecimal(record, inputSchema, (BigDecimal) input);
  } else if (Schema.Type.STRING == inputSchema.type()) {
    result = processString(record, inputSchema, (String) input);
  } else if (Schema.Type.BYTES == inputSchema.type()) {
    result = processBytes(record, inputSchema, (byte[]) input);
  } else if (Schema.Type.INT8 == inputSchema.type()) {
    result = processInt8(record, inputSchema, (byte) input);
  } else if (Schema.Type.INT16 == inputSchema.type()) {
    result = processInt16(record, inputSchema, (short) input);
  } else if (Schema.Type.INT32 == inputSchema.type()) {
    result = processInt32(record, inputSchema, (int) input);
  } else if (Schema.Type.INT64 == inputSchema.type()) {
    result = processInt64(record, inputSchema, (long) input);
  } else if (Schema.Type.FLOAT32 == inputSchema.type()) {
    result = processFloat32(record, inputSchema, (float) input);
  } else if (Schema.Type.FLOAT64 == inputSchema.type()) {
    result = processFloat64(record, inputSchema, (double) input);
  } else if (Schema.Type.ARRAY == inputSchema.type()) {
    result = processArray(record, inputSchema, (List<Object>) input);
  } else if (Schema.Type.MAP == inputSchema.type()) {
    result = processMap(record, inputSchema, (Map<Object, Object>) input);
  } else {
    throw new UnsupportedOperationException(
        String.format(
            "Schema is not supported. type='%s' name='%s'",
            inputSchema.type(),
            inputSchema.name()
        )
    );
  }

  return result;
}
 
Example #21
Source File: MsSqlTableMetadataProvider.java    From kafka-connect-cdc-mssql with Apache License 2.0 4 votes vote down vote up
Schema generateSchema(ResultSet resultSet,
                      final ChangeKey changeKey,
                      final String columnName) throws SQLException {
  boolean optional = resultSet.getBoolean(2);
  String dataType = resultSet.getString(3);
  int scale = resultSet.getInt(4);
  SchemaBuilder builder;

  log.trace("{}: columnName='{}' dataType='{}' scale={} optional={}", changeKey, columnName, dataType, scale, optional);

  switch (dataType) {
    case "bigint":
      builder = SchemaBuilder.int64();
      break;
    case "bit":
      builder = SchemaBuilder.bool();
      break;
    case "char":
    case "varchar":
    case "text":
    case "nchar":
    case "nvarchar":
    case "ntext":
    case "uniqueidentifier":
      builder = SchemaBuilder.string();
      break;
    case "smallmoney":
    case "money":
    case "decimal":
    case "numeric":
      builder = Decimal.builder(scale);
      break;
    case "binary":
    case "image":
    case "varbinary":
      builder = SchemaBuilder.bytes();
      break;
    case "date":
      builder = Date.builder();
      break;
    case "datetime":
    case "datetime2":
    case "smalldatetime":
      builder = Timestamp.builder();
      break;
    case "time":
      builder = Time.builder();
      break;
    case "int":
      builder = SchemaBuilder.int32();
      break;
    case "smallint":
      builder = SchemaBuilder.int16();
      break;
    case "tinyint":
      builder = SchemaBuilder.int8();
      break;
    case "real":
      builder = SchemaBuilder.float32();
      break;
    case "float":
      builder = SchemaBuilder.float64();
      break;

    default:
      throw new DataException(
          String.format("Could not process (dataType = '%s', optional = %s, scale = %d) for %s.",
              dataType, optional, scale, changeKey
          )
      );
  }

  log.trace("{}: columnName='{}' schema.type='{}' schema.name='{}'", changeKey, columnName, builder.type(), builder.name());

  builder.parameters(
      ImmutableMap.of(Change.ColumnValue.COLUMN_NAME, columnName)
  );

  if (optional) {
    builder.optional();
  }

  return builder.build();
}
 
Example #22
Source File: DecimalFieldConverter.java    From kafka-connect-mongodb with Apache License 2.0 4 votes vote down vote up
public DecimalFieldConverter(Format format) {
    super(Decimal.schema(0));
    this.format = format;
}
 
Example #23
Source File: DecimalFieldConverter.java    From kafka-connect-mongodb with Apache License 2.0 4 votes vote down vote up
public DecimalFieldConverter() {
    super(Decimal.schema(0));
    this.format = Format.DECIMAL128;
}
 
Example #24
Source File: SinkFieldConverterTest.java    From mongo-kafka with Apache License 2.0 4 votes vote down vote up
@TestFactory
@DisplayName("tests for logical type decimal field conversions (legacy)")
List<DynamicTest> testDecimalFieldConverterLegacy() {
  SinkFieldConverter converter =
      new DecimalFieldConverter(DecimalFieldConverter.Format.LEGACYDOUBLE);

  List<DynamicTest> tests = new ArrayList<>();
  asList(
          new BigDecimal("-1234567890.09876543210"),
          BigDecimal.ZERO,
          new BigDecimal("+1234567890.09876543210"))
      .forEach(
          el ->
              tests.add(
                  dynamicTest(
                      "conversion with " + converter.getClass().getSimpleName() + " for " + el,
                      () ->
                          assertEquals(
                              el.doubleValue(),
                              ((BsonDouble) converter.toBson(el)).getValue()))));

  tests.add(
      dynamicTest(
          "optional type conversions",
          () -> {
            Schema valueOptionalDefault =
                Decimal.builder(0).optional().defaultValue(BigDecimal.ZERO);
            assertAll(
                "checks",
                () ->
                    assertThrows(
                        DataException.class, () -> converter.toBson(null, Decimal.schema(0))),
                () ->
                    assertEquals(
                        new BsonNull(), converter.toBson(null, Decimal.builder(0).optional())),
                () ->
                    assertEquals(
                        ((BigDecimal) valueOptionalDefault.defaultValue()).doubleValue(),
                        ((BsonDouble) converter.toBson(null, valueOptionalDefault)).getValue()));
          }));
  return tests;
}
 
Example #25
Source File: SinkFieldConverterTest.java    From mongo-kafka with Apache License 2.0 4 votes vote down vote up
@TestFactory
@DisplayName("tests for logical type decimal field conversions (new)")
List<DynamicTest> testDecimalFieldConverterNew() {

  SinkFieldConverter converter = new DecimalFieldConverter();

  List<DynamicTest> tests = new ArrayList<>();
  asList(
          new BigDecimal("-1234567890.09876543210"),
          BigDecimal.ZERO,
          new BigDecimal("+1234567890.09876543210"))
      .forEach(
          el ->
              tests.add(
                  dynamicTest(
                      "conversion with " + converter.getClass().getSimpleName() + " for " + el,
                      () ->
                          assertEquals(
                              el,
                              ((BsonDecimal128) converter.toBson(el))
                                  .getValue()
                                  .bigDecimalValue()))));

  tests.add(
      dynamicTest(
          "optional type conversions",
          () -> {
            Schema valueOptionalDefault =
                Decimal.builder(0).optional().defaultValue(BigDecimal.ZERO);
            assertAll(
                "checks",
                () ->
                    assertThrows(
                        DataException.class, () -> converter.toBson(null, Decimal.schema(0))),
                () ->
                    assertEquals(
                        new BsonNull(), converter.toBson(null, Decimal.builder(0).optional())),
                () ->
                    assertEquals(
                        valueOptionalDefault.defaultValue(),
                        ((BsonDecimal128) converter.toBson(null, valueOptionalDefault))
                            .getValue()
                            .bigDecimalValue()));
          }));
  return tests;
}
 
Example #26
Source File: DecimalFieldConverter.java    From mongo-kafka with Apache License 2.0 4 votes vote down vote up
public DecimalFieldConverter(final Format format) {
  super(Decimal.schema(0));
  this.format = format;
}
 
Example #27
Source File: DecimalFieldConverter.java    From mongo-kafka with Apache License 2.0 4 votes vote down vote up
public DecimalFieldConverter() {
  super(Decimal.schema(0));
  this.format = Format.DECIMAL128;
}