org.apache.kafka.connect.data.Values Java Examples

The following examples show how to use org.apache.kafka.connect.data.Values. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: SchemaHelper.java From camel-kafka-connector with Apache License 2.0

5 votes

/**
 * Try to build a {@link SchemaBuilder} for a value of type {@link Object}
 * However, this will only build the schema only for known types, in case it can not return the precise SchemaBuilder type
 * it will return an optional {@link SchemaBuilder.Type}
 * @param value to return the SchemaBuilder for
 *
 * @return {@link SchemaBuilder} instance
 */
public static SchemaBuilder buildSchemaBuilderForType(final Object value) {
    Objects.requireNonNull(value);

    // gracefully try to infer the schema
    final Schema knownSchema = Values.inferSchema(value);

    if (knownSchema == null) {
        // let's now check for other types
        if (value instanceof Date) {
            return org.apache.kafka.connect.data.Date.builder();
        }
        if (value instanceof BigDecimal) {
            return Decimal.builder(((BigDecimal) value).scale());
        }
        // we re-check map and list since inferSchema function is not tolerant against map and list
        // for now we rely on inferSchema, however it makes some point to build a specific inferSchema method only for this connector
        if (value instanceof Map) {
            return new SchemaBuilder(Schema.Type.MAP);
        }
        if (value instanceof List) {
            return new SchemaBuilder(Schema.Type.ARRAY);
        }
        // if we do not fine any of schema out of the above, we just return an an optional byte schema
        return SchemaBuilder.bytes().optional();
    }
    return SchemaUtil.copySchemaBasics(knownSchema);
}

Example #2

Source File: KafkaConnectRandomIntIntegrationTest.java From hazelcast-jet-contrib with Apache License 2.0

5 votes

@Test
public void readFromRandomSource() throws Exception {
    System.setProperty("hazelcast.logging.type", "log4j");
    Properties randomProperties = new Properties();
    randomProperties.setProperty("name", "random-source-connector");
    randomProperties.setProperty("connector.class", "sasakitoa.kafka.connect.random.RandomSourceConnector");
    randomProperties.setProperty("generator.class", "sasakitoa.kafka.connect.random.generator.RandomInt");
    randomProperties.setProperty("tasks.max", "1");
    randomProperties.setProperty("messages.per.second", "1000");
    randomProperties.setProperty("topic", "test");
    randomProperties.setProperty("task.summary.enable", "true");

    Pipeline pipeline = Pipeline.create();
    pipeline.readFrom(KafkaConnectSources.connect(randomProperties))
            .withoutTimestamps()
            .map(record -> Values.convertToString(record.valueSchema(), record.value()))
            .writeTo(AssertionSinks.assertCollectedEventually(60,
                    list -> assertEquals(ITEM_COUNT, list.size())));

    JobConfig jobConfig = new JobConfig();
    jobConfig.addJar(Objects.requireNonNull(this.getClass()
                                                      .getClassLoader()
                                                      .getResource("random-connector-1.0-SNAPSHOT.jar"))
                                  .getPath()
    );

    Job job = createJetMember().newJob(pipeline, jobConfig);

    sleepAtLeastSeconds(5);

    try {
        job.join();
        fail("Job should have completed with an AssertionCompletedException, but completed normally");
    } catch (CompletionException e) {
        String errorMsg = e.getCause().getMessage();
        assertTrue("Job was expected to complete with AssertionCompletedException, but completed with: "
                + e.getCause(), errorMsg.contains(AssertionCompletedException.class.getName()));
    }
}

Example #3

Source File: DebeziumCDCWithJet.java From hazelcast-jet-demos with Apache License 2.0

5 votes

public static void main(String[] args) {
    JetInstance jet = JetBootstrap.getInstance();

    Configuration configuration = Configuration
            .create()
            .with("name", "mysql-demo-connector")
            .with("connector.class", "io.debezium.connector.mysql.MySqlConnector")
            /* begin connector properties */
            .with("database.hostname", "mysql")
            .with("database.port", "3306")
            .with("database.user", "debezium")
            .with("database.password", "dbz")
            .with("database.server.id", "184054")
            .with("database.server.name", "dbserver1")
            .with("database.whitelist", "inventory")
            .with("database.history.hazelcast.list.name", "test")
            .with("snapshot.mode", "schema_only")
            .build();

    Pipeline p = Pipeline.create();

    p.readFrom(DebeziumSources.cdc(configuration))
     .withoutTimestamps()
     .map(sourceRecord -> {
         String keyString = Values.convertToString(sourceRecord.keySchema(), sourceRecord.key());
         String valueString = Values.convertToString(sourceRecord.valueSchema(), sourceRecord.value());
         return Tuple2.tuple2(keyString, valueString);
     })
     .writeTo(Sinks.logger());

    jet.newJob(p).join();
}

Example #4

Source File: SetTypeDeserializer.java From debezium-incubator with Apache License 2.0

5 votes

@Override
@SuppressWarnings("unchecked")
public Object deserialize(AbstractType<?> abstractType, ByteBuffer bb) {
    Set<?> deserializedSet = (Set<?>) super.deserialize(abstractType, bb);
    List<?> deserializedList = (new ArrayList<>(deserializedSet));
    return Values.convertToList(getSchemaBuilder(abstractType).build(), deserializedList);
}

Example #5

Source File: CassandraTypeDeserializerTest.java From debezium-incubator with Apache License 2.0

5 votes

@Test
public void testTimeUUIDType() {
    UUID timeUUID = UUID.randomUUID();
    String expectedFixedUUID = Values.convertToString(CassandraTypeKafkaSchemaBuilders.UUID_TYPE, UuidUtil.asBytes(timeUUID));

    ByteBuffer serializedTimeUUID = TimeUUIDType.instance.decompose(timeUUID);

    Object deserializedTimeUUID = CassandraTypeDeserializer.deserialize(TimeUUIDType.instance, serializedTimeUUID);

    Assert.assertEquals(expectedFixedUUID, deserializedTimeUUID);
}

Example #6

Source File: CassandraTypeDeserializerTest.java From debezium-incubator with Apache License 2.0

5 votes

@Test
public void testUUIDType() {
    UUID uuid = UUID.randomUUID();
    String expectedFixedUUID = Values.convertToString(CassandraTypeKafkaSchemaBuilders.UUID_TYPE, UuidUtil.asBytes(uuid));

    ByteBuffer serializedUUID = UUIDType.instance.decompose(uuid);

    Object deserializedUUID = CassandraTypeDeserializer.deserialize(UUIDType.instance, serializedUUID);

    Assert.assertEquals(expectedFixedUUID, deserializedUUID);
}

Example #7