Java Code Examples for org.apache.flink.streaming.connectors.kafka.partitioner.FlinkFixedPartitioner

The following examples show how to use org.apache.flink.streaming.connectors.kafka.partitioner.FlinkFixedPartitioner. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
@SuppressWarnings("unchecked")
private Optional<FlinkKafkaPartitioner<Row>> getFlinkKafkaPartitioner(DescriptorProperties descriptorProperties) {
	return descriptorProperties
		.getOptionalString(CONNECTOR_SINK_PARTITIONER)
		.flatMap((String partitionerString) -> {
			switch (partitionerString) {
				case CONNECTOR_SINK_PARTITIONER_VALUE_FIXED:
					return Optional.of(new FlinkFixedPartitioner<>());
				case CONNECTOR_SINK_PARTITIONER_VALUE_ROUND_ROBIN:
					return Optional.empty();
				case CONNECTOR_SINK_PARTITIONER_VALUE_CUSTOM:
					final Class<? extends FlinkKafkaPartitioner> partitionerClass =
						descriptorProperties.getClass(CONNECTOR_SINK_PARTITIONER_CLASS, FlinkKafkaPartitioner.class);
					return Optional.of((FlinkKafkaPartitioner<Row>) InstantiationUtil.instantiate(partitionerClass));
				default:
					throw new TableException("Unsupported sink partitioner. Validator should have checked that.");
			}
		});
}
 
Example 2
Source Project: Flink-CEPplus   Source File: FlinkFixedPartitionerTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Test for when there are more sinks than partitions.
 * <pre>
 *   		Flink Sinks:		Kafka Partitions
 * 			1	---------------->	1
 * 			2   --------------/
 * 			3   -------------/
 * 			4	------------/
 * </pre>
 */
@Test
public void testMoreFlinkThanBrokers() {
	FlinkFixedPartitioner<String> part = new FlinkFixedPartitioner<>();

	int[] partitions = new int[]{0};

	part.open(0, 4);
	Assert.assertEquals(0, part.partition("abc1", null, null, null, partitions));

	part.open(1, 4);
	Assert.assertEquals(0, part.partition("abc2", null, null, null, partitions));

	part.open(2, 4);
	Assert.assertEquals(0, part.partition("abc3", null, null, null, partitions));
	Assert.assertEquals(0, part.partition("abc3", null, null, null, partitions)); // check if it is changing ;)

	part.open(3, 4);
	Assert.assertEquals(0, part.partition("abc4", null, null, null, partitions));
}
 
Example 3
Source Project: flink   Source File: KafkaTableSourceSinkFactoryBase.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
private Optional<FlinkKafkaPartitioner<Row>> getFlinkKafkaPartitioner(DescriptorProperties descriptorProperties) {
	return descriptorProperties
		.getOptionalString(CONNECTOR_SINK_PARTITIONER)
		.flatMap((String partitionerString) -> {
			switch (partitionerString) {
				case CONNECTOR_SINK_PARTITIONER_VALUE_FIXED:
					return Optional.of(new FlinkFixedPartitioner<>());
				case CONNECTOR_SINK_PARTITIONER_VALUE_ROUND_ROBIN:
					return Optional.empty();
				case CONNECTOR_SINK_PARTITIONER_VALUE_CUSTOM:
					final Class<? extends FlinkKafkaPartitioner> partitionerClass =
						descriptorProperties.getClass(CONNECTOR_SINK_PARTITIONER_CLASS, FlinkKafkaPartitioner.class);
					return Optional.of((FlinkKafkaPartitioner<Row>) InstantiationUtil.instantiate(partitionerClass));
				default:
					throw new TableException("Unsupported sink partitioner. Validator should have checked that.");
			}
		});
}
 
Example 4
Source Project: flink   Source File: FlinkFixedPartitionerTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Test for when there are more sinks than partitions.
 * <pre>
 *   		Flink Sinks:		Kafka Partitions
 * 			1	---------------->	1
 * 			2   --------------/
 * 			3   -------------/
 * 			4	------------/
 * </pre>
 */
@Test
public void testMoreFlinkThanBrokers() {
	FlinkFixedPartitioner<String> part = new FlinkFixedPartitioner<>();

	int[] partitions = new int[]{0};

	part.open(0, 4);
	Assert.assertEquals(0, part.partition("abc1", null, null, null, partitions));

	part.open(1, 4);
	Assert.assertEquals(0, part.partition("abc2", null, null, null, partitions));

	part.open(2, 4);
	Assert.assertEquals(0, part.partition("abc3", null, null, null, partitions));
	Assert.assertEquals(0, part.partition("abc3", null, null, null, partitions)); // check if it is changing ;)

	part.open(3, 4);
	Assert.assertEquals(0, part.partition("abc4", null, null, null, partitions));
}
 
Example 5
Source Project: flink   Source File: KafkaTableSourceSinkFactoryBase.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
private Optional<FlinkKafkaPartitioner<Row>> getFlinkKafkaPartitioner(DescriptorProperties descriptorProperties) {
	return descriptorProperties
		.getOptionalString(CONNECTOR_SINK_PARTITIONER)
		.flatMap((String partitionerString) -> {
			switch (partitionerString) {
				case CONNECTOR_SINK_PARTITIONER_VALUE_FIXED:
					return Optional.of(new FlinkFixedPartitioner<>());
				case CONNECTOR_SINK_PARTITIONER_VALUE_ROUND_ROBIN:
					return Optional.empty();
				case CONNECTOR_SINK_PARTITIONER_VALUE_CUSTOM:
					final Class<? extends FlinkKafkaPartitioner> partitionerClass =
						descriptorProperties.getClass(CONNECTOR_SINK_PARTITIONER_CLASS, FlinkKafkaPartitioner.class);
					return Optional.of((FlinkKafkaPartitioner<Row>) InstantiationUtil.instantiate(partitionerClass));
				default:
					throw new TableException("Unsupported sink partitioner. Validator should have checked that.");
			}
		});
}
 
Example 6
Source Project: flink   Source File: FlinkFixedPartitionerTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Test for when there are more sinks than partitions.
 * <pre>
 *   		Flink Sinks:		Kafka Partitions
 * 			1	---------------->	1
 * 			2   --------------/
 * 			3   -------------/
 * 			4	------------/
 * </pre>
 */
@Test
public void testMoreFlinkThanBrokers() {
	FlinkFixedPartitioner<String> part = new FlinkFixedPartitioner<>();

	int[] partitions = new int[]{0};

	part.open(0, 4);
	Assert.assertEquals(0, part.partition("abc1", null, null, null, partitions));

	part.open(1, 4);
	Assert.assertEquals(0, part.partition("abc2", null, null, null, partitions));

	part.open(2, 4);
	Assert.assertEquals(0, part.partition("abc3", null, null, null, partitions));
	Assert.assertEquals(0, part.partition("abc3", null, null, null, partitions)); // check if it is changing ;)

	part.open(3, 4);
	Assert.assertEquals(0, part.partition("abc4", null, null, null, partitions));
}
 
Example 7
Source Project: Flink-CEPplus   Source File: KafkaTestEnvironmentImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public <T> DataStreamSink<T> writeToKafkaWithTimestamps(DataStream<T> stream, String topic, KeyedSerializationSchema<T> serSchema, Properties props) {
	FlinkKafkaProducer011<T> prod = new FlinkKafkaProducer011<>(
		topic, serSchema, props, Optional.of(new FlinkFixedPartitioner<>()), producerSemantic, FlinkKafkaProducer011.DEFAULT_KAFKA_PRODUCERS_POOL_SIZE);

	prod.setWriteTimestampToKafka(true);

	return stream.addSink(prod);
}
 
Example 8
Source Project: Flink-CEPplus   Source File: KafkaTest.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public List<Descriptor> descriptors() {
	final Descriptor earliestDesc =
		new Kafka()
			.version("0.8")
			.startFromEarliest()
			.topic("WhateverTopic");

	final Descriptor specificOffsetsDesc =
		new Kafka()
			.version("0.11")
			.topic("MyTable")
			.startFromSpecificOffset(0, 42L)
			.startFromSpecificOffset(1, 300L)
			.property("zookeeper.stuff", "12")
			.property("kafka.stuff", "42");

	final Map<Integer, Long> offsets = new HashMap<>();
	offsets.put(0, 42L);
	offsets.put(1, 300L);

	final Properties properties = new Properties();
	properties.put("zookeeper.stuff", "12");
	properties.put("kafka.stuff", "42");

	final Descriptor specificOffsetsMapDesc =
		new Kafka()
			.version("0.11")
			.topic("MyTable")
			.startFromSpecificOffsets(offsets)
			.properties(properties)
			.sinkPartitionerCustom(FlinkFixedPartitioner.class);

	return Arrays.asList(earliestDesc, specificOffsetsDesc, specificOffsetsMapDesc);
}
 
Example 9
Source Project: Flink-CEPplus   Source File: FlinkFixedPartitionerTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests for when there are more partitions than sinks.
 * <pre>
 * 		Flink Sinks:		Kafka Partitions
 * 			1	---------------->	1
 * 			2	---------------->	2
 * 									3
 * 									4
 * 									5
 *
 * </pre>
 */
@Test
public void testFewerPartitions() {
	FlinkFixedPartitioner<String> part = new FlinkFixedPartitioner<>();

	int[] partitions = new int[]{0, 1, 2, 3, 4};
	part.open(0, 2);
	Assert.assertEquals(0, part.partition("abc1", null, null, null, partitions));
	Assert.assertEquals(0, part.partition("abc1", null, null, null, partitions));

	part.open(1, 2);
	Assert.assertEquals(1, part.partition("abc1", null, null, null, partitions));
	Assert.assertEquals(1, part.partition("abc1", null, null, null, partitions));
}
 
Example 10
Source Project: flink   Source File: FlinkFixedPartitionerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testMixedCase() {
	FlinkFixedPartitioner<String> part = new FlinkFixedPartitioner<>();
	int[] partitions = new int[]{0, 1};

	part.open(0, 3);
	Assert.assertEquals(0, part.partition("abc1", null, null, null, partitions));

	part.open(1, 3);
	Assert.assertEquals(1, part.partition("abc1", null, null, null, partitions));

	part.open(2, 3);
	Assert.assertEquals(0, part.partition("abc1", null, null, null, partitions));

}
 
Example 11
Source Project: flink   Source File: KafkaTest.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public List<Descriptor> descriptors() {
	final Descriptor earliestDesc =
		new Kafka()
			.version("0.8")
			.startFromEarliest()
			.topic("WhateverTopic");

	final Descriptor specificOffsetsDesc =
		new Kafka()
			.version("0.11")
			.topic("MyTable")
			.startFromSpecificOffset(0, 42L)
			.startFromSpecificOffset(1, 300L)
			.property("zookeeper.stuff", "12")
			.property("kafka.stuff", "42");

	final Map<Integer, Long> offsets = new HashMap<>();
	offsets.put(0, 42L);
	offsets.put(1, 300L);

	final Properties properties = new Properties();
	properties.put("zookeeper.stuff", "12");
	properties.put("kafka.stuff", "42");

	final Descriptor specificOffsetsMapDesc =
		new Kafka()
			.version("0.11")
			.topic("MyTable")
			.startFromSpecificOffsets(offsets)
			.properties(properties)
			.sinkPartitionerCustom(FlinkFixedPartitioner.class);

	return Arrays.asList(earliestDesc, specificOffsetsDesc, specificOffsetsMapDesc);
}
 
Example 12
Source Project: flink   Source File: FlinkFixedPartitionerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testMixedCase() {
	FlinkFixedPartitioner<String> part = new FlinkFixedPartitioner<>();
	int[] partitions = new int[]{0, 1};

	part.open(0, 3);
	Assert.assertEquals(0, part.partition("abc1", null, null, null, partitions));

	part.open(1, 3);
	Assert.assertEquals(1, part.partition("abc1", null, null, null, partitions));

	part.open(2, 3);
	Assert.assertEquals(0, part.partition("abc1", null, null, null, partitions));

}
 
Example 13
Source Project: PoseidonX   Source File: FlinkTaskInstanceBusiness.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * 构建 FlinkKafka09OutputComponent 对应的业务逻辑
 * @param flinkComponent
 */
private void dealFlinkKafka09OutputComponent(FlinkKafka09OutputComponent flinkComponent) {

    Properties kafkaProperties = new Properties();
    kafkaProperties.setProperty("bootstrap.servers",flinkComponent.getBrokerHostPorts());

    Kafka09JsonTableSink kafkaTableSink = new Kafka09JsonTableSink(
            flinkComponent.getTopic(),
            kafkaProperties,
            new FlinkFixedPartitioner());

    tableSinkMap.put(flinkComponent.getName(),kafkaTableSink);
}
 
Example 14
Source Project: df_data_service   Source File: FlinkAvroSQLClient.java    License: Apache License 2.0 5 votes vote down vote up
public static void tcFlinkAvroSQL(String KafkaServerHostPort, String SchemaRegistryHostPort,
                                  String srcTopic, String targetTopic,
                                  String consumerGroupId, String sinkKeys, String sqlState) {

    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    StreamTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);

    Properties properties = new Properties();
    properties.setProperty(ConstantApp.PK_KAFKA_HOST_PORT.replace("_", "."), KafkaServerHostPort);
    properties.setProperty(ConstantApp.PK_KAFKA_CONSUMER_GROURP, consumerGroupId);
    properties.setProperty(ConstantApp.PK_KAFKA_SCHEMA_REGISTRY_HOST_PORT.replace("_", "."), SchemaRegistryHostPort);
    properties.setProperty(ConstantApp.PK_FLINK_TABLE_SINK_KEYS, sinkKeys);

    String[] srcTopicList = srcTopic.split(",");
    for (int i = 0; i < srcTopicList.length; i++) {
        properties.setProperty(ConstantApp.PK_SCHEMA_SUB_INPUT, srcTopicList[i]);
        properties.setProperty(ConstantApp.PK_SCHEMA_ID_INPUT, SchemaRegistryClient.getLatestSchemaIDFromProperty(properties, ConstantApp.PK_SCHEMA_SUB_INPUT) + "");
        properties.setProperty(ConstantApp.PK_SCHEMA_STR_INPUT, SchemaRegistryClient.getLatestSchemaFromProperty(properties, ConstantApp.PK_SCHEMA_SUB_INPUT).toString());
        tableEnv.registerTableSource(srcTopicList[i], new Kafka010AvroTableSource(srcTopicList[i], properties));
    }

    try {
        Table result = tableEnv.sql(sqlState);
        SchemaRegistryClient.addSchemaFromTableResult(SchemaRegistryHostPort, targetTopic, result);
        // For old producer, we need to create topic-value subject as well
        SchemaRegistryClient.addSchemaFromTableResult(SchemaRegistryHostPort, targetTopic + "-value", result);

        // delivered properties for sink
        properties.setProperty(ConstantApp.PK_SCHEMA_SUB_OUTPUT, targetTopic);
        properties.setProperty(ConstantApp.PK_SCHEMA_ID_OUTPUT, SchemaRegistryClient.getLatestSchemaIDFromProperty(properties, ConstantApp.PK_SCHEMA_SUB_OUTPUT) + "");
        properties.setProperty(ConstantApp.PK_SCHEMA_STR_OUTPUT, SchemaRegistryClient.getLatestSchemaFromProperty(properties, ConstantApp.PK_SCHEMA_SUB_OUTPUT).toString());

        Kafka09AvroTableSink avro_sink =
                new Kafka09AvroTableSink(targetTopic, properties, new FlinkFixedPartitioner());
        result.writeToSink(avro_sink);
        env.execute("DF_FlinkSQL_Client_" + srcTopic + "-" + targetTopic);
    } catch (Exception e) {
        e.printStackTrace();
    }
}
 
Example 15
Source Project: df_data_service   Source File: UnitTestSuiteFlink.java    License: Apache License 2.0 5 votes vote down vote up
public static void testFlinkAvroSQLJson() {
    System.out.println("TestCase_Test Avro SQL to Json Sink");
    final String STATIC_USER_SCHEMA = "{"
            + "\"type\":\"record\","
            + "\"name\":\"myrecord\","
            + "\"fields\":["
            + "  { \"name\":\"symbol\", \"type\":\"string\" },"
            + "  { \"name\":\"name\", \"type\":\"string\" },"
            + "  { \"name\":\"exchangecode\", \"type\":\"string\" }"
            + "]}";

    String jarPath = DFInitService.class.getProtectionDomain().getCodeSource().getLocation().getPath();
    DFRemoteStreamEnvironment env = new DFRemoteStreamEnvironment("localhost", 6123, jarPath)
            .setParallelism(1);
    StreamTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
    Properties properties = new Properties();
    properties.setProperty("bootstrap.servers", "localhost:9092");
    properties.setProperty("group.id", "consumer_test");
    properties.setProperty("schema.subject", "test-value");
    properties.setProperty("schema.registry", "localhost:8081");
    properties.setProperty("useAvro", "avro");
    properties.setProperty("static.avro.schema",
            SchemaRegistryClient.getSchemaFromRegistry("http://localhost:8081", "test-value", "latest").toString());

    try {
        HashMap<String, String> hm = new HashMap<>();
        Kafka09AvroTableSource kafkaAvroTableSource =  new Kafka09AvroTableSource("test", properties);
        tableEnv.registerTableSource("Orders", kafkaAvroTableSource);

        Table result = tableEnv.sql("SELECT name, symbol, exchangecode FROM Orders");
        //Kafka09JsonTableSink json_sink = new Kafka09JsonTableSink ("test_json", properties, new FlinkFixedPartitioner());
        Kafka09AvroTableSink json_sink = new Kafka09AvroTableSink ("test_json", properties, new FlinkFixedPartitioner());

        // write the result Table to the TableSink
        result.writeToSink(json_sink);
        env.executeWithDFObj("Flink AVRO SQL KAFKA Test", new DFJobPOPJ().setJobConfig(hm) );
    } catch (Exception e) {
        e.printStackTrace();
    }
}
 
Example 16
Source Project: flink   Source File: KafkaDynamicTableFactoryTestBase.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testTableSink() {
	final DataType consumedDataType = SINK_SCHEMA.toPhysicalRowDataType();
	EncodingFormat<SerializationSchema<RowData>> encodingFormat =
			new TestFormatFactory.EncodingFormatMock(",");

	// Construct table sink using options and table sink factory.
	ObjectIdentifier objectIdentifier = ObjectIdentifier.of(
			"default",
			"default",
			"sinkTable");
	final CatalogTable sinkTable = createKafkaSinkCatalogTable();
	final DynamicTableSink actualSink = FactoryUtil.createTableSink(
			null,
			objectIdentifier,
			sinkTable,
			new Configuration(),
			Thread.currentThread().getContextClassLoader());

	final DynamicTableSink expectedSink = getExpectedSink(
			consumedDataType,
			TOPIC,
			KAFKA_PROPERTIES,
			Optional.of(new FlinkFixedPartitioner<>()),
			encodingFormat);
	assertEquals(expectedSink, actualSink);

	// Test sink format.
	final KafkaDynamicSinkBase actualKafkaSink = (KafkaDynamicSinkBase) actualSink;
	assertEquals(encodingFormat, actualKafkaSink.encodingFormat);

	// Test kafka producer.
	DynamicTableSink.SinkRuntimeProvider provider =
			actualKafkaSink.getSinkRuntimeProvider(new SinkRuntimeProviderContext(false));
	assertThat(provider, instanceOf(SinkFunctionProvider.class));
	final SinkFunctionProvider sinkFunctionProvider = (SinkFunctionProvider) provider;
	final SinkFunction<RowData> sinkFunction = sinkFunctionProvider.createSinkFunction();
	assertThat(sinkFunction, instanceOf(getExpectedProducerClass()));
}
 
Example 17
Source Project: flink   Source File: FlinkFixedPartitionerTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests for when there are more partitions than sinks.
 * <pre>
 * 		Flink Sinks:		Kafka Partitions
 * 			1	---------------->	1
 * 			2	---------------->	2
 * 									3
 * 									4
 * 									5
 *
 * </pre>
 */
@Test
public void testFewerPartitions() {
	FlinkFixedPartitioner<String> part = new FlinkFixedPartitioner<>();

	int[] partitions = new int[]{0, 1, 2, 3, 4};
	part.open(0, 2);
	Assert.assertEquals(0, part.partition("abc1", null, null, null, partitions));
	Assert.assertEquals(0, part.partition("abc1", null, null, null, partitions));

	part.open(1, 2);
	Assert.assertEquals(1, part.partition("abc1", null, null, null, partitions));
	Assert.assertEquals(1, part.partition("abc1", null, null, null, partitions));
}
 
Example 18
Source Project: flink   Source File: KafkaTest.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public List<Map<String, String>> properties() {
	final Map<String, String> props1 = new HashMap<>();
	props1.put("connector.property-version", "1");
	props1.put("connector.type", "kafka");
	props1.put("connector.version", "0.8");
	props1.put("connector.topic", "WhateverTopic");
	props1.put("connector.startup-mode", "earliest-offset");

	final Map<String, String> props2 = new HashMap<>();
	props2.put("connector.property-version", "1");
	props2.put("connector.type", "kafka");
	props2.put("connector.version", "0.11");
	props2.put("connector.topic", "MyTable");
	props2.put("connector.startup-mode", "specific-offsets");
	props2.put("connector.specific-offsets", "partition:0,offset:42;partition:1,offset:300");
	props2.put("connector.properties.zookeeper.stuff", "12");
	props2.put("connector.properties.kafka.stuff", "42");

	final Map<String, String> props3 = new HashMap<>();
	props3.put("connector.property-version", "1");
	props3.put("connector.type", "kafka");
	props3.put("connector.version", "0.11");
	props3.put("connector.topic", "MyTable");
	props3.put("connector.startup-mode", "specific-offsets");
	props3.put("connector.specific-offsets", "partition:0,offset:42;partition:1,offset:300");
	props3.put("connector.properties.zookeeper.stuff", "12");
	props3.put("connector.properties.kafka.stuff", "42");
	props3.put("connector.sink-partitioner", "custom");
	props3.put("connector.sink-partitioner-class", FlinkFixedPartitioner.class.getName());

	final Map<String, String> props4 = new HashMap<>();
	props4.put("connector.property-version", "1");
	props4.put("connector.type", "kafka");
	props4.put("connector.version", "0.11");
	props4.put("connector.topic", "MyTable");
	props4.put("connector.startup-mode", "timestamp");
	props4.put("connector.startup-timestamp-millis", "1577014729000");

	return Arrays.asList(props1, props2, props3, props4);
}
 
Example 19
Source Project: Flink-CEPplus   Source File: KafkaTest.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public List<Map<String, String>> properties() {
	final Map<String, String> props1 = new HashMap<>();
	props1.put("connector.property-version", "1");
	props1.put("connector.type", "kafka");
	props1.put("connector.version", "0.8");
	props1.put("connector.topic", "WhateverTopic");
	props1.put("connector.startup-mode", "earliest-offset");

	final Map<String, String> props2 = new HashMap<>();
	props2.put("connector.property-version", "1");
	props2.put("connector.type", "kafka");
	props2.put("connector.version", "0.11");
	props2.put("connector.topic", "MyTable");
	props2.put("connector.startup-mode", "specific-offsets");
	props2.put("connector.specific-offsets.0.partition", "0");
	props2.put("connector.specific-offsets.0.offset", "42");
	props2.put("connector.specific-offsets.1.partition", "1");
	props2.put("connector.specific-offsets.1.offset", "300");
	props2.put("connector.properties.0.key", "zookeeper.stuff");
	props2.put("connector.properties.0.value", "12");
	props2.put("connector.properties.1.key", "kafka.stuff");
	props2.put("connector.properties.1.value", "42");

	final Map<String, String> props3 = new HashMap<>();
	props3.put("connector.property-version", "1");
	props3.put("connector.type", "kafka");
	props3.put("connector.version", "0.11");
	props3.put("connector.topic", "MyTable");
	props3.put("connector.startup-mode", "specific-offsets");
	props3.put("connector.specific-offsets.0.partition", "0");
	props3.put("connector.specific-offsets.0.offset", "42");
	props3.put("connector.specific-offsets.1.partition", "1");
	props3.put("connector.specific-offsets.1.offset", "300");
	props3.put("connector.properties.0.key", "zookeeper.stuff");
	props3.put("connector.properties.0.value", "12");
	props3.put("connector.properties.1.key", "kafka.stuff");
	props3.put("connector.properties.1.value", "42");
	props3.put("connector.sink-partitioner", "custom");
	props3.put("connector.sink-partitioner-class", FlinkFixedPartitioner.class.getName());

	return Arrays.asList(props1, props2, props3);
}
 
Example 20
Source Project: flink   Source File: KafkaTableSourceSinkFactoryTestBase.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testTableSinkWithLegacyProperties() {
	// prepare parameters for Kafka table sink
	final TableSchema schema = TableSchema.builder()
		.field(FRUIT_NAME, DataTypes.STRING())
		.field(COUNT, DataTypes.DECIMAL(10, 4))
		.field(EVENT_TIME, DataTypes.TIMESTAMP(3))
		.build();

	final KafkaTableSinkBase expected = getExpectedKafkaTableSink(
		schema,
		TOPIC,
		KAFKA_PROPERTIES,
		Optional.of(new FlinkFixedPartitioner<>()),
		new TestSerializationSchema(schema.toRowType()));

	// construct table sink using descriptors and table sink factory
	final Map<String, String> legacyPropertiesMap = new HashMap<>();
	legacyPropertiesMap.putAll(createKafkaSinkProperties());

	// use legacy properties
	legacyPropertiesMap.remove("connector.specific-offsets");
	legacyPropertiesMap.remove("connector.properties.bootstrap.servers");
	legacyPropertiesMap.remove("connector.properties.group.id");

	// keep compatible with a specified update-mode
	legacyPropertiesMap.put("update-mode", "append");

	// legacy properties for specific-offsets and properties
	legacyPropertiesMap.put("connector.specific-offsets.0.partition", "0");
	legacyPropertiesMap.put("connector.specific-offsets.0.offset", "100");
	legacyPropertiesMap.put("connector.specific-offsets.1.partition", "1");
	legacyPropertiesMap.put("connector.specific-offsets.1.offset", "123");
	legacyPropertiesMap.put("connector.properties.0.key", "bootstrap.servers");
	legacyPropertiesMap.put("connector.properties.0.value", "dummy");
	legacyPropertiesMap.put("connector.properties.1.key", "group.id");
	legacyPropertiesMap.put("connector.properties.1.value", "dummy");

	final TableSink<?> actualSink = TableFactoryService.find(StreamTableSinkFactory.class, legacyPropertiesMap)
		.createStreamTableSink(legacyPropertiesMap);

	assertEquals(expected, actualSink);

	// test Kafka producer
	final KafkaTableSinkBase actualKafkaSink = (KafkaTableSinkBase) actualSink;
	final DataStreamMock streamMock = new DataStreamMock(new StreamExecutionEnvironmentMock(), schema.toRowType());
	actualKafkaSink.consumeDataStream(streamMock);
	assertTrue(getExpectedFlinkKafkaProducer().isAssignableFrom(streamMock.sinkFunction.getClass()));
}
 
Example 21
Source Project: flink   Source File: KafkaTest.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public List<Descriptor> descriptors() {
	final Descriptor earliestDesc =
		new Kafka()
			.version("0.8")
			.startFromEarliest()
			.topic("WhateverTopic");

	final Descriptor specificOffsetsDesc =
		new Kafka()
			.version("0.11")
			.topic("MyTable")
			.startFromSpecificOffset(0, 42L)
			.startFromSpecificOffset(1, 300L)
			.property("zookeeper.stuff", "12")
			.property("kafka.stuff", "42");

	final Map<Integer, Long> offsets = new HashMap<>();
	offsets.put(0, 42L);
	offsets.put(1, 300L);

	final Properties properties = new Properties();
	properties.put("zookeeper.stuff", "12");
	properties.put("kafka.stuff", "42");

	final Descriptor specificOffsetsMapDesc =
		new Kafka()
			.version("0.11")
			.topic("MyTable")
			.startFromSpecificOffsets(offsets)
			.properties(properties)
			.sinkPartitionerCustom(FlinkFixedPartitioner.class);

	final Descriptor timestampDesc =
		new Kafka()
			.version("0.11")
			.topic("MyTable")
			.startFromTimestamp(1577014729000L);

	return Arrays.asList(earliestDesc, specificOffsetsDesc, specificOffsetsMapDesc, timestampDesc);
}
 
Example 22
Source Project: flink   Source File: KafkaTest.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public List<Map<String, String>> properties() {
	final Map<String, String> props1 = new HashMap<>();
	props1.put("connector.property-version", "1");
	props1.put("connector.type", "kafka");
	props1.put("connector.version", "0.8");
	props1.put("connector.topic", "WhateverTopic");
	props1.put("connector.startup-mode", "earliest-offset");

	final Map<String, String> props2 = new HashMap<>();
	props2.put("connector.property-version", "1");
	props2.put("connector.type", "kafka");
	props2.put("connector.version", "0.11");
	props2.put("connector.topic", "MyTable");
	props2.put("connector.startup-mode", "specific-offsets");
	props2.put("connector.specific-offsets.0.partition", "0");
	props2.put("connector.specific-offsets.0.offset", "42");
	props2.put("connector.specific-offsets.1.partition", "1");
	props2.put("connector.specific-offsets.1.offset", "300");
	props2.put("connector.properties.0.key", "zookeeper.stuff");
	props2.put("connector.properties.0.value", "12");
	props2.put("connector.properties.1.key", "kafka.stuff");
	props2.put("connector.properties.1.value", "42");

	final Map<String, String> props3 = new HashMap<>();
	props3.put("connector.property-version", "1");
	props3.put("connector.type", "kafka");
	props3.put("connector.version", "0.11");
	props3.put("connector.topic", "MyTable");
	props3.put("connector.startup-mode", "specific-offsets");
	props3.put("connector.specific-offsets.0.partition", "0");
	props3.put("connector.specific-offsets.0.offset", "42");
	props3.put("connector.specific-offsets.1.partition", "1");
	props3.put("connector.specific-offsets.1.offset", "300");
	props3.put("connector.properties.0.key", "zookeeper.stuff");
	props3.put("connector.properties.0.value", "12");
	props3.put("connector.properties.1.key", "kafka.stuff");
	props3.put("connector.properties.1.value", "42");
	props3.put("connector.sink-partitioner", "custom");
	props3.put("connector.sink-partitioner-class", FlinkFixedPartitioner.class.getName());

	return Arrays.asList(props1, props2, props3);
}
 
Example 23
Source Project: flink   Source File: KafkaTableSourceSinkFactoryTestBase.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * This test can be unified with the corresponding source test once we have fixed FLINK-9870.
 */
@Test
public void testTableSink() {
	// prepare parameters for Kafka table sink

	final TableSchema schema = TableSchema.builder()
		.field(FRUIT_NAME, Types.STRING())
		.field(COUNT, Types.DECIMAL())
		.field(EVENT_TIME, Types.SQL_TIMESTAMP())
		.build();

	final KafkaTableSinkBase expected = getExpectedKafkaTableSink(
		schema,
		TOPIC,
		KAFKA_PROPERTIES,
		Optional.of(new FlinkFixedPartitioner<>()),
		new TestSerializationSchema(schema.toRowType()));

	// construct table sink using descriptors and table sink factory

	final TestTableDescriptor testDesc = new TestTableDescriptor(
			new Kafka()
				.version(getKafkaVersion())
				.topic(TOPIC)
				.properties(KAFKA_PROPERTIES)
				.sinkPartitionerFixed()
				.startFromSpecificOffsets(OFFSETS)) // test if they accepted although not needed
		.withFormat(new TestTableFormat())
		.withSchema(
			new Schema()
				.field(FRUIT_NAME, Types.STRING())
				.field(COUNT, Types.DECIMAL())
				.field(EVENT_TIME, Types.SQL_TIMESTAMP()))
		.inAppendMode();

	final Map<String, String> propertiesMap = testDesc.toProperties();
	final TableSink<?> actualSink = TableFactoryService.find(StreamTableSinkFactory.class, propertiesMap)
		.createStreamTableSink(propertiesMap);

	assertEquals(expected, actualSink);

	// test Kafka producer
	final KafkaTableSinkBase actualKafkaSink = (KafkaTableSinkBase) actualSink;
	final DataStreamMock streamMock = new DataStreamMock(new StreamExecutionEnvironmentMock(), schema.toRowType());
	actualKafkaSink.emitDataStream(streamMock);
	assertTrue(getExpectedFlinkKafkaProducer().isAssignableFrom(streamMock.sinkFunction.getClass()));
}
 
Example 24
Source Project: df_data_service   Source File: FlinkAvroTableAPIClient.java    License: Apache License 2.0 4 votes vote down vote up
public static void tcFlinkAvroTableAPI(String KafkaServerHostPort, String SchemaRegistryHostPort,
                                  String srcTopic, String targetTopic,
                                  String consumerGroupId, String sinkKeys, String transScript) {

    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    StreamTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);

    Properties properties = new Properties();
    properties.setProperty(ConstantApp.PK_KAFKA_HOST_PORT.replace("_", "."), KafkaServerHostPort);
    properties.setProperty(ConstantApp.PK_KAFKA_CONSUMER_GROURP, consumerGroupId);
    properties.setProperty(ConstantApp.PK_KAFKA_SCHEMA_REGISTRY_HOST_PORT.replace("_", "."), SchemaRegistryHostPort);
    properties.setProperty(ConstantApp.PK_FLINK_TABLE_SINK_KEYS, sinkKeys);

    String[] srcTopicList = srcTopic.split(",");
    for (int i = 0; i < srcTopicList.length; i++) {
        properties.setProperty(ConstantApp.PK_SCHEMA_SUB_INPUT, srcTopicList[i]);
        properties.setProperty(ConstantApp.PK_SCHEMA_ID_INPUT, SchemaRegistryClient.getLatestSchemaIDFromProperty(properties, ConstantApp.PK_SCHEMA_SUB_INPUT) + "");
        properties.setProperty(ConstantApp.PK_SCHEMA_STR_INPUT, SchemaRegistryClient.getLatestSchemaFromProperty(properties, ConstantApp.PK_SCHEMA_SUB_INPUT).toString());
        tableEnv.registerTableSource(srcTopic, new Kafka010AvroTableSource(srcTopicList[i], properties));
    }

    try {
        Table result;
        Table ingest = tableEnv.scan(srcTopic);
        String className = "dynamic.FlinkScript";
        String header = "package dynamic;\n" +
                "import org.apache.flink.table.api.Table;\n" +
                "import com.datafibers.util.*;\n";
        String javaCode = header +
                "public class FlinkScript implements DynamicRunner {\n" +
                "@Override \n" +
                "    public Table transTableObj(Table tbl) {\n" +
                "try {" +
                "return tbl." + transScript + ";\n" +
                "} catch (Exception e) {" +
                "};" +
                "return null;}}";
        // Dynamic code generation
        Class aClass = CompilerUtils.CACHED_COMPILER.loadFromJava(className, javaCode);
        DynamicRunner runner = (DynamicRunner) aClass.newInstance();
        result = runner.transTableObj(ingest);

        SchemaRegistryClient.addSchemaFromTableResult(SchemaRegistryHostPort, targetTopic, result);
        // delivered properties for sink
        properties.setProperty(ConstantApp.PK_SCHEMA_SUB_OUTPUT, targetTopic);
        properties.setProperty(ConstantApp.PK_SCHEMA_ID_OUTPUT, SchemaRegistryClient.getLatestSchemaIDFromProperty(properties, ConstantApp.PK_SCHEMA_SUB_OUTPUT) + "");
        properties.setProperty(ConstantApp.PK_SCHEMA_STR_OUTPUT, SchemaRegistryClient.getLatestSchemaFromProperty(properties, ConstantApp.PK_SCHEMA_SUB_OUTPUT).toString());

        Kafka09AvroTableSink avro_sink =
                new Kafka09AvroTableSink(targetTopic, properties, new FlinkFixedPartitioner());
        result.writeToSink(avro_sink);
        env.execute("DF_FlinkTableAPI_Client_" + srcTopic + "-" + targetTopic);
    } catch (Exception e) {
        e.printStackTrace();
    }
}
 
Example 25
Source Project: df_data_service   Source File: TCFlinkAvroSQL.java    License: Apache License 2.0 4 votes vote down vote up
public static void tcFlinkAvroSQL(String SchemaRegistryHostPort, String srcTopic, String targetTopic, String sqlState) {
    System.out.println("tcFlinkAvroSQL");
    String resultFile = "testResult";

    String jarPath = "C:/Users/dadu/Coding/df_data_service/target/df-data-service-1.1-SNAPSHOT-fat.jar";
    //String jarPath = "/Users/will/Documents/Coding/GitHub/df_data_service/target/df-data-service-1.1-SNAPSHOT-fat.jar";
    StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", 6123, jarPath)
            .setParallelism(1);
    StreamTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);

    Properties properties = new Properties();
    properties.setProperty(ConstantApp.PK_KAFKA_HOST_PORT.replace("_", "."), "localhost:9092");
    properties.setProperty(ConstantApp.PK_KAFKA_CONSUMER_GROURP, "consumer_test");
    //properties.setProperty(ConstantApp.PK_SCHEMA_SUB_OUTPUT, "test");
    properties.setProperty(ConstantApp.PK_KAFKA_SCHEMA_REGISTRY_HOST_PORT.replace("_", "."), SchemaRegistryHostPort);
    properties.setProperty(ConstantApp.PK_FLINK_TABLE_SINK_KEYS, "symbol");

    String[] srcTopicList = srcTopic.split(",");
    for (int i = 0; i < srcTopicList.length; i++) {
        properties.setProperty(ConstantApp.PK_SCHEMA_SUB_INPUT, srcTopicList[i]);
        properties.setProperty(ConstantApp.PK_SCHEMA_ID_INPUT, SchemaRegistryClient.getLatestSchemaIDFromProperty(properties, ConstantApp.PK_SCHEMA_SUB_INPUT) + "");
        properties.setProperty(ConstantApp.PK_SCHEMA_STR_INPUT, SchemaRegistryClient.getLatestSchemaFromProperty(properties, ConstantApp.PK_SCHEMA_SUB_INPUT).toString());
        tableEnv.registerTableSource(srcTopicList[i], new Kafka010AvroTableSource(srcTopicList[i], properties));
    }

    try {
        Table result = tableEnv.sql(sqlState);
        result.printSchema();
        System.out.println("generated avro schema is = " + SchemaRegistryClient.tableAPIToAvroSchema(result, targetTopic));
        SchemaRegistryClient.addSchemaFromTableResult(SchemaRegistryHostPort, targetTopic, result);

        // delivered properties
        properties.setProperty(ConstantApp.PK_SCHEMA_SUB_OUTPUT, targetTopic);
        properties.setProperty(ConstantApp.PK_SCHEMA_ID_OUTPUT, SchemaRegistryClient.getLatestSchemaIDFromProperty(properties, ConstantApp.PK_SCHEMA_SUB_OUTPUT) + "");
        properties.setProperty(ConstantApp.PK_SCHEMA_STR_OUTPUT, SchemaRegistryClient.getLatestSchemaFromProperty(properties, ConstantApp.PK_SCHEMA_SUB_OUTPUT).toString());

        System.out.println(Paths.get(resultFile).toAbsolutePath());
        Kafka09AvroTableSink avro_sink =
                new Kafka09AvroTableSink(targetTopic, properties, new FlinkFixedPartitioner());
        result.writeToSink(avro_sink);
        //result.writeToSink(new CsvTableSink(resultFile, "|", 1, FileSystem.WriteMode.OVERWRITE));
        env.execute("tcFlinkAvroSQL");
    } catch (Exception e) {
        e.printStackTrace();
    }
}
 
Example 26
Source Project: df_data_service   Source File: UnitTestSuiteFlink.java    License: Apache License 2.0 4 votes vote down vote up
public static void testFlinkAvroScriptWithStaticSchema() {
    System.out.println("TestCase_Test Avro Table API Script with static Schema");

    final String STATIC_USER_SCHEMA = "{"
            + "\"type\":\"record\","
            + "\"name\":\"myrecord\","
            + "\"fields\":["
            + "  { \"name\":\"symbol\", \"type\":\"string\" },"
            + "  { \"name\":\"name\", \"type\":\"string\" },"
            + "  { \"name\":\"exchangecode\", \"type\":\"string\" }"
            + "]}";

    String jarPath = DFInitService.class.getProtectionDomain().getCodeSource().getLocation().getPath();
    StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", 6123, jarPath)
            .setParallelism(1);
    StreamTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
    Properties properties = new Properties();
    properties.setProperty("bootstrap.servers", "localhost:9092");
    properties.setProperty("group.id", "consumer_test");
    properties.setProperty("schema.subject", "test-value");
    properties.setProperty("schema.registry", "localhost:8081");
    properties.setProperty("static.avro.schema", STATIC_USER_SCHEMA);

    try {
        Kafka09AvroTableSource kafkaAvroTableSource =  new Kafka09AvroTableSource("test", properties);
        tableEnv.registerTableSource("Orders", kafkaAvroTableSource);

        Table ingest = tableEnv.scan("Orders");

        String className = "dynamic.FlinkScript";

        String header = "package dynamic;\n" +
                "import org.apache.flink.table.api.Table;\n" +
                "import com.datafibers.util.*;\n";

        String transScript = "select(\"name\")";

        String javaCode = header +
                "public class FlinkScript implements DynamicRunner {\n" +
                "@Override \n" +
                "    public Table transTableObj(Table tbl) {\n" +
                "try {" +
                "return tbl."+ transScript + ";" +
                "} catch (Exception e) {" +
                "};" +
                "return null;}}";

        // Dynamic code generation
        Class aClass = CompilerUtils.CACHED_COMPILER.loadFromJava(className, javaCode);
        DynamicRunner runner = (DynamicRunner) aClass.newInstance();
        Table result = runner.transTableObj(ingest);

        Kafka09AvroTableSink sink =
                new Kafka09AvroTableSink ("test_json", properties, new FlinkFixedPartitioner());
        // write the result Table to the TableSink
        result.writeToSink(sink);
        env.execute("Flink AVRO SQL KAFKA Test");
    } catch (Exception e) {
        e.printStackTrace();
    }
}
 
Example 27
Source Project: flink   Source File: FlinkKafkaProducer010.java    License: Apache License 2.0 3 votes vote down vote up
/**
 * Creates a FlinkKafkaProducer for a given topic. the sink produces a DataStream to
 * the topic.
 *
 * <p>This constructor allows writing timestamps to Kafka, it follow approach (b) (see above)
 *
 * @param inStream The stream to write to Kafka
 * @param topicId ID of the Kafka topic.
 * @param serializationSchema User defined (keyless) serialization schema.
 * @param producerConfig Properties with the producer configuration.
 *
 * @deprecated Use {@link #FlinkKafkaProducer010(String, KeyedSerializationSchema, Properties)}
 * and call {@link #setWriteTimestampToKafka(boolean)}.
 */
@Deprecated
public static <T> FlinkKafkaProducer010Configuration<T> writeToKafkaWithTimestamps(DataStream<T> inStream,
																				String topicId,
																				SerializationSchema<T> serializationSchema,
																				Properties producerConfig) {
	return writeToKafkaWithTimestamps(inStream, topicId, new KeyedSerializationSchemaWrapper<>(serializationSchema), producerConfig, new FlinkFixedPartitioner<T>());
}
 
Example 28
Source Project: Flink-CEPplus   Source File: FlinkKafkaProducer.java    License: Apache License 2.0 3 votes vote down vote up
/**
 * Creates a FlinkKafkaProducer for a given topic. The sink produces a DataStream to
 * the topic.
 *
 * @param brokerList
 *			Comma separated addresses of the brokers
 * @param topicId
 * 			ID of the Kafka topic.
 * @param serializationSchema
 * 			User defined (keyless) serialization schema.
 */
public FlinkKafkaProducer(String brokerList, String topicId, SerializationSchema<IN> serializationSchema) {
	this(
		topicId,
		new KeyedSerializationSchemaWrapper<>(serializationSchema),
		getPropertiesFromBrokerList(brokerList),
		Optional.of(new FlinkFixedPartitioner<IN>()));
}
 
Example 29
Source Project: Flink-CEPplus   Source File: FlinkKafkaProducer.java    License: Apache License 2.0 3 votes vote down vote up
/**
 * Creates a FlinkKafkaProducer for a given topic. The sink produces a DataStream to
 * the topic.
 *
 * <p>Using this constructor, the default {@link FlinkFixedPartitioner} will be used as
 * the partitioner. This default partitioner maps each sink subtask to a single Kafka
 * partition (i.e. all records received by a sink subtask will end up in the same
 * Kafka partition).
 *
 * <p>To use a custom partitioner, please use
 * {@link #FlinkKafkaProducer(String, SerializationSchema, Properties, Optional)} instead.
 *
 * @param topicId
 * 			ID of the Kafka topic.
 * @param serializationSchema
 * 			User defined key-less serialization schema.
 * @param producerConfig
 * 			Properties with the producer configuration.
 */
public FlinkKafkaProducer(String topicId, SerializationSchema<IN> serializationSchema, Properties producerConfig) {
	this(
		topicId,
		new KeyedSerializationSchemaWrapper<>(serializationSchema),
		producerConfig,
		Optional.of(new FlinkFixedPartitioner<IN>()));
}
 
Example 30
Source Project: Flink-CEPplus   Source File: FlinkKafkaProducer.java    License: Apache License 2.0 3 votes vote down vote up
/**
 * Creates a FlinkKafkaProducer for a given topic. The sink produces a DataStream to
 * the topic.
 *
 * <p>Using this constructor, the default {@link FlinkFixedPartitioner} will be used as
 * the partitioner. This default partitioner maps each sink subtask to a single Kafka
 * partition (i.e. all records received by a sink subtask will end up in the same
 * Kafka partition).
 *
 * <p>To use a custom partitioner, please use
 * {@link #FlinkKafkaProducer(String, KeyedSerializationSchema, Properties, Optional)} instead.
 *
 * @param brokerList
 *			Comma separated addresses of the brokers
 * @param topicId
 * 			ID of the Kafka topic.
 * @param serializationSchema
 * 			User defined serialization schema supporting key/value messages
 */
public FlinkKafkaProducer(String brokerList, String topicId, KeyedSerializationSchema<IN> serializationSchema) {
	this(
		topicId,
		serializationSchema,
		getPropertiesFromBrokerList(brokerList),
		Optional.of(new FlinkFixedPartitioner<IN>()));
}