org.apache.flink.streaming.connectors.kafka.partitioner.FlinkFixedPartitioner Java Examples

The following examples show how to use org.apache.flink.streaming.connectors.kafka.partitioner.FlinkFixedPartitioner. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: KafkaTableSourceSinkFactoryBase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
private Optional<FlinkKafkaPartitioner<Row>> getFlinkKafkaPartitioner(DescriptorProperties descriptorProperties) {
	return descriptorProperties
		.getOptionalString(CONNECTOR_SINK_PARTITIONER)
		.flatMap((String partitionerString) -> {
			switch (partitionerString) {
				case CONNECTOR_SINK_PARTITIONER_VALUE_FIXED:
					return Optional.of(new FlinkFixedPartitioner<>());
				case CONNECTOR_SINK_PARTITIONER_VALUE_ROUND_ROBIN:
					return Optional.empty();
				case CONNECTOR_SINK_PARTITIONER_VALUE_CUSTOM:
					final Class<? extends FlinkKafkaPartitioner> partitionerClass =
						descriptorProperties.getClass(CONNECTOR_SINK_PARTITIONER_CLASS, FlinkKafkaPartitioner.class);
					return Optional.of((FlinkKafkaPartitioner<Row>) InstantiationUtil.instantiate(partitionerClass));
				default:
					throw new TableException("Unsupported sink partitioner. Validator should have checked that.");
			}
		});
}
 
Example #2
Source File: KafkaTableSourceSinkFactoryBase.java    From flink with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
private Optional<FlinkKafkaPartitioner<Row>> getFlinkKafkaPartitioner(DescriptorProperties descriptorProperties) {
	return descriptorProperties
		.getOptionalString(CONNECTOR_SINK_PARTITIONER)
		.flatMap((String partitionerString) -> {
			switch (partitionerString) {
				case CONNECTOR_SINK_PARTITIONER_VALUE_FIXED:
					return Optional.of(new FlinkFixedPartitioner<>());
				case CONNECTOR_SINK_PARTITIONER_VALUE_ROUND_ROBIN:
					return Optional.empty();
				case CONNECTOR_SINK_PARTITIONER_VALUE_CUSTOM:
					final Class<? extends FlinkKafkaPartitioner> partitionerClass =
						descriptorProperties.getClass(CONNECTOR_SINK_PARTITIONER_CLASS, FlinkKafkaPartitioner.class);
					return Optional.of((FlinkKafkaPartitioner<Row>) InstantiationUtil.instantiate(partitionerClass));
				default:
					throw new TableException("Unsupported sink partitioner. Validator should have checked that.");
			}
		});
}
 
Example #3
Source File: FlinkFixedPartitionerTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Test for when there are more sinks than partitions.
 * <pre>
 *   		Flink Sinks:		Kafka Partitions
 * 			1	---------------->	1
 * 			2   --------------/
 * 			3   -------------/
 * 			4	------------/
 * </pre>
 */
@Test
public void testMoreFlinkThanBrokers() {
	FlinkFixedPartitioner<String> part = new FlinkFixedPartitioner<>();

	int[] partitions = new int[]{0};

	part.open(0, 4);
	Assert.assertEquals(0, part.partition("abc1", null, null, null, partitions));

	part.open(1, 4);
	Assert.assertEquals(0, part.partition("abc2", null, null, null, partitions));

	part.open(2, 4);
	Assert.assertEquals(0, part.partition("abc3", null, null, null, partitions));
	Assert.assertEquals(0, part.partition("abc3", null, null, null, partitions)); // check if it is changing ;)

	part.open(3, 4);
	Assert.assertEquals(0, part.partition("abc4", null, null, null, partitions));
}
 
Example #4
Source File: FlinkFixedPartitionerTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Test for when there are more sinks than partitions.
 * <pre>
 *   		Flink Sinks:		Kafka Partitions
 * 			1	---------------->	1
 * 			2   --------------/
 * 			3   -------------/
 * 			4	------------/
 * </pre>
 */
@Test
public void testMoreFlinkThanBrokers() {
	FlinkFixedPartitioner<String> part = new FlinkFixedPartitioner<>();

	int[] partitions = new int[]{0};

	part.open(0, 4);
	Assert.assertEquals(0, part.partition("abc1", null, null, null, partitions));

	part.open(1, 4);
	Assert.assertEquals(0, part.partition("abc2", null, null, null, partitions));

	part.open(2, 4);
	Assert.assertEquals(0, part.partition("abc3", null, null, null, partitions));
	Assert.assertEquals(0, part.partition("abc3", null, null, null, partitions)); // check if it is changing ;)

	part.open(3, 4);
	Assert.assertEquals(0, part.partition("abc4", null, null, null, partitions));
}
 
Example #5
Source File: FlinkFixedPartitionerTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Test for when there are more sinks than partitions.
 * <pre>
 *   		Flink Sinks:		Kafka Partitions
 * 			1	---------------->	1
 * 			2   --------------/
 * 			3   -------------/
 * 			4	------------/
 * </pre>
 */
@Test
public void testMoreFlinkThanBrokers() {
	FlinkFixedPartitioner<String> part = new FlinkFixedPartitioner<>();

	int[] partitions = new int[]{0};

	part.open(0, 4);
	Assert.assertEquals(0, part.partition("abc1", null, null, null, partitions));

	part.open(1, 4);
	Assert.assertEquals(0, part.partition("abc2", null, null, null, partitions));

	part.open(2, 4);
	Assert.assertEquals(0, part.partition("abc3", null, null, null, partitions));
	Assert.assertEquals(0, part.partition("abc3", null, null, null, partitions)); // check if it is changing ;)

	part.open(3, 4);
	Assert.assertEquals(0, part.partition("abc4", null, null, null, partitions));
}
 
Example #6
Source File: KafkaTableSourceSinkFactoryBase.java    From flink with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
private Optional<FlinkKafkaPartitioner<Row>> getFlinkKafkaPartitioner(DescriptorProperties descriptorProperties) {
	return descriptorProperties
		.getOptionalString(CONNECTOR_SINK_PARTITIONER)
		.flatMap((String partitionerString) -> {
			switch (partitionerString) {
				case CONNECTOR_SINK_PARTITIONER_VALUE_FIXED:
					return Optional.of(new FlinkFixedPartitioner<>());
				case CONNECTOR_SINK_PARTITIONER_VALUE_ROUND_ROBIN:
					return Optional.empty();
				case CONNECTOR_SINK_PARTITIONER_VALUE_CUSTOM:
					final Class<? extends FlinkKafkaPartitioner> partitionerClass =
						descriptorProperties.getClass(CONNECTOR_SINK_PARTITIONER_CLASS, FlinkKafkaPartitioner.class);
					return Optional.of((FlinkKafkaPartitioner<Row>) InstantiationUtil.instantiate(partitionerClass));
				default:
					throw new TableException("Unsupported sink partitioner. Validator should have checked that.");
			}
		});
}
 
Example #7
Source File: KafkaTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public List<Descriptor> descriptors() {
	final Descriptor earliestDesc =
		new Kafka()
			.version("0.8")
			.startFromEarliest()
			.topic("WhateverTopic");

	final Descriptor specificOffsetsDesc =
		new Kafka()
			.version("0.11")
			.topic("MyTable")
			.startFromSpecificOffset(0, 42L)
			.startFromSpecificOffset(1, 300L)
			.property("zookeeper.stuff", "12")
			.property("kafka.stuff", "42");

	final Map<Integer, Long> offsets = new HashMap<>();
	offsets.put(0, 42L);
	offsets.put(1, 300L);

	final Properties properties = new Properties();
	properties.put("zookeeper.stuff", "12");
	properties.put("kafka.stuff", "42");

	final Descriptor specificOffsetsMapDesc =
		new Kafka()
			.version("0.11")
			.topic("MyTable")
			.startFromSpecificOffsets(offsets)
			.properties(properties)
			.sinkPartitionerCustom(FlinkFixedPartitioner.class);

	return Arrays.asList(earliestDesc, specificOffsetsDesc, specificOffsetsMapDesc);
}
 
Example #8
Source File: KafkaTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
public List<Descriptor> descriptors() {
	final Descriptor earliestDesc =
		new Kafka()
			.version("0.8")
			.startFromEarliest()
			.topic("WhateverTopic");

	final Descriptor specificOffsetsDesc =
		new Kafka()
			.version("0.11")
			.topic("MyTable")
			.startFromSpecificOffset(0, 42L)
			.startFromSpecificOffset(1, 300L)
			.property("zookeeper.stuff", "12")
			.property("kafka.stuff", "42");

	final Map<Integer, Long> offsets = new HashMap<>();
	offsets.put(0, 42L);
	offsets.put(1, 300L);

	final Properties properties = new Properties();
	properties.put("zookeeper.stuff", "12");
	properties.put("kafka.stuff", "42");

	final Descriptor specificOffsetsMapDesc =
		new Kafka()
			.version("0.11")
			.topic("MyTable")
			.startFromSpecificOffsets(offsets)
			.properties(properties)
			.sinkPartitionerCustom(FlinkFixedPartitioner.class);

	return Arrays.asList(earliestDesc, specificOffsetsDesc, specificOffsetsMapDesc);
}
 
Example #9
Source File: FlinkFixedPartitionerTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Tests for when there are more partitions than sinks.
 * <pre>
 * 		Flink Sinks:		Kafka Partitions
 * 			1	---------------->	1
 * 			2	---------------->	2
 * 									3
 * 									4
 * 									5
 *
 * </pre>
 */
@Test
public void testFewerPartitions() {
	FlinkFixedPartitioner<String> part = new FlinkFixedPartitioner<>();

	int[] partitions = new int[]{0, 1, 2, 3, 4};
	part.open(0, 2);
	Assert.assertEquals(0, part.partition("abc1", null, null, null, partitions));
	Assert.assertEquals(0, part.partition("abc1", null, null, null, partitions));

	part.open(1, 2);
	Assert.assertEquals(1, part.partition("abc1", null, null, null, partitions));
	Assert.assertEquals(1, part.partition("abc1", null, null, null, partitions));
}
 
Example #10
Source File: KafkaTestEnvironmentImpl.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
public <T> DataStreamSink<T> writeToKafkaWithTimestamps(DataStream<T> stream, String topic, KeyedSerializationSchema<T> serSchema, Properties props) {
	FlinkKafkaProducer011<T> prod = new FlinkKafkaProducer011<>(
		topic, serSchema, props, Optional.of(new FlinkFixedPartitioner<>()), producerSemantic, FlinkKafkaProducer011.DEFAULT_KAFKA_PRODUCERS_POOL_SIZE);

	prod.setWriteTimestampToKafka(true);

	return stream.addSink(prod);
}
 
Example #11
Source File: UnitTestSuiteFlink.java    From df_data_service with Apache License 2.0 5 votes vote down vote up
public static void testFlinkAvroSQLJson() {
    System.out.println("TestCase_Test Avro SQL to Json Sink");
    final String STATIC_USER_SCHEMA = "{"
            + "\"type\":\"record\","
            + "\"name\":\"myrecord\","
            + "\"fields\":["
            + "  { \"name\":\"symbol\", \"type\":\"string\" },"
            + "  { \"name\":\"name\", \"type\":\"string\" },"
            + "  { \"name\":\"exchangecode\", \"type\":\"string\" }"
            + "]}";

    String jarPath = DFInitService.class.getProtectionDomain().getCodeSource().getLocation().getPath();
    DFRemoteStreamEnvironment env = new DFRemoteStreamEnvironment("localhost", 6123, jarPath)
            .setParallelism(1);
    StreamTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
    Properties properties = new Properties();
    properties.setProperty("bootstrap.servers", "localhost:9092");
    properties.setProperty("group.id", "consumer_test");
    properties.setProperty("schema.subject", "test-value");
    properties.setProperty("schema.registry", "localhost:8081");
    properties.setProperty("useAvro", "avro");
    properties.setProperty("static.avro.schema",
            SchemaRegistryClient.getSchemaFromRegistry("http://localhost:8081", "test-value", "latest").toString());

    try {
        HashMap<String, String> hm = new HashMap<>();
        Kafka09AvroTableSource kafkaAvroTableSource =  new Kafka09AvroTableSource("test", properties);
        tableEnv.registerTableSource("Orders", kafkaAvroTableSource);

        Table result = tableEnv.sql("SELECT name, symbol, exchangecode FROM Orders");
        //Kafka09JsonTableSink json_sink = new Kafka09JsonTableSink ("test_json", properties, new FlinkFixedPartitioner());
        Kafka09AvroTableSink json_sink = new Kafka09AvroTableSink ("test_json", properties, new FlinkFixedPartitioner());

        // write the result Table to the TableSink
        result.writeToSink(json_sink);
        env.executeWithDFObj("Flink AVRO SQL KAFKA Test", new DFJobPOPJ().setJobConfig(hm) );
    } catch (Exception e) {
        e.printStackTrace();
    }
}
 
Example #12
Source File: FlinkAvroSQLClient.java    From df_data_service with Apache License 2.0 5 votes vote down vote up
public static void tcFlinkAvroSQL(String KafkaServerHostPort, String SchemaRegistryHostPort,
                                  String srcTopic, String targetTopic,
                                  String consumerGroupId, String sinkKeys, String sqlState) {

    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    StreamTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);

    Properties properties = new Properties();
    properties.setProperty(ConstantApp.PK_KAFKA_HOST_PORT.replace("_", "."), KafkaServerHostPort);
    properties.setProperty(ConstantApp.PK_KAFKA_CONSUMER_GROURP, consumerGroupId);
    properties.setProperty(ConstantApp.PK_KAFKA_SCHEMA_REGISTRY_HOST_PORT.replace("_", "."), SchemaRegistryHostPort);
    properties.setProperty(ConstantApp.PK_FLINK_TABLE_SINK_KEYS, sinkKeys);

    String[] srcTopicList = srcTopic.split(",");
    for (int i = 0; i < srcTopicList.length; i++) {
        properties.setProperty(ConstantApp.PK_SCHEMA_SUB_INPUT, srcTopicList[i]);
        properties.setProperty(ConstantApp.PK_SCHEMA_ID_INPUT, SchemaRegistryClient.getLatestSchemaIDFromProperty(properties, ConstantApp.PK_SCHEMA_SUB_INPUT) + "");
        properties.setProperty(ConstantApp.PK_SCHEMA_STR_INPUT, SchemaRegistryClient.getLatestSchemaFromProperty(properties, ConstantApp.PK_SCHEMA_SUB_INPUT).toString());
        tableEnv.registerTableSource(srcTopicList[i], new Kafka010AvroTableSource(srcTopicList[i], properties));
    }

    try {
        Table result = tableEnv.sql(sqlState);
        SchemaRegistryClient.addSchemaFromTableResult(SchemaRegistryHostPort, targetTopic, result);
        // For old producer, we need to create topic-value subject as well
        SchemaRegistryClient.addSchemaFromTableResult(SchemaRegistryHostPort, targetTopic + "-value", result);

        // delivered properties for sink
        properties.setProperty(ConstantApp.PK_SCHEMA_SUB_OUTPUT, targetTopic);
        properties.setProperty(ConstantApp.PK_SCHEMA_ID_OUTPUT, SchemaRegistryClient.getLatestSchemaIDFromProperty(properties, ConstantApp.PK_SCHEMA_SUB_OUTPUT) + "");
        properties.setProperty(ConstantApp.PK_SCHEMA_STR_OUTPUT, SchemaRegistryClient.getLatestSchemaFromProperty(properties, ConstantApp.PK_SCHEMA_SUB_OUTPUT).toString());

        Kafka09AvroTableSink avro_sink =
                new Kafka09AvroTableSink(targetTopic, properties, new FlinkFixedPartitioner());
        result.writeToSink(avro_sink);
        env.execute("DF_FlinkSQL_Client_" + srcTopic + "-" + targetTopic);
    } catch (Exception e) {
        e.printStackTrace();
    }
}
 
Example #13
Source File: FlinkTaskInstanceBusiness.java    From PoseidonX with Apache License 2.0 5 votes vote down vote up
/**
 * 构建 FlinkKafka09OutputComponent 对应的业务逻辑
 * @param flinkComponent
 */
private void dealFlinkKafka09OutputComponent(FlinkKafka09OutputComponent flinkComponent) {

    Properties kafkaProperties = new Properties();
    kafkaProperties.setProperty("bootstrap.servers",flinkComponent.getBrokerHostPorts());

    Kafka09JsonTableSink kafkaTableSink = new Kafka09JsonTableSink(
            flinkComponent.getTopic(),
            kafkaProperties,
            new FlinkFixedPartitioner());

    tableSinkMap.put(flinkComponent.getName(),kafkaTableSink);
}
 
Example #14
Source File: FlinkFixedPartitionerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testMixedCase() {
	FlinkFixedPartitioner<String> part = new FlinkFixedPartitioner<>();
	int[] partitions = new int[]{0, 1};

	part.open(0, 3);
	Assert.assertEquals(0, part.partition("abc1", null, null, null, partitions));

	part.open(1, 3);
	Assert.assertEquals(1, part.partition("abc1", null, null, null, partitions));

	part.open(2, 3);
	Assert.assertEquals(0, part.partition("abc1", null, null, null, partitions));

}
 
Example #15
Source File: FlinkFixedPartitionerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests for when there are more partitions than sinks.
 * <pre>
 * 		Flink Sinks:		Kafka Partitions
 * 			1	---------------->	1
 * 			2	---------------->	2
 * 									3
 * 									4
 * 									5
 *
 * </pre>
 */
@Test
public void testFewerPartitions() {
	FlinkFixedPartitioner<String> part = new FlinkFixedPartitioner<>();

	int[] partitions = new int[]{0, 1, 2, 3, 4};
	part.open(0, 2);
	Assert.assertEquals(0, part.partition("abc1", null, null, null, partitions));
	Assert.assertEquals(0, part.partition("abc1", null, null, null, partitions));

	part.open(1, 2);
	Assert.assertEquals(1, part.partition("abc1", null, null, null, partitions));
	Assert.assertEquals(1, part.partition("abc1", null, null, null, partitions));
}
 
Example #16
Source File: FlinkFixedPartitionerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testMixedCase() {
	FlinkFixedPartitioner<String> part = new FlinkFixedPartitioner<>();
	int[] partitions = new int[]{0, 1};

	part.open(0, 3);
	Assert.assertEquals(0, part.partition("abc1", null, null, null, partitions));

	part.open(1, 3);
	Assert.assertEquals(1, part.partition("abc1", null, null, null, partitions));

	part.open(2, 3);
	Assert.assertEquals(0, part.partition("abc1", null, null, null, partitions));

}
 
Example #17
Source File: KafkaDynamicTableFactoryTestBase.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testTableSink() {
	final DataType consumedDataType = SINK_SCHEMA.toPhysicalRowDataType();
	EncodingFormat<SerializationSchema<RowData>> encodingFormat =
			new TestFormatFactory.EncodingFormatMock(",");

	// Construct table sink using options and table sink factory.
	ObjectIdentifier objectIdentifier = ObjectIdentifier.of(
			"default",
			"default",
			"sinkTable");
	final CatalogTable sinkTable = createKafkaSinkCatalogTable();
	final DynamicTableSink actualSink = FactoryUtil.createTableSink(
			null,
			objectIdentifier,
			sinkTable,
			new Configuration(),
			Thread.currentThread().getContextClassLoader());

	final DynamicTableSink expectedSink = getExpectedSink(
			consumedDataType,
			TOPIC,
			KAFKA_PROPERTIES,
			Optional.of(new FlinkFixedPartitioner<>()),
			encodingFormat);
	assertEquals(expectedSink, actualSink);

	// Test sink format.
	final KafkaDynamicSinkBase actualKafkaSink = (KafkaDynamicSinkBase) actualSink;
	assertEquals(encodingFormat, actualKafkaSink.encodingFormat);

	// Test kafka producer.
	DynamicTableSink.SinkRuntimeProvider provider =
			actualKafkaSink.getSinkRuntimeProvider(new SinkRuntimeProviderContext(false));
	assertThat(provider, instanceOf(SinkFunctionProvider.class));
	final SinkFunctionProvider sinkFunctionProvider = (SinkFunctionProvider) provider;
	final SinkFunction<RowData> sinkFunction = sinkFunctionProvider.createSinkFunction();
	assertThat(sinkFunction, instanceOf(getExpectedProducerClass()));
}
 
Example #18
Source File: FlinkAvroTableAPIClient.java    From df_data_service with Apache License 2.0 4 votes vote down vote up
public static void tcFlinkAvroTableAPI(String KafkaServerHostPort, String SchemaRegistryHostPort,
                                  String srcTopic, String targetTopic,
                                  String consumerGroupId, String sinkKeys, String transScript) {

    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    StreamTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);

    Properties properties = new Properties();
    properties.setProperty(ConstantApp.PK_KAFKA_HOST_PORT.replace("_", "."), KafkaServerHostPort);
    properties.setProperty(ConstantApp.PK_KAFKA_CONSUMER_GROURP, consumerGroupId);
    properties.setProperty(ConstantApp.PK_KAFKA_SCHEMA_REGISTRY_HOST_PORT.replace("_", "."), SchemaRegistryHostPort);
    properties.setProperty(ConstantApp.PK_FLINK_TABLE_SINK_KEYS, sinkKeys);

    String[] srcTopicList = srcTopic.split(",");
    for (int i = 0; i < srcTopicList.length; i++) {
        properties.setProperty(ConstantApp.PK_SCHEMA_SUB_INPUT, srcTopicList[i]);
        properties.setProperty(ConstantApp.PK_SCHEMA_ID_INPUT, SchemaRegistryClient.getLatestSchemaIDFromProperty(properties, ConstantApp.PK_SCHEMA_SUB_INPUT) + "");
        properties.setProperty(ConstantApp.PK_SCHEMA_STR_INPUT, SchemaRegistryClient.getLatestSchemaFromProperty(properties, ConstantApp.PK_SCHEMA_SUB_INPUT).toString());
        tableEnv.registerTableSource(srcTopic, new Kafka010AvroTableSource(srcTopicList[i], properties));
    }

    try {
        Table result;
        Table ingest = tableEnv.scan(srcTopic);
        String className = "dynamic.FlinkScript";
        String header = "package dynamic;\n" +
                "import org.apache.flink.table.api.Table;\n" +
                "import com.datafibers.util.*;\n";
        String javaCode = header +
                "public class FlinkScript implements DynamicRunner {\n" +
                "@Override \n" +
                "    public Table transTableObj(Table tbl) {\n" +
                "try {" +
                "return tbl." + transScript + ";\n" +
                "} catch (Exception e) {" +
                "};" +
                "return null;}}";
        // Dynamic code generation
        Class aClass = CompilerUtils.CACHED_COMPILER.loadFromJava(className, javaCode);
        DynamicRunner runner = (DynamicRunner) aClass.newInstance();
        result = runner.transTableObj(ingest);

        SchemaRegistryClient.addSchemaFromTableResult(SchemaRegistryHostPort, targetTopic, result);
        // delivered properties for sink
        properties.setProperty(ConstantApp.PK_SCHEMA_SUB_OUTPUT, targetTopic);
        properties.setProperty(ConstantApp.PK_SCHEMA_ID_OUTPUT, SchemaRegistryClient.getLatestSchemaIDFromProperty(properties, ConstantApp.PK_SCHEMA_SUB_OUTPUT) + "");
        properties.setProperty(ConstantApp.PK_SCHEMA_STR_OUTPUT, SchemaRegistryClient.getLatestSchemaFromProperty(properties, ConstantApp.PK_SCHEMA_SUB_OUTPUT).toString());

        Kafka09AvroTableSink avro_sink =
                new Kafka09AvroTableSink(targetTopic, properties, new FlinkFixedPartitioner());
        result.writeToSink(avro_sink);
        env.execute("DF_FlinkTableAPI_Client_" + srcTopic + "-" + targetTopic);
    } catch (Exception e) {
        e.printStackTrace();
    }
}
 
Example #19
Source File: KafkaTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
public List<Map<String, String>> properties() {
	final Map<String, String> props1 = new HashMap<>();
	props1.put("connector.property-version", "1");
	props1.put("connector.type", "kafka");
	props1.put("connector.version", "0.8");
	props1.put("connector.topic", "WhateverTopic");
	props1.put("connector.startup-mode", "earliest-offset");

	final Map<String, String> props2 = new HashMap<>();
	props2.put("connector.property-version", "1");
	props2.put("connector.type", "kafka");
	props2.put("connector.version", "0.11");
	props2.put("connector.topic", "MyTable");
	props2.put("connector.startup-mode", "specific-offsets");
	props2.put("connector.specific-offsets", "partition:0,offset:42;partition:1,offset:300");
	props2.put("connector.properties.zookeeper.stuff", "12");
	props2.put("connector.properties.kafka.stuff", "42");

	final Map<String, String> props3 = new HashMap<>();
	props3.put("connector.property-version", "1");
	props3.put("connector.type", "kafka");
	props3.put("connector.version", "0.11");
	props3.put("connector.topic", "MyTable");
	props3.put("connector.startup-mode", "specific-offsets");
	props3.put("connector.specific-offsets", "partition:0,offset:42;partition:1,offset:300");
	props3.put("connector.properties.zookeeper.stuff", "12");
	props3.put("connector.properties.kafka.stuff", "42");
	props3.put("connector.sink-partitioner", "custom");
	props3.put("connector.sink-partitioner-class", FlinkFixedPartitioner.class.getName());

	final Map<String, String> props4 = new HashMap<>();
	props4.put("connector.property-version", "1");
	props4.put("connector.type", "kafka");
	props4.put("connector.version", "0.11");
	props4.put("connector.topic", "MyTable");
	props4.put("connector.startup-mode", "timestamp");
	props4.put("connector.startup-timestamp-millis", "1577014729000");

	return Arrays.asList(props1, props2, props3, props4);
}
 
Example #20
Source File: KafkaTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
public List<Map<String, String>> properties() {
	final Map<String, String> props1 = new HashMap<>();
	props1.put("connector.property-version", "1");
	props1.put("connector.type", "kafka");
	props1.put("connector.version", "0.8");
	props1.put("connector.topic", "WhateverTopic");
	props1.put("connector.startup-mode", "earliest-offset");

	final Map<String, String> props2 = new HashMap<>();
	props2.put("connector.property-version", "1");
	props2.put("connector.type", "kafka");
	props2.put("connector.version", "0.11");
	props2.put("connector.topic", "MyTable");
	props2.put("connector.startup-mode", "specific-offsets");
	props2.put("connector.specific-offsets.0.partition", "0");
	props2.put("connector.specific-offsets.0.offset", "42");
	props2.put("connector.specific-offsets.1.partition", "1");
	props2.put("connector.specific-offsets.1.offset", "300");
	props2.put("connector.properties.0.key", "zookeeper.stuff");
	props2.put("connector.properties.0.value", "12");
	props2.put("connector.properties.1.key", "kafka.stuff");
	props2.put("connector.properties.1.value", "42");

	final Map<String, String> props3 = new HashMap<>();
	props3.put("connector.property-version", "1");
	props3.put("connector.type", "kafka");
	props3.put("connector.version", "0.11");
	props3.put("connector.topic", "MyTable");
	props3.put("connector.startup-mode", "specific-offsets");
	props3.put("connector.specific-offsets.0.partition", "0");
	props3.put("connector.specific-offsets.0.offset", "42");
	props3.put("connector.specific-offsets.1.partition", "1");
	props3.put("connector.specific-offsets.1.offset", "300");
	props3.put("connector.properties.0.key", "zookeeper.stuff");
	props3.put("connector.properties.0.value", "12");
	props3.put("connector.properties.1.key", "kafka.stuff");
	props3.put("connector.properties.1.value", "42");
	props3.put("connector.sink-partitioner", "custom");
	props3.put("connector.sink-partitioner-class", FlinkFixedPartitioner.class.getName());

	return Arrays.asList(props1, props2, props3);
}
 
Example #21
Source File: KafkaTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@Override
public List<Map<String, String>> properties() {
	final Map<String, String> props1 = new HashMap<>();
	props1.put("connector.property-version", "1");
	props1.put("connector.type", "kafka");
	props1.put("connector.version", "0.8");
	props1.put("connector.topic", "WhateverTopic");
	props1.put("connector.startup-mode", "earliest-offset");

	final Map<String, String> props2 = new HashMap<>();
	props2.put("connector.property-version", "1");
	props2.put("connector.type", "kafka");
	props2.put("connector.version", "0.11");
	props2.put("connector.topic", "MyTable");
	props2.put("connector.startup-mode", "specific-offsets");
	props2.put("connector.specific-offsets.0.partition", "0");
	props2.put("connector.specific-offsets.0.offset", "42");
	props2.put("connector.specific-offsets.1.partition", "1");
	props2.put("connector.specific-offsets.1.offset", "300");
	props2.put("connector.properties.0.key", "zookeeper.stuff");
	props2.put("connector.properties.0.value", "12");
	props2.put("connector.properties.1.key", "kafka.stuff");
	props2.put("connector.properties.1.value", "42");

	final Map<String, String> props3 = new HashMap<>();
	props3.put("connector.property-version", "1");
	props3.put("connector.type", "kafka");
	props3.put("connector.version", "0.11");
	props3.put("connector.topic", "MyTable");
	props3.put("connector.startup-mode", "specific-offsets");
	props3.put("connector.specific-offsets.0.partition", "0");
	props3.put("connector.specific-offsets.0.offset", "42");
	props3.put("connector.specific-offsets.1.partition", "1");
	props3.put("connector.specific-offsets.1.offset", "300");
	props3.put("connector.properties.0.key", "zookeeper.stuff");
	props3.put("connector.properties.0.value", "12");
	props3.put("connector.properties.1.key", "kafka.stuff");
	props3.put("connector.properties.1.value", "42");
	props3.put("connector.sink-partitioner", "custom");
	props3.put("connector.sink-partitioner-class", FlinkFixedPartitioner.class.getName());

	return Arrays.asList(props1, props2, props3);
}
 
Example #22
Source File: KafkaTableSourceSinkFactoryTestBase.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * This test can be unified with the corresponding source test once we have fixed FLINK-9870.
 */
@Test
public void testTableSink() {
	// prepare parameters for Kafka table sink

	final TableSchema schema = TableSchema.builder()
		.field(FRUIT_NAME, Types.STRING())
		.field(COUNT, Types.DECIMAL())
		.field(EVENT_TIME, Types.SQL_TIMESTAMP())
		.build();

	final KafkaTableSinkBase expected = getExpectedKafkaTableSink(
		schema,
		TOPIC,
		KAFKA_PROPERTIES,
		Optional.of(new FlinkFixedPartitioner<>()),
		new TestSerializationSchema(schema.toRowType()));

	// construct table sink using descriptors and table sink factory

	final TestTableDescriptor testDesc = new TestTableDescriptor(
			new Kafka()
				.version(getKafkaVersion())
				.topic(TOPIC)
				.properties(KAFKA_PROPERTIES)
				.sinkPartitionerFixed()
				.startFromSpecificOffsets(OFFSETS)) // test if they accepted although not needed
		.withFormat(new TestTableFormat())
		.withSchema(
			new Schema()
				.field(FRUIT_NAME, Types.STRING())
				.field(COUNT, Types.DECIMAL())
				.field(EVENT_TIME, Types.SQL_TIMESTAMP()))
		.inAppendMode();

	final Map<String, String> propertiesMap = testDesc.toProperties();
	final TableSink<?> actualSink = TableFactoryService.find(StreamTableSinkFactory.class, propertiesMap)
		.createStreamTableSink(propertiesMap);

	assertEquals(expected, actualSink);

	// test Kafka producer
	final KafkaTableSinkBase actualKafkaSink = (KafkaTableSinkBase) actualSink;
	final DataStreamMock streamMock = new DataStreamMock(new StreamExecutionEnvironmentMock(), schema.toRowType());
	actualKafkaSink.emitDataStream(streamMock);
	assertTrue(getExpectedFlinkKafkaProducer().isAssignableFrom(streamMock.sinkFunction.getClass()));
}
 
Example #23
Source File: KafkaTableSourceSinkFactoryTestBase.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
public void testTableSinkWithLegacyProperties() {
	// prepare parameters for Kafka table sink
	final TableSchema schema = TableSchema.builder()
		.field(FRUIT_NAME, DataTypes.STRING())
		.field(COUNT, DataTypes.DECIMAL(10, 4))
		.field(EVENT_TIME, DataTypes.TIMESTAMP(3))
		.build();

	final KafkaTableSinkBase expected = getExpectedKafkaTableSink(
		schema,
		TOPIC,
		KAFKA_PROPERTIES,
		Optional.of(new FlinkFixedPartitioner<>()),
		new TestSerializationSchema(schema.toRowType()));

	// construct table sink using descriptors and table sink factory
	final Map<String, String> legacyPropertiesMap = new HashMap<>();
	legacyPropertiesMap.putAll(createKafkaSinkProperties());

	// use legacy properties
	legacyPropertiesMap.remove("connector.specific-offsets");
	legacyPropertiesMap.remove("connector.properties.bootstrap.servers");
	legacyPropertiesMap.remove("connector.properties.group.id");

	// keep compatible with a specified update-mode
	legacyPropertiesMap.put("update-mode", "append");

	// legacy properties for specific-offsets and properties
	legacyPropertiesMap.put("connector.specific-offsets.0.partition", "0");
	legacyPropertiesMap.put("connector.specific-offsets.0.offset", "100");
	legacyPropertiesMap.put("connector.specific-offsets.1.partition", "1");
	legacyPropertiesMap.put("connector.specific-offsets.1.offset", "123");
	legacyPropertiesMap.put("connector.properties.0.key", "bootstrap.servers");
	legacyPropertiesMap.put("connector.properties.0.value", "dummy");
	legacyPropertiesMap.put("connector.properties.1.key", "group.id");
	legacyPropertiesMap.put("connector.properties.1.value", "dummy");

	final TableSink<?> actualSink = TableFactoryService.find(StreamTableSinkFactory.class, legacyPropertiesMap)
		.createStreamTableSink(legacyPropertiesMap);

	assertEquals(expected, actualSink);

	// test Kafka producer
	final KafkaTableSinkBase actualKafkaSink = (KafkaTableSinkBase) actualSink;
	final DataStreamMock streamMock = new DataStreamMock(new StreamExecutionEnvironmentMock(), schema.toRowType());
	actualKafkaSink.consumeDataStream(streamMock);
	assertTrue(getExpectedFlinkKafkaProducer().isAssignableFrom(streamMock.sinkFunction.getClass()));
}
 
Example #24
Source File: UnitTestSuiteFlink.java    From df_data_service with Apache License 2.0 4 votes vote down vote up
public static void testFlinkAvroScriptWithStaticSchema() {
    System.out.println("TestCase_Test Avro Table API Script with static Schema");

    final String STATIC_USER_SCHEMA = "{"
            + "\"type\":\"record\","
            + "\"name\":\"myrecord\","
            + "\"fields\":["
            + "  { \"name\":\"symbol\", \"type\":\"string\" },"
            + "  { \"name\":\"name\", \"type\":\"string\" },"
            + "  { \"name\":\"exchangecode\", \"type\":\"string\" }"
            + "]}";

    String jarPath = DFInitService.class.getProtectionDomain().getCodeSource().getLocation().getPath();
    StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", 6123, jarPath)
            .setParallelism(1);
    StreamTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
    Properties properties = new Properties();
    properties.setProperty("bootstrap.servers", "localhost:9092");
    properties.setProperty("group.id", "consumer_test");
    properties.setProperty("schema.subject", "test-value");
    properties.setProperty("schema.registry", "localhost:8081");
    properties.setProperty("static.avro.schema", STATIC_USER_SCHEMA);

    try {
        Kafka09AvroTableSource kafkaAvroTableSource =  new Kafka09AvroTableSource("test", properties);
        tableEnv.registerTableSource("Orders", kafkaAvroTableSource);

        Table ingest = tableEnv.scan("Orders");

        String className = "dynamic.FlinkScript";

        String header = "package dynamic;\n" +
                "import org.apache.flink.table.api.Table;\n" +
                "import com.datafibers.util.*;\n";

        String transScript = "select(\"name\")";

        String javaCode = header +
                "public class FlinkScript implements DynamicRunner {\n" +
                "@Override \n" +
                "    public Table transTableObj(Table tbl) {\n" +
                "try {" +
                "return tbl."+ transScript + ";" +
                "} catch (Exception e) {" +
                "};" +
                "return null;}}";

        // Dynamic code generation
        Class aClass = CompilerUtils.CACHED_COMPILER.loadFromJava(className, javaCode);
        DynamicRunner runner = (DynamicRunner) aClass.newInstance();
        Table result = runner.transTableObj(ingest);

        Kafka09AvroTableSink sink =
                new Kafka09AvroTableSink ("test_json", properties, new FlinkFixedPartitioner());
        // write the result Table to the TableSink
        result.writeToSink(sink);
        env.execute("Flink AVRO SQL KAFKA Test");
    } catch (Exception e) {
        e.printStackTrace();
    }
}
 
Example #25
Source File: KafkaTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
public List<Descriptor> descriptors() {
	final Descriptor earliestDesc =
		new Kafka()
			.version("0.8")
			.startFromEarliest()
			.topic("WhateverTopic");

	final Descriptor specificOffsetsDesc =
		new Kafka()
			.version("0.11")
			.topic("MyTable")
			.startFromSpecificOffset(0, 42L)
			.startFromSpecificOffset(1, 300L)
			.property("zookeeper.stuff", "12")
			.property("kafka.stuff", "42");

	final Map<Integer, Long> offsets = new HashMap<>();
	offsets.put(0, 42L);
	offsets.put(1, 300L);

	final Properties properties = new Properties();
	properties.put("zookeeper.stuff", "12");
	properties.put("kafka.stuff", "42");

	final Descriptor specificOffsetsMapDesc =
		new Kafka()
			.version("0.11")
			.topic("MyTable")
			.startFromSpecificOffsets(offsets)
			.properties(properties)
			.sinkPartitionerCustom(FlinkFixedPartitioner.class);

	final Descriptor timestampDesc =
		new Kafka()
			.version("0.11")
			.topic("MyTable")
			.startFromTimestamp(1577014729000L);

	return Arrays.asList(earliestDesc, specificOffsetsDesc, specificOffsetsMapDesc, timestampDesc);
}
 
Example #26
Source File: TCFlinkAvroSQL.java    From df_data_service with Apache License 2.0 4 votes vote down vote up
public static void tcFlinkAvroSQL(String SchemaRegistryHostPort, String srcTopic, String targetTopic, String sqlState) {
    System.out.println("tcFlinkAvroSQL");
    String resultFile = "testResult";

    String jarPath = "C:/Users/dadu/Coding/df_data_service/target/df-data-service-1.1-SNAPSHOT-fat.jar";
    //String jarPath = "/Users/will/Documents/Coding/GitHub/df_data_service/target/df-data-service-1.1-SNAPSHOT-fat.jar";
    StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", 6123, jarPath)
            .setParallelism(1);
    StreamTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);

    Properties properties = new Properties();
    properties.setProperty(ConstantApp.PK_KAFKA_HOST_PORT.replace("_", "."), "localhost:9092");
    properties.setProperty(ConstantApp.PK_KAFKA_CONSUMER_GROURP, "consumer_test");
    //properties.setProperty(ConstantApp.PK_SCHEMA_SUB_OUTPUT, "test");
    properties.setProperty(ConstantApp.PK_KAFKA_SCHEMA_REGISTRY_HOST_PORT.replace("_", "."), SchemaRegistryHostPort);
    properties.setProperty(ConstantApp.PK_FLINK_TABLE_SINK_KEYS, "symbol");

    String[] srcTopicList = srcTopic.split(",");
    for (int i = 0; i < srcTopicList.length; i++) {
        properties.setProperty(ConstantApp.PK_SCHEMA_SUB_INPUT, srcTopicList[i]);
        properties.setProperty(ConstantApp.PK_SCHEMA_ID_INPUT, SchemaRegistryClient.getLatestSchemaIDFromProperty(properties, ConstantApp.PK_SCHEMA_SUB_INPUT) + "");
        properties.setProperty(ConstantApp.PK_SCHEMA_STR_INPUT, SchemaRegistryClient.getLatestSchemaFromProperty(properties, ConstantApp.PK_SCHEMA_SUB_INPUT).toString());
        tableEnv.registerTableSource(srcTopicList[i], new Kafka010AvroTableSource(srcTopicList[i], properties));
    }

    try {
        Table result = tableEnv.sql(sqlState);
        result.printSchema();
        System.out.println("generated avro schema is = " + SchemaRegistryClient.tableAPIToAvroSchema(result, targetTopic));
        SchemaRegistryClient.addSchemaFromTableResult(SchemaRegistryHostPort, targetTopic, result);

        // delivered properties
        properties.setProperty(ConstantApp.PK_SCHEMA_SUB_OUTPUT, targetTopic);
        properties.setProperty(ConstantApp.PK_SCHEMA_ID_OUTPUT, SchemaRegistryClient.getLatestSchemaIDFromProperty(properties, ConstantApp.PK_SCHEMA_SUB_OUTPUT) + "");
        properties.setProperty(ConstantApp.PK_SCHEMA_STR_OUTPUT, SchemaRegistryClient.getLatestSchemaFromProperty(properties, ConstantApp.PK_SCHEMA_SUB_OUTPUT).toString());

        System.out.println(Paths.get(resultFile).toAbsolutePath());
        Kafka09AvroTableSink avro_sink =
                new Kafka09AvroTableSink(targetTopic, properties, new FlinkFixedPartitioner());
        result.writeToSink(avro_sink);
        //result.writeToSink(new CsvTableSink(resultFile, "|", 1, FileSystem.WriteMode.OVERWRITE));
        env.execute("tcFlinkAvroSQL");
    } catch (Exception e) {
        e.printStackTrace();
    }
}
 
Example #27
Source File: FlinkKafkaProducer011.java    From flink with Apache License 2.0 3 votes vote down vote up
/**
 * Creates a FlinkKafkaProducer for a given topic. The sink produces a DataStream to
 * the topic.
 *
 * <p>Using this constructor, the default {@link FlinkFixedPartitioner} will be used as
 * the partitioner. This default partitioner maps each sink subtask to a single Kafka
 * partition (i.e. all records received by a sink subtask will end up in the same
 * Kafka partition).
 *
 * <p>To use a custom partitioner, please use
 * {@link #FlinkKafkaProducer011(String, SerializationSchema, Properties, Optional)} instead.
 *
 * @param topicId
 * 			ID of the Kafka topic.
 * @param serializationSchema
 * 			User defined key-less serialization schema.
 * @param producerConfig
 * 			Properties with the producer configuration.
 */
public FlinkKafkaProducer011(String topicId, SerializationSchema<IN> serializationSchema, Properties producerConfig) {
	this(
		topicId,
		new KeyedSerializationSchemaWrapper<>(serializationSchema),
		producerConfig,
		Optional.of(new FlinkFixedPartitioner<IN>()));
}
 
Example #28
Source File: FlinkKafkaProducer011.java    From flink with Apache License 2.0 3 votes vote down vote up
/**
 * Creates a FlinkKafkaProducer for a given topic. The sink produces a DataStream to
 * the topic.
 *
 * @param brokerList
 *			Comma separated addresses of the brokers
 * @param topicId
 * 			ID of the Kafka topic.
 * @param serializationSchema
 * 			User defined (keyless) serialization schema.
 */
public FlinkKafkaProducer011(String brokerList, String topicId, SerializationSchema<IN> serializationSchema) {
	this(
		topicId,
		new KeyedSerializationSchemaWrapper<>(serializationSchema),
		getPropertiesFromBrokerList(brokerList),
		Optional.of(new FlinkFixedPartitioner<IN>()));
}
 
Example #29
Source File: FlinkKafkaProducer.java    From flink with Apache License 2.0 3 votes vote down vote up
/**
 * Creates a FlinkKafkaProducer for a given topic. The sink produces a DataStream to
 * the topic.
 *
 * <p>Using this constructor, the default {@link FlinkFixedPartitioner} will be used as
 * the partitioner. This default partitioner maps each sink subtask to a single Kafka
 * partition (i.e. all records received by a sink subtask will end up in the same
 * Kafka partition).
 *
 * @param topicId
 * 			ID of the Kafka topic.
 * @param serializationSchema
 * 			User defined serialization schema supporting key/value messages
 * @param producerConfig
 * 			Properties with the producer configuration.
 * @param semantic
 * 			Defines semantic that will be used by this producer (see {@link FlinkKafkaProducer.Semantic}).
 *
 * @deprecated use {@link #FlinkKafkaProducer(String, KafkaSerializationSchema, Properties, FlinkKafkaProducer.Semantic)}
 */
@Deprecated
public FlinkKafkaProducer(
	String topicId,
	KeyedSerializationSchema<IN> serializationSchema,
	Properties producerConfig,
	FlinkKafkaProducer.Semantic semantic) {
	this(topicId,
		serializationSchema,
		producerConfig,
		Optional.of(new FlinkFixedPartitioner<IN>()),
		semantic,
		DEFAULT_KAFKA_PRODUCERS_POOL_SIZE);
}
 
Example #30
Source File: FlinkKafkaProducer011.java    From flink with Apache License 2.0 3 votes vote down vote up
/**
 * Creates a FlinkKafkaProducer for a given topic. The sink produces a DataStream to
 * the topic.
 *
 * <p>Using this constructor, the default {@link FlinkFixedPartitioner} will be used as
 * the partitioner. This default partitioner maps each sink subtask to a single Kafka
 * partition (i.e. all records received by a sink subtask will end up in the same
 * Kafka partition).
 *
 * <p>To use a custom partitioner, please use
 * {@link #FlinkKafkaProducer011(String, KeyedSerializationSchema, Properties, Optional)} instead.
 *
 * @param brokerList
 *			Comma separated addresses of the brokers
 * @param topicId
 * 			ID of the Kafka topic.
 * @param serializationSchema
 * 			User defined serialization schema supporting key/value messages
 */
public FlinkKafkaProducer011(String brokerList, String topicId, KeyedSerializationSchema<IN> serializationSchema) {
	this(
		topicId,
		serializationSchema,
		getPropertiesFromBrokerList(brokerList),
		Optional.of(new FlinkFixedPartitioner<IN>()));
}