Java Code Examples for org.apache.flink.api.common.serialization.SerializationSchema

@Override
public DynamicTableSink createDynamicTableSink(Context context) {
	TableSchema tableSchema = context.getCatalogTable().getSchema();
	ElasticsearchValidationUtils.validatePrimaryKey(tableSchema);
	final FactoryUtil.TableFactoryHelper helper = FactoryUtil.createTableFactoryHelper(this, context);

	final EncodingFormat<SerializationSchema<RowData>> format = helper.discoverEncodingFormat(
		SerializationFormatFactory.class,
		FORMAT_OPTION);

	helper.validate();
	Configuration configuration = new Configuration();
	context.getCatalogTable()
		.getOptions()
		.forEach(configuration::setString);
	Elasticsearch6Configuration config = new Elasticsearch6Configuration(configuration, context.getClassLoader());

	validate(config, configuration);

	return new Elasticsearch6DynamicSink(
		format,
		config,
		TableSchemaUtils.getPhysicalSchema(tableSchema));
}
 

private SerializationWithDeserializationSchemaMatcherBuilder(
	SerializationSchema<Row> serializationSchema,
	DeserializationSchema<Row> deserializationSchema) {
	try {
		// we serialize and deserialize the schema to test runtime behavior
		// when the schema is shipped to the cluster
		this.serializationSchema = deserializeObject(
			serializeObject(serializationSchema),
			this.getClass().getClassLoader());
		this.deserializationSchema = deserializeObject(
			serializeObject(deserializationSchema),
			this.getClass().getClassLoader());
	} catch (IOException | ClassNotFoundException e) {
		throw new RuntimeException(e);
	}
}
 

@Override
protected KafkaTableSinkBase getExpectedKafkaTableSink(
		TableSchema schema,
		String topic,
		Properties properties,
		Optional<FlinkKafkaPartitioner<Row>> partitioner,
		SerializationSchema<Row> serializationSchema) {

	return new Kafka011TableSink(
		schema,
		topic,
		properties,
		partitioner,
		serializationSchema
	);
}
 

private SerializationSchema<Row> getSerializationSchema(Map<String, String> properties) {
	final String formatType = properties.get(FORMAT_TYPE);
	// we could have added this check to the table factory context
	// but this approach allows to throw more helpful error messages
	// if the supported format has not been added
	if (formatType == null || !formatType.equals(SUPPORTED_FORMAT_TYPE)) {
		throw new ValidationException(
			"The Elasticsearch sink requires a '" + SUPPORTED_FORMAT_TYPE + "' format.");
	}

	@SuppressWarnings("unchecked")
	final SerializationSchemaFactory<Row> formatFactory = TableFactoryService.find(
		SerializationSchemaFactory.class,
		properties,
		this.getClass().getClassLoader());
	return formatFactory.createSerializationSchema(properties);
}
 

public ElasticsearchUpsertSinkFunction(
		String index,
		String docType,
		String keyDelimiter,
		String keyNullLiteral,
		SerializationSchema<Row> serializationSchema,
		XContentType contentType,
		RequestFactory requestFactory,
		int[] keyFieldIndices) {

	this.index = Preconditions.checkNotNull(index);
	this.docType = Preconditions.checkNotNull(docType);
	this.keyDelimiter = Preconditions.checkNotNull(keyDelimiter);
	this.serializationSchema = Preconditions.checkNotNull(serializationSchema);
	this.contentType = Preconditions.checkNotNull(contentType);
	this.keyFieldIndices = Preconditions.checkNotNull(keyFieldIndices);
	this.requestFactory = Preconditions.checkNotNull(requestFactory);
	this.keyNullLiteral = Preconditions.checkNotNull(keyNullLiteral);
}
 

@Override
public DynamicTableSink createDynamicTableSink(Context context) {
	TableSchema tableSchema = context.getCatalogTable().getSchema();
	ElasticsearchValidationUtils.validatePrimaryKey(tableSchema);

	final FactoryUtil.TableFactoryHelper helper = FactoryUtil.createTableFactoryHelper(this, context);

	final EncodingFormat<SerializationSchema<RowData>> format = helper.discoverEncodingFormat(
		SerializationFormatFactory.class,
		FORMAT_OPTION);

	helper.validate();
	Configuration configuration = new Configuration();
	context.getCatalogTable()
		.getOptions()
		.forEach(configuration::setString);
	Elasticsearch7Configuration config = new Elasticsearch7Configuration(configuration, context.getClassLoader());

	validate(config, configuration);

	return new Elasticsearch7DynamicSink(
		format,
		config,
		TableSchemaUtils.getPhysicalSchema(tableSchema));
}
 

@Override
protected ElasticsearchUpsertTableSinkBase getExpectedTableSink(
		boolean isAppendOnly,
		TableSchema schema,
		List<Host> hosts,
		String index,
		String docType,
		String keyDelimiter,
		String keyNullLiteral,
		SerializationSchema<Row> serializationSchema,
		XContentType contentType,
		ActionRequestFailureHandler failureHandler,
		Map<SinkOption, String> sinkOptions,
		IndexGenerator indexGenerator) {
	return new Elasticsearch7UpsertTableSink(
		isAppendOnly,
		schema,
		hosts,
		index,
		keyDelimiter,
		keyNullLiteral,
		serializationSchema,
		contentType,
		failureHandler,
		sinkOptions);
}
 

public TestElasticsearch6UpsertTableSink(
		boolean isAppendOnly,
		TableSchema schema,
		List<Host> hosts,
		String index,
		String docType,
		String keyDelimiter,
		String keyNullLiteral,
		SerializationSchema<Row> serializationSchema,
		XContentType contentType,
		ActionRequestFailureHandler failureHandler,
		Map<SinkOption, String> sinkOptions) {

	super(
		isAppendOnly,
		schema,
		hosts,
		index,
		docType,
		keyDelimiter,
		keyNullLiteral,
		serializationSchema,
		contentType,
		failureHandler,
		sinkOptions);
}
 

@Override
protected KafkaTableSinkBase getExpectedKafkaTableSink(
		TableSchema schema,
		String topic,
		Properties properties,
		Optional<FlinkKafkaPartitioner<Row>> partitioner,
		SerializationSchema<Row> serializationSchema) {

	return new Kafka010TableSink(
		schema,
		topic,
		properties,
		partitioner,
		serializationSchema
	);
}
 

/**
 * The flink pravega writer instance which can be added as a sink to a Flink job.
 *
 * @param clientConfig          The Pravega client configuration.
 * @param stream                The destination stream.
 * @param serializationSchema   The implementation for serializing every event into pravega's storage format.
 * @param eventRouter           The implementation to extract the partition key from the event.
 * @param writerMode            The Pravega writer mode.
 * @param txnLeaseRenewalPeriod Transaction lease renewal period in milliseconds.
 * @param enableWatermark       Flag to indicate whether Pravega watermark needs to be enabled or not.
 * @param enableMetrics         Flag to indicate whether metrics needs to be enabled or not.
 */
protected FlinkPravegaWriter(
        final ClientConfig clientConfig,
        final Stream stream,
        final SerializationSchema<T> serializationSchema,
        final PravegaEventRouter<T> eventRouter,
        final PravegaWriterMode writerMode,
        final long txnLeaseRenewalPeriod,
        final boolean enableWatermark,
        final boolean enableMetrics) {

    this.clientConfig = Preconditions.checkNotNull(clientConfig, "clientConfig");
    this.stream = Preconditions.checkNotNull(stream, "stream");
    this.serializationSchema = Preconditions.checkNotNull(serializationSchema, "serializationSchema");
    this.eventRouter = Preconditions.checkNotNull(eventRouter, "eventRouter");
    this.writerMode = Preconditions.checkNotNull(writerMode, "writerMode");
    Preconditions.checkArgument(txnLeaseRenewalPeriod > 0, "txnLeaseRenewalPeriod must be > 0");
    this.txnLeaseRenewalPeriod = txnLeaseRenewalPeriod;
    this.enableWatermark = enableWatermark;
    this.enableMetrics = enableMetrics;
    this.writerIdPrefix = UUID.randomUUID().toString();
}
 

@Override
protected KafkaTableSinkBase createKafkaTableSink(
		TableSchema schema,
		String topic,
		Properties properties,
		Optional<FlinkKafkaPartitioner<Row>> partitioner,
		SerializationSchema<Row> serializationSchema) {

	return new Kafka09TableSink(
		schema,
		topic,
		properties,
		partitioner,
		serializationSchema);
}
 

@Override
public EncodingFormat<SerializationSchema<RowData>> createEncodingFormat(
		DynamicTableFactory.Context context,
		ReadableConfig formatConfig) {
	FactoryUtil.validateFactoryOptions(this, formatConfig);
	return new EncodingFormatMock(formatConfig.get(DELIMITER));
}
 

@Override
protected FlinkKafkaProducerBase<Row> createKafkaProducer(
		String topic,
		Properties properties,
		SerializationSchema<Row> serializationSchema,
		Optional<FlinkKafkaPartitioner<Row>> partitioner) {
	return new FlinkKafkaProducer09<>(
		topic,
		serializationSchema,
		properties,
		partitioner.orElse(null));
}
 

@Override
protected KafkaTableSinkBase createKafkaTableSink(
	TableSchema schema,
	String topic,
	Properties properties,
	Optional<FlinkKafkaPartitioner<Row>> partitioner,
	SerializationSchema<Row> serializationSchema) {

	return new KafkaTableSink(
		schema,
		topic,
		properties,
		partitioner,
		serializationSchema);
}
 

public KafkaTableSink(
	TableSchema schema,
	String topic,
	Properties properties,
	Optional<FlinkKafkaPartitioner<Row>> partitioner,
	SerializationSchema<Row> serializationSchema) {

	super(schema, topic, properties, partitioner, serializationSchema);
}
 

@Override
protected KafkaTableSinkBase createKafkaTableSink(
		TableSchema schema,
		String topic,
		Properties properties,
		Optional<FlinkKafkaPartitioner<Row>> partitioner,
		SerializationSchema<Row> serializationSchema) {

	return new Kafka08TableSink(
		schema,
		topic,
		properties,
		partitioner,
		serializationSchema);
}
 

public Elasticsearch6SinkFunction(String index, Integer fieldIndex, String docType, String keyDelimiter, String keyNullLiteral, SerializationSchema<Row> serializationSchema, XContentType contentType, ElasticsearchUpsertTableSinkBase.RequestFactory requestFactory, int[] keyFieldIndices) {
    this.index = index;
    this.fieldIndex = fieldIndex;
    this.docType = Preconditions.checkNotNull(docType);
    this.keyDelimiter = Preconditions.checkNotNull(keyDelimiter);
    this.serializationSchema = Preconditions.checkNotNull(serializationSchema);
    this.contentType = Preconditions.checkNotNull(contentType);
    this.keyFieldIndices = Preconditions.checkNotNull(keyFieldIndices);
    this.requestFactory = Preconditions.checkNotNull(requestFactory);
    this.keyNullLiteral = Preconditions.checkNotNull(keyNullLiteral);
}
 

@Override
protected SinkFunction<RowData> createKafkaProducer(
		String topic,
		Properties properties,
		SerializationSchema<RowData> serializationSchema,
		Optional<FlinkKafkaPartitioner<RowData>> partitioner) {
	return new FlinkKafkaProducer<>(
			topic,
			serializationSchema,
			properties,
			partitioner);
}
 

@Override
protected ElasticsearchUpsertTableSinkBase createElasticsearchUpsertTableSink(
		boolean isAppendOnly,
		TableSchema schema,
		List<Host> hosts,
		String index,
		String docType,
		String keyDelimiter,
		String keyNullLiteral,
		SerializationSchema<Row> serializationSchema,
		XContentType contentType,
		ActionRequestFailureHandler failureHandler,
		Map<SinkOption, String> sinkOptions) {

	return new Elasticsearch6UpsertTableSink(
		isAppendOnly,
		schema,
		hosts,
		index,
		docType,
		keyDelimiter,
		keyNullLiteral,
		serializationSchema,
		contentType,
		failureHandler,
		sinkOptions);
}
 

@Override KafkaTableSinkBase newTableSink(TableSchema schema, String topic, Properties properties,
    Optional<FlinkKafkaPartitioner<Row>> partitioner, SerializationSchema<Row> serializationSchema) {
    return new Kafka010TableSink(
        schema,
        topic,
        properties,
        partitioner,
        serializationSchema
    );
}
 

@Override
public SerializationSchema<Row> createSerializationSchema(Map<String, String> properties) {
	final DescriptorProperties descriptorProperties = getValidatedProperties(properties);

	// create and configure
	if (descriptorProperties.containsKey(AvroValidator.FORMAT_RECORD_CLASS)) {
		return new AvroRowSerializationSchema(
			descriptorProperties.getClass(AvroValidator.FORMAT_RECORD_CLASS, SpecificRecord.class));
	} else {
		return new AvroRowSerializationSchema(descriptorProperties.getString(AvroValidator.FORMAT_AVRO_SCHEMA));
	}
}
 

@Override
protected KafkaTableSinkBase createKafkaTableSink(
	TableSchema schema,
	String topic,
	Properties properties,
	Optional<FlinkKafkaPartitioner<Row>> partitioner,
	SerializationSchema<Row> serializationSchema) {

	return new KafkaTableSink(
		schema,
		topic,
		properties,
		partitioner,
		serializationSchema);
}
 

@Override
protected KafkaTableSinkBase getExpectedKafkaTableSink(
	TableSchema schema,
	String topic,
	Properties properties,
	Optional<FlinkKafkaPartitioner<Row>> partitioner,
	SerializationSchema<Row> serializationSchema) {

	return new KafkaTableSink(
		schema,
		topic,
		properties,
		partitioner,
		serializationSchema);
}
 

@Test
public void givenProperBackupObject_whenSerializeIsInvoked_thenObjectIsProperlySerialized() throws IOException {
    InputMessage message = new InputMessage("Me", "User", LocalDateTime.now(), "Test Message");
    List<InputMessage> messages = Arrays.asList(message);
    Backup backup = new Backup(messages, LocalDateTime.now());
    byte[] backupSerialized = mapper.writeValueAsBytes(backup);
    SerializationSchema<Backup> serializationSchema = new BackupSerializationSchema();
    byte[] backupProcessed = serializationSchema.serialize(backup);
    
    assertArrayEquals(backupSerialized, backupProcessed);
}
 

@Test
public void testSpecificRecordWithConfluentSchemaRegistry() throws Exception {
	SerializationSchema<Address> deserializer = AvroSerializationSchema.forSpecific(Address.class);

	byte[] encodedAddress = writeRecord(address, Address.getClassSchema());
	byte[] serializedAddress = deserializer.serialize(address);
	assertArrayEquals(encodedAddress, serializedAddress);
}
 

public KafkaTableSink(
	TableSchema schema,
	String topic,
	Properties properties,
	Optional<FlinkKafkaPartitioner<Row>> partitioner,
	SerializationSchema<Row> serializationSchema) {

	super(schema, topic, properties, partitioner, serializationSchema);
}
 

@Test
public void testGenericRecord() throws Exception {
	SerializationSchema<GenericRecord> deserializationSchema =
		AvroSerializationSchema.forGeneric(
			address.getSchema()
		);

	byte[] encodedAddress = writeRecord(address, Address.getClassSchema());
	byte[] dataSerialized = deserializationSchema.serialize(address);
	assertArrayEquals(encodedAddress, dataSerialized);
}
 

/**
 * Creates the sink function for the current builder state.
 *
 * @param serializationSchema the deserialization schema to use.
 * @param eventRouter the event router to use.
 */
protected FlinkPravegaWriter<T> createSinkFunction(SerializationSchema<T> serializationSchema, PravegaEventRouter<T> eventRouter) {
    Preconditions.checkNotNull(serializationSchema, "serializationSchema");
    Preconditions.checkNotNull(eventRouter, "eventRouter");
    return new FlinkPravegaWriter<>(
            getPravegaConfig().getClientConfig(),
            resolveStream(),
            serializationSchema,
            eventRouter,
            writerMode,
            txnLeaseRenewalPeriod.toMilliseconds(),
            enableWatermark,
            isMetricsEnabled());
}
 

public Elasticsearch6UpsertTableSink(
		boolean isAppendOnly,
		TableSchema schema,
		List<Host> hosts,
		String index,
		String docType,
		String keyDelimiter,
		String keyNullLiteral,
		SerializationSchema<Row> serializationSchema,
		XContentType contentType,
		ActionRequestFailureHandler failureHandler,
		Map<SinkOption, String> sinkOptions) {

	super(
		isAppendOnly,
		schema,
		hosts,
		index,
		docType,
		keyDelimiter,
		keyNullLiteral,
		serializationSchema,
		contentType,
		failureHandler,
		sinkOptions,
		UPDATE_REQUEST_FACTORY);
}
 

@Override
protected ElasticsearchUpsertTableSinkBase createElasticsearchUpsertTableSink(
		boolean isAppendOnly,
		TableSchema schema,
		List<Host> hosts,
		String index,
		String docType,
		String keyDelimiter,
		String keyNullLiteral,
		SerializationSchema<Row> serializationSchema,
		XContentType contentType,
		ActionRequestFailureHandler failureHandler,
		Map<SinkOption, String> sinkOptions) {

	return new Elasticsearch6UpsertTableSink(
		isAppendOnly,
		schema,
		hosts,
		index,
		docType,
		keyDelimiter,
		keyNullLiteral,
		serializationSchema,
		contentType,
		failureHandler,
		sinkOptions);
}