org.apache.flink.table.descriptors.SchemaValidator Java Examples

The following examples show how to use org.apache.flink.table.descriptors.SchemaValidator. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: KafkaTableSourceSinkFactoryBase.java From Flink-CEPplus with Apache License 2.0

6 votes

@Override
public StreamTableSource<Row> createStreamTableSource(Map<String, String> properties) {
	final DescriptorProperties descriptorProperties = getValidatedProperties(properties);

	final String topic = descriptorProperties.getString(CONNECTOR_TOPIC);
	final DeserializationSchema<Row> deserializationSchema = getDeserializationSchema(properties);
	final StartupOptions startupOptions = getStartupOptions(descriptorProperties, topic);

	return createKafkaTableSource(
		descriptorProperties.getTableSchema(SCHEMA()),
		SchemaValidator.deriveProctimeAttribute(descriptorProperties),
		SchemaValidator.deriveRowtimeAttributes(descriptorProperties),
		SchemaValidator.deriveFieldMapping(
			descriptorProperties,
			Optional.of(deserializationSchema.getProducedType())),
		topic,
		getKafkaProperties(descriptorProperties),
		deserializationSchema,
		startupOptions.startupMode,
		startupOptions.specificOffsets);
}

Example #2

Source File: KafkaTableSourceSinkFactoryBase.java From Flink-CEPplus with Apache License 2.0

6 votes

@Override
public StreamTableSink<Row> createStreamTableSink(Map<String, String> properties) {
	final DescriptorProperties descriptorProperties = getValidatedProperties(properties);

	final TableSchema schema = descriptorProperties.getTableSchema(SCHEMA());
	final String topic = descriptorProperties.getString(CONNECTOR_TOPIC);
	final Optional<String> proctime = SchemaValidator.deriveProctimeAttribute(descriptorProperties);
	final List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors =
		SchemaValidator.deriveRowtimeAttributes(descriptorProperties);

	// see also FLINK-9870
	if (proctime.isPresent() || !rowtimeAttributeDescriptors.isEmpty() ||
			checkForCustomFieldMapping(descriptorProperties, schema)) {
		throw new TableException("Time attributes and custom field mappings are not supported yet.");
	}

	return createKafkaTableSink(
		schema,
		topic,
		getKafkaProperties(descriptorProperties),
		getFlinkKafkaPartitioner(descriptorProperties),
		getSerializationSchema(properties));
}

Example #3

Source File: KafkaTableSourceSinkFactoryBase.java From flink with Apache License 2.0

6 votes

@Override
public StreamTableSink<Row> createStreamTableSink(Map<String, String> properties) {
	final DescriptorProperties descriptorProperties = getValidatedProperties(properties);

	final TableSchema schema = TableSchemaUtils.getPhysicalSchema(
		descriptorProperties.getTableSchema(SCHEMA));
	final String topic = descriptorProperties.getString(CONNECTOR_TOPIC);
	final Optional<String> proctime = SchemaValidator.deriveProctimeAttribute(descriptorProperties);
	final List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors =
		SchemaValidator.deriveRowtimeAttributes(descriptorProperties);

	// see also FLINK-9870
	if (proctime.isPresent() || !rowtimeAttributeDescriptors.isEmpty() ||
			checkForCustomFieldMapping(descriptorProperties, schema)) {
		throw new TableException("Time attributes and custom field mappings are not supported yet.");
	}

	return createKafkaTableSink(
		schema,
		topic,
		getKafkaProperties(descriptorProperties),
		getFlinkKafkaPartitioner(descriptorProperties),
		getSerializationSchema(properties));
}

Example #4

Source File: KafkaTableSourceSinkFactoryBase.java From flink with Apache License 2.0

6 votes

@Override
public StreamTableSource<Row> createStreamTableSource(Map<String, String> properties) {
	final DescriptorProperties descriptorProperties = getValidatedProperties(properties);

	final String topic = descriptorProperties.getString(CONNECTOR_TOPIC);
	final DeserializationSchema<Row> deserializationSchema = getDeserializationSchema(properties);
	final StartupOptions startupOptions = getStartupOptions(descriptorProperties, topic);

	return createKafkaTableSource(
		TableSchemaUtils.getPhysicalSchema(descriptorProperties.getTableSchema(SCHEMA)),
		SchemaValidator.deriveProctimeAttribute(descriptorProperties),
		SchemaValidator.deriveRowtimeAttributes(descriptorProperties),
		SchemaValidator.deriveFieldMapping(
			descriptorProperties,
			Optional.of(deserializationSchema.getProducedType())),
		topic,
		getKafkaProperties(descriptorProperties),
		deserializationSchema,
		startupOptions.startupMode,
		startupOptions.specificOffsets,
		startupOptions.startupTimestampMillis);
}

Example #5

Source File: KafkaTableSourceSinkFactoryBase.java From flink with Apache License 2.0

6 votes

@Override
public StreamTableSource<Row> createStreamTableSource(Map<String, String> properties) {
	final DescriptorProperties descriptorProperties = getValidatedProperties(properties);

	final String topic = descriptorProperties.getString(CONNECTOR_TOPIC);
	final DeserializationSchema<Row> deserializationSchema = getDeserializationSchema(properties);
	final StartupOptions startupOptions = getStartupOptions(descriptorProperties, topic);

	return createKafkaTableSource(
		descriptorProperties.getTableSchema(SCHEMA),
		SchemaValidator.deriveProctimeAttribute(descriptorProperties),
		SchemaValidator.deriveRowtimeAttributes(descriptorProperties),
		SchemaValidator.deriveFieldMapping(
			descriptorProperties,
			Optional.of(deserializationSchema.getProducedType())),
		topic,
		getKafkaProperties(descriptorProperties),
		deserializationSchema,
		startupOptions.startupMode,
		startupOptions.specificOffsets);
}

Example #6

Source File: KafkaTableSourceSinkFactoryBase.java From flink with Apache License 2.0

6 votes

@Override
public StreamTableSink<Row> createStreamTableSink(Map<String, String> properties) {
	final DescriptorProperties descriptorProperties = getValidatedProperties(properties);

	final TableSchema schema = descriptorProperties.getTableSchema(SCHEMA);
	final String topic = descriptorProperties.getString(CONNECTOR_TOPIC);
	final Optional<String> proctime = SchemaValidator.deriveProctimeAttribute(descriptorProperties);
	final List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors =
		SchemaValidator.deriveRowtimeAttributes(descriptorProperties);

	// see also FLINK-9870
	if (proctime.isPresent() || !rowtimeAttributeDescriptors.isEmpty() ||
			checkForCustomFieldMapping(descriptorProperties, schema)) {
		throw new TableException("Time attributes and custom field mappings are not supported yet.");
	}

	return createKafkaTableSink(
		schema,
		topic,
		getKafkaProperties(descriptorProperties),
		getFlinkKafkaPartitioner(descriptorProperties),
		getSerializationSchema(properties));
}

Example #7

Source File: TestTableSourceFactoryBase.java From flink with Apache License 2.0

5 votes

@Override
public StreamTableSource<Row> createTableSource(TableSourceFactory.Context context) {
	TableSchema schema = context.getTable().getSchema();
	final DescriptorProperties params = new DescriptorProperties(true);
	params.putProperties(context.getTable().toProperties());
	final Optional<String> proctime = SchemaValidator.deriveProctimeAttribute(params);
	final List<RowtimeAttributeDescriptor> rowtime = SchemaValidator.deriveRowtimeAttributes(params);
	return new TestTableSource(
		schema,
		context.getTable().getProperties().get(testProperty),
		proctime.orElse(null),
		rowtime);
}

Example #8

Source File: ElasticsearchUpsertTableSinkFactoryBase.java From Flink-CEPplus with Apache License 2.0

5 votes

private DescriptorProperties getValidatedProperties(Map<String, String> properties) {
	final DescriptorProperties descriptorProperties = new DescriptorProperties(true);
	descriptorProperties.putProperties(properties);

	new StreamTableDescriptorValidator(true, false, true).validate(descriptorProperties);
	new SchemaValidator(true, false, false).validate(descriptorProperties);
	new ElasticsearchValidator().validate(descriptorProperties);

	return descriptorProperties;
}

Example #9

Source File: KafkaTableSourceSinkFactoryBase.java From flink with Apache License 2.0

5 votes

private boolean checkForCustomFieldMapping(DescriptorProperties descriptorProperties, TableSchema schema) {
	final Map<String, String> fieldMapping = SchemaValidator.deriveFieldMapping(
		descriptorProperties,
		Optional.of(schema.toRowType())); // until FLINK-9870 is fixed we assume that the table schema is the output type
	return fieldMapping.size() != schema.getFieldNames().length ||
		!fieldMapping.entrySet().stream().allMatch(mapping -> mapping.getKey().equals(mapping.getValue()));
}

Example #10

Source File: KafkaTableSourceSinkFactoryBase.java From flink with Apache License 2.0

5 votes

private DescriptorProperties getValidatedProperties(Map<String, String> properties) {
	final DescriptorProperties descriptorProperties = new DescriptorProperties(true);
	descriptorProperties.putProperties(properties);

	// allow Kafka timestamps to be used, watermarks can not be received from source
	new SchemaValidator(true, supportsKafkaTimestamps(), false).validate(descriptorProperties);
	new KafkaValidator().validate(descriptorProperties);

	return descriptorProperties;
}

Example #11

Source File: ElasticsearchUpsertTableSinkFactoryBase.java From flink with Apache License 2.0

5 votes

private DescriptorProperties getValidatedProperties(Map<String, String> properties) {
	final DescriptorProperties descriptorProperties = new DescriptorProperties(true);
	descriptorProperties.putProperties(properties);

	new StreamTableDescriptorValidator(true, false, true).validate(descriptorProperties);
	new SchemaValidator(true, false, false).validate(descriptorProperties);
	new ElasticsearchValidator().validate(descriptorProperties);

	return descriptorProperties;
}

Example #12

Source File: JdbcTableSourceSinkFactory.java From flink with Apache License 2.0

5 votes

private DescriptorProperties getValidatedProperties(Map<String, String> properties) {
	final DescriptorProperties descriptorProperties = new DescriptorProperties(true);
	descriptorProperties.putProperties(properties);

	new SchemaValidator(true, false, false).validate(descriptorProperties);
	new JdbcValidator().validate(descriptorProperties);

	return descriptorProperties;
}

Example #13

Source File: KuduTableFactory.java From bahir-flink with Apache License 2.0

5 votes

private DescriptorProperties getValidatedProps(Map<String, String> properties) {
    checkNotNull(properties.get(KUDU_MASTERS), "Missing required property " + KUDU_MASTERS);
    checkNotNull(properties.get(KUDU_TABLE), "Missing required property " + KUDU_TABLE);
    final DescriptorProperties descriptorProperties = new DescriptorProperties(true);
    descriptorProperties.putProperties(properties);
    new SchemaValidator(true, false, false).validate(descriptorProperties);
    return descriptorProperties;
}

Example #14

Source File: FlinkPravegaTableFactoryBase.java From flink-connectors with Apache License 2.0

5 votes

protected DescriptorProperties getValidatedProperties(Map<String, String> properties) {
    final DescriptorProperties descriptorProperties = new DescriptorProperties(true);
    descriptorProperties.putProperties(properties);

    boolean supportsSourceTimestamps = true;
    boolean supportsSourceWatermarks = true;
    new SchemaValidator(isStreamEnvironment(), supportsSourceTimestamps, supportsSourceWatermarks).validate(descriptorProperties);

    new PravegaValidator().validate(descriptorProperties);

    return descriptorProperties;
}

Example #15

Source File: TestTableSinkFactoryBase.java From flink with Apache License 2.0

5 votes

@Override
public StreamTableSink<Row> createStreamTableSink(Map<String, String> properties) {
	final DescriptorProperties params = new DescriptorProperties(true);
	params.putProperties(properties);
	return new TestTableSink(
			SchemaValidator.deriveTableSinkSchema(params),
			properties.get(testProperty));
}

Example #16

Source File: TestTableSourceFactoryBase.java From flink with Apache License 2.0

5 votes

@Override
public StreamTableSource<Row> createStreamTableSource(Map<String, String> properties) {
	final DescriptorProperties params = new DescriptorProperties(true);
	params.putProperties(properties);
	final Optional<String> proctime = SchemaValidator.deriveProctimeAttribute(params);
	final List<RowtimeAttributeDescriptor> rowtime = SchemaValidator.deriveRowtimeAttributes(params);
	return new TestTableSource(
		params.getTableSchema(SCHEMA),
		properties.get(testProperty),
		proctime.orElse(null),
		rowtime);
}

Example #17

Source File: CsvTableSinkFactoryBase.java From flink with Apache License 2.0

5 votes

protected CsvTableSink createTableSink(
		Boolean isStreaming,
		Map<String, String> properties) {

	DescriptorProperties params = new DescriptorProperties();
	params.putProperties(properties);

	// validate
	new FileSystemValidator().validate(params);
	new OldCsvValidator().validate(params);
	new SchemaValidator(isStreaming, false, false).validate(params);

	// build
	TableSchema formatSchema = params.getTableSchema(FORMAT_FIELDS);
	TableSchema tableSchema = params.getTableSchema(SCHEMA);

	if (!formatSchema.equals(tableSchema)) {
		throw new TableException(
				"Encodings that differ from the schema are not supported yet for CsvTableSink.");
	}

	String path = params.getString(CONNECTOR_PATH);
	String fieldDelimiter = params.getOptionalString(FORMAT_FIELD_DELIMITER).orElse(",");

	CsvTableSink csvTableSink = new CsvTableSink(path, fieldDelimiter);

	return (CsvTableSink) csvTableSink.configure(formatSchema.getFieldNames(), formatSchema.getFieldTypes());
}

Example #18

Source File: JDBCTableSourceSinkFactory.java From flink with Apache License 2.0

5 votes

private DescriptorProperties getValidatedProperties(Map<String, String> properties) {
	final DescriptorProperties descriptorProperties = new DescriptorProperties(true);
	descriptorProperties.putProperties(properties);

	new SchemaValidator(true, false, false).validate(descriptorProperties);
	new JDBCValidator().validate(descriptorProperties);

	return descriptorProperties;
}

Example #19

Source File: KafkaTableSourceSinkFactoryBase.java From flink with Apache License 2.0

5 votes

private boolean checkForCustomFieldMapping(DescriptorProperties descriptorProperties, TableSchema schema) {
	final Map<String, String> fieldMapping = SchemaValidator.deriveFieldMapping(
		descriptorProperties,
		Optional.of(schema.toRowType())); // until FLINK-9870 is fixed we assume that the table schema is the output type
	return fieldMapping.size() != schema.getFieldNames().length ||
		!fieldMapping.entrySet().stream().allMatch(mapping -> mapping.getKey().equals(mapping.getValue()));
}

Example #20

Source File: KafkaTableSourceSinkFactoryBase.java From flink with Apache License 2.0

5 votes

private DescriptorProperties getValidatedProperties(Map<String, String> properties) {
	final DescriptorProperties descriptorProperties = new DescriptorProperties(true);
	descriptorProperties.putProperties(properties);

	// allow Kafka timestamps to be used, watermarks can not be received from source
	new SchemaValidator(true, supportsKafkaTimestamps(), false).validate(descriptorProperties);
	new KafkaValidator().validate(descriptorProperties);

	return descriptorProperties;
}

Example #21

Source File: ElasticsearchUpsertTableSinkFactoryBase.java From flink with Apache License 2.0

5 votes

private DescriptorProperties getValidatedProperties(Map<String, String> properties) {
	final DescriptorProperties descriptorProperties = new DescriptorProperties(true);
	descriptorProperties.putProperties(properties);

	new StreamTableDescriptorValidator(true, false, true).validate(descriptorProperties);
	new SchemaValidator(true, false, false).validate(descriptorProperties);
	new ElasticsearchValidator().validate(descriptorProperties);

	return descriptorProperties;
}

Example #22

Source File: PulsarTableSourceSinkFactory.java From pulsar-flink with Apache License 2.0

5 votes

private boolean checkForCustomFieldMapping(DescriptorProperties descriptorProperties, TableSchema schema) {
    final Map<String, String> fieldMapping = SchemaValidator.deriveFieldMapping(
            descriptorProperties,
            Optional.of(schema.toRowType())); // until FLINK-9870 is fixed we assume that the table schema is the output type
    return fieldMapping.size() != schema.getFieldNames().length ||
            !fieldMapping.entrySet().stream().allMatch(mapping -> mapping.getKey().equals(mapping.getValue()));
}

Example #23

Source File: TestTableSinkFactoryBase.java From Flink-CEPplus with Apache License 2.0

5 votes

@Override
public StreamTableSink<Row> createStreamTableSink(Map<String, String> properties) {
	final DescriptorProperties params = new DescriptorProperties(true);
	params.putProperties(properties);
	return new TestTableSink(
			SchemaValidator.deriveTableSinkSchema(params),
			properties.get(testProperty));
}

Example #24

Source File: TestTableSourceFactoryBase.java From Flink-CEPplus with Apache License 2.0

5 votes

@Override
public StreamTableSource<Row> createStreamTableSource(Map<String, String> properties) {
	final DescriptorProperties params = new DescriptorProperties(true);
	params.putProperties(properties);
	final Optional<String> proctime = SchemaValidator.deriveProctimeAttribute(params);
	final List<RowtimeAttributeDescriptor> rowtime = SchemaValidator.deriveRowtimeAttributes(params);
	return new TestTableSource(
		params.getTableSchema(SCHEMA()),
		properties.get(testProperty),
		proctime.orElse(null),
		rowtime);
}

Example #25

Source File: KafkaTableSourceSinkFactoryBase.java From Flink-CEPplus with Apache License 2.0

5 votes

private boolean checkForCustomFieldMapping(DescriptorProperties descriptorProperties, TableSchema schema) {
	final Map<String, String> fieldMapping = SchemaValidator.deriveFieldMapping(
		descriptorProperties,
		Optional.of(schema.toRowType())); // until FLINK-9870 is fixed we assume that the table schema is the output type
	return fieldMapping.size() != schema.getFieldNames().length ||
		!fieldMapping.entrySet().stream().allMatch(mapping -> mapping.getKey().equals(mapping.getValue()));
}

Example #26

Source File: KafkaTableSourceSinkFactoryBase.java From Flink-CEPplus with Apache License 2.0

5 votes

private DescriptorProperties getValidatedProperties(Map<String, String> properties) {
	final DescriptorProperties descriptorProperties = new DescriptorProperties(true);
	descriptorProperties.putProperties(properties);

	// allow Kafka timestamps to be used, watermarks can not be received from source
	new SchemaValidator(true, supportsKafkaTimestamps(), false).validate(descriptorProperties);
	new KafkaValidator().validate(descriptorProperties);

	return descriptorProperties;
}

Example #27

Source File: CsvTableSourceFactoryBase.java From flink with Apache License 2.0

4 votes

protected CsvTableSource createTableSource(
		Boolean isStreaming,
		Map<String, String> properties) {

	DescriptorProperties params = new DescriptorProperties();
	params.putProperties(properties);

	// validate
	new FileSystemValidator().validate(params);
	new OldCsvValidator().validate(params);
	new SchemaValidator(isStreaming, false, false).validate(params);

	// build
	CsvTableSource.Builder csvTableSourceBuilder = new CsvTableSource.Builder();

	TableSchema formatSchema = params.getTableSchema(FORMAT_FIELDS);
	TableSchema tableSchema = params.getTableSchema(SCHEMA);

	// the CsvTableSource needs some rework first
	// for now the schema must be equal to the encoding
	if (!formatSchema.equals(tableSchema)) {
		throw new TableException(
				"Encodings that differ from the schema are not supported yet for CsvTableSources.");
	}

	params.getOptionalString(CONNECTOR_PATH).ifPresent(csvTableSourceBuilder::path);
	params.getOptionalString(FORMAT_FIELD_DELIMITER).ifPresent(csvTableSourceBuilder::fieldDelimiter);
	params.getOptionalString(FORMAT_LINE_DELIMITER).ifPresent(csvTableSourceBuilder::lineDelimiter);

	for (int i = 0; i < formatSchema.getFieldCount(); ++i) {
		csvTableSourceBuilder.field(formatSchema.getFieldNames()[i], formatSchema.getFieldTypes()[i]);
	}
	params.getOptionalCharacter(FORMAT_QUOTE_CHARACTER).ifPresent(csvTableSourceBuilder::quoteCharacter);
	params.getOptionalString(FORMAT_COMMENT_PREFIX).ifPresent(csvTableSourceBuilder::commentPrefix);
	params.getOptionalBoolean(FORMAT_IGNORE_FIRST_LINE).ifPresent(flag -> {
		if (flag) {
			csvTableSourceBuilder.ignoreFirstLine();
		}
	});

	params.getOptionalBoolean(FORMAT_IGNORE_PARSE_ERRORS).ifPresent(flag -> {
		if (flag) {
			csvTableSourceBuilder.ignoreParseErrors();
		}
	});

	return csvTableSourceBuilder.build();
}

Example #28

Source File: FlinkPravegaTableFactoryBase.java From flink-connectors with Apache License 2.0

4 votes

@SuppressWarnings("unchecked")
protected FlinkPravegaTableSource createFlinkPravegaTableSource(Map<String, String> properties) {
    final DescriptorProperties descriptorProperties = getValidatedProperties(properties);
    final TableSchema schema = TableSchemaUtils.getPhysicalSchema(descriptorProperties.getTableSchema(SCHEMA));
    final DeserializationSchema<Row> deserializationSchema = getDeserializationSchema(properties);

    ConnectorConfigurations connectorConfigurations = new ConnectorConfigurations();
    connectorConfigurations.parseConfigurations(descriptorProperties, ConnectorConfigurations.ConfigurationType.READER);

    // create source from the reader builder by using the supplied properties
    TableSourceReaderBuilder tableSourceReaderBuilder = new Pravega().tableSourceReaderBuilder();
    tableSourceReaderBuilder.withDeserializationSchema(deserializationSchema);

    if (connectorConfigurations.getAssignerWithTimeWindows().isPresent()) {
        tableSourceReaderBuilder.withTimestampAssigner(connectorConfigurations.getAssignerWithTimeWindows().get());
    }

    if (connectorConfigurations.getUid().isPresent()) {
        tableSourceReaderBuilder.uid(connectorConfigurations.getUid().get());
    }
    if (connectorConfigurations.getRgScope().isPresent()) {
        tableSourceReaderBuilder.withReaderGroupScope(connectorConfigurations.getRgScope().get());
    }
    if (connectorConfigurations.getRgName().isPresent()) {
        tableSourceReaderBuilder.withReaderGroupName(connectorConfigurations.getRgName().get());
    }
    if (connectorConfigurations.getRefreshInterval().isPresent()) {
        tableSourceReaderBuilder.withReaderGroupRefreshTime(Time.milliseconds(connectorConfigurations.getRefreshInterval().get()));
    }
    if (connectorConfigurations.getEventReadTimeoutInterval().isPresent()) {
        tableSourceReaderBuilder.withEventReadTimeout(Time.milliseconds(connectorConfigurations.getEventReadTimeoutInterval().get()));
    }
    if (connectorConfigurations.getCheckpointInitiateTimeoutInterval().isPresent()) {
        tableSourceReaderBuilder.withCheckpointInitiateTimeout(Time.milliseconds(connectorConfigurations.getCheckpointInitiateTimeoutInterval().get()));
    }

    tableSourceReaderBuilder.withPravegaConfig(connectorConfigurations.getPravegaConfig());
    if (connectorConfigurations.getMetrics().isPresent()) {
        tableSourceReaderBuilder.enableMetrics(connectorConfigurations.getMetrics().get());
    }
    tableSourceReaderBuilder.withPravegaConfig(connectorConfigurations.getPravegaConfig());
    for (StreamWithBoundaries streamWithBoundaries: connectorConfigurations.getReaderStreams()) {
        if (streamWithBoundaries.getFrom() != StreamCut.UNBOUNDED && streamWithBoundaries.getTo() != StreamCut.UNBOUNDED) {
            tableSourceReaderBuilder.forStream(streamWithBoundaries.getStream(), streamWithBoundaries.getFrom(), streamWithBoundaries.getTo());
        } else if (streamWithBoundaries.getFrom() != StreamCut.UNBOUNDED) {
            tableSourceReaderBuilder.forStream(streamWithBoundaries.getStream(), streamWithBoundaries.getFrom());
        } else {
            tableSourceReaderBuilder.forStream(streamWithBoundaries.getStream());
        }
    }

    FlinkPravegaTableSource flinkPravegaTableSource = new FlinkPravegaTableSource(
                                                                tableSourceReaderBuilder::buildSourceFunction,
                                                                tableSourceReaderBuilder::buildInputFormat, schema);
    flinkPravegaTableSource.setRowtimeAttributeDescriptors(SchemaValidator.deriveRowtimeAttributes(descriptorProperties));
    Optional<String> procTimeAttribute = SchemaValidator.deriveProctimeAttribute(descriptorProperties);
    if (procTimeAttribute.isPresent()) {
        flinkPravegaTableSource.setProctimeAttribute(procTimeAttribute.get());
    }

    return flinkPravegaTableSource;
}

Example #29

Source File: CsvTableSinkFactoryBase.java From flink with Apache License 2.0

4 votes

protected CsvTableSink createTableSink(
		Boolean isStreaming,
		Map<String, String> properties) {

	DescriptorProperties params = new DescriptorProperties();
	params.putProperties(properties);

	// validate
	new FileSystemValidator().validate(params);
	new OldCsvValidator().validate(params);
	new SchemaValidator(isStreaming, false, false).validate(params);

	// build
	TableSchema tableSchema = TableSchemaUtils.getPhysicalSchema(params.getTableSchema(SCHEMA));

	// if a schema is defined, no matter derive schema is set or not, will use the defined schema
	final boolean hasSchema = params.hasPrefix(FORMAT_FIELDS);
	if (hasSchema) {
		TableSchema formatSchema = params.getTableSchema(FORMAT_FIELDS);
		if (!getFieldLogicalTypes(formatSchema).equals(getFieldLogicalTypes(tableSchema))) {
			throw new TableException(String.format(
					"Encodings that differ from the schema are not supported yet for" +
							" CsvTableSink, format schema is '%s', but table schema is '%s'.",
					formatSchema,
					tableSchema));
		}
	}

	String path = params.getString(CONNECTOR_PATH);
	String fieldDelimiter = params.getOptionalString(FORMAT_FIELD_DELIMITER).orElse(",");
	Optional<String> writeModeParm = params.getOptionalString(FORMAT_WRITE_MODE);
	FileSystem.WriteMode writeMode =
			(writeModeParm.isPresent()) ? FileSystem.WriteMode.valueOf(writeModeParm.get()) : null;
	int numFiles = params.getOptionalInt(FORMAT_NUM_FILES).orElse(-1);

	// bridge to java.sql.Timestamp/Time/Date
	DataType[] dataTypes = Arrays.stream(tableSchema.getFieldDataTypes())
		.map(dt -> {
			switch (dt.getLogicalType().getTypeRoot()) {
				case TIMESTAMP_WITHOUT_TIME_ZONE:
					return dt.bridgedTo(Timestamp.class);
				case TIME_WITHOUT_TIME_ZONE:
					return dt.bridgedTo(Time.class);
				case DATE:
					return dt.bridgedTo(Date.class);
				default:
					return dt;
			}
		})
		.toArray(DataType[]::new);

	return new CsvTableSink(
		path,
		fieldDelimiter,
		numFiles,
		writeMode,
		tableSchema.getFieldNames(),
		dataTypes);
}

Example #30

Source File: CsvTableSourceFactoryBase.java From flink with Apache License 2.0

4 votes

protected CsvTableSource createTableSource(
		Boolean isStreaming,
		Map<String, String> properties) {

	DescriptorProperties params = new DescriptorProperties();
	params.putProperties(properties);

	// validate
	new FileSystemValidator().validate(params);
	new OldCsvValidator().validate(params);
	new SchemaValidator(isStreaming, false, false).validate(params);

	// build
	CsvTableSource.Builder csvTableSourceBuilder = new CsvTableSource.Builder();

	TableSchema tableSchema = TableSchemaUtils.getPhysicalSchema(params.getTableSchema(SCHEMA));

	// if a schema is defined, no matter derive schema is set or not, will use the defined schema
	final boolean hasSchema = params.hasPrefix(FORMAT_FIELDS);
	if (hasSchema) {
		TableSchema formatSchema = params.getTableSchema(FORMAT_FIELDS);
		// the CsvTableSource needs some rework first
		// for now the schema must be equal to the encoding
		// Ignore conversion classes in DataType
		if (!getFieldLogicalTypes(formatSchema).equals(getFieldLogicalTypes(tableSchema))) {
			throw new TableException(String.format(
					"Encodings that differ from the schema are not supported yet for" +
							" CsvTableSource, format schema is '%s', but table schema is '%s'.",
					formatSchema,
					tableSchema));
		}
	}

	params.getOptionalString(CONNECTOR_PATH).ifPresent(csvTableSourceBuilder::path);
	params.getOptionalString(FORMAT_FIELD_DELIMITER).ifPresent(csvTableSourceBuilder::fieldDelimiter);
	params.getOptionalString(FORMAT_LINE_DELIMITER).ifPresent(csvTableSourceBuilder::lineDelimiter);

	for (int i = 0; i < tableSchema.getFieldCount(); ++i) {
		csvTableSourceBuilder.field(tableSchema.getFieldNames()[i], tableSchema.getFieldDataTypes()[i]);
	}
	params.getOptionalCharacter(FORMAT_QUOTE_CHARACTER).ifPresent(csvTableSourceBuilder::quoteCharacter);
	params.getOptionalString(FORMAT_COMMENT_PREFIX).ifPresent(csvTableSourceBuilder::commentPrefix);
	params.getOptionalBoolean(FORMAT_IGNORE_FIRST_LINE).ifPresent(flag -> {
		if (flag) {
			csvTableSourceBuilder.ignoreFirstLine();
		}
	});

	params.getOptionalBoolean(FORMAT_IGNORE_PARSE_ERRORS).ifPresent(flag -> {
		if (flag) {
			csvTableSourceBuilder.ignoreParseErrors();
		}
	});

	return csvTableSourceBuilder.build();
}