Java Code Examples for org.apache.flink.table.descriptors.DescriptorProperties#hasPrefix()

@Override
public void validate(DescriptorProperties properties) {
    Map<String, String> names = properties.getIndexedProperty(SCHEMA, SCHEMA_NAME);
    Map<String, String> types = properties.getIndexedProperty(SCHEMA, SCHEMA_TYPE);

    boolean proctimeFound = false;

    int fieldsCount = Math.max(names.size(), types.size());

    for (int i = 0; i < fieldsCount; i++) {
        properties.validateString(SCHEMA + "." + i + "." + SCHEMA_NAME, false, 1);
        properties.validateType(SCHEMA + "." + i + "." + SCHEMA_TYPE, false, false);
        properties.validateString(SCHEMA + "." + i + "." + SCHEMA_FROM, true, 1);
        // either proctime or rowtime
        String proctime = SCHEMA + "." + i + "." + SCHEMA_PROCTIME;
        String rowtime = SCHEMA + "." + i + "." + ROWTIME;

        if (properties.containsKey(proctime)) {
            if (!isStreamEnvironment) {
                throw new ValidationException(
                        "Property $proctime is not allowed in a batch environment.");
            } else if (proctimeFound) {
                throw new ValidationException("A proctime attribute must only be defined once.");
            }
            // check proctime
            properties.validateBoolean(proctime, false);
            proctimeFound = properties.getBoolean(proctime);
            // no rowtime
            properties.validatePrefixExclusion(rowtime);
        } else if (properties.hasPrefix(rowtime)) {
            // check rowtime
            RowtimeValidator rowtimeValidator =
                    new RowtimeValidator(
                            supportsSourceTimestamps, supportsSourceWatermarks, SCHEMA + "." + i + ".");
            rowtimeValidator.validate(properties);
            // no proctime
            properties.validateExclusion(proctime);
        }
    }
}
 

protected CsvTableSink createTableSink(
		Boolean isStreaming,
		Map<String, String> properties) {

	DescriptorProperties params = new DescriptorProperties();
	params.putProperties(properties);

	// validate
	new FileSystemValidator().validate(params);
	new OldCsvValidator().validate(params);
	new SchemaValidator(isStreaming, false, false).validate(params);

	// build
	TableSchema tableSchema = TableSchemaUtils.getPhysicalSchema(params.getTableSchema(SCHEMA));

	// if a schema is defined, no matter derive schema is set or not, will use the defined schema
	final boolean hasSchema = params.hasPrefix(FORMAT_FIELDS);
	if (hasSchema) {
		TableSchema formatSchema = params.getTableSchema(FORMAT_FIELDS);
		if (!getFieldLogicalTypes(formatSchema).equals(getFieldLogicalTypes(tableSchema))) {
			throw new TableException(String.format(
					"Encodings that differ from the schema are not supported yet for" +
							" CsvTableSink, format schema is '%s', but table schema is '%s'.",
					formatSchema,
					tableSchema));
		}
	}

	String path = params.getString(CONNECTOR_PATH);
	String fieldDelimiter = params.getOptionalString(FORMAT_FIELD_DELIMITER).orElse(",");
	Optional<String> writeModeParm = params.getOptionalString(FORMAT_WRITE_MODE);
	FileSystem.WriteMode writeMode =
			(writeModeParm.isPresent()) ? FileSystem.WriteMode.valueOf(writeModeParm.get()) : null;
	int numFiles = params.getOptionalInt(FORMAT_NUM_FILES).orElse(-1);

	// bridge to java.sql.Timestamp/Time/Date
	DataType[] dataTypes = Arrays.stream(tableSchema.getFieldDataTypes())
		.map(dt -> {
			switch (dt.getLogicalType().getTypeRoot()) {
				case TIMESTAMP_WITHOUT_TIME_ZONE:
					return dt.bridgedTo(Timestamp.class);
				case TIME_WITHOUT_TIME_ZONE:
					return dt.bridgedTo(Time.class);
				case DATE:
					return dt.bridgedTo(Date.class);
				default:
					return dt;
			}
		})
		.toArray(DataType[]::new);

	return new CsvTableSink(
		path,
		fieldDelimiter,
		numFiles,
		writeMode,
		tableSchema.getFieldNames(),
		dataTypes);
}
 

protected CsvTableSource createTableSource(
		Boolean isStreaming,
		Map<String, String> properties) {

	DescriptorProperties params = new DescriptorProperties();
	params.putProperties(properties);

	// validate
	new FileSystemValidator().validate(params);
	new OldCsvValidator().validate(params);
	new SchemaValidator(isStreaming, false, false).validate(params);

	// build
	CsvTableSource.Builder csvTableSourceBuilder = new CsvTableSource.Builder();

	TableSchema tableSchema = TableSchemaUtils.getPhysicalSchema(params.getTableSchema(SCHEMA));

	// if a schema is defined, no matter derive schema is set or not, will use the defined schema
	final boolean hasSchema = params.hasPrefix(FORMAT_FIELDS);
	if (hasSchema) {
		TableSchema formatSchema = params.getTableSchema(FORMAT_FIELDS);
		// the CsvTableSource needs some rework first
		// for now the schema must be equal to the encoding
		// Ignore conversion classes in DataType
		if (!getFieldLogicalTypes(formatSchema).equals(getFieldLogicalTypes(tableSchema))) {
			throw new TableException(String.format(
					"Encodings that differ from the schema are not supported yet for" +
							" CsvTableSource, format schema is '%s', but table schema is '%s'.",
					formatSchema,
					tableSchema));
		}
	}

	params.getOptionalString(CONNECTOR_PATH).ifPresent(csvTableSourceBuilder::path);
	params.getOptionalString(FORMAT_FIELD_DELIMITER).ifPresent(csvTableSourceBuilder::fieldDelimiter);
	params.getOptionalString(FORMAT_LINE_DELIMITER).ifPresent(csvTableSourceBuilder::lineDelimiter);

	for (int i = 0; i < tableSchema.getFieldCount(); ++i) {
		csvTableSourceBuilder.field(tableSchema.getFieldNames()[i], tableSchema.getFieldDataTypes()[i]);
	}
	params.getOptionalCharacter(FORMAT_QUOTE_CHARACTER).ifPresent(csvTableSourceBuilder::quoteCharacter);
	params.getOptionalString(FORMAT_COMMENT_PREFIX).ifPresent(csvTableSourceBuilder::commentPrefix);
	params.getOptionalBoolean(FORMAT_IGNORE_FIRST_LINE).ifPresent(flag -> {
		if (flag) {
			csvTableSourceBuilder.ignoreFirstLine();
		}
	});

	params.getOptionalBoolean(FORMAT_IGNORE_PARSE_ERRORS).ifPresent(flag -> {
		if (flag) {
			csvTableSourceBuilder.ignoreParseErrors();
		}
	});

	return csvTableSourceBuilder.build();
}