Java Code Examples for org.apache.flink.table.descriptors.DescriptorProperties#containsKey()

The following examples show how to use org.apache.flink.table.descriptors.DescriptorProperties#containsKey() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: FlinkCalciteCatalogReader.java From flink with Apache License 2.0

6 votes

/**
 * Checks whether the {@link CatalogTable} uses legacy connector source options.
 */
private static boolean isLegacySourceOptions(
		CatalogTable catalogTable,
		CatalogSchemaTable schemaTable) {
	// normalize option keys
	DescriptorProperties properties = new DescriptorProperties(true);
	properties.putProperties(catalogTable.getOptions());
	if (properties.containsKey(ConnectorDescriptorValidator.CONNECTOR_TYPE)) {
		return true;
	} else {
		// try to create legacy table source using the options,
		// some legacy factories uses the new 'connector' key
		try {
			TableFactoryUtil.findAndCreateTableSource(
				schemaTable.getCatalog(),
				schemaTable.getTableIdentifier(),
				catalogTable,
				new Configuration());
			// success, then we will use the legacy factories
			return true;
		} catch (Throwable e) {
			// fail, then we will use new factories
			return false;
		}
	}
}

Example 2

Source File: JsonRowFormatFactory.java From flink with Apache License 2.0

5 votes

private TypeInformation<Row> createTypeInformation(DescriptorProperties descriptorProperties) {
	if (descriptorProperties.containsKey(JsonValidator.FORMAT_SCHEMA)) {
		return (RowTypeInfo) descriptorProperties.getType(JsonValidator.FORMAT_SCHEMA);
	} else if (descriptorProperties.containsKey(JsonValidator.FORMAT_JSON_SCHEMA)) {
		return JsonRowSchemaConverter.convert(descriptorProperties.getString(JsonValidator.FORMAT_JSON_SCHEMA));
	} else {
		return deriveSchema(descriptorProperties.asMap()).toRowType();
	}
}

Example 3

Source File: JsonRowFormatFactory.java From flink with Apache License 2.0

5 votes

private TypeInformation<Row> createTypeInformation(DescriptorProperties descriptorProperties) {
	if (descriptorProperties.containsKey(JsonValidator.FORMAT_SCHEMA)) {
		return (RowTypeInfo) descriptorProperties.getType(JsonValidator.FORMAT_SCHEMA);
	} else if (descriptorProperties.containsKey(JsonValidator.FORMAT_JSON_SCHEMA)) {
		return JsonRowSchemaConverter.convert(descriptorProperties.getString(JsonValidator.FORMAT_JSON_SCHEMA));
	} else {
		return deriveSchema(descriptorProperties.asMap()).toRowType();
	}
}

Example 4

Source File: AvroRowFormatFactory.java From flink with Apache License 2.0

5 votes

@Override
public SerializationSchema<Row> createSerializationSchema(Map<String, String> properties) {
	final DescriptorProperties descriptorProperties = getValidatedProperties(properties);

	// create and configure
	if (descriptorProperties.containsKey(AvroValidator.FORMAT_RECORD_CLASS)) {
		return new AvroRowSerializationSchema(
			descriptorProperties.getClass(AvroValidator.FORMAT_RECORD_CLASS, SpecificRecord.class));
	} else {
		return new AvroRowSerializationSchema(descriptorProperties.getString(AvroValidator.FORMAT_AVRO_SCHEMA));
	}
}

Example 5

Source File: AvroRowFormatFactory.java From flink with Apache License 2.0

5 votes

@Override
public DeserializationSchema<Row> createDeserializationSchema(Map<String, String> properties) {
	final DescriptorProperties descriptorProperties = getValidatedProperties(properties);

	// create and configure
	if (descriptorProperties.containsKey(AvroValidator.FORMAT_RECORD_CLASS)) {
		return new AvroRowDeserializationSchema(
			descriptorProperties.getClass(AvroValidator.FORMAT_RECORD_CLASS, SpecificRecord.class));
	} else {
		return new AvroRowDeserializationSchema(descriptorProperties.getString(AvroValidator.FORMAT_AVRO_SCHEMA));
	}
}

Example 6

Source File: CsvRowFormatFactory.java From flink with Apache License 2.0

5 votes

private static TypeInformation<Row> createTypeInformation(DescriptorProperties descriptorProperties) {
	if (descriptorProperties.containsKey(CsvValidator.FORMAT_SCHEMA)) {
		return (RowTypeInfo) descriptorProperties.getType(CsvValidator.FORMAT_SCHEMA);
	} else {
		return deriveSchema(descriptorProperties.asMap()).toRowType();
	}
}

Example 7

Source File: ElasticsearchUpsertTableSinkFactoryBase.java From flink with Apache License 2.0

5 votes

private List<Host> getHosts(DescriptorProperties descriptorProperties) {
	if (descriptorProperties.containsKey(CONNECTOR_HOSTS)) {
		return validateAndParseHostsString(descriptorProperties);
	} else {
		final List<Map<String, String>> hosts = descriptorProperties.getFixedIndexedProperties(
			CONNECTOR_HOSTS,
			Arrays.asList(CONNECTOR_HOSTS_HOSTNAME, CONNECTOR_HOSTS_PORT, CONNECTOR_HOSTS_PROTOCOL));
		return hosts.stream()
			.map(host -> new Host(
				descriptorProperties.getString(host.get(CONNECTOR_HOSTS_HOSTNAME)),
				descriptorProperties.getInt(host.get(CONNECTOR_HOSTS_PORT)),
				descriptorProperties.getString(host.get(CONNECTOR_HOSTS_PROTOCOL))))
			.collect(Collectors.toList());
	}
}

Example 8

Source File: ConnectorConfigurations.java From flink-connectors with Apache License 2.0

5 votes

private void populateWriterConfig(DescriptorProperties descriptorProperties) {
    Optional<String> streamScope = descriptorProperties.getOptionalString(CONNECTOR_WRITER_SCOPE);

    if (!defaultScope.isPresent() && !streamScope.isPresent()) {
        throw new ValidationException("Must supply either " + CONNECTOR_WRITER_SCOPE + " or " + CONNECTOR_CONNECTION_CONFIG_DEFAULT_SCOPE);
    }

    final String scopeVal = streamScope.isPresent() ? streamScope.get() : defaultScope.get();

    if (!descriptorProperties.containsKey(CONNECTOR_WRITER_STREAM)) {
        throw new ValidationException("Missing " + CONNECTOR_WRITER_STREAM + " configuration.");
    }
    final String streamName = descriptorProperties.getString(CONNECTOR_WRITER_STREAM);
    writerStream = Stream.of(scopeVal, streamName);

    txnLeaseRenewalInterval = descriptorProperties.getOptionalLong(CONNECTOR_WRITER_TXN_LEASE_RENEWAL_INTERVAL);

    if (!descriptorProperties.containsKey(CONNECTOR_WRITER_ROUTING_KEY_FILED_NAME)) {
        throw new ValidationException("Missing " + CONNECTOR_WRITER_ROUTING_KEY_FILED_NAME + " configuration.");
    }
    watermark = descriptorProperties.getOptionalBoolean(CONNECTOR_WRITER_ENABLE_WATERMARK);
    routingKey = descriptorProperties.getString(CONNECTOR_WRITER_ROUTING_KEY_FILED_NAME);

    Optional<String> optionalMode = descriptorProperties.getOptionalString(CONNECTOR_WRITER_MODE);
    if (optionalMode.isPresent()) {
        String mode = optionalMode.get();
        if (mode.equals(CONNECTOR_WRITER_MODE_VALUE_ATLEAST_ONCE)) {
            writerMode = Optional.of(PravegaWriterMode.ATLEAST_ONCE);
        } else if (mode.equals(CONNECTOR_WRITER_MODE_VALUE_EXACTLY_ONCE)) {
            writerMode = Optional.of(PravegaWriterMode.EXACTLY_ONCE);
        } else {
            throw new ValidationException("Invalid writer mode " + mode + " passed. Supported values: ("
                    + CONNECTOR_WRITER_MODE_VALUE_ATLEAST_ONCE + " or " + CONNECTOR_WRITER_MODE_VALUE_EXACTLY_ONCE + ")");
        }
    }
}

Example 9

Source File: PravegaValidator.java From flink-connectors with Apache License 2.0

5 votes

@Override
public void validate(DescriptorProperties properties) {
    super.validate(properties);
    properties.validateValue(CONNECTOR_TYPE, CONNECTOR_TYPE_VALUE_PRAVEGA, false);
    validateConnectionConfig(properties);
    if (properties.containsKey(CONNECTOR_READER)) {
        validateReaderConfigurations(properties);
    }
    if (properties.containsKey(CONNECTOR_WRITER)) {
        validateWriterConfigurations(properties);
    }
}

Example 10

Source File: CsvRowFormatFactory.java From Flink-CEPplus with Apache License 2.0

5 votes

private static TypeInformation<Row> createTypeInformation(DescriptorProperties descriptorProperties) {
	if (descriptorProperties.containsKey(CsvValidator.FORMAT_SCHEMA)) {
		return (RowTypeInfo) descriptorProperties.getType(CsvValidator.FORMAT_SCHEMA);
	} else {
		return deriveSchema(descriptorProperties.asMap()).toRowType();
	}
}

Example 11

Source File: AvroRowFormatFactory.java From flink with Apache License 2.0

5 votes

@Override
public SerializationSchema<Row> createSerializationSchema(Map<String, String> properties) {
	final DescriptorProperties descriptorProperties = getValidatedProperties(properties);

	// create and configure
	if (descriptorProperties.containsKey(AvroValidator.FORMAT_RECORD_CLASS)) {
		return new AvroRowSerializationSchema(
			descriptorProperties.getClass(AvroValidator.FORMAT_RECORD_CLASS, SpecificRecord.class));
	} else {
		return new AvroRowSerializationSchema(descriptorProperties.getString(AvroValidator.FORMAT_AVRO_SCHEMA));
	}
}

Example 12

Source File: AvroRowFormatFactory.java From flink with Apache License 2.0

5 votes

@Override
public DeserializationSchema<Row> createDeserializationSchema(Map<String, String> properties) {
	final DescriptorProperties descriptorProperties = getValidatedProperties(properties);

	// create and configure
	if (descriptorProperties.containsKey(AvroValidator.FORMAT_RECORD_CLASS)) {
		return new AvroRowDeserializationSchema(
			descriptorProperties.getClass(AvroValidator.FORMAT_RECORD_CLASS, SpecificRecord.class));
	} else {
		return new AvroRowDeserializationSchema(descriptorProperties.getString(AvroValidator.FORMAT_AVRO_SCHEMA));
	}
}

Example 13

Source File: CsvRowFormatFactory.java From flink with Apache License 2.0

5 votes

private static TypeInformation<Row> createTypeInformation(DescriptorProperties descriptorProperties) {
	if (descriptorProperties.containsKey(CsvValidator.FORMAT_SCHEMA)) {
		return (RowTypeInfo) descriptorProperties.getType(CsvValidator.FORMAT_SCHEMA);
	} else {
		return deriveSchema(descriptorProperties.asMap()).toRowType();
	}
}

Example 14

Source File: PulsarSchemaValidator.java From pulsar-flink with Apache License 2.0

5 votes

@Override
public void validate(DescriptorProperties properties) {
    Map<String, String> names = properties.getIndexedProperty(SCHEMA, SCHEMA_NAME);
    Map<String, String> types = properties.getIndexedProperty(SCHEMA, SCHEMA_TYPE);

    boolean proctimeFound = false;

    int fieldsCount = Math.max(names.size(), types.size());

    for (int i = 0; i < fieldsCount; i++) {
        properties.validateString(SCHEMA + "." + i + "." + SCHEMA_NAME, false, 1);
        properties.validateType(SCHEMA + "." + i + "." + SCHEMA_TYPE, false, false);
        properties.validateString(SCHEMA + "." + i + "." + SCHEMA_FROM, true, 1);
        // either proctime or rowtime
        String proctime = SCHEMA + "." + i + "." + SCHEMA_PROCTIME;
        String rowtime = SCHEMA + "." + i + "." + ROWTIME;

        if (properties.containsKey(proctime)) {
            if (!isStreamEnvironment) {
                throw new ValidationException(
                        "Property $proctime is not allowed in a batch environment.");
            } else if (proctimeFound) {
                throw new ValidationException("A proctime attribute must only be defined once.");
            }
            // check proctime
            properties.validateBoolean(proctime, false);
            proctimeFound = properties.getBoolean(proctime);
            // no rowtime
            properties.validatePrefixExclusion(rowtime);
        } else if (properties.hasPrefix(rowtime)) {
            // check rowtime
            RowtimeValidator rowtimeValidator =
                    new RowtimeValidator(
                            supportsSourceTimestamps, supportsSourceWatermarks, SCHEMA + "." + i + ".");
            rowtimeValidator.validate(properties);
            // no proctime
            properties.validateExclusion(proctime);
        }
    }
}

Example 15

Source File: PulsarCatalogValidator.java From pulsar-flink with Apache License 2.0

5 votes

private void validateStartingOffsets(DescriptorProperties properties) {
    if (properties.containsKey(CATALOG_STARTUP_MODE)) {
        String v = properties.getString(CATALOG_STARTUP_MODE);
        if (!v.equals("earliest") && !v.equals("latest")) {
            throw new ValidationException(CATALOG_STARTUP_MODE + " should be either earliest or latest");
        }
    }
}

Example 16

Source File: JsonRowFormatFactory.java From Flink-CEPplus with Apache License 2.0

5 votes

private TypeInformation<Row> createTypeInformation(DescriptorProperties descriptorProperties) {
	if (descriptorProperties.containsKey(JsonValidator.FORMAT_SCHEMA)) {
		return (RowTypeInfo) descriptorProperties.getType(JsonValidator.FORMAT_SCHEMA);
	} else if (descriptorProperties.containsKey(JsonValidator.FORMAT_JSON_SCHEMA)) {
		return JsonRowSchemaConverter.convert(descriptorProperties.getString(JsonValidator.FORMAT_JSON_SCHEMA));
	} else {
		return deriveSchema(descriptorProperties.asMap()).toRowType();
	}
}

Example 17

Source File: AvroRowFormatFactory.java From Flink-CEPplus with Apache License 2.0

5 votes

@Override
public SerializationSchema<Row> createSerializationSchema(Map<String, String> properties) {
	final DescriptorProperties descriptorProperties = getValidatedProperties(properties);

	// create and configure
	if (descriptorProperties.containsKey(AvroValidator.FORMAT_RECORD_CLASS)) {
		return new AvroRowSerializationSchema(
			descriptorProperties.getClass(AvroValidator.FORMAT_RECORD_CLASS, SpecificRecord.class));
	} else {
		return new AvroRowSerializationSchema(descriptorProperties.getString(AvroValidator.FORMAT_AVRO_SCHEMA));
	}
}

Example 18

Source File: AvroRowFormatFactory.java From Flink-CEPplus with Apache License 2.0

5 votes

@Override
public DeserializationSchema<Row> createDeserializationSchema(Map<String, String> properties) {
	final DescriptorProperties descriptorProperties = getValidatedProperties(properties);

	// create and configure
	if (descriptorProperties.containsKey(AvroValidator.FORMAT_RECORD_CLASS)) {
		return new AvroRowDeserializationSchema(
			descriptorProperties.getClass(AvroValidator.FORMAT_RECORD_CLASS, SpecificRecord.class));
	} else {
		return new AvroRowDeserializationSchema(descriptorProperties.getString(AvroValidator.FORMAT_AVRO_SCHEMA));
	}
}