org.apache.flink.table.utils.TableSchemaUtils Java Examples

The following examples show how to use org.apache.flink.table.utils.TableSchemaUtils. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: KafkaTableSourceBase.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Creates a generic Kafka {@link StreamTableSource}.
 *
 * @param schema                      Schema of the produced table.
 * @param proctimeAttribute           Field name of the processing time attribute.
 * @param rowtimeAttributeDescriptors Descriptor for a rowtime attribute
 * @param fieldMapping                Mapping for the fields of the table schema to
 *                                    fields of the physical returned type.
 * @param topic                       Kafka topic to consume.
 * @param properties                  Properties for the Kafka consumer.
 * @param deserializationSchema       Deserialization schema for decoding records from Kafka.
 * @param startupMode                 Startup mode for the contained consumer.
 * @param specificStartupOffsets      Specific startup offsets; only relevant when startup
 *                                    mode is {@link StartupMode#SPECIFIC_OFFSETS}.
 * @param startupTimestampMillis	  Startup timestamp for offsets; only relevant when startup
 *                                    mode is {@link StartupMode#TIMESTAMP}.
 */
protected KafkaTableSourceBase(
		TableSchema schema,
		Optional<String> proctimeAttribute,
		List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors,
		Optional<Map<String, String>> fieldMapping,
		String topic,
		Properties properties,
		DeserializationSchema<Row> deserializationSchema,
		StartupMode startupMode,
		Map<KafkaTopicPartition, Long> specificStartupOffsets,
		long startupTimestampMillis) {
	this.schema = TableSchemaUtils.checkNoGeneratedColumns(schema);
	this.proctimeAttribute = validateProctimeAttribute(proctimeAttribute);
	this.rowtimeAttributeDescriptors = validateRowtimeAttributeDescriptors(rowtimeAttributeDescriptors);
	this.fieldMapping = fieldMapping;
	this.topic = Preconditions.checkNotNull(topic, "Topic must not be null.");
	this.properties = Preconditions.checkNotNull(properties, "Properties must not be null.");
	this.deserializationSchema = Preconditions.checkNotNull(
		deserializationSchema, "Deserialization schema must not be null.");
	this.startupMode = Preconditions.checkNotNull(startupMode, "Startup mode must not be null.");
	this.specificStartupOffsets = Preconditions.checkNotNull(
		specificStartupOffsets, "Specific offsets must not be null.");
	this.startupTimestampMillis = startupTimestampMillis;
}
 
Example #2
Source File: Elasticsearch6DynamicSinkFactory.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public DynamicTableSink createDynamicTableSink(Context context) {
	TableSchema tableSchema = context.getCatalogTable().getSchema();
	ElasticsearchValidationUtils.validatePrimaryKey(tableSchema);
	final FactoryUtil.TableFactoryHelper helper = FactoryUtil.createTableFactoryHelper(this, context);

	final EncodingFormat<SerializationSchema<RowData>> format = helper.discoverEncodingFormat(
		SerializationFormatFactory.class,
		FORMAT_OPTION);

	helper.validate();
	Configuration configuration = new Configuration();
	context.getCatalogTable()
		.getOptions()
		.forEach(configuration::setString);
	Elasticsearch6Configuration config = new Elasticsearch6Configuration(configuration, context.getClassLoader());

	validate(config, configuration);

	return new Elasticsearch6DynamicSink(
		format,
		config,
		TableSchemaUtils.getPhysicalSchema(tableSchema));
}
 
Example #3
Source File: ElasticsearchUpsertTableSinkFactoryBase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public StreamTableSink<Tuple2<Boolean, Row>> createStreamTableSink(Map<String, String> properties) {
	final DescriptorProperties descriptorProperties = getValidatedProperties(properties);

	return createElasticsearchUpsertTableSink(
		descriptorProperties.isValue(UPDATE_MODE, UPDATE_MODE_VALUE_APPEND),
		TableSchemaUtils.getPhysicalSchema(descriptorProperties.getTableSchema(SCHEMA)),
		getHosts(descriptorProperties),
		descriptorProperties.getString(CONNECTOR_INDEX),
		descriptorProperties.getString(CONNECTOR_DOCUMENT_TYPE),
		descriptorProperties.getOptionalString(CONNECTOR_KEY_DELIMITER).orElse(DEFAULT_KEY_DELIMITER),
		descriptorProperties.getOptionalString(CONNECTOR_KEY_NULL_LITERAL).orElse(DEFAULT_KEY_NULL_LITERAL),
		getSerializationSchema(properties),
		SUPPORTED_CONTENT_TYPE,
		getFailureHandler(descriptorProperties),
		getSinkOptions(descriptorProperties));
}
 
Example #4
Source File: HBaseTableFactory.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public StreamTableSource<Row> createStreamTableSource(Map<String, String> properties) {
	final DescriptorProperties descriptorProperties = getValidatedProperties(properties);
	// create default configuration from current runtime env (`hbase-site.xml` in classpath) first,
	Configuration hbaseClientConf = HBaseConfigurationUtil.getHBaseConfiguration();
	String hbaseZk = descriptorProperties.getString(CONNECTOR_ZK_QUORUM);
	hbaseClientConf.set(HConstants.ZOOKEEPER_QUORUM, hbaseZk);
	descriptorProperties
		.getOptionalString(CONNECTOR_ZK_NODE_PARENT)
		.ifPresent(v -> hbaseClientConf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, v));

	String hTableName = descriptorProperties.getString(CONNECTOR_TABLE_NAME);
	TableSchema tableSchema = TableSchemaUtils.getPhysicalSchema(
		descriptorProperties.getTableSchema(SCHEMA));
	HBaseTableSchema hbaseSchema = validateTableSchema(tableSchema);
	return new HBaseTableSource(hbaseClientConf, hTableName, hbaseSchema, null);
}
 
Example #5
Source File: JdbcValidator.java    From flink with Apache License 2.0 6 votes vote down vote up
private void validateCommonProperties(DescriptorProperties properties) {
	properties.validateString(CONNECTOR_URL, false, 1);
	properties.validateString(CONNECTOR_TABLE, false, 1);
	properties.validateString(CONNECTOR_DRIVER, true);
	properties.validateString(CONNECTOR_USERNAME, true);
	properties.validateString(CONNECTOR_PASSWORD, true);

	final String url = properties.getString(CONNECTOR_URL);
	final Optional<JdbcDialect> dialect = JdbcDialects.get(url);
	Preconditions.checkState(dialect.isPresent(), "Cannot handle such jdbc url: " + url);

	TableSchema schema = TableSchemaUtils.getPhysicalSchema(properties.getTableSchema(SCHEMA));
	dialect.get().validate(schema);

	Optional<String> password = properties.getOptionalString(CONNECTOR_PASSWORD);
	if (password.isPresent()) {
		Preconditions.checkArgument(
			properties.getOptionalString(CONNECTOR_USERNAME).isPresent(),
			"Database username must be provided when database password is provided");
	}
}
 
Example #6
Source File: JdbcDynamicTableFactory.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public DynamicTableSink createDynamicTableSink(Context context) {
	final FactoryUtil.TableFactoryHelper helper = FactoryUtil.createTableFactoryHelper(this, context);
	final ReadableConfig config = helper.getOptions();

	helper.validate();
	validateConfigOptions(config);
	JdbcOptions jdbcOptions = getJdbcOptions(config);
	TableSchema physicalSchema = TableSchemaUtils.getPhysicalSchema(context.getCatalogTable().getSchema());

	return new JdbcDynamicTableSink(
		jdbcOptions,
		getJdbcExecutionOptions(config),
		getJdbcDmlOptions(jdbcOptions, physicalSchema),
		physicalSchema);
}
 
Example #7
Source File: JdbcTableSourceSinkFactory.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public StreamTableSink<Tuple2<Boolean, Row>> createStreamTableSink(Map<String, String> properties) {
	DescriptorProperties descriptorProperties = getValidatedProperties(properties);
	TableSchema schema = TableSchemaUtils.getPhysicalSchema(
		descriptorProperties.getTableSchema(SCHEMA));

	final JdbcUpsertTableSink.Builder builder = JdbcUpsertTableSink.builder()
		.setOptions(getJdbcOptions(descriptorProperties))
		.setTableSchema(schema);

	descriptorProperties.getOptionalInt(CONNECTOR_WRITE_FLUSH_MAX_ROWS).ifPresent(builder::setFlushMaxSize);
	descriptorProperties.getOptionalDuration(CONNECTOR_WRITE_FLUSH_INTERVAL).ifPresent(
		s -> builder.setFlushIntervalMills(s.toMillis()));
	descriptorProperties.getOptionalInt(CONNECTOR_WRITE_MAX_RETRIES).ifPresent(builder::setMaxRetryTimes);

	return builder.build();
}
 
Example #8
Source File: KafkaTableSourceSinkFactoryBase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public StreamTableSource<Row> createStreamTableSource(Map<String, String> properties) {
	final DescriptorProperties descriptorProperties = getValidatedProperties(properties);

	final String topic = descriptorProperties.getString(CONNECTOR_TOPIC);
	final DeserializationSchema<Row> deserializationSchema = getDeserializationSchema(properties);
	final StartupOptions startupOptions = getStartupOptions(descriptorProperties, topic);

	return createKafkaTableSource(
		TableSchemaUtils.getPhysicalSchema(descriptorProperties.getTableSchema(SCHEMA)),
		SchemaValidator.deriveProctimeAttribute(descriptorProperties),
		SchemaValidator.deriveRowtimeAttributes(descriptorProperties),
		SchemaValidator.deriveFieldMapping(
			descriptorProperties,
			Optional.of(deserializationSchema.getProducedType())),
		topic,
		getKafkaProperties(descriptorProperties),
		deserializationSchema,
		startupOptions.startupMode,
		startupOptions.specificOffsets,
		startupOptions.startupTimestampMillis);
}
 
Example #9
Source File: KafkaTableSourceSinkFactoryBase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public StreamTableSink<Row> createStreamTableSink(Map<String, String> properties) {
	final DescriptorProperties descriptorProperties = getValidatedProperties(properties);

	final TableSchema schema = TableSchemaUtils.getPhysicalSchema(
		descriptorProperties.getTableSchema(SCHEMA));
	final String topic = descriptorProperties.getString(CONNECTOR_TOPIC);
	final Optional<String> proctime = SchemaValidator.deriveProctimeAttribute(descriptorProperties);
	final List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors =
		SchemaValidator.deriveRowtimeAttributes(descriptorProperties);

	// see also FLINK-9870
	if (proctime.isPresent() || !rowtimeAttributeDescriptors.isEmpty() ||
			checkForCustomFieldMapping(descriptorProperties, schema)) {
		throw new TableException("Time attributes and custom field mappings are not supported yet.");
	}

	return createKafkaTableSink(
		schema,
		topic,
		getKafkaProperties(descriptorProperties),
		getFlinkKafkaPartitioner(descriptorProperties),
		getSerializationSchema(properties));
}
 
Example #10
Source File: Elasticsearch7DynamicSinkFactory.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public DynamicTableSink createDynamicTableSink(Context context) {
	TableSchema tableSchema = context.getCatalogTable().getSchema();
	ElasticsearchValidationUtils.validatePrimaryKey(tableSchema);

	final FactoryUtil.TableFactoryHelper helper = FactoryUtil.createTableFactoryHelper(this, context);

	final EncodingFormat<SerializationSchema<RowData>> format = helper.discoverEncodingFormat(
		SerializationFormatFactory.class,
		FORMAT_OPTION);

	helper.validate();
	Configuration configuration = new Configuration();
	context.getCatalogTable()
		.getOptions()
		.forEach(configuration::setString);
	Elasticsearch7Configuration config = new Elasticsearch7Configuration(configuration, context.getClassLoader());

	validate(config, configuration);

	return new Elasticsearch7DynamicSink(
		format,
		config,
		TableSchemaUtils.getPhysicalSchema(tableSchema));
}
 
Example #11
Source File: DataGenTableSourceFactory.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public DynamicTableSource createDynamicTableSource(Context context) {
	Configuration options = new Configuration();
	context.getCatalogTable().getOptions().forEach(options::setString);

	TableSchema tableSchema = TableSchemaUtils.getPhysicalSchema(context.getCatalogTable().getSchema());

	DataGenerator[] fieldGenerators = new DataGenerator[tableSchema.getFieldCount()];
	for (int i = 0; i < fieldGenerators.length; i++) {
		fieldGenerators[i] = createDataGenerator(
				tableSchema.getFieldName(i).get(),
				tableSchema.getFieldDataType(i).get(),
				options);
	}

	return new DataGenTableSource(fieldGenerators, tableSchema, options.get(ROWS_PER_SECOND));
}
 
Example #12
Source File: FlinkPravegaTableFactoryBase.java    From flink-connectors with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
protected FlinkPravegaTableSink createFlinkPravegaTableSink(Map<String, String> properties) {
    final DescriptorProperties descriptorProperties = getValidatedProperties(properties);
    final TableSchema schema = TableSchemaUtils.getPhysicalSchema(descriptorProperties.getTableSchema(SCHEMA));

    SerializationSchema<Row> serializationSchema = getSerializationSchema(properties);

    ConnectorConfigurations connectorConfigurations = new ConnectorConfigurations();
    connectorConfigurations.parseConfigurations(descriptorProperties, ConnectorConfigurations.ConfigurationType.WRITER);

    TableSinkWriterBuilder tableSinkWriterBuilder = new Pravega().tableSinkWriterBuilder();
    if (connectorConfigurations.getTxnLeaseRenewalInterval().isPresent()) {
        tableSinkWriterBuilder.withTxnLeaseRenewalPeriod(Time.milliseconds(connectorConfigurations.getTxnLeaseRenewalInterval().get().longValue()));
    }
    if (connectorConfigurations.getWriterMode().isPresent()) {
        tableSinkWriterBuilder.withWriterMode(connectorConfigurations.getWriterMode().get());
    }
    if (connectorConfigurations.getMetrics().isPresent()) {
        tableSinkWriterBuilder.enableMetrics(connectorConfigurations.getMetrics().get());
    }
    if (connectorConfigurations.getWatermark().isPresent()) {
        tableSinkWriterBuilder.enableWatermark(connectorConfigurations.getWatermark().get());
    }
    tableSinkWriterBuilder.withPravegaConfig(connectorConfigurations.getPravegaConfig());
    tableSinkWriterBuilder.withRoutingKeyField(connectorConfigurations.getRoutingKey());
    tableSinkWriterBuilder.withSerializationSchema(serializationSchema);
    tableSinkWriterBuilder.forStream(connectorConfigurations.getWriterStream());
    tableSinkWriterBuilder.withPravegaConfig(connectorConfigurations.getPravegaConfig());

    return new FlinkPravegaTableSink(tableSinkWriterBuilder::createSinkFunction,
            tableSinkWriterBuilder::createOutputFormat, schema);
}
 
Example #13
Source File: HiveTableSink.java    From flink with Apache License 2.0 5 votes vote down vote up
public HiveTableSink(
		boolean userMrWriter, boolean isBounded, JobConf jobConf, ObjectIdentifier identifier, CatalogTable table) {
	this.userMrWriter = userMrWriter;
	this.isBounded = isBounded;
	this.jobConf = jobConf;
	this.identifier = identifier;
	this.catalogTable = table;
	hiveVersion = Preconditions.checkNotNull(jobConf.get(HiveCatalogValidator.CATALOG_HIVE_VERSION),
			"Hive version is not defined");
	hiveShim = HiveShimLoader.loadHiveShim(hiveVersion);
	tableSchema = TableSchemaUtils.getPhysicalSchema(table.getSchema());
}
 
Example #14
Source File: KafkaTableSinkBase.java    From flink with Apache License 2.0 5 votes vote down vote up
protected KafkaTableSinkBase(
		TableSchema schema,
		String topic,
		Properties properties,
		Optional<FlinkKafkaPartitioner<Row>> partitioner,
		SerializationSchema<Row> serializationSchema) {
	this.schema = TableSchemaUtils.checkNoGeneratedColumns(schema);
	this.topic = Preconditions.checkNotNull(topic, "Topic must not be null.");
	this.properties = Preconditions.checkNotNull(properties, "Properties must not be null.");
	this.partitioner = Preconditions.checkNotNull(partitioner, "Partitioner must not be null.");
	this.serializationSchema = Preconditions.checkNotNull(serializationSchema, "Serialization schema must not be null.");
}
 
Example #15
Source File: ElasticsearchUpsertTableSinkBase.java    From flink with Apache License 2.0 5 votes vote down vote up
public ElasticsearchUpsertTableSinkBase(
		boolean isAppendOnly,
		TableSchema schema,
		List<Host> hosts,
		String index,
		String docType,
		String keyDelimiter,
		String keyNullLiteral,
		SerializationSchema<Row> serializationSchema,
		XContentType contentType,
		ActionRequestFailureHandler failureHandler,
		Map<SinkOption, String> sinkOptions,
		RequestFactory requestFactory) {

	this.isAppendOnly = isAppendOnly;
	this.schema = TableSchemaUtils.checkNoGeneratedColumns(schema);
	this.hosts = Preconditions.checkNotNull(hosts);
	this.index = Preconditions.checkNotNull(index);
	this.keyDelimiter = Preconditions.checkNotNull(keyDelimiter);
	this.keyNullLiteral = Preconditions.checkNotNull(keyNullLiteral);
	this.docType = Preconditions.checkNotNull(docType);
	this.serializationSchema = Preconditions.checkNotNull(serializationSchema);
	this.contentType = Preconditions.checkNotNull(contentType);
	this.failureHandler = Preconditions.checkNotNull(failureHandler);
	this.sinkOptions = Preconditions.checkNotNull(sinkOptions);
	this.requestFactory = Preconditions.checkNotNull(requestFactory);
}
 
Example #16
Source File: TestValuesTableFactory.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public DynamicTableSource createDynamicTableSource(Context context) {
	FactoryUtil.TableFactoryHelper helper = FactoryUtil.createTableFactoryHelper(this, context);
	helper.validate();
	ChangelogMode changelogMode = parseChangelogMode(helper.getOptions().get(CHANGELOG_MODE));
	String runtimeSource = helper.getOptions().get(RUNTIME_SOURCE);
	boolean isBounded = helper.getOptions().get(BOUNDED);
	String dataId = helper.getOptions().get(DATA_ID);
	String sourceClass = helper.getOptions().get(TABLE_SOURCE_CLASS);
	boolean isAsync = helper.getOptions().get(ASYNC_ENABLED);
	String lookupFunctionClass = helper.getOptions().get(LOOKUP_FUNCTION_CLASS);
	boolean nestedProjectionSupported = helper.getOptions().get(NESTED_PROJECTION_SUPPORTED);

	if (sourceClass.equals("DEFAULT")) {
		Collection<Row> data = registeredData.getOrDefault(dataId, Collections.emptyList());
		TableSchema physicalSchema = TableSchemaUtils.getPhysicalSchema(context.getCatalogTable().getSchema());
		return new TestValuesTableSource(
			physicalSchema,
			changelogMode,
			isBounded,
			runtimeSource,
			data,
			isAsync,
			lookupFunctionClass,
			nestedProjectionSupported,
			null);
	} else {
		try {
			return InstantiationUtil.instantiate(
				sourceClass,
				DynamicTableSource.class,
				Thread.currentThread().getContextClassLoader());
		} catch (FlinkException e) {
			throw new RuntimeException("Can't instantiate class " + sourceClass, e);
		}
	}
}
 
Example #17
Source File: CollectStreamTableSink.java    From flink with Apache License 2.0 5 votes vote down vote up
public CollectStreamTableSink(
		InetAddress targetAddress,
		int targetPort,
		TypeSerializer<Tuple2<Boolean, Row>> serializer,
		TableSchema tableSchema) {
	this.targetAddress = targetAddress;
	this.targetPort = targetPort;
	this.serializer = serializer;
	this.tableSchema = TableSchemaUtils.checkNoGeneratedColumns(tableSchema);
}
 
Example #18
Source File: HBaseTableFactory.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public StreamTableSink<Tuple2<Boolean, Row>> createStreamTableSink(Map<String, String> properties) {
	final DescriptorProperties descriptorProperties = getValidatedProperties(properties);
	HBaseOptions.Builder hbaseOptionsBuilder = HBaseOptions.builder();
	hbaseOptionsBuilder.setZkQuorum(descriptorProperties.getString(CONNECTOR_ZK_QUORUM));
	hbaseOptionsBuilder.setTableName(descriptorProperties.getString(CONNECTOR_TABLE_NAME));
	descriptorProperties
		.getOptionalString(CONNECTOR_ZK_NODE_PARENT)
		.ifPresent(hbaseOptionsBuilder::setZkNodeParent);

	TableSchema tableSchema = TableSchemaUtils.getPhysicalSchema(
		descriptorProperties.getTableSchema(SCHEMA));
	HBaseTableSchema hbaseSchema = validateTableSchema(tableSchema);

	HBaseWriteOptions.Builder writeBuilder = HBaseWriteOptions.builder();
	descriptorProperties
		.getOptionalInt(CONNECTOR_WRITE_BUFFER_FLUSH_MAX_ROWS)
		.ifPresent(writeBuilder::setBufferFlushMaxRows);
	descriptorProperties
		.getOptionalMemorySize(CONNECTOR_WRITE_BUFFER_FLUSH_MAX_SIZE)
		.ifPresent(v -> writeBuilder.setBufferFlushMaxSizeInBytes(v.getBytes()));
	descriptorProperties
		.getOptionalDuration(CONNECTOR_WRITE_BUFFER_FLUSH_INTERVAL)
		.ifPresent(v -> writeBuilder.setBufferFlushIntervalMillis(v.toMillis()));

	return new HBaseUpsertTableSink(
		hbaseSchema,
		hbaseOptionsBuilder.build(),
		writeBuilder.build()
	);
}
 
Example #19
Source File: JdbcDynamicTableFactory.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public DynamicTableSource createDynamicTableSource(Context context) {
	final FactoryUtil.TableFactoryHelper helper = FactoryUtil.createTableFactoryHelper(this, context);
	final ReadableConfig config = helper.getOptions();

	helper.validate();
	validateConfigOptions(config);
	TableSchema physicalSchema = TableSchemaUtils.getPhysicalSchema(context.getCatalogTable().getSchema());
	return new JdbcDynamicTableSource(
		getJdbcOptions(helper.getOptions()),
		getJdbcReadOptions(helper.getOptions()),
		getJdbcLookupOptions(helper.getOptions()),
		physicalSchema);
}
 
Example #20
Source File: JdbcTableSourceSinkFactory.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public StreamTableSource<Row> createStreamTableSource(Map<String, String> properties) {
	DescriptorProperties descriptorProperties = getValidatedProperties(properties);
	TableSchema schema = TableSchemaUtils.getPhysicalSchema(
		descriptorProperties.getTableSchema(SCHEMA));

	return JdbcTableSource.builder()
		.setOptions(getJdbcOptions(descriptorProperties))
		.setReadOptions(getJdbcReadOptions(descriptorProperties))
		.setLookupOptions(getJdbcLookupOptions(descriptorProperties))
		.setSchema(schema)
		.build();
}
 
Example #21
Source File: JdbcUpsertTableSink.java    From flink with Apache License 2.0 5 votes vote down vote up
private JdbcUpsertTableSink(
		TableSchema schema,
		JdbcOptions options,
		int flushMaxSize,
		long flushIntervalMills,
		int maxRetryTime) {
	this.schema = TableSchemaUtils.checkNoGeneratedColumns(schema);
	this.options = options;
	this.flushMaxSize = flushMaxSize;
	this.flushIntervalMills = flushIntervalMills;
	this.maxRetryTime = maxRetryTime;
}
 
Example #22
Source File: HBaseDynamicTableSource.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void applyProjection(int[][] projectedFields) {
	TableSchema projectSchema = TableSchemaUtils.projectSchema(
		hbaseSchema.convertsToTableSchema(),
		projectedFields);
	this.hbaseSchema = HBaseTableSchema.fromTableSchema(projectSchema);
}
 
Example #23
Source File: KuduTableUtils.java    From bahir-flink with Apache License 2.0 5 votes vote down vote up
public static TableSchema getSchemaWithSqlTimestamp(TableSchema schema) {
    TableSchema.Builder builder = new TableSchema.Builder();
    TableSchemaUtils.getPhysicalSchema(schema).getTableColumns().forEach(
            tableColumn -> {
                if (tableColumn.getType().getLogicalType() instanceof TimestampType) {
                    builder.field(tableColumn.getName(), tableColumn.getType().bridgedTo(Timestamp.class));
                } else {
                    builder.field(tableColumn.getName(), tableColumn.getType());
                }
            });
    return builder.build();
}
 
Example #24
Source File: FlinkPravegaTableSource.java    From flink-connectors with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a Pravega {@link TableSource}.
 * @param sourceFunctionFactory a factory for the {@link FlinkPravegaReader} to implement {@link StreamTableSource}
 * @param inputFormatFactory a factory for the {@link FlinkPravegaInputFormat} to implement {@link BatchTableSource}
 * @param schema the table schema
 */
protected FlinkPravegaTableSource(
        Supplier<FlinkPravegaReader<Row>> sourceFunctionFactory,
        Supplier<FlinkPravegaInputFormat<Row>> inputFormatFactory,
        TableSchema schema) {
    this.sourceFunctionFactory = checkNotNull(sourceFunctionFactory, "sourceFunctionFactory");
    this.inputFormatFactory = checkNotNull(inputFormatFactory, "inputFormatFactory");
    this.schema = TableSchemaUtils.checkNoGeneratedColumns(schema);
}
 
Example #25
Source File: TestTableSinkFactoryBase.java    From flink with Apache License 2.0 4 votes vote down vote up
public TestTableSink(TableSchema schema, String property) {
	this.schema = TableSchemaUtils.checkNoGeneratedColumns(schema);
	this.property = property;
}
 
Example #26
Source File: TestTableSourceFactoryBase.java    From flink with Apache License 2.0 4 votes vote down vote up
public TestTableSource(TableSchema schema, String property, String proctime, List<RowtimeAttributeDescriptor> rowtime) {
	this.schema = TableSchemaUtils.checkNoGeneratedColumns(schema);
	this.property = property;
	this.proctime = proctime;
	this.rowtime = rowtime;
}
 
Example #27
Source File: FlinkPravegaTableFactoryBase.java    From flink-connectors with Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("unchecked")
protected FlinkPravegaTableSource createFlinkPravegaTableSource(Map<String, String> properties) {
    final DescriptorProperties descriptorProperties = getValidatedProperties(properties);
    final TableSchema schema = TableSchemaUtils.getPhysicalSchema(descriptorProperties.getTableSchema(SCHEMA));
    final DeserializationSchema<Row> deserializationSchema = getDeserializationSchema(properties);

    ConnectorConfigurations connectorConfigurations = new ConnectorConfigurations();
    connectorConfigurations.parseConfigurations(descriptorProperties, ConnectorConfigurations.ConfigurationType.READER);

    // create source from the reader builder by using the supplied properties
    TableSourceReaderBuilder tableSourceReaderBuilder = new Pravega().tableSourceReaderBuilder();
    tableSourceReaderBuilder.withDeserializationSchema(deserializationSchema);

    if (connectorConfigurations.getAssignerWithTimeWindows().isPresent()) {
        tableSourceReaderBuilder.withTimestampAssigner(connectorConfigurations.getAssignerWithTimeWindows().get());
    }

    if (connectorConfigurations.getUid().isPresent()) {
        tableSourceReaderBuilder.uid(connectorConfigurations.getUid().get());
    }
    if (connectorConfigurations.getRgScope().isPresent()) {
        tableSourceReaderBuilder.withReaderGroupScope(connectorConfigurations.getRgScope().get());
    }
    if (connectorConfigurations.getRgName().isPresent()) {
        tableSourceReaderBuilder.withReaderGroupName(connectorConfigurations.getRgName().get());
    }
    if (connectorConfigurations.getRefreshInterval().isPresent()) {
        tableSourceReaderBuilder.withReaderGroupRefreshTime(Time.milliseconds(connectorConfigurations.getRefreshInterval().get()));
    }
    if (connectorConfigurations.getEventReadTimeoutInterval().isPresent()) {
        tableSourceReaderBuilder.withEventReadTimeout(Time.milliseconds(connectorConfigurations.getEventReadTimeoutInterval().get()));
    }
    if (connectorConfigurations.getCheckpointInitiateTimeoutInterval().isPresent()) {
        tableSourceReaderBuilder.withCheckpointInitiateTimeout(Time.milliseconds(connectorConfigurations.getCheckpointInitiateTimeoutInterval().get()));
    }

    tableSourceReaderBuilder.withPravegaConfig(connectorConfigurations.getPravegaConfig());
    if (connectorConfigurations.getMetrics().isPresent()) {
        tableSourceReaderBuilder.enableMetrics(connectorConfigurations.getMetrics().get());
    }
    tableSourceReaderBuilder.withPravegaConfig(connectorConfigurations.getPravegaConfig());
    for (StreamWithBoundaries streamWithBoundaries: connectorConfigurations.getReaderStreams()) {
        if (streamWithBoundaries.getFrom() != StreamCut.UNBOUNDED && streamWithBoundaries.getTo() != StreamCut.UNBOUNDED) {
            tableSourceReaderBuilder.forStream(streamWithBoundaries.getStream(), streamWithBoundaries.getFrom(), streamWithBoundaries.getTo());
        } else if (streamWithBoundaries.getFrom() != StreamCut.UNBOUNDED) {
            tableSourceReaderBuilder.forStream(streamWithBoundaries.getStream(), streamWithBoundaries.getFrom());
        } else {
            tableSourceReaderBuilder.forStream(streamWithBoundaries.getStream());
        }
    }

    FlinkPravegaTableSource flinkPravegaTableSource = new FlinkPravegaTableSource(
                                                                tableSourceReaderBuilder::buildSourceFunction,
                                                                tableSourceReaderBuilder::buildInputFormat, schema);
    flinkPravegaTableSource.setRowtimeAttributeDescriptors(SchemaValidator.deriveRowtimeAttributes(descriptorProperties));
    Optional<String> procTimeAttribute = SchemaValidator.deriveProctimeAttribute(descriptorProperties);
    if (procTimeAttribute.isPresent()) {
        flinkPravegaTableSource.setProctimeAttribute(procTimeAttribute.get());
    }

    return flinkPravegaTableSource;
}
 
Example #28
Source File: CollectBatchTableSink.java    From flink with Apache License 2.0 4 votes vote down vote up
public CollectBatchTableSink(String accumulatorName, TypeSerializer<Row> serializer, TableSchema tableSchema) {
	this.accumulatorName = accumulatorName;
	this.serializer = serializer;
	this.tableSchema = TableSchemaUtils.checkNoGeneratedColumns(tableSchema);
}
 
Example #29
Source File: TestValuesTableFactory.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
public void applyProjection(int[][] projectedFields) {
	this.physicalSchema = TableSchemaUtils.projectSchema(physicalSchema, projectedFields);
	this.projectedFields = Arrays.stream(projectedFields).mapToInt(f -> f[0]).toArray();
}
 
Example #30
Source File: CsvTableSourceFactoryBase.java    From flink with Apache License 2.0 4 votes vote down vote up
protected CsvTableSource createTableSource(
		Boolean isStreaming,
		Map<String, String> properties) {

	DescriptorProperties params = new DescriptorProperties();
	params.putProperties(properties);

	// validate
	new FileSystemValidator().validate(params);
	new OldCsvValidator().validate(params);
	new SchemaValidator(isStreaming, false, false).validate(params);

	// build
	CsvTableSource.Builder csvTableSourceBuilder = new CsvTableSource.Builder();

	TableSchema tableSchema = TableSchemaUtils.getPhysicalSchema(params.getTableSchema(SCHEMA));

	// if a schema is defined, no matter derive schema is set or not, will use the defined schema
	final boolean hasSchema = params.hasPrefix(FORMAT_FIELDS);
	if (hasSchema) {
		TableSchema formatSchema = params.getTableSchema(FORMAT_FIELDS);
		// the CsvTableSource needs some rework first
		// for now the schema must be equal to the encoding
		// Ignore conversion classes in DataType
		if (!getFieldLogicalTypes(formatSchema).equals(getFieldLogicalTypes(tableSchema))) {
			throw new TableException(String.format(
					"Encodings that differ from the schema are not supported yet for" +
							" CsvTableSource, format schema is '%s', but table schema is '%s'.",
					formatSchema,
					tableSchema));
		}
	}

	params.getOptionalString(CONNECTOR_PATH).ifPresent(csvTableSourceBuilder::path);
	params.getOptionalString(FORMAT_FIELD_DELIMITER).ifPresent(csvTableSourceBuilder::fieldDelimiter);
	params.getOptionalString(FORMAT_LINE_DELIMITER).ifPresent(csvTableSourceBuilder::lineDelimiter);

	for (int i = 0; i < tableSchema.getFieldCount(); ++i) {
		csvTableSourceBuilder.field(tableSchema.getFieldNames()[i], tableSchema.getFieldDataTypes()[i]);
	}
	params.getOptionalCharacter(FORMAT_QUOTE_CHARACTER).ifPresent(csvTableSourceBuilder::quoteCharacter);
	params.getOptionalString(FORMAT_COMMENT_PREFIX).ifPresent(csvTableSourceBuilder::commentPrefix);
	params.getOptionalBoolean(FORMAT_IGNORE_FIRST_LINE).ifPresent(flag -> {
		if (flag) {
			csvTableSourceBuilder.ignoreFirstLine();
		}
	});

	params.getOptionalBoolean(FORMAT_IGNORE_PARSE_ERRORS).ifPresent(flag -> {
		if (flag) {
			csvTableSourceBuilder.ignoreParseErrors();
		}
	});

	return csvTableSourceBuilder.build();
}