org.apache.flink.table.utils.TableSchemaUtils Java Examples
The following examples show how to use
org.apache.flink.table.utils.TableSchemaUtils.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: KafkaTableSourceBase.java From flink with Apache License 2.0 | 6 votes |
/** * Creates a generic Kafka {@link StreamTableSource}. * * @param schema Schema of the produced table. * @param proctimeAttribute Field name of the processing time attribute. * @param rowtimeAttributeDescriptors Descriptor for a rowtime attribute * @param fieldMapping Mapping for the fields of the table schema to * fields of the physical returned type. * @param topic Kafka topic to consume. * @param properties Properties for the Kafka consumer. * @param deserializationSchema Deserialization schema for decoding records from Kafka. * @param startupMode Startup mode for the contained consumer. * @param specificStartupOffsets Specific startup offsets; only relevant when startup * mode is {@link StartupMode#SPECIFIC_OFFSETS}. * @param startupTimestampMillis Startup timestamp for offsets; only relevant when startup * mode is {@link StartupMode#TIMESTAMP}. */ protected KafkaTableSourceBase( TableSchema schema, Optional<String> proctimeAttribute, List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors, Optional<Map<String, String>> fieldMapping, String topic, Properties properties, DeserializationSchema<Row> deserializationSchema, StartupMode startupMode, Map<KafkaTopicPartition, Long> specificStartupOffsets, long startupTimestampMillis) { this.schema = TableSchemaUtils.checkNoGeneratedColumns(schema); this.proctimeAttribute = validateProctimeAttribute(proctimeAttribute); this.rowtimeAttributeDescriptors = validateRowtimeAttributeDescriptors(rowtimeAttributeDescriptors); this.fieldMapping = fieldMapping; this.topic = Preconditions.checkNotNull(topic, "Topic must not be null."); this.properties = Preconditions.checkNotNull(properties, "Properties must not be null."); this.deserializationSchema = Preconditions.checkNotNull( deserializationSchema, "Deserialization schema must not be null."); this.startupMode = Preconditions.checkNotNull(startupMode, "Startup mode must not be null."); this.specificStartupOffsets = Preconditions.checkNotNull( specificStartupOffsets, "Specific offsets must not be null."); this.startupTimestampMillis = startupTimestampMillis; }
Example #2
Source File: Elasticsearch6DynamicSinkFactory.java From flink with Apache License 2.0 | 6 votes |
@Override public DynamicTableSink createDynamicTableSink(Context context) { TableSchema tableSchema = context.getCatalogTable().getSchema(); ElasticsearchValidationUtils.validatePrimaryKey(tableSchema); final FactoryUtil.TableFactoryHelper helper = FactoryUtil.createTableFactoryHelper(this, context); final EncodingFormat<SerializationSchema<RowData>> format = helper.discoverEncodingFormat( SerializationFormatFactory.class, FORMAT_OPTION); helper.validate(); Configuration configuration = new Configuration(); context.getCatalogTable() .getOptions() .forEach(configuration::setString); Elasticsearch6Configuration config = new Elasticsearch6Configuration(configuration, context.getClassLoader()); validate(config, configuration); return new Elasticsearch6DynamicSink( format, config, TableSchemaUtils.getPhysicalSchema(tableSchema)); }
Example #3
Source File: ElasticsearchUpsertTableSinkFactoryBase.java From flink with Apache License 2.0 | 6 votes |
@Override public StreamTableSink<Tuple2<Boolean, Row>> createStreamTableSink(Map<String, String> properties) { final DescriptorProperties descriptorProperties = getValidatedProperties(properties); return createElasticsearchUpsertTableSink( descriptorProperties.isValue(UPDATE_MODE, UPDATE_MODE_VALUE_APPEND), TableSchemaUtils.getPhysicalSchema(descriptorProperties.getTableSchema(SCHEMA)), getHosts(descriptorProperties), descriptorProperties.getString(CONNECTOR_INDEX), descriptorProperties.getString(CONNECTOR_DOCUMENT_TYPE), descriptorProperties.getOptionalString(CONNECTOR_KEY_DELIMITER).orElse(DEFAULT_KEY_DELIMITER), descriptorProperties.getOptionalString(CONNECTOR_KEY_NULL_LITERAL).orElse(DEFAULT_KEY_NULL_LITERAL), getSerializationSchema(properties), SUPPORTED_CONTENT_TYPE, getFailureHandler(descriptorProperties), getSinkOptions(descriptorProperties)); }
Example #4
Source File: HBaseTableFactory.java From flink with Apache License 2.0 | 6 votes |
@Override public StreamTableSource<Row> createStreamTableSource(Map<String, String> properties) { final DescriptorProperties descriptorProperties = getValidatedProperties(properties); // create default configuration from current runtime env (`hbase-site.xml` in classpath) first, Configuration hbaseClientConf = HBaseConfigurationUtil.getHBaseConfiguration(); String hbaseZk = descriptorProperties.getString(CONNECTOR_ZK_QUORUM); hbaseClientConf.set(HConstants.ZOOKEEPER_QUORUM, hbaseZk); descriptorProperties .getOptionalString(CONNECTOR_ZK_NODE_PARENT) .ifPresent(v -> hbaseClientConf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, v)); String hTableName = descriptorProperties.getString(CONNECTOR_TABLE_NAME); TableSchema tableSchema = TableSchemaUtils.getPhysicalSchema( descriptorProperties.getTableSchema(SCHEMA)); HBaseTableSchema hbaseSchema = validateTableSchema(tableSchema); return new HBaseTableSource(hbaseClientConf, hTableName, hbaseSchema, null); }
Example #5
Source File: JdbcValidator.java From flink with Apache License 2.0 | 6 votes |
private void validateCommonProperties(DescriptorProperties properties) { properties.validateString(CONNECTOR_URL, false, 1); properties.validateString(CONNECTOR_TABLE, false, 1); properties.validateString(CONNECTOR_DRIVER, true); properties.validateString(CONNECTOR_USERNAME, true); properties.validateString(CONNECTOR_PASSWORD, true); final String url = properties.getString(CONNECTOR_URL); final Optional<JdbcDialect> dialect = JdbcDialects.get(url); Preconditions.checkState(dialect.isPresent(), "Cannot handle such jdbc url: " + url); TableSchema schema = TableSchemaUtils.getPhysicalSchema(properties.getTableSchema(SCHEMA)); dialect.get().validate(schema); Optional<String> password = properties.getOptionalString(CONNECTOR_PASSWORD); if (password.isPresent()) { Preconditions.checkArgument( properties.getOptionalString(CONNECTOR_USERNAME).isPresent(), "Database username must be provided when database password is provided"); } }
Example #6
Source File: JdbcDynamicTableFactory.java From flink with Apache License 2.0 | 6 votes |
@Override public DynamicTableSink createDynamicTableSink(Context context) { final FactoryUtil.TableFactoryHelper helper = FactoryUtil.createTableFactoryHelper(this, context); final ReadableConfig config = helper.getOptions(); helper.validate(); validateConfigOptions(config); JdbcOptions jdbcOptions = getJdbcOptions(config); TableSchema physicalSchema = TableSchemaUtils.getPhysicalSchema(context.getCatalogTable().getSchema()); return new JdbcDynamicTableSink( jdbcOptions, getJdbcExecutionOptions(config), getJdbcDmlOptions(jdbcOptions, physicalSchema), physicalSchema); }
Example #7
Source File: JdbcTableSourceSinkFactory.java From flink with Apache License 2.0 | 6 votes |
@Override public StreamTableSink<Tuple2<Boolean, Row>> createStreamTableSink(Map<String, String> properties) { DescriptorProperties descriptorProperties = getValidatedProperties(properties); TableSchema schema = TableSchemaUtils.getPhysicalSchema( descriptorProperties.getTableSchema(SCHEMA)); final JdbcUpsertTableSink.Builder builder = JdbcUpsertTableSink.builder() .setOptions(getJdbcOptions(descriptorProperties)) .setTableSchema(schema); descriptorProperties.getOptionalInt(CONNECTOR_WRITE_FLUSH_MAX_ROWS).ifPresent(builder::setFlushMaxSize); descriptorProperties.getOptionalDuration(CONNECTOR_WRITE_FLUSH_INTERVAL).ifPresent( s -> builder.setFlushIntervalMills(s.toMillis())); descriptorProperties.getOptionalInt(CONNECTOR_WRITE_MAX_RETRIES).ifPresent(builder::setMaxRetryTimes); return builder.build(); }
Example #8
Source File: KafkaTableSourceSinkFactoryBase.java From flink with Apache License 2.0 | 6 votes |
@Override public StreamTableSource<Row> createStreamTableSource(Map<String, String> properties) { final DescriptorProperties descriptorProperties = getValidatedProperties(properties); final String topic = descriptorProperties.getString(CONNECTOR_TOPIC); final DeserializationSchema<Row> deserializationSchema = getDeserializationSchema(properties); final StartupOptions startupOptions = getStartupOptions(descriptorProperties, topic); return createKafkaTableSource( TableSchemaUtils.getPhysicalSchema(descriptorProperties.getTableSchema(SCHEMA)), SchemaValidator.deriveProctimeAttribute(descriptorProperties), SchemaValidator.deriveRowtimeAttributes(descriptorProperties), SchemaValidator.deriveFieldMapping( descriptorProperties, Optional.of(deserializationSchema.getProducedType())), topic, getKafkaProperties(descriptorProperties), deserializationSchema, startupOptions.startupMode, startupOptions.specificOffsets, startupOptions.startupTimestampMillis); }
Example #9
Source File: KafkaTableSourceSinkFactoryBase.java From flink with Apache License 2.0 | 6 votes |
@Override public StreamTableSink<Row> createStreamTableSink(Map<String, String> properties) { final DescriptorProperties descriptorProperties = getValidatedProperties(properties); final TableSchema schema = TableSchemaUtils.getPhysicalSchema( descriptorProperties.getTableSchema(SCHEMA)); final String topic = descriptorProperties.getString(CONNECTOR_TOPIC); final Optional<String> proctime = SchemaValidator.deriveProctimeAttribute(descriptorProperties); final List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors = SchemaValidator.deriveRowtimeAttributes(descriptorProperties); // see also FLINK-9870 if (proctime.isPresent() || !rowtimeAttributeDescriptors.isEmpty() || checkForCustomFieldMapping(descriptorProperties, schema)) { throw new TableException("Time attributes and custom field mappings are not supported yet."); } return createKafkaTableSink( schema, topic, getKafkaProperties(descriptorProperties), getFlinkKafkaPartitioner(descriptorProperties), getSerializationSchema(properties)); }
Example #10
Source File: Elasticsearch7DynamicSinkFactory.java From flink with Apache License 2.0 | 6 votes |
@Override public DynamicTableSink createDynamicTableSink(Context context) { TableSchema tableSchema = context.getCatalogTable().getSchema(); ElasticsearchValidationUtils.validatePrimaryKey(tableSchema); final FactoryUtil.TableFactoryHelper helper = FactoryUtil.createTableFactoryHelper(this, context); final EncodingFormat<SerializationSchema<RowData>> format = helper.discoverEncodingFormat( SerializationFormatFactory.class, FORMAT_OPTION); helper.validate(); Configuration configuration = new Configuration(); context.getCatalogTable() .getOptions() .forEach(configuration::setString); Elasticsearch7Configuration config = new Elasticsearch7Configuration(configuration, context.getClassLoader()); validate(config, configuration); return new Elasticsearch7DynamicSink( format, config, TableSchemaUtils.getPhysicalSchema(tableSchema)); }
Example #11
Source File: DataGenTableSourceFactory.java From flink with Apache License 2.0 | 6 votes |
@Override public DynamicTableSource createDynamicTableSource(Context context) { Configuration options = new Configuration(); context.getCatalogTable().getOptions().forEach(options::setString); TableSchema tableSchema = TableSchemaUtils.getPhysicalSchema(context.getCatalogTable().getSchema()); DataGenerator[] fieldGenerators = new DataGenerator[tableSchema.getFieldCount()]; for (int i = 0; i < fieldGenerators.length; i++) { fieldGenerators[i] = createDataGenerator( tableSchema.getFieldName(i).get(), tableSchema.getFieldDataType(i).get(), options); } return new DataGenTableSource(fieldGenerators, tableSchema, options.get(ROWS_PER_SECOND)); }
Example #12
Source File: FlinkPravegaTableFactoryBase.java From flink-connectors with Apache License 2.0 | 5 votes |
@SuppressWarnings("unchecked") protected FlinkPravegaTableSink createFlinkPravegaTableSink(Map<String, String> properties) { final DescriptorProperties descriptorProperties = getValidatedProperties(properties); final TableSchema schema = TableSchemaUtils.getPhysicalSchema(descriptorProperties.getTableSchema(SCHEMA)); SerializationSchema<Row> serializationSchema = getSerializationSchema(properties); ConnectorConfigurations connectorConfigurations = new ConnectorConfigurations(); connectorConfigurations.parseConfigurations(descriptorProperties, ConnectorConfigurations.ConfigurationType.WRITER); TableSinkWriterBuilder tableSinkWriterBuilder = new Pravega().tableSinkWriterBuilder(); if (connectorConfigurations.getTxnLeaseRenewalInterval().isPresent()) { tableSinkWriterBuilder.withTxnLeaseRenewalPeriod(Time.milliseconds(connectorConfigurations.getTxnLeaseRenewalInterval().get().longValue())); } if (connectorConfigurations.getWriterMode().isPresent()) { tableSinkWriterBuilder.withWriterMode(connectorConfigurations.getWriterMode().get()); } if (connectorConfigurations.getMetrics().isPresent()) { tableSinkWriterBuilder.enableMetrics(connectorConfigurations.getMetrics().get()); } if (connectorConfigurations.getWatermark().isPresent()) { tableSinkWriterBuilder.enableWatermark(connectorConfigurations.getWatermark().get()); } tableSinkWriterBuilder.withPravegaConfig(connectorConfigurations.getPravegaConfig()); tableSinkWriterBuilder.withRoutingKeyField(connectorConfigurations.getRoutingKey()); tableSinkWriterBuilder.withSerializationSchema(serializationSchema); tableSinkWriterBuilder.forStream(connectorConfigurations.getWriterStream()); tableSinkWriterBuilder.withPravegaConfig(connectorConfigurations.getPravegaConfig()); return new FlinkPravegaTableSink(tableSinkWriterBuilder::createSinkFunction, tableSinkWriterBuilder::createOutputFormat, schema); }
Example #13
Source File: HiveTableSink.java From flink with Apache License 2.0 | 5 votes |
public HiveTableSink( boolean userMrWriter, boolean isBounded, JobConf jobConf, ObjectIdentifier identifier, CatalogTable table) { this.userMrWriter = userMrWriter; this.isBounded = isBounded; this.jobConf = jobConf; this.identifier = identifier; this.catalogTable = table; hiveVersion = Preconditions.checkNotNull(jobConf.get(HiveCatalogValidator.CATALOG_HIVE_VERSION), "Hive version is not defined"); hiveShim = HiveShimLoader.loadHiveShim(hiveVersion); tableSchema = TableSchemaUtils.getPhysicalSchema(table.getSchema()); }
Example #14
Source File: KafkaTableSinkBase.java From flink with Apache License 2.0 | 5 votes |
protected KafkaTableSinkBase( TableSchema schema, String topic, Properties properties, Optional<FlinkKafkaPartitioner<Row>> partitioner, SerializationSchema<Row> serializationSchema) { this.schema = TableSchemaUtils.checkNoGeneratedColumns(schema); this.topic = Preconditions.checkNotNull(topic, "Topic must not be null."); this.properties = Preconditions.checkNotNull(properties, "Properties must not be null."); this.partitioner = Preconditions.checkNotNull(partitioner, "Partitioner must not be null."); this.serializationSchema = Preconditions.checkNotNull(serializationSchema, "Serialization schema must not be null."); }
Example #15
Source File: ElasticsearchUpsertTableSinkBase.java From flink with Apache License 2.0 | 5 votes |
public ElasticsearchUpsertTableSinkBase( boolean isAppendOnly, TableSchema schema, List<Host> hosts, String index, String docType, String keyDelimiter, String keyNullLiteral, SerializationSchema<Row> serializationSchema, XContentType contentType, ActionRequestFailureHandler failureHandler, Map<SinkOption, String> sinkOptions, RequestFactory requestFactory) { this.isAppendOnly = isAppendOnly; this.schema = TableSchemaUtils.checkNoGeneratedColumns(schema); this.hosts = Preconditions.checkNotNull(hosts); this.index = Preconditions.checkNotNull(index); this.keyDelimiter = Preconditions.checkNotNull(keyDelimiter); this.keyNullLiteral = Preconditions.checkNotNull(keyNullLiteral); this.docType = Preconditions.checkNotNull(docType); this.serializationSchema = Preconditions.checkNotNull(serializationSchema); this.contentType = Preconditions.checkNotNull(contentType); this.failureHandler = Preconditions.checkNotNull(failureHandler); this.sinkOptions = Preconditions.checkNotNull(sinkOptions); this.requestFactory = Preconditions.checkNotNull(requestFactory); }
Example #16
Source File: TestValuesTableFactory.java From flink with Apache License 2.0 | 5 votes |
@Override public DynamicTableSource createDynamicTableSource(Context context) { FactoryUtil.TableFactoryHelper helper = FactoryUtil.createTableFactoryHelper(this, context); helper.validate(); ChangelogMode changelogMode = parseChangelogMode(helper.getOptions().get(CHANGELOG_MODE)); String runtimeSource = helper.getOptions().get(RUNTIME_SOURCE); boolean isBounded = helper.getOptions().get(BOUNDED); String dataId = helper.getOptions().get(DATA_ID); String sourceClass = helper.getOptions().get(TABLE_SOURCE_CLASS); boolean isAsync = helper.getOptions().get(ASYNC_ENABLED); String lookupFunctionClass = helper.getOptions().get(LOOKUP_FUNCTION_CLASS); boolean nestedProjectionSupported = helper.getOptions().get(NESTED_PROJECTION_SUPPORTED); if (sourceClass.equals("DEFAULT")) { Collection<Row> data = registeredData.getOrDefault(dataId, Collections.emptyList()); TableSchema physicalSchema = TableSchemaUtils.getPhysicalSchema(context.getCatalogTable().getSchema()); return new TestValuesTableSource( physicalSchema, changelogMode, isBounded, runtimeSource, data, isAsync, lookupFunctionClass, nestedProjectionSupported, null); } else { try { return InstantiationUtil.instantiate( sourceClass, DynamicTableSource.class, Thread.currentThread().getContextClassLoader()); } catch (FlinkException e) { throw new RuntimeException("Can't instantiate class " + sourceClass, e); } } }
Example #17
Source File: CollectStreamTableSink.java From flink with Apache License 2.0 | 5 votes |
public CollectStreamTableSink( InetAddress targetAddress, int targetPort, TypeSerializer<Tuple2<Boolean, Row>> serializer, TableSchema tableSchema) { this.targetAddress = targetAddress; this.targetPort = targetPort; this.serializer = serializer; this.tableSchema = TableSchemaUtils.checkNoGeneratedColumns(tableSchema); }
Example #18
Source File: HBaseTableFactory.java From flink with Apache License 2.0 | 5 votes |
@Override public StreamTableSink<Tuple2<Boolean, Row>> createStreamTableSink(Map<String, String> properties) { final DescriptorProperties descriptorProperties = getValidatedProperties(properties); HBaseOptions.Builder hbaseOptionsBuilder = HBaseOptions.builder(); hbaseOptionsBuilder.setZkQuorum(descriptorProperties.getString(CONNECTOR_ZK_QUORUM)); hbaseOptionsBuilder.setTableName(descriptorProperties.getString(CONNECTOR_TABLE_NAME)); descriptorProperties .getOptionalString(CONNECTOR_ZK_NODE_PARENT) .ifPresent(hbaseOptionsBuilder::setZkNodeParent); TableSchema tableSchema = TableSchemaUtils.getPhysicalSchema( descriptorProperties.getTableSchema(SCHEMA)); HBaseTableSchema hbaseSchema = validateTableSchema(tableSchema); HBaseWriteOptions.Builder writeBuilder = HBaseWriteOptions.builder(); descriptorProperties .getOptionalInt(CONNECTOR_WRITE_BUFFER_FLUSH_MAX_ROWS) .ifPresent(writeBuilder::setBufferFlushMaxRows); descriptorProperties .getOptionalMemorySize(CONNECTOR_WRITE_BUFFER_FLUSH_MAX_SIZE) .ifPresent(v -> writeBuilder.setBufferFlushMaxSizeInBytes(v.getBytes())); descriptorProperties .getOptionalDuration(CONNECTOR_WRITE_BUFFER_FLUSH_INTERVAL) .ifPresent(v -> writeBuilder.setBufferFlushIntervalMillis(v.toMillis())); return new HBaseUpsertTableSink( hbaseSchema, hbaseOptionsBuilder.build(), writeBuilder.build() ); }
Example #19
Source File: JdbcDynamicTableFactory.java From flink with Apache License 2.0 | 5 votes |
@Override public DynamicTableSource createDynamicTableSource(Context context) { final FactoryUtil.TableFactoryHelper helper = FactoryUtil.createTableFactoryHelper(this, context); final ReadableConfig config = helper.getOptions(); helper.validate(); validateConfigOptions(config); TableSchema physicalSchema = TableSchemaUtils.getPhysicalSchema(context.getCatalogTable().getSchema()); return new JdbcDynamicTableSource( getJdbcOptions(helper.getOptions()), getJdbcReadOptions(helper.getOptions()), getJdbcLookupOptions(helper.getOptions()), physicalSchema); }
Example #20
Source File: JdbcTableSourceSinkFactory.java From flink with Apache License 2.0 | 5 votes |
@Override public StreamTableSource<Row> createStreamTableSource(Map<String, String> properties) { DescriptorProperties descriptorProperties = getValidatedProperties(properties); TableSchema schema = TableSchemaUtils.getPhysicalSchema( descriptorProperties.getTableSchema(SCHEMA)); return JdbcTableSource.builder() .setOptions(getJdbcOptions(descriptorProperties)) .setReadOptions(getJdbcReadOptions(descriptorProperties)) .setLookupOptions(getJdbcLookupOptions(descriptorProperties)) .setSchema(schema) .build(); }
Example #21
Source File: JdbcUpsertTableSink.java From flink with Apache License 2.0 | 5 votes |
private JdbcUpsertTableSink( TableSchema schema, JdbcOptions options, int flushMaxSize, long flushIntervalMills, int maxRetryTime) { this.schema = TableSchemaUtils.checkNoGeneratedColumns(schema); this.options = options; this.flushMaxSize = flushMaxSize; this.flushIntervalMills = flushIntervalMills; this.maxRetryTime = maxRetryTime; }
Example #22
Source File: HBaseDynamicTableSource.java From flink with Apache License 2.0 | 5 votes |
@Override public void applyProjection(int[][] projectedFields) { TableSchema projectSchema = TableSchemaUtils.projectSchema( hbaseSchema.convertsToTableSchema(), projectedFields); this.hbaseSchema = HBaseTableSchema.fromTableSchema(projectSchema); }
Example #23
Source File: KuduTableUtils.java From bahir-flink with Apache License 2.0 | 5 votes |
public static TableSchema getSchemaWithSqlTimestamp(TableSchema schema) { TableSchema.Builder builder = new TableSchema.Builder(); TableSchemaUtils.getPhysicalSchema(schema).getTableColumns().forEach( tableColumn -> { if (tableColumn.getType().getLogicalType() instanceof TimestampType) { builder.field(tableColumn.getName(), tableColumn.getType().bridgedTo(Timestamp.class)); } else { builder.field(tableColumn.getName(), tableColumn.getType()); } }); return builder.build(); }
Example #24
Source File: FlinkPravegaTableSource.java From flink-connectors with Apache License 2.0 | 5 votes |
/** * Creates a Pravega {@link TableSource}. * @param sourceFunctionFactory a factory for the {@link FlinkPravegaReader} to implement {@link StreamTableSource} * @param inputFormatFactory a factory for the {@link FlinkPravegaInputFormat} to implement {@link BatchTableSource} * @param schema the table schema */ protected FlinkPravegaTableSource( Supplier<FlinkPravegaReader<Row>> sourceFunctionFactory, Supplier<FlinkPravegaInputFormat<Row>> inputFormatFactory, TableSchema schema) { this.sourceFunctionFactory = checkNotNull(sourceFunctionFactory, "sourceFunctionFactory"); this.inputFormatFactory = checkNotNull(inputFormatFactory, "inputFormatFactory"); this.schema = TableSchemaUtils.checkNoGeneratedColumns(schema); }
Example #25
Source File: TestTableSinkFactoryBase.java From flink with Apache License 2.0 | 4 votes |
public TestTableSink(TableSchema schema, String property) { this.schema = TableSchemaUtils.checkNoGeneratedColumns(schema); this.property = property; }
Example #26
Source File: TestTableSourceFactoryBase.java From flink with Apache License 2.0 | 4 votes |
public TestTableSource(TableSchema schema, String property, String proctime, List<RowtimeAttributeDescriptor> rowtime) { this.schema = TableSchemaUtils.checkNoGeneratedColumns(schema); this.property = property; this.proctime = proctime; this.rowtime = rowtime; }
Example #27
Source File: FlinkPravegaTableFactoryBase.java From flink-connectors with Apache License 2.0 | 4 votes |
@SuppressWarnings("unchecked") protected FlinkPravegaTableSource createFlinkPravegaTableSource(Map<String, String> properties) { final DescriptorProperties descriptorProperties = getValidatedProperties(properties); final TableSchema schema = TableSchemaUtils.getPhysicalSchema(descriptorProperties.getTableSchema(SCHEMA)); final DeserializationSchema<Row> deserializationSchema = getDeserializationSchema(properties); ConnectorConfigurations connectorConfigurations = new ConnectorConfigurations(); connectorConfigurations.parseConfigurations(descriptorProperties, ConnectorConfigurations.ConfigurationType.READER); // create source from the reader builder by using the supplied properties TableSourceReaderBuilder tableSourceReaderBuilder = new Pravega().tableSourceReaderBuilder(); tableSourceReaderBuilder.withDeserializationSchema(deserializationSchema); if (connectorConfigurations.getAssignerWithTimeWindows().isPresent()) { tableSourceReaderBuilder.withTimestampAssigner(connectorConfigurations.getAssignerWithTimeWindows().get()); } if (connectorConfigurations.getUid().isPresent()) { tableSourceReaderBuilder.uid(connectorConfigurations.getUid().get()); } if (connectorConfigurations.getRgScope().isPresent()) { tableSourceReaderBuilder.withReaderGroupScope(connectorConfigurations.getRgScope().get()); } if (connectorConfigurations.getRgName().isPresent()) { tableSourceReaderBuilder.withReaderGroupName(connectorConfigurations.getRgName().get()); } if (connectorConfigurations.getRefreshInterval().isPresent()) { tableSourceReaderBuilder.withReaderGroupRefreshTime(Time.milliseconds(connectorConfigurations.getRefreshInterval().get())); } if (connectorConfigurations.getEventReadTimeoutInterval().isPresent()) { tableSourceReaderBuilder.withEventReadTimeout(Time.milliseconds(connectorConfigurations.getEventReadTimeoutInterval().get())); } if (connectorConfigurations.getCheckpointInitiateTimeoutInterval().isPresent()) { tableSourceReaderBuilder.withCheckpointInitiateTimeout(Time.milliseconds(connectorConfigurations.getCheckpointInitiateTimeoutInterval().get())); } tableSourceReaderBuilder.withPravegaConfig(connectorConfigurations.getPravegaConfig()); if (connectorConfigurations.getMetrics().isPresent()) { tableSourceReaderBuilder.enableMetrics(connectorConfigurations.getMetrics().get()); } tableSourceReaderBuilder.withPravegaConfig(connectorConfigurations.getPravegaConfig()); for (StreamWithBoundaries streamWithBoundaries: connectorConfigurations.getReaderStreams()) { if (streamWithBoundaries.getFrom() != StreamCut.UNBOUNDED && streamWithBoundaries.getTo() != StreamCut.UNBOUNDED) { tableSourceReaderBuilder.forStream(streamWithBoundaries.getStream(), streamWithBoundaries.getFrom(), streamWithBoundaries.getTo()); } else if (streamWithBoundaries.getFrom() != StreamCut.UNBOUNDED) { tableSourceReaderBuilder.forStream(streamWithBoundaries.getStream(), streamWithBoundaries.getFrom()); } else { tableSourceReaderBuilder.forStream(streamWithBoundaries.getStream()); } } FlinkPravegaTableSource flinkPravegaTableSource = new FlinkPravegaTableSource( tableSourceReaderBuilder::buildSourceFunction, tableSourceReaderBuilder::buildInputFormat, schema); flinkPravegaTableSource.setRowtimeAttributeDescriptors(SchemaValidator.deriveRowtimeAttributes(descriptorProperties)); Optional<String> procTimeAttribute = SchemaValidator.deriveProctimeAttribute(descriptorProperties); if (procTimeAttribute.isPresent()) { flinkPravegaTableSource.setProctimeAttribute(procTimeAttribute.get()); } return flinkPravegaTableSource; }
Example #28
Source File: CollectBatchTableSink.java From flink with Apache License 2.0 | 4 votes |
public CollectBatchTableSink(String accumulatorName, TypeSerializer<Row> serializer, TableSchema tableSchema) { this.accumulatorName = accumulatorName; this.serializer = serializer; this.tableSchema = TableSchemaUtils.checkNoGeneratedColumns(tableSchema); }
Example #29
Source File: TestValuesTableFactory.java From flink with Apache License 2.0 | 4 votes |
@Override public void applyProjection(int[][] projectedFields) { this.physicalSchema = TableSchemaUtils.projectSchema(physicalSchema, projectedFields); this.projectedFields = Arrays.stream(projectedFields).mapToInt(f -> f[0]).toArray(); }
Example #30
Source File: CsvTableSourceFactoryBase.java From flink with Apache License 2.0 | 4 votes |
protected CsvTableSource createTableSource( Boolean isStreaming, Map<String, String> properties) { DescriptorProperties params = new DescriptorProperties(); params.putProperties(properties); // validate new FileSystemValidator().validate(params); new OldCsvValidator().validate(params); new SchemaValidator(isStreaming, false, false).validate(params); // build CsvTableSource.Builder csvTableSourceBuilder = new CsvTableSource.Builder(); TableSchema tableSchema = TableSchemaUtils.getPhysicalSchema(params.getTableSchema(SCHEMA)); // if a schema is defined, no matter derive schema is set or not, will use the defined schema final boolean hasSchema = params.hasPrefix(FORMAT_FIELDS); if (hasSchema) { TableSchema formatSchema = params.getTableSchema(FORMAT_FIELDS); // the CsvTableSource needs some rework first // for now the schema must be equal to the encoding // Ignore conversion classes in DataType if (!getFieldLogicalTypes(formatSchema).equals(getFieldLogicalTypes(tableSchema))) { throw new TableException(String.format( "Encodings that differ from the schema are not supported yet for" + " CsvTableSource, format schema is '%s', but table schema is '%s'.", formatSchema, tableSchema)); } } params.getOptionalString(CONNECTOR_PATH).ifPresent(csvTableSourceBuilder::path); params.getOptionalString(FORMAT_FIELD_DELIMITER).ifPresent(csvTableSourceBuilder::fieldDelimiter); params.getOptionalString(FORMAT_LINE_DELIMITER).ifPresent(csvTableSourceBuilder::lineDelimiter); for (int i = 0; i < tableSchema.getFieldCount(); ++i) { csvTableSourceBuilder.field(tableSchema.getFieldNames()[i], tableSchema.getFieldDataTypes()[i]); } params.getOptionalCharacter(FORMAT_QUOTE_CHARACTER).ifPresent(csvTableSourceBuilder::quoteCharacter); params.getOptionalString(FORMAT_COMMENT_PREFIX).ifPresent(csvTableSourceBuilder::commentPrefix); params.getOptionalBoolean(FORMAT_IGNORE_FIRST_LINE).ifPresent(flag -> { if (flag) { csvTableSourceBuilder.ignoreFirstLine(); } }); params.getOptionalBoolean(FORMAT_IGNORE_PARSE_ERRORS).ifPresent(flag -> { if (flag) { csvTableSourceBuilder.ignoreParseErrors(); } }); return csvTableSourceBuilder.build(); }