Java Code Examples for org.apache.flink.table.api.TableSchema#getFieldDataTypes()

The following examples show how to use org.apache.flink.table.api.TableSchema#getFieldDataTypes() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: HiveTableOutputFormat.java    From flink with Apache License 2.0 6 votes vote down vote up
public HiveTableOutputFormat(JobConf jobConf, ObjectPath tablePath, CatalogTable table, HiveTablePartition hiveTablePartition,
							Properties tableProperties, boolean overwrite) {
	super(jobConf.getCredentials());

	Preconditions.checkNotNull(table, "table cannot be null");
	Preconditions.checkNotNull(hiveTablePartition, "HiveTablePartition cannot be null");
	Preconditions.checkNotNull(tableProperties, "Table properties cannot be null");

	HadoopUtils.mergeHadoopConf(jobConf);
	this.jobConf = jobConf;
	this.tablePath = tablePath;
	this.partitionColumns = table.getPartitionKeys();
	TableSchema tableSchema = table.getSchema();
	this.fieldNames = tableSchema.getFieldNames();
	this.fieldTypes = tableSchema.getFieldDataTypes();
	this.hiveTablePartition = hiveTablePartition;
	this.tableProperties = tableProperties;
	this.overwrite = overwrite;
	isPartitioned = partitionColumns != null && !partitionColumns.isEmpty();
	isDynamicPartition = isPartitioned && partitionColumns.size() > hiveTablePartition.getPartitionSpec().size();
	hiveVersion = Preconditions.checkNotNull(jobConf.get(HiveCatalogValidator.CATALOG_HIVE_VERSION),
			"Hive version is not defined");
}
 
Example 2
Source File: SelectTableSinkSchemaConverter.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Convert time attributes (proc time / event time) to normal timestamps,
 * and return a new {@link TableSchema}.
 */
static TableSchema convertTimeAttributeToRegularTimestamp(TableSchema tableSchema) {
	DataType[] oldTypes = tableSchema.getFieldDataTypes();
	String[] oldNames = tableSchema.getFieldNames();

	TableSchema.Builder builder = TableSchema.builder();
	for (int i = 0; i < tableSchema.getFieldCount(); i++) {
		DataType fieldType = oldTypes[i];
		String fieldName = oldNames[i];
		if (fieldType.getLogicalType() instanceof TimestampType) {
			TimestampType timestampType = (TimestampType) fieldType.getLogicalType();
			if (!timestampType.getKind().equals(TimestampKind.REGULAR)) {
				// converts `TIME ATTRIBUTE(ROWTIME)`/`TIME ATTRIBUTE(PROCTIME)` to `TIMESTAMP`
				builder.field(fieldName, Types.SQL_TIMESTAMP);
				continue;
			}
		}
		builder.field(fieldName, fieldType);
	}
	return builder.build();
}
 
Example 3
Source File: TypeMappingUtilsTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testCheckPhysicalLogicalTypeCompatible() {
	TableSchema tableSchema = TableSchema.builder()
							.field("a", DataTypes.VARCHAR(2))
							.field("b", DataTypes.DECIMAL(20, 2))
							.build();
	TableSink tableSink = new TestTableSink(tableSchema);
	LegacyTypeInformationType legacyDataType = (LegacyTypeInformationType) tableSink.getConsumedDataType()
													.getLogicalType();
	TypeInformation legacyTypeInfo = ((TupleTypeInfo) legacyDataType.getTypeInformation()).getTypeAt(1);
	DataType physicalType = TypeConversions.fromLegacyInfoToDataType(legacyTypeInfo);
	TableSchema physicSchema = DataTypeUtils.expandCompositeTypeToSchema(physicalType);
	DataType[] logicalDataTypes = tableSchema.getFieldDataTypes();
	DataType[] physicalDataTypes = physicSchema.getFieldDataTypes();
	for (int i = 0; i < logicalDataTypes.length; i++) {
		TypeMappingUtils.checkPhysicalLogicalTypeCompatible(
				physicalDataTypes[i].getLogicalType(),
				logicalDataTypes[i].getLogicalType(),
				"physicalField",
				"logicalField",
				false);
	}
}
 
Example 4
Source File: HiveBatchSource.java    From Alink with Apache License 2.0 6 votes vote down vote up
@Override
public DataType getProducedDataType() {
    TableSchema fullSchema = getTableSchema();
    DataType type;
    if (projectedFields == null) {
        type = fullSchema.toRowDataType();
    } else {
        String[] fullNames = fullSchema.getFieldNames();
        DataType[] fullTypes = fullSchema.getFieldDataTypes();
        type = TableSchema.builder().fields(
            Arrays.stream(projectedFields).mapToObj(i -> fullNames[i]).toArray(String[]::new),
            Arrays.stream(projectedFields).mapToObj(i -> fullTypes[i]).toArray(DataType[]::new))
            .build().toRowDataType();
    }
    return type.bridgedTo(BaseRow.class);
}
 
Example 5
Source File: JdbcTableSource.java    From flink with Apache License 2.0 6 votes vote down vote up
private JdbcTableSource(
	JdbcOptions options, JdbcReadOptions readOptions, JdbcLookupOptions lookupOptions,
	TableSchema schema, int[] selectFields) {
	this.options = options;
	this.readOptions = readOptions;
	this.lookupOptions = lookupOptions;
	this.schema = schema;

	this.selectFields = selectFields;

	final DataType[] schemaDataTypes = schema.getFieldDataTypes();
	final String[] schemaFieldNames = schema.getFieldNames();
	if (selectFields != null) {
		DataType[] dataTypes = new DataType[selectFields.length];
		String[] fieldNames = new String[selectFields.length];
		for (int i = 0; i < selectFields.length; i++) {
			dataTypes[i] = schemaDataTypes[selectFields[i]];
			fieldNames[i] = schemaFieldNames[selectFields[i]];
		}
		this.producedDataType =
				TableSchema.builder().fields(fieldNames, dataTypes).build().toRowDataType();
	} else {
		this.producedDataType = schema.toRowDataType();
	}
}
 
Example 6
Source File: SelectTableSinkSchemaConverter.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Convert time attributes (proc time / event time) to regular timestamp
 * and build a new {@link TableSchema}.
 */
public static TableSchema convertTimeAttributeToRegularTimestamp(TableSchema tableSchema) {
	DataType[] dataTypes = tableSchema.getFieldDataTypes();
	String[] oldNames = tableSchema.getFieldNames();

	TableSchema.Builder builder = TableSchema.builder();
	for (int i = 0; i < tableSchema.getFieldCount(); i++) {
		DataType fieldType = dataTypes[i];
		String fieldName = oldNames[i];
		if (fieldType.getLogicalType() instanceof TimestampType) {
			TimestampType timestampType = (TimestampType) fieldType.getLogicalType();
			if (!timestampType.getKind().equals(TimestampKind.REGULAR)) {
				// converts `TIME ATTRIBUTE(ROWTIME)`/`TIME ATTRIBUTE(PROCTIME)` to `TIMESTAMP(3)`
				builder.field(fieldName, DataTypes.TIMESTAMP(3));
				continue;
			}
		}
		builder.field(fieldName, fieldType);
	}
	return builder.build();
}
 
Example 7
Source File: SelectTableSinkSchemaConverter.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Change to default conversion class and build a new {@link TableSchema}.
 */
public static TableSchema changeDefaultConversionClass(TableSchema tableSchema) {
	DataType[] oldTypes = tableSchema.getFieldDataTypes();
	String[] fieldNames = tableSchema.getFieldNames();

	TableSchema.Builder builder = TableSchema.builder();
	for (int i = 0; i < tableSchema.getFieldCount(); i++) {
		DataType fieldType = LogicalTypeDataTypeConverter.fromLogicalTypeToDataType(
				LogicalTypeDataTypeConverter.fromDataTypeToLogicalType(oldTypes[i]));
		builder.field(fieldNames[i], fieldType);
	}
	return builder.build();
}
 
Example 8
Source File: HiveTableSource.java    From flink with Apache License 2.0 5 votes vote down vote up
private TableSchema getProducedTableSchema() {
	TableSchema fullSchema = getTableSchema();
	if (projectedFields == null) {
		return fullSchema;
	} else {
		String[] fullNames = fullSchema.getFieldNames();
		DataType[] fullTypes = fullSchema.getFieldDataTypes();
		return TableSchema.builder().fields(
				Arrays.stream(projectedFields).mapToObj(i -> fullNames[i]).toArray(String[]::new),
				Arrays.stream(projectedFields).mapToObj(i -> fullTypes[i]).toArray(DataType[]::new)).build();
	}
}
 
Example 9
Source File: LocalExecutor.java    From flink with Apache License 2.0 5 votes vote down vote up
private static TableSchema removeTimeAttributes(TableSchema schema) {
	final TableSchema.Builder builder = TableSchema.builder();
	for (int i = 0; i < schema.getFieldCount(); i++) {
		final DataType dataType = schema.getFieldDataTypes()[i];
		final DataType convertedType = DataTypeUtils.replaceLogicalType(
			dataType,
			LogicalTypeUtils.removeTimeAttributes(dataType.getLogicalType()));
		builder.field(schema.getFieldNames()[i], convertedType);
	}
	return builder.build();
}
 
Example 10
Source File: HiveTableUtil.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Create Hive columns from Flink TableSchema.
 */
public static List<FieldSchema> createHiveColumns(TableSchema schema) {
	String[] fieldNames = schema.getFieldNames();
	DataType[] fieldTypes = schema.getFieldDataTypes();

	List<FieldSchema> columns = new ArrayList<>(fieldNames.length);

	for (int i = 0; i < fieldNames.length; i++) {
		columns.add(
			new FieldSchema(fieldNames[i], HiveTypeUtil.toHiveTypeInfo(fieldTypes[i], true).getTypeName(), null));
	}

	return columns;
}
 
Example 11
Source File: Pravega.java    From flink-connectors with Apache License 2.0 5 votes vote down vote up
/**
 * Creates FlinkPravegaOutputFormat based on the given table schema and current builder state.
 * @param tableSchema the schema of the sink table
 */
public FlinkPravegaOutputFormat<Row> createOutputFormat(TableSchema tableSchema) {
    Preconditions.checkState(routingKeyFieldName != null, "The routing key field must be provided.");
    Preconditions.checkState(serializationSchema != null, "The serializationSchema must be provided.");
    PravegaEventRouter<Row> eventRouter = new FlinkPravegaTableSink.RowBasedRouter(routingKeyFieldName, tableSchema.getFieldNames(), tableSchema.getFieldDataTypes());
    return new FlinkPravegaOutputFormat<>(
            getPravegaConfig().getClientConfig(),
            resolveStream(),
            serializationSchema,
            eventRouter
    );
}
 
Example 12
Source File: HiveTableUtil.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Create Hive columns from Flink TableSchema.
 */
public static List<FieldSchema> createHiveColumns(TableSchema schema) {
	String[] fieldNames = schema.getFieldNames();
	DataType[] fieldTypes = schema.getFieldDataTypes();

	List<FieldSchema> columns = new ArrayList<>(fieldNames.length);

	for (int i = 0; i < fieldNames.length; i++) {
		columns.add(
			new FieldSchema(fieldNames[i], HiveTypeUtil.toHiveTypeName(fieldTypes[i]), null));
	}

	return columns;
}
 
Example 13
Source File: HiveDB.java    From Alink with Apache License 2.0 5 votes vote down vote up
@Override
public Table getBatchTable(String tableName, Params parameter, Long sessionId) throws Exception {
    ExecutionEnvironment env = MLEnvironmentFactory.get(sessionId).getExecutionEnvironment();
    HiveBatchSource hiveTableSource = getHiveBatchSource(tableName, parameter);
    DataSet<BaseRow> dataSet = hiveTableSource.getDataSet(env);
    TableSchema schema = hiveTableSource.getTableSchema();
    final DataType[] dataTypes = schema.getFieldDataTypes();
    DataSet<Row> rows = dataSet.map(new BaseRowToRow(dataTypes));
    Table tbl = DataSetConversionUtil.toTable(sessionId, rows, schema);
    if (getPartitionCols(tableName).size() > 0) { // remove static partition columns
        String[] fieldNames = getColNames(tableName);
        tbl = tbl.select(Strings.join(fieldNames, ","));
    }
    return tbl;
}
 
Example 14
Source File: QuicksqlServerMeta.java    From Quicksql with MIT License 5 votes vote down vote up
public QueryResult getFlinkQueryResult(Entry<TableSchema, List<Row>> sparkData) throws Exception {
    if (sparkData == null) {
        return new QueryResult(new ArrayList<>(), new ArrayList<>());
    }
    TableSchema tableSchema = sparkData.getKey();
    if (tableSchema == null || tableSchema.getFieldDataTypes().length != tableSchema.getFieldNames().length) {
        throw new SparkException("collect data error");
    }
    org.apache.flink.table.types.DataType[] fieldDataTypes = tableSchema.getFieldDataTypes();
    String[] fieldNames = tableSchema.getFieldNames();
    List<Row> value = sparkData.getValue();
    List<Object> data = new ArrayList<>();
    List<ColumnMetaData> meta = new ArrayList<>();
    value.stream().forEach(column -> {
        Object[] objects = new Object[column.getArity()];
        for (int i = 0; i < column.getArity(); i++) {
            objects[i] = column.getField(i);
        }
        data.add(Arrays.asList(objects));
    });
    for (int index = 0; index < fieldNames.length; index++) {
        ScalarType columnType = getColumnType2(fieldDataTypes[index]);
        meta.add(new ColumnMetaData(index, false, true, false, false,
            1, true, -1, fieldNames[index], fieldNames[index], null, -1, -1, null, null,
            columnType, true, false, false, columnType.columnClassName()));
    }
    return new QueryResult(meta, data);
}
 
Example 15
Source File: LocalExecutor.java    From flink with Apache License 2.0 5 votes vote down vote up
private static TableSchema removeTimeAttributes(TableSchema schema) {
	final TableSchema.Builder builder = TableSchema.builder();
	for (int i = 0; i < schema.getFieldCount(); i++) {
		final DataType dataType = schema.getFieldDataTypes()[i];
		final DataType convertedType = DataTypeUtils.replaceLogicalType(
			dataType,
			LogicalTypeUtils.removeTimeAttributes(dataType.getLogicalType()));
		builder.field(schema.getFieldNames()[i], convertedType);
	}
	return builder.build();
}
 
Example 16
Source File: Mapper.java    From Alink with Apache License 2.0 4 votes vote down vote up
public Mapper(TableSchema dataSchema, Params params) {
	this.dataFieldNames = dataSchema.getFieldNames();
	this.dataFieldTypes = dataSchema.getFieldDataTypes();
	this.params = (null == params) ? new Params() : params.clone();
}
 
Example 17
Source File: CatalogTableSchemaResolver.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Resolve the computed column's type for the given schema.
 *
 * @param tableSchema Table schema to derive table field names and data types
 * @return the resolved TableSchema
 */
public TableSchema resolve(TableSchema tableSchema) {
	final String rowtime;
	if (!tableSchema.getWatermarkSpecs().isEmpty()) {
		// TODO: [FLINK-14473] we only support top-level rowtime attribute right now
		rowtime = tableSchema.getWatermarkSpecs().get(0).getRowtimeAttribute();
		if (rowtime.contains(".")) {
			throw new ValidationException(
					String.format("Nested field '%s' as rowtime attribute is not supported right now.", rowtime));
		}
	} else {
		rowtime = null;
	}

	String[] fieldNames = tableSchema.getFieldNames();
	DataType[] fieldTypes = tableSchema.getFieldDataTypes();

	TableSchema.Builder builder = TableSchema.builder();
	for (int i = 0; i < tableSchema.getFieldCount(); ++i) {
		TableColumn tableColumn = tableSchema.getTableColumns().get(i);
		DataType fieldType = fieldTypes[i];

		if (tableColumn.isGenerated()) {
			fieldType = resolveExpressionDataType(tableColumn.getExpr().get(), tableSchema);
			if (isProctime(fieldType)) {
				if (fieldNames[i].equals(rowtime)) {
					throw new TableException("Watermark can not be defined for a processing time attribute column.");
				}
			}
		}

		if (isStreamingMode && fieldNames[i].equals(rowtime)) {
			TimestampType originalType = (TimestampType) fieldType.getLogicalType();
			LogicalType rowtimeType = new TimestampType(
					originalType.isNullable(),
					TimestampKind.ROWTIME,
					originalType.getPrecision());
			fieldType = TypeConversions.fromLogicalToDataType(rowtimeType);
		}

		if (tableColumn.isGenerated()) {
			builder.field(fieldNames[i], fieldType, tableColumn.getExpr().get());
		} else {
			builder.field(fieldNames[i], fieldType);
		}
	}

	tableSchema.getWatermarkSpecs().forEach(builder::watermark);
	tableSchema.getPrimaryKey().ifPresent(
			pk -> builder.primaryKey(pk.getName(), pk.getColumns().toArray(new String[0])));
	return builder.build();
}
 
Example 18
Source File: SqlToOperationConverter.java    From flink with Apache License 2.0 4 votes vote down vote up
/** Convert CREATE VIEW statement. */
private Operation convertCreateView(SqlCreateView sqlCreateView) {
	final SqlNode query = sqlCreateView.getQuery();
	final SqlNodeList fieldList = sqlCreateView.getFieldList();

	SqlNode validateQuery = flinkPlanner.validate(query);
	PlannerQueryOperation operation = toQueryOperation(flinkPlanner, validateQuery);
	TableSchema schema = operation.getTableSchema();

	// the view column list in CREATE VIEW is optional, if it's not empty, we should update
	// the column name with the names in view column list.
	if (!fieldList.getList().isEmpty()) {
		// alias column names
		String[] inputFieldNames = schema.getFieldNames();
		String[] aliasFieldNames = fieldList.getList().stream()
				.map(SqlNode::toString)
				.toArray(String[]::new);

		if (inputFieldNames.length != aliasFieldNames.length) {
			throw new SqlConversionException(String.format(
					"VIEW definition and input fields not match:\n\tDef fields: %s.\n\tInput fields: %s.",
					Arrays.toString(aliasFieldNames), Arrays.toString(inputFieldNames)));
		}

		DataType[] inputFieldTypes = schema.getFieldDataTypes();
		schema = TableSchema.builder().fields(aliasFieldNames, inputFieldTypes).build();
	}

	String originalQuery = getQuotedSqlString(query);
	String expandedQuery = getQuotedSqlString(validateQuery);
	String comment = sqlCreateView.getComment().map(c -> c.getNlsString().getValue()).orElse(null);
	CatalogView catalogView = new CatalogViewImpl(originalQuery,
			expandedQuery,
			schema,
			Collections.emptyMap(),
			comment);

	UnresolvedIdentifier unresolvedIdentifier = UnresolvedIdentifier.of(sqlCreateView.fullViewName());
	ObjectIdentifier identifier = catalogManager.qualifyIdentifier(unresolvedIdentifier);

	return new CreateViewOperation(
			identifier,
			catalogView,
			sqlCreateView.isIfNotExists(),
			sqlCreateView.isTemporary());
}
 
Example 19
Source File: DescriptorProperties.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Adds a table schema under the given key.
 */
public void putTableSchema(String key, TableSchema schema) {
	checkNotNull(key);
	checkNotNull(schema);

	final String[] fieldNames = schema.getFieldNames();
	final DataType[] fieldTypes = schema.getFieldDataTypes();
	final String[] fieldExpressions = schema.getTableColumns().stream()
		.map(column -> column.getExpr().orElse(null))
		.toArray(String[]::new);

	final List<List<String>> values = new ArrayList<>();
	for (int i = 0; i < schema.getFieldCount(); i++) {
		values.add(
			Arrays.asList(
				fieldNames[i],
				fieldTypes[i].getLogicalType().asSerializableString(),
				fieldExpressions[i]));
	}

	putIndexedOptionalProperties(
		key,
		Arrays.asList(NAME, DATA_TYPE, EXPR),
		values);

	if (!schema.getWatermarkSpecs().isEmpty()) {
		final List<List<String>> watermarkValues = new ArrayList<>();
		for (WatermarkSpec spec : schema.getWatermarkSpecs()) {
			watermarkValues.add(Arrays.asList(
				spec.getRowtimeAttribute(),
				spec.getWatermarkExpr(),
				spec.getWatermarkExprOutputType().getLogicalType().asSerializableString()));
		}
		putIndexedFixedProperties(
			key + '.' + WATERMARK,
			Arrays.asList(WATERMARK_ROWTIME, WATERMARK_STRATEGY_EXPR, WATERMARK_STRATEGY_DATA_TYPE),
			watermarkValues);
	}

	schema.getPrimaryKey().ifPresent(pk -> {
		putString(key + '.' + PRIMARY_KEY_NAME, pk.getName());
		putString(key + '.' + PRIMARY_KEY_COLUMNS, String.join(",", pk.getColumns()));
	});
}
 
Example 20
Source File: ModelMapper.java    From flink with Apache License 2.0 2 votes vote down vote up
/**
 * Constructs a ModelMapper.
 *
 * @param modelSchema The schema of the model rows passed to {@link #loadModel(List)}.
 * @param dataSchema The schema of the input data rows.
 * @param params The parameters of this ModelMapper.
 */
public ModelMapper(TableSchema modelSchema, TableSchema dataSchema, Params params) {
	super(dataSchema, params);
	this.modelFieldNames = modelSchema.getFieldNames();
	this.modelFieldTypes = modelSchema.getFieldDataTypes();
}