Java Code Examples for org.apache.flink.api.common.typeinfo.Types#ROW_NAMED

The following examples show how to use org.apache.flink.api.common.typeinfo.Types#ROW_NAMED . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: HBaseConnectorITCase.java    From flink with Apache License 2.0 6 votes vote down vote up
private static Map<String, String> hbaseTableProperties() {
	Map<String, String> properties = new HashMap<>();
	properties.put(CONNECTOR_TYPE, CONNECTOR_TYPE_VALUE_HBASE);
	properties.put(CONNECTOR_VERSION, CONNECTOR_VERSION_VALUE_143);
	properties.put(CONNECTOR_PROPERTY_VERSION, "1");
	properties.put(CONNECTOR_TABLE_NAME, TEST_TABLE_1);
	// get zk quorum from "hbase-site.xml" in classpath
	String hbaseZk = HBaseConfiguration.create().get(HConstants.ZOOKEEPER_QUORUM);
	properties.put(CONNECTOR_ZK_QUORUM, hbaseZk);
	// schema
	String[] columnNames = {FAMILY1, ROWKEY, FAMILY2, FAMILY3};
	TypeInformation<Row> f1 = Types.ROW_NAMED(new String[]{F1COL1}, Types.INT);
	TypeInformation<Row> f2 = Types.ROW_NAMED(new String[]{F2COL1, F2COL2}, Types.STRING, Types.LONG);
	TypeInformation<Row> f3 = Types.ROW_NAMED(new String[]{F3COL1, F3COL2, F3COL3}, Types.DOUBLE, Types.BOOLEAN, Types.STRING);
	TypeInformation[] columnTypes = new TypeInformation[]{f1, Types.INT, f2, f3};

	DescriptorProperties descriptorProperties = new DescriptorProperties(true);
	TableSchema tableSchema = new TableSchema(columnNames, columnTypes);
	descriptorProperties.putTableSchema(SCHEMA, tableSchema);
	descriptorProperties.putProperties(properties);
	return descriptorProperties.asMap();
}
 
Example 2
Source File: JsonRowSerializationSchemaTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testNestedSchema() throws IOException {
	final TypeInformation<Row> rowSchema = Types.ROW_NAMED(
		new String[] {"f1", "f2", "f3"},
		Types.INT, Types.BOOLEAN, Types.ROW(Types.INT, Types.DOUBLE));

	final Row row = new Row(3);
	row.setField(0, 42);
	row.setField(1, false);
	final Row nested = new Row(2);
	nested.setField(0, 22);
	nested.setField(1, 2.3);
	row.setField(2, nested);

	final Row resultRow = serializeAndDeserialize(rowSchema, row);
	assertEquals(row, resultRow);
}
 
Example 3
Source File: JsonRowSerializationSchemaTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testSerializationOfTwoRows() throws IOException {
	final TypeInformation<Row> rowSchema = Types.ROW_NAMED(
		new String[] {"f1", "f2", "f3"},
		Types.INT, Types.BOOLEAN, Types.STRING);

	final Row row1 = new Row(3);
	row1.setField(0, 1);
	row1.setField(1, true);
	row1.setField(2, "str");

	final JsonRowSerializationSchema serializationSchema = new JsonRowSerializationSchema(rowSchema);
	final JsonRowDeserializationSchema deserializationSchema = new JsonRowDeserializationSchema(rowSchema);

	byte[] bytes = serializationSchema.serialize(row1);
	assertEquals(row1, deserializationSchema.deserialize(bytes));

	final Row row2 = new Row(3);
	row2.setField(0, 10);
	row2.setField(1, false);
	row2.setField(2, "newStr");

	bytes = serializationSchema.serialize(row2);
	assertEquals(row2, deserializationSchema.deserialize(bytes));
}
 
Example 4
Source File: JsonRowSchemaConverterTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testComplexSchema() throws Exception {
	final URL url = getClass().getClassLoader().getResource("complex-schema.json");
	Objects.requireNonNull(url);
	final String schema = FileUtils.readFileUtf8(new File(url.getFile()));
	final TypeInformation<?> result = JsonRowSchemaConverter.convert(schema);

	final TypeInformation<?> expected = Types.ROW_NAMED(
		new String[] {"fn", "familyName", "additionalName", "tuples", "honorificPrefix", "url",
			"email", "tel", "sound", "org"},
		Types.STRING, Types.STRING, Types.BOOLEAN, Types.ROW(Types.BIG_DEC, Types.STRING, Types.STRING, Types.STRING),
		Types.OBJECT_ARRAY(Types.STRING), Types.STRING, Types.ROW_NAMED(new String[] {"type", "value"}, Types.STRING, Types.STRING),
		Types.ROW_NAMED(new String[] {"type", "value"}, Types.BIG_DEC, Types.STRING), Types.VOID,
		Types.ROW_NAMED(new String[] {"organizationUnit"}, Types.ROW()));

	assertEquals(expected, result);
}
 
Example 5
Source File: OrcTableSourceTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testGetReturnType() throws Exception {

	OrcTableSource orc = OrcTableSource.builder()
		.path(getPath(TEST_FILE_NESTED))
		.forOrcSchema(TEST_SCHEMA_NESTED)
		.build();

	TypeInformation<Row> returnType = orc.getReturnType();
	assertNotNull(returnType);
	assertTrue(returnType instanceof RowTypeInfo);
	RowTypeInfo rowType = (RowTypeInfo) returnType;

	TypeInformation<Row> expected = Types.ROW_NAMED(getNestedFieldNames(), getNestedFieldTypes());
	assertEquals(expected, rowType);
}
 
Example 6
Source File: OrcTableSourceTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testGetReturnType() throws Exception {

	OrcTableSource orc = OrcTableSource.builder()
		.path(getPath(TEST_FILE_NESTED))
		.forOrcSchema(TEST_SCHEMA_NESTED)
		.build();

	TypeInformation<Row> returnType = orc.getReturnType();
	assertNotNull(returnType);
	assertTrue(returnType instanceof RowTypeInfo);
	RowTypeInfo rowType = (RowTypeInfo) returnType;

	TypeInformation<Row> expected = Types.ROW_NAMED(getNestedFieldNames(), getNestedFieldTypes());
	assertEquals(expected, rowType);
}
 
Example 7
Source File: JsonRowSerializationSchemaTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testNestedSchema() {
	final TypeInformation<Row> rowSchema = Types.ROW_NAMED(
		new String[] {"f1", "f2", "f3"},
		Types.INT, Types.BOOLEAN, Types.ROW(Types.INT, Types.DOUBLE));

	final Row row = new Row(3);
	row.setField(0, 42);
	row.setField(1, false);
	final Row nested = new Row(2);
	nested.setField(0, 22);
	nested.setField(1, 2.3);
	row.setField(2, nested);

	final JsonRowSerializationSchema serializationSchema = new JsonRowSerializationSchema.Builder(rowSchema)
		.build();
	final JsonRowDeserializationSchema deserializationSchema = new JsonRowDeserializationSchema.Builder(rowSchema)
		.build();

	assertThat(row, whenSerializedWith(serializationSchema)
		.andDeserializedWith(deserializationSchema)
		.equalsTo(row));
}
 
Example 8
Source File: JsonRowSchemaConverterTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testComplexSchema() throws Exception {
	final URL url = getClass().getClassLoader().getResource("complex-schema.json");
	Objects.requireNonNull(url);
	final String schema = FileUtils.readFileUtf8(new File(url.getFile()));
	final TypeInformation<?> result = JsonRowSchemaConverter.convert(schema);

	final TypeInformation<?> expected = Types.ROW_NAMED(
		new String[] {"fn", "familyName", "additionalName", "tuples", "honorificPrefix", "url",
			"email", "tel", "sound", "org"},
		Types.STRING, Types.STRING, Types.BOOLEAN, Types.ROW(Types.BIG_DEC, Types.STRING, Types.STRING, Types.STRING),
		Types.OBJECT_ARRAY(Types.STRING), Types.STRING, Types.ROW_NAMED(new String[] {"type", "value"}, Types.STRING, Types.STRING),
		Types.ROW_NAMED(new String[] {"type", "value"}, Types.BIG_DEC, Types.STRING), Types.VOID,
		Types.ROW_NAMED(new String[] {"organizationUnit"}, Types.ROW()));

	assertEquals(expected, result);
}
 
Example 9
Source File: JsonRowSchemaConverterTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testReferenceSchema() throws Exception {
	final URL url = getClass().getClassLoader().getResource("reference-schema.json");
	Objects.requireNonNull(url);
	final String schema = FileUtils.readFileUtf8(new File(url.getFile()));
	final TypeInformation<?> result = JsonRowSchemaConverter.convert(schema);

	final TypeInformation<?> expected = Types.ROW_NAMED(
		new String[] {"billing_address", "shipping_address", "optional_address"},
		Types.ROW_NAMED(new String[] {"street_address", "city", "state"}, Types.STRING, Types.STRING, Types.STRING),
		Types.ROW_NAMED(new String[] {"street_address", "city", "state"}, Types.STRING, Types.STRING, Types.STRING),
		Types.ROW_NAMED(new String[] {"street_address", "city", "state"}, Types.STRING, Types.STRING, Types.STRING));

	assertEquals(expected, result);
}
 
Example 10
Source File: JsonRowSerializationSchemaTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testSerializationOfTwoRows() throws IOException {
	final TypeInformation<Row> rowSchema = Types.ROW_NAMED(
		new String[] {"f1", "f2", "f3"},
		Types.INT, Types.BOOLEAN, Types.STRING);

	final Row row1 = new Row(3);
	row1.setField(0, 1);
	row1.setField(1, true);
	row1.setField(2, "str");

	final JsonRowSerializationSchema serializationSchema = new JsonRowSerializationSchema.Builder(rowSchema)
		.build();
	final JsonRowDeserializationSchema deserializationSchema = new JsonRowDeserializationSchema.Builder(rowSchema)
		.build();

	byte[] bytes = serializationSchema.serialize(row1);
	assertEquals(row1, deserializationSchema.deserialize(bytes));

	final Row row2 = new Row(3);
	row2.setField(0, 10);
	row2.setField(1, false);
	row2.setField(2, "newStr");

	bytes = serializationSchema.serialize(row2);
	assertEquals(row2, deserializationSchema.deserialize(bytes));
}
 
Example 11
Source File: JsonRowSerializationSchemaTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test(expected = RuntimeException.class)
public void testSerializeRowWithInvalidNumberOfFields() {
	final TypeInformation<Row> rowSchema = Types.ROW_NAMED(
		new String[] {"f1", "f2", "f3"},
		Types.INT, Types.BOOLEAN, Types.STRING);

	final Row row = new Row(1);
	row.setField(0, 1);

	final JsonRowSerializationSchema serializationSchema = new JsonRowSerializationSchema(rowSchema);
	serializationSchema.serialize(row);
}
 
Example 12
Source File: LegacyTypeInfoDataTypeConverter.java    From flink with Apache License 2.0 5 votes vote down vote up
private static TypeInformation<?> convertToRowTypeInfo(FieldsDataType fieldsDataType) {
	final RowType rowType = (RowType) fieldsDataType.getLogicalType();

	final String[] fieldNames = rowType.getFields()
		.stream()
		.map(RowType.RowField::getName)
		.toArray(String[]::new);

	final TypeInformation<?>[] fieldTypes = fieldsDataType.getChildren()
		.stream()
		.map(LegacyTypeInfoDataTypeConverter::toLegacyTypeInfo)
		.toArray(TypeInformation[]::new);

	return Types.ROW_NAMED(fieldNames, fieldTypes);
}
 
Example 13
Source File: JsonRowSchemaConverter.java    From flink with Apache License 2.0 5 votes vote down vote up
private static TypeInformation<Row> convertObject(String location, JsonNode node, JsonNode root) {
	// validate properties
	if (!node.has(PROPERTIES)) {
		return Types.ROW();
	}
	if (!node.isObject()) {
		throw new IllegalArgumentException(
			"Invalid '" + PROPERTIES + "' property for object type in node: " + location);
	}
	final JsonNode props = node.get(PROPERTIES);
	final String[] names = new String[props.size()];
	final TypeInformation<?>[] types = new TypeInformation[props.size()];

	final Iterator<Map.Entry<String, JsonNode>> fieldIter = props.fields();
	int i = 0;
	while (fieldIter.hasNext()) {
		final Map.Entry<String, JsonNode> subNode = fieldIter.next();

		// set field name
		names[i] = subNode.getKey();

		// set type
		types[i] = convertType(location + '/' + subNode.getKey(), subNode.getValue(), root);

		i++;
	}

	// validate that object does not contain additional properties
	if (node.has(ADDITIONAL_PROPERTIES) && node.get(ADDITIONAL_PROPERTIES).isBoolean() &&
			node.get(ADDITIONAL_PROPERTIES).asBoolean()) {
		throw new IllegalArgumentException(
			"An object must not allow additional properties in node: " + location);
	}

	return Types.ROW_NAMED(names, types);
}
 
Example 14
Source File: ResultStore.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a result. Might start threads or opens sockets so every created result must be closed.
 */
public <T> DynamicResult<T> createResult(Environment env, TableSchema schema, ExecutionConfig config) {

	final TypeInformation<Row> outputType = Types.ROW_NAMED(schema.getFieldNames(), schema.getFieldTypes());

	if (env.getExecution().isStreamingExecution()) {
		// determine gateway address (and port if possible)
		final InetAddress gatewayAddress = getGatewayAddress(env.getDeployment());
		final int gatewayPort = getGatewayPort(env.getDeployment());

		if (env.getExecution().isChangelogMode()) {
			return new ChangelogCollectStreamResult<>(outputType, config, gatewayAddress, gatewayPort);
		} else {
			return new MaterializedCollectStreamResult<>(
				outputType,
				config,
				gatewayAddress,
				gatewayPort,
				env.getExecution().getMaxTableResultRows());
		}

	} else {
		// Batch Execution
		if (!env.getExecution().isTableMode()) {
			throw new SqlExecutionException("Results of batch queries can only be served in table mode.");
		}
		return new MaterializedCollectBatchResult<>(outputType, config);
	}
}
 
Example 15
Source File: OrcTableSourceTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private TypeInformation[] getNestedFieldTypes() {
	return new TypeInformation[]{
		Types.BOOLEAN, Types.BYTE, Types.SHORT, Types.INT, Types.LONG, Types.FLOAT, Types.DOUBLE,
		PrimitiveArrayTypeInfo.BYTE_PRIMITIVE_ARRAY_TYPE_INFO, Types.STRING,
		Types.ROW_NAMED(
			new String[]{"list"},
			ObjectArrayTypeInfo.getInfoFor(
				Types.ROW_NAMED(
					new String[]{"int1", "string1"},
					Types.INT, Types.STRING
				)
			)
		),
		ObjectArrayTypeInfo.getInfoFor(
			Types.ROW_NAMED(
				new String[]{"int1", "string1"},
				Types.INT, Types.STRING
			)
		),
		new MapTypeInfo<>(
			Types.STRING,
			Types.ROW_NAMED(
				new String[]{"int1", "string1"},
				Types.INT, Types.STRING
			)
		)
	};
}
 
Example 16
Source File: AvroSchemaConverter.java    From flink with Apache License 2.0 4 votes vote down vote up
private static TypeInformation<?> convertToTypeInfo(Schema schema) {
	switch (schema.getType()) {
		case RECORD:
			final List<Schema.Field> fields = schema.getFields();

			final TypeInformation<?>[] types = new TypeInformation<?>[fields.size()];
			final String[] names = new String[fields.size()];
			for (int i = 0; i < fields.size(); i++) {
				final Schema.Field field = fields.get(i);
				types[i] = convertToTypeInfo(field.schema());
				names[i] = field.name();
			}
			return Types.ROW_NAMED(names, types);
		case ENUM:
			return Types.STRING;
		case ARRAY:
			// result type might either be ObjectArrayTypeInfo or BasicArrayTypeInfo for Strings
			return Types.OBJECT_ARRAY(convertToTypeInfo(schema.getElementType()));
		case MAP:
			return Types.MAP(Types.STRING, convertToTypeInfo(schema.getValueType()));
		case UNION:
			final Schema actualSchema;
			if (schema.getTypes().size() == 2 && schema.getTypes().get(0).getType() == Schema.Type.NULL) {
				actualSchema = schema.getTypes().get(1);
			} else if (schema.getTypes().size() == 2 && schema.getTypes().get(1).getType() == Schema.Type.NULL) {
				actualSchema = schema.getTypes().get(0);
			} else if (schema.getTypes().size() == 1) {
				actualSchema = schema.getTypes().get(0);
			} else {
				// use Kryo for serialization
				return Types.GENERIC(Object.class);
			}
			return convertToTypeInfo(actualSchema);
		case FIXED:
			// logical decimal type
			if (schema.getLogicalType() instanceof LogicalTypes.Decimal) {
				return Types.BIG_DEC;
			}
			// convert fixed size binary data to primitive byte arrays
			return Types.PRIMITIVE_ARRAY(Types.BYTE);
		case STRING:
			// convert Avro's Utf8/CharSequence to String
			return Types.STRING;
		case BYTES:
			// logical decimal type
			if (schema.getLogicalType() instanceof LogicalTypes.Decimal) {
				return Types.BIG_DEC;
			}
			return Types.PRIMITIVE_ARRAY(Types.BYTE);
		case INT:
			// logical date and time type
			final org.apache.avro.LogicalType logicalType = schema.getLogicalType();
			if (logicalType == LogicalTypes.date()) {
				return Types.SQL_DATE;
			} else if (logicalType == LogicalTypes.timeMillis()) {
				return Types.SQL_TIME;
			}
			return Types.INT;
		case LONG:
			// logical timestamp type
			if (schema.getLogicalType() == LogicalTypes.timestampMillis()) {
				return Types.SQL_TIMESTAMP;
			}
			return Types.LONG;
		case FLOAT:
			return Types.FLOAT;
		case DOUBLE:
			return Types.DOUBLE;
		case BOOLEAN:
			return Types.BOOLEAN;
		case NULL:
			return Types.VOID;
	}
	throw new IllegalArgumentException("Unsupported Avro type '" + schema.getType() + "'.");
}
 
Example 17
Source File: AvroSchemaConverterTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
private void validateUserSchema(TypeInformation<?> actual) {
	final TypeInformation<Row> address = Types.ROW_NAMED(
		new String[]{
			"num",
			"street",
			"city",
			"state",
			"zip"},
		Types.INT,
		Types.STRING,
		Types.STRING,
		Types.STRING,
		Types.STRING);

	final TypeInformation<Row> user = Types.ROW_NAMED(
		new String[] {
			"name",
			"favorite_number",
			"favorite_color",
			"type_long_test",
			"type_double_test",
			"type_null_test",
			"type_bool_test",
			"type_array_string",
			"type_array_boolean",
			"type_nullable_array",
			"type_enum",
			"type_map",
			"type_fixed",
			"type_union",
			"type_nested",
			"type_bytes",
			"type_date",
			"type_time_millis",
			"type_time_micros",
			"type_timestamp_millis",
			"type_timestamp_micros",
			"type_decimal_bytes",
			"type_decimal_fixed"},
		Types.STRING,
		Types.INT,
		Types.STRING,
		Types.LONG,
		Types.DOUBLE,
		Types.VOID,
		Types.BOOLEAN,
		Types.OBJECT_ARRAY(Types.STRING),
		Types.OBJECT_ARRAY(Types.BOOLEAN),
		Types.OBJECT_ARRAY(Types.STRING),
		Types.STRING,
		Types.MAP(Types.STRING, Types.LONG),
		Types.PRIMITIVE_ARRAY(Types.BYTE),
		Types.GENERIC(Object.class),
		address,
		Types.PRIMITIVE_ARRAY(Types.BYTE),
		Types.SQL_DATE,
		Types.SQL_TIME,
		Types.INT,
		Types.SQL_TIMESTAMP,
		Types.LONG,
		Types.BIG_DEC,
		Types.BIG_DEC);

	assertEquals(user, actual);

	final RowTypeInfo userRowInfo = (RowTypeInfo) user;
	assertTrue(userRowInfo.schemaEquals(actual));
}
 
Example 18
Source File: CollectStreamTableSink.java    From zeppelin with Apache License 2.0 4 votes vote down vote up
@Override
public TypeInformation<Row> getRecordType() {
  return Types.ROW_NAMED(fieldNames, fieldTypes);
}
 
Example 19
Source File: CollectBatchTableSink.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@Override
public TypeInformation<Row> getOutputType() {
	return Types.ROW_NAMED(fieldNames, fieldTypes);
}
 
Example 20
Source File: TableSchema.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
/**
 * Converts a table schema into a (nested) type information describing a {@link Row}.
 */
public TypeInformation<Row> toRowType() {
	return Types.ROW_NAMED(fieldNames, fieldTypes);
}