Java Code Examples for org.apache.flink.api.common.typeinfo.BasicTypeInfo#SHORT_TYPE_INFO

private PredicateLeaf.Type toOrcType(TypeInformation<?> type) {
	if (type == BasicTypeInfo.BYTE_TYPE_INFO ||
		type == BasicTypeInfo.SHORT_TYPE_INFO ||
		type == BasicTypeInfo.INT_TYPE_INFO ||
		type == BasicTypeInfo.LONG_TYPE_INFO) {
		return PredicateLeaf.Type.LONG;
	} else if (type == BasicTypeInfo.FLOAT_TYPE_INFO ||
		type == BasicTypeInfo.DOUBLE_TYPE_INFO) {
		return PredicateLeaf.Type.FLOAT;
	} else if (type == BasicTypeInfo.BOOLEAN_TYPE_INFO) {
		return PredicateLeaf.Type.BOOLEAN;
	} else if (type == BasicTypeInfo.STRING_TYPE_INFO) {
		return PredicateLeaf.Type.STRING;
	} else if (type == SqlTimeTypeInfo.TIMESTAMP) {
		return PredicateLeaf.Type.TIMESTAMP;
	} else if (type == SqlTimeTypeInfo.DATE) {
		return PredicateLeaf.Type.DATE;
	} else if (type == BasicTypeInfo.BIG_DEC_TYPE_INFO) {
		return PredicateLeaf.Type.DECIMAL;
	} else {
		// unsupported type
		return null;
	}
}
 

private PredicateLeaf.Type toOrcType(TypeInformation<?> type) {
	if (type == BasicTypeInfo.BYTE_TYPE_INFO ||
		type == BasicTypeInfo.SHORT_TYPE_INFO ||
		type == BasicTypeInfo.INT_TYPE_INFO ||
		type == BasicTypeInfo.LONG_TYPE_INFO) {
		return PredicateLeaf.Type.LONG;
	} else if (type == BasicTypeInfo.FLOAT_TYPE_INFO ||
		type == BasicTypeInfo.DOUBLE_TYPE_INFO) {
		return PredicateLeaf.Type.FLOAT;
	} else if (type == BasicTypeInfo.BOOLEAN_TYPE_INFO) {
		return PredicateLeaf.Type.BOOLEAN;
	} else if (type == BasicTypeInfo.STRING_TYPE_INFO) {
		return PredicateLeaf.Type.STRING;
	} else if (type == SqlTimeTypeInfo.TIMESTAMP) {
		return PredicateLeaf.Type.TIMESTAMP;
	} else if (type == SqlTimeTypeInfo.DATE) {
		return PredicateLeaf.Type.DATE;
	} else if (type == BasicTypeInfo.BIG_DEC_TYPE_INFO) {
		return PredicateLeaf.Type.DECIMAL;
	} else {
		// unsupported type
		return null;
	}
}
 

private PredicateLeaf.Type toOrcType(TypeInformation<?> type) {
	if (type == BasicTypeInfo.BYTE_TYPE_INFO ||
		type == BasicTypeInfo.SHORT_TYPE_INFO ||
		type == BasicTypeInfo.INT_TYPE_INFO ||
		type == BasicTypeInfo.LONG_TYPE_INFO) {
		return PredicateLeaf.Type.LONG;
	} else if (type == BasicTypeInfo.FLOAT_TYPE_INFO ||
		type == BasicTypeInfo.DOUBLE_TYPE_INFO) {
		return PredicateLeaf.Type.FLOAT;
	} else if (type == BasicTypeInfo.BOOLEAN_TYPE_INFO) {
		return PredicateLeaf.Type.BOOLEAN;
	} else if (type == BasicTypeInfo.STRING_TYPE_INFO) {
		return PredicateLeaf.Type.STRING;
	} else if (type == SqlTimeTypeInfo.TIMESTAMP) {
		return PredicateLeaf.Type.TIMESTAMP;
	} else if (type == SqlTimeTypeInfo.DATE) {
		return PredicateLeaf.Type.DATE;
	} else if (type == BasicTypeInfo.BIG_DEC_TYPE_INFO) {
		return PredicateLeaf.Type.DECIMAL;
	} else {
		// unsupported type
		return null;
	}
}
 

private TypeInformation getFieldType(HCatFieldSchema fieldSchema) {

		switch(fieldSchema.getType()) {
			case INT:
				return BasicTypeInfo.INT_TYPE_INFO;
			case TINYINT:
				return BasicTypeInfo.BYTE_TYPE_INFO;
			case SMALLINT:
				return BasicTypeInfo.SHORT_TYPE_INFO;
			case BIGINT:
				return BasicTypeInfo.LONG_TYPE_INFO;
			case BOOLEAN:
				return BasicTypeInfo.BOOLEAN_TYPE_INFO;
			case FLOAT:
				return BasicTypeInfo.FLOAT_TYPE_INFO;
			case DOUBLE:
				return BasicTypeInfo.DOUBLE_TYPE_INFO;
			case STRING:
				return BasicTypeInfo.STRING_TYPE_INFO;
			case BINARY:
				return PrimitiveArrayTypeInfo.BYTE_PRIMITIVE_ARRAY_TYPE_INFO;
			case ARRAY:
				return new GenericTypeInfo(List.class);
			case MAP:
				return new GenericTypeInfo(Map.class);
			case STRUCT:
				return new GenericTypeInfo(List.class);
			default:
				throw new IllegalArgumentException("Unknown data type \"" + fieldSchema.getType() + "\" encountered.");
		}
	}
 

@Test
public void testRowSerializerWithComplexTypes() {
	TypeInformation<Row> typeInfo = new RowTypeInfo(
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.DOUBLE_TYPE_INFO,
		BasicTypeInfo.STRING_TYPE_INFO,
		new TupleTypeInfo<Tuple3<Integer, Boolean, Short>>(
			BasicTypeInfo.INT_TYPE_INFO,
			BasicTypeInfo.BOOLEAN_TYPE_INFO,
			BasicTypeInfo.SHORT_TYPE_INFO),
		TypeExtractor.createTypeInfo(MyPojo.class));

	MyPojo testPojo1 = new MyPojo();
	testPojo1.name = null;
	MyPojo testPojo2 = new MyPojo();
	testPojo2.name = "Test1";
	MyPojo testPojo3 = new MyPojo();
	testPojo3.name = "Test2";

	Row[] data = new Row[]{
		createRow(null, null, null, null, null),
		createRow(0, null, null, null, null),
		createRow(0, 0.0, null, null, null),
		createRow(0, 0.0, "a", null, null),
		createRow(1, 0.0, "a", null, null),
		createRow(1, 1.0, "a", null, null),
		createRow(1, 1.0, "b", null, null),
		createRow(1, 1.0, "b", new Tuple3<>(1, false, (short) 2), null),
		createRow(1, 1.0, "b", new Tuple3<>(2, false, (short) 2), null),
		createRow(1, 1.0, "b", new Tuple3<>(2, true, (short) 2), null),
		createRow(1, 1.0, "b", new Tuple3<>(2, true, (short) 3), null),
		createRow(1, 1.0, "b", new Tuple3<>(2, true, (short) 3), testPojo1),
		createRow(1, 1.0, "b", new Tuple3<>(2, true, (short) 3), testPojo2),
		createRow(1, 1.0, "b", new Tuple3<>(2, true, (short) 3), testPojo3)
	};

	TypeSerializer<Row> serializer = typeInfo.createSerializer(new ExecutionConfig());
	RowSerializerTestInstance testInstance = new RowSerializerTestInstance(serializer, data);
	testInstance.testAll();
}
 

private TypeInformation getFieldType(HCatFieldSchema fieldSchema) {

		switch(fieldSchema.getType()) {
			case INT:
				return BasicTypeInfo.INT_TYPE_INFO;
			case TINYINT:
				return BasicTypeInfo.BYTE_TYPE_INFO;
			case SMALLINT:
				return BasicTypeInfo.SHORT_TYPE_INFO;
			case BIGINT:
				return BasicTypeInfo.LONG_TYPE_INFO;
			case BOOLEAN:
				return BasicTypeInfo.BOOLEAN_TYPE_INFO;
			case FLOAT:
				return BasicTypeInfo.FLOAT_TYPE_INFO;
			case DOUBLE:
				return BasicTypeInfo.DOUBLE_TYPE_INFO;
			case STRING:
				return BasicTypeInfo.STRING_TYPE_INFO;
			case BINARY:
				return PrimitiveArrayTypeInfo.BYTE_PRIMITIVE_ARRAY_TYPE_INFO;
			case ARRAY:
				return new GenericTypeInfo(List.class);
			case MAP:
				return new GenericTypeInfo(Map.class);
			case STRUCT:
				return new GenericTypeInfo(List.class);
			default:
				throw new IllegalArgumentException("Unknown data type \"" + fieldSchema.getType() + "\" encountered.");
		}
	}
 

@Nullable
private Tuple2<Column, Comparable> extractColumnAndLiteral(BinaryComparison comp) {
	TypeInformation<?> typeInfo = getLiteralType(comp);
	String columnName = getColumnName(comp);

	// fetch literal and ensure it is comparable
	Object value = getLiteral(comp);
	// validate that literal is comparable
	if (!(value instanceof Comparable)) {
		LOG.warn("Encountered a non-comparable literal of type {}." +
			"Cannot push predicate [{}] into ParquetTablesource." +
			"This is a bug and should be reported.", value.getClass().getCanonicalName(), comp);
		return null;
	}

	if (typeInfo == BasicTypeInfo.BYTE_TYPE_INFO ||
		typeInfo == BasicTypeInfo.SHORT_TYPE_INFO ||
		typeInfo == BasicTypeInfo.INT_TYPE_INFO) {
		return new Tuple2<>(FilterApi.intColumn(columnName), (Integer) value);
	} else if (typeInfo == BasicTypeInfo.LONG_TYPE_INFO) {
		return new Tuple2<>(FilterApi.longColumn(columnName), (Long) value);
	} else if (typeInfo == BasicTypeInfo.FLOAT_TYPE_INFO) {
		return new Tuple2<>(FilterApi.floatColumn(columnName), (Float) value);
	} else if (typeInfo == BasicTypeInfo.BOOLEAN_TYPE_INFO) {
		return new Tuple2<>(FilterApi.booleanColumn(columnName), (Boolean) value);
	} else if (typeInfo == BasicTypeInfo.DOUBLE_TYPE_INFO) {
		return new Tuple2<>(FilterApi.doubleColumn(columnName), (Double) value);
	} else if (typeInfo == BasicTypeInfo.STRING_TYPE_INFO) {
		return new Tuple2<>(FilterApi.binaryColumn(columnName), Binary.fromString((String) value));
	} else {
		// unsupported type
		return null;
	}
}
 

@Test
public void testRowSerializerWithComplexTypes() {
	RowTypeInfo typeInfo = new RowTypeInfo(
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.DOUBLE_TYPE_INFO,
		BasicTypeInfo.STRING_TYPE_INFO,
		new TupleTypeInfo<Tuple3<Integer, Boolean, Short>>(
			BasicTypeInfo.INT_TYPE_INFO,
			BasicTypeInfo.BOOLEAN_TYPE_INFO,
			BasicTypeInfo.SHORT_TYPE_INFO),
		TypeExtractor.createTypeInfo(MyPojo.class));

	MyPojo testPojo1 = new MyPojo();
	testPojo1.name = null;
	MyPojo testPojo2 = new MyPojo();
	testPojo2.name = "Test1";
	MyPojo testPojo3 = new MyPojo();
	testPojo3.name = "Test2";

	Row[] data = new Row[]{
		createRow(null, null, null, null, null),
		createRow(0, null, null, null, null),
		createRow(0, 0.0, null, null, null),
		createRow(0, 0.0, "a", null, null),
		createRow(1, 0.0, "a", null, null),
		createRow(1, 1.0, "a", null, null),
		createRow(1, 1.0, "b", null, null),
		createRow(1, 1.0, "b", new Tuple3<>(1, false, (short) 2), null),
		createRow(1, 1.0, "b", new Tuple3<>(2, false, (short) 2), null),
		createRow(1, 1.0, "b", new Tuple3<>(2, true, (short) 2), null),
		createRow(1, 1.0, "b", new Tuple3<>(2, true, (short) 3), null),
		createRow(1, 1.0, "b", new Tuple3<>(2, true, (short) 3), testPojo1),
		createRow(1, 1.0, "b", new Tuple3<>(2, true, (short) 3), testPojo2),
		createRow(1, 1.0, "b", new Tuple3<>(2, true, (short) 3), testPojo3)
	};

	TypeSerializer<Row> serializer = typeInfo.createLegacySerializer(new ExecutionConfig());
	RowSerializerTestInstance testInstance = new RowSerializerTestInstance(serializer, data);
	testInstance.testAll();
}
 

private TypeInformation getFieldType(HCatFieldSchema fieldSchema) {

		switch(fieldSchema.getType()) {
			case INT:
				return BasicTypeInfo.INT_TYPE_INFO;
			case TINYINT:
				return BasicTypeInfo.BYTE_TYPE_INFO;
			case SMALLINT:
				return BasicTypeInfo.SHORT_TYPE_INFO;
			case BIGINT:
				return BasicTypeInfo.LONG_TYPE_INFO;
			case BOOLEAN:
				return BasicTypeInfo.BOOLEAN_TYPE_INFO;
			case FLOAT:
				return BasicTypeInfo.FLOAT_TYPE_INFO;
			case DOUBLE:
				return BasicTypeInfo.DOUBLE_TYPE_INFO;
			case STRING:
				return BasicTypeInfo.STRING_TYPE_INFO;
			case BINARY:
				return PrimitiveArrayTypeInfo.BYTE_PRIMITIVE_ARRAY_TYPE_INFO;
			case ARRAY:
				return new GenericTypeInfo(List.class);
			case MAP:
				return new GenericTypeInfo(Map.class);
			case STRUCT:
				return new GenericTypeInfo(List.class);
			default:
				throw new IllegalArgumentException("Unknown data type \"" + fieldSchema.getType() + "\" encountered.");
		}
	}
 

/**
 * Converts an ORC schema to a Flink TypeInformation.
 *
 * @param schema The ORC schema.
 * @return The TypeInformation that corresponds to the ORC schema.
 */
static TypeInformation schemaToTypeInfo(TypeDescription schema) {
	switch (schema.getCategory()) {
		case BOOLEAN:
			return BasicTypeInfo.BOOLEAN_TYPE_INFO;
		case BYTE:
			return BasicTypeInfo.BYTE_TYPE_INFO;
		case SHORT:
			return BasicTypeInfo.SHORT_TYPE_INFO;
		case INT:
			return BasicTypeInfo.INT_TYPE_INFO;
		case LONG:
			return BasicTypeInfo.LONG_TYPE_INFO;
		case FLOAT:
			return BasicTypeInfo.FLOAT_TYPE_INFO;
		case DOUBLE:
			return BasicTypeInfo.DOUBLE_TYPE_INFO;
		case DECIMAL:
			return BasicTypeInfo.BIG_DEC_TYPE_INFO;
		case STRING:
		case CHAR:
		case VARCHAR:
			return BasicTypeInfo.STRING_TYPE_INFO;
		case DATE:
			return SqlTimeTypeInfo.DATE;
		case TIMESTAMP:
			return SqlTimeTypeInfo.TIMESTAMP;
		case BINARY:
			return PrimitiveArrayTypeInfo.BYTE_PRIMITIVE_ARRAY_TYPE_INFO;
		case STRUCT:
			List<TypeDescription> fieldSchemas = schema.getChildren();
			TypeInformation[] fieldTypes = new TypeInformation[fieldSchemas.size()];
			for (int i = 0; i < fieldSchemas.size(); i++) {
				fieldTypes[i] = schemaToTypeInfo(fieldSchemas.get(i));
			}
			String[] fieldNames = schema.getFieldNames().toArray(new String[]{});
			return new RowTypeInfo(fieldTypes, fieldNames);
		case LIST:
			TypeDescription elementSchema = schema.getChildren().get(0);
			TypeInformation<?> elementType = schemaToTypeInfo(elementSchema);
			// arrays of primitive types are handled as object arrays to support null values
			return ObjectArrayTypeInfo.getInfoFor(elementType);
		case MAP:
			TypeDescription keySchema = schema.getChildren().get(0);
			TypeDescription valSchema = schema.getChildren().get(1);
			TypeInformation<?> keyType = schemaToTypeInfo(keySchema);
			TypeInformation<?> valType = schemaToTypeInfo(valSchema);
			return new MapTypeInfo<>(keyType, valType);
		case UNION:
			throw new UnsupportedOperationException("UNION type is not supported yet.");
		default:
			throw new IllegalArgumentException("Unknown type " + schema);
	}
}
 

@Test
public void testEmptyFields() throws Exception {
	String fileContent =
		",,,,,,,,\n" +
			",,,,,,,\n" +
			",,,,,,,,\n" +
			",,,,,,,\n" +
			",,,,,,,,\n" +
			",,,,,,,,\n" +
			",,,,,,,\n" +
			",,,,,,,,\n";

	FileInputSplit split = createTempFile(fileContent);

	TypeInformation[] fieldTypes = new TypeInformation[]{
		BasicTypeInfo.BOOLEAN_TYPE_INFO,
		BasicTypeInfo.BYTE_TYPE_INFO,
		BasicTypeInfo.DOUBLE_TYPE_INFO,
		BasicTypeInfo.FLOAT_TYPE_INFO,
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.LONG_TYPE_INFO,
		BasicTypeInfo.SHORT_TYPE_INFO,
		BasicTypeInfo.STRING_TYPE_INFO};

	RowCsvInputFormat format = new RowCsvInputFormat(PATH, fieldTypes, true);
	format.setFieldDelimiter(",");
	format.configure(new Configuration());
	format.open(split);

	Row result = new Row(8);
	int linesCnt = fileContent.split("\n").length;

	for (int i = 0; i < linesCnt; i++) {
		result = format.nextRecord(result);
		assertNull(result.getField(i));
	}

	// ensure no more rows
	assertNull(format.nextRecord(result));
	assertTrue(format.reachedEnd());
}
 

/**
 * Converts an ORC schema to a Flink TypeInformation.
 *
 * @param schema The ORC schema.
 * @return The TypeInformation that corresponds to the ORC schema.
 */
static TypeInformation schemaToTypeInfo(TypeDescription schema) {
	switch (schema.getCategory()) {
		case BOOLEAN:
			return BasicTypeInfo.BOOLEAN_TYPE_INFO;
		case BYTE:
			return BasicTypeInfo.BYTE_TYPE_INFO;
		case SHORT:
			return BasicTypeInfo.SHORT_TYPE_INFO;
		case INT:
			return BasicTypeInfo.INT_TYPE_INFO;
		case LONG:
			return BasicTypeInfo.LONG_TYPE_INFO;
		case FLOAT:
			return BasicTypeInfo.FLOAT_TYPE_INFO;
		case DOUBLE:
			return BasicTypeInfo.DOUBLE_TYPE_INFO;
		case DECIMAL:
			return BasicTypeInfo.BIG_DEC_TYPE_INFO;
		case STRING:
		case CHAR:
		case VARCHAR:
			return BasicTypeInfo.STRING_TYPE_INFO;
		case DATE:
			return SqlTimeTypeInfo.DATE;
		case TIMESTAMP:
			return SqlTimeTypeInfo.TIMESTAMP;
		case BINARY:
			return PrimitiveArrayTypeInfo.BYTE_PRIMITIVE_ARRAY_TYPE_INFO;
		case STRUCT:
			List<TypeDescription> fieldSchemas = schema.getChildren();
			TypeInformation[] fieldTypes = new TypeInformation[fieldSchemas.size()];
			for (int i = 0; i < fieldSchemas.size(); i++) {
				fieldTypes[i] = schemaToTypeInfo(fieldSchemas.get(i));
			}
			String[] fieldNames = schema.getFieldNames().toArray(new String[]{});
			return new RowTypeInfo(fieldTypes, fieldNames);
		case LIST:
			TypeDescription elementSchema = schema.getChildren().get(0);
			TypeInformation<?> elementType = schemaToTypeInfo(elementSchema);
			// arrays of primitive types are handled as object arrays to support null values
			return ObjectArrayTypeInfo.getInfoFor(elementType);
		case MAP:
			TypeDescription keySchema = schema.getChildren().get(0);
			TypeDescription valSchema = schema.getChildren().get(1);
			TypeInformation<?> keyType = schemaToTypeInfo(keySchema);
			TypeInformation<?> valType = schemaToTypeInfo(valSchema);
			return new MapTypeInfo<>(keyType, valType);
		case UNION:
			throw new UnsupportedOperationException("UNION type is not supported yet.");
		default:
			throw new IllegalArgumentException("Unknown type " + schema);
	}
}
 

@Test
public void testEmptyFields() throws Exception {
	String fileContent =
		",,,,,,,,\n" +
			",,,,,,,\n" +
			",,,,,,,,\n" +
			",,,,,,,\n" +
			",,,,,,,,\n" +
			",,,,,,,,\n" +
			",,,,,,,\n" +
			",,,,,,,,\n";

	FileInputSplit split = createTempFile(fileContent);

	TypeInformation[] fieldTypes = new TypeInformation[]{
		BasicTypeInfo.BOOLEAN_TYPE_INFO,
		BasicTypeInfo.BYTE_TYPE_INFO,
		BasicTypeInfo.DOUBLE_TYPE_INFO,
		BasicTypeInfo.FLOAT_TYPE_INFO,
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.LONG_TYPE_INFO,
		BasicTypeInfo.SHORT_TYPE_INFO,
		BasicTypeInfo.STRING_TYPE_INFO};

	RowCsvInputFormat format = new RowCsvInputFormat(PATH, fieldTypes, true);
	format.setFieldDelimiter(",");
	format.configure(new Configuration());
	format.open(split);

	Row result = new Row(8);
	int linesCnt = fileContent.split("\n").length;

	for (int i = 0; i < linesCnt; i++) {
		result = format.nextRecord(result);
		assertNull(result.getField(i));
	}

	// ensure no more rows
	assertNull(format.nextRecord(result));
	assertTrue(format.reachedEnd());
}
 

@Override
protected TypeInformation<Short> getValueTypeInfo() {
	return BasicTypeInfo.SHORT_TYPE_INFO;
}
 

@Test
public void testEmptyFields() throws Exception {
	String fileContent =
		",,,,,,,,\n" +
			",,,,,,,\n" +
			",,,,,,,,\n" +
			",,,,,,,\n" +
			",,,,,,,,\n" +
			",,,,,,,,\n" +
			",,,,,,,\n" +
			",,,,,,,,\n";

	FileInputSplit split = createTempFile(fileContent);

	TypeInformation[] fieldTypes = new TypeInformation[]{
		BasicTypeInfo.BOOLEAN_TYPE_INFO,
		BasicTypeInfo.BYTE_TYPE_INFO,
		BasicTypeInfo.DOUBLE_TYPE_INFO,
		BasicTypeInfo.FLOAT_TYPE_INFO,
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.LONG_TYPE_INFO,
		BasicTypeInfo.SHORT_TYPE_INFO,
		BasicTypeInfo.STRING_TYPE_INFO};

	RowCsvInputFormat format = new RowCsvInputFormat(PATH, fieldTypes, true);
	format.setFieldDelimiter(",");
	format.configure(new Configuration());
	format.open(split);

	Row result = new Row(8);
	int linesCnt = fileContent.split("\n").length;

	for (int i = 0; i < linesCnt; i++) {
		result = format.nextRecord(result);
		assertNull(result.getField(i));
	}

	// ensure no more rows
	assertNull(format.nextRecord(result));
	assertTrue(format.reachedEnd());
}
 

@Override
protected TypeInformation<Short> getValueTypeInfo() {
	return BasicTypeInfo.SHORT_TYPE_INFO;
}
 

@Override
protected TypeInformation<Short> getValueTypeInfo() {
	return BasicTypeInfo.SHORT_TYPE_INFO;
}
 

@Test
public void testEmptyFields() throws Exception {
	String fileContent =
			",,,,,,,,\n" +
					",,,,,,,\n" +
					",,,,,,,,\n" +
					",,,,,,,\n" +
					",,,,,,,,\n" +
					",,,,,,,,\n" +
					",,,,,,,\n" +
					",,,,,,,,\n";

	FileInputSplit split = createTempFile(fileContent);

	TypeInformation[] fieldTypes = new TypeInformation[]{
			BasicTypeInfo.BOOLEAN_TYPE_INFO,
			BasicTypeInfo.BYTE_TYPE_INFO,
			BasicTypeInfo.DOUBLE_TYPE_INFO,
			BasicTypeInfo.FLOAT_TYPE_INFO,
			BasicTypeInfo.INT_TYPE_INFO,
			BasicTypeInfo.LONG_TYPE_INFO,
			BasicTypeInfo.SHORT_TYPE_INFO,
			BasicTypeInfo.STRING_TYPE_INFO};

	RowCsvInputFormat.Builder builder = RowCsvInputFormat.builder(new RowTypeInfo(fieldTypes), PATH)
			.setFieldDelimiter(',')
			.setNullLiteral("");

	RowCsvInputFormat format = builder.build();
	format.configure(new Configuration());
	format.open(split);

	Row result = new Row(8);
	int linesCnt = fileContent.split("\n").length;

	for (int i = 0; i < linesCnt; i++) {
		result = format.nextRecord(result);
		assertNull(result.getField(i));
	}

	// ensure no more rows
	assertNull(format.nextRecord(result));
	assertTrue(format.reachedEnd());
}
 

@Nullable
private Tuple2<Column, Comparable> extractColumnAndLiteral(BinaryComparison comp) {
	String columnName = getColumnName(comp);
	ColumnPath columnPath = ColumnPath.fromDotString(columnName);
	TypeInformation<?> typeInfo = null;
	try {
		Type type = parquetSchema.getType(columnPath.toArray());
		typeInfo = ParquetSchemaConverter.convertParquetTypeToTypeInfo(type);
	} catch (InvalidRecordException e) {
		LOG.error("Pushed predicate on undefined field name {} in schema", columnName);
		return null;
	}

	// fetch literal and ensure it is comparable
	Object value = getLiteral(comp);
	// validate that literal is comparable
	if (!(value instanceof Comparable)) {
		LOG.warn("Encountered a non-comparable literal of type {}." +
			"Cannot push predicate [{}] into ParquetTablesource." +
			"This is a bug and should be reported.", value.getClass().getCanonicalName(), comp);
		return null;
	}

	if (typeInfo == BasicTypeInfo.BYTE_TYPE_INFO ||
		typeInfo == BasicTypeInfo.SHORT_TYPE_INFO ||
		typeInfo == BasicTypeInfo.INT_TYPE_INFO) {
		return new Tuple2<>(FilterApi.intColumn(columnName), ((Number) value).intValue());
	} else if (typeInfo == BasicTypeInfo.LONG_TYPE_INFO) {
		return new Tuple2<>(FilterApi.longColumn(columnName), ((Number) value).longValue());
	} else if (typeInfo == BasicTypeInfo.FLOAT_TYPE_INFO) {
		return new Tuple2<>(FilterApi.floatColumn(columnName), ((Number) value).floatValue());
	} else if (typeInfo == BasicTypeInfo.BOOLEAN_TYPE_INFO) {
		return new Tuple2<>(FilterApi.booleanColumn(columnName), (Boolean) value);
	} else if (typeInfo == BasicTypeInfo.DOUBLE_TYPE_INFO) {
		return new Tuple2<>(FilterApi.doubleColumn(columnName), ((Number) value).doubleValue());
	} else if (typeInfo == BasicTypeInfo.STRING_TYPE_INFO) {
		return new Tuple2<>(FilterApi.binaryColumn(columnName), Binary.fromString((String) value));
	} else {
		// unsupported type
		return null;
	}
}
 

/**
 * Converts an ORC schema to a Flink TypeInformation.
 *
 * @param schema The ORC schema.
 * @return The TypeInformation that corresponds to the ORC schema.
 */
static TypeInformation schemaToTypeInfo(TypeDescription schema) {
	switch (schema.getCategory()) {
		case BOOLEAN:
			return BasicTypeInfo.BOOLEAN_TYPE_INFO;
		case BYTE:
			return BasicTypeInfo.BYTE_TYPE_INFO;
		case SHORT:
			return BasicTypeInfo.SHORT_TYPE_INFO;
		case INT:
			return BasicTypeInfo.INT_TYPE_INFO;
		case LONG:
			return BasicTypeInfo.LONG_TYPE_INFO;
		case FLOAT:
			return BasicTypeInfo.FLOAT_TYPE_INFO;
		case DOUBLE:
			return BasicTypeInfo.DOUBLE_TYPE_INFO;
		case DECIMAL:
			return BasicTypeInfo.BIG_DEC_TYPE_INFO;
		case STRING:
		case CHAR:
		case VARCHAR:
			return BasicTypeInfo.STRING_TYPE_INFO;
		case DATE:
			return SqlTimeTypeInfo.DATE;
		case TIMESTAMP:
			return SqlTimeTypeInfo.TIMESTAMP;
		case BINARY:
			return PrimitiveArrayTypeInfo.BYTE_PRIMITIVE_ARRAY_TYPE_INFO;
		case STRUCT:
			List<TypeDescription> fieldSchemas = schema.getChildren();
			TypeInformation[] fieldTypes = new TypeInformation[fieldSchemas.size()];
			for (int i = 0; i < fieldSchemas.size(); i++) {
				fieldTypes[i] = schemaToTypeInfo(fieldSchemas.get(i));
			}
			String[] fieldNames = schema.getFieldNames().toArray(new String[]{});
			return new RowTypeInfo(fieldTypes, fieldNames);
		case LIST:
			TypeDescription elementSchema = schema.getChildren().get(0);
			TypeInformation<?> elementType = schemaToTypeInfo(elementSchema);
			// arrays of primitive types are handled as object arrays to support null values
			return ObjectArrayTypeInfo.getInfoFor(elementType);
		case MAP:
			TypeDescription keySchema = schema.getChildren().get(0);
			TypeDescription valSchema = schema.getChildren().get(1);
			TypeInformation<?> keyType = schemaToTypeInfo(keySchema);
			TypeInformation<?> valType = schemaToTypeInfo(valSchema);
			return new MapTypeInfo<>(keyType, valType);
		case UNION:
			throw new UnsupportedOperationException("UNION type is not supported yet.");
		default:
			throw new IllegalArgumentException("Unknown type " + schema);
	}
}