Java Code Examples for org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category#STRUCT
The following examples show how to use
org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category#STRUCT .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: HiveCoercionPolicy.java From presto with Apache License 2.0 | 6 votes |
|
private boolean canCoerceForStruct(HiveType fromHiveType, HiveType toHiveType)
{
if (fromHiveType.getCategory() != Category.STRUCT || toHiveType.getCategory() != Category.STRUCT) {
return false;
}
List<String> fromFieldNames = ((StructTypeInfo) fromHiveType.getTypeInfo()).getAllStructFieldNames();
List<String> toFieldNames = ((StructTypeInfo) toHiveType.getTypeInfo()).getAllStructFieldNames();
List<HiveType> fromFieldTypes = extractStructFieldTypes(fromHiveType);
List<HiveType> toFieldTypes = extractStructFieldTypes(toHiveType);
// Rule:
// * Fields may be added or dropped from the end.
// * For all other field indices, the corresponding fields must have
// the same name, and the type must be coercible.
for (int i = 0; i < min(fromFieldTypes.size(), toFieldTypes.size()); i++) {
if (!fromFieldNames.get(i).equals(toFieldNames.get(i))) {
return false;
}
if (!fromFieldTypes.get(i).equals(toFieldTypes.get(i)) && !canCoerce(fromFieldTypes.get(i), toFieldTypes.get(i))) {
return false;
}
}
return true;
}
Example 2
| Source File: ST_GeomFromGeoJson.java From spatial-framework-for-hadoop with Apache License 2.0 | 6 votes |
|
@Override
public Object evaluate(DeferredObject[] arguments) throws HiveException {
DeferredObject jsonDeferredObject = arguments[0];
String json = null;
if (jsonOI.getCategory() == Category.STRUCT){
//StructObjectInspector structOI = (StructObjectInspector)jsonOI;
// TODO support structs
} else {
PrimitiveObjectInspector primOI = (PrimitiveObjectInspector)jsonOI;
json = (String)primOI.getPrimitiveJavaObject(jsonDeferredObject.get());
}
try {
OGCGeometry ogcGeom = OGCGeometry.fromGeoJson(json);
return GeometryUtils.geometryToEsriShapeBytesWritable(ogcGeom);
} catch (Exception e) {
LogUtils.Log_InvalidText(LOG, json);
}
return null;
}
Example 3
| Source File: ST_GeomFromJson.java From spatial-framework-for-hadoop with Apache License 2.0 | 6 votes |
|
@Override
public Object evaluate(DeferredObject[] arguments) throws HiveException {
DeferredObject jsonDeferredObject = arguments[0];
String json = null;
if (jsonOI.getCategory() == Category.STRUCT){
//StructObjectInspector structOI = (StructObjectInspector)jsonOI;
// TODO support structs
} else {
PrimitiveObjectInspector primOI = (PrimitiveObjectInspector)jsonOI;
json = (String)primOI.getPrimitiveJavaObject(jsonDeferredObject.get());
}
try {
OGCGeometry ogcGeom = OGCGeometry.fromJson(json);
return GeometryUtils.geometryToEsriShapeBytesWritable(ogcGeom);
} catch (Exception e) {
}
return null;
}
Example 4
| Source File: HiveUtilities.java From dremio-oss with Apache License 2.0 | 5 votes |
|
public static StructObjectInspector getStructOI(final SerDe serDe) throws Exception {
ObjectInspector oi = serDe.getObjectInspector();
if (oi.getCategory() != Category.STRUCT) {
throw new UnsupportedOperationException(String.format("%s category not supported", oi.getCategory()));
}
return (StructObjectInspector) oi;
}
Example 5
| Source File: HiveORCVectorizedReader.java From dremio-oss with Apache License 2.0 | 5 votes |
|
private boolean isSupportedType(Category category) {
return (category == Category.PRIMITIVE ||
category == Category.LIST ||
category == Category.STRUCT ||
category == Category.MAP ||
category == Category.UNION);
}
Example 6
| Source File: HiveUtilities.java From dremio-oss with Apache License 2.0 | 5 votes |
|
public static StructObjectInspector getStructOI(final AbstractSerDe serDe) throws Exception {
ObjectInspector oi = serDe.getObjectInspector();
if (oi.getCategory() != Category.STRUCT) {
throw new UnsupportedOperationException(String.format("%s category not supported", oi.getCategory()));
}
return (StructObjectInspector) oi;
}
Example 7
| Source File: HiveORCVectorizedReader.java From dremio-oss with Apache License 2.0 | 5 votes |
|
private boolean isSupportedType(Category category) {
return (category == Category.PRIMITIVE ||
category == Category.LIST ||
category == Category.STRUCT ||
category == Category.MAP ||
category == Category.UNION);
}
Example 8
| Source File: HiveUtils.java From incubator-hivemall with Apache License 2.0 | 5 votes |
|
@Nonnull
public static StructObjectInspector asStructOI(@Nonnull final ObjectInspector oi)
throws UDFArgumentException {
if (oi.getCategory() != Category.STRUCT) {
throw new UDFArgumentException("Expected Struct OI but got: " + oi.getTypeName());
}
return (StructObjectInspector) oi;
}
Example 9
| Source File: ST_GeomFromGeoJson.java From spatial-framework-for-hadoop with Apache License 2.0 | 5 votes |
|
@Override
public ObjectInspector initialize(ObjectInspector[] arguments)
throws UDFArgumentException {
if (arguments.length != 1) {
throw new UDFArgumentLengthException("ST_GeomFromJson takes only one argument");
}
ObjectInspector argJsonOI = arguments[0];
if (argJsonOI.getCategory() == Category.PRIMITIVE)
{
PrimitiveObjectInspector poi = (PrimitiveObjectInspector)argJsonOI;
if (poi.getPrimitiveCategory() != PrimitiveCategory.STRING)
{
throw new UDFArgumentTypeException(0, "ST_GeomFromJson argument category must be either a string primitive or struct");
}
} else if (argJsonOI.getCategory() != Category.STRUCT) {
} else {
throw new UDFArgumentTypeException(0, "ST_GeomFromJson argument category must be either a string primitive or struct");
}
jsonOI = argJsonOI;
return GeometryUtils.geometryTransportObjectInspector;
}
Example 10
| Source File: ST_GeomFromJson.java From spatial-framework-for-hadoop with Apache License 2.0 | 5 votes |
|
@Override
public ObjectInspector initialize(ObjectInspector[] arguments)
throws UDFArgumentException {
if (arguments.length != 1) {
throw new UDFArgumentLengthException("ST_GeomFromJson takes only one argument");
}
ObjectInspector argJsonOI = arguments[0];
if (argJsonOI.getCategory() == Category.PRIMITIVE)
{
PrimitiveObjectInspector poi = (PrimitiveObjectInspector)argJsonOI;
if (poi.getPrimitiveCategory() != PrimitiveCategory.STRING)
{
throw new UDFArgumentTypeException(0, "ST_GeomFromJson argument category must be either a string primitive or struct");
}
} else if (argJsonOI.getCategory() != Category.STRUCT) {
} else {
throw new UDFArgumentTypeException(0, "ST_GeomFromJson argument category must be either a string primitive or struct");
}
jsonOI = argJsonOI;
return GeometryUtils.geometryTransportObjectInspector;
}
Example 11
| Source File: SingleLevelArrayMapKeyValuesSchemaConverter.java From presto with Apache License 2.0 | 4 votes |
|
private static Type convertType(String name, TypeInfo typeInfo, Repetition repetition)
{
if (typeInfo.getCategory() == Category.PRIMITIVE) {
if (typeInfo.equals(TypeInfoFactory.stringTypeInfo)) {
return Types.primitive(PrimitiveTypeName.BINARY, repetition).as(OriginalType.UTF8)
.named(name);
}
if (typeInfo.equals(TypeInfoFactory.intTypeInfo) ||
typeInfo.equals(TypeInfoFactory.shortTypeInfo) ||
typeInfo.equals(TypeInfoFactory.byteTypeInfo)) {
return Types.primitive(PrimitiveTypeName.INT32, repetition).named(name);
}
if (typeInfo.equals(TypeInfoFactory.longTypeInfo)) {
return Types.primitive(PrimitiveTypeName.INT64, repetition).named(name);
}
if (typeInfo.equals(TypeInfoFactory.doubleTypeInfo)) {
return Types.primitive(PrimitiveTypeName.DOUBLE, repetition).named(name);
}
if (typeInfo.equals(TypeInfoFactory.floatTypeInfo)) {
return Types.primitive(PrimitiveTypeName.FLOAT, repetition).named(name);
}
if (typeInfo.equals(TypeInfoFactory.booleanTypeInfo)) {
return Types.primitive(PrimitiveTypeName.BOOLEAN, repetition).named(name);
}
if (typeInfo.equals(TypeInfoFactory.binaryTypeInfo)) {
return Types.primitive(PrimitiveTypeName.BINARY, repetition).named(name);
}
if (typeInfo.equals(TypeInfoFactory.timestampTypeInfo)) {
return Types.primitive(PrimitiveTypeName.INT96, repetition).named(name);
}
if (typeInfo.equals(TypeInfoFactory.voidTypeInfo)) {
throw new UnsupportedOperationException("Void type not implemented");
}
if (typeInfo.getTypeName().toLowerCase(Locale.ENGLISH).startsWith(
serdeConstants.CHAR_TYPE_NAME)) {
if (repetition == Repetition.OPTIONAL) {
return Types.optional(PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named(name);
}
return Types.repeated(PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named(name);
}
if (typeInfo.getTypeName().toLowerCase(Locale.ENGLISH).startsWith(
serdeConstants.VARCHAR_TYPE_NAME)) {
if (repetition == Repetition.OPTIONAL) {
return Types.optional(PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named(name);
}
return Types.repeated(PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named(name);
}
if (typeInfo instanceof DecimalTypeInfo) {
DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) typeInfo;
int prec = decimalTypeInfo.precision();
int scale = decimalTypeInfo.scale();
int bytes = ParquetHiveSerDe.PRECISION_TO_BYTE_COUNT[prec - 1];
if (repetition == Repetition.OPTIONAL) {
return Types.optional(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY).length(bytes).as(OriginalType.DECIMAL).scale(scale).precision(prec).named(name);
}
return Types.repeated(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY).length(bytes).as(OriginalType.DECIMAL).scale(scale).precision(prec).named(name);
}
if (typeInfo.equals(TypeInfoFactory.dateTypeInfo)) {
return Types.primitive(PrimitiveTypeName.INT32, repetition).as(OriginalType.DATE).named(name);
}
if (typeInfo.equals(TypeInfoFactory.unknownTypeInfo)) {
throw new UnsupportedOperationException("Unknown type not implemented");
}
throw new IllegalArgumentException("Unknown type: " + typeInfo);
}
if (typeInfo.getCategory() == Category.LIST) {
return convertArrayType(name, (ListTypeInfo) typeInfo, repetition);
}
if (typeInfo.getCategory() == Category.STRUCT) {
return convertStructType(name, (StructTypeInfo) typeInfo, repetition);
}
if (typeInfo.getCategory() == Category.MAP) {
return convertMapType(name, (MapTypeInfo) typeInfo, repetition);
}
if (typeInfo.getCategory() == Category.UNION) {
throw new UnsupportedOperationException("Union type not implemented");
}
throw new IllegalArgumentException("Unknown type: " + typeInfo);
}
Example 12
| Source File: SingleLevelArraySchemaConverter.java From presto with Apache License 2.0 | 4 votes |
|
private static Type convertType(String name, TypeInfo typeInfo,
Repetition repetition)
{
if (typeInfo.getCategory() == Category.PRIMITIVE) {
if (typeInfo.equals(TypeInfoFactory.stringTypeInfo)) {
return Types.primitive(PrimitiveTypeName.BINARY, repetition).as(OriginalType.UTF8)
.named(name);
}
else if (typeInfo.equals(TypeInfoFactory.intTypeInfo) ||
typeInfo.equals(TypeInfoFactory.shortTypeInfo) ||
typeInfo.equals(TypeInfoFactory.byteTypeInfo)) {
return Types.primitive(PrimitiveTypeName.INT32, repetition).named(name);
}
else if (typeInfo.equals(TypeInfoFactory.longTypeInfo)) {
return Types.primitive(PrimitiveTypeName.INT64, repetition).named(name);
}
else if (typeInfo.equals(TypeInfoFactory.doubleTypeInfo)) {
return Types.primitive(PrimitiveTypeName.DOUBLE, repetition).named(name);
}
else if (typeInfo.equals(TypeInfoFactory.floatTypeInfo)) {
return Types.primitive(PrimitiveTypeName.FLOAT, repetition).named(name);
}
else if (typeInfo.equals(TypeInfoFactory.booleanTypeInfo)) {
return Types.primitive(PrimitiveTypeName.BOOLEAN, repetition).named(name);
}
else if (typeInfo.equals(TypeInfoFactory.binaryTypeInfo)) {
return Types.primitive(PrimitiveTypeName.BINARY, repetition).named(name);
}
else if (typeInfo.equals(TypeInfoFactory.timestampTypeInfo)) {
return Types.primitive(PrimitiveTypeName.INT96, repetition).named(name);
}
else if (typeInfo.equals(TypeInfoFactory.voidTypeInfo)) {
throw new UnsupportedOperationException("Void type not implemented");
}
else if (typeInfo.getTypeName().toLowerCase(Locale.ENGLISH).startsWith(
serdeConstants.CHAR_TYPE_NAME)) {
if (repetition == Repetition.OPTIONAL) {
return Types.optional(PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named(name);
}
else {
return Types.repeated(PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named(name);
}
}
else if (typeInfo.getTypeName().toLowerCase(Locale.ENGLISH).startsWith(
serdeConstants.VARCHAR_TYPE_NAME)) {
if (repetition == Repetition.OPTIONAL) {
return Types.optional(PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named(name);
}
else {
return Types.repeated(PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named(name);
}
}
else if (typeInfo instanceof DecimalTypeInfo) {
DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) typeInfo;
int prec = decimalTypeInfo.precision();
int scale = decimalTypeInfo.scale();
int bytes = ParquetHiveSerDe.PRECISION_TO_BYTE_COUNT[prec - 1];
if (repetition == Repetition.OPTIONAL) {
return Types.optional(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY).length(bytes).as(OriginalType.DECIMAL).scale(scale).precision(prec).named(name);
}
else {
return Types.repeated(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY).length(bytes).as(OriginalType.DECIMAL).scale(scale).precision(prec).named(name);
}
}
else if (typeInfo.equals(TypeInfoFactory.dateTypeInfo)) {
return Types.primitive(PrimitiveTypeName.INT32, repetition).as(OriginalType.DATE).named(name);
}
else if (typeInfo.equals(TypeInfoFactory.unknownTypeInfo)) {
throw new UnsupportedOperationException("Unknown type not implemented");
}
else {
throw new IllegalArgumentException("Unknown type: " + typeInfo);
}
}
else if (typeInfo.getCategory() == Category.LIST) {
return convertArrayType(name, (ListTypeInfo) typeInfo, repetition);
}
else if (typeInfo.getCategory() == Category.STRUCT) {
return convertStructType(name, (StructTypeInfo) typeInfo, repetition);
}
else if (typeInfo.getCategory() == Category.MAP) {
return convertMapType(name, (MapTypeInfo) typeInfo, repetition);
}
else if (typeInfo.getCategory() == Category.UNION) {
throw new UnsupportedOperationException("Union type not implemented");
}
else {
throw new IllegalArgumentException("Unknown type: " + typeInfo);
}
}
Example 13
| Source File: MapKeyValuesSchemaConverter.java From presto with Apache License 2.0 | 4 votes |
|
private static Type convertType(String name, TypeInfo typeInfo, Repetition repetition)
{
if (typeInfo.getCategory() == Category.PRIMITIVE) {
if (typeInfo.equals(TypeInfoFactory.stringTypeInfo)) {
return Types.primitive(PrimitiveTypeName.BINARY, repetition).as(OriginalType.UTF8)
.named(name);
}
else if (typeInfo.equals(TypeInfoFactory.intTypeInfo) ||
typeInfo.equals(TypeInfoFactory.shortTypeInfo) ||
typeInfo.equals(TypeInfoFactory.byteTypeInfo)) {
return Types.primitive(PrimitiveTypeName.INT32, repetition).named(name);
}
else if (typeInfo.equals(TypeInfoFactory.longTypeInfo)) {
return Types.primitive(PrimitiveTypeName.INT64, repetition).named(name);
}
else if (typeInfo.equals(TypeInfoFactory.doubleTypeInfo)) {
return Types.primitive(PrimitiveTypeName.DOUBLE, repetition).named(name);
}
else if (typeInfo.equals(TypeInfoFactory.floatTypeInfo)) {
return Types.primitive(PrimitiveTypeName.FLOAT, repetition).named(name);
}
else if (typeInfo.equals(TypeInfoFactory.booleanTypeInfo)) {
return Types.primitive(PrimitiveTypeName.BOOLEAN, repetition).named(name);
}
else if (typeInfo.equals(TypeInfoFactory.binaryTypeInfo)) {
return Types.primitive(PrimitiveTypeName.BINARY, repetition).named(name);
}
else if (typeInfo.equals(TypeInfoFactory.timestampTypeInfo)) {
return Types.primitive(PrimitiveTypeName.INT96, repetition).named(name);
}
else if (typeInfo.equals(TypeInfoFactory.voidTypeInfo)) {
throw new UnsupportedOperationException("Void type not implemented");
}
else if (typeInfo.getTypeName().toLowerCase(Locale.ENGLISH).startsWith(
serdeConstants.CHAR_TYPE_NAME)) {
return Types.optional(PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named(name);
}
else if (typeInfo.getTypeName().toLowerCase(Locale.ENGLISH).startsWith(
serdeConstants.VARCHAR_TYPE_NAME)) {
return Types.optional(PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named(name);
}
else if (typeInfo instanceof DecimalTypeInfo) {
DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) typeInfo;
int prec = decimalTypeInfo.precision();
int scale = decimalTypeInfo.scale();
int bytes = ParquetHiveSerDe.PRECISION_TO_BYTE_COUNT[prec - 1];
return Types.optional(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY).length(bytes).as(OriginalType.DECIMAL).scale(scale).precision(prec).named(name);
}
else if (typeInfo.equals(TypeInfoFactory.dateTypeInfo)) {
return Types.primitive(PrimitiveTypeName.INT32, repetition).as(OriginalType.DATE).named(name);
}
else if (typeInfo.equals(TypeInfoFactory.unknownTypeInfo)) {
throw new UnsupportedOperationException("Unknown type not implemented");
}
else {
throw new IllegalArgumentException("Unknown type: " + typeInfo);
}
}
else if (typeInfo.getCategory() == Category.LIST) {
return convertArrayType(name, (ListTypeInfo) typeInfo);
}
else if (typeInfo.getCategory() == Category.STRUCT) {
return convertStructType(name, (StructTypeInfo) typeInfo);
}
else if (typeInfo.getCategory() == Category.MAP) {
return convertMapType(name, (MapTypeInfo) typeInfo);
}
else if (typeInfo.getCategory() == Category.UNION) {
throw new UnsupportedOperationException("Union type not implemented");
}
else {
throw new IllegalArgumentException("Unknown type: " + typeInfo);
}
}
Example 14
| Source File: HiveORCVectorizedReader.java From dremio-oss with Apache License 2.0 | 4 votes |
|
private static boolean searchAllFields(final ObjectInspector rootOI,
final String name,
final int[] childCounts,
SearchResult position
) {
Category category = rootOI.getCategory();
if (category == Category.STRUCT) {
position.index++; // first child is immediately next to parent
StructObjectInspector sOi = (StructObjectInspector) rootOI;
for (StructField sf : sOi.getAllStructFieldRefs()) {
// We depend on the fact that caller takes care of calling current method
// once for each segment in the selected column path. So, we should always get
// searched field as immediate child
if (position.index >= childCounts.length) {
// input schema has more columns than what reader can read
return false;
}
if (sf.getFieldName().equalsIgnoreCase(name)) {
position.oI = sf.getFieldObjectInspector();
return true;
} else {
position.index += childCounts[position.index];
}
}
} else if (category == Category.MAP) {
position.index++; // first child is immediately next to parent
if (position.index >= childCounts.length) {
// input schema has more columns than what reader can read
return false;
}
if (name.equalsIgnoreCase(HiveUtilities.MAP_KEY_FIELD_NAME)) {
ObjectInspector kOi = ((MapObjectInspector) rootOI).getMapKeyObjectInspector();
position.oI = kOi;
return true;
}
position.index += childCounts[position.index];
if (position.index >= childCounts.length) {
// input schema has more columns than what reader can read
return false;
}
if (name.equalsIgnoreCase(HiveUtilities.MAP_VALUE_FIELD_NAME)) {
ObjectInspector vOi = ((MapObjectInspector) rootOI).getMapValueObjectInspector();
position.oI = vOi;
return true;
}
}
return false;
}
Example 15
| Source File: HiveORCVectorizedReader.java From dremio-oss with Apache License 2.0 | 4 votes |
|
private static boolean searchAllFields(final ObjectInspector rootOI,
final String name,
final int[] childCounts,
SearchResult position
) {
Category category = rootOI.getCategory();
if (category == Category.STRUCT) {
position.index++; // first child is immediately next to parent
StructObjectInspector sOi = (StructObjectInspector) rootOI;
for (StructField sf : sOi.getAllStructFieldRefs()) {
// We depend on the fact that caller takes care of calling current method
// once for each segment in the selected column path. So, we should always get
// searched field as immediate child
if (sf.getFieldName().equalsIgnoreCase(name)) {
position.oI = sf.getFieldObjectInspector();
return true;
} else {
if (position.index >= childCounts.length) {
return false;
}
position.index += childCounts[position.index];
}
}
} else if (category == Category.MAP) {
position.index++; // first child is immediately next to parent
if (name.equalsIgnoreCase(HiveUtilities.MAP_KEY_FIELD_NAME)) {
ObjectInspector kOi = ((MapObjectInspector) rootOI).getMapKeyObjectInspector();
position.oI = kOi;
return true;
}
if (position.index >= childCounts.length) {
return false;
}
position.index += childCounts[position.index];
if (name.equalsIgnoreCase(HiveUtilities.MAP_VALUE_FIELD_NAME)) {
ObjectInspector vOi = ((MapObjectInspector) rootOI).getMapValueObjectInspector();
position.oI = vOi;
return true;
}
}
return false;
}
Example 16
| Source File: HiveUtils.java From incubator-hivemall with Apache License 2.0 | 4 votes |
|
public static boolean isStructOI(@Nonnull final ObjectInspector oi) {
return oi.getCategory() == Category.STRUCT;
}
