Java Code Examples for org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector

The following are top voted examples for showing how to use org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector. These examples are extracted from open source projects. You can vote up the examples you like and your votes will be used in our system to generate more good examples.
Example 1
Project: hive-jq-udtf   File: JsonQueryUDTF.java   Source Code and License 8 votes vote down vote up
private StructObjectInspector initialize(final ObjectInspector jsonArg, final ObjectInspector jqArg, final List<ObjectInspector> nameAndTypeArgs) throws UDFArgumentException {
	this.in = Arguments.asString(jsonArg, "JSON");

	try {
		this.jq = JsonQuery.compile(Arguments.asConstantNonNullString(jqArg, "JQ"));
	} catch (final JsonQueryException e) {
		throw new UDFArgumentException("JQ is invalid: " + e.getMessage());
	}

	this.marshaller = ResultObjectMarshallers.create(Arguments.asConstantNonNullStrings(nameAndTypeArgs, "TYPE or NAME:TYPE"));

	this.scope = new Scope();
	this.mapper = new ObjectMapper(new JsonFactory().enable(Feature.ALLOW_UNQUOTED_CONTROL_CHARS));

	return marshaller.objectInspector();
}
 
Example 2
Project: dataplatform-schema-lib   File: OrcSerdeFactory.java   Source Code and License 7 votes vote down vote up
public static OrcSerde getFromTypeInfo( final Configuration config , final TypeInfo typeInfo )throws IOException{
  ObjectInspector objectInspector = TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo( typeInfo );
  if( !( typeInfo instanceof StructTypeInfo ) ){
    throw new IOException( "Input type info is not StructTypeInfo : " + typeInfo.toString() );
  }
  String columnsName = "";
  String columnsType = "";
  List<TypeInfo> typeInfoList = ( (StructTypeInfo)typeInfo ).getAllStructFieldTypeInfos();
  List<StructField> structField = (List<StructField>)( ( (StructObjectInspector)objectInspector ).getAllStructFieldRefs() );
  for( int i = 0 ; i < structField.size() ; i++ ){
    if( ! columnsName.isEmpty() ){
      columnsName = columnsName.concat( "," );
      columnsType = columnsType.concat( "," );
    }
    columnsName = columnsName.concat( structField.get(i).getFieldName() );
    columnsType = columnsType.concat( typeInfoList.get(i).toString() );
  }

  OrcSerde serde = new OrcSerde();
  Properties table = new Properties();
  table.setProperty( serdeConstants.LIST_COLUMNS , columnsName );
  table.setProperty( serdeConstants.LIST_COLUMN_TYPES , columnsType );
  serde.initialize( config , table );

  return serde;
}
 
Example 3
Project: dremio-oss   File: HiveTestUDFImpls.java   Source Code and License 6 votes vote down vote up
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
  if (arguments.length != 1) {
    throw new UDFArgumentLengthException(String.format("%s needs 1 argument, got %d", udfName, arguments.length));
  }

  if (arguments[0].getCategory() != Category.PRIMITIVE ||
      ((PrimitiveObjectInspector) arguments[0]).getPrimitiveCategory() != inputType) {
    String actual = arguments[0].getCategory() + (arguments[0].getCategory() == Category.PRIMITIVE ?
        "[" + ((PrimitiveObjectInspector) arguments[0]).getPrimitiveCategory() + "]" : "");
    throw new UDFArgumentException(
        String.format("%s only takes primitive type %s, got %s", udfName, inputType, actual));
  }
  argumentOI = arguments[0];
  return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(outputType);
}
 
Example 4
Project: multiple-dimension-spread   File: MDSMapObjectInspector.java   Source Code and License 6 votes vote down vote up
public MDSMapObjectInspector( final MapTypeInfo typeInfo ){
  TypeInfo keyTypeInfo = typeInfo.getMapKeyTypeInfo();
  if( keyTypeInfo.getCategory() == ObjectInspector.Category.PRIMITIVE && ( (PrimitiveTypeInfo)keyTypeInfo ).getPrimitiveCategory() == PrimitiveCategory.STRING ){
    keyObjectInspector = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
  }
  else{
    throw new RuntimeException( "Map key type is string only." );
  }

  valueObjectInspector = MDSObjectInspectorFactory.craeteObjectInspectorFromTypeInfo( typeInfo.getMapValueTypeInfo() ); 

  if( valueObjectInspector.getCategory() == ObjectInspector.Category.PRIMITIVE ){
    getField = new PrimitiveGetField( (PrimitiveObjectInspector)valueObjectInspector );
  }
  else if( valueObjectInspector.getCategory() == ObjectInspector.Category.UNION ){
    getField = new UnionGetField( (UnionTypeInfo)( typeInfo.getMapValueTypeInfo() ) );
  }
  else{
    getField = new NestedGetField();
  }
}
 
Example 5
Project: hive-jq-udtf   File: JsonQueryUDTFTest.java   Source Code and License 6 votes vote down vote up
@Test
public void testMultiColumn() throws HiveException {
	final JsonQueryUDTF sut = new JsonQueryUDTF();
	final StructObjectInspector oi = sut.initialize(new ObjectInspector[] {
			PrimitiveObjectInspectorFactory.writableStringObjectInspector,
			toConstantOI(".region as $region | .timezones[] | {name: ($region + \"/\" + .name), offset}"),
			toConstantOI("name:string"),
			toConstantOI("offset:int"),
	});
	assertEquals("struct<name:string,offset:int>", oi.getTypeName());

	final List<Object> results = evaluate(sut, toObject(TEST_JSON));
	assertEquals(3, results.size());

	final HivePath namePath = new HivePath(oi, ".name");
	final HivePath offsetPath = new HivePath(oi, ".offset");

	assertEquals("Asia/Tokyo", namePath.extract(results.get(0)).asString());
	assertEquals(540, offsetPath.extract(results.get(0)).asInt());

	assertEquals("Asia/Taipei", namePath.extract(results.get(1)).asString());
	assertEquals(480, offsetPath.extract(results.get(1)).asInt());

	assertEquals("Asia/Kamchatka", namePath.extract(results.get(2)).asString());
	assertEquals(720, offsetPath.extract(results.get(2)).asInt());
}
 
Example 6
Project: monarch   File: MonarchPredicateHandler.java   Source Code and License 6 votes vote down vote up
public static boolean isMonarchTypeSupported(final ObjectInspector oi) {
  if (ObjectInspector.Category.PRIMITIVE.equals(oi.getCategory())) {
    /** handle primitive type definitions like decimal(20,20) or varchar(100) **/
    String typeStr = oi.getTypeName();
    final int argPos = typeStr.indexOf('(');
    if (argPos > 0) {
      typeStr = typeStr.substring(0, argPos);
    }
    return TYPE_HIVE_TO_MONARCH_MAP.containsKey(typeStr);
  } else if (oi instanceof ListObjectInspector) {
    ListObjectInspector loi = (ListObjectInspector)oi;
    return isMonarchTypeSupported(loi.getListElementObjectInspector());
  } else if (oi instanceof MapObjectInspector) {
    MapObjectInspector moi = (MapObjectInspector)oi;
    return isMonarchTypeSupported(moi.getMapKeyObjectInspector()) &&
      isMonarchTypeSupported(moi.getMapValueObjectInspector());
  } else if (oi instanceof StructObjectInspector) {
    return ((StructObjectInspector) oi).getAllStructFieldRefs().stream()
      .map(StructField::getFieldObjectInspector)
      .allMatch(MonarchPredicateHandler::isMonarchTypeSupported);
  } else if (oi instanceof UnionObjectInspector) {
    return ((UnionObjectInspector) oi).getObjectInspectors().stream()
      .allMatch(MonarchPredicateHandler::isMonarchTypeSupported);
  }
  return false;
}
 
Example 7
Project: dremio-oss   File: HiveFunctionRegistry.java   Source Code and License 6 votes vote down vote up
private HiveFuncHolder matchAndCreateUDFHolder(String udfName,
                                               Class<? extends UDF> udfClazz,
                                               CompleteType[] argTypes,
                                               ObjectInspector[] argOIs) {
  try {
    GenericUDF udfInstance = new GenericUDFBridge(udfName, false/* is operator */, udfClazz.getName());
    ObjectInspector returnOI = udfInstance.initialize(argOIs);

    return new HiveFuncHolder(
      udfName,
      udfClazz,
      argTypes,
      returnOI,
      CompleteType.fromMinorType(ObjectInspectorHelper.getMinorType(returnOI)),
      nonDeterministicUDFs.contains(udfClazz));
  } catch (Exception e) { /*ignore this*/ }

  return null;
}
 
Example 8
Project: QDrill   File: ObjectInspectorHelper.java   Source Code and License 6 votes vote down vote up
public static MinorType getDrillType(ObjectInspector oi) {
  switch(oi.getCategory()) {
    case PRIMITIVE: {
      PrimitiveObjectInspector poi = (PrimitiveObjectInspector)oi;
      if (TYPE_HIVE2DRILL.containsKey(poi.getPrimitiveCategory())) {
        return TYPE_HIVE2DRILL.get(poi.getPrimitiveCategory());
      }
      throw new UnsupportedOperationException();
    }

    case MAP:
    case LIST:
    case STRUCT:
    default:
      throw new UnsupportedOperationException();
  }
}
 
Example 9
Project: QDrill   File: HiveTestUDFImpls.java   Source Code and License 6 votes vote down vote up
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
  if (arguments.length != 1) {
    throw new UDFArgumentLengthException(String.format("%s needs 1 argument, got %d", udfName, arguments.length));
  }

  if (arguments[0].getCategory() != Category.PRIMITIVE ||
      ((PrimitiveObjectInspector) arguments[0]).getPrimitiveCategory() != inputType) {
    String actual = arguments[0].getCategory() + (arguments[0].getCategory() == Category.PRIMITIVE ?
        "[" + ((PrimitiveObjectInspector) arguments[0]).getPrimitiveCategory() + "]" : "");
    throw new UDFArgumentException(
        String.format("%s only takes primitive type %s, got %s", udfName, inputType, actual));
  }
  argumentOI = arguments[0];
  return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(outputType);
}
 
Example 10
Project: hive-udf-backports   File: GenericUDFNextDay.java   Source Code and License 6 votes vote down vote up
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
  checkArgsSize(arguments, 2, 2);

  checkArgPrimitive(arguments, 0);
  checkArgPrimitive(arguments, 1);

  checkArgGroups(arguments, 0, inputTypes, STRING_GROUP, DATE_GROUP, VOID_GROUP);
  checkArgGroups(arguments, 1, inputTypes, STRING_GROUP, VOID_GROUP);

  obtainDateConverter(arguments, 0, inputTypes, converters);
  obtainStringConverter(arguments, 1, inputTypes, converters);

  if (arguments[1] instanceof ConstantObjectInspector) {
    String dayOfWeek = getConstantStringValue(arguments, 1);
    isDayOfWeekConst = true;
    dayOfWeekIntConst = getIntDayOfWeek(dayOfWeek);
  }

  ObjectInspector outputOI = PrimitiveObjectInspectorFactory.writableStringObjectInspector;
  return outputOI;
}
 
Example 11
Project: hive-udf-backports   File: BackportUtils.java   Source Code and License 6 votes vote down vote up
public static void obtainLongConverter(ObjectInspector[] arguments, int i,
        PrimitiveCategory[] inputTypes, Converter[] converters) throws UDFArgumentTypeException {
    PrimitiveObjectInspector inOi = (PrimitiveObjectInspector) arguments[i];
    PrimitiveCategory inputType = inOi.getPrimitiveCategory();
    switch (inputType) {
        case BYTE:
        case SHORT:
        case INT:
        case LONG:
            break;
        default:
            throw new UDFArgumentTypeException(i,
                "_FUNC_ only takes LONG/INT/SHORT/BYTE types as " + getArgOrder(i)
                        + " argument, got " + inputType);
    }

    Converter converter = ObjectInspectorConverters.getConverter(arguments[i],
        PrimitiveObjectInspectorFactory.writableIntObjectInspector);
    converters[i] = converter;
    inputTypes[i] = inputType;
}
 
Example 12
Project: hive-udf-backports   File: GenericUDAFBinarySetFunctions.java   Source Code and License 6 votes vote down vote up
private static void checkArgumentTypes(TypeInfo[] parameters) throws UDFArgumentTypeException {
  if (parameters.length != 2) {
    throw new UDFArgumentTypeException(parameters.length - 1,
        "Exactly two arguments are expected.");
  }

  if (parameters[0].getCategory() != ObjectInspector.Category.PRIMITIVE) {
    throw new UDFArgumentTypeException(0, "Only primitive type arguments are accepted but "
        + parameters[0].getTypeName() + " is passed.");
  }

  if (parameters[1].getCategory() != ObjectInspector.Category.PRIMITIVE) {
    throw new UDFArgumentTypeException(1, "Only primitive type arguments are accepted but "
        + parameters[1].getTypeName() + " is passed.");
  }

  if (!acceptedPrimitiveCategory(((PrimitiveTypeInfo) parameters[0]).getPrimitiveCategory())) {
    throw new UDFArgumentTypeException(0, "Only numeric type arguments are accepted but "
        + parameters[0].getTypeName() + " is passed.");

  }
  if (!acceptedPrimitiveCategory(((PrimitiveTypeInfo) parameters[1]).getPrimitiveCategory())) {
    throw new UDFArgumentTypeException(1, "Only numeric type arguments are accepted but "
        + parameters[1].getTypeName() + " is passed.");
  }
}
 
Example 13
Project: hive-jq-udtf   File: JsonQueryUDTFTest.java   Source Code and License 6 votes vote down vote up
@Test
public void testMissingFieldsInConversions() throws HiveException {
	final JsonQueryUDTF sut = new JsonQueryUDTF();

	final StructObjectInspector oi = sut.initialize(new ObjectInspector[] {
			PrimitiveObjectInspectorFactory.writableStringObjectInspector,
			toConstantOI("{foo: 10}"),
			toConstantOI("foo:int"),
			toConstantOI("bar:int"),
	});

	final List<Object> results = evaluate(sut, toObject(null));
	assertEquals(1, results.size());

	assertEquals(10, new HivePath(oi, ".foo").extract(results.get(0)).asInt());
	assertTrue(new HivePath(oi, ".bar").extract(results.get(0)).isNull());
}
 
Example 14
Project: hive-udf-backports   File: GenericUDFAddMonths.java   Source Code and License 6 votes vote down vote up
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
  checkArgsSize(arguments, 2, 2);

  checkArgPrimitive(arguments, 0);
  checkArgPrimitive(arguments, 1);

  checkArgGroups(arguments, 0, inputTypes, STRING_GROUP, DATE_GROUP, VOID_GROUP);
  checkArgGroups(arguments, 1, inputTypes, NUMERIC_GROUP, VOID_GROUP);

  obtainDateConverter(arguments, 0, inputTypes, converters);
  obtainIntConverter(arguments, 1, inputTypes, converters);

  if (arguments[1] instanceof ConstantObjectInspector) {
    numMonthsConst = getConstantIntValue(arguments, 1);
    isNumMonthsConst = true;
  }

  ObjectInspector outputOI = PrimitiveObjectInspectorFactory.writableStringObjectInspector;
  return outputOI;
}
 
Example 15
Project: dremio-oss   File: ObjectInspectorHelper.java   Source Code and License 6 votes vote down vote up
public static MinorType getMinorType(ObjectInspector oi) {
  switch(oi.getCategory()) {
    case PRIMITIVE: {
      PrimitiveObjectInspector poi = (PrimitiveObjectInspector)oi;
      if (TYPE_HIVE2MINOR.containsKey(poi.getPrimitiveCategory())) {
        return TYPE_HIVE2MINOR.get(poi.getPrimitiveCategory());
      }
      throw new UnsupportedOperationException();
    }

    case MAP:
    case LIST:
    case STRUCT:
    default:
      throw new UnsupportedOperationException();
  }
}
 
Example 16
Project: hive-udf-backports   File: BackportUtils.java   Source Code and License 6 votes vote down vote up
public static void obtainTimestampConverter(ObjectInspector[] arguments, int i,
        PrimitiveCategory[] inputTypes, Converter[] converters) throws UDFArgumentTypeException {
    PrimitiveObjectInspector inOi = (PrimitiveObjectInspector) arguments[i];
    PrimitiveCategory inputType = inOi.getPrimitiveCategory();
    ObjectInspector outOi;
    switch (inputType) {
        case STRING:
        case VARCHAR:
        case CHAR:
        case TIMESTAMP:
        case DATE:
            break;
        default:
            throw new UDFArgumentTypeException(i,
                "_FUNC_ only takes STRING_GROUP or DATE_GROUP types as " + getArgOrder(i)
                        + " argument, got " + inputType);
    }
    outOi = PrimitiveObjectInspectorFactory.writableTimestampObjectInspector;
    converters[i] = ObjectInspectorConverters.getConverter(inOi, outOi);
    inputTypes[i] = inputType;
}
 
Example 17
Project: hive-udf-backports   File: BackportUtils.java   Source Code and License 6 votes vote down vote up
public static void obtainIntConverter(ObjectInspector[] arguments, int i,
        PrimitiveCategory[] inputTypes, Converter[] converters) throws UDFArgumentTypeException {
    PrimitiveObjectInspector inOi = (PrimitiveObjectInspector) arguments[i];
    PrimitiveCategory inputType = inOi.getPrimitiveCategory();
    switch (inputType) {
        case BYTE:
        case SHORT:
        case INT:
        case VOID:
            break;
        default:
            throw new UDFArgumentTypeException(i, "_FUNC_ only takes INT/SHORT/BYTE types as "
                    + getArgOrder(i) + " argument, got " + inputType);
    }

    Converter converter = ObjectInspectorConverters.getConverter(arguments[i],
        PrimitiveObjectInspectorFactory.writableIntObjectInspector);
    converters[i] = converter;
    inputTypes[i] = inputType;
}
 
Example 18
Project: hive-jq-udtf   File: JsonQueryUDTF.java   Source Code and License 5 votes vote down vote up
private static Pair<String, ObjectInspector> parseNameAndType(final String nameAndType, final boolean requireName) throws UDFArgumentException {
	try {
		final Matcher m = NAME_AND_TYPE_PATTERN.matcher(nameAndType);
		if (m.matches()) {
			final String name = m.group(1);
			final ObjectInspector oi = ObjectInspectors.newObjectInspectorFromHiveType(nameAndType.substring(name.length() + 1));
			return Pair.of(name, oi);
		}
		if (requireName)
			throw new UDFArgumentException("Can't parse NAME:TYPE from \"" + nameAndType + "\". NAME is required.");
		return Pair.of(null, ObjectInspectors.newObjectInspectorFromHiveType(nameAndType));
	} catch (final Exception e) {
		throw new UDFArgumentException("Can't parse NAME:TYPE or TYPE from \"" + nameAndType + "\". " + e.getMessage());
	}
}
 
Example 19
Project: multiple-dimension-spread   File: InHiveExpr.java   Source Code and License 5 votes vote down vote up
public static PrimitiveObjectInspector getPrimitiveObjectInspector( final ExprNodeDesc exprNode ){
  if( ! ( exprNode instanceof ExprNodeConstantDesc ) ){
    return null;
  }
  ExprNodeConstantDesc constDesc = (ExprNodeConstantDesc)exprNode;
  ObjectInspector objectInspector = constDesc.getWritableObjectInspector();
  if( objectInspector.getCategory() != ObjectInspector.Category.PRIMITIVE ){
    return null;
  }
  return (PrimitiveObjectInspector)objectInspector;
}
 
Example 20
Project: hive-jq-udtf   File: ObjectInspectors.java   Source Code and License 5 votes vote down vote up
private static ObjectInspector newObjectInspectorFromHiveType(final ASTNode type) {
	// matching by token names, because token IDs (which are static final) drastically change between versions.
	switch (type.getToken().getText()) {
		case "TOK_STRING":
			return PrimitiveObjectInspectorFactory.writableStringObjectInspector;
		case "TOK_INT":
			return PrimitiveObjectInspectorFactory.writableIntObjectInspector;
		case "TOK_DOUBLE":
			return PrimitiveObjectInspectorFactory.writableDoubleObjectInspector;
		case "TOK_FLOAT":
			return PrimitiveObjectInspectorFactory.writableFloatObjectInspector;
		case "TOK_BIGINT":
			return PrimitiveObjectInspectorFactory.writableLongObjectInspector;
		case "TOK_BOOLEAN": {
			return PrimitiveObjectInspectorFactory.writableBooleanObjectInspector;
		}
		case "TOK_STRUCT": {
			final ASTNode tabColList = (ASTNode) type.getChild(0);
			final List<String> names = new ArrayList<>();
			final List<ObjectInspector> ois = new ArrayList<>();
			for (final Node tabCol : tabColList.getChildren()) {
				final ASTNode a = (ASTNode) tabCol;
				names.add(a.getChild(0).toString());
				ois.add(newObjectInspectorFromHiveType((ASTNode) a.getChild(1)));
			}
			return ObjectInspectorFactory.getStandardStructObjectInspector(names, ois);
		}
		case "TOK_MAP": {
			final ObjectInspector keyType = newObjectInspectorFromHiveType((ASTNode) type.getChild(0));
			final ObjectInspector valueType = newObjectInspectorFromHiveType((ASTNode) type.getChild(1));
			return ObjectInspectorFactory.getStandardMapObjectInspector(keyType, valueType);
		}
		case "TOK_LIST": {
			final ObjectInspector itemType = newObjectInspectorFromHiveType((ASTNode) type.getChild(0));
			return ObjectInspectorFactory.getStandardListObjectInspector(itemType);
		}
		default:
			throw new IllegalArgumentException("unsupported type: " + type.toStringTree());
	}
}
 
Example 21
Project: dremio-oss   File: ObjectInspectorHelper.java   Source Code and License 5 votes vote down vote up
public static JBlock initReturnValueHolder(ClassGenerator<?> g, JCodeModel m, JVar returnValueHolder, ObjectInspector oi, MinorType returnType) {
    JBlock block = new JBlock(false, false);
    switch(oi.getCategory()) {
      case PRIMITIVE: {
        PrimitiveObjectInspector poi = (PrimitiveObjectInspector)oi;
        switch(poi.getPrimitiveCategory()) {
<#list typesOI.map as entry>
          case ${entry.hiveType}:{
            JType holderClass = TypeHelper.getHolderType(m, returnType, DataMode.OPTIONAL);
            block.assign(returnValueHolder, JExpr._new(holderClass));

          <#if entry.hiveType == "VARCHAR" || entry.hiveType == "STRING" || entry.hiveType == "BINARY" || entry.hiveType == "CHAR">
            block.assign( //
                returnValueHolder.ref("buffer"), //
                JExpr
                  .direct("context")
                  .invoke("getManagedBuffer")
                  .invoke("reallocIfNeeded")
                    .arg(JExpr.lit(1024))
            );
          </#if>
            return block;
          }
</#list>
          default:
            throw new UnsupportedOperationException(String.format("Received unknown/unsupported type '%s'", poi.getPrimitiveCategory().toString()));
        }
      }

      case MAP:
      case LIST:
      case STRUCT:
      default:
        throw new UnsupportedOperationException(String.format("Received unknown/unsupported type '%s'", oi.getCategory().toString()));
    }
  }
 
Example 22
Project: multiple-dimension-spread   File: MDSListObjectInspector.java   Source Code and License 5 votes vote down vote up
public MDSListObjectInspector( final ListTypeInfo typeInfo ){
  valueObjectInspector = MDSObjectInspectorFactory.craeteObjectInspectorFromTypeInfo( typeInfo.getListElementTypeInfo() ); 
  if( valueObjectInspector.getCategory() == ObjectInspector.Category.PRIMITIVE ){
    getField = new PrimitiveGetField( (PrimitiveObjectInspector)valueObjectInspector );
  }
  else if( valueObjectInspector.getCategory() == ObjectInspector.Category.UNION ){
    getField = new UnionGetField( (UnionTypeInfo)( typeInfo.getListElementTypeInfo() ) );
  }
  else{
    getField = new NestedGetField();
  }
}
 
Example 23
Project: monarch   File: OrcUtils.java   Source Code and License 5 votes vote down vote up
/**
 * Convert the value using the ObjectInspector. The Writable values are converted to their
 * respective Java objects from using the provided inspector.
 *
 * @param oi the field object inspector
 * @param value the value
 * @return the corresponding Java object value
 */
public static Object convert(final ObjectInspector oi, final Object value) {
  if (value == null) {
    return null;
  }
  Object outValue = null;
  switch (oi.getCategory()) {
    case PRIMITIVE:
      outValue = OrcReadFunctionMap.get(oi.getTypeName()).apply(value);
      break;
    case LIST:
      final ListObjectInspector loi = (ListObjectInspector) oi;
      final ObjectInspector eoi = loi.getListElementObjectInspector();
      outValue =
          loi.getList(value).stream().map(e -> convert(eoi, e)).collect(Collectors.toList());
      break;
    case MAP:
      final MapObjectInspector moi = (MapObjectInspector) oi;
      final ObjectInspector koi = moi.getMapKeyObjectInspector();
      final ObjectInspector voi = moi.getMapValueObjectInspector();
      outValue = moi.getMap(value).entrySet().stream()
          .collect(Collectors.toMap(e -> convert(koi, e.getKey()),
              e -> convert(voi, e.getValue()), throwingMerger(), LinkedHashMap::new));
      break;
    case STRUCT:
      final StructObjectInspector soi = (StructObjectInspector) oi;
      outValue = soi.getAllStructFieldRefs().stream()
          .map(e -> convert(e.getFieldObjectInspector(), soi.getStructFieldData(value, e)))
          .toArray();
      break;
    case UNION:
      final UnionObjectInspector uoi = (UnionObjectInspector) oi;
      final List<? extends ObjectInspector> ois = uoi.getObjectInspectors();
      final byte tag = uoi.getTag(value);
      outValue = new Object[] {tag, convert(ois.get(tag), uoi.getField(value))};
      break;
  }
  return outValue;
}
 
Example 24
Project: hive-udf-backports   File: BackportUtils.java   Source Code and License 5 votes vote down vote up
public static Boolean getConstantBooleanValue(ObjectInspector[] arguments, int i)
        throws UDFArgumentTypeException {
    Object constValue = ((ConstantObjectInspector) arguments[i]).getWritableConstantValue();
    if (constValue == null) {
        return false;
    }
    if (constValue instanceof BooleanWritable) {
        return ((BooleanWritable) constValue).get();
    } else {
        throw new UDFArgumentTypeException(i, "_FUNC_ only takes BOOLEAN types as "
                + getArgOrder(i) + " argument, got " + constValue.getClass());
    }
}
 
Example 25
Project: Transwarp-Sample-Code   File: udtfCheck.java   Source Code and License 5 votes vote down vote up
@Override
public StructObjectInspector initialize(ObjectInspector[] args) throws UDFArgumentException {
    stringOI = (PrimitiveObjectInspector) args[0];

    List<String> fieldNames = new ArrayList<String>(2);
    List<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>(2);
    fieldNames.add("id");
    fieldNames.add("name");
    fieldOIs.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
    fieldOIs.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);
}
 
Example 26
Project: dremio-oss   File: HiveFieldConverter.java   Source Code and License 5 votes vote down vote up
@Override
public void setSafeValue(ObjectInspector oi, Object hiveFieldValue, ValueVector outputVV, int outputIndex) {
  final Text value = ((HiveVarcharObjectInspector)oi).getPrimitiveWritableObject(hiveFieldValue).getTextValue();
  final byte[] valueBytes = value.getBytes();
  final int valueLen = value.getLength();
  ((NullableVarCharVector) outputVV).getMutator().setSafe(outputIndex, valueBytes, 0, valueLen);
}
 
Example 27
Project: QDrill   File: HiveFuncHolder.java   Source Code and License 5 votes vote down vote up
/**
 * Create holder for GenericUDF
 * @param genericUdfClazz implementation class
 * @param argTypes
 * @param returnOI
 * @param returnType
 */
public HiveFuncHolder(Class<? extends GenericUDF> genericUdfClazz, MajorType[] argTypes,
                      ObjectInspector returnOI, MajorType returnType, boolean isRandom) {
  this.genericUdfClazz = genericUdfClazz;
  this.argTypes = argTypes;
  this.returnOI = returnOI;
  this.returnType = returnType;
  this.isRandom = isRandom;
}
 
Example 28
Project: dremio-oss   File: ObjectInspectorHelper.java   Source Code and License 5 votes vote down vote up
public static ObjectInspector getObjectInspector(DataMode mode, MinorType minorType, boolean varCharToStringReplacement) {
  try {
    if (mode == DataMode.REQUIRED) {
      if (OIMAP_REQUIRED.containsKey(minorType)) {
        if (varCharToStringReplacement && minorType == MinorType.VARCHAR) {
          return (ObjectInspector) ((Class) OIMAP_REQUIRED.get(minorType).toArray()[1]).newInstance();
        } else {
          return (ObjectInspector) ((Class) OIMAP_REQUIRED.get(minorType).toArray()[0]).newInstance();
        }
      }
    } else if (mode == DataMode.OPTIONAL) {
      if (OIMAP_OPTIONAL.containsKey(minorType)) {
        if (varCharToStringReplacement && minorType == MinorType.VARCHAR) {
          return (ObjectInspector) ((Class) OIMAP_OPTIONAL.get(minorType).toArray()[1]).newInstance();
        } else {
          return (ObjectInspector) ((Class) OIMAP_OPTIONAL.get(minorType).toArray()[0]).newInstance();
        }
      }
    } else {
      throw new UnsupportedOperationException("Repeated types are not supported as arguement to Hive UDFs");
    }
  } catch(InstantiationException | IllegalAccessException e) {
    throw new RuntimeException("Failed to instantiate ObjectInspector", e);
  }

  throw new UnsupportedOperationException(
      String.format("Type %s[%s] not supported as arguement to Hive UDFs", minorType.toString(), mode.toString()));
}
 
Example 29
Project: hive-jq-udtf   File: JsonQueryUDTF.java   Source Code and License 5 votes vote down vote up
public static String asConstantNonNullString(final ObjectInspector oi, final String name) throws UDFArgumentException {
	if (!(oi instanceof WritableConstantStringObjectInspector))
		throw new UDFArgumentException(name + " must be a constant string.");
	final Text text = ((WritableConstantStringObjectInspector) oi).getWritableConstantValue();
	if (text == null)
		throw new UDFArgumentException(name + " must not be NULL.");
	return text.toString();
}
 
Example 30
Project: QDrill   File: HiveFieldConverter.java   Source Code and License 5 votes vote down vote up
@Override
public void setSafeValue(ObjectInspector oi, Object hiveFieldValue, ValueVector outputVV, int outputIndex) {
  DecimalUtility.getSparseFromBigDecimal(
      ((HiveDecimalObjectInspector)oi).getPrimitiveJavaObject(hiveFieldValue).bigDecimalValue(),
      holder.buffer, holder.start, holder.scale, holder.precision, Decimal38SparseHolder.nDecimalDigits);
  ((NullableDecimal38SparseVector) outputVV).getMutator().setSafe(outputIndex, holder);
}
 
Example 31
Project: monarch   File: AWriter.java   Source Code and License 5 votes vote down vote up
AWriter(FileSystem fs, Path path, OrcFile.WriterOptions opts) throws IOException {
  super(fs, path, opts);
  this.inspector = (ObjectInspector) ReflectionUtils.getFieldValue(opts, "inspector");
  int batchSize = (int) ReflectionUtils.getFieldValue(opts, "batchSize");
  internalBatch = opts.getSchema().createRowBatch(batchSize);
  if (inspector instanceof StructObjectInspector) {
    List<? extends StructField> fieldList =
        ((StructObjectInspector) inspector).getAllStructFieldRefs();
    fields = new StructField[fieldList.size()];
    fieldList.toArray(fields);
  } else {
    fields = null;
  }
}
 
Example 32
Project: hive-udf-backports   File: BackportUtils.java   Source Code and License 5 votes vote down vote up
public static void obtainBinaryConverter(ObjectInspector[] arguments, int i,
        PrimitiveCategory[] inputTypes, Converter[] converters) throws UDFArgumentTypeException {
    PrimitiveObjectInspector inOi = (PrimitiveObjectInspector) arguments[i];
    PrimitiveCategory inputType = inOi.getPrimitiveCategory();

    Converter converter = ObjectInspectorConverters.getConverter(arguments[i],
        PrimitiveObjectInspectorFactory.writableBinaryObjectInspector);
    converters[i] = converter;
    inputTypes[i] = inputType;
}
 
Example 33
Project: dremio-oss   File: HiveFuncHolder.java   Source Code and License 5 votes vote down vote up
/**
 * Start generating code
 * @return workspace variables
 */
@Override
public JVar[] renderStart(ClassGenerator<?> g, CompleteType resolvedOutput, HoldingContainer[] inputVariables){
  JVar[] workspaceJVars = new JVar[5];

  workspaceJVars[0] = g.declareClassField("returnOI", g.getModel()._ref(ObjectInspector.class));
  workspaceJVars[1] = g.declareClassField("udfInstance", g.getModel()._ref(GenericUDF.class));
  workspaceJVars[2] = g.declareClassField("deferredObjects", g.getModel()._ref(DeferredObject[].class));
  workspaceJVars[3] = g.declareClassField("arguments", g.getModel()._ref(DeferredObject[].class));
  workspaceJVars[4] = g.declareClassField("returnValueHolder", TypeHelper.getHolderType(g.getModel(), returnType.toMinorType(), DataMode.OPTIONAL));

  return workspaceJVars;
}
 
Example 34
Project: hive-udf-backports   File: BackportUtils.java   Source Code and License 5 votes vote down vote up
public static void obtainDoubleConverter(ObjectInspector[] arguments, int i,
        PrimitiveCategory[] inputTypes, Converter[] converters) throws UDFArgumentTypeException {
    PrimitiveObjectInspector inOi = (PrimitiveObjectInspector) arguments[i];
    PrimitiveCategory inputType = inOi.getPrimitiveCategory();
    Converter converter = ObjectInspectorConverters.getConverter(arguments[i],
        PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
    converters[i] = converter;
    inputTypes[i] = inputType;
}
 
Example 35
Project: QDrill   File: HiveFieldConverter.java   Source Code and License 5 votes vote down vote up
@Override
public void setSafeValue(ObjectInspector oi, Object hiveFieldValue, ValueVector outputVV, int outputIndex) {
  final Text value = ((HiveVarcharObjectInspector)oi).getPrimitiveWritableObject(hiveFieldValue).getTextValue();
  final byte[] valueBytes = value.getBytes();
  final int valueLen = value.getLength();
  ((NullableVarCharVector) outputVV).getMutator().setSafe(outputIndex, valueBytes, 0, valueLen);
}
 
Example 36
Project: dataplatform-schema-lib   File: HivePrimitiveConverterFactory.java   Source Code and License 5 votes vote down vote up
public static IHivePrimitiveConverter get( final ObjectInspector objectInspector ){

    switch( objectInspector.getCategory() ){
      case PRIMITIVE:
        PrimitiveObjectInspector primitiveInspector = (PrimitiveObjectInspector)objectInspector;
        switch( primitiveInspector.getPrimitiveCategory() ){
          case BINARY:
            return new HiveBytesPrimitiveConverter( (BinaryObjectInspector)objectInspector );
          case BOOLEAN:
            return new HiveBooleanPrimitiveConverter( (BooleanObjectInspector)objectInspector );
          case BYTE:
            return new HiveBytePrimitiveConverter( (ByteObjectInspector)objectInspector );
          case DOUBLE:
            return new HiveDoublePrimitiveConverter( (DoubleObjectInspector)objectInspector );
          case FLOAT:
            return new HiveFloatPrimitiveConverter( (FloatObjectInspector)objectInspector );
          case INT:
            return new HiveIntegerPrimitiveConverter( (IntObjectInspector)objectInspector );
          case LONG:
            return new HiveLongPrimitiveConverter( (LongObjectInspector)objectInspector );
          case SHORT:
            return new HiveShortPrimitiveConverter( (ShortObjectInspector)objectInspector );
          case STRING:
            return new HiveStringPrimitiveConverter( (StringObjectInspector)objectInspector );
          case DATE:
          case TIMESTAMP:
          case VOID:
          case UNKNOWN:
          default:
            return new HiveDefaultPrimitiveConverter();
        }
      default :
        return new HiveDefaultPrimitiveConverter();
    }
  }
 
Example 37
Project: dremio-oss   File: HiveFuncHolder.java   Source Code and License 5 votes vote down vote up
/**
 * Create holder for GenericUDF
 * @param genericUdfClazz implementation class
 * @param argTypes
 * @param returnOI
 * @param returnType
 */
public HiveFuncHolder(
    Class<? extends GenericUDF> genericUdfClazz,
    CompleteType[] argTypes,
    ObjectInspector returnOI,
    CompleteType returnType,
    boolean isRandom) {
  this.genericUdfClazz = genericUdfClazz;
  this.argTypes = argTypes;
  this.returnOI = returnOI;
  this.returnType = returnType;
  this.isRandom = isRandom;
}
 
Example 38
Project: dataplatform-schema-lib   File: HiveParserFactory.java   Source Code and License 5 votes vote down vote up
public static boolean hasParser( final ObjectInspector objectInspector ){
  switch( objectInspector.getCategory() ){
    case LIST:
    case MAP:
    case STRUCT:
      return true;
    case UNION:
    default:
      return false;
  }
}
 
Example 39
Project: dataplatform-schema-lib   File: OrcParserFactory.java   Source Code and License 5 votes vote down vote up
public static IParser get( final ObjectInspector objectInspector , final Object row ){

    switch( objectInspector.getCategory() ){
      case LIST:
        return new OrcListParser( row , (ListObjectInspector)objectInspector );
      case MAP:
        return new OrcMapParser( row , (MapObjectInspector)objectInspector );
      case STRUCT:
        return new OrcStructParser( row , (StructObjectInspector)objectInspector );
      case UNION:
      default:
        return new OrcNullParser();
    }
  }
 
Example 40
Project: dataplatform-schema-lib   File: OrcParserFactory.java   Source Code and License 5 votes vote down vote up
public static boolean hasParser( final ObjectInspector objectInspector ){
  switch( objectInspector.getCategory() ){
    case LIST:
    case MAP:
    case STRUCT:
      return true;
    case UNION:
    default:
      return false;
  }
}