org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo Java Examples

The following examples show how to use org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TsFileSerDeTest.java    From incubator-iotdb with Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() {
  tsFileSerDer = new TsFileSerDe();
  columnNames = Arrays.asList("time_stamp", "sensor_1");
  columnTypes = new ArrayList<>();
  PrimitiveTypeInfo typeInfo1 = new PrimitiveTypeInfo();
  typeInfo1.setTypeName("bigint");
  columnTypes.add(typeInfo1);
  PrimitiveTypeInfo typeInfo2 = new PrimitiveTypeInfo();
  typeInfo2.setTypeName("bigint");
  columnTypes.add(typeInfo2);
  tbl = new Properties();
  String delimiter = ",";
  tbl.setProperty(serdeConstants.COLUMN_NAME_DELIMITER, delimiter);
  tbl.setProperty(serdeConstants.LIST_COLUMNS, String.join(delimiter, columnNames));
  tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, "bigint,bigint");
  tbl.setProperty(TsFileSerDe.DEVICE_ID, "device_1");
  job = new JobConf();
  try {
    tsFileSerDer.initialize(job, tbl);
  } catch (SerDeException e) {
    e.printStackTrace();

  }
}
 
Example #2
Source File: BlurObjectInspectorGenerator.java    From incubator-retired-blur with Apache License 2.0 6 votes vote down vote up
private ObjectInspector createObjectInspectorWorker(TypeInfo ti) throws SerDeException {
  switch (ti.getCategory()) {
  case PRIMITIVE:
    PrimitiveTypeInfo pti = (PrimitiveTypeInfo) ti;
    return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(pti);
  case STRUCT:
    StructTypeInfo sti = (StructTypeInfo) ti;
    List<ObjectInspector> ois = new ArrayList<ObjectInspector>(sti.getAllStructFieldTypeInfos().size());
    for (TypeInfo typeInfo : sti.getAllStructFieldTypeInfos()) {
      ois.add(createObjectInspectorWorker(typeInfo));
    }
    return ObjectInspectorFactory.getStandardStructObjectInspector(sti.getAllStructFieldNames(), ois);
  case LIST:
    ListTypeInfo lti = (ListTypeInfo) ti;
    TypeInfo listElementTypeInfo = lti.getListElementTypeInfo();
    return ObjectInspectorFactory.getStandardListObjectInspector(createObjectInspectorWorker(listElementTypeInfo));
  default:
    throw new SerDeException("No Hive categories matched for [" + ti + "]");
  }
}
 
Example #3
Source File: HiveWriteUtils.java    From presto with Apache License 2.0 6 votes vote down vote up
private static boolean isWritableType(TypeInfo typeInfo)
{
    switch (typeInfo.getCategory()) {
        case PRIMITIVE:
            PrimitiveCategory primitiveCategory = ((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory();
            return isWritablePrimitiveType(primitiveCategory);
        case MAP:
            MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo;
            return isWritableType(mapTypeInfo.getMapKeyTypeInfo()) && isWritableType(mapTypeInfo.getMapValueTypeInfo());
        case LIST:
            ListTypeInfo listTypeInfo = (ListTypeInfo) typeInfo;
            return isWritableType(listTypeInfo.getListElementTypeInfo());
        case STRUCT:
            StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo;
            return structTypeInfo.getAllStructFieldTypeInfos().stream().allMatch(HiveWriteUtils::isWritableType);
    }
    return false;
}
 
Example #4
Source File: CobolDeserializer.java    From Cobol-to-Hive with Apache License 2.0 6 votes vote down vote up
private Object worker(String columnName, TypeInfo columnType){
	
	switch(columnType.getCategory()) {
		
		case STRUCT:
			return deserializeStruct(columnName, (StructTypeInfo) columnType);
		case UNION:
			return deserializeUnion(columnName,(UnionTypeInfo) columnType);
		case LIST:
		return deserializeList(columnName, (ListTypeInfo) columnType);
		case MAP:
		throw new RuntimeException("map type is not possible for cobol layout" + columnType.getCategory());
		case PRIMITIVE:
		return deserializePrimitive(columnName, (PrimitiveTypeInfo) columnType);
		default:
		throw new RuntimeException("Unknown TypeInfo: " + columnType.getCategory());
	}
}
 
Example #5
Source File: HiveBucketing.java    From presto with Apache License 2.0 6 votes vote down vote up
private static boolean containsTimestampBucketedV2(TypeInfo type)
{
    switch (type.getCategory()) {
        case PRIMITIVE:
            return ((PrimitiveTypeInfo) type).getPrimitiveCategory() == TIMESTAMP;
        case LIST:
            return containsTimestampBucketedV2(((ListTypeInfo) type).getListElementTypeInfo());
        case MAP:
            MapTypeInfo mapTypeInfo = (MapTypeInfo) type;
            // Note: we do not check map value type because HiveBucketingV2#hashOfMap hashes map values with v1
            return containsTimestampBucketedV2(mapTypeInfo.getMapKeyTypeInfo());
        default:
            // TODO: support more types, e.g. ROW
            throw new UnsupportedOperationException("Computation of Hive bucket hashCode is not supported for Hive category: " + type.getCategory());
    }
}
 
Example #6
Source File: ExpressionHelper.java    From aws-glue-data-catalog-client-for-apache-hive-metastore with Apache License 2.0 6 votes vote down vote up
private static void fieldEscaper(List<ExprNodeDesc> exprNodes, ExprNodeDesc parent, Set<String> columnNamesInNotInExpression) {
  if (exprNodes == null || exprNodes.isEmpty()) {
    return;
  } else {
    for (ExprNodeDesc nodeDesc : exprNodes) {
      String nodeType = nodeDesc.getTypeString().toLowerCase();
      if (QUOTED_TYPES.contains(nodeType)) {
        PrimitiveTypeInfo tInfo = new PrimitiveTypeInfo();
        tInfo.setTypeName(HIVE_STRING_TYPE_NAME);
        nodeDesc.setTypeInfo(tInfo);
      }
      addColumnNamesOfNotInExpressionToSet(nodeDesc, parent, columnNamesInNotInExpression);
      fieldEscaper(nodeDesc.getChildren(), nodeDesc, columnNamesInNotInExpression);
    }
  }
}
 
Example #7
Source File: TsFileSerDe.java    From incubator-iotdb with Apache License 2.0 6 votes vote down vote up
private ObjectInspector createObjectInspectorWorker(TypeInfo ti) throws TsFileSerDeException {
  if(!supportedCategories(ti)) {
    throw new TsFileSerDeException("Don't yet support this type: " + ti);
  }
  ObjectInspector result;
  switch(ti.getCategory()) {
    case PRIMITIVE:
      PrimitiveTypeInfo pti = (PrimitiveTypeInfo) ti;
      result = PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(pti);
      break;
    // these types is not supported in TsFile
    case LIST:
    case MAP:
    case STRUCT:
    case UNION:
      throw new TsFileSerDeException("The type is not supported in TsFile: " + ti);
    default:
      throw new TsFileSerDeException("No Hive categories matched: " + ti);
  }
  return result;
}
 
Example #8
Source File: HiveUtils.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
public static boolean isNumberTypeInfo(@Nonnull TypeInfo typeInfo) {
    if (typeInfo.getCategory() != ObjectInspector.Category.PRIMITIVE) {
        return false;
    }
    switch (((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory()) {
        case BYTE:
        case SHORT:
        case INT:
        case LONG:
        case FLOAT:
        case DOUBLE:
        case DECIMAL:
            return true;
        default:
            return false;
    }
}
 
Example #9
Source File: KuromojiUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testFiveArgumenString() throws UDFArgumentException, IOException {
    GenericUDF udf = new KuromojiUDF();
    ObjectInspector[] argOIs = new ObjectInspector[5];
    // line
    argOIs[0] = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    // mode
    PrimitiveTypeInfo stringType = new PrimitiveTypeInfo();
    stringType.setTypeName("string");
    argOIs[1] = PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
        stringType, null);
    // stopWords
    argOIs[2] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, null);
    // stopTags
    argOIs[3] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, null);
    // userDictUrl
    argOIs[4] = PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
        stringType, null);
    udf.initialize(argOIs);
    udf.close();
}
 
Example #10
Source File: KuromojiUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testFiveArgumentArray() throws UDFArgumentException, IOException {
    GenericUDF udf = new KuromojiUDF();
    ObjectInspector[] argOIs = new ObjectInspector[5];
    // line
    argOIs[0] = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    // mode
    PrimitiveTypeInfo stringType = new PrimitiveTypeInfo();
    stringType.setTypeName("string");
    argOIs[1] = PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
        stringType, null);
    // stopWords
    argOIs[2] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, null);
    // stopTags
    argOIs[3] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, null);
    // userDictUrl
    argOIs[4] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, null);
    udf.initialize(argOIs);
    udf.close();
}
 
Example #11
Source File: MDSMapObjectInspector.java    From multiple-dimension-spread with Apache License 2.0 6 votes vote down vote up
public MDSMapObjectInspector( final MapTypeInfo typeInfo ){
  TypeInfo keyTypeInfo = typeInfo.getMapKeyTypeInfo();
  if( keyTypeInfo.getCategory() == ObjectInspector.Category.PRIMITIVE && ( (PrimitiveTypeInfo)keyTypeInfo ).getPrimitiveCategory() == PrimitiveCategory.STRING ){
    keyObjectInspector = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
  }
  else{
    throw new RuntimeException( "Map key type is string only." );
  }

  valueObjectInspector = MDSObjectInspectorFactory.craeteObjectInspectorFromTypeInfo( typeInfo.getMapValueTypeInfo() ); 

  if( valueObjectInspector.getCategory() == ObjectInspector.Category.PRIMITIVE ){
    getField = new PrimitiveGetField( (PrimitiveObjectInspector)valueObjectInspector );
  }
  else if( valueObjectInspector.getCategory() == ObjectInspector.Category.UNION ){
    getField = new UnionGetField( (UnionTypeInfo)( typeInfo.getMapValueTypeInfo() ) );
  }
  else{
    getField = new NestedGetField();
  }
}
 
Example #12
Source File: KuromojiUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testFourArgument() throws UDFArgumentException, IOException {
    GenericUDF udf = new KuromojiUDF();
    ObjectInspector[] argOIs = new ObjectInspector[4];
    // line
    argOIs[0] = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    // mode
    PrimitiveTypeInfo stringType = new PrimitiveTypeInfo();
    stringType.setTypeName("string");
    argOIs[1] = PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
        stringType, null);
    // stopWords
    argOIs[2] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, null);
    // stopTags
    argOIs[3] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, null);
    udf.initialize(argOIs);
    udf.close();
}
 
Example #13
Source File: KuromojiUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test
public void testThreeArgument() throws UDFArgumentException, IOException {
    GenericUDF udf = new KuromojiUDF();
    ObjectInspector[] argOIs = new ObjectInspector[3];
    // line
    argOIs[0] = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    // mode
    PrimitiveTypeInfo stringType = new PrimitiveTypeInfo();
    stringType.setTypeName("string");
    argOIs[1] = PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
        stringType, null);
    // stopWords
    argOIs[2] = ObjectInspectorFactory.getStandardConstantListObjectInspector(
        PrimitiveObjectInspectorFactory.javaStringObjectInspector, null);
    udf.initialize(argOIs);
    udf.close();
}
 
Example #14
Source File: KuromojiUDFTest.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Test(expected = UDFArgumentException.class)
public void testInvalidMode() throws IOException, HiveException {
    GenericUDF udf = new KuromojiUDF();
    ObjectInspector[] argOIs = new ObjectInspector[2];
    // line
    argOIs[0] = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    // mode
    PrimitiveTypeInfo stringType = new PrimitiveTypeInfo();
    stringType.setTypeName("string");
    argOIs[1] = PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
        stringType, new Text("unsupported mode"));
    udf.initialize(argOIs);

    DeferredObject[] args = new DeferredObject[1];
    args[0] = new DeferredObject() {
        public Text get() throws HiveException {
            return new Text("クロモジのJapaneseAnalyzerを使ってみる。テスト。");
        }

        @Override
        public void prepare(int arg) throws HiveException {}
    };
    udf.evaluate(args);

    udf.close();
}
 
Example #15
Source File: EmoSerDe.java    From emodb with Apache License 2.0 6 votes vote down vote up
private Object deserializeDate(Object value, PrimitiveTypeInfo type)
        throws SerDeException {
    long ts;
    // Dates can be either ISO8601 Strings or numeric timestamps.  Any other data type or format cannot be
    // deserialized.
    if (value instanceof String) {
        try {
            ts = JsonHelper.parseTimestamp((String) value).getTime();
        } catch (Exception e) {
            throw new SerDeException("Invalid time string: " + value);
        }
    } else if (value instanceof Number) {
        ts = ((Number) value).longValue();
    } else if (value instanceof java.util.Date) {
        ts = ((java.util.Date) value).getTime();
    } else {
        throw new SerDeException("Invalid time value: " + value);
    }

    if (type.getPrimitiveCategory() == PrimitiveObjectInspector.PrimitiveCategory.DATE) {
        return new Date(ts);
    } else {
        return new Timestamp(ts);
    }
}
 
Example #16
Source File: EmoSerDe.java    From emodb with Apache License 2.0 6 votes vote down vote up
private Object deserializeNumber(Object value, PrimitiveTypeInfo type)
        throws SerDeException {
    // Note that only numbers and booleans are supported.  All other types cannot be deserialized.  In particular
    // String representations of numbers are not parsed.
    Number number;
    if (value instanceof Number) {
        number = (Number) value;
    } else if (value instanceof Boolean) {
        number = ((Boolean) value) ? (byte) 1 : 0;
    } else {
        throw new SerDeException("Value is not a " + type + ": " + value);
    }

    switch (type.getPrimitiveCategory()) {
        case BYTE:   return number.byteValue();
        case SHORT:  return number.shortValue();
        case INT:    return number.intValue();
        case LONG:   return number.longValue();
        case FLOAT:  return number.floatValue();
        case DOUBLE: return number.doubleValue();
    }

    throw new SerDeException("Primitive number did not match any expected categories"); // Unreachable
}
 
Example #17
Source File: EmoSerDe.java    From emodb with Apache License 2.0 6 votes vote down vote up
/**
 * Deserializes a primitive to its corresponding Java type, doing a best-effort conversion when necessary.
 */
private Object deserializePrimitive(PrimitiveTypeInfo type, Object value)
        throws SerDeException {
    switch (type.getPrimitiveCategory()) {
        case VOID:
            return null;
        case STRING:
            return deserializeString(value);
        case BOOLEAN:
            return deserializeBoolean(value);
        case BYTE:
        case SHORT:
        case INT:
        case LONG:
        case FLOAT:
        case DOUBLE:
            return deserializeNumber(value, type);
        case DATE:
        case TIMESTAMP:
            return deserializeDate(value, type);
        default:
            throw new SerDeException("Unsupported type: " + type.getPrimitiveCategory());
    }
}
 
Example #18
Source File: EmoSerDe.java    From emodb with Apache License 2.0 6 votes vote down vote up
/**
 * Determines if the given primitive is supported by this deserializer.  At this time the only exclusions are
 * BINARY, DECIMAL, VARCHAR, CHAR, and UNKNOWN.
 */
private boolean isSupportedPrimitive(PrimitiveTypeInfo type) {
    switch (type.getPrimitiveCategory()) {
        case VOID:
        case STRING:
        case BOOLEAN:
        case BYTE:
        case SHORT:
        case INT:
        case LONG:
        case FLOAT:
        case DOUBLE:
        case DATE:
        case TIMESTAMP:
            return true;
        default:
            return false;
    }
}
 
Example #19
Source File: HiveTypeUtil.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Convert Hive data type to a Flink data type.
 *
 * @param hiveType a Hive data type
 * @return the corresponding Flink data type
 */
public static DataType toFlinkType(TypeInfo hiveType) {
	checkNotNull(hiveType, "hiveType cannot be null");

	switch (hiveType.getCategory()) {
		case PRIMITIVE:
			return toFlinkPrimitiveType((PrimitiveTypeInfo) hiveType);
		case LIST:
			ListTypeInfo listTypeInfo = (ListTypeInfo) hiveType;
			return DataTypes.ARRAY(toFlinkType(listTypeInfo.getListElementTypeInfo()));
		case MAP:
			MapTypeInfo mapTypeInfo = (MapTypeInfo) hiveType;
			return DataTypes.MAP(toFlinkType(mapTypeInfo.getMapKeyTypeInfo()), toFlinkType(mapTypeInfo.getMapValueTypeInfo()));
		case STRUCT:
			StructTypeInfo structTypeInfo = (StructTypeInfo) hiveType;

			List<String> names = structTypeInfo.getAllStructFieldNames();
			List<TypeInfo> typeInfos = structTypeInfo.getAllStructFieldTypeInfos();

			DataTypes.Field[] fields = new DataTypes.Field[names.size()];

			for (int i = 0; i < fields.length; i++) {
				fields[i] = DataTypes.FIELD(names.get(i), toFlinkType(typeInfos.get(i)));
			}

			return DataTypes.ROW(fields);
		default:
			throw new UnsupportedOperationException(
				String.format("Flink doesn't support Hive data type %s yet.", hiveType));
	}
}
 
Example #20
Source File: Funnel.java    From hive-funnel-udf with Apache License 2.0 5 votes vote down vote up
@Override
public FunnelEvaluator getEvaluator(GenericUDAFParameterInfo info) throws SemanticException {
    // Get the parameters
    TypeInfo [] parameters = info.getParameters();

    // Check number of arguments
    if (parameters.length < 3) {
        throw new UDFArgumentLengthException("Please specify the action column, the timestamp column, and at least one funnel.");
    }

    // Check the action_column type and enforce that all funnel steps are the same type
    if (parameters[0].getCategory() != ObjectInspector.Category.PRIMITIVE) {
        throw new UDFArgumentTypeException(0, "Only primitive type arguments are accepted but " + parameters[0].getTypeName() + " was passed.");
    }
    PrimitiveCategory actionColumnCategory = ((PrimitiveTypeInfo) parameters[0]).getPrimitiveCategory();

    // Check the timestamp_column type
    if (parameters[1].getCategory() != ObjectInspector.Category.PRIMITIVE) {
        throw new UDFArgumentTypeException(1, "Only primitive type arguments are accepted but " + parameters[0].getTypeName() + " was passed.");
    }

    // Check that all funnel steps are the same type as the action_column
    for (int i = 2; i < parameters.length; i++) {
        switch (parameters[i].getCategory()) {
            case LIST:
                // Check that the list is of primitives of the same type as the action column
                TypeInfo typeInfo = ((ListTypeInfo) parameters[i]).getListElementTypeInfo();
                if (typeInfo.getCategory() != ObjectInspector.Category.PRIMITIVE || ((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory() != actionColumnCategory) {
                    throw new UDFArgumentTypeException(i, "Funnel list parameter " + Integer.toString(i) + " of type " + parameters[i].getTypeName() + " does not match expected type " + parameters[0].getTypeName() + ".");
                }
                break;
            default:
                throw new UDFArgumentTypeException(i, "Funnel list parameter " + Integer.toString(i) + " of type " + parameters[i].getTypeName() + " should be a list.");
        }
    }

    return new FunnelEvaluator();
}
 
Example #21
Source File: Merge.java    From hive-funnel-udf with Apache License 2.0 5 votes vote down vote up
@Override
public MergeEvaluator getEvaluator(GenericUDAFParameterInfo info) throws SemanticException {
    // Get the parameters
    TypeInfo [] parameters = info.getParameters();

    // Check number of arguments
    if (parameters.length != 1) {
        throw new UDFArgumentLengthException("Please specify the funnel column.");
    }

    // Check if the parameter is not a list
    if (parameters[0].getCategory() != ObjectInspector.Category.LIST) {
        throw new UDFArgumentTypeException(0, "Only list type arguments are accepted but " + parameters[0].getTypeName() + " was passed as the first parameter.");
    }

    // Check that the list is an array of primitives
    if (((ListTypeInfo) parameters[0]).getListElementTypeInfo().getCategory() != ObjectInspector.Category.PRIMITIVE) {
        throw new UDFArgumentTypeException(0, "A long array argument should be passed, but " + parameters[0].getTypeName() + " was passed instead.");
    }

    // Check that the list is of type long
    // May want to add support for int/double/float later
    switch (((PrimitiveTypeInfo) ((ListTypeInfo) parameters[0]).getListElementTypeInfo()).getPrimitiveCategory()) {
        case LONG:
            break;
        default:
            throw new UDFArgumentTypeException(0, "A long array argument should be passed, but " + parameters[0].getTypeName() + " was passed instead.");
    }

    return new MergeEvaluator();
}
 
Example #22
Source File: HiveUtils.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
public static boolean isIntegerTypeInfo(@Nonnull TypeInfo typeInfo) {
    if (typeInfo.getCategory() != ObjectInspector.Category.PRIMITIVE) {
        return false;
    }
    switch (((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory()) {
        case BYTE:
        case SHORT:
        case INT:
        case LONG:
            return true;
        default:
            return false;
    }
}
 
Example #23
Source File: CobolStringField.java    From Cobol-to-Hive with Apache License 2.0 5 votes vote down vote up
@Override
	public Object deserialize(byte[] rowBytes) {
		byte[] temp = super.transcodeField(super.getBytes(rowBytes));
		String s1 = new String(temp);
//		System.out.println(name+"\t - "+s1+"\t:"+offset+"\t@"+length);
		switch (((PrimitiveTypeInfo) this.typeInfo).getPrimitiveCategory()) {
		case STRING:
			return s1;
		case VARCHAR:
			return new HiveVarchar(s1, this.length);
			//return s1;
		}
		return null;
	}
 
Example #24
Source File: KuromojiUDFTest.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
public void testExpectedMode() throws UDFArgumentException, IOException {
    GenericUDF udf = new KuromojiUDF();
    ObjectInspector[] argOIs = new ObjectInspector[2];
    // line
    argOIs[0] = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    // mode
    PrimitiveTypeInfo stringType = new PrimitiveTypeInfo();
    stringType.setTypeName("string");
    argOIs[1] = PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
        stringType, new Text("normal"));
    udf.initialize(argOIs);
    udf.close();
}
 
Example #25
Source File: KuromojiUDFTest.java    From incubator-hivemall with Apache License 2.0 5 votes vote down vote up
@Test
public void testTwoArgument() throws UDFArgumentException, IOException {
    GenericUDF udf = new KuromojiUDF();
    ObjectInspector[] argOIs = new ObjectInspector[2];
    // line
    argOIs[0] = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    // mode
    PrimitiveTypeInfo stringType = new PrimitiveTypeInfo();
    stringType.setTypeName("string");
    argOIs[1] = PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
        stringType, null);
    udf.initialize(argOIs);
    udf.close();
}
 
Example #26
Source File: ConvertersTest.java    From HiveRunner with Apache License 2.0 5 votes vote down vote up
private void assertConversionException(Object value, PrimitiveTypeInfo typeInfo) {
  try {
    System.out.println(Converters.convert(value, typeInfo));
  } catch (ConversionException e) {
    return;
  }
  fail("Expected " + ConversionException.class.getSimpleName() + " for value " + value + " ("
      + value.getClass().getSimpleName() + ") to " + typeInfo.getTypeName());
}
 
Example #27
Source File: XmlObjectInspectorFactory.java    From Hive-XML-SerDe with Apache License 2.0 5 votes vote down vote up
/**
 * Returns the standard java object inspector
 * 
 * @param typeInfo
 *            the type info
 * @param xmlProcessor
 *            the XML processor
 * @return the standard java object inspector
 */
public static ObjectInspector getStandardJavaObjectInspectorFromTypeInfo(TypeInfo typeInfo, XmlProcessor xmlProcessor) {
    switch (typeInfo.getCategory()) {
        case PRIMITIVE: {
            return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory());
        }
        case LIST: {
            ObjectInspector listElementObjectInspector = getStandardJavaObjectInspectorFromTypeInfo(((ListTypeInfo) typeInfo).getListElementTypeInfo(),
                xmlProcessor);
            return new XmlListObjectInspector(listElementObjectInspector, xmlProcessor);
        }
        case MAP: {
            MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo;
            ObjectInspector mapKeyObjectInspector = getStandardJavaObjectInspectorFromTypeInfo(mapTypeInfo.getMapKeyTypeInfo(),
                xmlProcessor);
            ObjectInspector mapValueObjectInspector = getStandardJavaObjectInspectorFromTypeInfo(mapTypeInfo.getMapValueTypeInfo(),
                xmlProcessor);
            return new XmlMapObjectInspector(mapKeyObjectInspector, mapValueObjectInspector, xmlProcessor);
        }
        case STRUCT: {
            StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo;
            List<String> structFieldNames = structTypeInfo.getAllStructFieldNames();
            List<TypeInfo> fieldTypeInfos = structTypeInfo.getAllStructFieldTypeInfos();
            List<ObjectInspector> structFieldObjectInspectors = new ArrayList<ObjectInspector>(fieldTypeInfos.size());
            for (int fieldIndex = 0; fieldIndex < fieldTypeInfos.size(); ++fieldIndex) {
                structFieldObjectInspectors.add(getStandardJavaObjectInspectorFromTypeInfo(fieldTypeInfos.get(fieldIndex), xmlProcessor));
            }
            return getStandardStructObjectInspector(structFieldNames, structFieldObjectInspectors, xmlProcessor);
        }
        default: {
            throw new IllegalStateException();
        }
    }
}
 
Example #28
Source File: Converters.java    From HiveRunner with Apache License 2.0 5 votes vote down vote up
/**
 * Attempts to convert the input value into the target type. If the input value is {@code null} then {@code null} is
 * returned. If the input value is a String then an attempt is made to convert it into the target type. If the input
 * value is not a {@link String} then it is assumed the user has explicitly chosen the required type and no attempt is
 * made to perform a conversion. This may result in Hive throwing an error if the incorrect type was chosen.
 *
 * @param value The input value.
 * @param typeInfo The target Table's column type.
 */
public static Object convert(Object value, PrimitiveTypeInfo typeInfo) {
  if (value == null) {
    return null;
  }
  if (value instanceof String) {
    return CONVERTER.convert((String) value, type(typeInfo));
  }
  return value;
}
 
Example #29
Source File: EmoSerDe.java    From emodb with Apache License 2.0 5 votes vote down vote up
/**
 * Deserializes a raw value to the provided type.
 */
private Object deserialize(TypeInfo type, Object rawValue)
        throws SerDeException {
    Object value = null;

    if (rawValue != null) {
        switch (type.getCategory()) {
            case PRIMITIVE:
                value = deserializePrimitive((PrimitiveTypeInfo) type, rawValue);
                break;
            case STRUCT:
                value = deserializeStruct((StructTypeInfo) type, rawValue);
                break;
            case MAP:
                value = deserializeMap((MapTypeInfo) type, rawValue);
                break;
            case LIST:
                value = deserializeList((ListTypeInfo) type, rawValue);
                break;
            case UNION:
                value = deserializeUnion((UnionTypeInfo) type, rawValue);
                break;
        }
    }

    return value;
}
 
Example #30
Source File: HiveInspectors.java    From flink with Apache License 2.0 5 votes vote down vote up
private static ObjectInspector getObjectInspector(TypeInfo type) {
	switch (type.getCategory()) {

		case PRIMITIVE:
			PrimitiveTypeInfo primitiveType = (PrimitiveTypeInfo) type;
			return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(primitiveType);
		case LIST:
			ListTypeInfo listType = (ListTypeInfo) type;
			return ObjectInspectorFactory.getStandardListObjectInspector(
					getObjectInspector(listType.getListElementTypeInfo()));
		case MAP:
			MapTypeInfo mapType = (MapTypeInfo) type;
			return ObjectInspectorFactory.getStandardMapObjectInspector(
					getObjectInspector(mapType.getMapKeyTypeInfo()), getObjectInspector(mapType.getMapValueTypeInfo()));
		case STRUCT:
			StructTypeInfo structType = (StructTypeInfo) type;
			List<TypeInfo> fieldTypes = structType.getAllStructFieldTypeInfos();

			List<ObjectInspector> fieldInspectors = new ArrayList<ObjectInspector>();
			for (TypeInfo fieldType : fieldTypes) {
				fieldInspectors.add(getObjectInspector(fieldType));
			}

			return ObjectInspectorFactory.getStandardStructObjectInspector(
					structType.getAllStructFieldNames(), fieldInspectors);
		default:
			throw new CatalogException("Unsupported Hive type category " + type.getCategory());
	}
}