org.apache.hadoop.hive.common.type.HiveChar Java Examples

The following examples show how to use org.apache.hadoop.hive.common.type.HiveChar. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: HiveTypeUtil.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public TypeInfo visit(CharType charType) {
	// Flink and Hive have different length limit for CHAR. Promote it to STRING if it exceeds the limits of
	// Hive and we're told not to check precision. This can be useful when calling Hive UDF to process data.
	if (charType.getLength() > HiveChar.MAX_CHAR_LENGTH || charType.getLength() < 1) {
		if (checkPrecision) {
			throw new CatalogException(
					String.format("HiveCatalog doesn't support char type with length of '%d'. " +
									"The supported length is [%d, %d]",
							charType.getLength(), 1, HiveChar.MAX_CHAR_LENGTH));
		} else {
			return TypeInfoFactory.stringTypeInfo;
		}
	}
	return TypeInfoFactory.getCharTypeInfo(charType.getLength());
}
 
Example #2
Source File: HiveCatalogDataTypeTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testDataTypes() throws Exception {
	DataType[] types = new DataType[] {
		DataTypes.TINYINT(),
		DataTypes.SMALLINT(),
		DataTypes.INT(),
		DataTypes.BIGINT(),
		DataTypes.FLOAT(),
		DataTypes.DOUBLE(),
		DataTypes.BOOLEAN(),
		DataTypes.STRING(),
		DataTypes.BYTES(),
		DataTypes.DATE(),
		DataTypes.TIMESTAMP(9),
		DataTypes.CHAR(HiveChar.MAX_CHAR_LENGTH),
		DataTypes.VARCHAR(HiveVarchar.MAX_VARCHAR_LENGTH),
		DataTypes.DECIMAL(5, 3)
	};

	verifyDataTypes(types);
}
 
Example #3
Source File: HdfsSerDeImportService.java    From hadoop-etl-udfs with MIT License 6 votes vote down vote up
private static Object getJavaObjectFromFieldData(Object data, ObjectInspector objInsp) {
    if (data == null) {
        return null;
    }
    if (objInsp.getCategory() == Category.PRIMITIVE) {
        Object obj = ObjectInspectorUtils.copyToStandardJavaObject(data, objInsp);
        if (obj instanceof HiveDecimal) {
            obj = ((HiveDecimal) obj).bigDecimalValue();
        } else if (obj instanceof HiveVarchar || obj instanceof HiveChar) {
            obj = obj.toString();
        } else if (obj instanceof byte[]) {
            obj = Hex.encodeHexString((byte[]) obj);
        }
        return obj;
    } else if (objInsp.getCategory() == Category.LIST) {
        return getJsonArrayFromFieldData(data, objInsp, Json.createBuilderFactory(null)).build().toString();
    } else {
        return getJsonObjectFromFieldData(data, objInsp, Json.createBuilderFactory(null)).build().toString();
    }
}
 
Example #4
Source File: HdfsSerDeImportService.java    From hadoop-etl-udfs with MIT License 6 votes vote down vote up
private static Object getJavaObjectFromPrimitiveData(Object data, ObjectInspector objInsp) {
    assert(objInsp.getCategory() == Category.PRIMITIVE);
    if (data == null) {
        return null;
    }
    if (data instanceof BytesWritable && objInsp instanceof WritableHiveDecimalObjectInspector) {
        // BytesWritable cannot be directly cast to HiveDecimalWritable
        WritableHiveDecimalObjectInspector oi = (WritableHiveDecimalObjectInspector) objInsp;
        data = oi.create(((BytesWritable) data).getBytes(), oi.scale());
    }
    Object obj = ObjectInspectorUtils.copyToStandardJavaObject(data, objInsp);
    if (obj instanceof HiveDecimal) {
        obj = ((HiveDecimal) obj).bigDecimalValue();
    } else if (obj instanceof HiveVarchar || obj instanceof HiveChar) {
        obj = obj.toString();
    } else if (obj instanceof byte[]) {
        obj = Hex.encodeHexString((byte[]) obj);
    }
    return obj;
}
 
Example #5
Source File: HiveCatalogDataTypeTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testDataTypes() throws Exception {
	DataType[] types = new DataType[] {
		DataTypes.TINYINT(),
		DataTypes.SMALLINT(),
		DataTypes.INT(),
		DataTypes.BIGINT(),
		DataTypes.FLOAT(),
		DataTypes.DOUBLE(),
		DataTypes.BOOLEAN(),
		DataTypes.STRING(),
		DataTypes.BYTES(),
		DataTypes.DATE(),
		DataTypes.TIMESTAMP(),
		DataTypes.CHAR(HiveChar.MAX_CHAR_LENGTH),
		DataTypes.VARCHAR(HiveVarchar.MAX_VARCHAR_LENGTH),
		DataTypes.DECIMAL(5, 3)
	};

	verifyDataTypes(types);
}
 
Example #6
Source File: HCatalogExportTest.java    From aliyun-maxcompute-data-collectors with Apache License 2.0 6 votes vote down vote up
public void testStringTypes() throws Exception {
  final int TOTAL_RECORDS = 1 * 10;
  String table = getTableName().toUpperCase();
  ColumnGenerator[] cols = new ColumnGenerator[] {
    HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
      "char(14)", Types.CHAR, HCatFieldSchema.Type.STRING, 0, 0,
      "string to test", "string to test", KeyType.NOT_A_KEY),
    HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1),
        "char(14)", Types.CHAR, HCatFieldSchema.Type.CHAR, 14, 0,
        new HiveChar("string to test", 14), "string to test",
        KeyType.NOT_A_KEY),
    HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(2),
        "char(14)", Types.CHAR, HCatFieldSchema.Type.VARCHAR, 14, 0,
        new HiveVarchar("string to test", 14), "string to test",
        KeyType.NOT_A_KEY),
    HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(3),
      "longvarchar", Types.LONGVARCHAR, HCatFieldSchema.Type.STRING, 0, 0,
      "string to test", "string to test", KeyType.NOT_A_KEY),
  };
  List<String> addlArgsArray = new ArrayList<String>();
  runHCatExport(addlArgsArray, TOTAL_RECORDS, table, cols);
}
 
Example #7
Source File: HCatalogImportTest.java    From aliyun-maxcompute-data-collectors with Apache License 2.0 6 votes vote down vote up
public void testStringTypes() throws Exception {
  final int TOTAL_RECORDS = 1 * 10;
  String table = getTableName().toUpperCase();
  ColumnGenerator[] cols = new ColumnGenerator[] {
    HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
      "char(14)", Types.CHAR, HCatFieldSchema.Type.STRING, 0, 0,
      "string to test", "string to test", KeyType.NOT_A_KEY),
    HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1),
        "char(14)", Types.CHAR, HCatFieldSchema.Type.CHAR, 14, 0,
        new HiveChar("string to test", 14), "string to test",
        KeyType.NOT_A_KEY),
    HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(2),
        "char(14)", Types.CHAR, HCatFieldSchema.Type.VARCHAR, 14, 0,
        new HiveVarchar("string to test", 14), "string to test",
        KeyType.NOT_A_KEY),
    HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(3),
      "longvarchar", Types.LONGVARCHAR, HCatFieldSchema.Type.STRING, 0, 0,
      "string to test", "string to test", KeyType.NOT_A_KEY),
  };
  List<String> addlArgsArray = new ArrayList<String>();
  setExtraArgs(addlArgsArray);
  runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
}
 
Example #8
Source File: HiveORCSearchArgumentBuilder.java    From pxf with Apache License 2.0 6 votes vote down vote up
private static Object boxLiteral(Object literal) {
    if (literal instanceof String ||
            literal instanceof Long ||
            literal instanceof Double ||
            literal instanceof Date ||
            literal instanceof Timestamp ||
            literal instanceof HiveDecimal ||
            literal instanceof BigDecimal ||
            literal instanceof Boolean) {
        return literal;
    } else if (literal instanceof HiveChar ||
            literal instanceof HiveVarchar) {
        return StringUtils.stripEnd(literal.toString(), null);
    } else if (literal instanceof Byte ||
            literal instanceof Short ||
            literal instanceof Integer) {
        return ((Number) literal).longValue();
    } else if (literal instanceof Float) {
        // to avoid change in precision when upcasting float to double
        // we convert the literal to string and parse it as double. (HIVE-8460)
        return Double.parseDouble(literal.toString());
    } else {
        throw new IllegalArgumentException("Unknown type for literal " +
                literal);
    }
}
 
Example #9
Source File: SqoopHCatImportHelper.java    From aliyun-maxcompute-data-collectors with Apache License 2.0 6 votes vote down vote up
private Object convertClobType(Object val, HCatFieldSchema hfs) {
  HCatFieldSchema.Type hfsType = hfs.getType();
  ClobRef cr = (ClobRef) val;
  String s = cr.isExternal() ? cr.toString() : cr.getData();

  if (hfsType == HCatFieldSchema.Type.STRING) {
    return s;
  } else if (hfsType == HCatFieldSchema.Type.VARCHAR) {
    VarcharTypeInfo vti = (VarcharTypeInfo) hfs.getTypeInfo();
    HiveVarchar hvc = new HiveVarchar(s, vti.getLength());
    return hvc;
  } else if (hfsType == HCatFieldSchema.Type.CHAR) {
    CharTypeInfo cti = (CharTypeInfo) hfs.getTypeInfo();
    HiveChar hc = new HiveChar(s, cti.getLength());
    return hc;
  }
  return null;
}
 
Example #10
Source File: DataToStringsSketchUDAF.java    From incubator-datasketches-hive with Apache License 2.0 5 votes vote down vote up
@Override
public String extractValue(final Object data, final ObjectInspector objectInspector)
    throws HiveException {
  final Object value = inputObjectInspector.getPrimitiveJavaObject(data);
  if (value instanceof String) {
    return (String) value;
  } else if (value instanceof HiveChar) {
    return ((HiveChar) value).getValue();
  } else if (value instanceof HiveVarchar) {
    return ((HiveVarchar) value).getValue();
  } else {
    throw new UDFArgumentTypeException(0, "unsupported type " + value.getClass().getName());
  }
}
 
Example #11
Source File: DataToStringsSketchUDAF.java    From incubator-datasketches-hive with Apache License 2.0 5 votes vote down vote up
@Override
public String extractValue(final Object data, final ObjectInspector objectInspector)
    throws HiveException {
  final Object value = inputObjectInspector.getPrimitiveJavaObject(data);
  if (value instanceof String) {
    return (String) value;
  } else if (value instanceof HiveChar) {
    return ((HiveChar) value).getValue();
  } else if (value instanceof HiveVarchar) {
    return ((HiveVarchar) value).getValue();
  } else {
    throw new UDFArgumentTypeException(0, "unsupported type " + value.getClass().getName());
  }
}
 
Example #12
Source File: WritableTypeConverter.java    From tajo with Apache License 2.0 5 votes vote down vote up
public static Writable convertDatum2Writable(Datum value) {
  switch(value.kind()) {
    case INT1: return new ByteWritable(value.asByte());
    case INT2: return new ShortWritable(value.asInt2());
    case INT4: return new IntWritable(value.asInt4());
    case INT8: return new LongWritable(value.asInt8());

    case FLOAT4: return new FloatWritable(value.asFloat4());
    case FLOAT8: return new DoubleWritable(value.asFloat8());

    // NOTE: value should be DateDatum
    case DATE: return new DateWritable(value.asInt4() - DateTimeConstants.UNIX_EPOCH_JDATE);

    // NOTE: value should be TimestampDatum
    case TIMESTAMP:
      TimestampWritable result = new TimestampWritable();
      result.setTime(DateTimeUtil.julianTimeToJavaTime(value.asInt8()));
      return result;

    case CHAR: {
      String str = value.asChars();
      return new HiveCharWritable(new HiveChar(str, str.length()));
    }
    case TEXT: return new Text(value.asChars());
    case VARBINARY: return new BytesWritable(value.asByteArray());

    case NULL_TYPE: return null;
  }

  throw new TajoRuntimeException(new NotImplementedException(TypeStringEncoder.encode(value.type())));
}
 
Example #13
Source File: HiveCatalogDataTypeTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testCharTypeLength() throws Exception {
	DataType[] types = new DataType[] {
		DataTypes.CHAR(HiveChar.MAX_CHAR_LENGTH + 1)
	};

	exception.expect(CatalogException.class);
	verifyDataTypes(types);
}
 
Example #14
Source File: SqoopHCatImportHelper.java    From aliyun-maxcompute-data-collectors with Apache License 2.0 5 votes vote down vote up
private Object convertBooleanTypes(Object val, HCatFieldSchema hfs) {
  HCatFieldSchema.Type hfsType = hfs.getType();
  Boolean b = (Boolean) val;
  if (hfsType == HCatFieldSchema.Type.BOOLEAN) {
    return b;
  } else if (hfsType == HCatFieldSchema.Type.TINYINT) {
    return (byte) (b ? 1 : 0);
  } else if (hfsType == HCatFieldSchema.Type.SMALLINT) {
    return (short) (b ? 1 : 0);
  } else if (hfsType == HCatFieldSchema.Type.INT) {
    return (int) (b ? 1 : 0);
  } else if (hfsType == HCatFieldSchema.Type.BIGINT) {
    return (long) (b ? 1 : 0);
  } else if (hfsType == HCatFieldSchema.Type.FLOAT) {
    return (float) (b ? 1 : 0);
  } else if (hfsType == HCatFieldSchema.Type.DOUBLE) {
    return (double) (b ? 1 : 0);
  } else if (hfsType == HCatFieldSchema.Type.STRING) {
    return val.toString();
  } else if (hfsType == HCatFieldSchema.Type.VARCHAR) {
    VarcharTypeInfo vti = (VarcharTypeInfo) hfs.getTypeInfo();
    HiveVarchar hvc = new HiveVarchar(val.toString(), vti.getLength());
    return hvc;
  } else if (hfsType == HCatFieldSchema.Type.CHAR) {
    CharTypeInfo cti = (CharTypeInfo) hfs.getTypeInfo();
    HiveChar hChar = new HiveChar(val.toString(), cti.getLength());
    return hChar;
  }
  return null;
}
 
Example #15
Source File: SqoopHCatImportHelper.java    From aliyun-maxcompute-data-collectors with Apache License 2.0 5 votes vote down vote up
private Object convertStringTypes(Object val, HCatFieldSchema hfs) {
  HCatFieldSchema.Type hfsType = hfs.getType();
  if (hfsType == HCatFieldSchema.Type.STRING
      || hfsType == HCatFieldSchema.Type.VARCHAR
      || hfsType == HCatFieldSchema.Type.CHAR) {
    String str = val.toString();
    if (doHiveDelimsReplacement) {
      str = FieldFormatter.hiveStringReplaceDelims(str,
        hiveDelimsReplacement, hiveDelimiters);
    }
    if (hfsType == HCatFieldSchema.Type.STRING) {
      return str;
    } else if (hfsType == HCatFieldSchema.Type.VARCHAR) {
      VarcharTypeInfo vti = (VarcharTypeInfo) hfs.getTypeInfo();
      HiveVarchar hvc = new HiveVarchar(str, vti.getLength());
      return hvc;
    } else if (hfsType == HCatFieldSchema.Type.CHAR) {
      CharTypeInfo cti = (CharTypeInfo) hfs.getTypeInfo();
      HiveChar hc = new HiveChar(val.toString(), cti.getLength());
      return hc;
    }
  } else if (hfsType == HCatFieldSchema.Type.DECIMAL) {
    BigDecimal bd = new BigDecimal(val.toString(), MathContext.DECIMAL128);
    HiveDecimal hd = HiveDecimal.create(bd);
    return hd;
  }
  return null;
}
 
Example #16
Source File: HiveORCSearchArgumentBuilder.java    From pxf with Apache License 2.0 5 votes vote down vote up
/**
 * Get the type of the given expression node.
 *
 * @param literal the object
 * @return int, string, or float or null if we don't know the type
 */
private PredicateLeaf.Type getType(Object literal) {
    if (literal instanceof Byte ||
            literal instanceof Short ||
            literal instanceof Integer ||
            literal instanceof Long) {
        return PredicateLeaf.Type.LONG;
    } else if (literal instanceof HiveChar ||
            literal instanceof HiveVarchar ||
            literal instanceof String) {
        return PredicateLeaf.Type.STRING;
    } else if (literal instanceof Float ||
            literal instanceof Double) {
        return PredicateLeaf.Type.FLOAT;
    } else if (literal instanceof Date) {
        return PredicateLeaf.Type.DATE;
    } else if (literal instanceof Timestamp) {
        return PredicateLeaf.Type.TIMESTAMP;
    } else if (literal instanceof HiveDecimal ||
            literal instanceof BigDecimal) {
        return PredicateLeaf.Type.DECIMAL;
    } else if (literal instanceof Boolean) {
        return PredicateLeaf.Type.BOOLEAN;
    } else if (literal instanceof List) {
        @SuppressWarnings("unchecked")
        List<Object> l = (List<Object>) literal;
        if (l.size() > 0)
            return getType(l.get(0));
    }
    throw new IllegalArgumentException(String.format("Unknown type for literal %s", literal));
}
 
Example #17
Source File: CacheablePrimitiveObjectInspectorConverter.java    From transport with BSD 2-Clause "Simplified" License 5 votes vote down vote up
@Override
public Object convert(Object input) {
  if (input == null) {
    return null;
  }
  HiveCharWritable hc = new HiveCharWritable();
  switch (inputOI.getPrimitiveCategory()) {
    case BOOLEAN:
      return outputOI.set(hc,
          ((BooleanObjectInspector) inputOI).get(input)
              ? new HiveChar("TRUE", -1) : new HiveChar("FALSE", -1));
    default:
      return outputOI.set(hc, PrimitiveObjectInspectorUtils.getHiveChar(input, inputOI));
  }
}
 
Example #18
Source File: HiveCatalogDataTypeTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testCharTypeLength() throws Exception {
	DataType[] types = new DataType[] {
		DataTypes.CHAR(HiveChar.MAX_CHAR_LENGTH + 1)
	};

	exception.expect(CatalogException.class);
	exception.expectMessage("HiveCatalog doesn't support char type with length of '256'. The maximum length is 255");
	verifyDataTypes(types);
}
 
Example #19
Source File: HiveInspectors.java    From flink with Apache License 2.0 5 votes vote down vote up
private static ConstantObjectInspector getPrimitiveJavaConstantObjectInspector(PrimitiveTypeInfo typeInfo, Object value) {
	switch (typeInfo.getPrimitiveCategory()) {
		case BOOLEAN:
			return new JavaConstantBooleanObjectInspector((Boolean) value);
		case BYTE:
			return new JavaConstantByteObjectInspector((Byte) value);
		case SHORT:
			return new JavaConstantShortObjectInspector((Short) value);
		case INT:
			return new JavaConstantIntObjectInspector((Integer) value);
		case LONG:
			return new JavaConstantLongObjectInspector((Long) value);
		case FLOAT:
			return new JavaConstantFloatObjectInspector((Float) value);
		case DOUBLE:
			return new JavaConstantDoubleObjectInspector((Double) value);
		case STRING:
			return new JavaConstantStringObjectInspector((String) value);
		case CHAR:
			return new JavaConstantHiveCharObjectInspector((HiveChar) value);
		case VARCHAR:
			return new JavaConstantHiveVarcharObjectInspector((HiveVarchar) value);
		case DATE:
			return new JavaConstantDateObjectInspector((Date) value);
		case TIMESTAMP:
			return new JavaConstantTimestampObjectInspector((Timestamp) value);
		case DECIMAL:
			return new JavaConstantHiveDecimalObjectInspector((HiveDecimal) value);
		case BINARY:
			return new JavaConstantBinaryObjectInspector((byte[]) value);
		case UNKNOWN:
		case VOID:
			// If type is null, we use the Java Constant String to replace
			return new JavaConstantStringObjectInspector((String) value);
		default:
			throw new FlinkHiveUDFException(
				String.format("Cannot find ConstantObjectInspector for %s", typeInfo));
	}
}
 
Example #20
Source File: HiveTypeUtil.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public TypeInfo visit(CharType charType) {
	if (charType.getLength() > HiveChar.MAX_CHAR_LENGTH) {
		throw new CatalogException(
				String.format("HiveCatalog doesn't support char type with length of '%d'. " +
							"The maximum length is %d",
							charType.getLength(), HiveChar.MAX_CHAR_LENGTH));
	}
	return TypeInfoFactory.getCharTypeInfo(charType.getLength());
}
 
Example #21
Source File: ConvertersTest.java    From HiveRunner with Apache License 2.0 5 votes vote down vote up
@Test
public void otherTypeInfo() {
  assertEquals(HiveDecimal.create("1.234"), Converters.convert("1.234", decimalTypeInfo));
  assertEquals(new HiveChar("foo", -1), Converters.convert("foo", charTypeInfo));
  assertTrue(new HiveVarchar("foo", -1).equals((HiveVarchar) Converters.convert("foo", varcharTypeInfo)));
  assertEquals("foo", Converters.convert("foo", unknownTypeInfo));
  assertEquals("foo", Converters.convert("foo", voidTypeInfo));
}
 
Example #22
Source File: Converters.java    From HiveRunner with Apache License 2.0 4 votes vote down vote up
@Override
public Object convert(@SuppressWarnings("rawtypes") Class type, Object value) {
  return new HiveChar(value.toString(), -1);
}
 
Example #23
Source File: SMSerDe.java    From spliceengine with GNU Affero General Public License v3.0 4 votes vote down vote up
/**
   * This method takes an object representing a row of data from Hive, and
   * uses the ObjectInspector to get the data for each column and serialize
   * it.
   */
  //@Override
  public Writable serialize(Object obj, ObjectInspector oi)
          throws SerDeException {
  	ExecRow row = null;
  	int[] execRowFormatIds = null;
      try {
	List<NameType> nameTypes = sqlUtil.getTableStructure(tableName);
	execRowFormatIds = sqlUtil.getExecRowFormatIds(colNames, nameTypes);
	row = sqlUtil.getExecRow(execRowFormatIds);
	if (row == null)
		throw new SerDeException("ExecRow Cannot be Null");
} catch (SQLException | StandardException | IOException e1) {
	throw new SerDeException(e1);
}    	
  	if (Log.isTraceEnabled())
  		SpliceLogUtils.trace(Log, "serialize with obj=%s, oi=%s",obj,oi);
      if (oi.getCategory() != ObjectInspector.Category.STRUCT) {
          throw new SerDeException(getClass().toString()
                  + " can only serialize struct types, but we got: "
                  + oi.getTypeName());
      }

      StructObjectInspector soi = (StructObjectInspector) oi;
      List<? extends StructField> fields = soi.getAllStructFieldRefs();

      try {

      	DataValueDescriptor dvd;
          for (int i = 0; i < fields.size(); i++) {
              StructField field = fields.get(i);
              dvd = row.getColumn(i+1);
              ObjectInspector fieldOI = field.getFieldObjectInspector();
              Object fieldObj = soi.getStructFieldData(obj, field);
              PrimitiveObjectInspector primOI = (PrimitiveObjectInspector) fieldOI;
              Object data = primOI.getPrimitiveJavaObject(fieldObj);

              PrimitiveCategory primitiveCategory = primOI.getPrimitiveCategory();
              switch (primitiveCategory) {
                  case BYTE:
                      dvd.setValue(((Byte) data).byteValue());
                      break;
                  case INT:
                      dvd.setValue(((Integer) data).intValue());
                      break;
                  case VARCHAR:
                      dvd.setValue(((HiveVarchar)data).getValue());
                      break;
                  case CHAR:
                      dvd.setValue(((HiveChar)data).getValue());
                      break;
                  case STRING:
                      dvd.setValue((String) data);
                      break;
                  case BINARY:
                      dvd.setValue((SerializationUtils.serialize((Serializable) data))); // is this right?  Should just be a byte[]
                      break;
                  case BOOLEAN:
                      dvd.setValue(((Boolean) data).booleanValue());
                      break;
                  case DECIMAL:
                      dvd.setValue(((HiveDecimal) data).doubleValue());
                      break;
                  case DOUBLE:
                      dvd.setValue(((Double) data).doubleValue());
                      break;
                  case FLOAT:
                      dvd.setValue(((Float) data).floatValue());
                      break;
                  case LONG:
                      dvd.setValue(((Long) data).longValue());
                      break;
                  case SHORT:
                      dvd.setValue(((Short) data).shortValue());
                      break;
                  case TIMESTAMP:
                      dvd.setValue((Timestamp) data);
                      break;
                  case DATE:
                      dvd.setValue((java.sql.Date) data);
                      break;
                  default:
                      throw new SerDeException(String.format("Hive Type %s Not Supported Yet",primOI.getPrimitiveCategory()));
              }
          }

      } catch (StandardException e) {
          // TODO Auto-generated catch block
          throw new RuntimeException("Serialized Object To Java Type Error");
      }
      ExecRowWritable rowWritable = new ExecRowWritable(WriteReadUtils.getExecRowFromTypeFormatIds(execRowFormatIds));
      rowWritable.set(row);
      return rowWritable;
  }
 
Example #24
Source File: OrcTester.java    From spliceengine with GNU Affero General Public License v3.0 4 votes vote down vote up
private static Object preprocessWriteValueOld(TypeInfo typeInfo, Object value) throws IOException
    {
        if (value == null) {
            return null;
        }
        switch (typeInfo.getCategory()) {
            case PRIMITIVE:
                PrimitiveObjectInspector.PrimitiveCategory primitiveCategory = ((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory();
                switch (primitiveCategory) {
                    case BOOLEAN:
                        return value;
                    case BYTE:
                        return ((Number) value).byteValue();
                    case SHORT:
                        return ((Number) value).shortValue();
                    case INT:
                        return ((Number) value).intValue();
                    case LONG:
                        return ((Number) value).longValue();
                    case FLOAT:
                        return ((Number) value).floatValue();
                    case DOUBLE:
                        return ((Number) value).doubleValue();
                    case DECIMAL:
                        return HiveDecimal.create(((Decimal) value).toBigDecimal().bigDecimal());
                    case STRING:
                        return value;
                    case CHAR:
                        return new HiveChar(value.toString(), ((CharTypeInfo) typeInfo).getLength());
                    case DATE:
                        LocalDate localDate = LocalDate.ofEpochDay((int)value);
                        ZonedDateTime zonedDateTime = localDate.atStartOfDay(ZoneId.systemDefault());

                        long millis = zonedDateTime.toEpochSecond() * 1000;
                        Date date = new Date(0);
                        // mills must be set separately to avoid masking
                        date.setTime(millis);
                        return date;
                    case TIMESTAMP:
                        long millisUtc = ((Long)value).intValue();
                        return new Timestamp(millisUtc);
                    case BINARY:
                        return ((String) value).getBytes();
//                        return (byte[])value;
                }
                break;
            case MAP:
                MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo;
                TypeInfo keyTypeInfo = mapTypeInfo.getMapKeyTypeInfo();
                TypeInfo valueTypeInfo = mapTypeInfo.getMapValueTypeInfo();
                Map<Object, Object> newMap = new HashMap<>();
                for (Entry<?, ?> entry : ((Map<?, ?>) value).entrySet()) {
                    newMap.put(preprocessWriteValueOld(keyTypeInfo, entry.getKey()), preprocessWriteValueOld(valueTypeInfo, entry.getValue()));
                }
                return newMap;
            case LIST:
                ListTypeInfo listTypeInfo = (ListTypeInfo) typeInfo;
                TypeInfo elementTypeInfo = listTypeInfo.getListElementTypeInfo();
                List<Object> newList = new ArrayList<>(((Collection<?>) value).size());
                for (Object element : (Iterable<?>) value) {
                    newList.add(preprocessWriteValueOld(elementTypeInfo, element));
                }
                return newList;
            case STRUCT:
                StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo;
                List<?> fieldValues = (List<?>) value;
                List<TypeInfo> fieldTypeInfos = structTypeInfo.getAllStructFieldTypeInfos();
                List<Object> newStruct = new ArrayList<>();
                for (int fieldId = 0; fieldId < fieldValues.size(); fieldId++) {
                    newStruct.add(preprocessWriteValueOld(fieldTypeInfos.get(fieldId), fieldValues.get(fieldId)));
                }
                return newStruct;
        }
        throw new IOException(format("Unsupported Hive type: %s", typeInfo));
    }
 
Example #25
Source File: HiveTestUDFImpls.java    From dremio-oss with Apache License 2.0 4 votes vote down vote up
@Override
public Object evaluate(DeferredObject[] arguments) throws HiveException {
  if (arguments[0] == null || arguments[0].get() == null) {
    return null;
  }

  Object input = arguments[0].get();
  switch(inputType) {
    case BOOLEAN:
      return ((BooleanObjectInspector)argumentOI).get(input) ? Boolean.TRUE : Boolean.FALSE;
    case BYTE:
      return new Byte(((ByteObjectInspector)argumentOI).get(input));
    case SHORT:
      return new Short(((ShortObjectInspector)argumentOI).get(input));
    case INT:
      return new Integer(((IntObjectInspector)argumentOI).get(input));
    case LONG:
      return new Long(((LongObjectInspector)argumentOI).get(input));
    case FLOAT:
      return new Float(((FloatObjectInspector)argumentOI).get(input));
    case DOUBLE:
      return new Double(((DoubleObjectInspector)argumentOI).get(input));
    case STRING:
      return PrimitiveObjectInspectorUtils.getString(input, (StringObjectInspector)argumentOI);
    case BINARY:
      return PrimitiveObjectInspectorUtils.getBinary(input, (BinaryObjectInspector) argumentOI).getBytes();
    case VARCHAR:
      if (outputType == PrimitiveCategory.CHAR) {
        HiveVarchar hiveVarchar = PrimitiveObjectInspectorUtils.getHiveVarchar(input, (HiveVarcharObjectInspector) argumentOI);
        return new HiveChar(hiveVarchar.getValue(), HiveChar.MAX_CHAR_LENGTH);
      } else {
        return PrimitiveObjectInspectorUtils.getHiveVarchar(input, (HiveVarcharObjectInspector)argumentOI);
      }
    case CHAR:
      return PrimitiveObjectInspectorUtils.getHiveChar(input, (HiveCharObjectInspector) argumentOI);
    case DATE:
      return PrimitiveObjectInspectorUtils.getDate(input, (DateObjectInspector) argumentOI);
    case TIMESTAMP:
      return PrimitiveObjectInspectorUtils.getTimestamp(input, (TimestampObjectInspector) argumentOI);
    case DECIMAL:
      // return type is a HiveVarchar
      HiveDecimal decimalValue =
          PrimitiveObjectInspectorUtils.getHiveDecimal(input, (HiveDecimalObjectInspector) argumentOI);
      return new HiveVarchar(decimalValue.toString(), HiveVarchar.MAX_VARCHAR_LENGTH);
  }

  throw new UnsupportedOperationException(String.format("Unexpected input type '%s' in Test UDF", inputType));
}
 
Example #26
Source File: JsonSerdeUtilsTest.java    From incubator-hivemall with Apache License 2.0 4 votes vote down vote up
@Test
public void testRW() throws Exception {
    List<Object> rlist = new ArrayList<Object>(13);
    {
        rlist.add(new Byte("123"));
        rlist.add(new Short("456"));
        rlist.add(new Integer(789));
        rlist.add(new Long(1000L));
        rlist.add(new Double(5.3D));
        rlist.add(new Float(2.39F));
        rlist.add(new String("hcat\nand\nhadoop"));
        rlist.add(null);

        List<Object> innerStruct = new ArrayList<Object>(2);
        innerStruct.add(new String("abc"));
        innerStruct.add(new String("def"));
        rlist.add(innerStruct);

        List<Integer> innerList = new ArrayList<Integer>();
        innerList.add(314);
        innerList.add(007);
        rlist.add(innerList);

        Map<Short, String> map = new HashMap<Short, String>(3);
        map.put(new Short("2"), "hcat is cool");
        map.put(new Short("3"), "is it?");
        map.put(new Short("4"), "or is it not?");
        rlist.add(map);

        rlist.add(new Boolean(true));

        List<Object> c1 = new ArrayList<Object>();
        List<Object> c1_1 = new ArrayList<Object>();
        c1_1.add(new Integer(12));
        List<Object> i2 = new ArrayList<Object>();
        List<Integer> ii1 = new ArrayList<Integer>();
        ii1.add(new Integer(13));
        ii1.add(new Integer(14));
        i2.add(ii1);
        Map<String, List<?>> ii2 = new HashMap<String, List<?>>();
        List<Integer> iii1 = new ArrayList<Integer>();
        iii1.add(new Integer(15));
        ii2.put("phew", iii1);
        i2.add(ii2);
        c1_1.add(i2);
        c1.add(c1_1);
        rlist.add(c1);
        rlist.add(HiveDecimal.create(new BigDecimal("123.45")));//prec 5, scale 2
        rlist.add(new HiveChar("hive\nchar", 10));
        rlist.add(new HiveVarchar("hive\nvarchar", 20));
        rlist.add(Date.valueOf("2014-01-07"));
        rlist.add(new Timestamp(System.currentTimeMillis()));
        rlist.add("hive\nbinary".getBytes("UTF-8"));
    }

    DefaultHCatRecord r = new DefaultHCatRecord(rlist);

    List<String> columnNames =
            Arrays.asList("ti,si,i,bi,d,f,s,n,r,l,m,b,c1,bd,hc,hvc,dt,ts,bin".split(","));
    List<TypeInfo> columnTypes = TypeInfoUtils.getTypeInfosFromTypeString(
        "tinyint,smallint,int,bigint,double,float,string,string,"
                + "struct<a:string,b:string>,array<int>,map<smallint,string>,boolean,"
                + "array<struct<i1:int,i2:struct<ii1:array<int>,ii2:map<string,struct<iii1:int>>>>>,"
                + "decimal(5,2),char(10),varchar(20),date,timestamp,binary");

    StructTypeInfo rowTypeInfo =
            (StructTypeInfo) TypeInfoFactory.getStructTypeInfo(columnNames, columnTypes);
    HCatRecordObjectInspector objInspector =
            HCatRecordObjectInspectorFactory.getHCatRecordObjectInspector(rowTypeInfo);

    Text serialized = JsonSerdeUtils.serialize(r, objInspector, columnNames);
    List<Object> deserialized =
            JsonSerdeUtils.deserialize(serialized, columnNames, columnTypes);

    assertRecordEquals(rlist, deserialized);
}
 
Example #27
Source File: HiveJsonStructReader.java    From incubator-hivemall with Apache License 2.0 4 votes vote down vote up
private Object getObjectOfCorrespondingPrimitiveType(String s, PrimitiveObjectInspector oi)
        throws IOException {
    PrimitiveTypeInfo typeInfo = oi.getTypeInfo();
    if (writeablePrimitives) {
        Converter c = ObjectInspectorConverters.getConverter(
            PrimitiveObjectInspectorFactory.javaStringObjectInspector, oi);
        return c.convert(s);
    }

    switch (typeInfo.getPrimitiveCategory()) {
        case INT:
            return Integer.valueOf(s);
        case BYTE:
            return Byte.valueOf(s);
        case SHORT:
            return Short.valueOf(s);
        case LONG:
            return Long.valueOf(s);
        case BOOLEAN:
            return (s.equalsIgnoreCase("true"));
        case FLOAT:
            return Float.valueOf(s);
        case DOUBLE:
            return Double.valueOf(s);
        case STRING:
            return s;
        case BINARY:
            try {
                String t = Text.decode(s.getBytes(), 0, s.getBytes().length);
                return t.getBytes();
            } catch (CharacterCodingException e) {
                LOG.warn("Error generating json binary type from object.", e);
                return null;
            }
        case DATE:
            return Date.valueOf(s);
        case TIMESTAMP:
            return Timestamp.valueOf(s);
        case DECIMAL:
            return HiveDecimal.create(s);
        case VARCHAR:
            return new HiveVarchar(s, ((BaseCharTypeInfo) typeInfo).getLength());
        case CHAR:
            return new HiveChar(s, ((BaseCharTypeInfo) typeInfo).getLength());
        default:
            throw new IOException(
                "Could not convert from string to " + typeInfo.getPrimitiveCategory());
    }
}
 
Example #28
Source File: JsonSerdeUtils.java    From incubator-hivemall with Apache License 2.0 4 votes vote down vote up
@Nonnull
private static Object getObjectOfCorrespondingPrimitiveType(String s,
        PrimitiveTypeInfo mapKeyType) throws IOException {
    switch (Type.getPrimitiveHType(mapKeyType)) {
        case INT:
            return Integer.valueOf(s);
        case TINYINT:
            return Byte.valueOf(s);
        case SMALLINT:
            return Short.valueOf(s);
        case BIGINT:
            return Long.valueOf(s);
        case BOOLEAN:
            return (s.equalsIgnoreCase("true"));
        case FLOAT:
            return Float.valueOf(s);
        case DOUBLE:
            return Double.valueOf(s);
        case STRING:
            return s;
        case BINARY:
            try {
                String t = Text.decode(s.getBytes(), 0, s.getBytes().length);
                return t.getBytes();
            } catch (CharacterCodingException e) {
                throw new IOException("Error generating json binary type from object.", e);
            }
        case DATE:
            return Date.valueOf(s);
        case TIMESTAMP:
            return Timestamp.valueOf(s);
        case DECIMAL:
            return HiveDecimal.create(s);
        case VARCHAR:
            return new HiveVarchar(s, ((BaseCharTypeInfo) mapKeyType).getLength());
        case CHAR:
            return new HiveChar(s, ((BaseCharTypeInfo) mapKeyType).getLength());
        default:
            throw new IOException(
                "Could not convert from string to map type " + mapKeyType.getTypeName());
    }
}
 
Example #29
Source File: ExcelSerde.java    From hadoopoffice with Apache License 2.0 4 votes vote down vote up
/**
 * Deserializes an object of type @see #getSerializedClass()
 * Note: Some Java types, such as Decimal, are converted to Hive specific datatypes. 
 * 
 * @param arg0 object of type @see #getSerializedClass()
 * @return Array containing objects of type primitive Java (e.g. string, byte, integer)/Hive (e.g HiveDecimal, HiveVarChar)
 * 
 */
@Override
public Object deserialize(Writable arg0) throws SerDeException {
	if ((arg0 == null) || (arg0 instanceof NullWritable)) {
		return this.nullRow;
	}
	Object[] primitiveRow = this.readConverter
			.getDataAccordingToSchema((SpreadSheetCellDAO[]) ((ArrayWritable) arg0).get());
	// check if supported type and convert to hive type, if necessary
	for (int i = 0; i < primitiveRow.length; i++) {
		PrimitiveTypeInfo ti = (PrimitiveTypeInfo) this.columnTypes.get(i);
		switch (ti.getPrimitiveCategory()) {
		case STRING:
			primitiveRow[i] = primitiveRow[i];
			break;
		case BYTE:
			primitiveRow[i] = primitiveRow[i];
			break;
		case SHORT:
			primitiveRow[i] = primitiveRow[i];
			break;
		case INT:
			primitiveRow[i] = primitiveRow[i];
			break;
		case LONG:
			primitiveRow[i] = primitiveRow[i];
			break;
		case FLOAT:
			primitiveRow[i] = primitiveRow[i];
			break;
		case DOUBLE:
			primitiveRow[i] = primitiveRow[i];
			break;
		case BOOLEAN:
			primitiveRow[i] = primitiveRow[i];
			break;
		case TIMESTAMP:
			primitiveRow[i] = primitiveRow[i];
			break;
		case DATE:
			if (primitiveRow[i] != null) {
				primitiveRow[i] = new java.sql.Date(((Date) primitiveRow[i]).getTime());
			}
			break;
		case DECIMAL:
			if (primitiveRow[i] != null) {
				primitiveRow[i] = HiveDecimal.create((BigDecimal) primitiveRow[i]);
			}
			break;
		case CHAR:
			if (primitiveRow[i] != null) {
				primitiveRow[i] = new HiveChar((String) primitiveRow[i], ((CharTypeInfo) ti).getLength());
			}
			break;
		case VARCHAR:
			if (primitiveRow[i] != null) {
				primitiveRow[i] = new HiveVarchar((String) primitiveRow[i], ((VarcharTypeInfo) ti).getLength());
			}
			break;
		default:
			throw new SerDeException("Unsupported type " + ti);
		}
	}
	if (this.columnNames.size()>primitiveRow.length) { // can happen in rare cases where a row does not contain all columns
		Object[] tempRow = new Object[this.columnNames.size()];
		for (int i=0;i<primitiveRow.length;i++) {
			tempRow[i]=primitiveRow[i];
		}
		primitiveRow=tempRow;
	}
	return primitiveRow;
}
 
Example #30
Source File: OrcTester.java    From presto with Apache License 2.0 4 votes vote down vote up
private static Object preprocessWriteValueHive(Type type, Object value)
{
    if (value == null) {
        return null;
    }

    if (type.equals(BOOLEAN)) {
        return value;
    }
    if (type.equals(TINYINT)) {
        return ((Number) value).byteValue();
    }
    if (type.equals(SMALLINT)) {
        return ((Number) value).shortValue();
    }
    if (type.equals(INTEGER)) {
        return ((Number) value).intValue();
    }
    if (type.equals(BIGINT)) {
        return ((Number) value).longValue();
    }
    if (type.equals(REAL)) {
        return ((Number) value).floatValue();
    }
    if (type.equals(DOUBLE)) {
        return ((Number) value).doubleValue();
    }
    if (type instanceof VarcharType) {
        return value;
    }
    if (type instanceof CharType) {
        return new HiveChar((String) value, ((CharType) type).getLength());
    }
    if (type.equals(VARBINARY)) {
        return ((SqlVarbinary) value).getBytes();
    }
    if (type.equals(DATE)) {
        int days = ((SqlDate) value).getDays();
        LocalDate localDate = LocalDate.ofEpochDay(days);
        ZonedDateTime zonedDateTime = localDate.atStartOfDay(ZoneId.systemDefault());

        long millis = SECONDS.toMillis(zonedDateTime.toEpochSecond());
        Date date = new Date(0);
        // millis must be set separately to avoid masking
        date.setTime(millis);
        return date;
    }
    if (type.equals(TIMESTAMP)) {
        long millisUtc = ((SqlTimestamp) value).getMillisUtc();
        return new Timestamp(millisUtc);
    }
    if (type instanceof DecimalType) {
        return HiveDecimal.create(((SqlDecimal) value).toBigDecimal());
    }
    if (type instanceof ArrayType) {
        Type elementType = type.getTypeParameters().get(0);
        return ((List<?>) value).stream()
                .map(element -> preprocessWriteValueHive(elementType, element))
                .collect(toList());
    }
    if (type instanceof MapType) {
        Type keyType = type.getTypeParameters().get(0);
        Type valueType = type.getTypeParameters().get(1);
        Map<Object, Object> newMap = new HashMap<>();
        for (Entry<?, ?> entry : ((Map<?, ?>) value).entrySet()) {
            newMap.put(preprocessWriteValueHive(keyType, entry.getKey()), preprocessWriteValueHive(valueType, entry.getValue()));
        }
        return newMap;
    }
    if (type instanceof RowType) {
        List<?> fieldValues = (List<?>) value;
        List<Type> fieldTypes = type.getTypeParameters();
        List<Object> newStruct = new ArrayList<>();
        for (int fieldId = 0; fieldId < fieldValues.size(); fieldId++) {
            newStruct.add(preprocessWriteValueHive(fieldTypes.get(fieldId), fieldValues.get(fieldId)));
        }
        return newStruct;
    }
    throw new IllegalArgumentException("unsupported type: " + type);
}