Java Code Examples for org.apache.hadoop.hive.serde2.io.HiveDecimalWritable

The following examples show how to use org.apache.hadoop.hive.serde2.io.HiveDecimalWritable. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: nifi   Source File: TestNiFiOrcUtils.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void test_getWritable() throws Exception {
    assertTrue(NiFiOrcUtils.convertToORCObject(null, 1, true) instanceof IntWritable);
    assertTrue(NiFiOrcUtils.convertToORCObject(null, 1L, true) instanceof LongWritable);
    assertTrue(NiFiOrcUtils.convertToORCObject(null, 1.0f, true) instanceof FloatWritable);
    assertTrue(NiFiOrcUtils.convertToORCObject(null, 1.0, true) instanceof DoubleWritable);
    assertTrue(NiFiOrcUtils.convertToORCObject(null, BigDecimal.valueOf(1L), true) instanceof HiveDecimalWritable);
    assertTrue(NiFiOrcUtils.convertToORCObject(null, new int[]{1, 2, 3}, true) instanceof List);
    assertTrue(NiFiOrcUtils.convertToORCObject(null, Arrays.asList(1, 2, 3), true) instanceof List);
    Map<String, Float> map = new HashMap<>();
    map.put("Hello", 1.0f);
    map.put("World", 2.0f);

    Object convMap = NiFiOrcUtils.convertToORCObject(TypeInfoUtils.getTypeInfoFromTypeString("map<string,float>"), map, true);
    assertTrue(convMap instanceof Map);
    ((Map) convMap).forEach((key, value) -> {
        assertTrue(key instanceof Text);
        assertTrue(value instanceof FloatWritable);
    });
}
 
Example 2
Source Project: nifi   Source File: TestNiFiOrcUtils.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void test_getWritable() throws Exception {
    assertTrue(NiFiOrcUtils.convertToORCObject(null, 1) instanceof IntWritable);
    assertTrue(NiFiOrcUtils.convertToORCObject(null, 1L) instanceof LongWritable);
    assertTrue(NiFiOrcUtils.convertToORCObject(null, 1.0f) instanceof FloatWritable);
    assertTrue(NiFiOrcUtils.convertToORCObject(null, 1.0) instanceof DoubleWritable);
    assertTrue(NiFiOrcUtils.convertToORCObject(null, BigDecimal.valueOf(1.0D)) instanceof HiveDecimalWritable);
    assertTrue(NiFiOrcUtils.convertToORCObject(null, new int[]{1, 2, 3}) instanceof List);
    assertTrue(NiFiOrcUtils.convertToORCObject(null, Arrays.asList(1, 2, 3)) instanceof List);
    Map<String, Float> map = new HashMap<>();
    map.put("Hello", 1.0f);
    map.put("World", 2.0f);

    Object convMap = NiFiOrcUtils.convertToORCObject(TypeInfoUtils.getTypeInfoFromTypeString("map<string,float>"), map);
    assertTrue(convMap instanceof Map);
    ((Map) convMap).forEach((key, value) -> {
        assertTrue(key instanceof Text);
        assertTrue(value instanceof FloatWritable);
    });
}
 
Example 3
Source Project: presto   Source File: DecimalUtils.java    License: Apache License 2.0 5 votes vote down vote up
public static long getShortDecimalValue(HiveDecimalWritable writable, int columnScale)
{
    byte[] bytes = writable.getInternalStorage();
    long value = getShortDecimalValue(bytes);
    value = rescale(value, writable.getScale(), columnScale);
    return value;
}
 
Example 4
Source Project: dremio-oss   Source File: ObjectInspectors.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public HiveDecimalWritable getPrimitiveWritableObject(Object o) {
<#if mode == "Optional">
  if (o == null) {
    return null;
  }
  final NullableDecimalHolder h = (NullableDecimalHolder) o;
<#else>
  final DecimalHolder h = (DecimalHolder) o;
</#if>
  h.start = (h.start / org.apache.arrow.vector.util.DecimalUtility.DECIMAL_BYTE_LENGTH);
  return new HiveDecimalWritable(
      HiveDecimal.create(DecimalUtility.getBigDecimalFromArrowBuf(h.buffer, h.start, h.scale)));
}
 
Example 5
Source Project: spork   Source File: OrcUtils.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public HiveDecimalWritable getPrimitiveWritableObject(Object o) {
    if (o instanceof BigDecimal) {
        return o == null ? null : new HiveDecimalWritable(HiveDecimal.create((BigDecimal)o));
    } else { // BigInteger
        return o == null ? null : new HiveDecimalWritable(HiveDecimal.create((BigInteger)o));
    }
}
 
Example 6
Source Project: presto   Source File: OrcTester.java    License: Apache License 2.0 4 votes vote down vote up
private static Object decodeRecordReaderValue(Type type, Object actualValue)
{
    if (actualValue instanceof BooleanWritable) {
        actualValue = ((BooleanWritable) actualValue).get();
    }
    else if (actualValue instanceof ByteWritable) {
        actualValue = ((ByteWritable) actualValue).get();
    }
    else if (actualValue instanceof BytesWritable) {
        actualValue = new SqlVarbinary(((BytesWritable) actualValue).copyBytes());
    }
    else if (actualValue instanceof DateWritable) {
        actualValue = new SqlDate(((DateWritable) actualValue).getDays());
    }
    else if (actualValue instanceof DoubleWritable) {
        actualValue = ((DoubleWritable) actualValue).get();
    }
    else if (actualValue instanceof FloatWritable) {
        actualValue = ((FloatWritable) actualValue).get();
    }
    else if (actualValue instanceof IntWritable) {
        actualValue = ((IntWritable) actualValue).get();
    }
    else if (actualValue instanceof HiveCharWritable) {
        actualValue = ((HiveCharWritable) actualValue).getPaddedValue().toString();
    }
    else if (actualValue instanceof LongWritable) {
        actualValue = ((LongWritable) actualValue).get();
    }
    else if (actualValue instanceof ShortWritable) {
        actualValue = ((ShortWritable) actualValue).get();
    }
    else if (actualValue instanceof HiveDecimalWritable) {
        DecimalType decimalType = (DecimalType) type;
        HiveDecimalWritable writable = (HiveDecimalWritable) actualValue;
        // writable messes with the scale so rescale the values to the Presto type
        BigInteger rescaledValue = rescale(writable.getHiveDecimal().unscaledValue(), writable.getScale(), decimalType.getScale());
        actualValue = new SqlDecimal(rescaledValue, decimalType.getPrecision(), decimalType.getScale());
    }
    else if (actualValue instanceof Text) {
        actualValue = actualValue.toString();
    }
    else if (actualValue instanceof TimestampWritable) {
        TimestampWritable timestamp = (TimestampWritable) actualValue;
        actualValue = sqlTimestampOf((timestamp.getSeconds() * 1000) + (timestamp.getNanos() / 1000000L), SESSION);
    }
    else if (actualValue instanceof OrcStruct) {
        List<Object> fields = new ArrayList<>();
        OrcStruct structObject = (OrcStruct) actualValue;
        for (int fieldId = 0; fieldId < structObject.getNumFields(); fieldId++) {
            fields.add(OrcUtil.getFieldValue(structObject, fieldId));
        }
        actualValue = decodeRecordReaderStruct(type, fields);
    }
    else if (actualValue instanceof List) {
        actualValue = decodeRecordReaderList(type, ((List<?>) actualValue));
    }
    else if (actualValue instanceof Map) {
        actualValue = decodeRecordReaderMap(type, (Map<?, ?>) actualValue);
    }
    return actualValue;
}
 
Example 7
Source Project: presto   Source File: RcFileTester.java    License: Apache License 2.0 4 votes vote down vote up
private static Object decodeRecordReaderValue(Type type, Object actualValue)
{
    if (actualValue instanceof LazyPrimitive) {
        actualValue = ((LazyPrimitive<?, ?>) actualValue).getWritableObject();
    }
    if (actualValue instanceof BooleanWritable) {
        actualValue = ((BooleanWritable) actualValue).get();
    }
    else if (actualValue instanceof ByteWritable) {
        actualValue = ((ByteWritable) actualValue).get();
    }
    else if (actualValue instanceof BytesWritable) {
        actualValue = new SqlVarbinary(((BytesWritable) actualValue).copyBytes());
    }
    else if (actualValue instanceof DateWritable) {
        actualValue = new SqlDate(((DateWritable) actualValue).getDays());
    }
    else if (actualValue instanceof DoubleWritable) {
        actualValue = ((DoubleWritable) actualValue).get();
    }
    else if (actualValue instanceof FloatWritable) {
        actualValue = ((FloatWritable) actualValue).get();
    }
    else if (actualValue instanceof IntWritable) {
        actualValue = ((IntWritable) actualValue).get();
    }
    else if (actualValue instanceof LongWritable) {
        actualValue = ((LongWritable) actualValue).get();
    }
    else if (actualValue instanceof ShortWritable) {
        actualValue = ((ShortWritable) actualValue).get();
    }
    else if (actualValue instanceof HiveDecimalWritable) {
        DecimalType decimalType = (DecimalType) type;
        HiveDecimalWritable writable = (HiveDecimalWritable) actualValue;
        // writable messes with the scale so rescale the values to the Presto type
        BigInteger rescaledValue = rescale(writable.getHiveDecimal().unscaledValue(), writable.getScale(), decimalType.getScale());
        actualValue = new SqlDecimal(rescaledValue, decimalType.getPrecision(), decimalType.getScale());
    }
    else if (actualValue instanceof Text) {
        actualValue = actualValue.toString();
    }
    else if (actualValue instanceof TimestampWritable) {
        TimestampWritable timestamp = (TimestampWritable) actualValue;
        if (SESSION.isLegacyTimestamp()) {
            actualValue = SqlTimestamp.legacyFromMillis(3, (timestamp.getSeconds() * 1000) + (timestamp.getNanos() / 1000000L), UTC_KEY);
        }
        else {
            actualValue = SqlTimestamp.fromMillis(3, (timestamp.getSeconds() * 1000) + (timestamp.getNanos() / 1000000L));
        }
    }
    else if (actualValue instanceof StructObject) {
        StructObject structObject = (StructObject) actualValue;
        actualValue = decodeRecordReaderStruct(type, structObject.getFieldsAsList());
    }
    else if (actualValue instanceof LazyBinaryArray) {
        actualValue = decodeRecordReaderList(type, ((LazyBinaryArray) actualValue).getList());
    }
    else if (actualValue instanceof LazyBinaryMap) {
        actualValue = decodeRecordReaderMap(type, ((LazyBinaryMap) actualValue).getMap());
    }
    else if (actualValue instanceof LazyArray) {
        actualValue = decodeRecordReaderList(type, ((LazyArray) actualValue).getList());
    }
    else if (actualValue instanceof LazyMap) {
        actualValue = decodeRecordReaderMap(type, ((LazyMap) actualValue).getMap());
    }
    else if (actualValue instanceof List) {
        actualValue = decodeRecordReaderList(type, ((List<?>) actualValue));
    }
    return actualValue;
}
 
Example 8
Source Project: presto   Source File: SerDeUtils.java    License: Apache License 2.0 4 votes vote down vote up
private static void serializePrimitive(Type type, BlockBuilder builder, Object object, PrimitiveObjectInspector inspector)
{
    requireNonNull(builder, "parent builder is null");

    if (object == null) {
        builder.appendNull();
        return;
    }

    switch (inspector.getPrimitiveCategory()) {
        case BOOLEAN:
            BooleanType.BOOLEAN.writeBoolean(builder, ((BooleanObjectInspector) inspector).get(object));
            return;
        case BYTE:
            TinyintType.TINYINT.writeLong(builder, ((ByteObjectInspector) inspector).get(object));
            return;
        case SHORT:
            SmallintType.SMALLINT.writeLong(builder, ((ShortObjectInspector) inspector).get(object));
            return;
        case INT:
            IntegerType.INTEGER.writeLong(builder, ((IntObjectInspector) inspector).get(object));
            return;
        case LONG:
            BigintType.BIGINT.writeLong(builder, ((LongObjectInspector) inspector).get(object));
            return;
        case FLOAT:
            RealType.REAL.writeLong(builder, floatToRawIntBits(((FloatObjectInspector) inspector).get(object)));
            return;
        case DOUBLE:
            DoubleType.DOUBLE.writeDouble(builder, ((DoubleObjectInspector) inspector).get(object));
            return;
        case STRING:
            type.writeSlice(builder, Slices.utf8Slice(((StringObjectInspector) inspector).getPrimitiveJavaObject(object)));
            return;
        case VARCHAR:
            type.writeSlice(builder, Slices.utf8Slice(((HiveVarcharObjectInspector) inspector).getPrimitiveJavaObject(object).getValue()));
            return;
        case CHAR:
            CharType charType = (CharType) type;
            HiveChar hiveChar = ((HiveCharObjectInspector) inspector).getPrimitiveJavaObject(object);
            type.writeSlice(builder, truncateToLengthAndTrimSpaces(Slices.utf8Slice(hiveChar.getValue()), charType.getLength()));
            return;
        case DATE:
            DateType.DATE.writeLong(builder, formatDateAsLong(object, (DateObjectInspector) inspector));
            return;
        case TIMESTAMP:
            TimestampType.TIMESTAMP.writeLong(builder, formatTimestampAsLong(object, (TimestampObjectInspector) inspector));
            return;
        case BINARY:
            VARBINARY.writeSlice(builder, Slices.wrappedBuffer(((BinaryObjectInspector) inspector).getPrimitiveJavaObject(object)));
            return;
        case DECIMAL:
            DecimalType decimalType = (DecimalType) type;
            HiveDecimalWritable hiveDecimal = ((HiveDecimalObjectInspector) inspector).getPrimitiveWritableObject(object);
            if (decimalType.isShort()) {
                decimalType.writeLong(builder, DecimalUtils.getShortDecimalValue(hiveDecimal, decimalType.getScale()));
            }
            else {
                decimalType.writeSlice(builder, DecimalUtils.getLongDecimalValue(hiveDecimal, decimalType.getScale()));
            }
            return;
    }
    throw new RuntimeException("Unknown primitive type: " + inspector.getPrimitiveCategory());
}
 
Example 9
Source Project: presto   Source File: DecimalUtils.java    License: Apache License 2.0 4 votes vote down vote up
public static Slice getLongDecimalValue(HiveDecimalWritable writable, int columnScale)
{
    BigInteger value = new BigInteger(writable.getInternalStorage());
    value = rescale(value, writable.getScale(), columnScale);
    return encodeUnscaledValue(value);
}
 
Example 10
Source Project: Flink-CEPplus   Source File: OrcBatchReader.java    License: Apache License 2.0 4 votes vote down vote up
private static BigDecimal readBigDecimal(HiveDecimalWritable hiveDecimalWritable) {
	HiveDecimal hiveDecimal = hiveDecimalWritable.getHiveDecimal();
	return hiveDecimal.bigDecimalValue();
}
 
Example 11
Source Project: Flink-CEPplus   Source File: OrcRowInputFormat.java    License: Apache License 2.0 4 votes vote down vote up
Object castLiteral(Serializable literal) {

			switch (literalType) {
				case LONG:
					if (literal instanceof Byte) {
						return new Long((Byte) literal);
					} else if (literal instanceof Short) {
						return new Long((Short) literal);
					} else if (literal instanceof Integer) {
						return new Long((Integer) literal);
					} else if (literal instanceof Long) {
						return literal;
					} else {
						throw new IllegalArgumentException("A predicate on a LONG column requires an integer " +
							"literal, i.e., Byte, Short, Integer, or Long.");
					}
				case FLOAT:
					if (literal instanceof Float) {
						return new Double((Float) literal);
					} else if (literal instanceof Double) {
						return literal;
					} else if (literal instanceof BigDecimal) {
						return ((BigDecimal) literal).doubleValue();
					} else {
						throw new IllegalArgumentException("A predicate on a FLOAT column requires a floating " +
							"literal, i.e., Float or Double.");
					}
				case STRING:
					if (literal instanceof String) {
						return literal;
					} else {
						throw new IllegalArgumentException("A predicate on a STRING column requires a floating " +
							"literal, i.e., Float or Double.");
					}
				case BOOLEAN:
					if (literal instanceof Boolean) {
						return literal;
					} else {
						throw new IllegalArgumentException("A predicate on a BOOLEAN column requires a Boolean literal.");
					}
				case DATE:
					if (literal instanceof Date) {
						return literal;
					} else {
						throw new IllegalArgumentException("A predicate on a DATE column requires a java.sql.Date literal.");
					}
				case TIMESTAMP:
					if (literal instanceof Timestamp) {
						return literal;
					} else {
						throw new IllegalArgumentException("A predicate on a TIMESTAMP column requires a java.sql.Timestamp literal.");
					}
				case DECIMAL:
					if (literal instanceof BigDecimal) {
						return new HiveDecimalWritable(HiveDecimal.create((BigDecimal) literal));
					} else {
						throw new IllegalArgumentException("A predicate on a DECIMAL column requires a BigDecimal literal.");
					}
				default:
					throw new IllegalArgumentException("Unknown literal type " + literalType);
			}
		}
 
Example 12
Source Project: flink   Source File: HiveInspectors.java    License: Apache License 2.0 4 votes vote down vote up
public static ObjectInspector getObjectInspector(Class clazz) {
	TypeInfo typeInfo;

	if (clazz.equals(String.class) || clazz.equals(Text.class)) {

		typeInfo = TypeInfoFactory.stringTypeInfo;
	} else if (clazz.equals(Boolean.class) || clazz.equals(BooleanWritable.class)) {

		typeInfo = TypeInfoFactory.booleanTypeInfo;
	} else if (clazz.equals(Byte.class) || clazz.equals(ByteWritable.class)) {

		typeInfo = TypeInfoFactory.byteTypeInfo;
	} else if (clazz.equals(Short.class) || clazz.equals(ShortWritable.class)) {

		typeInfo = TypeInfoFactory.shortTypeInfo;
	} else if (clazz.equals(Integer.class) || clazz.equals(IntWritable.class)) {

		typeInfo = TypeInfoFactory.intTypeInfo;
	} else if (clazz.equals(Long.class) || clazz.equals(LongWritable.class)) {

		typeInfo = TypeInfoFactory.longTypeInfo;
	} else if (clazz.equals(Float.class) || clazz.equals(FloatWritable.class)) {

		typeInfo = TypeInfoFactory.floatTypeInfo;
	} else if (clazz.equals(Double.class) || clazz.equals(DoubleWritable.class)) {

		typeInfo = TypeInfoFactory.doubleTypeInfo;
	} else if (clazz.equals(Date.class) || clazz.equals(DateWritable.class)) {

		typeInfo = TypeInfoFactory.dateTypeInfo;
	} else if (clazz.equals(Timestamp.class) || clazz.equals(TimestampWritable.class)) {

		typeInfo = TypeInfoFactory.timestampTypeInfo;
	} else if (clazz.equals(byte[].class) || clazz.equals(BytesWritable.class)) {

		typeInfo = TypeInfoFactory.binaryTypeInfo;
	} else if (clazz.equals(HiveChar.class) || clazz.equals(HiveCharWritable.class)) {

		typeInfo = TypeInfoFactory.charTypeInfo;
	} else if (clazz.equals(HiveVarchar.class) || clazz.equals(HiveVarcharWritable.class)) {

		typeInfo = TypeInfoFactory.varcharTypeInfo;
	} else if (clazz.equals(HiveDecimal.class) || clazz.equals(HiveDecimalWritable.class)) {

		typeInfo = TypeInfoFactory.decimalTypeInfo;
	} else {
		throw new FlinkHiveUDFException(
			String.format("Class %s is not supported yet", clazz.getName()));
	}

	return getObjectInspector(typeInfo);
}
 
Example 13
Source Project: flink   Source File: OrcBatchReader.java    License: Apache License 2.0 4 votes vote down vote up
private static BigDecimal readBigDecimal(HiveDecimalWritable hiveDecimalWritable) {
	HiveDecimal hiveDecimal = hiveDecimalWritable.getHiveDecimal();
	return hiveDecimal.bigDecimalValue();
}
 
Example 14
Source Project: flink   Source File: OrcRowInputFormat.java    License: Apache License 2.0 4 votes vote down vote up
Object castLiteral(Serializable literal) {

			switch (literalType) {
				case LONG:
					if (literal instanceof Byte) {
						return new Long((Byte) literal);
					} else if (literal instanceof Short) {
						return new Long((Short) literal);
					} else if (literal instanceof Integer) {
						return new Long((Integer) literal);
					} else if (literal instanceof Long) {
						return literal;
					} else {
						throw new IllegalArgumentException("A predicate on a LONG column requires an integer " +
							"literal, i.e., Byte, Short, Integer, or Long.");
					}
				case FLOAT:
					if (literal instanceof Float) {
						return new Double((Float) literal);
					} else if (literal instanceof Double) {
						return literal;
					} else if (literal instanceof BigDecimal) {
						return ((BigDecimal) literal).doubleValue();
					} else {
						throw new IllegalArgumentException("A predicate on a FLOAT column requires a floating " +
							"literal, i.e., Float or Double.");
					}
				case STRING:
					if (literal instanceof String) {
						return literal;
					} else {
						throw new IllegalArgumentException("A predicate on a STRING column requires a floating " +
							"literal, i.e., Float or Double.");
					}
				case BOOLEAN:
					if (literal instanceof Boolean) {
						return literal;
					} else {
						throw new IllegalArgumentException("A predicate on a BOOLEAN column requires a Boolean literal.");
					}
				case DATE:
					if (literal instanceof Date) {
						return literal;
					} else {
						throw new IllegalArgumentException("A predicate on a DATE column requires a java.sql.Date literal.");
					}
				case TIMESTAMP:
					if (literal instanceof Timestamp) {
						return literal;
					} else {
						throw new IllegalArgumentException("A predicate on a TIMESTAMP column requires a java.sql.Timestamp literal.");
					}
				case DECIMAL:
					if (literal instanceof BigDecimal) {
						return new HiveDecimalWritable(HiveDecimal.create((BigDecimal) literal));
					} else {
						throw new IllegalArgumentException("A predicate on a DECIMAL column requires a BigDecimal literal.");
					}
				default:
					throw new IllegalArgumentException("Unknown literal type " + literalType);
			}
		}
 
Example 15
Source Project: dremio-oss   Source File: ORCSearchArgumentGenerator.java    License: Apache License 2.0 4 votes vote down vote up
private static HiveDecimalWritable getDecimal(RexLiteral literal) {
  return new HiveDecimalWritable(HiveDecimal.create((BigDecimal) literal.getValue()));
}
 
Example 16
Source Project: hudi   Source File: HoodieRealtimeRecordReaderUtils.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Convert the projected read from delta record into an array writable.
 */
public static Writable avroToArrayWritable(Object value, Schema schema) {

  if (value == null) {
    return null;
  }

  switch (schema.getType()) {
    case STRING:
      return new Text(value.toString());
    case BYTES:
      return new BytesWritable(((ByteBuffer)value).array());
    case INT:
      return new IntWritable((Integer) value);
    case LONG:
      return new LongWritable((Long) value);
    case FLOAT:
      return new FloatWritable((Float) value);
    case DOUBLE:
      return new DoubleWritable((Double) value);
    case BOOLEAN:
      return new BooleanWritable((Boolean) value);
    case NULL:
      return null;
    case RECORD:
      GenericRecord record = (GenericRecord) value;
      Writable[] recordValues = new Writable[schema.getFields().size()];
      int recordValueIndex = 0;
      for (Schema.Field field : schema.getFields()) {
        recordValues[recordValueIndex++] = avroToArrayWritable(record.get(field.name()), field.schema());
      }
      return new ArrayWritable(Writable.class, recordValues);
    case ENUM:
      return new Text(value.toString());
    case ARRAY:
      GenericArray arrayValue = (GenericArray) value;
      Writable[] arrayValues = new Writable[arrayValue.size()];
      int arrayValueIndex = 0;
      for (Object obj : arrayValue) {
        arrayValues[arrayValueIndex++] = avroToArrayWritable(obj, schema.getElementType());
      }
      // Hive 1.x will fail here, it requires values2 to be wrapped into another ArrayWritable
      return new ArrayWritable(Writable.class, arrayValues);
    case MAP:
      Map mapValue = (Map) value;
      Writable[] mapValues = new Writable[mapValue.size()];
      int mapValueIndex = 0;
      for (Object entry : mapValue.entrySet()) {
        Map.Entry mapEntry = (Map.Entry) entry;
        Writable[] nestedMapValues = new Writable[2];
        nestedMapValues[0] = new Text(mapEntry.getKey().toString());
        nestedMapValues[1] = avroToArrayWritable(mapEntry.getValue(), schema.getValueType());
        mapValues[mapValueIndex++] = new ArrayWritable(Writable.class, nestedMapValues);
      }
      // Hive 1.x will fail here, it requires values3 to be wrapped into another ArrayWritable
      return new ArrayWritable(Writable.class, mapValues);
    case UNION:
      List<Schema> types = schema.getTypes();
      if (types.size() != 2) {
        throw new IllegalArgumentException("Only support union with 2 fields");
      }
      Schema s1 = types.get(0);
      Schema s2 = types.get(1);
      if (s1.getType() == Schema.Type.NULL) {
        return avroToArrayWritable(value, s2);
      } else if (s2.getType() == Schema.Type.NULL) {
        return avroToArrayWritable(value, s1);
      } else {
        throw new IllegalArgumentException("Only support union with null");
      }
    case FIXED:
      if (schema.getLogicalType() != null && schema.getLogicalType().getName().equals("decimal")) {
        LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) LogicalTypes.fromSchema(schema);
        HiveDecimalWritable writable = new HiveDecimalWritable(((GenericFixed) value).bytes(),
            decimal.getScale());
        return HiveDecimalWritable.enforcePrecisionScale(writable,
            decimal.getPrecision(),
            decimal.getScale());
      }
      return new BytesWritable(((GenericFixed) value).bytes());
    default:
      return null;
  }
}
 
Example 17
Source Project: flink   Source File: OrcBatchReader.java    License: Apache License 2.0 4 votes vote down vote up
private static BigDecimal readBigDecimal(HiveDecimalWritable hiveDecimalWritable) {
	HiveDecimal hiveDecimal = hiveDecimalWritable.getHiveDecimal();
	return hiveDecimal.bigDecimalValue();
}
 
Example 18
Source Project: flink   Source File: OrcSplitReader.java    License: Apache License 2.0 4 votes vote down vote up
Object castLiteral(Serializable literal) {

			switch (literalType) {
				case LONG:
					if (literal instanceof Byte) {
						return new Long((Byte) literal);
					} else if (literal instanceof Short) {
						return new Long((Short) literal);
					} else if (literal instanceof Integer) {
						return new Long((Integer) literal);
					} else if (literal instanceof Long) {
						return literal;
					} else {
						throw new IllegalArgumentException("A predicate on a LONG column requires an integer " +
								"literal, i.e., Byte, Short, Integer, or Long.");
					}
				case FLOAT:
					if (literal instanceof Float) {
						return new Double((Float) literal);
					} else if (literal instanceof Double) {
						return literal;
					} else if (literal instanceof BigDecimal) {
						return ((BigDecimal) literal).doubleValue();
					} else {
						throw new IllegalArgumentException("A predicate on a FLOAT column requires a floating " +
								"literal, i.e., Float or Double.");
					}
				case STRING:
					if (literal instanceof String) {
						return literal;
					} else {
						throw new IllegalArgumentException("A predicate on a STRING column requires a floating " +
								"literal, i.e., Float or Double.");
					}
				case BOOLEAN:
					if (literal instanceof Boolean) {
						return literal;
					} else {
						throw new IllegalArgumentException("A predicate on a BOOLEAN column requires a Boolean literal.");
					}
				case DATE:
					if (literal instanceof Date) {
						return literal;
					} else {
						throw new IllegalArgumentException("A predicate on a DATE column requires a java.sql.Date literal.");
					}
				case TIMESTAMP:
					if (literal instanceof Timestamp) {
						return literal;
					} else {
						throw new IllegalArgumentException("A predicate on a TIMESTAMP column requires a java.sql.Timestamp literal.");
					}
				case DECIMAL:
					if (literal instanceof BigDecimal) {
						return new HiveDecimalWritable(HiveDecimal.create((BigDecimal) literal));
					} else {
						throw new IllegalArgumentException("A predicate on a DECIMAL column requires a BigDecimal literal.");
					}
				default:
					throw new IllegalArgumentException("Unknown literal type " + literalType);
			}
		}
 
Example 19
Source Project: incubator-gobblin   Source File: OrcUtils.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Copy the value of {@param from} object into {@param to} with supporting of type-widening that ORC allowed.
 */
public static void handlePrimitiveWritableComparable(WritableComparable from, WritableComparable to) {
  if (from instanceof ByteWritable) {
    if (to instanceof ByteWritable) {
      ((ByteWritable) to).set(((ByteWritable) from).get());
      return;
    } else if (to instanceof ShortWritable) {
      ((ShortWritable) to).set(((ByteWritable) from).get());
      return;
    } else if (to instanceof IntWritable) {
      ((IntWritable) to).set(((ByteWritable) from).get());
      return;
    } else if (to instanceof LongWritable) {
      ((LongWritable) to).set(((ByteWritable) from).get());
      return;
    } else if (to instanceof DoubleWritable) {
      ((DoubleWritable) to).set(((ByteWritable) from).get());
      return;
    }
  } else if (from instanceof ShortWritable) {
    if (to instanceof ShortWritable) {
      ((ShortWritable) to).set(((ShortWritable) from).get());
      return;
    } else if (to instanceof IntWritable) {
      ((IntWritable) to).set(((ShortWritable) from).get());
      return;
    } else if (to instanceof LongWritable) {
      ((LongWritable) to).set(((ShortWritable) from).get());
      return;
    } else if (to instanceof DoubleWritable) {
      ((DoubleWritable) to).set(((ShortWritable) from).get());
      return;
    }
  } else if (from instanceof IntWritable) {
    if (to instanceof IntWritable) {
      ((IntWritable) to).set(((IntWritable) from).get());
      return;
    } else if (to instanceof LongWritable) {
      ((LongWritable) to).set(((IntWritable) from).get());
      return;
    } else if (to instanceof DoubleWritable) {
      ((DoubleWritable) to).set(((IntWritable) from).get());
      return;
    }
  } else if (from instanceof LongWritable) {
    if (to instanceof LongWritable) {
      ((LongWritable) to).set(((LongWritable) from).get());
      return;
    } else if (to instanceof DoubleWritable) {
      ((DoubleWritable) to).set(((LongWritable) from).get());
      return;
    }
    // Following from this branch, type-widening is not allowed and only value-copy will happen.
  } else if (from instanceof DoubleWritable) {
    if (to instanceof DoubleWritable) {
      ((DoubleWritable) to).set(((DoubleWritable) from).get());
      return;
    }
  } else if (from instanceof BytesWritable) {
    if (to instanceof BytesWritable) {
      ((BytesWritable) to).set((BytesWritable) from);
      return;
    }
  } else if (from instanceof FloatWritable) {
    if (to instanceof FloatWritable) {
      ((FloatWritable) to).set(((FloatWritable) from).get());
      return;
    }
  } else if (from instanceof Text) {
    if (to instanceof Text) {
      ((Text) to).set((Text) from);
      return;
    }
  } else if (from instanceof DateWritable) {
    if (to instanceof DateWritable) {
      ((DateWritable) to).set(((DateWritable) from).get());
      return;
    }
  } else if (from instanceof OrcTimestamp && to instanceof OrcTimestamp) {
    ((OrcTimestamp) to).set(((OrcTimestamp) from).toString());
    return;
  } else if (from instanceof HiveDecimalWritable && to instanceof HiveDecimalWritable) {
    ((HiveDecimalWritable) to).set(((HiveDecimalWritable) from).getHiveDecimal());
    return;
  } else if (from instanceof BooleanWritable && to instanceof BooleanWritable) {
    ((BooleanWritable) to).set(((BooleanWritable) from).get());
    return;
  }
  throw new UnsupportedOperationException(String
      .format("The conversion of primitive-type WritableComparable object from %s to %s is not supported",
          from.getClass(), to.getClass()));
}
 
Example 20
Source Project: spork   Source File: TestOrcStorage.java    License: Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("rawtypes")
private void compareData(Object expected, Object actual) {
    if (expected instanceof Text) {
        assertEquals(String.class, actual.getClass());
        assertEquals(expected.toString(), actual);
    } else if (expected instanceof ShortWritable) {
        assertEquals(Integer.class, actual.getClass());
        assertEquals((int)((ShortWritable) expected).get(), actual);
    } else if (expected instanceof IntWritable) {
        assertEquals(Integer.class, actual.getClass());
        assertEquals(((IntWritable) expected).get(), actual);
    } else if (expected instanceof LongWritable) {
        assertEquals(Long.class, actual.getClass());
        assertEquals(((LongWritable) expected).get(), actual);
    } else if (expected instanceof FloatWritable) {
        assertEquals(Float.class, actual.getClass());
        assertEquals(((FloatWritable) expected).get(), actual);
    } else if (expected instanceof HiveDecimalWritable) {
        assertEquals(BigDecimal.class, actual.getClass());
        assertEquals(((HiveDecimalWritable) expected).toString(), actual.toString());
    } else if (expected instanceof DoubleWritable) {
        assertEquals(Double.class, actual.getClass());
        assertEquals(((DoubleWritable) expected).get(), actual);
    } else if (expected instanceof BooleanWritable) {
        assertEquals(Boolean.class, actual.getClass());
        assertEquals(((BooleanWritable) expected).get(), actual);
    } else if (expected instanceof TimestampWritable) {
        assertEquals(DateTime.class, actual.getClass());
        assertEquals(((TimestampWritable) expected).getTimestamp().getTime(),
                ((DateTime) actual).getMillis());
    } else if (expected instanceof BytesWritable) {
        assertEquals(DataByteArray.class, actual.getClass());
        BytesWritable bw = (BytesWritable) expected;
        assertEquals(new DataByteArray(bw.getBytes(), 0, bw.getLength()), actual);
    } else if (expected instanceof ByteWritable) {
        assertEquals(Integer.class, actual.getClass());
        assertEquals((int) ((ByteWritable) expected).get(), actual);
    } else if (expected instanceof OrcStruct) {
        assertEquals(BinSedesTuple.class, actual.getClass());
        // TODO: compare actual values. No getters in OrcStruct
    } else if (expected instanceof ArrayList) {
        assertEquals(DefaultDataBag.class, actual.getClass());
        // TODO: compare actual values. No getters in OrcStruct
    } else if (expected instanceof HashMap) {
        assertEquals(HashMap.class, actual.getClass());
        assertEquals(((HashMap) expected).size(), ((HashMap) actual).size());
        // TODO: compare actual values. No getters in OrcStruct
    } else if (expected == null) {
        assertEquals(expected, actual);
    } else {
        Assert.fail("Unknown object type: " + expected.getClass().getName());
    }
}
 
Example 21
Source Project: nifi   Source File: PutORCTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testWriteORCWithAvroLogicalTypes() throws IOException, InitializationException {
    final String avroSchema = IOUtils.toString(new FileInputStream("src/test/resources/user_logical_types.avsc"), StandardCharsets.UTF_8);
    schema = new Schema.Parser().parse(avroSchema);
    LocalTime nowTime = LocalTime.now();
    LocalDateTime nowDateTime = LocalDateTime.now();
    LocalDate nowDate = LocalDate.now();

    final int timeMillis = nowTime.get(ChronoField.MILLI_OF_DAY);
    final Timestamp timestampMillis = Timestamp.valueOf(nowDateTime);
    final Date dt = Date.valueOf(nowDate);
    final BigDecimal bigDecimal = new BigDecimal("92.12");

    configure(proc, 10, (numUsers, readerFactory) -> {
        for (int i = 0; i < numUsers; i++) {
            readerFactory.addRecord(
                    i,
                    timeMillis,
                    timestampMillis,
                    dt,
                    bigDecimal);
        }
        return null;
    });

    final String filename = "testORCWithDefaults-" + System.currentTimeMillis();

    final Map<String, String> flowFileAttributes = new HashMap<>();
    flowFileAttributes.put(CoreAttributes.FILENAME.key(), filename);

    testRunner.setProperty(PutORC.HIVE_TABLE_NAME, "myTable");

    testRunner.enqueue("trigger", flowFileAttributes);
    testRunner.run();
    testRunner.assertAllFlowFilesTransferred(PutORC.REL_SUCCESS, 1);

    final Path orcFile = new Path(DIRECTORY + "/" + filename);

    // verify the successful flow file has the expected attributes
    final MockFlowFile mockFlowFile = testRunner.getFlowFilesForRelationship(PutORC.REL_SUCCESS).get(0);
    mockFlowFile.assertAttributeEquals(PutORC.ABSOLUTE_HDFS_PATH_ATTRIBUTE, orcFile.getParent().toString());
    mockFlowFile.assertAttributeEquals(CoreAttributes.FILENAME.key(), filename);
    mockFlowFile.assertAttributeEquals(PutORC.RECORD_COUNT_ATTR, "10");
    // DDL will be created with field names normalized (lowercased, e.g.) for Hive by default
    mockFlowFile.assertAttributeEquals(PutORC.HIVE_DDL_ATTRIBUTE,
            "CREATE EXTERNAL TABLE IF NOT EXISTS `myTable` (`id` INT, `timemillis` INT, `timestampmillis` TIMESTAMP, `dt` DATE, `dec` DECIMAL) STORED AS ORC");

    // verify we generated a provenance event
    final List<ProvenanceEventRecord> provEvents = testRunner.getProvenanceEvents();
    assertEquals(1, provEvents.size());

    // verify it was a SEND event with the correct URI
    final ProvenanceEventRecord provEvent = provEvents.get(0);
    assertEquals(ProvenanceEventType.SEND, provEvent.getEventType());
    // If it runs with a real HDFS, the protocol will be "hdfs://", but with a local filesystem, just assert the filename.
    Assert.assertTrue(provEvent.getTransitUri().endsWith(DIRECTORY + "/" + filename));

    // verify the content of the ORC file by reading it back in
    verifyORCUsers(orcFile, 10, (x, currUser) -> {
                assertEquals((int) currUser, ((IntWritable) x.get(0)).get());
                assertEquals(timeMillis, ((IntWritable) x.get(1)).get());
                assertEquals(timestampMillis, ((TimestampWritableV2) x.get(2)).getTimestamp().toSqlTimestamp());
                final DateFormat noTimeOfDayDateFormat = new SimpleDateFormat("yyyy-MM-dd");
                noTimeOfDayDateFormat.setTimeZone(TimeZone.getTimeZone("gmt"));
                assertEquals(noTimeOfDayDateFormat.format(dt), ((DateWritableV2) x.get(3)).get().toString());
                assertEquals(bigDecimal, ((HiveDecimalWritable) x.get(4)).getHiveDecimal().bigDecimalValue());
                return null;
            }
    );

    // verify we don't have the temp dot file after success
    final File tempOrcFile = new File(DIRECTORY + "/." + filename);
    Assert.assertFalse(tempOrcFile.exists());

    // verify we DO have the CRC file after success
    final File crcAvroORCFile = new File(DIRECTORY + "/." + filename + ".crc");
    Assert.assertTrue(crcAvroORCFile.exists());
}
 
Example 22
Source Project: pentaho-hadoop-shims   Source File: OrcConverter.java    License: Apache License 2.0 4 votes vote down vote up
protected static Object convertFromSourceToTargetDataType( ColumnVector columnVector, int currentBatchRow,
                                                           int orcValueMetaInterface ) {

  if ( columnVector.isNull[ currentBatchRow ] ) {
    return null;
  }
  switch ( orcValueMetaInterface ) {
    case ValueMetaInterface.TYPE_INET:
      try {
        return InetAddress.getByName( new String( ( (BytesColumnVector) columnVector ).vector[ currentBatchRow ],
          ( (BytesColumnVector) columnVector ).start[ currentBatchRow ],
          ( (BytesColumnVector) columnVector ).length[ currentBatchRow ] ) );
      } catch ( UnknownHostException e ) {
        e.printStackTrace();
      }

    case ValueMetaInterface.TYPE_STRING:
      return new String( ( (BytesColumnVector) columnVector ).vector[ currentBatchRow ],
        ( (BytesColumnVector) columnVector ).start[ currentBatchRow ],
        ( (BytesColumnVector) columnVector ).length[ currentBatchRow ] );

    case ValueMetaInterface.TYPE_INTEGER:
      return (long) ( (LongColumnVector) columnVector ).vector[ currentBatchRow ];

    case ValueMetaInterface.TYPE_NUMBER:
      return ( (DoubleColumnVector) columnVector ).vector[ currentBatchRow ];

    case ValueMetaInterface.TYPE_BIGNUMBER:
      HiveDecimalWritable obj = ( (DecimalColumnVector) columnVector ).vector[ currentBatchRow ];
      return obj.getHiveDecimal().bigDecimalValue();

    case ValueMetaInterface.TYPE_TIMESTAMP:
      Timestamp timestamp = new Timestamp( ( (TimestampColumnVector) columnVector ).time[ currentBatchRow ] );
      timestamp.setNanos( ( (TimestampColumnVector) columnVector ).nanos[ currentBatchRow ] );
      return timestamp;

    case ValueMetaInterface.TYPE_DATE:
      LocalDate localDate =
        LocalDate.ofEpochDay( 0 ).plusDays( ( (LongColumnVector) columnVector ).vector[ currentBatchRow ] );
      Date dateValue = Date.from( localDate.atStartOfDay( ZoneId.systemDefault() ).toInstant() );
      return dateValue;

    case ValueMetaInterface.TYPE_BOOLEAN:
      return ( (LongColumnVector) columnVector ).vector[ currentBatchRow ] == 0 ? false : true;

    case ValueMetaInterface.TYPE_BINARY:
      byte[] origBytes = ( (BytesColumnVector) columnVector ).vector[ currentBatchRow ];
      int startPos = ( (BytesColumnVector) columnVector ).start[ currentBatchRow ];
      byte[] newBytes = Arrays.copyOfRange( origBytes, startPos,
        startPos + ( (BytesColumnVector) columnVector ).length[ currentBatchRow ] );
      return newBytes;
  }

  //if none of the cases match return a null
  return null;
}
 
Example 23
Source Project: elasticsearch-hadoop   Source File: HiveTypeToJsonTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testDecimal() {
    assertEquals("\"1\"", hiveTypeToJson(new MyHiveType(new HiveDecimalWritable(HiveDecimal.create(BigDecimal.ONE)),
            decimalTypeInfo)));
}