org.apache.spark.sql.types.Decimal Java Examples

The following examples show how to use org.apache.spark.sql.types.Decimal. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ColumnarBatchRow.java    From spliceengine with GNU Affero General Public License v3.0 6 votes vote down vote up
@Override
public BigDecimal getDecimal(int i) {
    if (isNullAt(i)) {
        return null;
    }
    DataType dt = structType.fields()[i].dataType();
    int precision = ((DecimalType) dt).precision();
    int scale = ((DecimalType) dt).scale();
    if (DecimalType.isByteArrayDecimalType(dt)) {
        byte[] bytes = row.getBinary(i);
        BigInteger bigInteger = new BigInteger(bytes);
        BigDecimal javaDecimal = new BigDecimal(bigInteger, scale);
        return Decimal.apply(javaDecimal, precision, scale).toJavaBigDecimal();
    } else {
        return Decimal.apply(DecimalType.is32BitDecimalType(dt) ? getInt(i) : getLong(i), precision, scale).toJavaBigDecimal();
    }
}
 
Example #2
Source File: SparkValueWriters.java    From iceberg with Apache License 2.0 6 votes vote down vote up
@Override
public void write(Decimal d, Encoder encoder) throws IOException {
  Preconditions.checkArgument(d.scale() == scale,
      "Cannot write value as decimal(%s,%s), wrong scale: %s", precision, scale, d);
  Preconditions.checkArgument(d.precision() <= precision,
      "Cannot write value as decimal(%s,%s), too large: %s", precision, scale, d);

  BigDecimal decimal = d.toJavaBigDecimal();

  byte fillByte = (byte) (decimal.signum() < 0 ? 0xFF : 0x00);
  byte[] unscaled = decimal.unscaledValue().toByteArray();
  byte[] buf = bytes.get();
  int offset = length - unscaled.length;

  for (int i = 0; i < length; i += 1) {
    if (i < offset) {
      buf[i] = fillByte;
    } else {
      buf[i] = unscaled[i - offset];
    }
  }

  encoder.writeFixed(buf);
}
 
Example #3
Source File: SparkOrcReader.java    From iceberg with Apache License 2.0 6 votes vote down vote up
@Override
public void convert(UnsafeArrayWriter writer, int element,
                    ColumnVector vector, int row) {
  if (vector.isRepeating) {
    row = 0;
  }
  if (!vector.noNulls && vector.isNull[row]) {
    writer.setNull(element);
  } else {
    BigDecimal v = ((DecimalColumnVector) vector).vector[row]
        .getHiveDecimal().bigDecimalValue();
    writer.write(element,
        new Decimal().set(new scala.math.BigDecimal(v), precision, scale),
        precision, scale);
  }
}
 
Example #4
Source File: SparkOrcReader.java    From iceberg with Apache License 2.0 6 votes vote down vote up
@Override
public void convert(UnsafeRowWriter writer, int column, ColumnVector vector,
                    int row) {
  if (vector.isRepeating) {
    row = 0;
  }
  if (!vector.noNulls && vector.isNull[row]) {
    writer.setNullAt(column);
  } else {
    BigDecimal v = ((DecimalColumnVector) vector).vector[row]
        .getHiveDecimal().bigDecimalValue();
    writer.write(column,
        new Decimal().set(new scala.math.BigDecimal(v), precision, scale),
        precision, scale);
  }
}
 
Example #5
Source File: SparkParquetWriters.java    From iceberg with Apache License 2.0 6 votes vote down vote up
@Override
public void write(int repetitionLevel, Decimal decimal) {
  Preconditions.checkArgument(decimal.scale() == scale,
      "Cannot write value as decimal(%s,%s), wrong scale: %s", precision, scale, decimal);
  Preconditions.checkArgument(decimal.precision() <= precision,
      "Cannot write value as decimal(%s,%s), too large: %s", precision, scale, decimal);

  BigDecimal bigDecimal = decimal.toJavaBigDecimal();

  byte fillByte = (byte) (bigDecimal.signum() < 0 ? 0xFF : 0x00);
  byte[] unscaled = bigDecimal.unscaledValue().toByteArray();
  byte[] buf = bytes.get();
  int offset = length - unscaled.length;

  for (int i = 0; i < length; i += 1) {
    if (i < offset) {
      buf[i] = fillByte;
    } else {
      buf[i] = unscaled[i - offset];
    }
  }

  column.writeBinary(repetitionLevel, Binary.fromReusedByteArray(buf));
}
 
Example #6
Source File: SparkValueWriters.java    From iceberg with Apache License 2.0 6 votes vote down vote up
@Override
public void write(Decimal d, Encoder encoder) throws IOException {
  Preconditions.checkArgument(d.scale() == scale,
      "Cannot write value as decimal(%s,%s), wrong scale: %s", precision, scale, d);
  Preconditions.checkArgument(d.precision() <= precision,
      "Cannot write value as decimal(%s,%s), too large: %s", precision, scale, d);

  BigDecimal decimal = d.toJavaBigDecimal();

  byte fillByte = (byte) (decimal.signum() < 0 ? 0xFF : 0x00);
  byte[] unscaled = decimal.unscaledValue().toByteArray();
  byte[] buf = bytes.get();
  int offset = length - unscaled.length;

  for (int i = 0; i < length; i += 1) {
    if (i < offset) {
      buf[i] = fillByte;
    } else {
      buf[i] = unscaled[i - offset];
    }
  }

  encoder.writeFixed(buf);
}
 
Example #7
Source File: StructInternalRow.java    From iceberg with Apache License 2.0 5 votes vote down vote up
private ArrayData collectionToArrayData(Type elementType, Collection<?> values) {
  switch (elementType.typeId()) {
    case BOOLEAN:
    case INTEGER:
    case DATE:
    case TIME:
    case LONG:
    case TIMESTAMP:
    case FLOAT:
    case DOUBLE:
      return fillArray(values, array -> (pos, value) -> array[pos] = value);
    case STRING:
      return fillArray(values, array ->
          (BiConsumer<Integer, CharSequence>) (pos, seq) -> array[pos] = UTF8String.fromString(seq.toString()));
    case FIXED:
    case BINARY:
      return fillArray(values, array ->
          (BiConsumer<Integer, ByteBuffer>) (pos, buf) -> array[pos] = ByteBuffers.toByteArray(buf));
    case DECIMAL:
      return fillArray(values, array ->
          (BiConsumer<Integer, BigDecimal>) (pos, dec) -> array[pos] = Decimal.apply(dec));
    case STRUCT:
      return fillArray(values, array -> (BiConsumer<Integer, StructLike>) (pos, tuple) ->
          array[pos] = new StructInternalRow(elementType.asStructType(), tuple));
    case LIST:
      return fillArray(values, array -> (BiConsumer<Integer, Collection<?>>) (pos, list) ->
          array[pos] = collectionToArrayData(elementType.asListType(), list));
    case MAP:
      return fillArray(values, array -> (BiConsumer<Integer, Map<?, ?>>) (pos, map) ->
          array[pos] = mapToMapData(elementType.asMapType(), map));
    default:
      throw new UnsupportedOperationException("Unsupported array element type: " + elementType);
  }
}
 
Example #8
Source File: PartitionKey.java    From iceberg with Apache License 2.0 5 votes vote down vote up
@Override
public Object get(InternalRow row) {
  if (row.isNullAt(p)) {
    return null;
  }
  return ((Decimal) row.get(p, type)).toJavaBigDecimal();
}
 
Example #9
Source File: Reader.java    From iceberg with Apache License 2.0 5 votes vote down vote up
/**
 * Converts the objects into instances used by Spark's InternalRow.
 *
 * @param value a data value
 * @param type the Spark data type
 * @return the value converted to the representation expected by Spark's InternalRow.
 */
private static Object convert(Object value, DataType type) {
  if (type instanceof StringType) {
    return UTF8String.fromString(value.toString());
  } else if (type instanceof BinaryType) {
    ByteBuffer buffer = (ByteBuffer) value;
    return buffer.get(new byte[buffer.remaining()]);
  } else if (type instanceof DecimalType) {
    return Decimal.fromDecimal(value);
  }
  return value;
}
 
Example #10
Source File: SparkOrcReader.java    From iceberg with Apache License 2.0 5 votes vote down vote up
static Converter buildConverter(BufferHolder holder, TypeDescription schema) {
  switch (schema.getCategory()) {
    case BOOLEAN:
      return new BooleanConverter();
    case BYTE:
      return new ByteConverter();
    case SHORT:
      return new ShortConverter();
    case DATE:
    case INT:
      return new IntConverter();
    case LONG:
      return new LongConverter();
    case FLOAT:
      return new FloatConverter();
    case DOUBLE:
      return new DoubleConverter();
    case TIMESTAMP:
      return new TimestampConverter();
    case DECIMAL:
      if (schema.getPrecision() <= Decimal.MAX_LONG_DIGITS()) {
        return new Decimal18Converter(schema.getPrecision(), schema.getScale());
      } else {
        return new Decimal38Converter(schema.getPrecision(), schema.getScale());
      }
    case BINARY:
    case STRING:
    case CHAR:
    case VARCHAR:
      return new BinaryConverter(holder);
    case STRUCT:
      return new StructConverter(holder, schema);
    case LIST:
      return new ListConverter(holder, schema);
    case MAP:
      return new MapConverter(holder, schema);
    default:
      throw new IllegalArgumentException("Unhandled type " + schema);
  }
}
 
Example #11
Source File: SparkOrcReader.java    From iceberg with Apache License 2.0 5 votes vote down vote up
@Override
public void convert(UnsafeArrayWriter writer, int element,
                    ColumnVector vector, int row) {
  if (vector.isRepeating) {
    row = 0;
  }
  if (!vector.noNulls && vector.isNull[row]) {
    writer.setNull(element);
  } else {
    HiveDecimalWritable v = ((DecimalColumnVector) vector).vector[row];
    writer.write(element,
        new Decimal().set(hack.unscaledLong(v), precision, v.scale()),
        precision, scale);
  }
}
 
Example #12
Source File: SparkOrcReader.java    From iceberg with Apache License 2.0 5 votes vote down vote up
@Override
public void convert(UnsafeRowWriter writer, int column, ColumnVector vector,
                    int row) {
  if (vector.isRepeating) {
    row = 0;
  }
  if (!vector.noNulls && vector.isNull[row]) {
    writer.setNullAt(column);
  } else {
    HiveDecimalWritable v = ((DecimalColumnVector) vector).vector[row];
    writer.write(column,
        new Decimal().set(hack.unscaledLong(v), precision, v.scale()),
        precision, scale);
  }
}
 
Example #13
Source File: SparkExpressions.java    From iceberg with Apache License 2.0 5 votes vote down vote up
private static Object valueFromSpark(Literal lit) {
  if (lit.value() instanceof UTF8String) {
    return lit.value().toString();
  } else if (lit.value() instanceof Decimal) {
    return ((Decimal) lit.value()).toJavaBigDecimal();
  }
  return lit.value();
}
 
Example #14
Source File: PartitionKey.java    From iceberg with Apache License 2.0 5 votes vote down vote up
@Override
public Object get(InternalRow row) {
  if (row.isNullAt(position())) {
    return null;
  }
  return ((Decimal) row.get(position(), type())).toJavaBigDecimal();
}
 
Example #15
Source File: RowDataReader.java    From iceberg with Apache License 2.0 5 votes vote down vote up
private static Object convertConstant(Type type, Object value) {
  if (value == null) {
    return null;
  }

  switch (type.typeId()) {
    case DECIMAL:
      return Decimal.apply((BigDecimal) value);
    case STRING:
      if (value instanceof Utf8) {
        Utf8 utf8 = (Utf8) value;
        return UTF8String.fromBytes(utf8.getBytes(), 0, utf8.getByteLength());
      }
      return UTF8String.fromString(value.toString());
    case FIXED:
      if (value instanceof byte[]) {
        return value;
      } else if (value instanceof GenericData.Fixed) {
        return ((GenericData.Fixed) value).bytes();
      }
      return ByteBuffers.toByteArray((ByteBuffer) value);
    case BINARY:
      return ByteBuffers.toByteArray((ByteBuffer) value);
    default:
  }
  return value;
}
 
Example #16
Source File: FlightArrowColumnVector.java    From flight-spark-source with Apache License 2.0 5 votes vote down vote up
@Override
public Decimal getDecimal(int rowId, int precision, int scale) {
  if (isNullAt(rowId)) {
    return null;
  }
  return accessor.getDecimal(rowId, precision, scale);
}
 
Example #17
Source File: ArrowVectorAccessors.java    From iceberg with Apache License 2.0 5 votes vote down vote up
@Override
final Decimal getDecimal(int rowId, int precision, int scale) {
  int dictId = offsetVector.get(rowId);
  if (cache[dictId] == null) {
    cache[dictId] = Decimal.apply(parquetDictionary.decodeToInt(dictId), precision, scale);
  }
  return cache[dictId];
}
 
Example #18
Source File: SparkParquetWriters.java    From iceberg with Apache License 2.0 5 votes vote down vote up
@Override
public void write(int repetitionLevel, Decimal decimal) {
  Preconditions.checkArgument(decimal.scale() == scale,
      "Cannot write value as decimal(%s,%s), wrong scale: %s", precision, scale, decimal);
  Preconditions.checkArgument(decimal.precision() <= precision,
      "Cannot write value as decimal(%s,%s), too large: %s", precision, scale, decimal);

  column.writeInteger(repetitionLevel, (int) decimal.toUnscaledLong());
}
 
Example #19
Source File: SparkParquetWriters.java    From iceberg with Apache License 2.0 5 votes vote down vote up
@Override
public void write(int repetitionLevel, Decimal decimal) {
  Preconditions.checkArgument(decimal.scale() == scale,
      "Cannot write value as decimal(%s,%s), wrong scale: %s", precision, scale, decimal);
  Preconditions.checkArgument(decimal.precision() <= precision,
      "Cannot write value as decimal(%s,%s), too large: %s", precision, scale, decimal);

  column.writeLong(repetitionLevel, decimal.toUnscaledLong());
}
 
Example #20
Source File: IcebergArrowColumnVector.java    From iceberg with Apache License 2.0 5 votes vote down vote up
@Override
public Decimal getDecimal(int rowId, int precision, int scale) {
  if (isNullAt(rowId)) {
    return null;
  }
  return accessor.getDecimal(rowId, precision, scale);
}
 
Example #21
Source File: FlightArrowColumnVector.java    From flight-spark-source with Apache License 2.0 5 votes vote down vote up
@Override
final Decimal getDecimal(int rowId, int precision, int scale) {
  if (isNullAt(rowId)) {
    return null;
  }
  return Decimal.apply(accessor.getObject(rowId), precision, scale);
}
 
Example #22
Source File: ArrowVectorAccessors.java    From iceberg with Apache License 2.0 5 votes vote down vote up
@Override
final Decimal getDecimal(int rowId, int precision, int scale) {
  int dictId = offsetVector.get(rowId);
  if (cache[dictId] == null) {
    cache[dictId] = Decimal.apply(parquetDictionary.decodeToLong(dictId), precision, scale);
  }
  return cache[dictId];
}
 
Example #23
Source File: ArrowVectorAccessors.java    From iceberg with Apache License 2.0 5 votes vote down vote up
@Override
final Decimal getDecimal(int rowId, int precision, int scale) {
  int dictId = offsetVector.get(rowId);
  if (cache[dictId] == null) {
    cache[dictId] = Decimal.apply(
        new BigInteger(parquetDictionary.decodeToBinary(dictId).getBytes()).longValue(),
        precision,
        scale);
  }
  return cache[dictId];
}
 
Example #24
Source File: SparkParquetWriters.java    From iceberg with Apache License 2.0 4 votes vote down vote up
private static PrimitiveWriter<Decimal> decimalAsLong(ColumnDescriptor desc,
                                                      int precision, int scale) {
  return new LongDecimalWriter(desc, precision, scale);
}
 
Example #25
Source File: FlightArrowColumnVector.java    From flight-spark-source with Apache License 2.0 4 votes vote down vote up
Decimal getDecimal(int rowId, int precision, int scale) {
  throw new UnsupportedOperationException();
}
 
Example #26
Source File: SparkParquetReaders.java    From iceberg with Apache License 2.0 4 votes vote down vote up
@Override
public Decimal read(Decimal ignored) {
  Binary binary = column.nextBinary();
  return Decimal.fromDecimal(new BigDecimal(new BigInteger(binary.getBytes()), scale));
}
 
Example #27
Source File: SparkParquetReaders.java    From iceberg with Apache License 2.0 4 votes vote down vote up
@Override
public Decimal read(Decimal ignored) {
  return Decimal.apply(column.nextInteger(), precision, scale);
}
 
Example #28
Source File: SparkParquetReaders.java    From iceberg with Apache License 2.0 4 votes vote down vote up
@Override
public Decimal read(Decimal ignored) {
  return Decimal.apply(column.nextLong(), precision, scale);
}
 
Example #29
Source File: SparkParquetReaders.java    From iceberg with Apache License 2.0 4 votes vote down vote up
@Override
public Decimal getDecimal(int ordinal, int precision, int scale) {
  return (Decimal) values[ordinal];
}
 
Example #30
Source File: SparkValueReaders.java    From iceberg with Apache License 2.0 4 votes vote down vote up
static ValueReader<Decimal> decimal(ValueReader<byte[]> unscaledReader, int scale) {
  return new DecimalReader(unscaledReader, scale);
}