org.apache.iceberg.types.Types.DoubleType Java Examples
The following examples show how to use
org.apache.iceberg.types.Types.DoubleType.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
| Source File: ArrowSchemaUtilTest.java From iceberg with Apache License 2.0 | 6 votes |
|
@Test
public void convertPrimitive() {
Schema iceberg = new Schema(
Types.NestedField.optional(0, INTEGER_FIELD, IntegerType.get()),
Types.NestedField.optional(1, BOOLEAN_FIELD, BooleanType.get()),
Types.NestedField.required(2, DOUBLE_FIELD, DoubleType.get()),
Types.NestedField.required(3, STRING_FIELD, StringType.get()),
Types.NestedField.optional(4, DATE_FIELD, DateType.get()),
Types.NestedField.optional(5, TIMESTAMP_FIELD, TimestampType.withZone()),
Types.NestedField.optional(6, LONG_FIELD, LongType.get()),
Types.NestedField.optional(7, FLOAT_FIELD, FloatType.get()),
Types.NestedField.optional(8, TIME_FIELD, TimeType.get()),
Types.NestedField.optional(9, BINARY_FIELD, Types.BinaryType.get()),
Types.NestedField.optional(10, DECIMAL_FIELD, Types.DecimalType.of(1, 1)),
Types.NestedField.optional(12, LIST_FIELD, Types.ListType.ofOptional(13, Types.IntegerType.get())),
Types.NestedField.required(14, MAP_FIELD, Types.MapType.ofOptional(15, 16,
StringType.get(), IntegerType.get())),
Types.NestedField.optional(17, FIXED_WIDTH_BINARY_FIELD, Types.FixedType.ofLength(10)));
org.apache.arrow.vector.types.pojo.Schema arrow = ArrowSchemaUtil.convert(iceberg);
validate(iceberg, arrow);
}
Example #2
| Source File: SchemaUtilTest.java From iceberg with Apache License 2.0 | 6 votes |
|
@Test
public void testPrimitive() throws IOException {
Schema icebergSchema = new Schema(
optional(1, "b", BooleanType.get()),
optional(2, "i", IntegerType.get()),
optional(3, "l", LongType.get()),
optional(4, "f", FloatType.get()),
optional(5, "d", DoubleType.get()),
optional(6, "dec", DecimalType.of(0, 2)),
optional(7, "s", StringType.get()),
optional(8, "bi", BinaryType.get())
);
ResourceSchema pigSchema = SchemaUtil.convert(icebergSchema);
assertEquals(
"b:boolean,i:int,l:long,f:float,d:double,dec:bigdecimal,s:chararray,bi:bytearray", pigSchema.toString());
}
Example #3
| Source File: TestPartitionFields.java From presto with Apache License 2.0 | 5 votes |
|
private static PartitionSpec partitionSpec(Consumer<PartitionSpec.Builder> consumer)
{
Schema schema = new Schema(
NestedField.required(1, "order_key", LongType.get()),
NestedField.required(2, "ts", TimestampType.withoutZone()),
NestedField.required(3, "price", DoubleType.get()),
NestedField.optional(4, "comment", StringType.get()),
NestedField.optional(5, "notes", ListType.ofRequired(6, StringType.get())));
PartitionSpec.Builder builder = PartitionSpec.builderFor(schema);
consumer.accept(builder);
return builder.build();
}
Example #4
| Source File: SchemaUtilTest.java From iceberg with Apache License 2.0 | 5 votes |
|
@Test
public void testComplex() throws IOException {
convertToPigSchema(
new Schema(
optional(1, "bag", ListType.ofOptional(2, BooleanType.get())),
optional(3, "map", MapType.ofOptional(4, 5, StringType.get(), DoubleType.get())),
optional(6, "tuple",
StructType.of(optional(7, "i", IntegerType.get()), optional(8, "f", FloatType.get())))
), "bag:{(boolean)},map:[double],tuple:(i:int,f:float)", null
);
}
Example #5
| Source File: SchemaUtilTest.java From iceberg with Apache License 2.0 | 5 votes |
|
@Test
public void nestedBags() throws IOException {
convertToPigSchema(new Schema(
optional(1, "nested",
ListType.ofOptional(2,
ListType.ofOptional(3,
ListType.ofOptional(4, DoubleType.get()))))
), "nested:{({({(double)})})}", "");
}
Example #6
| Source File: SchemaUtilTest.java From iceberg with Apache License 2.0 | 5 votes |
|
@Test
public void mapConversions() throws IOException {
// consistent behavior for maps conversions. The below test case, correctly does not specify map key types
convertToPigSchema(
new Schema(
required(
1, "a",
MapType.ofRequired(
2, 3,
StringType.get(),
ListType.ofRequired(
4, StructType.of(
required(5, "b", LongType.get()),
required(6, "c", StringType.get())))))),
"a:[{(b:long,c:chararray)}]",
"We do not specify the map key type here");
// struct<a:map<string,map<string,double>>> -> (a:[[double]])
// As per https://pig.apache.org/docs/latest/basic.html#map-schema. It seems that
// we only need to specify value type as keys are always of type chararray
convertToPigSchema(
new Schema(
StructType.of(
required(1, "a", MapType.ofRequired(
2, 3,
StringType.get(),
MapType.ofRequired(4, 5, StringType.get(), DoubleType.get())))
).fields()),
"a:[[double]]",
"A map key type does not need to be specified");
}
Example #7
| Source File: SchemaUtilTest.java From iceberg with Apache License 2.0 | 4 votes |
|
@Test(expected = FrontendException.class)
public void invalidMap() throws IOException {
convertToPigSchema(new Schema(
optional(1, "invalid", MapType.ofOptional(2, 3, IntegerType.get(), DoubleType.get()))
), "", "");
}
Example #8
| Source File: TestMetrics.java From iceberg with Apache License 2.0 | 4 votes |
|
@Test
public void testMetricsForTopLevelWithMultipleRowGroup() throws Exception {
Assume.assumeTrue("Skip test for formats that do not support small row groups", supportsSmallRowGroups());
int recordCount = 201;
List<Record> records = new ArrayList<>(recordCount);
for (int i = 0; i < recordCount; i++) {
Record newRecord = GenericRecord.create(SIMPLE_SCHEMA);
newRecord.setField("booleanCol", i == 0 ? false : true);
newRecord.setField("intCol", i + 1);
newRecord.setField("longCol", i == 0 ? null : i + 1L);
newRecord.setField("floatCol", i + 1.0F);
newRecord.setField("doubleCol", i == 0 ? null : i + 1.0D);
newRecord.setField("decimalCol", i == 0 ? null : new BigDecimal(i + "").add(new BigDecimal("1.00")));
newRecord.setField("stringCol", "AAA");
newRecord.setField("dateCol", DateTimeUtil.dateFromDays(i + 1));
newRecord.setField("timeCol", DateTimeUtil.timeFromMicros(i + 1L));
newRecord.setField("timestampColAboveEpoch", DateTimeUtil.timestampFromMicros(i + 1L));
newRecord.setField("fixedCol", fixed);
newRecord.setField("binaryCol", ByteBuffer.wrap("S".getBytes()));
newRecord.setField("timestampColBelowEpoch", DateTimeUtil.timestampFromMicros((i + 1L) * -1L));
records.add(newRecord);
}
// create file with multiple row groups. by using smaller number of bytes
InputFile recordsFile = writeRecordsWithSmallRowGroups(SIMPLE_SCHEMA, records.toArray(new Record[0]));
Assert.assertNotNull(recordsFile);
// rowgroup size should be > 1
Assert.assertEquals(3, splitCount(recordsFile));
Metrics metrics = getMetrics(recordsFile);
Assert.assertEquals(201L, (long) metrics.recordCount());
assertCounts(1, 201L, 0L, metrics);
assertBounds(1, Types.BooleanType.get(), false, true, metrics);
assertBounds(2, Types.IntegerType.get(), 1, 201, metrics);
assertCounts(3, 201L, 1L, metrics);
assertBounds(3, Types.LongType.get(), 2L, 201L, metrics);
assertCounts(4, 201L, 0L, metrics);
assertBounds(4, Types.FloatType.get(), 1.0F, 201.0F, metrics);
assertCounts(5, 201L, 1L, metrics);
assertBounds(5, Types.DoubleType.get(), 2.0D, 201.0D, metrics);
assertCounts(6, 201L, 1L, metrics);
assertBounds(6, Types.DecimalType.of(10, 2), new BigDecimal("2.00"),
new BigDecimal("201.00"), metrics);
}
