org.apache.iceberg.types.Types.LongType Java Examples

The following examples show how to use org.apache.iceberg.types.Types.LongType. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: SchemaUtilTest.java    From iceberg with Apache License 2.0 6 votes vote down vote up
@Test
public void testTupleInMap() throws IOException {
  Schema icebergSchema = new Schema(
      optional(
          1, "nested_list",
          MapType.ofOptional(
              2, 3,
              StringType.get(),
              ListType.ofOptional(
                  4, StructType.of(
                      required(5, "id", LongType.get()),
                      optional(6, "data", StringType.get()))))));

  ResourceSchema pigSchema = SchemaUtil.convert(icebergSchema);
  // The output should contain a nested struct within a list within a map, I think.
  assertEquals("nested_list:[{(id:long,data:chararray)}]", pigSchema.toString());
}
 
Example #2
Source File: SchemaUtilTest.java    From iceberg with Apache License 2.0 6 votes vote down vote up
@Test
public void testPrimitive() throws IOException {
  Schema icebergSchema = new Schema(
      optional(1, "b", BooleanType.get()),
      optional(2, "i", IntegerType.get()),
      optional(3, "l", LongType.get()),
      optional(4, "f", FloatType.get()),
      optional(5, "d", DoubleType.get()),
      optional(6, "dec", DecimalType.of(0, 2)),
      optional(7, "s", StringType.get()),
      optional(8, "bi", BinaryType.get())
  );

  ResourceSchema pigSchema = SchemaUtil.convert(icebergSchema);
  assertEquals(
      "b:boolean,i:int,l:long,f:float,d:double,dec:bigdecimal,s:chararray,bi:bytearray", pigSchema.toString());
}
 
Example #3
Source File: ArrowSchemaUtilTest.java    From iceberg with Apache License 2.0 6 votes vote down vote up
@Test
public void convertPrimitive() {
  Schema iceberg = new Schema(
      Types.NestedField.optional(0, INTEGER_FIELD, IntegerType.get()),
      Types.NestedField.optional(1, BOOLEAN_FIELD, BooleanType.get()),
      Types.NestedField.required(2, DOUBLE_FIELD, DoubleType.get()),
      Types.NestedField.required(3, STRING_FIELD, StringType.get()),
      Types.NestedField.optional(4, DATE_FIELD, DateType.get()),
      Types.NestedField.optional(5, TIMESTAMP_FIELD, TimestampType.withZone()),
      Types.NestedField.optional(6, LONG_FIELD, LongType.get()),
      Types.NestedField.optional(7, FLOAT_FIELD, FloatType.get()),
      Types.NestedField.optional(8, TIME_FIELD, TimeType.get()),
      Types.NestedField.optional(9, BINARY_FIELD, Types.BinaryType.get()),
      Types.NestedField.optional(10, DECIMAL_FIELD, Types.DecimalType.of(1, 1)),
      Types.NestedField.optional(12, LIST_FIELD, Types.ListType.ofOptional(13, Types.IntegerType.get())),
      Types.NestedField.required(14, MAP_FIELD, Types.MapType.ofOptional(15, 16,
          StringType.get(), IntegerType.get())),
      Types.NestedField.optional(17, FIXED_WIDTH_BINARY_FIELD, Types.FixedType.ofLength(10)));

  org.apache.arrow.vector.types.pojo.Schema arrow = ArrowSchemaUtil.convert(iceberg);

  validate(iceberg, arrow);
}
 
Example #4
Source File: AvroDataTest.java    From iceberg with Apache License 2.0 5 votes vote down vote up
@Test
public void testMixedTypes() throws IOException {
  StructType structType = StructType.of(
      required(0, "id", LongType.get()),
      optional(1, "list_of_maps",
          ListType.ofOptional(2, MapType.ofOptional(3, 4,
              Types.StringType.get(),
              SUPPORTED_PRIMITIVES))),
      optional(5, "map_of_lists",
          MapType.ofOptional(6, 7,
              Types.StringType.get(),
              ListType.ofOptional(8, SUPPORTED_PRIMITIVES))),
      required(9, "list_of_lists",
          ListType.ofOptional(10, ListType.ofOptional(11, SUPPORTED_PRIMITIVES))),
      required(12, "map_of_maps",
          MapType.ofOptional(13, 14,
              Types.StringType.get(),
              MapType.ofOptional(15, 16,
                  Types.StringType.get(),
                  SUPPORTED_PRIMITIVES))),
      required(17, "list_of_struct_of_nested_types", ListType.ofOptional(19, StructType.of(
          Types.NestedField.required(20, "m1", MapType.ofOptional(21, 22,
              Types.StringType.get(),
              SUPPORTED_PRIMITIVES)),
          Types.NestedField.optional(23, "l1", ListType.ofRequired(24, SUPPORTED_PRIMITIVES)),
          Types.NestedField.required(25, "l2", ListType.ofRequired(26, SUPPORTED_PRIMITIVES)),
          Types.NestedField.optional(27, "m2", MapType.ofOptional(28, 29,
              Types.StringType.get(),
              SUPPORTED_PRIMITIVES))
      )))
  );

  Schema schema = new Schema(TypeUtil.assignFreshIds(structType, new AtomicInteger(0)::incrementAndGet)
      .asStructType().fields());

  writeAndValidate(schema);
}
 
Example #5
Source File: TestPartitionFields.java    From presto with Apache License 2.0 5 votes vote down vote up
private static PartitionSpec partitionSpec(Consumer<PartitionSpec.Builder> consumer)
{
    Schema schema = new Schema(
            NestedField.required(1, "order_key", LongType.get()),
            NestedField.required(2, "ts", TimestampType.withoutZone()),
            NestedField.required(3, "price", DoubleType.get()),
            NestedField.optional(4, "comment", StringType.get()),
            NestedField.optional(5, "notes", ListType.ofRequired(6, StringType.get())));

    PartitionSpec.Builder builder = PartitionSpec.builderFor(schema);
    consumer.accept(builder);
    return builder.build();
}
 
Example #6
Source File: AvroDataTest.java    From iceberg with Apache License 2.0 5 votes vote down vote up
@Test
public void testMapOfStructs() throws IOException {
  Schema schema = new Schema(
      required(0, "id", LongType.get()),
      optional(1, "data", MapType.ofOptional(2, 3,
          Types.StringType.get(),
          SUPPORTED_PRIMITIVES)));

  writeAndValidate(schema);
}
 
Example #7
Source File: AvroDataTest.java    From iceberg with Apache License 2.0 5 votes vote down vote up
@Test
public void testComplexMapKey() throws IOException {
  Schema schema = new Schema(
      required(0, "id", LongType.get()),
      optional(1, "data", MapType.ofOptional(2, 3,
          Types.StructType.of(
              required(4, "i", Types.IntegerType.get()),
              optional(5, "s", Types.StringType.get())),
          Types.StringType.get())));

  writeAndValidate(schema);
}
 
Example #8
Source File: AvroDataTest.java    From iceberg with Apache License 2.0 5 votes vote down vote up
@Test
public void testNumericMapKey() throws IOException {
  Schema schema = new Schema(
      required(0, "id", LongType.get()),
      optional(1, "data", MapType.ofOptional(2, 3,
          Types.LongType.get(),
          Types.StringType.get())));

  writeAndValidate(schema);
}
 
Example #9
Source File: AvroDataTest.java    From iceberg with Apache License 2.0 5 votes vote down vote up
@Test
public void testMap() throws IOException {
  Schema schema = new Schema(
      required(0, "id", LongType.get()),
      optional(1, "data", MapType.ofOptional(2, 3,
          Types.StringType.get(),
          Types.StringType.get())));

  writeAndValidate(schema);
}
 
Example #10
Source File: AvroDataTest.java    From iceberg with Apache License 2.0 5 votes vote down vote up
@Test
public void testArrayOfStructs() throws IOException {
  Schema schema = new Schema(
      required(0, "id", LongType.get()),
      optional(1, "data", ListType.ofOptional(2, SUPPORTED_PRIMITIVES)));

  writeAndValidate(schema);
}
 
Example #11
Source File: AvroDataTest.java    From iceberg with Apache License 2.0 5 votes vote down vote up
@Test
public void testArray() throws IOException {
  Schema schema = new Schema(
      required(0, "id", LongType.get()),
      optional(1, "data", ListType.ofOptional(2, Types.StringType.get())));

  writeAndValidate(schema);
}
 
Example #12
Source File: TestMetrics.java    From iceberg with Apache License 2.0 5 votes vote down vote up
@Test
public void testMetricsForNestedStructFieldsWithMultipleRowGroup() throws IOException {
  Assume.assumeTrue("Skip test for formats that do not support small row groups", supportsSmallRowGroups());

  int recordCount = 201;
  List<Record> records = Lists.newArrayListWithExpectedSize(recordCount);

  for (int i = 0; i < recordCount; i++) {
    Record newLeafStruct = GenericRecord.create(LEAF_STRUCT_TYPE);
    newLeafStruct.setField("leafLongCol", i + 1L);
    newLeafStruct.setField("leafBinaryCol", ByteBuffer.wrap("A".getBytes()));
    Record newNestedStruct = GenericRecord.create(NESTED_STRUCT_TYPE);
    newNestedStruct.setField("longCol", i + 1L);
    newNestedStruct.setField("leafStructCol", newLeafStruct);
    Record newRecord = GenericRecord.create(NESTED_SCHEMA);
    newRecord.setField("intCol", i + 1);
    newRecord.setField("nestedStructCol", newNestedStruct);
    records.add(newRecord);
  }

  // create file with multiple row groups. by using smaller number of bytes
  InputFile recordsFile = writeRecordsWithSmallRowGroups(NESTED_SCHEMA, records.toArray(new Record[0]));

  Assert.assertNotNull(recordsFile);
  // rowgroup size should be > 1
  Assert.assertEquals(3, splitCount(recordsFile));

  Metrics metrics = getMetrics(recordsFile);
  Assert.assertEquals(201L, (long) metrics.recordCount());
  assertCounts(1, 201L, 0L, metrics);
  assertBounds(1, IntegerType.get(), 1, 201, metrics);
  assertCounts(3, 201L, 0L, metrics);
  assertBounds(3, LongType.get(), 1L, 201L, metrics);
  assertCounts(5, 201L, 0L, metrics);
  assertBounds(5, LongType.get(), 1L, 201L, metrics);
  assertCounts(6, 201L, 0L, metrics);
  assertBounds(6, BinaryType.get(),
      ByteBuffer.wrap("A".getBytes()), ByteBuffer.wrap("A".getBytes()), metrics);
}
 
Example #13
Source File: TestMetrics.java    From iceberg with Apache License 2.0 5 votes vote down vote up
@Test
public void testMetricsForNestedStructFields() throws IOException {

  Record leafStruct = GenericRecord.create(LEAF_STRUCT_TYPE);
  leafStruct.setField("leafLongCol", 20L);
  leafStruct.setField("leafBinaryCol", ByteBuffer.wrap("A".getBytes()));
  Record nestedStruct = GenericRecord.create(NESTED_STRUCT_TYPE);
  nestedStruct.setField("longCol", 100L);
  nestedStruct.setField("leafStructCol", leafStruct);
  Record record = GenericRecord.create(NESTED_SCHEMA);
  record.setField("intCol", Integer.MAX_VALUE);
  record.setField("nestedStructCol", nestedStruct);

  InputFile recordsFile = writeRecords(NESTED_SCHEMA, record);

  Metrics metrics = getMetrics(recordsFile);
  Assert.assertEquals(1L, (long) metrics.recordCount());
  assertCounts(1, 1L, 0L, metrics);
  assertBounds(1, IntegerType.get(), Integer.MAX_VALUE, Integer.MAX_VALUE, metrics);
  assertCounts(3, 1L, 0L, metrics);
  assertBounds(3, LongType.get(), 100L, 100L, metrics);
  assertCounts(5, 1L, 0L, metrics);
  assertBounds(5, LongType.get(), 20L, 20L, metrics);
  assertCounts(6, 1L, 0L, metrics);
  assertBounds(6, BinaryType.get(),
      ByteBuffer.wrap("A".getBytes()), ByteBuffer.wrap("A".getBytes()), metrics);
}
 
Example #14
Source File: SchemaUtilTest.java    From iceberg with Apache License 2.0 5 votes vote down vote up
@Test
public void testLongInBag() throws IOException {
  Schema icebergSchema = new Schema(
      optional(
          1, "nested_list",
          MapType.ofOptional(
              2, 3,
              StringType.get(),
              ListType.ofRequired(5, LongType.get()))));
  SchemaUtil.convert(icebergSchema);
}
 
Example #15
Source File: SchemaUtilTest.java    From iceberg with Apache License 2.0 5 votes vote down vote up
@Test
public void mapConversions() throws IOException {
  // consistent behavior for maps conversions. The below test case, correctly does not specify map key types
  convertToPigSchema(
      new Schema(
          required(
              1, "a",
              MapType.ofRequired(
                  2, 3,
                  StringType.get(),
                  ListType.ofRequired(
                      4, StructType.of(
                          required(5, "b", LongType.get()),
                          required(6, "c", StringType.get())))))),
      "a:[{(b:long,c:chararray)}]",
      "We do not specify the map key type here");
  // struct<a:map<string,map<string,double>>> -> (a:[[double]])
  // As per https://pig.apache.org/docs/latest/basic.html#map-schema. It seems that
  // we  only need to specify value type as keys are always of type chararray
  convertToPigSchema(
      new Schema(
          StructType.of(
              required(1, "a", MapType.ofRequired(
                  2, 3,
                  StringType.get(),
                  MapType.ofRequired(4, 5, StringType.get(), DoubleType.get())))
          ).fields()),
      "a:[[double]]",
      "A map key type does not need to be specified");
}
 
Example #16
Source File: ArrowSchemaUtilTest.java    From iceberg with Apache License 2.0 5 votes vote down vote up
@Test
public void convertComplex() {
  Schema iceberg = new Schema(
      Types.NestedField.optional(0, "m", MapType.ofOptional(
          1, 2, StringType.get(),
          LongType.get())
      ),
      Types.NestedField.required(3, "m2", MapType.ofOptional(
          4, 5, StringType.get(),
          ListType.ofOptional(6, TimestampType.withoutZone()))
      )
  );
  org.apache.arrow.vector.types.pojo.Schema arrow = ArrowSchemaUtil.convert(iceberg);
  Assert.assertEquals(iceberg.columns().size(), arrow.getFields().size());
}
 
Example #17
Source File: AvroDataTest.java    From iceberg with Apache License 2.0 5 votes vote down vote up
@Test
public void testMixedTypes() throws IOException {
  StructType structType = StructType.of(
      required(0, "id", LongType.get()),
      optional(1, "list_of_maps",
          ListType.ofOptional(2, MapType.ofOptional(3, 4,
              Types.StringType.get(),
              SUPPORTED_PRIMITIVES))),
      optional(5, "map_of_lists",
          MapType.ofOptional(6, 7,
              Types.StringType.get(),
              ListType.ofOptional(8, SUPPORTED_PRIMITIVES))),
      required(9, "list_of_lists",
          ListType.ofOptional(10, ListType.ofOptional(11, SUPPORTED_PRIMITIVES))),
      required(12, "map_of_maps",
          MapType.ofOptional(13, 14,
              Types.StringType.get(),
              MapType.ofOptional(15, 16,
                  Types.StringType.get(),
                  SUPPORTED_PRIMITIVES))),
      required(17, "list_of_struct_of_nested_types", ListType.ofOptional(19, StructType.of(
          Types.NestedField.required(20, "m1", MapType.ofOptional(21, 22,
              Types.StringType.get(),
              SUPPORTED_PRIMITIVES)),
          Types.NestedField.optional(23, "l1", ListType.ofRequired(24, SUPPORTED_PRIMITIVES)),
          Types.NestedField.required(25, "l2", ListType.ofRequired(26, SUPPORTED_PRIMITIVES)),
          Types.NestedField.optional(27, "m2", MapType.ofOptional(28, 29,
              Types.StringType.get(),
              SUPPORTED_PRIMITIVES))
      )))
  );

  Schema schema = new Schema(TypeUtil.assignFreshIds(structType, new AtomicInteger(0)::incrementAndGet)
      .asStructType().fields());

  writeAndValidate(schema);
}
 
Example #18
Source File: AvroDataTest.java    From iceberg with Apache License 2.0 5 votes vote down vote up
@Test
public void testMapOfStructs() throws IOException {
  Schema schema = TypeUtil.assignIncreasingFreshIds(new Schema(
      required(0, "id", LongType.get()),
      optional(1, "data", MapType.ofOptional(2, 3,
          Types.StringType.get(),
          SUPPORTED_PRIMITIVES))));

  writeAndValidate(schema);
}
 
Example #19
Source File: AvroDataTest.java    From iceberg with Apache License 2.0 5 votes vote down vote up
@Test
public void testComplexMapKey() throws IOException {
  Schema schema = new Schema(
      required(0, "id", LongType.get()),
      optional(1, "data", MapType.ofOptional(2, 3,
          Types.StructType.of(
              required(4, "i", Types.IntegerType.get()),
              optional(5, "s", Types.StringType.get())),
          Types.StringType.get())));

  writeAndValidate(schema);
}
 
Example #20
Source File: AvroDataTest.java    From iceberg with Apache License 2.0 5 votes vote down vote up
@Test
public void testNumericMapKey() throws IOException {
  Schema schema = new Schema(
      required(0, "id", LongType.get()),
      optional(1, "data", MapType.ofOptional(2, 3,
          Types.LongType.get(),
          Types.StringType.get())));

  writeAndValidate(schema);
}
 
Example #21
Source File: AvroDataTest.java    From iceberg with Apache License 2.0 5 votes vote down vote up
@Test
public void testMap() throws IOException {
  Schema schema = new Schema(
      required(0, "id", LongType.get()),
      optional(1, "data", MapType.ofOptional(2, 3,
          Types.StringType.get(),
          Types.StringType.get())));

  writeAndValidate(schema);
}
 
Example #22
Source File: AvroDataTest.java    From iceberg with Apache License 2.0 5 votes vote down vote up
@Test
public void testArrayOfStructs() throws IOException {
  Schema schema = TypeUtil.assignIncreasingFreshIds(new Schema(
      required(0, "id", LongType.get()),
      optional(1, "data", ListType.ofOptional(2, SUPPORTED_PRIMITIVES))));

  writeAndValidate(schema);
}
 
Example #23
Source File: AvroDataTest.java    From iceberg with Apache License 2.0 5 votes vote down vote up
@Test
public void testArray() throws IOException {
  Schema schema = new Schema(
      required(0, "id", LongType.get()),
      optional(1, "data", ListType.ofOptional(2, Types.StringType.get())));

  writeAndValidate(schema);
}
 
Example #24
Source File: DataTest.java    From iceberg with Apache License 2.0 5 votes vote down vote up
@Test
public void testMixedTypes() throws IOException {
  StructType structType = StructType.of(
      required(0, "id", LongType.get()),
      optional(1, "list_of_maps",
          ListType.ofOptional(2, MapType.ofOptional(3, 4,
              Types.StringType.get(),
              SUPPORTED_PRIMITIVES))),
      optional(5, "map_of_lists",
          MapType.ofOptional(6, 7,
              Types.StringType.get(),
              ListType.ofOptional(8, SUPPORTED_PRIMITIVES))),
      required(9, "list_of_lists",
          ListType.ofOptional(10, ListType.ofOptional(11, SUPPORTED_PRIMITIVES))),
      required(12, "map_of_maps",
          MapType.ofOptional(13, 14,
              Types.StringType.get(),
              MapType.ofOptional(15, 16,
                  Types.StringType.get(),
                  SUPPORTED_PRIMITIVES))),
      required(17, "list_of_struct_of_nested_types", ListType.ofOptional(19, StructType.of(
          Types.NestedField.required(20, "m1", MapType.ofOptional(21, 22,
              Types.StringType.get(),
              SUPPORTED_PRIMITIVES)),
          Types.NestedField.optional(23, "l1", ListType.ofRequired(24, SUPPORTED_PRIMITIVES)),
          Types.NestedField.required(25, "l2", ListType.ofRequired(26, SUPPORTED_PRIMITIVES)),
          Types.NestedField.optional(27, "m2", MapType.ofOptional(28, 29,
              Types.StringType.get(),
              SUPPORTED_PRIMITIVES))
      )))
  );

  Schema schema = new Schema(TypeUtil.assignFreshIds(structType, new AtomicInteger(0)::incrementAndGet)
      .asStructType().fields());

  writeAndValidate(schema);
}
 
Example #25
Source File: DataTest.java    From iceberg with Apache License 2.0 5 votes vote down vote up
@Test
public void testMapOfStructs() throws IOException {
  Schema schema = new Schema(
      required(0, "id", LongType.get()),
      optional(1, "data", MapType.ofOptional(2, 3,
          Types.StringType.get(),
          SUPPORTED_PRIMITIVES)));

  writeAndValidate(schema);
}
 
Example #26
Source File: DataTest.java    From iceberg with Apache License 2.0 5 votes vote down vote up
@Test
public void testComplexMapKey() throws IOException {
  Schema schema = new Schema(
      required(0, "id", LongType.get()),
      optional(1, "data", MapType.ofOptional(2, 3,
          StructType.of(
              required(4, "i", Types.IntegerType.get()),
              optional(5, "s", Types.StringType.get())),
          Types.StringType.get())));

  writeAndValidate(schema);
}
 
Example #27
Source File: DataTest.java    From iceberg with Apache License 2.0 5 votes vote down vote up
@Test
public void testNumericMapKey() throws IOException {
  Schema schema = new Schema(
      required(0, "id", LongType.get()),
      optional(1, "data", MapType.ofOptional(2, 3,
          LongType.get(),
          Types.StringType.get())));

  writeAndValidate(schema);
}
 
Example #28
Source File: DataTest.java    From iceberg with Apache License 2.0 5 votes vote down vote up
@Test
public void testMap() throws IOException {
  Schema schema = new Schema(
      required(0, "id", LongType.get()),
      optional(1, "data", MapType.ofOptional(2, 3,
          Types.StringType.get(),
          Types.StringType.get())));

  writeAndValidate(schema);
}
 
Example #29
Source File: DataTest.java    From iceberg with Apache License 2.0 5 votes vote down vote up
@Test
public void testArrayOfStructs() throws IOException {
  Schema schema = new Schema(
      required(0, "id", LongType.get()),
      optional(1, "data", ListType.ofOptional(2, SUPPORTED_PRIMITIVES)));

  writeAndValidate(schema);
}
 
Example #30
Source File: DataTest.java    From iceberg with Apache License 2.0 5 votes vote down vote up
@Test
public void testArray() throws IOException {
  Schema schema = new Schema(
      required(0, "id", LongType.get()),
      optional(1, "data", ListType.ofOptional(2, Types.StringType.get())));

  writeAndValidate(schema);
}