Java Code Examples for org.apache.iceberg.types.Types.NestedField#optional()

The following examples show how to use org.apache.iceberg.types.Types.NestedField#optional() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: IcebergMetadata.java From presto with Apache License 2.0

6 votes

private static Schema toIcebergSchema(List<ColumnMetadata> columns)
{
    List<NestedField> icebergColumns = new ArrayList<>();
    for (ColumnMetadata column : columns) {
        if (!column.isHidden()) {
            int index = icebergColumns.size();
            Type type = toIcebergType(column.getType());
            NestedField field = column.isNullable()
                    ? NestedField.optional(index, column.getName(), type, column.getComment())
                    : NestedField.required(index, column.getName(), type, column.getComment());
            icebergColumns.add(field);
        }
    }
    Schema schema = new Schema(icebergColumns);
    AtomicInteger nextFieldId = new AtomicInteger(1);
    return TypeUtil.assignFreshIds(schema, nextFieldId::getAndIncrement);
}

Example 2

Source File: TestSchemaConverter.java From dremio-oss with Apache License 2.0

6 votes

@Test
public void map() {
  org.apache.iceberg.Schema icebergSchema = new org.apache.iceberg.Schema(
    NestedField.optional(1, "map",
      Types.MapType.ofOptional(2, 3, Types.IntegerType.get(), Types.FloatType.get()))
  );

  List<Field> children = Arrays.asList(
    CompleteType.INT.toField("key"),
    CompleteType.FLOAT.toField("value")
  );
  BatchSchema schema = BatchSchema.newBuilder()
    .addField(new CompleteType(new ArrowType.Map(false), children).toField("map"))
    .build();

  BatchSchema result = schemaConverter.fromIceberg(icebergSchema);
  // dremio silently drops map type columns
  assertEquals(result.getFieldCount(), 0);

  org.apache.iceberg.Schema icebergResult = schemaConverter.toIceberg(schema);
  assertEquals(icebergSchema.toString(), icebergResult.toString());
}

Example 3

Source File: TestPartitionFields.java From presto with Apache License 2.0

5 votes

private static PartitionSpec partitionSpec(Consumer<PartitionSpec.Builder> consumer)
{
    Schema schema = new Schema(
            NestedField.required(1, "order_key", LongType.get()),
            NestedField.required(2, "ts", TimestampType.withoutZone()),
            NestedField.required(3, "price", DoubleType.get()),
            NestedField.optional(4, "comment", StringType.get()),
            NestedField.optional(5, "notes", ListType.ofRequired(6, StringType.get())));

    PartitionSpec.Builder builder = PartitionSpec.builderFor(schema);
    consumer.accept(builder);
    return builder.build();
}

Example 4

Source File: SchemaConverter.java From dremio-oss with Apache License 2.0

5 votes

public static NestedField toIcebergColumn(Field field) {
  try {
    return NestedField.optional(0, field.getName(), toIcebergType(CompleteType.fromField(field)));
  } catch (Exception e) {
    throw UserException.unsupportedError(e)
      .message("conversion from arrow type to iceberg type failed for field " + field.getName())
      .buildSilently();
  }
}

Example 5

Source File: TestIcebergTableDrop.java From dremio-oss with Apache License 2.0

5 votes

@Before
public void setUp() {
  schema = new Schema(
    NestedField.optional(1, "n_nationkey", Types.IntegerType.get()),
    NestedField.optional(2, "n_name", Types.StringType.get()),
    NestedField.optional(3, "n_regionkey", Types.IntegerType.get()),
    NestedField.optional(4, "n_comment", Types.StringType.get())
  );
}

Example 6

Source File: TestSchemaConverter.java From dremio-oss with Apache License 2.0

5 votes

@Test
public void primitiveBasic() {
  org.apache.iceberg.Schema icebergSchema = new org.apache.iceberg.Schema(
    NestedField.optional(1, "boolean", Types.BooleanType.get()),
    NestedField.optional(2, "int", Types.IntegerType.get()),
    NestedField.optional(3, "long", Types.LongType.get()),
    NestedField.optional(4, "float", Types.FloatType.get()),
    NestedField.optional(5, "double", Types.DoubleType.get()),
    NestedField.optional(6, "decimal_38_16", Types.DecimalType.of(38, 16)),
    NestedField.optional(7, "string", Types.StringType.get()),
    NestedField.optional(8, "binary", Types.BinaryType.get()),
    NestedField.optional(9, "date", Types.DateType.get()),
    NestedField.optional(10, "time", Types.TimeType.get()),
    NestedField.optional(11, "fixed_32", Types.FixedType.ofLength(32)),
    NestedField.optional(12, "timestamp", Types.TimestampType.withZone())
  );

  BatchSchema schema = BatchSchema.newBuilder()
    .addField(CompleteType.BIT.toField("boolean"))
    .addField(CompleteType.INT.toField("int"))
    .addField(CompleteType.BIGINT.toField("long"))
    .addField(CompleteType.FLOAT.toField("float"))
    .addField(CompleteType.DOUBLE.toField("double"))
    .addField(new CompleteType(new Decimal(38, 16)).toField("decimal_38_16"))
    .addField(CompleteType.VARCHAR.toField("string"))
    .addField(CompleteType.VARBINARY.toField("binary"))
    .addField(CompleteType.DATE.toField("date"))
    .addField(CompleteType.TIME.toField("time"))
    .addField(new CompleteType(new ArrowType.FixedSizeBinary(32)).toField("fixed_32"))
    .addField(CompleteType.TIMESTAMP.toField("timestamp"))
    .build();

  assertEquals(schema, schemaConverter.fromIceberg(icebergSchema));
  assertEquals(icebergSchema.toString(), schemaConverter.toIceberg(schema).toString());
}

Example 7

Source File: TestSchemaConverter.java From dremio-oss with Apache License 2.0

5 votes

@Test
public void missingArrowTypes() {
  org.apache.iceberg.Schema icebergSchema = new org.apache.iceberg.Schema(
    NestedField.optional(1, "uuid", Types.UUIDType.get())
  );

  BatchSchema schema = BatchSchema.newBuilder()
    .addField(new CompleteType(new FixedSizeBinary(16)).toField("uuid"))
    .build();

  BatchSchema result = schemaConverter.fromIceberg(icebergSchema);
  assertEquals(result, schema);
}

Example 8

Source File: TestSchemaConverter.java From dremio-oss with Apache License 2.0

5 votes

@Test
public void testPartitionComparatorField() {
  BatchSchema inputschema = BatchSchema.newBuilder()
    .addField(CompleteType.BIT.toField("boolean"))
    .addField(CompleteType.INT.toField("int"))
    .addField(CompleteType.BIT.toField(WriterPrel.PARTITION_COMPARATOR_FIELD))
    .build();

  org.apache.iceberg.Schema expectedSchema = new org.apache.iceberg.Schema(
    NestedField.optional(1, "boolean", Types.BooleanType.get()),
    NestedField.optional(2, "int", Types.IntegerType.get()));

  SchemaConverter convert = new SchemaConverter();
  assertEquals(convert.toIceberg(inputschema).toString(), expectedSchema.toString());
}

Example 9

Source File: TestSchemaConverter.java From dremio-oss with Apache License 2.0

5 votes

@Test
public void unsupportedIcebergTypes() {
  org.apache.iceberg.Schema schema = new org.apache.iceberg.Schema(
    NestedField.optional(1, "timestamp_nozone_field", Types.TimestampType.withoutZone())
  );

  expectedEx.expect(UserException.class);
  expectedEx.expectMessage("conversion from iceberg type to arrow type failed for field timestamp_nozone_field");
  SchemaConverter convert = new SchemaConverter();
  convert.fromIceberg(schema);
}

Example 10

Source File: TestRefresh.java From dremio-oss with Apache License 2.0

5 votes

@Before
public void setUp() {
  schema = new Schema(
    NestedField.optional(1, "n_nationkey", Types.IntegerType.get()),
    NestedField.optional(2, "n_name", Types.StringType.get()),
    NestedField.optional(3, "n_regionkey", Types.IntegerType.get()),
    NestedField.optional(4, "n_comment", Types.StringType.get())
  );
}

Example 11

Source File: TestIcebergPartitions.java From dremio-oss with Apache License 2.0

5 votes

@Before
public void setUp() {
  schema = new Schema(
    NestedField.optional(1, ID, Types.IntegerType.get()),
    NestedField.optional(2, NAME, Types.StringType.get())
  );

  spec = PartitionSpec
    .builderFor(schema)
    .identity(ID)
    .identity(NAME)
    .build();
}

Example 12

Source File: TestSchemaConverter.java From dremio-oss with Apache License 2.0

4 votes

@Test
public void mixed() throws Exception {
  BatchSchema schema = BatchSchema.newBuilder()
    .addField(CompleteType.INT.toField("rownum"))
    .addField(CompleteType.VARCHAR.toField("name"))
    .addField(CompleteType.INT.toField("age"))
    .addField(CompleteType.FLOAT.toField("gpa"))
    .addField(CompleteType.BIGINT.toField("studentnum"))
    .addField(CompleteType.TIMESTAMP.toField("create_time"))
    .addField(CompleteType.VARCHAR.asList().toField("interests"))
    .addField(CompleteType.struct(
      CompleteType.VARCHAR.toField("color"),
      CompleteType.VARCHAR.toField("sport"),
      CompleteType.VARCHAR.toField("food")
    ).toField("favorites"))
    .build();

  org.apache.iceberg.Schema expectedSchema = new org.apache.iceberg.Schema(
    NestedField.optional(1, "rownum", Types.IntegerType.get()),
    NestedField.optional(2, "name", Types.StringType.get()),
    NestedField.optional(3, "age", Types.IntegerType.get()),
    NestedField.optional(4, "gpa", Types.FloatType.get()),
    NestedField.optional(5, "studentnum", Types.LongType.get()),
    NestedField.optional(6, "create_time", Types.TimestampType.withZone()),
    NestedField.optional(7, "interests",
      Types.ListType.ofOptional(9, Types.StringType.get())),
    NestedField.optional(8, "favorites",
      Types.StructType.of(
        NestedField.optional(10, "color", Types.StringType.get()),
        NestedField.optional(11, "sport", Types.StringType.get()),
        NestedField.optional(12, "food", Types.StringType.get())
      ))
  );

  org.apache.iceberg.Schema icebergResult = schemaConverter.toIceberg(schema);
  assertEquals(expectedSchema.toString(), icebergResult.toString());

  TemporaryFolder folder = new TemporaryFolder();
  folder.create();

  String rootPath = folder.getRoot().toString();
  Configuration conf = new Configuration();
  IcebergCatalog catalog = new IcebergCatalog(rootPath, conf);
  catalog.beginCreateTable(schema, Collections.emptyList());
  catalog.endCreateTable();

  Table table = new HadoopTables(conf).load(rootPath);
  assertEquals(expectedSchema.toString(), table.schema().toString());
}