Java Code Examples for org.apache.iceberg.types.Types#ListType

private static List<OrcType> toOrcListType(int nextFieldTypeIndex, Types.ListType listType, Map<String, String> attributes)
{
    nextFieldTypeIndex++;
    Map<String, String> elementAttributes = ImmutableMap.<String, String>builder()
            .put(ORC_ICEBERG_ID_KEY, Integer.toString(listType.elementId()))
            .put(ORC_ICEBERG_REQUIRED_KEY, Boolean.toString(listType.isElementRequired()))
            .build();
    List<OrcType> itemTypes = toOrcType(nextFieldTypeIndex, listType.elementType(), elementAttributes);

    List<OrcType> orcTypes = new ArrayList<>();
    orcTypes.add(new OrcType(
            OrcType.OrcTypeKind.LIST,
            ImmutableList.of(new OrcColumnId(nextFieldTypeIndex)),
            ImmutableList.of("item"),
            Optional.empty(),
            Optional.empty(),
            Optional.empty(),
            attributes));

    orcTypes.addAll(itemTypes);
    return orcTypes;
}
 

public static <T> T visit(Type iType, TypeDescription schema, OrcSchemaWithTypeVisitor<T> visitor) {
  switch (schema.getCategory()) {
    case STRUCT:
      return visitRecord(iType != null ? iType.asStructType() : null, schema, visitor);

    case UNION:
      throw new UnsupportedOperationException("Cannot handle " + schema);

    case LIST:
      Types.ListType list = iType != null ? iType.asListType() : null;
      return visitor.list(
          list, schema,
          visit(list.elementType(), schema.getChildren().get(0), visitor));

    case MAP:
      Types.MapType map = iType != null ? iType.asMapType() : null;
      return visitor.map(
          map, schema,
          visit(map != null ? map.keyType() : null, schema.getChildren().get(0), visitor),
          visit(map != null ? map.valueType() : null, schema.getChildren().get(1), visitor));

    default:
      return visitor.primitive(iType != null ? iType.asPrimitiveType() : null, schema);
  }
}
 

private static <T> T visitArray(Type type, Schema array, AvroSchemaWithTypeVisitor<T> visitor) {
  if (array.getLogicalType() instanceof LogicalMap || (type != null && type.isMapType())) {
    Preconditions.checkState(
        AvroSchemaUtil.isKeyValueSchema(array.getElementType()),
        "Cannot visit invalid logical map type: %s", array);
    Types.MapType map = type != null ? type.asMapType() : null;
    List<Schema.Field> keyValueFields = array.getElementType().getFields();
    return visitor.map(map, array,
        visit(map != null ? map.keyType() : null, keyValueFields.get(0).schema(), visitor),
        visit(map != null ? map.valueType() : null, keyValueFields.get(1).schema(), visitor));

  } else {
    Types.ListType list = type != null ? type.asListType() : null;
    return visitor.array(list, array,
        visit(list != null ? list.elementType() : null, array.getElementType(), visitor));
  }
}
 

@Override
public ParquetValueReader<?> list(Types.ListType expectedList, GroupType array,
                                  ParquetValueReader<?> elementReader) {
  GroupType repeated = array.getFields().get(0).asGroupType();
  String[] repeatedPath = currentPath();

  int repeatedD = type.getMaxDefinitionLevel(repeatedPath) - 1;
  int repeatedR = type.getMaxRepetitionLevel(repeatedPath) - 1;

  Type elementType = repeated.getType(0);
  int elementD = type.getMaxDefinitionLevel(path(elementType.getName())) - 1;

  return new ListReader<>(repeatedD, repeatedR, ParquetValueReaders.option(elementType, elementD, elementReader));
}
 

@Test
public void testAddNestedListOfStructs() {
  Schema schema = new Schema(required(1, "id", Types.IntegerType.get()));
  Types.ListType list = Types.ListType.ofOptional(1,
      Types.StructType.of(
          required(9, "lat", Types.IntegerType.get()),
          optional(8, "long", Types.IntegerType.get())
      )
  );

  Schema expected = new Schema(
      required(1, "id", Types.IntegerType.get()),
      optional(2, "locations", Types.ListType.ofOptional(3,
          Types.StructType.of(
              required(4, "lat", Types.IntegerType.get()),
              optional(5, "long", Types.IntegerType.get())
          )
      ))
  );

  Schema result = new SchemaUpdate(schema, 1)
      .addColumn("locations", list)
      .apply();

  Assert.assertEquals("Should add map and reassign column IDs",
      expected.asStruct(), result.asStruct());
}
 

static void assertEquals(Types.ListType list, List<?> expected, List<?> actual) {
  Type elementType = list.elementType();

  Assert.assertEquals("List size should match", expected.size(), actual.size());

  for (int i = 0; i < expected.size(); i += 1) {
    Object expectedValue = expected.get(i);
    Object actualValue = actual.get(i);

    assertEquals(elementType, expectedValue, actualValue);
  }
}
 

private static void assertEqualsSafe(Types.ListType list, Collection<?> expected, List actual) {
  Type elementType = list.elementType();
  List<?> expectedElements = Lists.newArrayList(expected);
  for (int i = 0; i < expectedElements.size(); i += 1) {
    Object expectedValue = expectedElements.get(i);
    Object actualValue = actual.get(i);

    assertEqualsSafe(elementType, expectedValue, actualValue);
  }
}
 

private static void assertEqualsUnsafe(Types.ListType list, Collection<?> expected, ArrayData actual) {
  Type elementType = list.elementType();
  List<?> expectedElements = Lists.newArrayList(expected);
  for (int i = 0; i < expectedElements.size(); i += 1) {
    Object expectedValue = expectedElements.get(i);
    Object actualValue = actual.get(i, convert(elementType));

    assertEqualsUnsafe(elementType, expectedValue, actualValue);
  }
}
 

private static void assertEqualsSafe(Types.ListType list, Collection<?> expected, List<?> actual) {
  Type elementType = list.elementType();
  List<?> expectedElements = Lists.newArrayList(expected);
  for (int i = 0; i < expectedElements.size(); i += 1) {
    Object expectedValue = expectedElements.get(i);
    Object actualValue = actual.get(i);

    assertEqualsSafe(elementType, expectedValue, actualValue);
  }
}
 

private static Types.ListType listFromJson(JsonNode json) {
  int elementId = JsonUtil.getInt(ELEMENT_ID, json);
  Type elementType = typeFromJson(json.get(ELEMENT));
  boolean isRequired = JsonUtil.getBool(ELEMENT_REQUIRED, json);

  if (isRequired) {
    return Types.ListType.ofRequired(elementId, elementType);
  } else {
    return Types.ListType.ofOptional(elementId, elementType);
  }
}
 

public static void assertEquals(Types.ListType list, List<?> expected, List<?> actual) {
  Type elementType = list.elementType();

  Assert.assertEquals("List size should match", expected.size(), actual.size());

  for (int i = 0; i < expected.size(); i += 1) {
    Object expectedValue = expected.get(i);
    Object actualValue = actual.get(i);

    assertEquals(elementType, expectedValue, actualValue);
  }
}
 

@Override
public ParquetValueReader<?> list(Types.ListType expectedList, GroupType array,
                                  ParquetValueReader<?> elementReader) {
  GroupType repeated = array.getFields().get(0).asGroupType();
  String[] repeatedPath = currentPath();

  int repeatedD = type.getMaxDefinitionLevel(repeatedPath) - 1;
  int repeatedR = type.getMaxRepetitionLevel(repeatedPath) - 1;

  Type elementType = repeated.getType(0);
  int elementD = type.getMaxDefinitionLevel(path(elementType.getName())) - 1;

  return new ListReader<>(repeatedD, repeatedR, ParquetValueReaders.option(elementType, elementD, elementReader));
}
 

static void toJson(Types.ListType list, JsonGenerator generator) throws IOException {
  generator.writeStartObject();

  generator.writeStringField(TYPE, LIST);

  generator.writeNumberField(ELEMENT_ID, list.elementId());
  generator.writeFieldName(ELEMENT);
  toJson(list.elementType(), generator);
  generator.writeBooleanField(ELEMENT_REQUIRED, !list.isElementOptional());

  generator.writeEndObject();
}
 

@Override
public OrcValueReader<?> list(Types.ListType iList, TypeDescription array, OrcValueReader<?> elementReader) {
  return SparkOrcValueReaders.array(elementReader);
}
 

/**
 * convert iceberg to hive type.
 * @param type iceberg type.
 * @return hive type string.
 */
public static String fromIcebergToHiveType(final org.apache.iceberg.types.Type type) {
    switch (type.typeId()) {
        case BOOLEAN:
            return serdeConstants.BOOLEAN_TYPE_NAME;
        case INTEGER:
            return serdeConstants.INT_TYPE_NAME;
        case LONG:
            return serdeConstants.BIGINT_TYPE_NAME;
        case FLOAT:
            return serdeConstants.FLOAT_TYPE_NAME;
        case DOUBLE:
            return serdeConstants.DOUBLE_TYPE_NAME;
        case DATE:
            return serdeConstants.DATE_TYPE_NAME;
        case TIME:
            throw new UnsupportedOperationException("Hive does not support time fields");
        case TIMESTAMP:
            return serdeConstants.TIMESTAMP_TYPE_NAME;
        case STRING:
        case UUID:
            return serdeConstants.STRING_TYPE_NAME;
        case FIXED:
            return serdeConstants.BINARY_TYPE_NAME;
        case BINARY:
            return serdeConstants.BINARY_TYPE_NAME;
        case DECIMAL:
            final Types.DecimalType decimalType = (Types.DecimalType) type;
            return String.format("decimal(%s,%s)", decimalType.precision(), decimalType.scale());
        case STRUCT:
            final Types.StructType structType = type.asStructType();
            final String nameToType = (String) structType.fields().stream().map((f) -> {
                return String.format("%s:%s", f.name(), fromIcebergToHiveType(f.type()));
            }).collect(Collectors.joining(","));
            return String.format("struct<%s>", nameToType);
        case LIST:
            final Types.ListType listType = type.asListType();
            return String.format("array<%s>", fromIcebergToHiveType(listType.elementType()));
        case MAP:
            final Types.MapType mapType = type.asMapType();
            return String.format("map<%s,%s>", fromIcebergToHiveType(mapType.keyType()),
                fromIcebergToHiveType(mapType.valueType()));
        default:
            throw new UnsupportedOperationException(type + " is not supported");
    }
}
 

@Override
public TypeDescription list(Types.ListType iList, TypeDescription array, TypeDescription element) {
  columnsInContainers.addAll(flatten(element));
  return array;
}
 

private static Types.NestedField convertOrcToIceberg(TypeDescription orcType, String name,
                                                     TypeUtil.NextID nextID) {

  final int icebergID = icebergID(orcType).orElseGet(nextID::get);
  final boolean isRequired = isRequired(orcType);

  switch (orcType.getCategory()) {
    case BOOLEAN:
      return getIcebergType(icebergID, name, Types.BooleanType.get(), isRequired);
    case BYTE:
    case SHORT:
    case INT:
      return getIcebergType(icebergID, name, Types.IntegerType.get(), isRequired);
    case LONG:
      String longAttributeValue = orcType.getAttributeValue(ICEBERG_LONG_TYPE_ATTRIBUTE);
      LongType longType = longAttributeValue == null ? LongType.LONG : LongType.valueOf(longAttributeValue);
      switch (longType) {
        case TIME:
          return getIcebergType(icebergID, name, Types.TimeType.get(), isRequired);
        case LONG:
          return getIcebergType(icebergID, name, Types.LongType.get(), isRequired);
        default:
          throw new IllegalStateException("Invalid Long type found in ORC type attribute");
      }
    case FLOAT:
      return getIcebergType(icebergID, name, Types.FloatType.get(), isRequired);
    case DOUBLE:
      return getIcebergType(icebergID, name, Types.DoubleType.get(), isRequired);
    case STRING:
    case CHAR:
    case VARCHAR:
      return getIcebergType(icebergID, name, Types.StringType.get(), isRequired);
    case BINARY:
      String binaryAttributeValue = orcType.getAttributeValue(ICEBERG_BINARY_TYPE_ATTRIBUTE);
      BinaryType binaryType = binaryAttributeValue == null ? BinaryType.BINARY :
          BinaryType.valueOf(binaryAttributeValue);
      switch (binaryType) {
        case UUID:
          return getIcebergType(icebergID, name, Types.UUIDType.get(), isRequired);
        case FIXED:
          int fixedLength = Integer.parseInt(orcType.getAttributeValue(ICEBERG_FIELD_LENGTH));
          return getIcebergType(icebergID, name, Types.FixedType.ofLength(fixedLength), isRequired);
        case BINARY:
          return getIcebergType(icebergID, name, Types.BinaryType.get(), isRequired);
        default:
          throw new IllegalStateException("Invalid Binary type found in ORC type attribute");
      }
    case DATE:
      return getIcebergType(icebergID, name, Types.DateType.get(), isRequired);
    case TIMESTAMP:
      return getIcebergType(icebergID, name, Types.TimestampType.withoutZone(), isRequired);
    case TIMESTAMP_INSTANT:
      return getIcebergType(icebergID, name, Types.TimestampType.withZone(), isRequired);
    case DECIMAL:
      return getIcebergType(icebergID, name,
          Types.DecimalType.of(orcType.getPrecision(), orcType.getScale()),
          isRequired);
    case STRUCT: {
      List<String> fieldNames = orcType.getFieldNames();
      List<TypeDescription> fieldTypes = orcType.getChildren();
      List<Types.NestedField> fields = new ArrayList<>(fieldNames.size());
      for (int c = 0; c < fieldNames.size(); ++c) {
        String childName = fieldNames.get(c);
        TypeDescription type = fieldTypes.get(c);
        Types.NestedField field = convertOrcToIceberg(type, childName, nextID);
        fields.add(field);
      }

      return getIcebergType(icebergID, name, Types.StructType.of(fields), isRequired);
    }
    case LIST: {
      TypeDescription elementType = orcType.getChildren().get(0);
      Types.NestedField element = convertOrcToIceberg(elementType, "element", nextID);

      Types.ListType listTypeWithElem = isRequired(elementType) ?
          Types.ListType.ofRequired(element.fieldId(), element.type()) :
          Types.ListType.ofOptional(element.fieldId(), element.type());
      return isRequired ?
          Types.NestedField.required(icebergID, name, listTypeWithElem) :
          Types.NestedField.optional(icebergID, name, listTypeWithElem);
    }
    case MAP: {
      TypeDescription keyType = orcType.getChildren().get(0);
      Types.NestedField key = convertOrcToIceberg(keyType, "key", nextID);
      TypeDescription valueType = orcType.getChildren().get(1);
      Types.NestedField value = convertOrcToIceberg(valueType, "value", nextID);

      Types.MapType mapTypeWithKV = isRequired(valueType) ?
          Types.MapType.ofRequired(key.fieldId(), value.fieldId(), key.type(), value.type()) :
          Types.MapType.ofOptional(key.fieldId(), value.fieldId(), key.type(), value.type());

      return getIcebergType(icebergID, name, mapTypeWithKV, isRequired);
    }
    default:
      // We don't have an answer for union types.
      throw new IllegalArgumentException("Can't handle " + orcType);
  }
}
 

private static TypeDescription convert(Integer fieldId, Type type, boolean isRequired) {
  final TypeDescription orcType;

  switch (type.typeId()) {
    case BOOLEAN:
      orcType = TypeDescription.createBoolean();
      break;
    case INTEGER:
      orcType = TypeDescription.createInt();
      break;
    case TIME:
      orcType = TypeDescription.createLong();
      orcType.setAttribute(ICEBERG_LONG_TYPE_ATTRIBUTE, LongType.TIME.toString());
      break;
    case LONG:
      orcType = TypeDescription.createLong();
      orcType.setAttribute(ICEBERG_LONG_TYPE_ATTRIBUTE, LongType.LONG.toString());
      break;
    case FLOAT:
      orcType = TypeDescription.createFloat();
      break;
    case DOUBLE:
      orcType = TypeDescription.createDouble();
      break;
    case DATE:
      orcType = TypeDescription.createDate();
      break;
    case TIMESTAMP:
      Types.TimestampType tsType = (Types.TimestampType) type;
      if (tsType.shouldAdjustToUTC()) {
        orcType = TypeDescription.createTimestampInstant();
      } else {
        orcType = TypeDescription.createTimestamp();
      }
      break;
    case STRING:
      orcType = TypeDescription.createString();
      break;
    case UUID:
      orcType = TypeDescription.createBinary();
      orcType.setAttribute(ICEBERG_BINARY_TYPE_ATTRIBUTE, BinaryType.UUID.toString());
      break;
    case FIXED:
      orcType = TypeDescription.createBinary();
      orcType.setAttribute(ICEBERG_BINARY_TYPE_ATTRIBUTE, BinaryType.FIXED.toString());
      orcType.setAttribute(ICEBERG_FIELD_LENGTH, Integer.toString(((Types.FixedType) type).length()));
      break;
    case BINARY:
      orcType = TypeDescription.createBinary();
      orcType.setAttribute(ICEBERG_BINARY_TYPE_ATTRIBUTE, BinaryType.BINARY.toString());
      break;
    case DECIMAL: {
      Types.DecimalType decimal = (Types.DecimalType) type;
      orcType = TypeDescription.createDecimal()
          .withScale(decimal.scale())
          .withPrecision(decimal.precision());
      break;
    }
    case STRUCT: {
      orcType = TypeDescription.createStruct();
      for (Types.NestedField field : type.asStructType().fields()) {
        TypeDescription childType = convert(field.fieldId(), field.type(), field.isRequired());
        orcType.addField(field.name(), childType);
      }
      break;
    }
    case LIST: {
      Types.ListType list = (Types.ListType) type;
      TypeDescription elementType = convert(list.elementId(), list.elementType(),
          list.isElementRequired());
      orcType = TypeDescription.createList(elementType);
      break;
    }
    case MAP: {
      Types.MapType map = (Types.MapType) type;
      TypeDescription keyType = convert(map.keyId(), map.keyType(), true);
      TypeDescription valueType = convert(map.valueId(), map.valueType(), map.isValueRequired());
      orcType = TypeDescription.createMap(keyType, valueType);
      break;
    }
    default:
      throw new IllegalArgumentException("Unhandled type " + type.typeId());
  }

  // Set Iceberg column attributes for mapping
  orcType.setAttribute(ICEBERG_ID_ATTRIBUTE, String.valueOf(fieldId));
  orcType.setAttribute(ICEBERG_REQUIRED_ATTRIBUTE, String.valueOf(isRequired));
  return orcType;
}
 

public T list(Types.ListType iList, GroupType array, T element) {
  return null;
}
 

public static Type toPrestoType(org.apache.iceberg.types.Type type, TypeManager typeManager)
{
    switch (type.typeId()) {
        case BOOLEAN:
            return BooleanType.BOOLEAN;
        case BINARY:
        case FIXED:
            return VarbinaryType.VARBINARY;
        case DATE:
            return DateType.DATE;
        case DECIMAL:
            Types.DecimalType decimalType = (Types.DecimalType) type;
            return DecimalType.createDecimalType(decimalType.precision(), decimalType.scale());
        case DOUBLE:
            return DoubleType.DOUBLE;
        case LONG:
            return BigintType.BIGINT;
        case FLOAT:
            return RealType.REAL;
        case INTEGER:
            return IntegerType.INTEGER;
        case TIME:
            return TimeType.TIME;
        case TIMESTAMP:
            Types.TimestampType timestampType = (Types.TimestampType) type;
            if (timestampType.shouldAdjustToUTC()) {
                return TimestampWithTimeZoneType.TIMESTAMP_WITH_TIME_ZONE;
            }
            return TimestampType.TIMESTAMP;
        case UUID:
        case STRING:
            return VarcharType.createUnboundedVarcharType();
        case LIST:
            Types.ListType listType = (Types.ListType) type;
            return new ArrayType(toPrestoType(listType.elementType(), typeManager));
        case MAP:
            Types.MapType mapType = (Types.MapType) type;
            TypeSignature keyType = toPrestoType(mapType.keyType(), typeManager).getTypeSignature();
            TypeSignature valueType = toPrestoType(mapType.valueType(), typeManager).getTypeSignature();
            return typeManager.getParameterizedType(StandardTypes.MAP, ImmutableList.of(TypeSignatureParameter.typeParameter(keyType), TypeSignatureParameter.typeParameter(valueType)));
        case STRUCT:
            List<Types.NestedField> fields = ((Types.StructType) type).fields();
            return RowType.from(fields.stream()
                    .map(field -> new RowType.Field(Optional.of(field.name()), toPrestoType(field.type(), typeManager)))
                    .collect(toImmutableList()));
        default:
            throw new UnsupportedOperationException(format("Cannot convert from Iceberg type '%s' (%s) to Presto type", type, type.typeId()));
    }
}