Java Code Examples for org.apache.iceberg.types.Types#MapType

The following examples show how to use org.apache.iceberg.types.Types#MapType . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: SchemaParser.java    From iceberg with Apache License 2.0 6 votes vote down vote up
static void toJson(Types.MapType map, JsonGenerator generator) throws IOException {
  generator.writeStartObject();

  generator.writeStringField(TYPE, MAP);

  generator.writeNumberField(KEY_ID, map.keyId());
  generator.writeFieldName(KEY);
  toJson(map.keyType(), generator);

  generator.writeNumberField(VALUE_ID, map.valueId());
  generator.writeFieldName(VALUE);
  toJson(map.valueType(), generator);
  generator.writeBooleanField(VALUE_REQUIRED, !map.isValueOptional());

  generator.writeEndObject();
}
 
Example 2
Source File: AvroSchemaWithTypeVisitor.java    From iceberg with Apache License 2.0 6 votes vote down vote up
private static <T> T visitArray(Type type, Schema array, AvroSchemaWithTypeVisitor<T> visitor) {
  if (array.getLogicalType() instanceof LogicalMap || (type != null && type.isMapType())) {
    Preconditions.checkState(
        AvroSchemaUtil.isKeyValueSchema(array.getElementType()),
        "Cannot visit invalid logical map type: %s", array);
    Types.MapType map = type != null ? type.asMapType() : null;
    List<Schema.Field> keyValueFields = array.getElementType().getFields();
    return visitor.map(map, array,
        visit(map != null ? map.keyType() : null, keyValueFields.get(0).schema(), visitor),
        visit(map != null ? map.valueType() : null, keyValueFields.get(1).schema(), visitor));

  } else {
    Types.ListType list = type != null ? type.asListType() : null;
    return visitor.array(list, array,
        visit(list != null ? list.elementType() : null, array.getElementType(), visitor));
  }
}
 
Example 3
Source File: AvroSchemaWithTypeVisitor.java    From iceberg with Apache License 2.0 6 votes vote down vote up
public static <T> T visit(Type iType, Schema schema, AvroSchemaWithTypeVisitor<T> visitor) {
  switch (schema.getType()) {
    case RECORD:
      return visitRecord(iType != null ? iType.asStructType() : null, schema, visitor);

    case UNION:
      return visitUnion(iType, schema, visitor);

    case ARRAY:
      return visitArray(iType, schema, visitor);

    case MAP:
      Types.MapType map = iType != null ? iType.asMapType() : null;
      return visitor.map(map, schema,
          visit(map != null ? map.valueType() : null, schema.getValueType(), visitor));

    default:
      return visitor.primitive(iType != null ? iType.asPrimitiveType() : null, schema);
  }
}
 
Example 4
Source File: OrcSchemaWithTypeVisitor.java    From iceberg with Apache License 2.0 6 votes vote down vote up
public static <T> T visit(Type iType, TypeDescription schema, OrcSchemaWithTypeVisitor<T> visitor) {
  switch (schema.getCategory()) {
    case STRUCT:
      return visitRecord(iType != null ? iType.asStructType() : null, schema, visitor);

    case UNION:
      throw new UnsupportedOperationException("Cannot handle " + schema);

    case LIST:
      Types.ListType list = iType != null ? iType.asListType() : null;
      return visitor.list(
          list, schema,
          visit(list.elementType(), schema.getChildren().get(0), visitor));

    case MAP:
      Types.MapType map = iType != null ? iType.asMapType() : null;
      return visitor.map(
          map, schema,
          visit(map != null ? map.keyType() : null, schema.getChildren().get(0), visitor),
          visit(map != null ? map.valueType() : null, schema.getChildren().get(1), visitor));

    default:
      return visitor.primitive(iType != null ? iType.asPrimitiveType() : null, schema);
  }
}
 
Example 5
Source File: TestHelpers.java    From iceberg with Apache License 2.0 6 votes vote down vote up
private static void assertEqualsUnsafe(Types.MapType map, Map<?, ?> expected, MapData actual) {
  Type keyType = map.keyType();
  Type valueType = map.valueType();

  List<Map.Entry<?, ?>> expectedElements = Lists.newArrayList(expected.entrySet());
  ArrayData actualKeys = actual.keyArray();
  ArrayData actualValues = actual.valueArray();

  for (int i = 0; i < expectedElements.size(); i += 1) {
    Map.Entry<?, ?> expectedPair = expectedElements.get(i);
    Object actualKey = actualKeys.get(i, convert(keyType));
    Object actualValue = actualValues.get(i, convert(keyType));

    assertEqualsUnsafe(keyType, expectedPair.getKey(), actualKey);
    assertEqualsUnsafe(valueType, expectedPair.getValue(), actualValue);
  }
}
 
Example 6
Source File: TestHelpers.java    From iceberg with Apache License 2.0 6 votes vote down vote up
private static void assertEqualsSafe(Types.MapType map,
                                     Map<?, ?> expected, Map<?, ?> actual) {
  Type keyType = map.keyType();
  Type valueType = map.valueType();

  for (Object expectedKey : expected.keySet()) {
    Object matchingKey = null;
    for (Object actualKey : actual.keySet()) {
      try {
        assertEqualsSafe(keyType, expectedKey, actualKey);
        matchingKey = actualKey;
      } catch (AssertionError e) {
        // failed
      }
    }

    Assert.assertNotNull("Should have a matching key", matchingKey);
    assertEqualsSafe(valueType, expected.get(expectedKey), actual.get(matchingKey));
  }
}
 
Example 7
Source File: GenericsHelpers.java    From iceberg with Apache License 2.0 6 votes vote down vote up
private static void assertEqualsSafe(Types.MapType map,
                                     Map<?, ?> expected, Map<?, ?> actual) {
  Type keyType = map.keyType();
  Type valueType = map.valueType();
  Assert.assertEquals("Should have the same number of keys", expected.keySet().size(), actual.keySet().size());

  for (Object expectedKey : expected.keySet()) {
    Object matchingKey = null;
    for (Object actualKey : actual.keySet()) {
      try {
        assertEqualsSafe(keyType, expectedKey, actualKey);
        matchingKey = actualKey;
        break;
      } catch (AssertionError e) {
        // failed
      }
    }

    Assert.assertNotNull("Should have a matching key", matchingKey);
    assertEqualsSafe(valueType, expected.get(expectedKey), actual.get(matchingKey));
  }
}
 
Example 8
Source File: GenericsHelpers.java    From iceberg with Apache License 2.0 6 votes vote down vote up
private static void assertEqualsUnsafe(Types.MapType map, Map<?, ?> expected, MapData actual) {
  Type keyType = map.keyType();
  Type valueType = map.valueType();

  List<Map.Entry<?, ?>> expectedElements = Lists.newArrayList(expected.entrySet());
  ArrayData actualKeys = actual.keyArray();
  ArrayData actualValues = actual.valueArray();

  for (int i = 0; i < expectedElements.size(); i += 1) {
    Map.Entry<?, ?> expectedPair = expectedElements.get(i);
    Object actualKey = actualKeys.get(i, convert(keyType));
    Object actualValue = actualValues.get(i, convert(keyType));

    assertEqualsUnsafe(keyType, expectedPair.getKey(), actualKey);
    assertEqualsUnsafe(valueType, expectedPair.getValue(), actualValue);
  }
}
 
Example 9
Source File: TestSchemaUpdate.java    From iceberg with Apache License 2.0 5 votes vote down vote up
@Test
public void testAddNestedMapOfStructs() {
  Schema schema = new Schema(required(1, "id", Types.IntegerType.get()));
  Types.MapType map = Types.MapType.ofOptional(1, 2,
      Types.StructType.of(
          required(20, "address", Types.StringType.get()),
          required(21, "city", Types.StringType.get()),
          required(22, "state", Types.StringType.get()),
          required(23, "zip", Types.IntegerType.get())
      ),
      Types.StructType.of(
          required(9, "lat", Types.IntegerType.get()),
          optional(8, "long", Types.IntegerType.get())
      )
  );

  Schema expected = new Schema(
      required(1, "id", Types.IntegerType.get()),
      optional(2, "locations", Types.MapType.ofOptional(3, 4,
          Types.StructType.of(
              required(5, "address", Types.StringType.get()),
              required(6, "city", Types.StringType.get()),
              required(7, "state", Types.StringType.get()),
              required(8, "zip", Types.IntegerType.get())
          ),
          Types.StructType.of(
              required(9, "lat", Types.IntegerType.get()),
              optional(10, "long", Types.IntegerType.get())
          )
      ))
  );

  Schema result = new SchemaUpdate(schema, 1)
      .addColumn("locations", map)
      .apply();

  Assert.assertEquals("Should add map and reassign column IDs",
      expected.asStruct(), result.asStruct());
}
 
Example 10
Source File: AvroTestHelpers.java    From iceberg with Apache License 2.0 5 votes vote down vote up
static void assertEquals(Types.MapType map, Map<?, ?> expected, Map<?, ?> actual) {
  Type valueType = map.valueType();

  Assert.assertEquals("Map size should match", expected.size(), actual.size());

  for (Object expectedKey : expected.keySet()) {
    Object expectedValue = expected.get(expectedKey);
    Object actualValue = actual.get(expectedKey);

    assertEquals(valueType, expectedValue, actualValue);
  }
}
 
Example 11
Source File: StructInternalRow.java    From iceberg with Apache License 2.0 5 votes vote down vote up
private MapData mapToMapData(Types.MapType mapType, Map<?, ?> map) {
  // make a defensive copy to ensure entries do not change
  List<Map.Entry<?, ?>> entries = ImmutableList.copyOf(map.entrySet());
  return new ArrayBasedMapData(
      collectionToArrayData(mapType.keyType(), Lists.transform(entries, Map.Entry::getKey)),
      collectionToArrayData(mapType.valueType(), Lists.transform(entries, Map.Entry::getValue)));
}
 
Example 12
Source File: DataTestHelpers.java    From iceberg with Apache License 2.0 5 votes vote down vote up
public static void assertEquals(Types.MapType map, Map<?, ?> expected, Map<?, ?> actual) {
  Type valueType = map.valueType();

  Assert.assertEquals("Map size should match", expected.size(), actual.size());

  for (Object expectedKey : expected.keySet()) {
    Object expectedValue = expected.get(expectedKey);
    Object actualValue = actual.get(expectedKey);

    assertEquals(valueType, expectedValue, actualValue);
  }
}
 
Example 13
Source File: TypeConverter.java    From presto with Apache License 2.0 5 votes vote down vote up
private static List<OrcType> toOrcMapType(int nextFieldTypeIndex, Types.MapType mapType, Map<String, String> attributes)
{
    nextFieldTypeIndex++;
    Map<String, String> keyAttributes = ImmutableMap.<String, String>builder()
            .put(ORC_ICEBERG_ID_KEY, Integer.toString(mapType.keyId()))
            .put(ORC_ICEBERG_REQUIRED_KEY, Boolean.toString(true))
            .build();
    List<OrcType> keyTypes = toOrcType(nextFieldTypeIndex, mapType.keyType(), keyAttributes);
    Map<String, String> valueAttributes = ImmutableMap.<String, String>builder()
            .put(ORC_ICEBERG_ID_KEY, Integer.toString(mapType.valueId()))
            .put(ORC_ICEBERG_REQUIRED_KEY, Boolean.toString(mapType.isValueRequired()))
            .build();
    List<OrcType> valueTypes = toOrcType(nextFieldTypeIndex + keyTypes.size(), mapType.valueType(), valueAttributes);

    List<OrcType> orcTypes = new ArrayList<>();
    orcTypes.add(new OrcType(
            OrcType.OrcTypeKind.MAP,
            ImmutableList.of(new OrcColumnId(nextFieldTypeIndex), new OrcColumnId(nextFieldTypeIndex + keyTypes.size())),
            ImmutableList.of("key", "value"),
            Optional.empty(),
            Optional.empty(),
            Optional.empty(),
            attributes));

    orcTypes.addAll(keyTypes);
    orcTypes.addAll(valueTypes);
    return orcTypes;
}
 
Example 14
Source File: TestHelpers.java    From iceberg with Apache License 2.0 4 votes vote down vote up
private static void assertEqualsMaps(String prefix, Types.MapType type,
                                     MapData expected, Map<?, ?> actual) {
  if (expected == null || actual == null) {
    Assert.assertEquals(prefix, expected, actual);
  } else {
    Type keyType = type.keyType();
    Type valueType = type.valueType();
    ArrayData expectedKeyArray = expected.keyArray();
    ArrayData expectedValueArray = expected.valueArray();
    Assert.assertEquals(prefix + " length", expected.numElements(), actual.size());
    for (int e = 0; e < expected.numElements(); ++e) {
      Object expectedKey = getValue(expectedKeyArray, e, keyType);
      Object actualValue = actual.get(expectedKey);
      if (actualValue == null) {
        Assert.assertEquals(prefix + ".key=" + expectedKey + " has null", true,
            expected.valueArray().isNullAt(e));
      } else {
        switch (valueType.typeId()) {
          case BOOLEAN:
          case INTEGER:
          case LONG:
          case FLOAT:
          case DOUBLE:
          case STRING:
          case DECIMAL:
          case DATE:
          case TIMESTAMP:
            Assert.assertEquals(prefix + ".key=" + expectedKey + " - " + valueType,
                getValue(expectedValueArray, e, valueType),
                actual.get(expectedKey));
            break;
          case UUID:
          case FIXED:
          case BINARY:
            assertEqualBytes(prefix + ".key=" + expectedKey,
                (byte[]) getValue(expectedValueArray, e, valueType),
                (byte[]) actual.get(expectedKey));
            break;
          case STRUCT: {
            Types.StructType st = (Types.StructType) valueType;
            assertEquals(prefix + ".key=" + expectedKey, st,
                expectedValueArray.getStruct(e, st.fields().size()),
                (Row) actual.get(expectedKey));
            break;
          }
          case LIST:
            assertEqualsLists(prefix + ".key=" + expectedKey,
                valueType.asListType(),
                expectedValueArray.getArray(e),
                toList((Seq<?>) actual.get(expectedKey)));
            break;
          case MAP:
            assertEqualsMaps(prefix + ".key=" + expectedKey, valueType.asMapType(),
                expectedValueArray.getMap(e),
                toJavaMap((scala.collection.Map<?, ?>) actual.get(expectedKey)));
            break;
          default:
            throw new IllegalArgumentException("Unhandled type " + valueType);
        }
      }
    }
  }
}
 
Example 15
Source File: SchemaUtil.java    From iceberg with Apache License 2.0 4 votes vote down vote up
private static ResourceSchema convertComplex(Type type) throws IOException {
  ResourceSchema result = new ResourceSchema();

  switch (type.typeId()) {
    case STRUCT:
      Types.StructType structType = type.asStructType();

      List<ResourceFieldSchema> fields = Lists.newArrayList();

      for (Types.NestedField f : structType.fields()) {
        fields.add(convert(f));
      }

      result.setFields(fields.toArray(new ResourceFieldSchema[0]));

      return result;
    case LIST:
      Types.ListType listType = type.asListType();

      ResourceFieldSchema [] elementFieldSchemas = new ResourceFieldSchema[]{convert(listType.elementType())};

      if (listType.elementType().isStructType()) {
        result.setFields(elementFieldSchemas);
      } else {
        //Wrap non-struct types in tuples
        ResourceSchema elementSchema = new ResourceSchema();
        elementSchema.setFields(elementFieldSchemas);

        ResourceFieldSchema tupleSchema = new ResourceFieldSchema();
        tupleSchema.setType(DataType.TUPLE);
        tupleSchema.setSchema(elementSchema);

        result.setFields(new ResourceFieldSchema[]{tupleSchema});
      }

      return result;
    case MAP:
      Types.MapType mapType = type.asMapType();

      if (mapType.keyType().typeId() != Type.TypeID.STRING) {
        throw new FrontendException("Unsupported map key type: " + mapType.keyType());
      }
      result.setFields(new ResourceFieldSchema[]{convert(mapType.valueType())});

      return result;
    default:
      throw new FrontendException("Unsupported complex type: " + type);
  }
}
 
Example 16
Source File: ORCSchemaUtil.java    From iceberg with Apache License 2.0 4 votes vote down vote up
private static Types.NestedField convertOrcToIceberg(TypeDescription orcType, String name,
                                                     TypeUtil.NextID nextID) {

  final int icebergID = icebergID(orcType).orElseGet(nextID::get);
  final boolean isRequired = isRequired(orcType);

  switch (orcType.getCategory()) {
    case BOOLEAN:
      return getIcebergType(icebergID, name, Types.BooleanType.get(), isRequired);
    case BYTE:
    case SHORT:
    case INT:
      return getIcebergType(icebergID, name, Types.IntegerType.get(), isRequired);
    case LONG:
      String longAttributeValue = orcType.getAttributeValue(ICEBERG_LONG_TYPE_ATTRIBUTE);
      LongType longType = longAttributeValue == null ? LongType.LONG : LongType.valueOf(longAttributeValue);
      switch (longType) {
        case TIME:
          return getIcebergType(icebergID, name, Types.TimeType.get(), isRequired);
        case LONG:
          return getIcebergType(icebergID, name, Types.LongType.get(), isRequired);
        default:
          throw new IllegalStateException("Invalid Long type found in ORC type attribute");
      }
    case FLOAT:
      return getIcebergType(icebergID, name, Types.FloatType.get(), isRequired);
    case DOUBLE:
      return getIcebergType(icebergID, name, Types.DoubleType.get(), isRequired);
    case STRING:
    case CHAR:
    case VARCHAR:
      return getIcebergType(icebergID, name, Types.StringType.get(), isRequired);
    case BINARY:
      String binaryAttributeValue = orcType.getAttributeValue(ICEBERG_BINARY_TYPE_ATTRIBUTE);
      BinaryType binaryType = binaryAttributeValue == null ? BinaryType.BINARY :
          BinaryType.valueOf(binaryAttributeValue);
      switch (binaryType) {
        case UUID:
          return getIcebergType(icebergID, name, Types.UUIDType.get(), isRequired);
        case FIXED:
          int fixedLength = Integer.parseInt(orcType.getAttributeValue(ICEBERG_FIELD_LENGTH));
          return getIcebergType(icebergID, name, Types.FixedType.ofLength(fixedLength), isRequired);
        case BINARY:
          return getIcebergType(icebergID, name, Types.BinaryType.get(), isRequired);
        default:
          throw new IllegalStateException("Invalid Binary type found in ORC type attribute");
      }
    case DATE:
      return getIcebergType(icebergID, name, Types.DateType.get(), isRequired);
    case TIMESTAMP:
      return getIcebergType(icebergID, name, Types.TimestampType.withoutZone(), isRequired);
    case TIMESTAMP_INSTANT:
      return getIcebergType(icebergID, name, Types.TimestampType.withZone(), isRequired);
    case DECIMAL:
      return getIcebergType(icebergID, name,
          Types.DecimalType.of(orcType.getPrecision(), orcType.getScale()),
          isRequired);
    case STRUCT: {
      List<String> fieldNames = orcType.getFieldNames();
      List<TypeDescription> fieldTypes = orcType.getChildren();
      List<Types.NestedField> fields = new ArrayList<>(fieldNames.size());
      for (int c = 0; c < fieldNames.size(); ++c) {
        String childName = fieldNames.get(c);
        TypeDescription type = fieldTypes.get(c);
        Types.NestedField field = convertOrcToIceberg(type, childName, nextID);
        fields.add(field);
      }

      return getIcebergType(icebergID, name, Types.StructType.of(fields), isRequired);
    }
    case LIST: {
      TypeDescription elementType = orcType.getChildren().get(0);
      Types.NestedField element = convertOrcToIceberg(elementType, "element", nextID);

      Types.ListType listTypeWithElem = isRequired(elementType) ?
          Types.ListType.ofRequired(element.fieldId(), element.type()) :
          Types.ListType.ofOptional(element.fieldId(), element.type());
      return isRequired ?
          Types.NestedField.required(icebergID, name, listTypeWithElem) :
          Types.NestedField.optional(icebergID, name, listTypeWithElem);
    }
    case MAP: {
      TypeDescription keyType = orcType.getChildren().get(0);
      Types.NestedField key = convertOrcToIceberg(keyType, "key", nextID);
      TypeDescription valueType = orcType.getChildren().get(1);
      Types.NestedField value = convertOrcToIceberg(valueType, "value", nextID);

      Types.MapType mapTypeWithKV = isRequired(valueType) ?
          Types.MapType.ofRequired(key.fieldId(), value.fieldId(), key.type(), value.type()) :
          Types.MapType.ofOptional(key.fieldId(), value.fieldId(), key.type(), value.type());

      return getIcebergType(icebergID, name, mapTypeWithKV, isRequired);
    }
    default:
      // We don't have an answer for union types.
      throw new IllegalArgumentException("Can't handle " + orcType);
  }
}
 
Example 17
Source File: ORCSchemaUtil.java    From iceberg with Apache License 2.0 4 votes vote down vote up
private static TypeDescription convert(Integer fieldId, Type type, boolean isRequired) {
  final TypeDescription orcType;

  switch (type.typeId()) {
    case BOOLEAN:
      orcType = TypeDescription.createBoolean();
      break;
    case INTEGER:
      orcType = TypeDescription.createInt();
      break;
    case TIME:
      orcType = TypeDescription.createLong();
      orcType.setAttribute(ICEBERG_LONG_TYPE_ATTRIBUTE, LongType.TIME.toString());
      break;
    case LONG:
      orcType = TypeDescription.createLong();
      orcType.setAttribute(ICEBERG_LONG_TYPE_ATTRIBUTE, LongType.LONG.toString());
      break;
    case FLOAT:
      orcType = TypeDescription.createFloat();
      break;
    case DOUBLE:
      orcType = TypeDescription.createDouble();
      break;
    case DATE:
      orcType = TypeDescription.createDate();
      break;
    case TIMESTAMP:
      Types.TimestampType tsType = (Types.TimestampType) type;
      if (tsType.shouldAdjustToUTC()) {
        orcType = TypeDescription.createTimestampInstant();
      } else {
        orcType = TypeDescription.createTimestamp();
      }
      break;
    case STRING:
      orcType = TypeDescription.createString();
      break;
    case UUID:
      orcType = TypeDescription.createBinary();
      orcType.setAttribute(ICEBERG_BINARY_TYPE_ATTRIBUTE, BinaryType.UUID.toString());
      break;
    case FIXED:
      orcType = TypeDescription.createBinary();
      orcType.setAttribute(ICEBERG_BINARY_TYPE_ATTRIBUTE, BinaryType.FIXED.toString());
      orcType.setAttribute(ICEBERG_FIELD_LENGTH, Integer.toString(((Types.FixedType) type).length()));
      break;
    case BINARY:
      orcType = TypeDescription.createBinary();
      orcType.setAttribute(ICEBERG_BINARY_TYPE_ATTRIBUTE, BinaryType.BINARY.toString());
      break;
    case DECIMAL: {
      Types.DecimalType decimal = (Types.DecimalType) type;
      orcType = TypeDescription.createDecimal()
          .withScale(decimal.scale())
          .withPrecision(decimal.precision());
      break;
    }
    case STRUCT: {
      orcType = TypeDescription.createStruct();
      for (Types.NestedField field : type.asStructType().fields()) {
        TypeDescription childType = convert(field.fieldId(), field.type(), field.isRequired());
        orcType.addField(field.name(), childType);
      }
      break;
    }
    case LIST: {
      Types.ListType list = (Types.ListType) type;
      TypeDescription elementType = convert(list.elementId(), list.elementType(),
          list.isElementRequired());
      orcType = TypeDescription.createList(elementType);
      break;
    }
    case MAP: {
      Types.MapType map = (Types.MapType) type;
      TypeDescription keyType = convert(map.keyId(), map.keyType(), true);
      TypeDescription valueType = convert(map.valueId(), map.valueType(), map.isValueRequired());
      orcType = TypeDescription.createMap(keyType, valueType);
      break;
    }
    default:
      throw new IllegalArgumentException("Unhandled type " + type.typeId());
  }

  // Set Iceberg column attributes for mapping
  orcType.setAttribute(ICEBERG_ID_ATTRIBUTE, String.valueOf(fieldId));
  orcType.setAttribute(ICEBERG_REQUIRED_ATTRIBUTE, String.valueOf(isRequired));
  return orcType;
}
 
Example 18
Source File: HiveTypeConverter.java    From iceberg with Apache License 2.0 4 votes vote down vote up
public static String convert(Type type) {
  switch (type.typeId()) {
    case BOOLEAN:
      return "boolean";
    case INTEGER:
      return "int";
    case LONG:
      return "bigint";
    case FLOAT:
      return "float";
    case DOUBLE:
      return "double";
    case DATE:
      return "date";
    case TIME:
      return "string";
    case TIMESTAMP:
      return "timestamp";
    case STRING:
    case UUID:
      return "string";
    case FIXED:
      return "binary";
    case BINARY:
      return "binary";
    case DECIMAL:
      final Types.DecimalType decimalType = (Types.DecimalType) type;
      // TODO may be just decimal?
      return String.format("decimal(%s,%s)", decimalType.precision(), decimalType.scale());
    case STRUCT:
      final Types.StructType structType = type.asStructType();
      final String nameToType = structType.fields().stream()
          .map(f -> String.format("%s:%s", f.name(), convert(f.type())))
          .collect(Collectors.joining(","));
      return String.format("struct<%s>", nameToType);
    case LIST:
      final Types.ListType listType = type.asListType();
      return String.format("array<%s>", convert(listType.elementType()));
    case MAP:
      final Types.MapType mapType = type.asMapType();
      return String.format("map<%s,%s>", convert(mapType.keyType()), convert(mapType.valueType()));
    default:
      throw new UnsupportedOperationException(type + " is not supported");
  }
}
 
Example 19
Source File: HiveTypeConverter.java    From metacat with Apache License 2.0 4 votes vote down vote up
/**
 * convert iceberg to hive type.
 * @param type iceberg type.
 * @return hive type string.
 */
public static String fromIcebergToHiveType(final org.apache.iceberg.types.Type type) {
    switch (type.typeId()) {
        case BOOLEAN:
            return serdeConstants.BOOLEAN_TYPE_NAME;
        case INTEGER:
            return serdeConstants.INT_TYPE_NAME;
        case LONG:
            return serdeConstants.BIGINT_TYPE_NAME;
        case FLOAT:
            return serdeConstants.FLOAT_TYPE_NAME;
        case DOUBLE:
            return serdeConstants.DOUBLE_TYPE_NAME;
        case DATE:
            return serdeConstants.DATE_TYPE_NAME;
        case TIME:
            throw new UnsupportedOperationException("Hive does not support time fields");
        case TIMESTAMP:
            return serdeConstants.TIMESTAMP_TYPE_NAME;
        case STRING:
        case UUID:
            return serdeConstants.STRING_TYPE_NAME;
        case FIXED:
            return serdeConstants.BINARY_TYPE_NAME;
        case BINARY:
            return serdeConstants.BINARY_TYPE_NAME;
        case DECIMAL:
            final Types.DecimalType decimalType = (Types.DecimalType) type;
            return String.format("decimal(%s,%s)", decimalType.precision(), decimalType.scale());
        case STRUCT:
            final Types.StructType structType = type.asStructType();
            final String nameToType = (String) structType.fields().stream().map((f) -> {
                return String.format("%s:%s", f.name(), fromIcebergToHiveType(f.type()));
            }).collect(Collectors.joining(","));
            return String.format("struct<%s>", nameToType);
        case LIST:
            final Types.ListType listType = type.asListType();
            return String.format("array<%s>", fromIcebergToHiveType(listType.elementType()));
        case MAP:
            final Types.MapType mapType = type.asMapType();
            return String.format("map<%s,%s>", fromIcebergToHiveType(mapType.keyType()),
                fromIcebergToHiveType(mapType.valueType()));
        default:
            throw new UnsupportedOperationException(type + " is not supported");
    }
}
 
Example 20
Source File: TypeConverter.java    From presto with Apache License 2.0 4 votes vote down vote up
public static Type toPrestoType(org.apache.iceberg.types.Type type, TypeManager typeManager)
{
    switch (type.typeId()) {
        case BOOLEAN:
            return BooleanType.BOOLEAN;
        case BINARY:
        case FIXED:
            return VarbinaryType.VARBINARY;
        case DATE:
            return DateType.DATE;
        case DECIMAL:
            Types.DecimalType decimalType = (Types.DecimalType) type;
            return DecimalType.createDecimalType(decimalType.precision(), decimalType.scale());
        case DOUBLE:
            return DoubleType.DOUBLE;
        case LONG:
            return BigintType.BIGINT;
        case FLOAT:
            return RealType.REAL;
        case INTEGER:
            return IntegerType.INTEGER;
        case TIME:
            return TimeType.TIME;
        case TIMESTAMP:
            Types.TimestampType timestampType = (Types.TimestampType) type;
            if (timestampType.shouldAdjustToUTC()) {
                return TimestampWithTimeZoneType.TIMESTAMP_WITH_TIME_ZONE;
            }
            return TimestampType.TIMESTAMP;
        case UUID:
        case STRING:
            return VarcharType.createUnboundedVarcharType();
        case LIST:
            Types.ListType listType = (Types.ListType) type;
            return new ArrayType(toPrestoType(listType.elementType(), typeManager));
        case MAP:
            Types.MapType mapType = (Types.MapType) type;
            TypeSignature keyType = toPrestoType(mapType.keyType(), typeManager).getTypeSignature();
            TypeSignature valueType = toPrestoType(mapType.valueType(), typeManager).getTypeSignature();
            return typeManager.getParameterizedType(StandardTypes.MAP, ImmutableList.of(TypeSignatureParameter.typeParameter(keyType), TypeSignatureParameter.typeParameter(valueType)));
        case STRUCT:
            List<Types.NestedField> fields = ((Types.StructType) type).fields();
            return RowType.from(fields.stream()
                    .map(field -> new RowType.Field(Optional.of(field.name()), toPrestoType(field.type(), typeManager)))
                    .collect(toImmutableList()));
        default:
            throw new UnsupportedOperationException(format("Cannot convert from Iceberg type '%s' (%s) to Presto type", type, type.typeId()));
    }
}