Java Code Examples for org.apache.iceberg.types.Type#PrimitiveType
The following examples show how to use
org.apache.iceberg.types.Type#PrimitiveType .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: ManifestFileUtil.java From iceberg with Apache License 2.0 | 6 votes |
public static boolean canContainAny(ManifestFile manifest, Iterable<StructLike> partitions, Function<Integer, PartitionSpec> specLookup) { if (manifest.partitions() == null) { return true; } Types.StructType partitionType = specLookup.apply(manifest.partitionSpecId()).partitionType(); List<ManifestFile.PartitionFieldSummary> fieldSummaries = manifest.partitions(); List<Types.NestedField> fields = partitionType.fields(); List<FieldSummary<?>> summaries = Lists.newArrayListWithExpectedSize(fieldSummaries.size()); for (int pos = 0; pos < fieldSummaries.size(); pos += 1) { Type.PrimitiveType primitive = fields.get(pos).type().asPrimitiveType(); summaries.add(new FieldSummary<>(primitive, fieldSummaries.get(pos))); } for (StructLike partition : partitions) { if (canContain(summaries, partition)) { return true; } } return false; }
Example 2
Source File: PartitionTable.java From presto with Apache License 2.0 | 5 votes |
private List<Type> partitionTypes(List<PartitionField> partitionFields) { ImmutableList.Builder<Type> partitionTypeBuilder = ImmutableList.builder(); for (PartitionField partitionField : partitionFields) { Type.PrimitiveType sourceType = idToTypeMapping.get(partitionField.sourceId()); Type type = partitionField.transform().getResultType(sourceType); partitionTypeBuilder.add(type); } return partitionTypeBuilder.build(); }
Example 3
Source File: RandomUtil.java From iceberg with Apache License 2.0 | 5 votes |
public static Object generateDictionaryEncodablePrimitive(Type.PrimitiveType primitive, Random random) { int value = random.nextInt(3); switch (primitive.typeId()) { case BOOLEAN: return true; // doesn't really matter for booleans since they are not dictionary encoded case INTEGER: case DATE: return value; case FLOAT: return (float) value; case DOUBLE: return (double) value; case LONG: case TIME: case TIMESTAMP: return (long) value; case STRING: return String.valueOf(value); case FIXED: byte[] fixed = new byte[((Types.FixedType) primitive).length()]; Arrays.fill(fixed, (byte) value); return fixed; case BINARY: byte[] binary = new byte[value + 1]; Arrays.fill(binary, (byte) value); return binary; case DECIMAL: Types.DecimalType type = (Types.DecimalType) primitive; BigInteger unscaled = new BigInteger(String.valueOf(value + 1)); return new BigDecimal(unscaled, type.scale()); default: throw new IllegalArgumentException( "Cannot generate random value for unknown type: " + primitive); } }
Example 4
Source File: RandomData.java From iceberg with Apache License 2.0 | 5 votes |
@Override protected Object randomValue(Type.PrimitiveType primitive, Random rand) { this.rowCount += 1; if (rowCount > dictionaryEncodedRows) { return RandomUtil.generatePrimitive(primitive, rand); } else { return RandomUtil.generateDictionaryEncodablePrimitive(primitive, rand); } }
Example 5
Source File: RandomData.java From iceberg with Apache License 2.0 | 5 votes |
@Override protected Object randomValue(Type.PrimitiveType primitive, Random rand) { this.rowCount += 1; if (rowCount > dictionaryEncodedRows) { return RandomUtil.generatePrimitive(primitive, rand); } else { return RandomUtil.generateDictionaryEncodablePrimitive(primitive, rand); } }
Example 6
Source File: ManifestFileUtil.java From iceberg with Apache License 2.0 | 5 votes |
@SuppressWarnings("unchecked") FieldSummary(Type.PrimitiveType primitive, ManifestFile.PartitionFieldSummary summary) { this.comparator = Comparators.forType(primitive); this.javaClass = (Class<T>) primitive.typeId().javaClass(); this.lowerBound = Conversions.fromByteBuffer(primitive, summary.lowerBound()); this.upperBound = Conversions.fromByteBuffer(primitive, summary.upperBound()); this.containsNull = summary.containsNull(); }
Example 7
Source File: TestSchemaUpdate.java From iceberg with Apache License 2.0 | 5 votes |
@Test public void testUpdateFailure() { Set<Pair<Type.PrimitiveType, Type.PrimitiveType>> allowedUpdates = Sets.newHashSet( Pair.of(Types.IntegerType.get(), Types.LongType.get()), Pair.of(Types.FloatType.get(), Types.DoubleType.get()), Pair.of(Types.DecimalType.of(9, 2), Types.DecimalType.of(18, 2)) ); List<Type.PrimitiveType> primitives = Lists.newArrayList( Types.BooleanType.get(), Types.IntegerType.get(), Types.LongType.get(), Types.FloatType.get(), Types.DoubleType.get(), Types.DateType.get(), Types.TimeType.get(), Types.TimestampType.withZone(), Types.TimestampType.withoutZone(), Types.StringType.get(), Types.UUIDType.get(), Types.BinaryType.get(), Types.FixedType.ofLength(3), Types.FixedType.ofLength(4), Types.DecimalType.of(9, 2), Types.DecimalType.of(9, 3), Types.DecimalType.of(18, 2) ); for (Type.PrimitiveType fromType : primitives) { for (Type.PrimitiveType toType : primitives) { Schema fromSchema = new Schema(required(1, "col", fromType)); if (fromType.equals(toType) || allowedUpdates.contains(Pair.of(fromType, toType))) { Schema expected = new Schema(required(1, "col", toType)); Schema result = new SchemaUpdate(fromSchema, 1).updateColumn("col", toType).apply(); Assert.assertEquals("Should allow update", expected.asStruct(), result.asStruct()); continue; } String typeChange = fromType.toString() + " -> " + toType.toString(); AssertHelpers.assertThrows("Should reject update: " + typeChange, IllegalArgumentException.class, "change column type: col: " + typeChange, () -> new SchemaUpdate(fromSchema, 1).updateColumn("col", toType)); } } }
Example 8
Source File: RandomGenericData.java From iceberg with Apache License 2.0 | 4 votes |
protected Object randomValue(Type.PrimitiveType primitive, Random rand) { return RandomUtil.generatePrimitive(primitive, rand); }
Example 9
Source File: RandomData.java From iceberg with Apache License 2.0 | 4 votes |
protected Object randomValue(Type.PrimitiveType primitive, Random rand) { return RandomUtil.generatePrimitive(primitive, random); }
Example 10
Source File: RandomData.java From iceberg with Apache License 2.0 | 4 votes |
@Override protected Object randomValue(Type.PrimitiveType primitive, Random random) { return RandomUtil.generateDictionaryEncodablePrimitive(primitive, random); }
Example 11
Source File: RandomData.java From iceberg with Apache License 2.0 | 4 votes |
@Override protected Object randomValue(Type.PrimitiveType primitive, Random random) { return RandomUtil.generateDictionaryEncodablePrimitive(primitive, random); }
Example 12
Source File: SchemaParser.java From iceberg with Apache License 2.0 | 4 votes |
static void toJson(Type.PrimitiveType primitive, JsonGenerator generator) throws IOException { generator.writeString(primitive.toString()); }
Example 13
Source File: UpdateSchema.java From iceberg with Apache License 2.0 | 2 votes |
/** * Update a column in the schema to a new primitive type. * <p> * The name is used to find the column to update using {@link Schema#findField(String)}. * <p> * Only updates that widen types are allowed. * <p> * Columns may be updated and renamed in the same schema update. * * @param name name of the column to rename * @param newType replacement type for the column * @return this for method chaining * @throws IllegalArgumentException If name doesn't identify a column in the schema or if this * change introduces a type incompatibility or if it conflicts * with other additions, renames, or updates. */ UpdateSchema updateColumn(String name, Type.PrimitiveType newType);
Example 14
Source File: UpdateSchema.java From iceberg with Apache License 2.0 | 2 votes |
/** * Update a column in the schema to a new primitive type. * <p> * The name is used to find the column to update using {@link Schema#findField(String)}. * <p> * Only updates that widen types are allowed. * <p> * Columns may be updated and renamed in the same schema update. * * @param name name of the column to rename * @param newType replacement type for the column * @param newDoc replacement documentation string for the column * @return this for method chaining * @throws IllegalArgumentException If name doesn't identify a column in the schema or if this * change introduces a type incompatibility or if it conflicts * with other additions, renames, or updates. */ default UpdateSchema updateColumn(String name, Type.PrimitiveType newType, String newDoc) { return updateColumn(name, newType).updateColumnDoc(name, newDoc); }