org.apache.avro.LogicalType Java Examples
The following examples show how to use
org.apache.avro.LogicalType.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ParquetAvro.java From iceberg with Apache License 2.0 | 7 votes |
@Override @SuppressWarnings("unchecked") public Conversion<Object> getConversionFor(LogicalType logicalType) { if (logicalType == null) { return null; } if (logicalType instanceof LogicalTypes.Decimal) { LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) logicalType; if (decimal.getPrecision() <= 9) { return (Conversion<Object>) intDecimalConversion; } else if (decimal.getPrecision() <= 18) { return (Conversion<Object>) longDecimalConversion; } else { return (Conversion<Object>) fixedDecimalConversion; } } else if ("uuid".equals(logicalType.getName())) { return (Conversion<Object>) uuidConversion; } return super.getConversionFor(logicalType); }
Example #2
Source File: ParquetAvro.java From iceberg with Apache License 2.0 | 6 votes |
@Override @SuppressWarnings("unchecked") public <T> Conversion<T> getConversionByClass(Class<T> datumClass, LogicalType logicalType) { if (logicalType == null) { return null; } if (logicalType instanceof ParquetDecimal) { ParquetDecimal decimal = (ParquetDecimal) logicalType; if (decimal.precision() <= 9) { return (Conversion<T>) intDecimalConversion; } else if (decimal.precision() <= 18) { return (Conversion<T>) longDecimalConversion; } else { return (Conversion<T>) fixedDecimalConversion; } } else if ("uuid".equals(logicalType.getName())) { return (Conversion<T>) uuidConversion; } return super.getConversionByClass(datumClass, logicalType); }
Example #3
Source File: AvroSchemaConverter.java From parquet-mr with Apache License 2.0 | 6 votes |
private LogicalTypeAnnotation convertLogicalType(LogicalType logicalType) { if (logicalType == null) { return null; } else if (logicalType instanceof LogicalTypes.Decimal) { LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) logicalType; return decimalType(decimal.getScale(), decimal.getPrecision()); } else if (logicalType instanceof LogicalTypes.Date) { return dateType(); } else if (logicalType instanceof LogicalTypes.TimeMillis) { return timeType(true, MILLIS); } else if (logicalType instanceof LogicalTypes.TimeMicros) { return timeType(true, MICROS); } else if (logicalType instanceof LogicalTypes.TimestampMillis) { return timestampType(true, MILLIS); } else if (logicalType instanceof LogicalTypes.TimestampMicros) { return timestampType(true, MICROS); } else if (logicalType.getName().equals(LogicalTypes.uuid().getName()) && writeParquetUUID) { return uuidType(); } return null; }
Example #4
Source File: AvroWriteSupport.java From parquet-mr with Apache License 2.0 | 6 votes |
private <D> Object convert(Schema schema, LogicalType logicalType, Conversion<D> conversion, Object datum) { if (conversion == null) { return datum; } Class<D> fromClass = conversion.getConvertedType(); switch (schema.getType()) { case RECORD: return conversion.toRecord(fromClass.cast(datum), schema, logicalType); case ENUM: return conversion.toEnumSymbol(fromClass.cast(datum), schema, logicalType); case ARRAY: return conversion.toArray(fromClass.cast(datum), schema, logicalType); case MAP: return conversion.toMap(fromClass.cast(datum), schema, logicalType); case FIXED: return conversion.toFixed(fromClass.cast(datum), schema, logicalType); case STRING: return conversion.toCharSequence(fromClass.cast(datum), schema, logicalType); case BYTES: return conversion.toBytes(fromClass.cast(datum), schema, logicalType); case INT: return conversion.toInt(fromClass.cast(datum), schema, logicalType); case LONG: return conversion.toLong(fromClass.cast(datum), schema, logicalType); case FLOAT: return conversion.toFloat(fromClass.cast(datum), schema, logicalType); case DOUBLE: return conversion.toDouble(fromClass.cast(datum), schema, logicalType); case BOOLEAN: return conversion.toBoolean(fromClass.cast(datum), schema, logicalType); } return datum; }
Example #5
Source File: AvroRecordConverter.java From DataflowTemplates with Apache License 2.0 | 6 votes |
private Optional<Timestamp> readTimestamp( GenericRecord record, Schema.Type avroType, LogicalType logicalType, String fieldName) { switch (avroType) { case LONG: if (LogicalTypes.timestampMillis().equals(logicalType)) { return Optional.ofNullable((Long) record.get(fieldName)) .map(x -> Timestamp.ofTimeMicroseconds(1000L * x)); } if (LogicalTypes.timestampMicros().equals(logicalType)) { return Optional.ofNullable((Long) record.get(fieldName)) .map(Timestamp::ofTimeMicroseconds); } // Default to micro-seconds. return Optional.ofNullable((Long) record.get(fieldName)).map(Timestamp::ofTimeMicroseconds); case STRING: return Optional.ofNullable((Utf8) record.get(fieldName)) .map(Utf8::toString) .map(Timestamp::parseTimestamp); default: throw new IllegalArgumentException("Cannot interpret " + avroType + " as TIMESTAMP"); } }
Example #6
Source File: AvroRowSerializationSchema.java From flink with Apache License 2.0 | 6 votes |
private byte[] convertFromDecimal(Schema schema, BigDecimal decimal) { final LogicalType logicalType = schema.getLogicalType(); if (logicalType instanceof LogicalTypes.Decimal) { final LogicalTypes.Decimal decimalType = (LogicalTypes.Decimal) logicalType; // rescale to target type final BigDecimal rescaled = decimal.setScale(decimalType.getScale(), BigDecimal.ROUND_UNNECESSARY); // byte array must contain the two's-complement representation of the // unscaled integer value in big-endian byte order return decimal.unscaledValue().toByteArray(); } else { throw new RuntimeException("Unsupported decimal type."); } }
Example #7
Source File: ParquetAvro.java From iceberg with Apache License 2.0 | 6 votes |
@Override @SuppressWarnings("unchecked") public <T> Conversion<T> getConversionByClass(Class<T> datumClass, LogicalType logicalType) { if (logicalType == null) { return null; } if (logicalType instanceof ParquetDecimal) { ParquetDecimal decimal = (ParquetDecimal) logicalType; if (decimal.precision() <= 9) { return (Conversion<T>) intDecimalConversion; } else if (decimal.precision() <= 18) { return (Conversion<T>) longDecimalConversion; } else { return (Conversion<T>) fixedDecimalConversion; } } else if ("uuid".equals(logicalType.getName())) { return (Conversion<T>) uuidConversion; } return super.getConversionByClass(datumClass, logicalType); }
Example #8
Source File: TestCircularReferences.java From parquet-mr with Apache License 2.0 | 6 votes |
@Override public IndexedRecord fromRecord(IndexedRecord value, Schema schema, LogicalType type) { // read side long id = getId(value, schema); // keep track of this for later references references.put(id, value); // call any callbacks waiting to resolve this id List<Callback> callbacks = callbacksById.get(id); for (Callback callback : callbacks) { callback.set(value); } return value; }
Example #9
Source File: ParquetAvro.java From iceberg with Apache License 2.0 | 6 votes |
@Override @SuppressWarnings("unchecked") public Conversion<Object> getConversionFor(LogicalType logicalType) { if (logicalType == null) { return null; } if (logicalType instanceof LogicalTypes.Decimal) { LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) logicalType; if (decimal.getPrecision() <= 9) { return (Conversion<Object>) intDecimalConversion; } else if (decimal.getPrecision() <= 18) { return (Conversion<Object>) longDecimalConversion; } else { return (Conversion<Object>) fixedDecimalConversion; } } else if ("uuid".equals(logicalType.getName())) { return (Conversion<Object>) uuidConversion; } return super.getConversionFor(logicalType); }
Example #10
Source File: AvroRecordConverter.java From DataflowTemplates with Apache License 2.0 | 6 votes |
private Optional<Date> readDate( GenericRecord record, Schema.Type avroType, LogicalType logicalType, String fieldName) { switch (avroType) { case INT: if (logicalType == null || !LogicalTypes.date().equals(logicalType)) { throw new IllegalArgumentException( "Cannot interpret Avrotype INT Logicaltype " + logicalType + " as DATE"); } // Avro Date is number of days since Jan 1, 1970. // Have to convert to Java Date first before creating google.cloud.core.Date return Optional.ofNullable((Integer) record.get(fieldName)) .map(x -> new java.util.Date((long) x * 24L * 3600L * 1000L)) .map(Date::fromJavaUtilDate); case STRING: return Optional.ofNullable((Utf8) record.get(fieldName)) .map(Utf8::toString) .map(Date::parseDate); default: throw new IllegalArgumentException("Cannot interpret " + avroType + " as DATE"); } }
Example #11
Source File: AvroSchemaConverter190Int96Avro18.java From datacollector with Apache License 2.0 | 6 votes |
private LogicalType convertOriginalTypeToLogicalType(OriginalType annotation, DecimalMetadata meta) { if (annotation == null) { return null; } switch (annotation) { case DECIMAL: return LogicalTypes.decimal(meta.getPrecision(), meta.getScale()); case DATE: return LogicalTypes.date(); case TIME_MILLIS: return LogicalTypes.timeMillis(); case TIME_MICROS: return LogicalTypes.timeMicros(); case TIMESTAMP_MILLIS: return LogicalTypes.timestampMillis(); case TIMESTAMP_MICROS: return LogicalTypes.timestampMicros(); default: return null; } }
Example #12
Source File: AvroSchemaConverter190Int96Avro18.java From datacollector with Apache License 2.0 | 6 votes |
private OriginalType convertLogicalType(LogicalType logicalType) { if (logicalType == null) { return null; } else if (logicalType instanceof LogicalTypes.Decimal) { return OriginalType.DECIMAL; } else if (logicalType instanceof LogicalTypes.Date) { return OriginalType.DATE; } else if (logicalType instanceof LogicalTypes.TimeMillis) { return OriginalType.TIME_MILLIS; } else if (logicalType instanceof LogicalTypes.TimeMicros) { return OriginalType.TIME_MICROS; } else if (logicalType instanceof LogicalTypes.TimestampMillis) { return OriginalType.TIMESTAMP_MILLIS; } else if (logicalType instanceof LogicalTypes.TimestampMicros) { return OriginalType.TIMESTAMP_MICROS; } return null; }
Example #13
Source File: BigQueryAvroUtils.java From beam with Apache License 2.0 | 6 votes |
private static List<Object> convertRepeatedField( Schema schema, TableFieldSchema fieldSchema, Object v) { Type arrayType = schema.getType(); verify( arrayType == Type.ARRAY, "BigQuery REPEATED field %s should be Avro ARRAY, not %s", fieldSchema.getName(), arrayType); // REPEATED fields are represented as Avro arrays. if (v == null) { // Handle the case of an empty repeated field. return new ArrayList<>(); } @SuppressWarnings("unchecked") List<Object> elements = (List<Object>) v; ArrayList<Object> values = new ArrayList<>(); Type elementType = schema.getElementType().getType(); LogicalType elementLogicalType = schema.getElementType().getLogicalType(); for (Object element : elements) { values.add(convertRequiredField(elementType, elementLogicalType, fieldSchema, element)); } return values; }
Example #14
Source File: AvroWriteSupportInt96Avro18.java From datacollector with Apache License 2.0 | 6 votes |
private <D> Object convert(Schema schema, LogicalType logicalType, Conversion<D> conversion, Object datum) { if (conversion == null) { return datum; } Class<D> fromClass = conversion.getConvertedType(); switch (schema.getType()) { case RECORD: return conversion.toRecord(fromClass.cast(datum), schema, logicalType); case ENUM: return conversion.toEnumSymbol(fromClass.cast(datum), schema, logicalType); case ARRAY: return conversion.toArray(fromClass.cast(datum), schema, logicalType); case MAP: return conversion.toMap(fromClass.cast(datum), schema, logicalType); case FIXED: return conversion.toFixed(fromClass.cast(datum), schema, logicalType); case STRING: return conversion.toCharSequence(fromClass.cast(datum), schema, logicalType); case BYTES: return conversion.toBytes(fromClass.cast(datum), schema, logicalType); case INT: return conversion.toInt(fromClass.cast(datum), schema, logicalType); case LONG: return conversion.toLong(fromClass.cast(datum), schema, logicalType); case FLOAT: return conversion.toFloat(fromClass.cast(datum), schema, logicalType); case DOUBLE: return conversion.toDouble(fromClass.cast(datum), schema, logicalType); case BOOLEAN: return conversion.toBoolean(fromClass.cast(datum), schema, logicalType); default: break; } return datum; }
Example #15
Source File: TalendTypeTest.java From components with Apache License 2.0 | 5 votes |
/** * Checks {@link TalendType#convertFromAvro(Schema)} throws {@link UnsupportedOperationException} with following message * "Unrecognized type", * when unknown logical type is passed */ @Test public void testConvertFromAvroUnsupportedLogicalType() { thrown.expect(UnsupportedOperationException.class); thrown.expectMessage("Unrecognized type unsupported"); LogicalType unsupported = new LogicalType("unsupported"); Schema fieldSchema = unsupported.addToSchema(AvroUtils._string()); TalendType.convertFromAvro(fieldSchema); }
Example #16
Source File: UUIDConversion.java From iceberg with Apache License 2.0 | 5 votes |
@Override public GenericFixed toFixed(UUID value, Schema schema, LogicalType type) { ByteBuffer buffer = ByteBuffer.allocate(16); buffer.order(ByteOrder.BIG_ENDIAN); buffer.putLong(value.getMostSignificantBits()); buffer.putLong(value.getLeastSignificantBits()); return new GenericData.Fixed(schema, buffer.array()); }
Example #17
Source File: AvroRowSerializationSchema.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
private int convertFromDate(Schema schema, Date date) { final LogicalType logicalType = schema.getLogicalType(); if (logicalType == LogicalTypes.date()) { // adopted from Apache Calcite final long time = date.getTime(); final long converted = time + (long) LOCAL_TZ.getOffset(time); return (int) (converted / 86400000L); } else { throw new RuntimeException("Unsupported date type."); } }
Example #18
Source File: AvroRowSerializationSchema.java From flink with Apache License 2.0 | 5 votes |
private long convertFromTimestamp(Schema schema, Timestamp date) { final LogicalType logicalType = schema.getLogicalType(); if (logicalType == LogicalTypes.timestampMillis()) { // adopted from Apache Calcite final long time = date.getTime(); return time + (long) LOCAL_TZ.getOffset(time); } else { throw new RuntimeException("Unsupported timestamp type."); } }
Example #19
Source File: AvroRowSerializationSchema.java From flink with Apache License 2.0 | 5 votes |
private int convertFromTime(Schema schema, Time date) { final LogicalType logicalType = schema.getLogicalType(); if (logicalType == LogicalTypes.timeMillis()) { // adopted from Apache Calcite final long time = date.getTime(); final long converted = time + (long) LOCAL_TZ.getOffset(time); return (int) (converted % 86400000L); } else { throw new RuntimeException("Unsupported time type."); } }
Example #20
Source File: AvroRowSerializationSchema.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
private byte[] convertFromDecimal(Schema schema, BigDecimal decimal) { final LogicalType logicalType = schema.getLogicalType(); if (logicalType instanceof LogicalTypes.Decimal) { final LogicalTypes.Decimal decimalType = (LogicalTypes.Decimal) logicalType; // rescale to target type final BigDecimal rescaled = decimal.setScale(decimalType.getScale(), BigDecimal.ROUND_UNNECESSARY); // byte array must contain the two's-complement representation of the // unscaled integer value in big-endian byte order return decimal.unscaledValue().toByteArray(); } else { throw new RuntimeException("Unsupported decimal type."); } }
Example #21
Source File: AvroRowSerializationSchema.java From flink with Apache License 2.0 | 5 votes |
private int convertFromDate(Schema schema, Date date) { final LogicalType logicalType = schema.getLogicalType(); if (logicalType == LogicalTypes.date()) { // adopted from Apache Calcite final long time = date.getTime(); final long converted = time + (long) LOCAL_TZ.getOffset(time); return (int) (converted / 86400000L); } else { throw new RuntimeException("Unsupported date type."); } }
Example #22
Source File: AvroRowSerializationSchema.java From flink with Apache License 2.0 | 5 votes |
private byte[] convertFromDecimal(Schema schema, BigDecimal decimal) { final LogicalType logicalType = schema.getLogicalType(); if (logicalType instanceof LogicalTypes.Decimal) { final LogicalTypes.Decimal decimalType = (LogicalTypes.Decimal) logicalType; // rescale to target type final BigDecimal rescaled = decimal.setScale(decimalType.getScale(), BigDecimal.ROUND_UNNECESSARY); // byte array must contain the two's-complement representation of the // unscaled integer value in big-endian byte order return decimal.unscaledValue().toByteArray(); } else { throw new RuntimeException("Unsupported decimal type."); } }
Example #23
Source File: TypeConverterUtils.java From components with Apache License 2.0 | 5 votes |
/** * @param srcSchema THe schema to investigate. * @return The Date/Time output type that corresponds to the logical type of the schema, or null if none. */ public static TypeConverterOutputTypes getDateTimeTypeFromLogicalType(Schema srcSchema) { LogicalType srcLogicalType = AvroUtils.unwrapIfNullable(srcSchema).getLogicalType(); if (srcLogicalType instanceof LogicalTypes.Date) { return TypeConverterOutputTypes.Date; } else if (srcLogicalType instanceof LogicalTypes.TimeMillis) { return TypeConverterOutputTypes.Time; } else if (srcLogicalType instanceof LogicalTypes.TimestampMillis) { return TypeConverterOutputTypes.DateTime; } else { return null; } }
Example #24
Source File: AvroSchemaUtil.java From iceberg with Apache License 2.0 | 5 votes |
public static boolean isTimestamptz(Schema schema) { LogicalType logicalType = schema.getLogicalType(); if (logicalType != null && logicalType instanceof LogicalTypes.TimestampMicros) { // timestamptz is adjusted to UTC Object value = schema.getObjectProp(ADJUST_TO_UTC_PROP); if (value instanceof Boolean) { return (Boolean) value; } else if (value instanceof String) { return Boolean.parseBoolean((String) value); } } return false; }
Example #25
Source File: AvroSchemaUtil.java From iceberg with Apache License 2.0 | 5 votes |
public static boolean isTimestamptz(Schema schema) { LogicalType logicalType = schema.getLogicalType(); if (logicalType != null && logicalType instanceof LogicalTypes.TimestampMicros) { // timestamptz is adjusted to UTC Object value = schema.getObjectProp(ADJUST_TO_UTC_PROP); if (value instanceof Boolean) { return (Boolean) value; } else if (value instanceof String) { return Boolean.parseBoolean((String) value); } } return false; }
Example #26
Source File: UUIDConversion.java From iceberg with Apache License 2.0 | 5 votes |
@Override public UUID fromFixed(GenericFixed value, Schema schema, LogicalType type) { ByteBuffer buffer = ByteBuffer.wrap(value.bytes()); buffer.order(ByteOrder.BIG_ENDIAN); long mostSigBits = buffer.getLong(); long leastSigBits = buffer.getLong(); return new UUID(mostSigBits, leastSigBits); }
Example #27
Source File: UUIDConversion.java From iceberg with Apache License 2.0 | 5 votes |
@Override public GenericFixed toFixed(UUID value, Schema schema, LogicalType type) { ByteBuffer buffer = ByteBuffer.allocate(16); buffer.order(ByteOrder.BIG_ENDIAN); buffer.putLong(value.getMostSignificantBits()); buffer.putLong(value.getLeastSignificantBits()); return new GenericData.Fixed(schema, buffer.array()); }
Example #28
Source File: ConvertAvroTypeToSQL.java From components with Apache License 2.0 | 5 votes |
public int convertToSQLType(Schema schema) { Schema.Type type = schema.getType(); // The standard Avro Type LogicalType logicalType = schema.getLogicalType(); // The logical type for Data by example String javaType = schema.getProp(SchemaConstants.JAVA_CLASS_FLAG); // And the Talend java type if standard Avro type is Union if (logicalType == null && type == Schema.Type.UNION) { for (Schema s : schema.getTypes()) { logicalType = null; if (s.getType() != Schema.Type.NULL) { type = s.getType(); javaType = s.getProp(SchemaConstants.JAVA_CLASS_FLAG); logicalType = s.getLogicalType(); if (javaType == null && logicalType == null) { type = s.getType(); // Get Avro type if JAVA_CLASS_FLAG is not defined } break; } } } int sqlType = Types.NULL; if (logicalType != null) { sqlType = convertAvroLogicialType(logicalType); } else if (javaType == null) { sqlType = convertRawAvroType(type); } else { sqlType = convertTalendAvroType(javaType); } if(this.config.CONVERT_SQLTYPE_TO_ANOTHER_SQLTYPE.containsKey(sqlType)){ sqlType = this.config.CONVERT_SQLTYPE_TO_ANOTHER_SQLTYPE.get(sqlType); } return sqlType; }
Example #29
Source File: AvroUtilsTest.java From beam with Apache License 2.0 | 5 votes |
private static GenericRecord getGenericRecord() { LogicalType decimalType = LogicalTypes.decimal(Integer.MAX_VALUE) .addToSchema(org.apache.avro.Schema.create(Type.BYTES)) .getLogicalType(); ByteBuffer encodedDecimal = new Conversions.DecimalConversion().toBytes(BIG_DECIMAL, null, decimalType); return new GenericRecordBuilder(getAvroSchema()) .set("bool", true) .set("int", 43) .set("long", 44L) .set("float", (float) 44.1) .set("double", (double) 44.2) .set("string", new Utf8("string")) .set("bytes", ByteBuffer.wrap(BYTE_ARRAY)) .set("decimal", encodedDecimal) .set("timestampMillis", DATE_TIME.getMillis()) .set("row", getSubGenericRecord("row")) .set("array", ImmutableList.of(getSubGenericRecord("array"), getSubGenericRecord("array"))) .set( "map", ImmutableMap.of( new Utf8("k1"), getSubGenericRecord("map"), new Utf8("k2"), getSubGenericRecord("map"))) .build(); }
Example #30
Source File: PulsarMetadata.java From pulsar with Apache License 2.0 | 5 votes |
@VisibleForTesting static Type convertType(Schema.Type avroType, LogicalType logicalType) { switch (avroType) { case BOOLEAN: return BooleanType.BOOLEAN; case INT: if (logicalType == LogicalTypes.timeMillis()) { return TIME; } else if (logicalType == LogicalTypes.date()) { return DATE; } return IntegerType.INTEGER; case LONG: if (logicalType == LogicalTypes.timestampMillis()) { return TIMESTAMP; } return BigintType.BIGINT; case FLOAT: return RealType.REAL; case DOUBLE: return DoubleType.DOUBLE; case BYTES: return VarbinaryType.VARBINARY; case STRING: return VarcharType.VARCHAR; case ENUM: return VarcharType.VARCHAR; default: log.error("Cannot convert type: %s", avroType); return null; } }