org.apache.avro.LogicalTypes Java Examples

The following examples show how to use org.apache.avro.LogicalTypes. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ParquetAvro.java    From iceberg with Apache License 2.0 7 votes vote down vote up
@Override
@SuppressWarnings("unchecked")
public Conversion<Object> getConversionFor(LogicalType logicalType) {
  if (logicalType == null) {
    return null;
  }

  if (logicalType instanceof LogicalTypes.Decimal) {
    LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) logicalType;
    if (decimal.getPrecision() <= 9) {
      return (Conversion<Object>) intDecimalConversion;
    } else if (decimal.getPrecision() <= 18) {
      return (Conversion<Object>) longDecimalConversion;
    } else {
      return (Conversion<Object>) fixedDecimalConversion;
    }
  } else if ("uuid".equals(logicalType.getName())) {
    return (Conversion<Object>) uuidConversion;
  }
  return super.getConversionFor(logicalType);
}
 
Example #2
Source File: GenerateAvroJavaTask.java    From gradle-avro-plugin with Apache License 2.0 6 votes vote down vote up
@TaskAction
protected void process() {
    getLogger().debug("Using outputCharacterEncoding {}", getOutputCharacterEncoding().getOrNull());
    getLogger().debug("Using stringType {}", stringTypeProvider.get().name());
    getLogger().debug("Using fieldVisibility {}", fieldVisibilityProvider.get().name());
    getLogger().debug("Using templateDirectory '{}'", getTemplateDirectory().getOrNull());
    getLogger().debug("Using createSetters {}", isCreateSetters().get());
    getLogger().debug("Using createOptionalGetters {}", isCreateOptionalGetters().get());
    getLogger().debug("Using gettersReturnOptional {}", isGettersReturnOptional().get());
    getLogger().debug("Using optionalGettersForNullableFieldsOnly {}", isOptionalGettersForNullableFieldsOnly().get());
    getLogger().debug("Using enableDecimalLogicalType {}", isEnableDecimalLogicalType().get());
    getLogger().debug("Using logicalTypeFactories {}",
        logicalTypeFactories.get().entrySet().stream().collect(Collectors.toMap(
            Map.Entry::getKey,
            (Map.Entry<String, Class<? extends LogicalTypes.LogicalTypeFactory>> e) -> e.getValue().getName()
        )));
    getLogger().debug("Using customConversions {}",
        customConversions.get().stream().map(v -> ((Class) v).getName()).collect(Collectors.toList()));
    getLogger().info("Found {} files", getInputs().getSourceFiles().getFiles().size());
    failOnUnsupportedFiles();
    processFiles();
}
 
Example #3
Source File: TypeConverterUtils.java    From components with Apache License 2.0 6 votes vote down vote up
/**
 * Generate a schema from output type and format
 *
 * @param outputType
 * @param outputFormat
 * @return
 */
public static Schema getSchema(TypeConverterOutputTypes outputType, String outputFormat) {
    Schema result = Schema.create(outputType.getTargetType());
    switch (outputType) {
    case Date:
        result = LogicalTypes.date().addToSchema(result);
        break;
    case Time:
        result = LogicalTypes.timeMillis().addToSchema(result);
        break;
    case DateTime:
        result = LogicalTypes.timestampMillis().addToSchema(result);
        break;
    }
    return result;
}
 
Example #4
Source File: TestAvroTypeUtil.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testConvertAvroRecordToMapWithFieldTypeOfBinaryAndLogicalTypeDecimal() {
    // Create a field schema like {"type":"binary","name":"amount","logicalType":"decimal","precision":18,"scale":8}
    final LogicalTypes.Decimal decimalType = LogicalTypes.decimal(18, 8);
    final Schema fieldSchema = Schema.create(Type.BYTES);
    decimalType.addToSchema(fieldSchema);

    // Create a field named "amount" using the field schema above
    final Schema.Field field = new Schema.Field("amount", fieldSchema, null, (Object)null);

    // Create an overall record schema with the amount field
    final Schema avroSchema = Schema.createRecord(Collections.singletonList(field));

    // Create an example Avro record with the amount field of type binary and a logical type of decimal
    final BigDecimal expectedBigDecimal = new BigDecimal("1234567890.12345678");
    final GenericRecord genericRecord = new GenericData.Record(avroSchema);
    genericRecord.put("amount", new Conversions.DecimalConversion().toBytes(expectedBigDecimal, fieldSchema, decimalType));

    // Convert the Avro schema to a Record schema
    thenConvertAvroSchemaToRecordSchema(avroSchema, expectedBigDecimal, genericRecord);
}
 
Example #5
Source File: TestAvroUtils.java    From envelope with Apache License 2.0 6 votes vote down vote up
@Test
public void toTypeSchemaDecimalNullable() throws Exception {
  // Defaults to precision 10, scale 0
  Schema schema = AvroUtils.typeFor(DataTypes.createDecimalType());

  assertEquals("Invalid type", Schema.Type.UNION, schema.getType());
  assertEquals("Invalid union size", 2, schema.getTypes().size());

  for (Schema s : schema.getTypes()) {
    assertThat("Invalid union types", s.getType(), anyOf(is(Schema.Type.BYTES), is(Schema.Type.NULL)));
    if (s.getType().equals(Schema.Type.BYTES)) {
      assertEquals("Invalid LogicalType", LogicalTypes.decimal(10, 0), s.getLogicalType());
    }
  }

  //System.out.println(schema.toString(true));
}
 
Example #6
Source File: TestGenericLogicalTypes.java    From parquet-mr with Apache License 2.0 6 votes vote down vote up
@Test
public void testWriteUUIDReadStringSchema() throws IOException {
  Schema uuidSchema = record("R",
      field("uuid", LogicalTypes.uuid().addToSchema(Schema.create(STRING))));
  GenericRecord u1 = instance(uuidSchema, "uuid", UUID.randomUUID());
  GenericRecord u2 = instance(uuidSchema, "uuid", UUID.randomUUID());

  Schema stringUuidSchema = Schema.create(STRING);
  stringUuidSchema.addProp(GenericData.STRING_PROP, "String");
  Schema stringSchema = record("R", field("uuid", stringUuidSchema));
  GenericRecord s1 = instance(stringSchema, "uuid", u1.get("uuid").toString());
  GenericRecord s2 = instance(stringSchema, "uuid", u2.get("uuid").toString());

  File test = write(GENERIC, uuidSchema, u1, u2);
  Assert.assertEquals("Should read UUIDs as Strings",
      Arrays.asList(s1, s2), read(GENERIC, stringSchema, test));
}
 
Example #7
Source File: TestAvroTypeUtil.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testConvertAvroRecordToMapWithFieldTypeOfFixedAndLogicalTypeDecimal() {
   // Create a field schema like {"type":"fixed","name":"amount","size":16,"logicalType":"decimal","precision":18,"scale":8}
   final LogicalTypes.Decimal decimalType = LogicalTypes.decimal(18, 8);
    final Schema fieldSchema = Schema.createFixed("amount", null, null, 16);
    decimalType.addToSchema(fieldSchema);

    // Create a field named "amount" using the field schema above
    final Schema.Field field = new Schema.Field("amount", fieldSchema, null, (Object)null);

    // Create an overall record schema with the amount field
    final Schema avroSchema = Schema.createRecord(Collections.singletonList(field));

    // Create an example Avro record with the amount field of type fixed and a logical type of decimal
    final BigDecimal expectedBigDecimal = new BigDecimal("1234567890.12345678");
    final GenericRecord genericRecord = new GenericData.Record(avroSchema);
    genericRecord.put("amount", new Conversions.DecimalConversion().toFixed(expectedBigDecimal, fieldSchema, decimalType));

    // Convert the Avro schema to a Record schema
    thenConvertAvroSchemaToRecordSchema(avroSchema, expectedBigDecimal, genericRecord);
}
 
Example #8
Source File: TestReflectLogicalTypes.java    From parquet-mr with Apache License 2.0 6 votes vote down vote up
@Test
public void testReadUUIDArray() throws IOException {
  Schema uuidArraySchema = SchemaBuilder.record(RecordWithUUIDArray.class.getName())
      .fields()
      .name("uuids").type().array().items().stringType().noDefault()
      .endRecord();
  LogicalTypes.uuid().addToSchema(
      uuidArraySchema.getField("uuids").schema().getElementType());

  UUID u1 = UUID.randomUUID();
  UUID u2 = UUID.randomUUID();

  GenericRecord r = new GenericData.Record(uuidArraySchema);
  r.put("uuids", Arrays.asList(u1.toString(), u2.toString()));

  RecordWithUUIDArray expected = new RecordWithUUIDArray();
  expected.uuids = new UUID[] {u1, u2};

  File test = write(uuidArraySchema, r);

  Assert.assertEquals("Should convert Strings to UUIDs",
      expected,
      read(REFLECT, uuidArraySchema, test).get(0));
}
 
Example #9
Source File: AvroRecordConverter.java    From DataflowTemplates with Apache License 2.0 6 votes vote down vote up
private Optional<Date> readDate(
    GenericRecord record, Schema.Type avroType, LogicalType logicalType, String fieldName) {
  switch (avroType) {
    case INT:
      if (logicalType == null || !LogicalTypes.date().equals(logicalType)) {
        throw new IllegalArgumentException(
            "Cannot interpret Avrotype INT Logicaltype " + logicalType + " as DATE");
      }
      // Avro Date is number of days since Jan 1, 1970.
      // Have to convert to Java Date first before creating google.cloud.core.Date
      return Optional.ofNullable((Integer) record.get(fieldName))
          .map(x -> new java.util.Date((long) x * 24L * 3600L * 1000L))
          .map(Date::fromJavaUtilDate);
    case STRING:
      return Optional.ofNullable((Utf8) record.get(fieldName))
          .map(Utf8::toString)
          .map(Date::parseDate);
    default:
      throw new IllegalArgumentException("Cannot interpret " + avroType + " as DATE");
  }
}
 
Example #10
Source File: AvroRecordConverter.java    From DataflowTemplates with Apache License 2.0 6 votes vote down vote up
private Optional<Timestamp> readTimestamp(
    GenericRecord record, Schema.Type avroType, LogicalType logicalType, String fieldName) {
  switch (avroType) {
    case LONG:
      if (LogicalTypes.timestampMillis().equals(logicalType)) {
        return Optional.ofNullable((Long) record.get(fieldName))
            .map(x -> Timestamp.ofTimeMicroseconds(1000L * x));
      }
      if (LogicalTypes.timestampMicros().equals(logicalType)) {
        return Optional.ofNullable((Long) record.get(fieldName))
            .map(Timestamp::ofTimeMicroseconds);
      }
      // Default to micro-seconds.
      return Optional.ofNullable((Long) record.get(fieldName)).map(Timestamp::ofTimeMicroseconds);
    case STRING:
      return Optional.ofNullable((Utf8) record.get(fieldName))
          .map(Utf8::toString)
          .map(Timestamp::parseTimestamp);
    default:
      throw new IllegalArgumentException("Cannot interpret " + avroType + " as TIMESTAMP");
  }
}
 
Example #11
Source File: TestGenericLogicalTypes.java    From parquet-mr with Apache License 2.0 6 votes vote down vote up
@Test
public void testWriteNullableUUID() throws IOException {
  Schema nullableUuidSchema = record("R",
      optionalField("uuid", LogicalTypes.uuid().addToSchema(Schema.create(STRING))));
  GenericRecord u1 = instance(nullableUuidSchema, "uuid", UUID.randomUUID());
  GenericRecord u2 = instance(nullableUuidSchema, "uuid", null);

  Schema stringUuidSchema = Schema.create(STRING);
  stringUuidSchema.addProp(GenericData.STRING_PROP, "String");
  Schema nullableStringSchema = record("R", optionalField("uuid", stringUuidSchema));
  GenericRecord s1 = instance(nullableStringSchema, "uuid", u1.get("uuid").toString());
  GenericRecord s2 = instance(nullableStringSchema, "uuid", null);

  File test = write(GENERIC, nullableUuidSchema, u1, u2);
  Assert.assertEquals("Should read UUIDs as Strings",
      Arrays.asList(s1, s2), read(GENERIC, nullableStringSchema, test));
}
 
Example #12
Source File: GenerateAvroJavaTask.java    From gradle-avro-plugin with Apache License 2.0 6 votes vote down vote up
/**
 * Registers the logical types to be used in this run.
 * This must be called before the Schemas are parsed, or they will not be applied correctly.
 * Since {@link LogicalTypes} is a static registry, this may result in side-effects.
 */
private void registerLogicalTypes() {
    Map<String, Class<? extends LogicalTypes.LogicalTypeFactory>> logicalTypeFactoryMap = logicalTypeFactories.get();
    Set<Map.Entry<String, Class<? extends LogicalTypes.LogicalTypeFactory>>> logicalTypeFactoryEntries =
        logicalTypeFactoryMap.entrySet();
    for (Map.Entry<String, Class<? extends LogicalTypes.LogicalTypeFactory>> entry : logicalTypeFactoryEntries) {
        String logicalTypeName = entry.getKey();
        Class<? extends LogicalTypes.LogicalTypeFactory> logicalTypeFactoryClass = entry.getValue();
        try {
            LogicalTypes.LogicalTypeFactory logicalTypeFactory = logicalTypeFactoryClass.getDeclaredConstructor().newInstance();
            LogicalTypes.register(logicalTypeName, logicalTypeFactory);
        } catch (ReflectiveOperationException ex) {
            getLogger().error("Could not instantiate logicalTypeFactory class \"" + logicalTypeFactoryClass.getName() + "\"");
        }
    }
}
 
Example #13
Source File: TestGenericLogicalTypes.java    From parquet-mr with Apache License 2.0 6 votes vote down vote up
@Test
public void testReadUUIDWithParquetUUID() throws IOException {
  Schema uuidSchema = record("R",
      field("uuid", LogicalTypes.uuid().addToSchema(Schema.create(STRING))));
  GenericRecord u1 = instance(uuidSchema, "uuid", UUID.randomUUID());
  GenericRecord u2 = instance(uuidSchema, "uuid", UUID.randomUUID());
  File test = write(conf(AvroWriteSupport.WRITE_PARQUET_UUID, true), uuidSchema, u1, u2);

  Assert.assertEquals("Should read UUID objects",
      Arrays.asList(u1, u2), read(GENERIC, uuidSchema, test));

  GenericRecord s1 = instance(uuidSchema, "uuid", u1.get("uuid").toString());
  GenericRecord s2 = instance(uuidSchema, "uuid", u2.get("uuid").toString());

  Assert.assertEquals("Should read UUID as Strings",
      Arrays.asList(s1, s2), read(GenericData.get(), uuidSchema, test));

}
 
Example #14
Source File: TestReflectLogicalTypes.java    From parquet-mr with Apache License 2.0 6 votes vote down vote up
@Test
public void testReadUUIDList() throws IOException {
  Schema uuidListSchema = SchemaBuilder.record(RecordWithUUIDList.class.getName())
      .fields()
      .name("uuids").type().array().items().stringType().noDefault()
      .endRecord();
  uuidListSchema.getField("uuids").schema().addProp(
      SpecificData.CLASS_PROP, List.class.getName());
  LogicalTypes.uuid().addToSchema(
      uuidListSchema.getField("uuids").schema().getElementType());

  UUID u1 = UUID.randomUUID();
  UUID u2 = UUID.randomUUID();

  GenericRecord r = new GenericData.Record(uuidListSchema);
  r.put("uuids", Arrays.asList(u1.toString(), u2.toString()));

  RecordWithUUIDList expected = new RecordWithUUIDList();
  expected.uuids = Arrays.asList(u1, u2);

  File test = write(uuidListSchema, r);

  Assert.assertEquals("Should convert Strings to UUIDs",
      expected, read(REFLECT, uuidListSchema, test).get(0));
}
 
Example #15
Source File: TestAvroUtils.java    From envelope with Apache License 2.0 5 votes vote down vote up
@Test
public void toTypeSchemaDateNotNullable() throws Exception {
  Schema schema = AvroUtils.typeFor(DataTypes.DateType, false);

  assertEquals("Invalid type", Schema.Type.INT, schema.getType());
  assertEquals("Invalid LogicalType", LogicalTypes.date(), schema.getLogicalType());

  //System.out.println(schema.toString(true));
}
 
Example #16
Source File: TestReflectLogicalTypes.java    From parquet-mr with Apache License 2.0 5 votes vote down vote up
@Test
public void testReadUUIDGenericRecordWithParquetUUID() throws IOException {
  Schema uuidSchema = SchemaBuilder.record("RecordWithUUID")
      .fields().requiredString("uuid").endRecord();
  LogicalTypes.uuid().addToSchema(uuidSchema.getField("uuid").schema());

  UUID u1 = UUID.randomUUID();
  UUID u2 = UUID.randomUUID();

  RecordWithStringUUID r1 = new RecordWithStringUUID();
  r1.uuid = u1.toString();
  RecordWithStringUUID r2 = new RecordWithStringUUID();
  r2.uuid = u2.toString();

  List<GenericData.Record> expected = Arrays.asList(
      new GenericData.Record(uuidSchema), new GenericData.Record(uuidSchema));
  expected.get(0).put("uuid", u1);
  expected.get(1).put("uuid", u2);

  File test = write(
      AvroTestUtil.conf(AvroWriteSupport.WRITE_PARQUET_UUID, true),
      uuidSchema, r1, r2);

  Assert.assertEquals("Should convert Strings to UUIDs",
      expected, read(REFLECT, uuidSchema, test));

  Schema uuidStringSchema = SchemaBuilder
      .record(RecordWithStringUUID.class.getName())
      .fields().requiredString("uuid").endRecord();
  LogicalTypes.uuid().addToSchema(uuidStringSchema.getField("uuid").schema());

  Assert.assertEquals("Should not convert to UUID if accessor is String",
      Arrays.asList(r1, r2),
      read(REFLECT, uuidStringSchema, test));
}
 
Example #17
Source File: AvroSchemaTest.java    From component-runtime with Apache License 2.0 5 votes vote down vote up
@Test
void checkDateConversionFromExternalAvro() {
    final org.apache.avro.Schema avro = SchemaBuilder
            .record("test")
            .fields()
            .name("date")
            .type(LogicalTypes.timestampMillis().addToSchema(Schema.create(Schema.Type.LONG)))
            .noDefault()
            .endRecord();
    assertEquals(DATETIME, new AvroSchema(avro).getEntries().iterator().next().getType());
}
 
Example #18
Source File: TestReflectLogicalTypes.java    From parquet-mr with Apache License 2.0 5 votes vote down vote up
@Test
public void testWriteUUID() throws IOException {
  Schema uuidSchema = SchemaBuilder.record(RecordWithUUID.class.getName())
      .fields().requiredString("uuid").endRecord();
  LogicalTypes.uuid().addToSchema(uuidSchema.getField("uuid").schema());

  UUID u1 = UUID.randomUUID();
  UUID u2 = UUID.randomUUID();

  RecordWithUUID r1 = new RecordWithUUID();
  r1.uuid = u1;
  RecordWithUUID r2 = new RecordWithUUID();
  r2.uuid = u2;

  List<RecordWithStringUUID> expected = Arrays.asList(
      new RecordWithStringUUID(), new RecordWithStringUUID());
  expected.get(0).uuid = u1.toString();
  expected.get(1).uuid = u2.toString();

  File test = write(REFLECT, uuidSchema, r1, r2);

  // verify that the field's type overrides the logical type
  Schema uuidStringSchema = SchemaBuilder
      .record(RecordWithStringUUID.class.getName())
      .fields().requiredString("uuid").endRecord();

  Assert.assertEquals("Should read uuid as String without UUID conversion",
      expected,
      read(REFLECT, uuidStringSchema, test));

  LogicalTypes.uuid().addToSchema(uuidStringSchema.getField("uuid").schema());
  Assert.assertEquals("Should read uuid as String without UUID logical type",
      expected,
      read(ReflectData.get(), uuidStringSchema, test));
}
 
Example #19
Source File: TestReflectLogicalTypes.java    From parquet-mr with Apache License 2.0 5 votes vote down vote up
@Test
public void testWriteUUIDList() throws IOException {
  Schema uuidListSchema = SchemaBuilder.record(RecordWithUUIDList.class.getName())
      .fields()
      .name("uuids").type().array().items().stringType().noDefault()
      .endRecord();
  uuidListSchema.getField("uuids").schema().addProp(
      SpecificData.CLASS_PROP, List.class.getName());
  LogicalTypes.uuid().addToSchema(
      uuidListSchema.getField("uuids").schema().getElementType());

  Schema stringArraySchema = SchemaBuilder.record("RecordWithUUIDArray")
      .fields()
      .name("uuids").type().array().items().stringType().noDefault()
      .endRecord();
  stringArraySchema.getField("uuids").schema()
      .addProp(SpecificData.CLASS_PROP, List.class.getName());

  UUID u1 = UUID.randomUUID();
  UUID u2 = UUID.randomUUID();

  GenericRecord expected = new GenericData.Record(stringArraySchema);
  expected.put("uuids", Arrays.asList(u1.toString(), u2.toString()));

  RecordWithUUIDList r = new RecordWithUUIDList();
  r.uuids = Arrays.asList(u1, u2);

  File test = write(REFLECT, uuidListSchema, r);

  Assert.assertEquals("Should read UUIDs as Strings",
      expected,
      read(REFLECT, stringArraySchema, test).get(0));
}
 
Example #20
Source File: PulsarMetadata.java    From pulsar with Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
static Type convertType(Schema.Type avroType, LogicalType logicalType) {
    switch (avroType) {
        case BOOLEAN:
            return BooleanType.BOOLEAN;
        case INT:
            if (logicalType == LogicalTypes.timeMillis()) {
                return TIME;
            } else if (logicalType == LogicalTypes.date()) {
                return DATE;
            }
            return IntegerType.INTEGER;
        case LONG:
            if (logicalType == LogicalTypes.timestampMillis()) {
                return TIMESTAMP;
            }
            return BigintType.BIGINT;
        case FLOAT:
            return RealType.REAL;
        case DOUBLE:
            return DoubleType.DOUBLE;
        case BYTES:
            return VarbinaryType.VARBINARY;
        case STRING:
            return VarcharType.VARCHAR;
        case ENUM:
            return VarcharType.VARCHAR;
        default:
            log.error("Cannot convert type: %s", avroType);
            return null;
    }
}
 
Example #21
Source File: TestAvroSchemaConverter.java    From parquet-mr with Apache License 2.0 5 votes vote down vote up
@Test
public void testUUIDTypeWithParquetUUID() throws Exception {
  Schema uuid = LogicalTypes.uuid().addToSchema(Schema.create(STRING));
  Schema expected = Schema.createRecord("myrecord", null, null, false,
      Arrays.asList(new Schema.Field("uuid", uuid, null, null)));

  testRoundTripConversion(AvroTestUtil.conf(AvroWriteSupport.WRITE_PARQUET_UUID, true),
      expected,
      "message myrecord {\n" +
          "  required fixed_len_byte_array(16) uuid (UUID);\n" +
          "}\n");
}
 
Example #22
Source File: TypeConverterUtils.java    From components with Apache License 2.0 5 votes vote down vote up
/**
 * @param srcSchema THe schema to investigate.
 * @return The Date/Time output type that corresponds to the logical type of the schema, or null if none.
 */
public static TypeConverterOutputTypes getDateTimeTypeFromLogicalType(Schema srcSchema) {
    LogicalType srcLogicalType = AvroUtils.unwrapIfNullable(srcSchema).getLogicalType();
    if (srcLogicalType instanceof LogicalTypes.Date) {
        return TypeConverterOutputTypes.Date;
    } else if (srcLogicalType instanceof LogicalTypes.TimeMillis) {
        return TypeConverterOutputTypes.Time;
    } else if (srcLogicalType instanceof LogicalTypes.TimestampMillis) {
        return TypeConverterOutputTypes.DateTime;
    } else {
        return null;
    }
}
 
Example #23
Source File: TestGenericLogicalTypes.java    From parquet-mr with Apache License 2.0 5 votes vote down vote up
@Test
public void testWriteUUIDReadStringMissingLogicalType() throws IOException {
  Schema uuidSchema = record("R",
      field("uuid", LogicalTypes.uuid().addToSchema(Schema.create(STRING))));
  GenericRecord u1 = instance(uuidSchema, "uuid", UUID.randomUUID());
  GenericRecord u2 = instance(uuidSchema, "uuid", UUID.randomUUID());

  GenericRecord s1 = instance(uuidSchema, "uuid", new Utf8(u1.get("uuid").toString()));
  GenericRecord s2 = instance(uuidSchema, "uuid", new Utf8(u2.get("uuid").toString()));

  File test = write(GENERIC, uuidSchema, u1, u2);
  Assert.assertEquals("Should read UUIDs as Strings",
      Arrays.asList(s1, s2), read(GenericData.get(), uuidSchema, test));
}
 
Example #24
Source File: LogicalTypeValidatingVisitor.java    From data-highway with Apache License 2.0 5 votes vote down vote up
@Override
public void onVisit(Schema schema, Collection<String> breadcrumb) {
  try {
    LogicalTypes.fromSchema(schema);
  } catch (Exception e) {
    String path = breadcrumb.stream().collect(joining("/", "/", ""));
    throw new IllegalArgumentException("Invalid logical type declared at " + path, e);
  }
}
 
Example #25
Source File: SnowflakeWriter.java    From components with Apache License 2.0 5 votes vote down vote up
private Object formatIfAnySnowflakeDateType(Object inputValue, Schema s) {
    if (LogicalTypes.fromSchemaIgnoreInvalid(s) == LogicalTypes.timeMillis()) {
        return formatter.formatTimeMillis(inputValue);
    } else if (LogicalTypes.fromSchemaIgnoreInvalid(s) == LogicalTypes.date()) {
        return formatter.formatDate(inputValue);
    } else if (LogicalTypes.fromSchemaIgnoreInvalid(s) == LogicalTypes.timestampMillis()) {
        return formatter.formatTimestampMillis(inputValue);
    } else {
        return inputValue;
    }
}
 
Example #26
Source File: AvroSchema.java    From component-runtime with Apache License 2.0 5 votes vote down vote up
private Type doMapType(final Schema schema) {
    switch (schema.getType()) {
    case LONG:
        if (Boolean.parseBoolean(readProp(schema, Type.DATETIME.name()))
                || LogicalTypes.timestampMillis().equals(LogicalTypes.fromSchemaIgnoreInvalid(schema))) {
            return Type.DATETIME;
        }
        return Type.LONG;
    default:
        return Type.valueOf(schema.getType().name());
    }
}
 
Example #27
Source File: TestSchemaConversions.java    From iceberg with Apache License 2.0 5 votes vote down vote up
@Test
public void testList() {
  Type list = Types.ListType.ofRequired(34, Types.UUIDType.get());
  Schema schema = addElementId(34, SchemaBuilder.array().items(
      LogicalTypes.uuid().addToSchema(Schema.createFixed("uuid_fixed", null, null, 16))));

  Assert.assertEquals("Avro schema to list",
      list, AvroSchemaUtil.convert(schema));
  Assert.assertEquals("List to Avro schema",
      schema, AvroSchemaUtil.convert(list));
}
 
Example #28
Source File: AvroRowSerializationSchema.java    From flink with Apache License 2.0 5 votes vote down vote up
private byte[] convertFromDecimal(Schema schema, BigDecimal decimal) {
	final LogicalType logicalType = schema.getLogicalType();
	if (logicalType instanceof LogicalTypes.Decimal) {
		final LogicalTypes.Decimal decimalType = (LogicalTypes.Decimal) logicalType;
		// rescale to target type
		final BigDecimal rescaled = decimal.setScale(decimalType.getScale(), BigDecimal.ROUND_UNNECESSARY);
		// byte array must contain the two's-complement representation of the
		// unscaled integer value in big-endian byte order
		return decimal.unscaledValue().toByteArray();
	} else {
		throw new RuntimeException("Unsupported decimal type.");
	}
}
 
Example #29
Source File: AvroRowSerializationSchema.java    From flink with Apache License 2.0 5 votes vote down vote up
private int convertFromDate(Schema schema, Date date) {
	final LogicalType logicalType = schema.getLogicalType();
	if (logicalType == LogicalTypes.date()) {
		// adopted from Apache Calcite
		final long time = date.getTime();
		final long converted = time + (long) LOCAL_TZ.getOffset(time);
		return (int) (converted / 86400000L);
	} else {
		throw new RuntimeException("Unsupported date type.");
	}
}
 
Example #30
Source File: DataSourceUtils.java    From hudi with Apache License 2.0 5 votes vote down vote up
/**
 * Given an Avro field schema checks whether the field is of Logical Date Type or not.
 *
 * @param fieldSchema avro field schema
 * @return boolean indicating whether fieldSchema is of Avro's Date Logical Type
 */
private static boolean isLogicalTypeDate(Schema fieldSchema) {
  if (fieldSchema.getType() == Schema.Type.UNION) {
    return fieldSchema.getTypes().stream().anyMatch(schema -> schema.getLogicalType() == LogicalTypes.date());
  }
  return fieldSchema.getLogicalType() == LogicalTypes.date();
}