org.apache.avro.Schema Java Examples

/**
 * For the schema that is a UNION type with NULL and Record type, it provides Records type.
 * @param inputSchema
 * @return
 */
private static Schema getActualRecord(Schema inputSchema) {
  if (Type.RECORD.equals(inputSchema.getType())) {
    return inputSchema;
  }

  Preconditions.checkArgument(Type.UNION.equals(inputSchema.getType()), "Nested schema is only support with either record or union type of null with record");
  Preconditions.checkArgument(inputSchema.getTypes().size() <= 2,
      "For union type in nested record, it should only have NULL and Record type");

  for (Schema inner : inputSchema.getTypes()) {
    if (Type.NULL.equals(inner.getType())) {
      continue;
    }
    Preconditions.checkArgument(Type.RECORD.equals(inner.getType()), "For union type in nested record, it should only have NULL and Record type");
    return inner;

  }
  throw new IllegalArgumentException(inputSchema + " is not supported.");
}
 

public AvroFileInputStream(FileStatus status) throws IOException {
  pos = 0;
  buffer = new byte[0];
  GenericDatumReader<Object> reader = new GenericDatumReader<Object>();
  FileContext fc = FileContext.getFileContext(new Configuration());
  fileReader =
    DataFileReader.openReader(new AvroFSInput(fc, status.getPath()),reader);
  Schema schema = fileReader.getSchema();
  writer = new GenericDatumWriter<Object>(schema);
  output = new ByteArrayOutputStream();
  JsonGenerator generator =
    new JsonFactory().createJsonGenerator(output, JsonEncoding.UTF8);
  MinimalPrettyPrinter prettyPrinter = new MinimalPrettyPrinter();
  prettyPrinter.setRootValueSeparator(System.getProperty("line.separator"));
  generator.setPrettyPrinter(prettyPrinter);
  encoder = EncoderFactory.get().jsonEncoder(schema, generator);
}
 

private void validateSchemaIsSupported(Schema schema) {
  for (Field field : schema.getFields()) {
    Type type = field.schema().getType();

    if (type.equals(Type.UNION)) {
      List<Schema> types = field.schema().getTypes();

      if (types.size() != 2) {
        throw new RuntimeException("Union type in Avro serializer schema must only contain two types");
      }

      if (types.get(0).getType().equals(Type.NULL)) {
        type = types.get(1).getType();
      }
      else {
        type = types.get(0).getType();
      }
    }

    if (!supportedTypes.contains(type)) {
      throw new RuntimeException("Avro serializer for Kafka output does not support Avro schema type: " + type);
    }
  }
}
 

@Test(expected = Exception.class)
public void demoteDoubleToLong() throws Exception {
  Schema schema = getSchemaFieldAssembler().requiredDouble(EVOLUTION_COLUMN)
      .endRecord();
  Schema evolvedSchema = getSchemaFieldAssembler().requiredLong(EVOLUTION_COLUMN)
      .endRecord();
  FieldDataWrapper beforeEvolution = new FieldDataWrapper(EVOLUTION_COLUMN, 1d);
  FieldDataWrapper afterEvolution = new FieldDataWrapper(EVOLUTION_COLUMN, 2l);
  List<String> expectedData = Lists.newArrayList(
      "1\t1.0\t1",
      "2\t2.0\t2"
  );

  runTest(schema, evolvedSchema, beforeEvolution, afterEvolution);
  runDataChecks(evolvedSchema, expectedData);
}
 

@Test
public void testBasic() throws Exception {

    NetSuiteRuntime netSuiteRuntime = new TestNetSuiteRuntimeImpl(webServiceMockTestFixture.getClientFactory());
    NetSuiteDatasetRuntime dataSetRuntime = netSuiteRuntime.getDatasetRuntime(mockTestFixture.getConnectionProperties());

    mockGetRequestResults(null);

    TypeDesc typeDesc = clientService.getMetaDataSource().getTypeInfo("Opportunity");

    Schema schema = dataSetRuntime.getSchema(typeDesc.getTypeName());

    NsObjectOutputTransducer transducer = new NsObjectOutputTransducer(
            webServiceMockTestFixture.getClientService(), typeDesc.getTypeName());

    List<IndexedRecord> indexedRecordList = makeIndexedRecords(clientService, schema,
            new AbstractNetSuiteTestBase.SimpleObjectComposer<>(Opportunity.class), 10);

    for (IndexedRecord indexedRecord : indexedRecordList) {
        Opportunity record = (Opportunity) transducer.write(indexedRecord);
        assertNsObject(typeDesc, record);
    }
}
 

@Test
public void testConvertIntToString() {
    // No format
    String conv = testConvertValue(Schema.create(Schema.Type.INT), 123,
            TypeConverterProperties.TypeConverterOutputTypes.String, null, String.class);
    assertThat(conv, is("123"));

    // One format
    conv = testConvertValue(Schema.create(Schema.Type.INT), 123, TypeConverterProperties.TypeConverterOutputTypes.String,
            "'#'#", String.class);
    assertThat(conv, is("#123"));

    // Another format
    conv = testConvertValue(Schema.create(Schema.Type.INT), 123456, TypeConverterProperties.TypeConverterOutputTypes.String,
            "#,###.00", String.class);
    assertThat(conv, is("123,456.00"));
}
 

/**
 * Convert an Avro schema to a Pig schema
 */
public static ResourceSchema convert(Schema schema) throws IOException {

    if (AvroStorageUtils.containsGenericUnion(schema))
        throw new IOException ("We don't accept schema containing generic unions.");

    Set<Schema> visitedRecords = new HashSet<Schema>();
    ResourceFieldSchema inSchema = inconvert(schema, FIELD, visitedRecords);

    ResourceSchema tupleSchema;
    if (inSchema.getType() == DataType.TUPLE) {
        tupleSchema = inSchema.getSchema();
    } else { // other typs
        ResourceFieldSchema tupleWrapper = AvroStorageUtils.wrapAsTuple(inSchema);

        ResourceSchema topSchema = new ResourceSchema();
        topSchema.setFields(new ResourceFieldSchema[] { tupleWrapper });

        tupleSchema = topSchema;

    }
    return tupleSchema;
}
 

@Test
public void testReadUUIDWithParquetUUID() throws IOException {
  Schema uuidSchema = record("R",
      field("uuid", LogicalTypes.uuid().addToSchema(Schema.create(STRING))));
  GenericRecord u1 = instance(uuidSchema, "uuid", UUID.randomUUID());
  GenericRecord u2 = instance(uuidSchema, "uuid", UUID.randomUUID());
  File test = write(conf(AvroWriteSupport.WRITE_PARQUET_UUID, true), uuidSchema, u1, u2);

  Assert.assertEquals("Should read UUID objects",
      Arrays.asList(u1, u2), read(GENERIC, uuidSchema, test));

  GenericRecord s1 = instance(uuidSchema, "uuid", u1.get("uuid").toString());
  GenericRecord s2 = instance(uuidSchema, "uuid", u2.get("uuid").toString());

  Assert.assertEquals("Should read UUID as Strings",
      Arrays.asList(s1, s2), read(GenericData.get(), uuidSchema, test));

}
 

public static Schema getListOperationRejectRESTSchema() {
    return SchemaBuilder.builder().record("REST").fields() //
            .name(FIELD_LIST_ID)//
            .prop(SchemaConstants.TALEND_IS_LOCKED, "true")//
            .type().nullable().intType().noDefault() //
            .name(FIELD_LEAD_ID)//
            .prop(SchemaConstants.TALEND_IS_LOCKED, "true")//
            .type().nullable().intType().noDefault() //
            .name(FIELD_STATUS)//
            .prop(SchemaConstants.TALEND_IS_LOCKED, "true")//
            .type().nullable().stringType().noDefault() //
            .name(FIELD_ERROR_MSG)//
            .prop(SchemaConstants.TALEND_IS_LOCKED, "true")//
            .type().nullable().stringType().noDefault() //
            .endRecord();
}
 

private AvroExternalTable(AvroExternalTable.Builder builder) throws IOException {
  super(builder);

  if (builder.moveDataToTmpHdfsDir) {
    this.dataLocationInHdfs = moveDataFileToSeparateHdfsDir(builder.dataLocationInHdfs, builder.extensionToBeMoved);
    this.deleteDataAfterDone = true;
  } else {
    this.dataLocationInHdfs = builder.dataLocationInHdfs;
    this.deleteDataAfterDone = false;
  }

  if (StringUtils.isNotBlank(builder.schemaLocationInHdfs)) {
    this.schemaLocationInHdfs = builder.schemaLocationInHdfs;
    this.attributes = getAttributesFromAvroSchemaFile();
    this.deleteSchemaAfterDone = false;
  } else {
    Schema schema = getSchemaFromAvroDataFile();
    this.attributes = parseSchema(schema);
    this.schemaLocationInHdfs = writeSchemaToHdfs(schema);
    this.deleteSchemaAfterDone = true;
  }
}
 

@Test
void createAndUpdateSchema() throws Exception {
    String artifactId = TestUtils.generateArtifactId();

    Schema schema = new Schema.Parser().parse("{\"type\":\"record\",\"name\":\"myrecord1\",\"fields\":[{\"name\":\"foo1\",\"type\":\"string\"}]}");
    createArtifactViaConfluentClient(schema, artifactId);

    Schema updatedSchema = new Schema.Parser().parse("{\"type\":\"record\",\"name\":\"myrecord2\",\"fields\":[{\"name\":\"foo2\",\"type\":\"long\"}]}");
    createArtifactViaConfluentClient(updatedSchema, artifactId);

    assertThrows(SchemaParseException.class, () -> new Schema.Parser().parse("<type>record</type>\n<name>test</name>"));
    assertThat(confluentService.getAllVersions(artifactId), hasItems(1, 2));

    confluentService.deleteSubject(artifactId);
    waitForSubjectDeleted(artifactId);
}
 

@Test
public void testWriteAndReadUsingReflectDataSchemaWithDataModel() {
  Schema testRecordSchema = ReflectData.get().getSchema(TestRecord.class);

  List<GenericRecord> records = generateGenericRecords(1000);
  mainPipeline
      .apply(Create.of(records).withCoder(AvroCoder.of(testRecordSchema)))
      .apply(
          FileIO.<GenericRecord>write()
              .via(ParquetIO.sink(testRecordSchema))
              .to(temporaryFolder.getRoot().getAbsolutePath()));
  mainPipeline.run().waitUntilFinish();

  PCollection<GenericRecord> readBack =
      readPipeline.apply(
          ParquetIO.read(testRecordSchema)
              .withAvroDataModel(GenericData.get())
              .from(temporaryFolder.getRoot().getAbsolutePath() + "/*"));

  PAssert.that(readBack).containsInAnyOrder(records);
  readPipeline.run().waitUntilFinish();
}
 

@Test
public void testDecorateRecordWithPrimitiveField() {
  Schema inputRecordSchema = SchemaBuilder.record("test").fields()
          .name("integer1")
          .prop("innerProp", "innerVal")
          .type().intBuilder().endInt().noDefault()
          .requiredString("string1")
          .endRecord();

  GenericRecord inputRecord = new GenericData.Record(inputRecordSchema);
  inputRecord.put("integer1", 10);
  inputRecord.put("string1", "hello");

  Schema outputRecordSchema = AvroUtils.decorateRecordSchema(inputRecordSchema, Collections.singletonList(new Schema.Field("newField", SchemaBuilder.builder().intType(), "test field", null)));
  Map<String, Object> newFields = new HashMap<>();
  newFields.put("newField", 5);

  GenericRecord outputRecord = AvroUtils.decorateRecord(inputRecord, newFields, outputRecordSchema);
  Assert.assertEquals(outputRecord.get("newField"), 5);
  Assert.assertEquals(outputRecord.get("integer1"), 10);
  Assert.assertEquals(outputRecord.get("string1"), "hello");

}
 

private Type doMapType(final Schema schema) {
    switch (schema.getType()) {
    case LONG:
        if (Boolean.parseBoolean(readProp(schema, Type.DATETIME.name()))
                || LogicalTypes.timestampMillis().equals(LogicalTypes.fromSchemaIgnoreInvalid(schema))) {
            return Type.DATETIME;
        }
        return Type.LONG;
    default:
        return Type.valueOf(schema.getType().name());
    }
}
 

@Test
public void testGetSchemaModuleSelection() throws IOException {
    datasetProperties.sourceType.setValue(SalesforceDatasetProperties.SourceType.MODULE_SELECTION);
    datasetProperties.moduleName.setValue("Account");
    datasetProperties.selectColumnIds.setValue(Arrays.asList(
            "Id", "Name", "AccountNumber", "BillingCity"));

    doReturn(SCHEMA_1).when(dataprepSource).guessSchema(anyString());

    Schema schema = SalesforceSchemaUtils.getSchema(datasetProperties, dataprepSource, null);
    assertNotNull(schema);
    assertEquals(SCHEMA_1, schema);
}
 

@OnEnabled
public void onEnabled(final ConfigurationContext context) {
    final int cacheSize = context.getProperty(CACHE_SIZE).asInteger();
    compiledAvroSchemaCache = Caffeine.newBuilder()
            .maximumSize(cacheSize)
            .build(schemaText -> new Schema.Parser().parse(schemaText));

    final int capacity = context.getProperty(ENCODER_POOL_SIZE).evaluateAttributeExpressions().asInteger();
    encoderPool = new LinkedBlockingQueue<>(capacity);
}
 

@Test
public void testSchemaToTypeInfoConversion() throws Exception {
  // Simple non-nested case:
  Schema avroSchema = SchemaBuilder.record("test")
      .fields()
      .name("id")
      .type()
      .intType()
      .noDefault()
      .name("timestamp")
      .type()
      .stringType()
      .noDefault()
      .endRecord();

  TypeInfo orcSchema = OrcTestTools.convertAvroSchemaToOrcSchema(avroSchema);
  String targetOrcSchemaString = "struct<id:int,timestamp:string>";
  Assert.assertEquals(targetOrcSchemaString, orcSchema.toString());

  // Nested case:
  avroSchema = SchemaBuilder.record("nested")
      .fields()
      .name("nestedId")
      .type()
      .array()
      .items()
      .stringType()
      .noDefault()
      .name("timestamp")
      .type()
      .stringType()
      .noDefault()
      .endRecord();
  orcSchema = OrcTestTools.convertAvroSchemaToOrcSchema(avroSchema);
  TypeDescription targetTypeDescription = TypeDescription.createStruct()
      .addField("nestedId", TypeDescription.createList(TypeDescription.createString()))
      .addField("timestamp", TypeDescription.createString());
  Assert.assertEquals(orcSchema.toString().toLowerCase(), targetTypeDescription.toString().toLowerCase());
}
 

@Test
public void uniqueFieldNames() {
    String xsd =
            "<xs:schema xmlns:xs='http://www.w3.org/2001/XMLSchema'>" +
            "  <xs:complexType name='type'>" +
            "    <xs:sequence>" +
            "      <xs:element name='field' type='xs:string'/>" +
            "    </xs:sequence>" +
            "    <xs:attribute name='field' type='xs:string'/>" +
            "  </xs:complexType>" +
            "  <xs:element name='root' type='type'/>" +
            "</xs:schema>";

    Schema schema = Converter.createSchema(xsd);

    assertEquals(2, schema.getFields().size());
    Schema.Field field = schema.getField("field");
    assertNotNull(field);
    assertEquals("" + new Source("field", true), field.getProp(Source.SOURCE));

    Schema.Field field0 = schema.getField("field0");
    assertEquals("" + new Source("field", false), field0.getProp(Source.SOURCE));

    String xml = "<root field='value'><field>value0</field></root>";
    GenericData.Record record = Converter.createDatum(schema, xml);

    assertEquals("value", record.get("field"));
    assertEquals("value0", record.get("field0"));
}
 

private void map(IndexedRecord input, ProcessContext context) throws IOException {
    PyObject output = pyFn.__call__(new PyUnicode(input.toString()));

    if (jsonGenericRecordConverter == null) {
        JsonSchemaInferrer jsonSchemaInferrer = new JsonSchemaInferrer(new ObjectMapper());
        Schema jsonSchema = jsonSchemaInferrer.inferSchema(output.toString());
        jsonGenericRecordConverter = new JsonGenericRecordConverter(jsonSchema);
    }

    GenericRecord outputRecord = jsonGenericRecordConverter.convertToAvro(output.toString());
    context.output(outputRecord);
}
 

@Override
public Schema getSchema() {
    // Simple schema container.
    final Schema[] s = new Schema[1];
    // Try to get one record and determine its schema in a callback.
    getSample(1, new Consumer<IndexedRecord>() {

        @Override
        public void accept(IndexedRecord in) {
            s[0] = in.getSchema();
        }
    });
    // Return the discovered schema.
    return s[0];
}
 

private void setFieldFromNode(Schema schema, GenericData.Record record, Node node) {
    if (node.getNodeType() != Node.ELEMENT_NODE)
        return;

    Element child = (Element) node;
    boolean setRecordFromNode = false;
    final String fieldName = child.getLocalName();
    Schema.Field field = getFieldBySource(schema, new Source(fieldName, false));
    if(field == null) {
      field = getNestedFieldBySource(schema, new Source(fieldName, false));
      setRecordFromNode = true;
    }

    if (field != null) {
        boolean array = field.schema().getType() == Schema.Type.ARRAY;
      Object datum = createNodeDatum(!array ? field.schema() : field.schema().getElementType(), child, setRecordFromNode);

        if (!array)
            record.put(field.name(), datum);
        else {
            @SuppressWarnings("unchecked") List<Object> values = (List<Object>) record.get(field.name());
            values.add(datum);
        }
    } else {
        Schema.Field anyField = schema.getField(Source.WILDCARD);
        if (anyField == null)
            throw new ConverterException("Could not find field " + fieldName + " in Avro Schema " + schema.getName() +  " , neither as specific field nor 'any' element");

        @SuppressWarnings("unchecked") Map<String, String> map = (HashMap<String, String>) record.get(Source.WILDCARD);
        map.put(fieldName, getContentAsText(child));
    }
}
 

@Override
public void open(InitializationContext context) throws Exception {
	final Schema schema = AvroSchemaConverter.convertToSchema(rowType);
	this.record = new GenericData.Record(schema);
	this.datumReader = new SpecificDatumReader<>(schema);
	this.inputStream = new MutableByteArrayInputStream();
	this.decoder = DecoderFactory.get().binaryDecoder(this.inputStream, null);
}
 

/**
     * 解析被generic schema封装的实际数据
     *
     * @param schema  schema对象
     * @param payload 实际数据
     * @return List<GenericRecord>
     * @throws Exception
     */
    public List<GenericRecord> decode(Db2GenericMessage msg) throws IOException {

        ByteBuffer buffer = ByteBuffer.wrap(msg.getPayload());

        if (buffer.get() != Constants.MAGIC_BYTE) {
            logger.error("Unknown magic byte!");
        }

        int id = buffer.getInt();

        Schema schema = null;
        try {
            schema = ThreadLocalCache.get(Constants.CacheNames.TABLE_SCHEMA_VERSION_CACHE, Utils.buildDataTableSchemaCacheKey(msg.getSchemaName(), msg.getTableName(), String.valueOf(id)));
            if (schema == null) {
                schema = schemaRegistry.getById(id);
                ThreadLocalCache.put(Constants.CacheNames.TABLE_SCHEMA_VERSION_CACHE, Utils.buildDataTableSchemaCacheKey(msg.getSchemaName(), msg.getTableName(), String.valueOf(id)), schema);
            }

        } catch (RestClientException e) {
            logger.error("Schema Registry RestClientException: " + e);
        }

        int length = buffer.limit() - 5;
        int start = buffer.position() + buffer.arrayOffset();

//        logger.debug("Schema:" + schema.toString() + " schema payload:" + new String(msg.getPayload(), "utf-8"));
        List<GenericRecord> list = new LinkedList<>();
        DatumReader<GenericRecord> reader = new GenericDatumReader<>(schema);
        BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(buffer.array(), start, length, null);
        GenericRecord genericRecord = reader.read(null, decoder);
        list.add((GenericRecord) genericRecord);

        return list;
    }
 

@Test
public void testGetCustomObjects() throws Exception {
    iprops.customObjectAction.setValue(CustomObjectAction.get);
    doThrow(new MarketoException("REST", "error")).when(client).executeGetRequest(any(Schema.class));
    mktoRR = client.getCustomObjects(iprops, null);
    assertFalse(mktoRR.isSuccess());
    assertFalse(mktoRR.getErrorsString().isEmpty());
    //
    doReturn(new MarketoRecordResult()).when(client).executeGetRequest(any(Schema.class));
    mktoRR = client.getCustomObjects(iprops, null);
    assertFalse(mktoRR.isSuccess());
    assertTrue(mktoRR.getErrorsString().isEmpty());
    //
    MarketoRecordResult mrr = new MarketoRecordResult();
    mrr.setSuccess(true);
    mrr.setRemainCount(0);
    mrr.setRecordCount(1);
    List<IndexedRecord> records = new ArrayList<>();
    IndexedRecord record = new Record(MarketoConstants.getCustomObjectRecordSchema());
    record.put(0, "mkto-123456");
    record.put(1, 0);
    record.put(2, new Date());
    record.put(3, new Date());
    records.add(record);
    mrr.setRecords(records);
    doReturn(mrr).when(client).executeGetRequest(any(Schema.class));
    mktoRR = client.getCustomObjects(iprops, null);
    assertTrue(mktoRR.isSuccess());
    assertTrue(mktoRR.getErrorsString().isEmpty());
}
 

@Test
public void testGetLeadActivity() throws Exception {
    doReturn(getLeadActivityResult()).when(port).getLeadActivity(any(ParamsGetLeadActivity.class),
            any(AuthenticationHeader.class));
    iprops.inputOperation.setValue(InputOperation.getLeadActivity);
    iprops.afterInputOperation();
    Field attr = new Field("attrName", AvroUtils._string(), "", null);
    iprops.schemaInput.schema
            .setValue(MarketoUtils.newSchema(iprops.schemaInput.schema.getValue(), "test", Collections.singletonList(attr)));
    iprops.beforeMappingInput();
    iprops.leadKeyTypeSOAP.setValue(LeadKeyTypeSOAP.IDNUM);
    mktoRR = client.getLeadActivity(iprops, null);
    assertNotNull(mktoRR);
    assertTrue(mktoRR.isSuccess());
    List<IndexedRecord> records = mktoRR.getRecords();
    assertNotNull(records);
    IndexedRecord record = records.get(0);
    assertNotNull(record);
    Schema refSchema = iprops.schemaInput.schema.getValue();
    assertEquals(refSchema, record.getSchema());
    assertEquals("ABC-123-DEF", record.get(refSchema.getField("marketoGUID").pos()));
    assertEquals(123456L, record.get(refSchema.getField("Id").pos()));
    assertEquals("mktgAssetName", record.get(refSchema.getField("MktgAssetName").pos()));
    assertTrue(record.get(refSchema.getField("ActivityDateTime").pos()) instanceof Long);
    assertEquals("activityType", record.get(refSchema.getField("ActivityType").pos()));
    assertEquals("mktgAssetName", record.get(refSchema.getField("MktgAssetName").pos()));
    assertEquals("mktPersonId", record.get(refSchema.getField("MktPersonId").pos()));
    assertEquals("campaign", record.get(refSchema.getField("Campaign").pos()));
    assertEquals("foreignSysId", record.get(refSchema.getField("ForeignSysId").pos()));
    assertEquals("personName", record.get(refSchema.getField("PersonName").pos()));
    assertEquals("orgName", record.get(refSchema.getField("OrgName").pos()));
    assertEquals("foreignSysOrgId", record.get(refSchema.getField("ForeignSysOrgId").pos()));
    assertEquals("attrValue", record.get(refSchema.getField("attrName").pos()));
    //
    doThrow(new RuntimeException("error")).when(port).getLeadActivity(any(ParamsGetLeadActivity.class),
            any(AuthenticationHeader.class));
    mktoRR = client.getLeadActivity(iprops, null);
    assertNotNull(mktoRR);
    assertFalse(mktoRR.isSuccess());
}
 

@Override
protected Schema fetchSchemaByKey(Integer key) throws SchemaRegistryException {
  try {
    return this.schemaRegistryClient.getByID(key);
  } catch (IOException | RestClientException e) {
    throw new SchemaRegistryException(e);
  }
}
 

/**
 * @param pentahoType
 * @param avroData
 * @param fieldSchema
 * @return
 */
public Object convertToKettleValue( AvroInputField pentahoType, ByteBuffer avroData, Schema fieldSchema ) {
  Object pentahoData = null;
  if ( avroData != null ) {
    try {
      switch ( pentahoType.getPentahoType() ) {
        case ValueMetaInterface.TYPE_BIGNUMBER:
          Conversions.DecimalConversion converter = new Conversions.DecimalConversion();
          Schema schema = fieldSchema;
          if ( schema.getType().equals( Schema.Type.UNION ) ) {
            List<Schema> schemas = schema.getTypes();
            for ( Schema s : schemas ) {
              if ( !s.getName().equalsIgnoreCase( "null" ) ) {
                schema = s;
                break;
              }
            }
          }
          Object precision = schema.getObjectProp( AvroSpec.DECIMAL_PRECISION );
          Object scale = schema.getObjectProp( AvroSpec.DECIMAL_SCALE );
          LogicalTypes.Decimal decimalType =
            LogicalTypes.decimal( Integer.parseInt( precision.toString() ), Integer.parseInt( scale.toString() ) );
          pentahoData = converter.fromBytes( avroData, m_schemaToUse, decimalType );
          break;
        case ValueMetaInterface.TYPE_BINARY:
          pentahoData = new byte[ avroData.remaining() ];
          avroData.get( (byte[]) pentahoData );
          break;
      }
    } catch ( Exception e ) {
      // If unable to do the type conversion just ignore. null will be returned.
    }
  }
  return pentahoData;
}
 

@Test
public void testResolveTypeObjectToGeneric() {
  Class<Object> type = Object.class;
  Schema schema = SchemaBuilder.record("User").fields()
      .requiredString("name")
      .requiredString("color")
      .endRecord();
  Class expResult = GenericData.Record.class;
  Class result = DataModelUtil.resolveType(type, schema);
  assertEquals(expResult, result);
}
 

@Test
public void testConfluentAvroDeserializerForSchemaEvolution() throws IOException, RestClientException, SchemaRegistryException {
  WorkUnitState mockWorkUnitState = getMockWorkUnitState(0L, 10L);
  mockWorkUnitState.setProp("schema.registry.url", TEST_URL);

  Schema schemaV1 = SchemaBuilder.record(TEST_RECORD_NAME)
      .namespace(TEST_NAMESPACE).fields()
      .name(TEST_FIELD_NAME).type().stringType().noDefault()
      .endRecord();

  Schema schemaV2 = SchemaBuilder.record(TEST_RECORD_NAME)
      .namespace(TEST_NAMESPACE).fields()
      .name(TEST_FIELD_NAME).type().stringType().noDefault()
      .optionalString(TEST_FIELD_NAME2).endRecord();

  GenericRecord testGenericRecord = new GenericRecordBuilder(schemaV1).set(TEST_FIELD_NAME, "testValue").build();

  SchemaRegistryClient mockSchemaRegistryClient = mock(SchemaRegistryClient.class);
  when(mockSchemaRegistryClient.getByID(any(Integer.class))).thenReturn(schemaV1);

  Serializer<Object> kafkaEncoder = new KafkaAvroSerializer(mockSchemaRegistryClient);
  Deserializer<Object> kafkaDecoder = new KafkaAvroDeserializer(mockSchemaRegistryClient);

  ByteBuffer testGenericRecordByteBuffer =
      ByteBuffer.wrap(kafkaEncoder.serialize(TEST_TOPIC_NAME, testGenericRecord));

  KafkaSchemaRegistry<Integer, Schema> mockKafkaSchemaRegistry = mock(KafkaSchemaRegistry.class);
  when(mockKafkaSchemaRegistry.getLatestSchemaByTopic(TEST_TOPIC_NAME)).thenReturn(schemaV2);

  KafkaDeserializerExtractor kafkaDecoderExtractor = new KafkaDeserializerExtractor(mockWorkUnitState,
      Optional.fromNullable(Deserializers.CONFLUENT_AVRO), kafkaDecoder, mockKafkaSchemaRegistry);

  when(kafkaDecoderExtractor.getSchema()).thenReturn(schemaV2);

  ByteArrayBasedKafkaRecord mockMessageAndOffset = getMockMessageAndOffset(testGenericRecordByteBuffer);

  GenericRecord received = (GenericRecord) kafkaDecoderExtractor.decodeRecord(mockMessageAndOffset);
  Assert.assertEquals(received.toString(), "{\"testField\": \"testValue\", \"testField2\": null}");

}
 

private void assertOutputAvroRecords(List<Map<String, Object>> expectedRecords, MockFlowFile resultFlowFile) throws IOException {
    assertEquals(String.valueOf(expectedRecords.size()), resultFlowFile.getAttribute(PutHiveStreaming.HIVE_STREAMING_RECORD_COUNT_ATTR));

    final DataFileStream<GenericRecord> reader = new DataFileStream<>(
            new ByteArrayInputStream(resultFlowFile.toByteArray()),
            new GenericDatumReader<GenericRecord>());

    Schema schema = reader.getSchema();

    // Verify that the schema is preserved
    assertTrue(schema.equals(new Schema.Parser().parse(new File("src/test/resources/user.avsc"))));

    GenericRecord record = null;
    for (Map<String, Object> expectedRecord : expectedRecords) {
        assertTrue(reader.hasNext());
        record = reader.next(record);
        final String name = record.get("name").toString();
        final Integer favorite_number = (Integer) record.get("favorite_number");
        assertNotNull(name);
        assertNotNull(favorite_number);
        assertNull(record.get("favorite_color"));
        assertNull(record.get("scale"));

        assertEquals(expectedRecord.get("name"), name);
        assertEquals(expectedRecord.get("favorite_number"), favorite_number);
    }
    assertFalse(reader.hasNext());
}