Java Code Examples for org.apache.avro.Schema

The following examples show how to use org.apache.avro.Schema. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: components   Source File: MarketoConstants.java    License: Apache License 2.0 6 votes vote down vote up
public static Schema getListOperationRejectRESTSchema() {
    return SchemaBuilder.builder().record("REST").fields() //
            .name(FIELD_LIST_ID)//
            .prop(SchemaConstants.TALEND_IS_LOCKED, "true")//
            .type().nullable().intType().noDefault() //
            .name(FIELD_LEAD_ID)//
            .prop(SchemaConstants.TALEND_IS_LOCKED, "true")//
            .type().nullable().intType().noDefault() //
            .name(FIELD_STATUS)//
            .prop(SchemaConstants.TALEND_IS_LOCKED, "true")//
            .type().nullable().stringType().noDefault() //
            .name(FIELD_ERROR_MSG)//
            .prop(SchemaConstants.TALEND_IS_LOCKED, "true")//
            .type().nullable().stringType().noDefault() //
            .endRecord();
}
 
Example 2
@Test(expected = Exception.class)
public void demoteDoubleToLong() throws Exception {
  Schema schema = getSchemaFieldAssembler().requiredDouble(EVOLUTION_COLUMN)
      .endRecord();
  Schema evolvedSchema = getSchemaFieldAssembler().requiredLong(EVOLUTION_COLUMN)
      .endRecord();
  FieldDataWrapper beforeEvolution = new FieldDataWrapper(EVOLUTION_COLUMN, 1d);
  FieldDataWrapper afterEvolution = new FieldDataWrapper(EVOLUTION_COLUMN, 2l);
  List<String> expectedData = Lists.newArrayList(
      "1\t1.0\t1",
      "2\t2.0\t2"
  );

  runTest(schema, evolvedSchema, beforeEvolution, afterEvolution);
  runDataChecks(evolvedSchema, expectedData);
}
 
Example 3
Source Project: big-c   Source File: Display.java    License: Apache License 2.0 6 votes vote down vote up
public AvroFileInputStream(FileStatus status) throws IOException {
  pos = 0;
  buffer = new byte[0];
  GenericDatumReader<Object> reader = new GenericDatumReader<Object>();
  FileContext fc = FileContext.getFileContext(new Configuration());
  fileReader =
    DataFileReader.openReader(new AvroFSInput(fc, status.getPath()),reader);
  Schema schema = fileReader.getSchema();
  writer = new GenericDatumWriter<Object>(schema);
  output = new ByteArrayOutputStream();
  JsonGenerator generator =
    new JsonFactory().createJsonGenerator(output, JsonEncoding.UTF8);
  MinimalPrettyPrinter prettyPrinter = new MinimalPrettyPrinter();
  prettyPrinter.setRootValueSeparator(System.getProperty("line.separator"));
  generator.setPrettyPrinter(prettyPrinter);
  encoder = EncoderFactory.get().jsonEncoder(schema, generator);
}
 
Example 4
Source Project: envelope   Source File: AvroSerializer.java    License: Apache License 2.0 6 votes vote down vote up
private void validateSchemaIsSupported(Schema schema) {
  for (Field field : schema.getFields()) {
    Type type = field.schema().getType();

    if (type.equals(Type.UNION)) {
      List<Schema> types = field.schema().getTypes();

      if (types.size() != 2) {
        throw new RuntimeException("Union type in Avro serializer schema must only contain two types");
      }

      if (types.get(0).getType().equals(Type.NULL)) {
        type = types.get(1).getType();
      }
      else {
        type = types.get(0).getType();
      }
    }

    if (!supportedTypes.contains(type)) {
      throw new RuntimeException("Avro serializer for Kafka output does not support Avro schema type: " + type);
    }
  }
}
 
Example 5
Source Project: components   Source File: NetSuiteOutputTransducerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testBasic() throws Exception {

    NetSuiteRuntime netSuiteRuntime = new TestNetSuiteRuntimeImpl(webServiceMockTestFixture.getClientFactory());
    NetSuiteDatasetRuntime dataSetRuntime = netSuiteRuntime.getDatasetRuntime(mockTestFixture.getConnectionProperties());

    mockGetRequestResults(null);

    TypeDesc typeDesc = clientService.getMetaDataSource().getTypeInfo("Opportunity");

    Schema schema = dataSetRuntime.getSchema(typeDesc.getTypeName());

    NsObjectOutputTransducer transducer = new NsObjectOutputTransducer(
            webServiceMockTestFixture.getClientService(), typeDesc.getTypeName());

    List<IndexedRecord> indexedRecordList = makeIndexedRecords(clientService, schema,
            new AbstractNetSuiteTestBase.SimpleObjectComposer<>(Opportunity.class), 10);

    for (IndexedRecord indexedRecord : indexedRecordList) {
        Opportunity record = (Opportunity) transducer.write(indexedRecord);
        assertNsObject(typeDesc, record);
    }
}
 
Example 6
Source Project: components   Source File: TypeConverterUtilsTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testConvertIntToString() {
    // No format
    String conv = testConvertValue(Schema.create(Schema.Type.INT), 123,
            TypeConverterProperties.TypeConverterOutputTypes.String, null, String.class);
    assertThat(conv, is("123"));

    // One format
    conv = testConvertValue(Schema.create(Schema.Type.INT), 123, TypeConverterProperties.TypeConverterOutputTypes.String,
            "'#'#", String.class);
    assertThat(conv, is("#123"));

    // Another format
    conv = testConvertValue(Schema.create(Schema.Type.INT), 123456, TypeConverterProperties.TypeConverterOutputTypes.String,
            "#,###.00", String.class);
    assertThat(conv, is("123,456.00"));
}
 
Example 7
Source Project: spork   Source File: AvroSchema2Pig.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Convert an Avro schema to a Pig schema
 */
public static ResourceSchema convert(Schema schema) throws IOException {

    if (AvroStorageUtils.containsGenericUnion(schema))
        throw new IOException ("We don't accept schema containing generic unions.");

    Set<Schema> visitedRecords = new HashSet<Schema>();
    ResourceFieldSchema inSchema = inconvert(schema, FIELD, visitedRecords);

    ResourceSchema tupleSchema;
    if (inSchema.getType() == DataType.TUPLE) {
        tupleSchema = inSchema.getSchema();
    } else { // other typs
        ResourceFieldSchema tupleWrapper = AvroStorageUtils.wrapAsTuple(inSchema);

        ResourceSchema topSchema = new ResourceSchema();
        topSchema.setFields(new ResourceFieldSchema[] { tupleWrapper });

        tupleSchema = topSchema;

    }
    return tupleSchema;
}
 
Example 8
Source Project: incubator-gobblin   Source File: AvroUtilsTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testDecorateRecordWithPrimitiveField() {
  Schema inputRecordSchema = SchemaBuilder.record("test").fields()
          .name("integer1")
          .prop("innerProp", "innerVal")
          .type().intBuilder().endInt().noDefault()
          .requiredString("string1")
          .endRecord();

  GenericRecord inputRecord = new GenericData.Record(inputRecordSchema);
  inputRecord.put("integer1", 10);
  inputRecord.put("string1", "hello");

  Schema outputRecordSchema = AvroUtils.decorateRecordSchema(inputRecordSchema, Collections.singletonList(new Schema.Field("newField", SchemaBuilder.builder().intType(), "test field", null)));
  Map<String, Object> newFields = new HashMap<>();
  newFields.put("newField", 5);

  GenericRecord outputRecord = AvroUtils.decorateRecord(inputRecord, newFields, outputRecordSchema);
  Assert.assertEquals(outputRecord.get("newField"), 5);
  Assert.assertEquals(outputRecord.get("integer1"), 10);
  Assert.assertEquals(outputRecord.get("string1"), "hello");

}
 
Example 9
Source Project: parquet-mr   Source File: TestGenericLogicalTypes.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testReadUUIDWithParquetUUID() throws IOException {
  Schema uuidSchema = record("R",
      field("uuid", LogicalTypes.uuid().addToSchema(Schema.create(STRING))));
  GenericRecord u1 = instance(uuidSchema, "uuid", UUID.randomUUID());
  GenericRecord u2 = instance(uuidSchema, "uuid", UUID.randomUUID());
  File test = write(conf(AvroWriteSupport.WRITE_PARQUET_UUID, true), uuidSchema, u1, u2);

  Assert.assertEquals("Should read UUID objects",
      Arrays.asList(u1, u2), read(GENERIC, uuidSchema, test));

  GenericRecord s1 = instance(uuidSchema, "uuid", u1.get("uuid").toString());
  GenericRecord s2 = instance(uuidSchema, "uuid", u2.get("uuid").toString());

  Assert.assertEquals("Should read UUID as Strings",
      Arrays.asList(s1, s2), read(GenericData.get(), uuidSchema, test));

}
 
Example 10
Source Project: beam   Source File: ParquetIOTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testWriteAndReadUsingReflectDataSchemaWithDataModel() {
  Schema testRecordSchema = ReflectData.get().getSchema(TestRecord.class);

  List<GenericRecord> records = generateGenericRecords(1000);
  mainPipeline
      .apply(Create.of(records).withCoder(AvroCoder.of(testRecordSchema)))
      .apply(
          FileIO.<GenericRecord>write()
              .via(ParquetIO.sink(testRecordSchema))
              .to(temporaryFolder.getRoot().getAbsolutePath()));
  mainPipeline.run().waitUntilFinish();

  PCollection<GenericRecord> readBack =
      readPipeline.apply(
          ParquetIO.read(testRecordSchema)
              .withAvroDataModel(GenericData.get())
              .from(temporaryFolder.getRoot().getAbsolutePath() + "/*"));

  PAssert.that(readBack).containsInAnyOrder(records);
  readPipeline.run().waitUntilFinish();
}
 
Example 11
Source Project: apicurio-registry   Source File: SchemasConfluentIT.java    License: Apache License 2.0 6 votes vote down vote up
@Test
void createAndUpdateSchema() throws Exception {
    String artifactId = TestUtils.generateArtifactId();

    Schema schema = new Schema.Parser().parse("{\"type\":\"record\",\"name\":\"myrecord1\",\"fields\":[{\"name\":\"foo1\",\"type\":\"string\"}]}");
    createArtifactViaConfluentClient(schema, artifactId);

    Schema updatedSchema = new Schema.Parser().parse("{\"type\":\"record\",\"name\":\"myrecord2\",\"fields\":[{\"name\":\"foo2\",\"type\":\"long\"}]}");
    createArtifactViaConfluentClient(updatedSchema, artifactId);

    assertThrows(SchemaParseException.class, () -> new Schema.Parser().parse("<type>record</type>\n<name>test</name>"));
    assertThat(confluentService.getAllVersions(artifactId), hasItems(1, 2));

    confluentService.deleteSubject(artifactId);
    waitForSubjectDeleted(artifactId);
}
 
Example 12
Source Project: incubator-gobblin   Source File: AvroExternalTable.java    License: Apache License 2.0 6 votes vote down vote up
private AvroExternalTable(AvroExternalTable.Builder builder) throws IOException {
  super(builder);

  if (builder.moveDataToTmpHdfsDir) {
    this.dataLocationInHdfs = moveDataFileToSeparateHdfsDir(builder.dataLocationInHdfs, builder.extensionToBeMoved);
    this.deleteDataAfterDone = true;
  } else {
    this.dataLocationInHdfs = builder.dataLocationInHdfs;
    this.deleteDataAfterDone = false;
  }

  if (StringUtils.isNotBlank(builder.schemaLocationInHdfs)) {
    this.schemaLocationInHdfs = builder.schemaLocationInHdfs;
    this.attributes = getAttributesFromAvroSchemaFile();
    this.deleteSchemaAfterDone = false;
  } else {
    Schema schema = getSchemaFromAvroDataFile();
    this.attributes = parseSchema(schema);
    this.schemaLocationInHdfs = writeSchemaToHdfs(schema);
    this.deleteSchemaAfterDone = true;
  }
}
 
Example 13
Source Project: incubator-gobblin   Source File: AvroFieldsPickConverter.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * For the schema that is a UNION type with NULL and Record type, it provides Records type.
 * @param inputSchema
 * @return
 */
private static Schema getActualRecord(Schema inputSchema) {
  if (Type.RECORD.equals(inputSchema.getType())) {
    return inputSchema;
  }

  Preconditions.checkArgument(Type.UNION.equals(inputSchema.getType()), "Nested schema is only support with either record or union type of null with record");
  Preconditions.checkArgument(inputSchema.getTypes().size() <= 2,
      "For union type in nested record, it should only have NULL and Record type");

  for (Schema inner : inputSchema.getTypes()) {
    if (Type.NULL.equals(inner.getType())) {
      continue;
    }
    Preconditions.checkArgument(Type.RECORD.equals(inner.getType()), "For union type in nested record, it should only have NULL and Record type");
    return inner;

  }
  throw new IllegalArgumentException(inputSchema + " is not supported.");
}
 
Example 14
Source Project: component-runtime   Source File: AvroSchema.java    License: Apache License 2.0 5 votes vote down vote up
private Type doMapType(final Schema schema) {
    switch (schema.getType()) {
    case LONG:
        if (Boolean.parseBoolean(readProp(schema, Type.DATETIME.name()))
                || LogicalTypes.timestampMillis().equals(LogicalTypes.fromSchemaIgnoreInvalid(schema))) {
            return Type.DATETIME;
        }
        return Type.LONG;
    default:
        return Type.valueOf(schema.getType().name());
    }
}
 
Example 15
Source Project: Cubert   Source File: AvroSchemaManager.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Look up schema using type name or field name
 */
public Schema getSchema(String name) {
    Schema schema = typeName2Schema.get(name);
    schema = (schema == null) ? name2Schema.get(name) : schema;
    return schema;

}
 
Example 16
Source Project: parquet-mr   Source File: AvroWriteSupport.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Calls an appropriate write method based on the value.
 * Value MUST not be null.
 *
 * @param type the Parquet type
 * @param avroSchema the Avro schema
 * @param value a non-null value to write
 */
private void writeValue(Type type, Schema avroSchema, Object value) {
  Schema nonNullAvroSchema = AvroSchemaConverter.getNonNull(avroSchema);
  LogicalType logicalType = nonNullAvroSchema.getLogicalType();
  if (logicalType != null) {
    Conversion<?> conversion = model.getConversionByClass(
        value.getClass(), logicalType);
    writeValueWithoutConversion(type, nonNullAvroSchema,
        convert(nonNullAvroSchema, logicalType, conversion, value));
  } else {
    writeValueWithoutConversion(type, nonNullAvroSchema, value);
  }
}
 
Example 17
Source Project: simplesource   Source File: AvroGenericUtils.java    License: Apache License 2.0 5 votes vote down vote up
private static Schema generateSchema(final Schema aggregateSchema) {
    return SchemaBuilder
            .record(aggregateSchema.getName() + "OptionalAggregateWithSequence").namespace(aggregateSchema.getNamespace())
            .fields()
            .name(AGGREGATION).type(toNullableSchema(aggregateSchema)).withDefault(null)
            .name(SEQUENCE).type().longType().noDefault()
            .endRecord();
}
 
Example 18
Source Project: flink   Source File: AvroSchemaConverter.java    License: Apache License 2.0 5 votes vote down vote up
private static SchemaBuilder.BaseTypeBuilder<Schema> getNullableBuilder(LogicalType logicalType) {
	SchemaBuilder.TypeBuilder<Schema> builder = SchemaBuilder.builder();
	if (logicalType.isNullable()) {
		return builder.nullable();
	}
	return builder;
}
 
Example 19
Source Project: xml-avro   Source File: SchemaBuilder.java    License: Apache License 2.0 5 votes vote down vote up
private Schema createRecordSchema(String name, XSComplexTypeDefinition type) {
    Schema record = Schema.createRecord(name, null, null, false);
    schemas.put(name, record);

    record.setFields(createFields(type));
    return record;
}
 
Example 20
Source Project: kite   Source File: SchemaCommand.java    License: Apache License 2.0 5 votes vote down vote up
@Override
@edu.umd.cs.findbugs.annotations.SuppressWarnings(
    value={"NP_GUARANTEED_DEREF", "NP_NULL_ON_SOME_PATH"},
    justification="Null case checked by precondition")
public int run() throws IOException {
  Preconditions.checkArgument(
      datasets != null && !datasets.isEmpty(),
      "Missing dataset name");
  if (merge || datasets.size() == 1) {
    Schema mergedSchema = null;
    for (String uriOrPath : datasets) {
      mergedSchema = merge(mergedSchema, schema(uriOrPath));
    }

    Preconditions.checkNotNull(mergedSchema, "No valid schema found");

    output(mergedSchema.toString(!minimize), console, outputPath);

  } else {
    Preconditions.checkArgument(outputPath == null,
        "Cannot output multiple schemas to one file");
    for (String name : datasets) {
      console.info("Dataset \"{}\" schema: {}",
          name, schema(name).toString(!minimize));
    }
  }
  return 0;
}
 
Example 21
Source Project: kite   Source File: DatasetKeyInputFormat.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings({"deprecation", "unchecked"})
private static <E> View<E> load(Configuration conf) {
  Class<E> type;
  try {
    type = (Class<E>)conf.getClass(KITE_TYPE, GenericData.Record.class);
  } catch (RuntimeException e) {
    if (e.getCause() instanceof ClassNotFoundException) {
      throw new TypeNotFoundException(String.format(
          "The Java class %s for the entity type could not be found",
          conf.get(KITE_TYPE)),
          e.getCause());
    } else {
      throw e;
    }
  }

  DefaultConfiguration.set(conf);

  String schemaStr = conf.get(KITE_READER_SCHEMA);
  Schema projection = null;
  if (schemaStr != null) {
    projection = new Schema.Parser().parse(schemaStr);
  }

  String inputUri = conf.get(KITE_INPUT_URI);
  if (projection != null) {
    return Datasets.load(inputUri).asSchema(projection).asType(type);
  } else {
    return Datasets.load(inputUri, type);
  }
}
 
Example 22
Source Project: nifi   Source File: TestPutHiveStreaming.java    License: Apache License 2.0 5 votes vote down vote up
private void assertOutputAvroRecords(List<Map<String, Object>> expectedRecords, MockFlowFile resultFlowFile) throws IOException {
    assertEquals(String.valueOf(expectedRecords.size()), resultFlowFile.getAttribute(PutHiveStreaming.HIVE_STREAMING_RECORD_COUNT_ATTR));

    final DataFileStream<GenericRecord> reader = new DataFileStream<>(
            new ByteArrayInputStream(resultFlowFile.toByteArray()),
            new GenericDatumReader<GenericRecord>());

    Schema schema = reader.getSchema();

    // Verify that the schema is preserved
    assertTrue(schema.equals(new Schema.Parser().parse(new File("src/test/resources/user.avsc"))));

    GenericRecord record = null;
    for (Map<String, Object> expectedRecord : expectedRecords) {
        assertTrue(reader.hasNext());
        record = reader.next(record);
        final String name = record.get("name").toString();
        final Integer favorite_number = (Integer) record.get("favorite_number");
        assertNotNull(name);
        assertNotNull(favorite_number);
        assertNull(record.get("favorite_color"));
        assertNull(record.get("scale"));

        assertEquals(expectedRecord.get("name"), name);
        assertEquals(expectedRecord.get("favorite_number"), favorite_number);
    }
    assertFalse(reader.hasNext());
}
 
Example 23
Source Project: kite   Source File: TestDataModelUtil.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testResolveTypeObjectToGeneric() {
  Class<Object> type = Object.class;
  Schema schema = SchemaBuilder.record("User").fields()
      .requiredString("name")
      .requiredString("color")
      .endRecord();
  Class expResult = GenericData.Record.class;
  Class result = DataModelUtil.resolveType(type, schema);
  assertEquals(expResult, result);
}
 
Example 24
@Override
protected Schema fetchSchemaByKey(Integer key) throws SchemaRegistryException {
  try {
    return this.schemaRegistryClient.getByID(key);
  } catch (IOException | RestClientException e) {
    throw new SchemaRegistryException(e);
  }
}
 
Example 25
Source Project: components   Source File: MarketoSOAPClientTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testGetLeadActivity() throws Exception {
    doReturn(getLeadActivityResult()).when(port).getLeadActivity(any(ParamsGetLeadActivity.class),
            any(AuthenticationHeader.class));
    iprops.inputOperation.setValue(InputOperation.getLeadActivity);
    iprops.afterInputOperation();
    Field attr = new Field("attrName", AvroUtils._string(), "", null);
    iprops.schemaInput.schema
            .setValue(MarketoUtils.newSchema(iprops.schemaInput.schema.getValue(), "test", Collections.singletonList(attr)));
    iprops.beforeMappingInput();
    iprops.leadKeyTypeSOAP.setValue(LeadKeyTypeSOAP.IDNUM);
    mktoRR = client.getLeadActivity(iprops, null);
    assertNotNull(mktoRR);
    assertTrue(mktoRR.isSuccess());
    List<IndexedRecord> records = mktoRR.getRecords();
    assertNotNull(records);
    IndexedRecord record = records.get(0);
    assertNotNull(record);
    Schema refSchema = iprops.schemaInput.schema.getValue();
    assertEquals(refSchema, record.getSchema());
    assertEquals("ABC-123-DEF", record.get(refSchema.getField("marketoGUID").pos()));
    assertEquals(123456L, record.get(refSchema.getField("Id").pos()));
    assertEquals("mktgAssetName", record.get(refSchema.getField("MktgAssetName").pos()));
    assertTrue(record.get(refSchema.getField("ActivityDateTime").pos()) instanceof Long);
    assertEquals("activityType", record.get(refSchema.getField("ActivityType").pos()));
    assertEquals("mktgAssetName", record.get(refSchema.getField("MktgAssetName").pos()));
    assertEquals("mktPersonId", record.get(refSchema.getField("MktPersonId").pos()));
    assertEquals("campaign", record.get(refSchema.getField("Campaign").pos()));
    assertEquals("foreignSysId", record.get(refSchema.getField("ForeignSysId").pos()));
    assertEquals("personName", record.get(refSchema.getField("PersonName").pos()));
    assertEquals("orgName", record.get(refSchema.getField("OrgName").pos()));
    assertEquals("foreignSysOrgId", record.get(refSchema.getField("ForeignSysOrgId").pos()));
    assertEquals("attrValue", record.get(refSchema.getField("attrName").pos()));
    //
    doThrow(new RuntimeException("error")).when(port).getLeadActivity(any(ParamsGetLeadActivity.class),
            any(AuthenticationHeader.class));
    mktoRR = client.getLeadActivity(iprops, null);
    assertNotNull(mktoRR);
    assertFalse(mktoRR.isSuccess());
}
 
Example 26
@Test
public void testConfluentAvroDeserializerForSchemaEvolution() throws IOException, RestClientException, SchemaRegistryException {
  WorkUnitState mockWorkUnitState = getMockWorkUnitState(0L, 10L);
  mockWorkUnitState.setProp("schema.registry.url", TEST_URL);

  Schema schemaV1 = SchemaBuilder.record(TEST_RECORD_NAME)
      .namespace(TEST_NAMESPACE).fields()
      .name(TEST_FIELD_NAME).type().stringType().noDefault()
      .endRecord();

  Schema schemaV2 = SchemaBuilder.record(TEST_RECORD_NAME)
      .namespace(TEST_NAMESPACE).fields()
      .name(TEST_FIELD_NAME).type().stringType().noDefault()
      .optionalString(TEST_FIELD_NAME2).endRecord();

  GenericRecord testGenericRecord = new GenericRecordBuilder(schemaV1).set(TEST_FIELD_NAME, "testValue").build();

  SchemaRegistryClient mockSchemaRegistryClient = mock(SchemaRegistryClient.class);
  when(mockSchemaRegistryClient.getByID(any(Integer.class))).thenReturn(schemaV1);

  Serializer<Object> kafkaEncoder = new KafkaAvroSerializer(mockSchemaRegistryClient);
  Deserializer<Object> kafkaDecoder = new KafkaAvroDeserializer(mockSchemaRegistryClient);

  ByteBuffer testGenericRecordByteBuffer =
      ByteBuffer.wrap(kafkaEncoder.serialize(TEST_TOPIC_NAME, testGenericRecord));

  KafkaSchemaRegistry<Integer, Schema> mockKafkaSchemaRegistry = mock(KafkaSchemaRegistry.class);
  when(mockKafkaSchemaRegistry.getLatestSchemaByTopic(TEST_TOPIC_NAME)).thenReturn(schemaV2);

  KafkaDeserializerExtractor kafkaDecoderExtractor = new KafkaDeserializerExtractor(mockWorkUnitState,
      Optional.fromNullable(Deserializers.CONFLUENT_AVRO), kafkaDecoder, mockKafkaSchemaRegistry);

  when(kafkaDecoderExtractor.getSchema()).thenReturn(schemaV2);

  ByteArrayBasedKafkaRecord mockMessageAndOffset = getMockMessageAndOffset(testGenericRecordByteBuffer);

  GenericRecord received = (GenericRecord) kafkaDecoderExtractor.decodeRecord(mockMessageAndOffset);
  Assert.assertEquals(received.toString(), "{\"testField\": \"testValue\", \"testField2\": null}");

}
 
Example 27
Source Project: nifi   Source File: AvroRecordSetWriter.java    License: Apache License 2.0 5 votes vote down vote up
@OnEnabled
public void onEnabled(final ConfigurationContext context) {
    final int cacheSize = context.getProperty(CACHE_SIZE).asInteger();
    compiledAvroSchemaCache = Caffeine.newBuilder()
            .maximumSize(cacheSize)
            .build(schemaText -> new Schema.Parser().parse(schemaText));

    final int capacity = context.getProperty(ENCODER_POOL_SIZE).evaluateAttributeExpressions().asInteger();
    encoderPool = new LinkedBlockingQueue<>(capacity);
}
 
Example 28
Source Project: xml-avro   Source File: ConverterTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void uniqueFieldNames() {
    String xsd =
            "<xs:schema xmlns:xs='http://www.w3.org/2001/XMLSchema'>" +
            "  <xs:complexType name='type'>" +
            "    <xs:sequence>" +
            "      <xs:element name='field' type='xs:string'/>" +
            "    </xs:sequence>" +
            "    <xs:attribute name='field' type='xs:string'/>" +
            "  </xs:complexType>" +
            "  <xs:element name='root' type='type'/>" +
            "</xs:schema>";

    Schema schema = Converter.createSchema(xsd);

    assertEquals(2, schema.getFields().size());
    Schema.Field field = schema.getField("field");
    assertNotNull(field);
    assertEquals("" + new Source("field", true), field.getProp(Source.SOURCE));

    Schema.Field field0 = schema.getField("field0");
    assertEquals("" + new Source("field", false), field0.getProp(Source.SOURCE));

    String xml = "<root field='value'><field>value0</field></root>";
    GenericData.Record record = Converter.createDatum(schema, xml);

    assertEquals("value", record.get("field"));
    assertEquals("value0", record.get("field0"));
}
 
Example 29
@Test
public void testNonSchemaConstructableNewInstance() {
  Schema schema = Mockito.mock(Schema.class);
  Object instance = AvroCompatibilityHelper.newInstance(Pojo.class, schema);
  Assert.assertNotNull(instance);
  Assert.assertTrue(instance instanceof  Pojo);
}
 
Example 30
Source Project: components   Source File: PythonRowDoFn.java    License: Apache License 2.0 5 votes vote down vote up
private void map(IndexedRecord input, ProcessContext context) throws IOException {
    PyObject output = pyFn.__call__(new PyUnicode(input.toString()));

    if (jsonGenericRecordConverter == null) {
        JsonSchemaInferrer jsonSchemaInferrer = new JsonSchemaInferrer(new ObjectMapper());
        Schema jsonSchema = jsonSchemaInferrer.inferSchema(output.toString());
        jsonGenericRecordConverter = new JsonGenericRecordConverter(jsonSchema);
    }

    GenericRecord outputRecord = jsonGenericRecordConverter.convertToAvro(output.toString());
    context.output(outputRecord);
}