Java Code Examples for org.apache.avro.Schema#toString()

The following examples show how to use org.apache.avro.Schema#toString() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
private void doTestSchemaResolution(String givenSchemaLocation, String expectedSchemaLocation) throws IOException {
    AvroSchemaResolver avroSchemaResolver = new AvroSchemaResolver(null);
    Schema schema = new Schema.Parser().parse(getResourceText(givenSchemaLocation));
    LOG.info("schema = %s", schema);

    Schema effectiveSchema = avroSchemaResolver.handleUnionFieldsWithNull(schema, new HashSet<>());
    LOG.info("effectiveSchema = %s", effectiveSchema);
    String returnedSchemaText = effectiveSchema.toString();
    Assert.assertEquals(getResourceText(expectedSchemaLocation).replace(" ", ""),
                        returnedSchemaText.replace(" ", ""));

    // double check whether the effective schema is semantically right parsing
    Schema.Parser parser = new Schema.Parser();
    Schema parsedReturnedSchema = parser.parse(returnedSchemaText);
    Assert.assertEquals(effectiveSchema, parsedReturnedSchema);
}
 
Example 2
Source Project: kite   File: Conversions.java    License: Apache License 2.0 6 votes vote down vote up
public static Object convert(Object obj, Schema schema) {
  Class<?> returnType = SchemaUtil.getClassForType(schema.getType());
  if (returnType != null) {
    return convert(obj, returnType);
  }

  switch (schema.getType()) {
    case UNION:
      for (Schema possibleSchema : schema.getTypes()) {
        try {
          return convert(obj,
              SchemaUtil.getClassForType(possibleSchema.getType()));
        } catch (ClassCastException e) {
          // didn't work, try the next one
        }
      }
      break;
  }
  throw new ClassCastException(
      "Cannot convert using schema: " + schema.toString(true));
}
 
Example 3
private int processSchemaFiles() {
    Set<File> inputFiles = filterSources(new FileExtensionSpec(SCHEMA_EXTENSION)).getFiles();
    ProcessingState processingState = resolver.resolve(inputFiles);
    for (Schema schema : processingState.getSchemas()) {
        try {
            String outputPath = schema.getNamespace().replaceAll(Pattern.quote("."), "/")
                + "/" + schema.getName() + "." + SCHEMA_EXTENSION;
            File outputFile = new File(getOutputDir().get().getAsFile(), outputPath);
            String schemaJson = schema.toString(true);
            FileUtils.writeJsonFile(outputFile, schemaJson);
            getLogger().debug("Wrote {}", outputFile.getPath());
        } catch (IOException ex) {
            throw new GradleException(String.format("Failed to write resolved schema definition for %s", schema.getFullName()), ex);
        }
    }
    return processingState.getProcessedTotal();
}
 
Example 4
@Test
public void testUnionSchemasPropRetention() throws Exception {
    AvroSchemaResolver avroSchemaResolver = new AvroSchemaResolver(null);
    Schema schema = new Schema.Parser().parse(getResourceText("/avro/composites/unions-with-props.avsc"));
    LOG.info("schema = %s", schema);

    Schema effectiveSchema = avroSchemaResolver.handleUnionFieldsWithNull(schema, new HashSet<>());
    LOG.info("effectiveSchema = %s", effectiveSchema);
    String returnedSchemaText = effectiveSchema.toString();
    Assert.assertEquals("foo", effectiveSchema.getField("name").getProp("someProp"));
    Assert.assertEquals("bar", effectiveSchema.getField("address").getProp("otherProp"));
    Assert.assertEquals("baz", effectiveSchema.getField("address").schema().getField("pincode").getProp("anotherProp"));
    Assert.assertEquals("quux", effectiveSchema.getField("secondaryAddress").getProp("moreProps"));

    // double check whether the effective schema is semantically right parsing
    Schema.Parser parser = new Schema.Parser();
    Schema parsedReturnedSchema = parser.parse(returnedSchemaText);
    Assert.assertEquals(effectiveSchema, parsedReturnedSchema);
}
 
Example 5
Source Project: kite   File: Compatibility.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Checks that the {@code existing} {@link DatasetDescriptor} is compatible
 * with {@code test}.
 *
 * @param existing the current {@code DatasetDescriptor} for a dataset
 * @param test a new {@code DatasetDescriptor} for the same dataset
 */
public static void checkCompatible(DatasetDescriptor existing,
                                   DatasetDescriptor test) {
  checkNotChanged("format", existing.getFormat(), test.getFormat());

  checkNotChanged("partitioning",
      existing.isPartitioned(), test.isPartitioned());

  if (existing.isPartitioned()) {
    checkStrategyUpdate(
        existing.getPartitionStrategy(),
        test.getPartitionStrategy(),
        test.getSchema());
  }

  // check can read records written with old schema using new schema
  Schema oldSchema = existing.getSchema();
  Schema testSchema = test.getSchema();
  if (!SchemaValidationUtil.canRead(oldSchema, testSchema)) {
    throw new IncompatibleSchemaException("Schema cannot read data " +
        "written using existing schema. Schema: " + testSchema.toString(true) +
        "\nExisting schema: " + oldSchema.toString(true));
  }

}
 
Example 6
Source Project: nifi   File: TestConvertAvroToJSON.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSingleSchemalessAvroMessage() throws IOException {
    final TestRunner runner = TestRunners.newTestRunner(new ConvertAvroToJSON());
    Schema schema = new Schema.Parser().parse(new File("src/test/resources/user.avsc"));
    String stringSchema = schema.toString();
    runner.setProperty(ConvertAvroToJSON.SCHEMA, stringSchema);

    final GenericRecord user1 = new GenericData.Record(schema);
    user1.put("name", "Alyssa");
    user1.put("favorite_number", 256);

    final ByteArrayOutputStream out1 = new ByteArrayOutputStream();
    final BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out1, null);
    final DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<>(schema);
    datumWriter.write(user1, encoder);

    encoder.flush();
    out1.flush();
    byte[] test = out1.toByteArray();
    runner.enqueue(test);

    runner.run();

    runner.assertAllFlowFilesTransferred(ConvertAvroToJSON.REL_SUCCESS, 1);
    final MockFlowFile out = runner.getFlowFilesForRelationship(ConvertAvroToJSON.REL_SUCCESS).get(0);
    out.assertContentEquals("{\"name\": \"Alyssa\", \"favorite_number\": 256, \"favorite_color\": null}");
}
 
Example 7
Source Project: flink   File: AvroSerializationSchema.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates an Avro deserialization schema.
 *
 * @param recordClazz class to serialize. Should be one of:
 *                    {@link org.apache.avro.specific.SpecificRecord},
 *                    {@link org.apache.avro.generic.GenericRecord}.
 * @param schema      writer Avro schema. Should be provided if recordClazz is
 *                    {@link GenericRecord}
 */
protected AvroSerializationSchema(Class<T> recordClazz, @Nullable Schema schema) {
	Preconditions.checkNotNull(recordClazz, "Avro record class must not be null.");
	this.recordClazz = recordClazz;
	this.schema = schema;
	if (schema != null) {
		this.schemaString = schema.toString();
	} else {
		this.schemaString = null;
	}
}
 
Example 8
Source Project: iceberg   File: PartitionData.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Used by Avro reflection to instantiate this class when reading manifest files.
 */
PartitionData(Schema schema) {
  this.partitionType = AvroSchemaUtil.convert(schema).asNestedType().asStructType();
  this.size = partitionType.fields().size();
  this.data = new Object[size];
  this.stringSchema = schema.toString();
  this.schema = schema;
}
 
Example 9
@Test
public void testSingleSchemalessAvroMessage_noContainer() throws IOException {
    final TestRunner runner = TestRunners.newTestRunner(new ConvertAvroToJSON());
    runner.setProperty(ConvertAvroToJSON.CONTAINER_OPTIONS, ConvertAvroToJSON.CONTAINER_NONE);
    Schema schema = new Schema.Parser().parse(new File("src/test/resources/user.avsc"));
    String stringSchema = schema.toString();
    runner.setProperty(ConvertAvroToJSON.SCHEMA, stringSchema);

    final GenericRecord user1 = new GenericData.Record(schema);
    user1.put("name", "Alyssa");
    user1.put("favorite_number", 256);

    final ByteArrayOutputStream out1 = new ByteArrayOutputStream();
    final BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out1, null);
    final DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<>(schema);
    datumWriter.write(user1, encoder);

    encoder.flush();
    out1.flush();
    byte[] test = out1.toByteArray();
    runner.enqueue(test);

    runner.run();

    runner.assertAllFlowFilesTransferred(ConvertAvroToJSON.REL_SUCCESS, 1);
    final MockFlowFile out = runner.getFlowFilesForRelationship(ConvertAvroToJSON.REL_SUCCESS).get(0);
    out.assertContentEquals("{\"name\": \"Alyssa\", \"favorite_number\": 256, \"favorite_color\": null}");
}
 
Example 10
Source Project: localization_nifi   File: JsonUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Reads provided {@link InputStream} as ISON into Avro
 * {@link GenericRecord} applying provided {@link Schema} returning the
 * resulting GenericRecord.
 */
public static GenericRecord read(InputStream jsonIs, Schema schema) {
    DataInputStream din = new DataInputStream(jsonIs);
    try {
        Decoder decoder = DecoderFactory.get().jsonDecoder(schema, din);
        DatumReader<GenericData.Record> reader = new GenericDatumReader<>(schema);
        return reader.read(null, decoder);
    } catch (Exception e) {
        throw new IllegalStateException("Failed to parse incoming Json input stream into Avro GenericRecord. "
                + "Possible reason: the value may not be a valid JSON or incompatible schema is provided. Schema was '"
                + schema.toString(true) + "'.", e);
    }
}
 
Example 11
Source Project: samza   File: AvroSchemaGenRelConverter.java    License: Apache License 2.0 5 votes vote down vote up
private Schema computePayloadSchema(String streamName, SamzaSqlRelMessage relMessage) {
  SamzaSqlRelRecord relRecord = relMessage.getSamzaSqlRelRecord();
  List<Schema.Field> keyFields = new ArrayList<>();
  List<String> fieldNames = relRecord.getFieldNames();
  List<Object> values = relRecord.getFieldValues();

  for (int index = 0; index < fieldNames.size(); index++) {
    if (fieldNames.get(index).equals(SamzaSqlRelMessage.KEY_NAME) || values.get(index) == null) {
      continue;
    }

    Object value = values.get(index);
    Schema avroType;
    if (value instanceof GenericData.Record) {
      avroType = ((GenericData.Record) value).getSchema();
    } else {
      avroType = ReflectData.get().getSchema(value.getClass());
    }
    keyFields.add(new Schema.Field(fieldNames.get(index), avroType, "", null));
  }

  Schema ks = Schema.createRecord(streamName, "", streamName + "_namespace", false);
  ks.setFields(keyFields);
  String schemaStr = ks.toString();
  Schema schema;
  // See whether we have a schema object corresponding to the schemaValue and reuse it.
  // CachedSchemaRegistryClient doesn't like if we recreate schema objects.
  if (schemas.containsKey(schemaStr)) {
    schema = schemas.get(schemaStr);
  } else {
    schema = Schema.parse(schemaStr);
    schemas.put(schemaStr, schema);
  }

  return schema;
}
 
Example 12
Source Project: iceberg   File: PartitionData.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Used by Avro reflection to instantiate this class when reading manifest files.
 */
public PartitionData(Schema schema) {
  this.partitionType = AvroSchemaUtil.convert(schema).asNestedType().asStructType();
  this.size = partitionType.fields().size();
  this.data = new Object[size];
  this.stringSchema = schema.toString();
  this.schema = schema;
}
 
Example 13
@Override
public void write(Kryo kryo, Output output, Schema object) {
	String schemaAsString = object.toString(false);
	output.writeString(schemaAsString);
}
 
Example 14
Source Project: flink   File: AvroKryoSerializerUtils.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void write(Kryo kryo, Output output, Schema object) {
	String schemaAsString = object.toString(false);
	output.writeString(schemaAsString);
}
 
Example 15
Source Project: components   File: GeneratorFunctions.java    License: Apache License 2.0 4 votes vote down vote up
public EnumGenerator(Schema enumSchema) {
    jsonSchema = enumSchema.toString();
    schema = enumSchema;
}
 
Example 16
Source Project: jstorm   File: GenericAvroSerializer.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public String getFingerprint(Schema schema) {
    return schema.toString();
}
 
Example 17
public FixedFlowInputBoundedSource withSchema(Schema schema) {
    this.schemaString = schema.toString();
    return this;
}
 
Example 18
public AvroGenericRecordSource(int numEvents, int numKeys, Schema schema) {
	super(numEvents, numKeys);
	this.producedType = new GenericRecordAvroTypeInfo(schema);
	this.myPojoSchema = schema;
	this.schemaString = schema.toString();
}
 
Example 19
/**
 * @param input avro object
 * @return textual representation of the schema of the given {@code input} avro object
 */
@Override
protected String getSchemaText(Object input) {
    Schema schema = AvroUtils.computeSchema(input);
    return schema.toString();
}
 
Example 20
public TheMerger(Schema schema)
{
  this.schema = schema;
  this.schemaString = schema.toString();
}