Java Code Examples for org.apache.avro.Schema#create()

The following examples show how to use org.apache.avro.Schema#create() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: components   File: TJDBCRowProperties.java    License: Apache License 2.0 6 votes vote down vote up
public void updateOutputSchemas() {
    Schema inputSchema = main.schema.getValue();

    schemaFlow.schema.setValue(inputSchema);

    final List<Schema.Field> additionalRejectFields = new ArrayList<Schema.Field>();

    Schema.Field field = new Schema.Field("errorCode", Schema.create(Schema.Type.STRING), null, (Object) null);
    field.addProp(SchemaConstants.TALEND_IS_LOCKED, "false");
    field.addProp(SchemaConstants.TALEND_FIELD_GENERATED, "true");
    field.addProp(SchemaConstants.TALEND_COLUMN_DB_LENGTH, "255");
    additionalRejectFields.add(field);

    field = new Schema.Field("errorMessage", Schema.create(Schema.Type.STRING), null, (Object) null);
    field.addProp(SchemaConstants.TALEND_IS_LOCKED, "false");
    field.addProp(SchemaConstants.TALEND_FIELD_GENERATED, "true");
    field.addProp(SchemaConstants.TALEND_COLUMN_DB_LENGTH, "255");
    additionalRejectFields.add(field);

    Schema rejectSchema = CommonUtils.newSchema(inputSchema, "rejectOutput", additionalRejectFields);

    schemaReject.schema.setValue(rejectSchema);
}
 
Example 2
@Test
public void shouldReadMapOfJavaStrings() {
    // given
    Schema stringMapSchema = Schema.createMap(Schema.create(Schema.Type.STRING));
    Schema javaStringSchema = Schema.create(Schema.Type.STRING);
    GenericData.setStringType(javaStringSchema, GenericData.StringType.String);
    Schema javaStringMapSchema = Schema.createMap(javaStringSchema);

    Map<String, String> stringMap = new HashMap<>(0);
    stringMap.put("1", "abc");
    stringMap.put("2", "aaa");

    // when
    Map<Utf8, String> resultJavaStringMap = deserializeGenericFast(stringMapSchema, javaStringMapSchema,
            serializeGeneric(stringMap, javaStringMapSchema));

    // then
    Assert.assertEquals(2, resultJavaStringMap.size());
    Assert.assertEquals("abc", resultJavaStringMap.get(new Utf8("1")));
    Assert.assertEquals("aaa", resultJavaStringMap.get(new Utf8("2")));
}
 
Example 3
@Test
public void shouldReadArrayOfJavaStrings() {
    // given
    Schema javaStringSchema = Schema.create(Schema.Type.STRING);
    GenericData.setStringType(javaStringSchema, GenericData.StringType.String);
    Schema javaStringArraySchema = Schema.createArray(javaStringSchema);

    GenericData.Array<String> javaStringArray = new GenericData.Array<>(0, javaStringArraySchema);
    javaStringArray.add("aaa");
    javaStringArray.add("abc");

    GenericData.Array<String> resultJavaStringArray = deserializeGenericFast(javaStringArraySchema, javaStringArraySchema,
            serializeGeneric(javaStringArray));

    // then
    Assert.assertEquals(2, resultJavaStringArray.size());
    Assert.assertEquals("aaa", resultJavaStringArray.get(0));
    Assert.assertEquals("abc", resultJavaStringArray.get(1));
}
 
Example 4
Source Project: Cubert   File: GenerateDictionary.java    License: Apache License 2.0 6 votes vote down vote up
public static Schema getSchema()
{
    Field[] fields =
            {
                    new Schema.Field("colname",
                                     Schema.create(Type.STRING),
                                     null,
                                     null),
                    new Schema.Field("colvalue",
                                     Schema.create(Type.STRING),
                                     null,
                                     null),
                    new Schema.Field("code", Schema.create(Type.INT), null, null) };

    Schema schema = Schema.createRecord("dictionary", null, null, false);
    schema.setFields(Arrays.asList(fields));

    return schema;
}
 
Example 5
Source Project: xml-avro   File: SchemaBuilder.java    License: Apache License 2.0 6 votes vote down vote up
private Schema createTypeSchema(XSTypeDefinition type, boolean optional, boolean array) {
    typeLevel++;
    Schema schema;

    if (type.getTypeCategory() == XSTypeDefinition.SIMPLE_TYPE)
        schema = Schema.create(getPrimitiveType((XSSimpleTypeDefinition) type));
    else {
        String name = complexTypeName(type);
        debug("Creating schema for type " + name);

        schema = schemas.get(name);
        if (schema == null) schema = createRecordSchema(name, (XSComplexTypeDefinition) type);
    }

    if (array || isGroupTypeWithMultipleOccurs(type))
        schema = Schema.createArray(schema);
    else if (optional) {
        Schema nullSchema = Schema.create(Schema.Type.NULL);
        schema = Schema.createUnion(Arrays.asList(nullSchema, schema));
    }

    typeLevel--;
    return schema;
}
 
Example 6
@Test
public void testWriteUUIDReadStringSchema() throws IOException {
  Schema uuidSchema = record("R",
      field("uuid", LogicalTypes.uuid().addToSchema(Schema.create(STRING))));
  GenericRecord u1 = instance(uuidSchema, "uuid", UUID.randomUUID());
  GenericRecord u2 = instance(uuidSchema, "uuid", UUID.randomUUID());

  Schema stringUuidSchema = Schema.create(STRING);
  stringUuidSchema.addProp(GenericData.STRING_PROP, "String");
  Schema stringSchema = record("R", field("uuid", stringUuidSchema));
  GenericRecord s1 = instance(stringSchema, "uuid", u1.get("uuid").toString());
  GenericRecord s2 = instance(stringSchema, "uuid", u2.get("uuid").toString());

  File test = write(GENERIC, uuidSchema, u1, u2);
  Assert.assertEquals("Should read UUIDs as Strings",
      Arrays.asList(s1, s2), read(GENERIC, stringSchema, test));
}
 
Example 7
Source Project: components   File: AvroTypeConverter.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Converts Talend type to Avro type schema
 * 
 * @param talendType data integration native type
 * @param logicalType avro logical type
 * @return field schema
 * @throws {@link UnsupportedOperationException} in case of unsupported Talend type or logical type
 */
public static Schema convertToAvro(TalendType talendType, String logicalType) {
    Schema fieldSchema = LogicalTypeUtils.getSchemaByLogicalType(logicalType);
    if (fieldSchema != null) {
        return fieldSchema;
    }

    switch (talendType) {
    case STRING:
        return Schema.create(Schema.Type.STRING);
    case BOOLEAN:
        return Schema.create(Schema.Type.BOOLEAN);
    case INTEGER:
        return Schema.create(Schema.Type.INT);
    case LONG:
        return Schema.create(Schema.Type.LONG);
    case DOUBLE:
        return Schema.create(Schema.Type.DOUBLE);
    case FLOAT:
        return Schema.create(Schema.Type.FLOAT);
    case BYTE:
        return AvroUtils._byte();
    case SHORT:
        return AvroUtils._short();
    case CHARACTER:
        return AvroUtils._character();
    case BIG_DECIMAL:
        return AvroUtils._decimal();
    case DATE:
        return AvroUtils._logicalTimestamp();
    default:
        throw new UnsupportedOperationException("Unrecognized type " + talendType);
    }
}
 
Example 8
@Test(groups = "deserializationTest")
public void testIsSupportedForFastDeserializer() {
  Set<Schema.Type> supportedSchemaTypes = new HashSet<>();
  supportedSchemaTypes.add(Schema.Type.RECORD);
  supportedSchemaTypes.add(Schema.Type.MAP);
  supportedSchemaTypes.add(Schema.Type.ARRAY);

  Map<Schema.Type, Schema> schemaTypes = new HashMap<>();
  /**
   * Those types could be created by {@link Schema#create(org.apache.avro.Schema.Type)} function.
   */
  schemaTypes.put(Schema.Type.RECORD, Schema.parse("{\"type\": \"record\", \"name\": \"test\", \"fields\":[]}"));
  schemaTypes.put(Schema.Type.MAP, Schema.parse("{\"type\": \"map\", \"values\": \"string\"}"));
  schemaTypes.put(Schema.Type.ARRAY, Schema.parse("{\"type\": \"array\", \"items\": \"string\"}"));
  schemaTypes.put(Schema.Type.ENUM, Schema.parse("{\"type\": \"enum\", \"name\": \"test_enum\", \"symbols\":[]}"));
  schemaTypes.put(Schema.Type.UNION, Schema.parse("[\"null\", \"string\"]"));
  schemaTypes.put(Schema.Type.FIXED, Schema.parse("{\"type\": \"fixed\", \"size\": 16, \"name\": \"test_fixed\"}"));

  for (Schema.Type type : Schema.Type.values()) {
    Schema schema = schemaTypes.containsKey(type) ? schemaTypes.get(type) : Schema.create(type);
    if (supportedSchemaTypes.contains(type)) {
      Assert.assertTrue(FastSerdeCache.isSupportedForFastDeserializer(type));
      FastDeserializerGeneratorBase.getClassName(schema, schema, "");
    } else {
      Assert.assertFalse(FastSerdeCache.isSupportedForFastDeserializer(type));
    }
  }
}
 
Example 9
@Test
public void testRecordWithFixed () {
    List<Schema.Field> fields = new ArrayList<>();
    Map<String, Object> data = new HashMap<>();
    for (int i = 0; i < PRIMITIVE_VALUES.length; ++i) {
        Schema.Field field = new Schema.Field(NAMES[i], Schema.create(SCHEMA_TYPES[i]), null, null);
        fields.add(field);
        data.put(NAMES[i], PRIMITIVE_VALUES[i]);
    }

    //add fixed to test case
    fields.add(new Schema.Field("fixed", Schema.createFixed("fixedSchema", null, null, 10), null, null));
    data.put("fixed", "bytes".getBytes());

    //add array to test case
    fields.add(new Schema.Field("array", Schema.createArray(Schema.create(Schema.Type.INT)), null, null));
    List<Integer> integerList = new ArrayList<>();
    integerList.add(1);
    integerList.add(2);
    data.put("array", integerList);

    Schema schema = Schema.createRecord(fields);
    GenericRecord expected = new GenericData.Record(schema);
    for (int i = 0; i < PRIMITIVE_VALUES.length; ++i) {
        expected.put(NAMES[i], PRIMITIVE_VALUES[i]);
    }
    expected.put("fixed", new GenericData.Fixed(Schema.createFixed("fixedSchema", null, null, 10), "bytes".getBytes()));
    expected.put("array", new GenericData.Array<Integer>(Schema.createArray(Schema.create(Schema.Type.INT)), integerList));
    StreamlineEvent streamlineEvent = StreamlineEventImpl.builder().fieldsAndValues(data).dataSourceId("dataSourceId").build();
    Assert.assertEquals(expected, StreamlineEventSerializer.getAvroRecord(streamlineEvent, schema));
}
 
Example 10
Source Project: beam   File: AvroGenerators.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Schema generate(SourceOfRandomness random, GenerationStatus status) {
  Schema.Type type;

  if (nesting(status) >= MAX_NESTING) {
    type = random.choose(PRIMITIVE_TYPES);
  } else {
    type = random.choose(ALL_TYPES);
  }

  if (PRIMITIVE_TYPES.contains(type)) {
    return Schema.create(type);
  } else {
    nestingInc(status);

    if (type == Schema.Type.FIXED) {
      int size = random.choose(Arrays.asList(1, 5, 12));
      return Schema.createFixed("fixed_" + branch(status), "", "", size);
    } else if (type == Schema.Type.UNION) {
      // only nullable fields, everything else isn't supported in row conversion code
      return UnionSchemaGenerator.INSTANCE.generate(random, status);
    } else if (type == Schema.Type.ENUM) {
      return EnumSchemaGenerator.INSTANCE.generate(random, status);
    } else if (type == Schema.Type.RECORD) {
      return RecordSchemaGenerator.INSTANCE.generate(random, status);
    } else if (type == Schema.Type.MAP) {
      return Schema.createMap(generate(random, status));
    } else if (type == Schema.Type.ARRAY) {
      return Schema.createArray(generate(random, status));
    } else {
      throw new AssertionError("Unexpected AVRO type: " + type);
    }
  }
}
 
Example 11
Source Project: nifi   File: AvroTypeUtil.java    License: Apache License 2.0 5 votes vote down vote up
private static Schema nullable(final Schema schema) {
    if (schema.getType() == Type.UNION) {
        final List<Schema> unionTypes = new ArrayList<>(schema.getTypes());
        final Schema nullSchema = Schema.create(Type.NULL);
        if (unionTypes.contains(nullSchema)) {
            return schema;
        }

        unionTypes.add(nullSchema);
        return Schema.createUnion(unionTypes);
    }

    return Schema.createUnion(Schema.create(Type.NULL), schema);
}
 
Example 12
private void updateOutputSchemas() {
    final Schema inputSchema = module.main.schema.getValue();

    Schema.Field field = null;

    final List<Schema.Field> additionalMainFields = new ArrayList<Schema.Field>();

    if (OutputAction.UPSERT.equals(outputAction.getValue()) && outputUpsertKey.getValue()
            && inputSchema.getField(upsertKeyColumn.getValue()) == null) {
        field = new Schema.Field("UpsertColumnValue", Schema.create(Schema.Type.STRING), null, (Object) null);
        field.addProp(SchemaConstants.TALEND_IS_LOCKED, "false");
        field.addProp(SchemaConstants.TALEND_FIELD_GENERATED, "true");
        field.addProp(SchemaConstants.TALEND_COLUMN_DB_LENGTH, "255");
        additionalMainFields.add(field);
    }

    field = new Schema.Field("salesforce_id", Schema.create(Schema.Type.STRING), null, (Object) null);
    field.addProp(SchemaConstants.TALEND_IS_LOCKED, "false");
    field.addProp(SchemaConstants.TALEND_FIELD_GENERATED, "true");
    field.addProp(SchemaConstants.TALEND_COLUMN_DB_LENGTH, "255");
    additionalMainFields.add(field);

    field = new Schema.Field("salesforce_created", Schema.create(Schema.Type.STRING), null, (Object) null);
    field.addProp(SchemaConstants.TALEND_IS_LOCKED, "false");
    field.addProp(SchemaConstants.TALEND_FIELD_GENERATED, "true");
    field.addProp(SchemaConstants.TALEND_COLUMN_DB_LENGTH, "255");
    additionalMainFields.add(field);

    Schema mainOutputSchema = newSchema(inputSchema, "output", additionalMainFields);
    schemaFlow.schema.setValue(mainOutputSchema);

    final List<Schema.Field> additionalRejectFields = new ArrayList<Schema.Field>();

    if (OutputAction.UPSERT.equals(outputAction.getValue()) && outputUpsertKey.getValue()
            && inputSchema.getField(upsertKeyColumn.getValue()) == null) {
        field = new Schema.Field("UpsertColumnValue", Schema.create(Schema.Type.STRING), null, (Object) null);
        field.addProp(SchemaConstants.TALEND_IS_LOCKED, "false");
        field.addProp(SchemaConstants.TALEND_FIELD_GENERATED, "true");
        field.addProp(SchemaConstants.TALEND_COLUMN_DB_LENGTH, "255");
        additionalRejectFields.add(field);
    }

    field = new Schema.Field("error", Schema.create(Schema.Type.STRING), null, (Object) null);
    field.addProp(SchemaConstants.TALEND_IS_LOCKED, "false");
    field.addProp(SchemaConstants.TALEND_FIELD_GENERATED, "true");
    field.addProp(SchemaConstants.TALEND_COLUMN_DB_LENGTH, "255");
    additionalRejectFields.add(field);

    Schema rejectSchema = newSchema(inputSchema, "rejectOutput", additionalRejectFields);
    schemaReject.schema.setValue(rejectSchema);
}
 
Example 13
Source Project: funcj   File: AvroSchemaCodecFormat.java    License: MIT License 4 votes vote down vote up
@Override
public Object encodePrim(double value, Object out) {
    return Schema.create(Schema.Type.DOUBLE);
}
 
Example 14
@Test
public void shouldWritePrimitives() {
    // given
    Schema javaLangStringSchema = Schema.create(Schema.Type.STRING);
    GenericData.setStringType(javaLangStringSchema, GenericData.StringType.String);
    Schema recordSchema = createRecord("testRecord",
            createField("testInt", Schema.create(Schema.Type.INT)),
            createPrimitiveUnionFieldSchema("testIntUnion", Schema.Type.INT),
            createField("testString", Schema.create(Schema.Type.STRING)),
            createPrimitiveUnionFieldSchema("testStringUnion", Schema.Type.STRING),
            createField("testJavaString", javaLangStringSchema),
            createUnionField("testJavaStringUnion", javaLangStringSchema),
            createField("testLong", Schema.create(Schema.Type.LONG)),
            createPrimitiveUnionFieldSchema("testLongUnion", Schema.Type.LONG),
            createField("testDouble", Schema.create(Schema.Type.DOUBLE)),
            createPrimitiveUnionFieldSchema("testDoubleUnion", Schema.Type.DOUBLE),
            createField("testFloat", Schema.create(Schema.Type.FLOAT)),
            createPrimitiveUnionFieldSchema("testFloatUnion", Schema.Type.FLOAT),
            createField("testBoolean", Schema.create(Schema.Type.BOOLEAN)),
            createPrimitiveUnionFieldSchema("testBooleanUnion", Schema.Type.BOOLEAN),
            createField("testBytes", Schema.create(Schema.Type.BYTES)),
            createPrimitiveUnionFieldSchema("testBytesUnion", Schema.Type.BYTES));

    GenericRecordBuilder builder = new GenericRecordBuilder(recordSchema);
    builder.set("testInt", 1);
    builder.set("testIntUnion", 1);
    builder.set("testString", "aaa");
    builder.set("testStringUnion", "aaa");
    builder.set("testJavaString", "aaa");
    builder.set("testJavaStringUnion", "aaa");
    builder.set("testLong", 1L);
    builder.set("testLongUnion", 1L);
    builder.set("testDouble", 1.0);
    builder.set("testDoubleUnion", 1.0);
    builder.set("testFloat", 1.0f);
    builder.set("testFloatUnion", 1.0f);
    builder.set("testBoolean", true);
    builder.set("testBooleanUnion", true);
    builder.set("testBytes", ByteBuffer.wrap(new byte[]{0x01, 0x02}));
    builder.set("testBytesUnion", ByteBuffer.wrap(new byte[]{0x01, 0x02}));

    // when
    GenericRecord record = deserializeGeneric(recordSchema, serializeGenericFast(builder.build()));

    // then
    Assert.assertEquals(1, record.get("testInt"));
    Assert.assertEquals(1, record.get("testIntUnion"));
    Assert.assertEquals("aaa", record.get("testString").toString());
    Assert.assertEquals("aaa", record.get("testStringUnion").toString());
    Assert.assertEquals("aaa", record.get("testJavaString"));
    Assert.assertEquals("aaa", record.get("testJavaStringUnion"));
    Assert.assertEquals(1L, record.get("testLong"));
    Assert.assertEquals(1L, record.get("testLongUnion"));
    Assert.assertEquals(1.0, record.get("testDouble"));
    Assert.assertEquals(1.0, record.get("testDoubleUnion"));
    Assert.assertEquals(1.0f, record.get("testFloat"));
    Assert.assertEquals(1.0f, record.get("testFloatUnion"));
    Assert.assertEquals(true, record.get("testBoolean"));
    Assert.assertEquals(true, record.get("testBooleanUnion"));
    Assert.assertEquals(ByteBuffer.wrap(new byte[]{0x01, 0x02}), record.get("testBytes"));
    Assert.assertEquals(ByteBuffer.wrap(new byte[]{0x01, 0x02}), record.get("testBytesUnion"));

}
 
Example 15
Source Project: incubator-gobblin   File: AvroFlattener.java    License: Apache License 2.0 4 votes vote down vote up
/***
 * Flatten the Schema to un-nest recursive Records (to make it optimal for ORC)
 * @param schema Schema to flatten
 * @param shouldPopulateLineage is set to true if the field is going to be flattened and moved up the hierarchy -
 *                              so that lineage information can be tagged to it; which happens when there is a
 *                              Record within a Record OR Record within Option within Record and so on,
 *                              however not when there is a Record within Map or Array
 * @param flattenComplexTypes Flatten complex types recursively other than Record and Option
 * @return Flattened Avro Schema
 */
private Schema flatten(Schema schema, boolean shouldPopulateLineage, boolean flattenComplexTypes) {
  Schema flattenedSchema;

  // Process all Schema Types
  // (Primitives are simply cloned)
  switch (schema.getType()) {
    case ARRAY:
      // Array might be an array of recursive Records, flatten them
      if (flattenComplexTypes) {
        flattenedSchema = Schema.createArray(flatten(schema.getElementType(), false));
      } else {
        flattenedSchema = Schema.createArray(schema.getElementType());
      }
      break;
    case BOOLEAN:
      flattenedSchema = Schema.create(schema.getType());
      break;
    case BYTES:
      flattenedSchema = Schema.create(schema.getType());
      break;
    case DOUBLE:
      flattenedSchema = Schema.create(schema.getType());
      break;
    case ENUM:
      flattenedSchema =
          Schema.createEnum(schema.getName(), schema.getDoc(), schema.getNamespace(), schema.getEnumSymbols());
      break;
    case FIXED:
      flattenedSchema =
          Schema.createFixed(schema.getName(), schema.getDoc(), schema.getNamespace(), schema.getFixedSize());
      break;
    case FLOAT:
      flattenedSchema = Schema.create(schema.getType());
      break;
    case INT:
      flattenedSchema = Schema.create(schema.getType());
      break;
    case LONG:
      flattenedSchema = Schema.create(schema.getType());
      break;
    case MAP:
      if (flattenComplexTypes) {
        flattenedSchema = Schema.createMap(flatten(schema.getValueType(), false));
      } else {
        flattenedSchema = Schema.createMap(schema.getValueType());
      }
      break;
    case NULL:
      flattenedSchema = Schema.create(schema.getType());
      break;
    case RECORD:
      flattenedSchema = flattenRecord(schema, shouldPopulateLineage, flattenComplexTypes);
      break;
    case STRING:
      flattenedSchema = Schema.create(schema.getType());
      break;
    case UNION:
      flattenedSchema = flattenUnion(schema, shouldPopulateLineage, flattenComplexTypes);
      break;
    default:
      String exceptionMessage = String.format("Schema flattening failed for \"%s\" ", schema);
      LOG.error(exceptionMessage);

      throw new AvroRuntimeException(exceptionMessage);
  }

  // Copy schema metadata
  copyProperties(schema, flattenedSchema);

  return flattenedSchema;
}
 
Example 16
@Override
protected Schema createStringType() {
  return Schema.create(STRING);
}
 
Example 17
Source Project: funcj   File: GenerateSchema.java    License: MIT License 4 votes vote down vote up
@Override
public Schema floatP(IList<String> path, String name) {
    return Schema.create(Schema.Type.FLOAT);
}
 
Example 18
Source Project: parquet-mr   File: AvroTestUtil.java    License: Apache License 2.0 4 votes vote down vote up
public static Schema primitive(Schema.Type type) {
  return Schema.create(type);
}
 
Example 19
Source Project: parquet-mr   File: AvroJson.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public Schema text(TextNode ignored) {
  return Schema.create(Schema.Type.STRING);
}
 
Example 20
public AvroMap(Map<Object, Object> map, Schema mapSchema) {
  _map = map;
  _keySchema = Schema.create(STRING);
  _valueSchema = mapSchema.getValueType();
}