Java Code Examples for org.apache.avro.Schema#addProp()

The following examples show how to use org.apache.avro.Schema#addProp() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: TestGenericLogicalTypes.java From parquet-mr with Apache License 2.0

6 votes

@Test
public void testWriteNullableUUID() throws IOException {
  Schema nullableUuidSchema = record("R",
      optionalField("uuid", LogicalTypes.uuid().addToSchema(Schema.create(STRING))));
  GenericRecord u1 = instance(nullableUuidSchema, "uuid", UUID.randomUUID());
  GenericRecord u2 = instance(nullableUuidSchema, "uuid", null);

  Schema stringUuidSchema = Schema.create(STRING);
  stringUuidSchema.addProp(GenericData.STRING_PROP, "String");
  Schema nullableStringSchema = record("R", optionalField("uuid", stringUuidSchema));
  GenericRecord s1 = instance(nullableStringSchema, "uuid", u1.get("uuid").toString());
  GenericRecord s2 = instance(nullableStringSchema, "uuid", null);

  File test = write(GENERIC, nullableUuidSchema, u1, u2);
  Assert.assertEquals("Should read UUIDs as Strings",
      Arrays.asList(s1, s2), read(GENERIC, nullableStringSchema, test));
}

Example 2

Source File: GoogleDriveDeleteProperties.java From components with Apache License 2.0

6 votes

public void setupProperties() {
    super.setupProperties();

    Schema schema = SchemaBuilder.builder().record(GoogleDriveDeleteDefinition.COMPONENT_NAME).fields() //
            .name(GoogleDriveDeleteDefinition.RETURN_FILE_ID)//
            .prop(SchemaConstants.TALEND_IS_LOCKED, "true")//
            .type().nullable().stringType().noDefault() //
            .endRecord();
    schema.addProp(SchemaConstants.TALEND_IS_LOCKED, "true");
    schemaMain.schema.setValue(schema);

    deleteMode.setPossibleValues(AccessMethod.values());
    deleteMode.setValue(AccessMethod.Name);
    file.setValue("");
    useTrash.setValue(true);
}

Example 3

Source File: GenericAvroSchema.java From pulsar with Apache License 2.0

6 votes

@Override
protected SchemaReader<GenericRecord> loadReader(BytesSchemaVersion schemaVersion) {
     SchemaInfo schemaInfo = getSchemaInfoByVersion(schemaVersion.get());
     if (schemaInfo != null) {
         log.info("Load schema reader for version({}), schema is : {}",
             SchemaUtils.getStringSchemaVersion(schemaVersion.get()),
             schemaInfo);
         Schema writerSchema = parseAvroSchema(schemaInfo.getSchemaDefinition());
         Schema readerSchema = useProvidedSchemaAsReaderSchema ? schema : writerSchema;
         readerSchema.addProp(OFFSET_PROP, schemaInfo.getProperties().getOrDefault(OFFSET_PROP, "0"));

         return new GenericAvroReader(
                 writerSchema,
                 readerSchema,
                 schemaVersion.get());
     } else {
         log.warn("No schema found for version({}), use latest schema : {}",
             SchemaUtils.getStringSchemaVersion(schemaVersion.get()),
             this.schemaInfo);
         return reader;
     }
}

Example 4

Source File: CopyableSchemaTest.java From incubator-gobblin with Apache License 2.0

5 votes

@Test
public void testCopy() throws CopyNotSupportedException {
  Schema schema = new Schema.Parser().parse(AVRO_SCHEMA);
  CopyableSchema copyableSchema = new CopyableSchema(schema);
  Schema copy = copyableSchema.copy();
  Assert.assertEquals(schema, copy);
  copy.addProp("foo", "bar");
  Assert.assertNotEquals(schema, copy);
}

Example 5

Source File: ParquetAvro.java From iceberg with Apache License 2.0

5 votes

private static Schema copyRecord(Schema record, List<Schema.Field> newFields) {
  Schema copy = Schema.createRecord(record.getName(),
      record.getDoc(), record.getNamespace(), record.isError(), newFields);

  for (Map.Entry<String, Object> prop : record.getObjectProps().entrySet()) {
    copy.addProp(prop.getKey(), prop.getValue());
  }

  return copy;
}

Example 6

Source File: AvroFlattener.java From incubator-gobblin with Apache License 2.0

5 votes

/***
 * Copy properties to an Avro Schema
 * @param props Properties to copy to Avro Schema
 * @param schema Avro Schema to copy properties to
 */
private static void copyProperties(Map<String, JsonNode> props, Schema schema) {
  Preconditions.checkNotNull(schema);

  // (if null, don't copy but do not throw exception)
  if (null != props) {
    for (Map.Entry<String, JsonNode> prop : props.entrySet()) {
      schema.addProp(prop.getKey(), prop.getValue());
    }
  }
}

Example 7

Source File: PruneColumns.java From iceberg with Apache License 2.0

5 votes

private Schema mapWithIds(Schema map, Integer keyId, Integer valueId) {
  if (!AvroSchemaUtil.hasProperty(map, AvroSchemaUtil.KEY_ID_PROP) ||
      !AvroSchemaUtil.hasProperty(map, AvroSchemaUtil.VALUE_ID_PROP)) {
    Schema result = Schema.createMap(map.getValueType());
    result.addProp(AvroSchemaUtil.KEY_ID_PROP, keyId);
    result.addProp(AvroSchemaUtil.VALUE_ID_PROP, valueId);
    return result;
  }
  return map;
}

Example 8

Source File: AvroUtils.java From incubator-gobblin with Apache License 2.0

5 votes

/***
 * Copy properties to an Avro Schema
 * @param props Properties to copy to Avro Schema
 * @param schema Avro Schema to copy properties to
 */
private static void copyProperties(Map<String, JsonNode> props, Schema schema) {
  Preconditions.checkNotNull(schema);

  // (if null, don't copy but do not throw exception)
  if (null != props) {
    for (Map.Entry<String, JsonNode> prop : props.entrySet()) {
      schema.addProp(prop.getKey(), prop.getValue());
    }
  }
}

Example 9

Source File: TJiraInputProperties.java From components with Apache License 2.0

5 votes

/**
 * Sets initial value of schema property
 */
void setupSchema() {
    Schema stringSchema = AvroUtils._string();

    // create Schema for JSON
    Schema.Field jsonField = new Schema.Field("json", stringSchema, null, (Object) null, Order.ASCENDING);
    Schema initialSchema = Schema.createRecord("jira", null, null, false, Collections.singletonList(jsonField));
    initialSchema.addProp(TALEND_IS_LOCKED, "true");

    schema.schema.setValue(initialSchema);
}

Example 10

Source File: MarketoConstants.java From components with Apache License 2.0

5 votes

public static Schema triggerCampaignSchemaFlow() {
    Schema s = SchemaBuilder.builder().record("triggerCampaign").fields() //
            .name(FIELD_CAMPAIGN_ID)//
            .prop(SchemaConstants.TALEND_COLUMN_IS_KEY, "true")//
            .prop(SchemaConstants.TALEND_IS_LOCKED, "true")//
            .type().nullable().intType().noDefault() //
            .endRecord();
    s.addProp(SchemaConstants.TALEND_IS_LOCKED, "true");
    return s;
}

Example 11

Source File: MarketoConstants.java From components with Apache License 2.0

5 votes

public static Schema scheduleCampaignSchema() {
    Schema s = SchemaBuilder.builder().record("scheduleCampaign").fields() //
            .name(FIELD_CAMPAIGN_ID)//
            .prop(SchemaConstants.TALEND_COLUMN_IS_KEY, "true")//
            .prop(SchemaConstants.TALEND_IS_LOCKED, "true")//
            .type().nullable().intType().noDefault() //
            .name(FIELD_STATUS).type().nullable().stringType().noDefault()//
            .endRecord();
    s.addProp(SchemaConstants.TALEND_IS_LOCKED, "true");
    return s;
}

Example 12

Source File: PruneColumns.java From iceberg with Apache License 2.0

5 votes

private static Schema copyRecord(Schema record, List<Schema.Field> newFields) {
  Schema copy = Schema.createRecord(record.getName(),
      record.getDoc(), record.getNamespace(), record.isError(), newFields);

  for (Map.Entry<String, Object> prop : record.getObjectProps().entrySet()) {
    copy.addProp(prop.getKey(), prop.getValue());
  }

  return copy;
}

Example 13

Source File: AvroTestHelpers.java From iceberg with Apache License 2.0

4 votes

static Schema addValueId(int id, Schema schema) {
  schema.addProp(AvroSchemaUtil.VALUE_ID_PROP, id);
  return schema;
}

Example 14

Source File: AvroUtils.java From incubator-gobblin with Apache License 2.0

4 votes

public static Schema addSchemaCreationTime(Schema inputSchema, Schema outputSchema) {
  if (inputSchema.getProp(SCHEMA_CREATION_TIME_KEY) != null && outputSchema.getProp(SCHEMA_CREATION_TIME_KEY) == null) {
    outputSchema.addProp(SCHEMA_CREATION_TIME_KEY, inputSchema.getProp(SCHEMA_CREATION_TIME_KEY));
  }
  return outputSchema;
}

Example 15

Source File: SObjectAdapterFactoryTest.java From components with Apache License 2.0

4 votes

@Test
public void testConvertToAvroForNestedObjects() throws Exception {

    Schema schema = SchemaBuilder.builder().record("Schema").fields() //
            .name("Id").prop(SchemaConstants.TALEND_COLUMN_IS_KEY, "true").type().stringType().noDefault() //
            .name("Name").type(AvroUtils._string()).noDefault() //
            .name("FieldA").type(AvroUtils._int()).noDefault() //
            .name("Member_Name").type(AvroUtils._string()).noDefault() //
            .name("Member_FieldA").type(AvroUtils._string()).noDefault() //
            .name("Member_Contact_Name").type(AvroUtils._string()).noDefault() //
            .name("Member_Contact_FieldA").type(AvroUtils._string()).noDefault() //
            .endRecord();
    schema.addProp(SalesforceSchemaConstants.COLUMNNAME_DELIMTER, "_");
    schema.addProp(SalesforceSchemaConstants.VALUE_DELIMITER, "|");

    converter.setSchema(schema);

    SObject sObject = new SObject("Campaign");
    sObject.addField("Id", "12345");
    sObject.addField("Name", "Qwerty");

    // Intentionally add another value to check that it is skipped by converter
    sObject.addField("FieldA", 300);
    sObject.addField("FieldA", 400);

    SObject sObject1 = new SObject("CampaignMember");
    sObject1.addField("Id", "10001");
    sObject1.addField("Name", "Member A");
    sObject1.addField("FieldA", 10);
    sObject.addField("Member", sObject1);

    SObject sObject11 = new SObject("Contact");
    sObject11.addField("Id", "20001");
    sObject11.addField("Name", "Contact A");
    sObject11.addField("FieldA", "foo");
    sObject1.addField("Contact", sObject11);

    SObject sObject2 = new SObject("CampaignMember");
    sObject2.addField("Id", "10002");
    sObject2.addField("Name", "Member B");
    sObject2.addField("FieldA", 20);
    sObject.addField("Member", sObject2);

    SObject sObject21 = new SObject("Contact");
    sObject21.addField("Id", "20002");
    sObject21.addField("Name", "Contact B");
    sObject21.addField("FieldA", "bar");
    sObject2.addField("Contact", sObject21);

    IndexedRecord indexedRecord = converter.convertToAvro(sObject);
    assertNotNull(indexedRecord);
    assertNotNull(indexedRecord.getSchema());
    assertEquals(schema, indexedRecord.getSchema());

    assertEquals("12345", indexedRecord.get(0));
    assertEquals("Qwerty", indexedRecord.get(1));
    assertEquals(Integer.valueOf(300), indexedRecord.get(2));
    assertEquals("Member A|Member B", indexedRecord.get(3));
    assertEquals("10|20", indexedRecord.get(4));
    assertEquals("Contact A|Contact B", indexedRecord.get(5));
    assertEquals("foo|bar", indexedRecord.get(6));
}

Example 16

Source File: AvroHiveSchemaGenerator.java From datacollector with Apache License 2.0

4 votes

private static Schema traverse(Map.Entry<String, HiveTypeInfo> node)
    throws StageException {
  switch(node.getValue().getHiveType()){
    case STRING:
      return Schema.create(Schema.Type.STRING);

    case BOOLEAN:
      return Schema.create(Schema.Type.BOOLEAN);

    case INT:
      return Schema.create(Schema.Type.INT);

    case BIGINT:
      return Schema.create(Schema.Type.LONG);

    case FLOAT:
      return Schema.create(Schema.Type.FLOAT);

    case DOUBLE:
      return Schema.create(Schema.Type.DOUBLE);

    case BINARY:
      return Schema.create(Schema.Type.BYTES);

    case TIMESTAMP:
      Schema timestampSchema = Schema.create(Schema.Type.LONG);
      timestampSchema.addProp(AvroTypeUtil.LOGICAL_TYPE, AvroTypeUtil.LOGICAL_TYPE_TIMESTAMP_MILLIS);
      return timestampSchema;

    case DATE:
      Schema dateSchema = Schema.create(Schema.Type.INT);
      dateSchema.addProp(AvroTypeUtil.LOGICAL_TYPE, AvroTypeUtil.LOGICAL_TYPE_DATE);
      return dateSchema;

    case DECIMAL:
      Utils.checkArgument(node.getValue() instanceof DecimalTypeInfo, "Invalid type used in HiveTypeInfo");
      DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo)node.getValue();
      Schema schema = Schema.create(Schema.Type.BYTES);
      schema.addProp(AvroTypeUtil.LOGICAL_TYPE, AvroTypeUtil.LOGICAL_TYPE_DECIMAL);
      schema.addProp(AvroTypeUtil.LOGICAL_TYPE_ATTR_PRECISION, getJsonNode(decimalTypeInfo.getPrecision()));
      schema.addProp(AvroTypeUtil.LOGICAL_TYPE_ATTR_SCALE, getJsonNode(decimalTypeInfo.getScale()));
      return schema;

    default:
      // Accessing Unsupported Type (Map, Union, Array, Enum, NULL, Record)
      throw new StageException(Errors.HIVE_24, node.getValue().getHiveType());
  }
}

Example 17

Source File: TestSchemaConversions.java From iceberg with Apache License 2.0

4 votes

private Schema addAdjustToUtc(Schema schema, boolean adjustToUTC) {
  schema.addProp(AvroSchemaUtil.ADJUST_TO_UTC_PROP, adjustToUTC);
  return schema;
}

Example 18

Source File: SObjectAdapterFactoryTest.java From components with Apache License 2.0

4 votes

@Test
public void testConvertToAvroForAggregateResult() throws Exception {

    Schema schema = SchemaBuilder.builder().record("Schema").fields() //
            .name("Id").prop(SchemaConstants.TALEND_COLUMN_IS_KEY, "true").type().stringType().noDefault() //
            .name("Name").type(AvroUtils._string()).noDefault() //
            .name("Field_A").type(AvroUtils._int()).noDefault() //
            .name("Field_B").type(AvroUtils._boolean()).noDefault() //
            .name("Field_C").type(AvroUtils._date()).noDefault() //
            .name("Field_D").type(AvroUtils._double()).noDefault() //
            .name("Field_E_Value").type(AvroUtils._string()).noDefault() //
            .name("Field_F").type(AvroUtils._string()).noDefault() //
            .name("Field_G").type(AvroUtils._string()).noDefault() //
            .name("Field_H").type(AvroUtils._string()).noDefault() //
            .name("Field_I").type(AvroUtils._string()).noDefault() //
            .name("Field_J").type(AvroUtils._string()).noDefault() //
            .name("Field_K").type(AvroUtils._string()).noDefault() //
            .name("Field_L").type(AvroUtils._string()).noDefault() //
            .endRecord();
    schema.getField("Field_C").addProp(SchemaConstants.TALEND_COLUMN_PATTERN,
            "yyyy-MM-dd'T'HH:mm:ss'.000Z'");
    schema.addProp(SalesforceSchemaConstants.COLUMNNAME_DELIMTER, "_");
    schema.addProp(SalesforceSchemaConstants.VALUE_DELIMITER, "|");

    converter.setSchema(schema);

    SObject sObject = new SObject("AggregateResult");

    sObject.addField("Id", "12345");
    sObject.addField("Name", "Qwerty");
    sObject.addField("Field_A", "42");
    sObject.addField("Field_B", "true");
    sObject.addField("Field_C", dateFormat.parse("2017-06-15T18:26:34.000Z"));
    sObject.addField("Field_D", 10200.45);

    SObject sObject1 = new SObject();
    sObject1.addField("Value", 245);
    sObject.addField("Field_E", sObject1);
    SObject sObject2 = new SObject();
    sObject2.addField("Value", 542);
    sObject.addField("Field_E", sObject2);

    sObject.addField("Field_F", new BigDecimal("20000000000000000000000000.123456789"));
    sObject.addField("Field_G", 899.5f);
    sObject.addField("Field_H", Boolean.TRUE);
    sObject.addField("Field_I", new byte[]{0x0a, 0x0b, 0x0c, 0x0d});
    sObject.addField("Field_J", 102030405060708090L);
    sObject.addField("Field_K", SalesforceRuntime.convertDateToCalendar(
            dateFormat.parse("2017-06-16T10:45:02.000Z"),false));
    sObject.addField("Field_L", null);

    IndexedRecord indexedRecord = converter.convertToAvro(sObject);
    assertNotNull(indexedRecord);
    assertNotNull(indexedRecord.getSchema());
    assertEquals(schema, indexedRecord.getSchema());

    assertEquals("12345", indexedRecord.get(0));
    assertEquals("Qwerty", indexedRecord.get(1));
    assertEquals(Integer.valueOf(42), indexedRecord.get(2));
    assertEquals(Boolean.TRUE, indexedRecord.get(3));
    assertEquals(dateFormat.parse("2017-06-15T18:26:34.000Z").getTime(), indexedRecord.get(4));
    assertEquals(Double.valueOf(10200.45), indexedRecord.get(5));
    assertEquals("245|542", indexedRecord.get(6));
    assertEquals("20000000000000000000000000.123456789", indexedRecord.get(7));
    assertEquals("899.5", indexedRecord.get(8));
    assertEquals("true", indexedRecord.get(9));
    assertEquals("CgsMDQ==", indexedRecord.get(10));
    assertEquals("102030405060708090", indexedRecord.get(11));
    assertEquals("2017-06-16T07:45:02.000Z", indexedRecord.get(12));
    assertNull(indexedRecord.get(13));
}

Example 19

Source File: AvroPropertyMapper.java From component-runtime with Apache License 2.0

4 votes

default Schema setProp(final Schema schema, final String name, final String value) {
    schema.addProp("talend.component." + name, value);
    return schema;
}

Example 20

Source File: LogicalTypeValidatingVisitorTest.java From data-highway with Apache License 2.0

4 votes

@Test
public void validLogicalType() throws Exception {
  Schema schema = SchemaBuilder.builder().intType();
  schema.addProp(LogicalType.LOGICAL_TYPE_PROP, "date");
  underTest.onVisit(schema, breadcrumb);
}