Java Code Examples for org.apache.avro.Schema#addProp()

The following examples show how to use org.apache.avro.Schema#addProp() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source Project: pulsar File: GenericAvroSchema.java License: Apache License 2.0

6 votes

@Override
protected SchemaReader<GenericRecord> loadReader(BytesSchemaVersion schemaVersion) {
     SchemaInfo schemaInfo = getSchemaInfoByVersion(schemaVersion.get());
     if (schemaInfo != null) {
         log.info("Load schema reader for version({}), schema is : {}",
             SchemaUtils.getStringSchemaVersion(schemaVersion.get()),
             schemaInfo);
         Schema writerSchema = parseAvroSchema(schemaInfo.getSchemaDefinition());
         Schema readerSchema = useProvidedSchemaAsReaderSchema ? schema : writerSchema;
         readerSchema.addProp(OFFSET_PROP, schemaInfo.getProperties().getOrDefault(OFFSET_PROP, "0"));

         return new GenericAvroReader(
                 writerSchema,
                 readerSchema,
                 schemaVersion.get());
     } else {
         log.warn("No schema found for version({}), use latest schema : {}",
             SchemaUtils.getStringSchemaVersion(schemaVersion.get()),
             this.schemaInfo);
         return reader;
     }
}

Example 2

Source Project: components File: GoogleDriveDeleteProperties.java License: Apache License 2.0

6 votes

public void setupProperties() {
    super.setupProperties();

    Schema schema = SchemaBuilder.builder().record(GoogleDriveDeleteDefinition.COMPONENT_NAME).fields() //
            .name(GoogleDriveDeleteDefinition.RETURN_FILE_ID)//
            .prop(SchemaConstants.TALEND_IS_LOCKED, "true")//
            .type().nullable().stringType().noDefault() //
            .endRecord();
    schema.addProp(SchemaConstants.TALEND_IS_LOCKED, "true");
    schemaMain.schema.setValue(schema);

    deleteMode.setPossibleValues(AccessMethod.values());
    deleteMode.setValue(AccessMethod.Name);
    file.setValue("");
    useTrash.setValue(true);
}

Example 3

Source Project: parquet-mr File: TestGenericLogicalTypes.java License: Apache License 2.0

6 votes

@Test
public void testWriteNullableUUID() throws IOException {
  Schema nullableUuidSchema = record("R",
      optionalField("uuid", LogicalTypes.uuid().addToSchema(Schema.create(STRING))));
  GenericRecord u1 = instance(nullableUuidSchema, "uuid", UUID.randomUUID());
  GenericRecord u2 = instance(nullableUuidSchema, "uuid", null);

  Schema stringUuidSchema = Schema.create(STRING);
  stringUuidSchema.addProp(GenericData.STRING_PROP, "String");
  Schema nullableStringSchema = record("R", optionalField("uuid", stringUuidSchema));
  GenericRecord s1 = instance(nullableStringSchema, "uuid", u1.get("uuid").toString());
  GenericRecord s2 = instance(nullableStringSchema, "uuid", null);

  File test = write(GENERIC, nullableUuidSchema, u1, u2);
  Assert.assertEquals("Should read UUIDs as Strings",
      Arrays.asList(s1, s2), read(GENERIC, nullableStringSchema, test));
}

Example 4

Source Project: components File: MarketoConstants.java License: Apache License 2.0

5 votes

public static Schema scheduleCampaignSchema() {
    Schema s = SchemaBuilder.builder().record("scheduleCampaign").fields() //
            .name(FIELD_CAMPAIGN_ID)//
            .prop(SchemaConstants.TALEND_COLUMN_IS_KEY, "true")//
            .prop(SchemaConstants.TALEND_IS_LOCKED, "true")//
            .type().nullable().intType().noDefault() //
            .name(FIELD_STATUS).type().nullable().stringType().noDefault()//
            .endRecord();
    s.addProp(SchemaConstants.TALEND_IS_LOCKED, "true");
    return s;
}

Example 5

Source Project: incubator-gobblin File: CopyableSchemaTest.java License: Apache License 2.0

5 votes

@Test
public void testCopy() throws CopyNotSupportedException {
  Schema schema = new Schema.Parser().parse(AVRO_SCHEMA);
  CopyableSchema copyableSchema = new CopyableSchema(schema);
  Schema copy = copyableSchema.copy();
  Assert.assertEquals(schema, copy);
  copy.addProp("foo", "bar");
  Assert.assertNotEquals(schema, copy);
}

Example 6

Source Project: iceberg File: ParquetAvro.java License: Apache License 2.0

5 votes

private static Schema copyRecord(Schema record, List<Schema.Field> newFields) {
  Schema copy = Schema.createRecord(record.getName(),
      record.getDoc(), record.getNamespace(), record.isError(), newFields);

  for (Map.Entry<String, Object> prop : record.getObjectProps().entrySet()) {
    copy.addProp(prop.getKey(), prop.getValue());
  }

  return copy;
}

Example 7

Source Project: iceberg File: PruneColumns.java License: Apache License 2.0

5 votes

private Schema mapWithIds(Schema map, Integer keyId, Integer valueId) {
  if (!AvroSchemaUtil.hasProperty(map, AvroSchemaUtil.KEY_ID_PROP) ||
      !AvroSchemaUtil.hasProperty(map, AvroSchemaUtil.VALUE_ID_PROP)) {
    Schema result = Schema.createMap(map.getValueType());
    result.addProp(AvroSchemaUtil.KEY_ID_PROP, keyId);
    result.addProp(AvroSchemaUtil.VALUE_ID_PROP, valueId);
    return result;
  }
  return map;
}

Example 8

Source Project: incubator-gobblin File: AvroUtils.java License: Apache License 2.0

5 votes

/***
 * Copy properties to an Avro Schema
 * @param props Properties to copy to Avro Schema
 * @param schema Avro Schema to copy properties to
 */
private static void copyProperties(Map<String, JsonNode> props, Schema schema) {
  Preconditions.checkNotNull(schema);

  // (if null, don't copy but do not throw exception)
  if (null != props) {
    for (Map.Entry<String, JsonNode> prop : props.entrySet()) {
      schema.addProp(prop.getKey(), prop.getValue());
    }
  }
}

Example 9

Source Project: components File: TJiraInputProperties.java License: Apache License 2.0

5 votes

/**
 * Sets initial value of schema property
 */
void setupSchema() {
    Schema stringSchema = AvroUtils._string();

    // create Schema for JSON
    Schema.Field jsonField = new Schema.Field("json", stringSchema, null, (Object) null, Order.ASCENDING);
    Schema initialSchema = Schema.createRecord("jira", null, null, false, Collections.singletonList(jsonField));
    initialSchema.addProp(TALEND_IS_LOCKED, "true");

    schema.schema.setValue(initialSchema);
}

Example 10

Source Project: components File: MarketoConstants.java License: Apache License 2.0

5 votes

public static Schema triggerCampaignSchemaFlow() {
    Schema s = SchemaBuilder.builder().record("triggerCampaign").fields() //
            .name(FIELD_CAMPAIGN_ID)//
            .prop(SchemaConstants.TALEND_COLUMN_IS_KEY, "true")//
            .prop(SchemaConstants.TALEND_IS_LOCKED, "true")//
            .type().nullable().intType().noDefault() //
            .endRecord();
    s.addProp(SchemaConstants.TALEND_IS_LOCKED, "true");
    return s;
}

Example 11

Source Project: iceberg File: PruneColumns.java License: Apache License 2.0

5 votes

private static Schema copyRecord(Schema record, List<Schema.Field> newFields) {
  Schema copy = Schema.createRecord(record.getName(),
      record.getDoc(), record.getNamespace(), record.isError(), newFields);

  for (Map.Entry<String, Object> prop : record.getObjectProps().entrySet()) {
    copy.addProp(prop.getKey(), prop.getValue());
  }

  return copy;
}

Example 12

Source Project: incubator-gobblin File: AvroFlattener.java License: Apache License 2.0

5 votes

/***
 * Copy properties to an Avro Schema
 * @param props Properties to copy to Avro Schema
 * @param schema Avro Schema to copy properties to
 */
private static void copyProperties(Map<String, JsonNode> props, Schema schema) {
  Preconditions.checkNotNull(schema);

  // (if null, don't copy but do not throw exception)
  if (null != props) {
    for (Map.Entry<String, JsonNode> prop : props.entrySet()) {
      schema.addProp(prop.getKey(), prop.getValue());
    }
  }
}

Example 13

Source Project: data-highway File: LogicalTypeValidatingVisitorTest.java License: Apache License 2.0

4 votes

@Test
public void validLogicalType() throws Exception {
  Schema schema = SchemaBuilder.builder().intType();
  schema.addProp(LogicalType.LOGICAL_TYPE_PROP, "date");
  underTest.onVisit(schema, breadcrumb);
}

Example 14

Source Project: component-runtime File: AvroPropertyMapper.java License: Apache License 2.0

4 votes

default Schema setProp(final Schema schema, final String name, final String value) {
    schema.addProp("talend.component." + name, value);
    return schema;
}

Example 15

Source Project: components File: SObjectAdapterFactoryTest.java License: Apache License 2.0

4 votes

@Test
public void testConvertToAvroForAggregateResult() throws Exception {

    Schema schema = SchemaBuilder.builder().record("Schema").fields() //
            .name("Id").prop(SchemaConstants.TALEND_COLUMN_IS_KEY, "true").type().stringType().noDefault() //
            .name("Name").type(AvroUtils._string()).noDefault() //
            .name("Field_A").type(AvroUtils._int()).noDefault() //
            .name("Field_B").type(AvroUtils._boolean()).noDefault() //
            .name("Field_C").type(AvroUtils._date()).noDefault() //
            .name("Field_D").type(AvroUtils._double()).noDefault() //
            .name("Field_E_Value").type(AvroUtils._string()).noDefault() //
            .name("Field_F").type(AvroUtils._string()).noDefault() //
            .name("Field_G").type(AvroUtils._string()).noDefault() //
            .name("Field_H").type(AvroUtils._string()).noDefault() //
            .name("Field_I").type(AvroUtils._string()).noDefault() //
            .name("Field_J").type(AvroUtils._string()).noDefault() //
            .name("Field_K").type(AvroUtils._string()).noDefault() //
            .name("Field_L").type(AvroUtils._string()).noDefault() //
            .endRecord();
    schema.getField("Field_C").addProp(SchemaConstants.TALEND_COLUMN_PATTERN,
            "yyyy-MM-dd'T'HH:mm:ss'.000Z'");
    schema.addProp(SalesforceSchemaConstants.COLUMNNAME_DELIMTER, "_");
    schema.addProp(SalesforceSchemaConstants.VALUE_DELIMITER, "|");

    converter.setSchema(schema);

    SObject sObject = new SObject("AggregateResult");

    sObject.addField("Id", "12345");
    sObject.addField("Name", "Qwerty");
    sObject.addField("Field_A", "42");
    sObject.addField("Field_B", "true");
    sObject.addField("Field_C", dateFormat.parse("2017-06-15T18:26:34.000Z"));
    sObject.addField("Field_D", 10200.45);

    SObject sObject1 = new SObject();
    sObject1.addField("Value", 245);
    sObject.addField("Field_E", sObject1);
    SObject sObject2 = new SObject();
    sObject2.addField("Value", 542);
    sObject.addField("Field_E", sObject2);

    sObject.addField("Field_F", new BigDecimal("20000000000000000000000000.123456789"));
    sObject.addField("Field_G", 899.5f);
    sObject.addField("Field_H", Boolean.TRUE);
    sObject.addField("Field_I", new byte[]{0x0a, 0x0b, 0x0c, 0x0d});
    sObject.addField("Field_J", 102030405060708090L);
    sObject.addField("Field_K", SalesforceRuntime.convertDateToCalendar(
            dateFormat.parse("2017-06-16T10:45:02.000Z"),false));
    sObject.addField("Field_L", null);

    IndexedRecord indexedRecord = converter.convertToAvro(sObject);
    assertNotNull(indexedRecord);
    assertNotNull(indexedRecord.getSchema());
    assertEquals(schema, indexedRecord.getSchema());

    assertEquals("12345", indexedRecord.get(0));
    assertEquals("Qwerty", indexedRecord.get(1));
    assertEquals(Integer.valueOf(42), indexedRecord.get(2));
    assertEquals(Boolean.TRUE, indexedRecord.get(3));
    assertEquals(dateFormat.parse("2017-06-15T18:26:34.000Z").getTime(), indexedRecord.get(4));
    assertEquals(Double.valueOf(10200.45), indexedRecord.get(5));
    assertEquals("245|542", indexedRecord.get(6));
    assertEquals("20000000000000000000000000.123456789", indexedRecord.get(7));
    assertEquals("899.5", indexedRecord.get(8));
    assertEquals("true", indexedRecord.get(9));
    assertEquals("CgsMDQ==", indexedRecord.get(10));
    assertEquals("102030405060708090", indexedRecord.get(11));
    assertEquals("2017-06-16T07:45:02.000Z", indexedRecord.get(12));
    assertNull(indexedRecord.get(13));
}

Example 16

Source Project: iceberg File: AvroTestHelpers.java License: Apache License 2.0

4 votes

static Schema addValueId(int id, Schema schema) {
  schema.addProp(AvroSchemaUtil.VALUE_ID_PROP, id);
  return schema;
}

Example 17

Source Project: iceberg File: TestSchemaConversions.java License: Apache License 2.0

4 votes

private Schema addAdjustToUtc(Schema schema, boolean adjustToUTC) {
  schema.addProp(AvroSchemaUtil.ADJUST_TO_UTC_PROP, adjustToUTC);
  return schema;
}

Example 18

Source Project: datacollector File: AvroHiveSchemaGenerator.java License: Apache License 2.0

4 votes

private static Schema traverse(Map.Entry<String, HiveTypeInfo> node)
    throws StageException {
  switch(node.getValue().getHiveType()){
    case STRING:
      return Schema.create(Schema.Type.STRING);

    case BOOLEAN:
      return Schema.create(Schema.Type.BOOLEAN);

    case INT:
      return Schema.create(Schema.Type.INT);

    case BIGINT:
      return Schema.create(Schema.Type.LONG);

    case FLOAT:
      return Schema.create(Schema.Type.FLOAT);

    case DOUBLE:
      return Schema.create(Schema.Type.DOUBLE);

    case BINARY:
      return Schema.create(Schema.Type.BYTES);

    case TIMESTAMP:
      Schema timestampSchema = Schema.create(Schema.Type.LONG);
      timestampSchema.addProp(AvroTypeUtil.LOGICAL_TYPE, AvroTypeUtil.LOGICAL_TYPE_TIMESTAMP_MILLIS);
      return timestampSchema;

    case DATE:
      Schema dateSchema = Schema.create(Schema.Type.INT);
      dateSchema.addProp(AvroTypeUtil.LOGICAL_TYPE, AvroTypeUtil.LOGICAL_TYPE_DATE);
      return dateSchema;

    case DECIMAL:
      Utils.checkArgument(node.getValue() instanceof DecimalTypeInfo, "Invalid type used in HiveTypeInfo");
      DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo)node.getValue();
      Schema schema = Schema.create(Schema.Type.BYTES);
      schema.addProp(AvroTypeUtil.LOGICAL_TYPE, AvroTypeUtil.LOGICAL_TYPE_DECIMAL);
      schema.addProp(AvroTypeUtil.LOGICAL_TYPE_ATTR_PRECISION, getJsonNode(decimalTypeInfo.getPrecision()));
      schema.addProp(AvroTypeUtil.LOGICAL_TYPE_ATTR_SCALE, getJsonNode(decimalTypeInfo.getScale()));
      return schema;

    default:
      // Accessing Unsupported Type (Map, Union, Array, Enum, NULL, Record)
      throw new StageException(Errors.HIVE_24, node.getValue().getHiveType());
  }
}

Example 19

Source Project: components File: SObjectAdapterFactoryTest.java License: Apache License 2.0

4 votes

@Test
public void testConvertToAvroForNestedObjects() throws Exception {

    Schema schema = SchemaBuilder.builder().record("Schema").fields() //
            .name("Id").prop(SchemaConstants.TALEND_COLUMN_IS_KEY, "true").type().stringType().noDefault() //
            .name("Name").type(AvroUtils._string()).noDefault() //
            .name("FieldA").type(AvroUtils._int()).noDefault() //
            .name("Member_Name").type(AvroUtils._string()).noDefault() //
            .name("Member_FieldA").type(AvroUtils._string()).noDefault() //
            .name("Member_Contact_Name").type(AvroUtils._string()).noDefault() //
            .name("Member_Contact_FieldA").type(AvroUtils._string()).noDefault() //
            .endRecord();
    schema.addProp(SalesforceSchemaConstants.COLUMNNAME_DELIMTER, "_");
    schema.addProp(SalesforceSchemaConstants.VALUE_DELIMITER, "|");

    converter.setSchema(schema);

    SObject sObject = new SObject("Campaign");
    sObject.addField("Id", "12345");
    sObject.addField("Name", "Qwerty");

    // Intentionally add another value to check that it is skipped by converter
    sObject.addField("FieldA", 300);
    sObject.addField("FieldA", 400);

    SObject sObject1 = new SObject("CampaignMember");
    sObject1.addField("Id", "10001");
    sObject1.addField("Name", "Member A");
    sObject1.addField("FieldA", 10);
    sObject.addField("Member", sObject1);

    SObject sObject11 = new SObject("Contact");
    sObject11.addField("Id", "20001");
    sObject11.addField("Name", "Contact A");
    sObject11.addField("FieldA", "foo");
    sObject1.addField("Contact", sObject11);

    SObject sObject2 = new SObject("CampaignMember");
    sObject2.addField("Id", "10002");
    sObject2.addField("Name", "Member B");
    sObject2.addField("FieldA", 20);
    sObject.addField("Member", sObject2);

    SObject sObject21 = new SObject("Contact");
    sObject21.addField("Id", "20002");
    sObject21.addField("Name", "Contact B");
    sObject21.addField("FieldA", "bar");
    sObject2.addField("Contact", sObject21);

    IndexedRecord indexedRecord = converter.convertToAvro(sObject);
    assertNotNull(indexedRecord);
    assertNotNull(indexedRecord.getSchema());
    assertEquals(schema, indexedRecord.getSchema());

    assertEquals("12345", indexedRecord.get(0));
    assertEquals("Qwerty", indexedRecord.get(1));
    assertEquals(Integer.valueOf(300), indexedRecord.get(2));
    assertEquals("Member A|Member B", indexedRecord.get(3));
    assertEquals("10|20", indexedRecord.get(4));
    assertEquals("Contact A|Contact B", indexedRecord.get(5));
    assertEquals("foo|bar", indexedRecord.get(6));
}

Example 20

Source Project: incubator-gobblin File: AvroUtils.java License: Apache License 2.0

4 votes

public static Schema addSchemaCreationTime(Schema inputSchema, Schema outputSchema) {
  if (inputSchema.getProp(SCHEMA_CREATION_TIME_KEY) != null && outputSchema.getProp(SCHEMA_CREATION_TIME_KEY) == null) {
    outputSchema.addProp(SCHEMA_CREATION_TIME_KEY, inputSchema.getProp(SCHEMA_CREATION_TIME_KEY));
  }
  return outputSchema;
}