Java Code Examples for org.apache.avro.Schema#addProp()

The following examples show how to use org.apache.avro.Schema#addProp() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: pulsar   File: GenericAvroSchema.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected SchemaReader<GenericRecord> loadReader(BytesSchemaVersion schemaVersion) {
     SchemaInfo schemaInfo = getSchemaInfoByVersion(schemaVersion.get());
     if (schemaInfo != null) {
         log.info("Load schema reader for version({}), schema is : {}",
             SchemaUtils.getStringSchemaVersion(schemaVersion.get()),
             schemaInfo);
         Schema writerSchema = parseAvroSchema(schemaInfo.getSchemaDefinition());
         Schema readerSchema = useProvidedSchemaAsReaderSchema ? schema : writerSchema;
         readerSchema.addProp(OFFSET_PROP, schemaInfo.getProperties().getOrDefault(OFFSET_PROP, "0"));

         return new GenericAvroReader(
                 writerSchema,
                 readerSchema,
                 schemaVersion.get());
     } else {
         log.warn("No schema found for version({}), use latest schema : {}",
             SchemaUtils.getStringSchemaVersion(schemaVersion.get()),
             this.schemaInfo);
         return reader;
     }
}
 
Example 2
public void setupProperties() {
    super.setupProperties();

    Schema schema = SchemaBuilder.builder().record(GoogleDriveDeleteDefinition.COMPONENT_NAME).fields() //
            .name(GoogleDriveDeleteDefinition.RETURN_FILE_ID)//
            .prop(SchemaConstants.TALEND_IS_LOCKED, "true")//
            .type().nullable().stringType().noDefault() //
            .endRecord();
    schema.addProp(SchemaConstants.TALEND_IS_LOCKED, "true");
    schemaMain.schema.setValue(schema);

    deleteMode.setPossibleValues(AccessMethod.values());
    deleteMode.setValue(AccessMethod.Name);
    file.setValue("");
    useTrash.setValue(true);
}
 
Example 3
@Test
public void testWriteNullableUUID() throws IOException {
  Schema nullableUuidSchema = record("R",
      optionalField("uuid", LogicalTypes.uuid().addToSchema(Schema.create(STRING))));
  GenericRecord u1 = instance(nullableUuidSchema, "uuid", UUID.randomUUID());
  GenericRecord u2 = instance(nullableUuidSchema, "uuid", null);

  Schema stringUuidSchema = Schema.create(STRING);
  stringUuidSchema.addProp(GenericData.STRING_PROP, "String");
  Schema nullableStringSchema = record("R", optionalField("uuid", stringUuidSchema));
  GenericRecord s1 = instance(nullableStringSchema, "uuid", u1.get("uuid").toString());
  GenericRecord s2 = instance(nullableStringSchema, "uuid", null);

  File test = write(GENERIC, nullableUuidSchema, u1, u2);
  Assert.assertEquals("Should read UUIDs as Strings",
      Arrays.asList(s1, s2), read(GENERIC, nullableStringSchema, test));
}
 
Example 4
Source Project: components   File: MarketoConstants.java    License: Apache License 2.0 5 votes vote down vote up
public static Schema scheduleCampaignSchema() {
    Schema s = SchemaBuilder.builder().record("scheduleCampaign").fields() //
            .name(FIELD_CAMPAIGN_ID)//
            .prop(SchemaConstants.TALEND_COLUMN_IS_KEY, "true")//
            .prop(SchemaConstants.TALEND_IS_LOCKED, "true")//
            .type().nullable().intType().noDefault() //
            .name(FIELD_STATUS).type().nullable().stringType().noDefault()//
            .endRecord();
    s.addProp(SchemaConstants.TALEND_IS_LOCKED, "true");
    return s;
}
 
Example 5
@Test
public void testCopy() throws CopyNotSupportedException {
  Schema schema = new Schema.Parser().parse(AVRO_SCHEMA);
  CopyableSchema copyableSchema = new CopyableSchema(schema);
  Schema copy = copyableSchema.copy();
  Assert.assertEquals(schema, copy);
  copy.addProp("foo", "bar");
  Assert.assertNotEquals(schema, copy);
}
 
Example 6
Source Project: iceberg   File: ParquetAvro.java    License: Apache License 2.0 5 votes vote down vote up
private static Schema copyRecord(Schema record, List<Schema.Field> newFields) {
  Schema copy = Schema.createRecord(record.getName(),
      record.getDoc(), record.getNamespace(), record.isError(), newFields);

  for (Map.Entry<String, Object> prop : record.getObjectProps().entrySet()) {
    copy.addProp(prop.getKey(), prop.getValue());
  }

  return copy;
}
 
Example 7
Source Project: iceberg   File: PruneColumns.java    License: Apache License 2.0 5 votes vote down vote up
private Schema mapWithIds(Schema map, Integer keyId, Integer valueId) {
  if (!AvroSchemaUtil.hasProperty(map, AvroSchemaUtil.KEY_ID_PROP) ||
      !AvroSchemaUtil.hasProperty(map, AvroSchemaUtil.VALUE_ID_PROP)) {
    Schema result = Schema.createMap(map.getValueType());
    result.addProp(AvroSchemaUtil.KEY_ID_PROP, keyId);
    result.addProp(AvroSchemaUtil.VALUE_ID_PROP, valueId);
    return result;
  }
  return map;
}
 
Example 8
Source Project: incubator-gobblin   File: AvroUtils.java    License: Apache License 2.0 5 votes vote down vote up
/***
 * Copy properties to an Avro Schema
 * @param props Properties to copy to Avro Schema
 * @param schema Avro Schema to copy properties to
 */
private static void copyProperties(Map<String, JsonNode> props, Schema schema) {
  Preconditions.checkNotNull(schema);

  // (if null, don't copy but do not throw exception)
  if (null != props) {
    for (Map.Entry<String, JsonNode> prop : props.entrySet()) {
      schema.addProp(prop.getKey(), prop.getValue());
    }
  }
}
 
Example 9
Source Project: components   File: TJiraInputProperties.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Sets initial value of schema property
 */
void setupSchema() {
    Schema stringSchema = AvroUtils._string();

    // create Schema for JSON
    Schema.Field jsonField = new Schema.Field("json", stringSchema, null, (Object) null, Order.ASCENDING);
    Schema initialSchema = Schema.createRecord("jira", null, null, false, Collections.singletonList(jsonField));
    initialSchema.addProp(TALEND_IS_LOCKED, "true");

    schema.schema.setValue(initialSchema);
}
 
Example 10
Source Project: components   File: MarketoConstants.java    License: Apache License 2.0 5 votes vote down vote up
public static Schema triggerCampaignSchemaFlow() {
    Schema s = SchemaBuilder.builder().record("triggerCampaign").fields() //
            .name(FIELD_CAMPAIGN_ID)//
            .prop(SchemaConstants.TALEND_COLUMN_IS_KEY, "true")//
            .prop(SchemaConstants.TALEND_IS_LOCKED, "true")//
            .type().nullable().intType().noDefault() //
            .endRecord();
    s.addProp(SchemaConstants.TALEND_IS_LOCKED, "true");
    return s;
}
 
Example 11
Source Project: iceberg   File: PruneColumns.java    License: Apache License 2.0 5 votes vote down vote up
private static Schema copyRecord(Schema record, List<Schema.Field> newFields) {
  Schema copy = Schema.createRecord(record.getName(),
      record.getDoc(), record.getNamespace(), record.isError(), newFields);

  for (Map.Entry<String, Object> prop : record.getObjectProps().entrySet()) {
    copy.addProp(prop.getKey(), prop.getValue());
  }

  return copy;
}
 
Example 12
Source Project: incubator-gobblin   File: AvroFlattener.java    License: Apache License 2.0 5 votes vote down vote up
/***
 * Copy properties to an Avro Schema
 * @param props Properties to copy to Avro Schema
 * @param schema Avro Schema to copy properties to
 */
private static void copyProperties(Map<String, JsonNode> props, Schema schema) {
  Preconditions.checkNotNull(schema);

  // (if null, don't copy but do not throw exception)
  if (null != props) {
    for (Map.Entry<String, JsonNode> prop : props.entrySet()) {
      schema.addProp(prop.getKey(), prop.getValue());
    }
  }
}
 
Example 13
@Test
public void validLogicalType() throws Exception {
  Schema schema = SchemaBuilder.builder().intType();
  schema.addProp(LogicalType.LOGICAL_TYPE_PROP, "date");
  underTest.onVisit(schema, breadcrumb);
}
 
Example 14
default Schema setProp(final Schema schema, final String name, final String value) {
    schema.addProp("talend.component." + name, value);
    return schema;
}
 
Example 15
@Test
public void testConvertToAvroForAggregateResult() throws Exception {

    Schema schema = SchemaBuilder.builder().record("Schema").fields() //
            .name("Id").prop(SchemaConstants.TALEND_COLUMN_IS_KEY, "true").type().stringType().noDefault() //
            .name("Name").type(AvroUtils._string()).noDefault() //
            .name("Field_A").type(AvroUtils._int()).noDefault() //
            .name("Field_B").type(AvroUtils._boolean()).noDefault() //
            .name("Field_C").type(AvroUtils._date()).noDefault() //
            .name("Field_D").type(AvroUtils._double()).noDefault() //
            .name("Field_E_Value").type(AvroUtils._string()).noDefault() //
            .name("Field_F").type(AvroUtils._string()).noDefault() //
            .name("Field_G").type(AvroUtils._string()).noDefault() //
            .name("Field_H").type(AvroUtils._string()).noDefault() //
            .name("Field_I").type(AvroUtils._string()).noDefault() //
            .name("Field_J").type(AvroUtils._string()).noDefault() //
            .name("Field_K").type(AvroUtils._string()).noDefault() //
            .name("Field_L").type(AvroUtils._string()).noDefault() //
            .endRecord();
    schema.getField("Field_C").addProp(SchemaConstants.TALEND_COLUMN_PATTERN,
            "yyyy-MM-dd'T'HH:mm:ss'.000Z'");
    schema.addProp(SalesforceSchemaConstants.COLUMNNAME_DELIMTER, "_");
    schema.addProp(SalesforceSchemaConstants.VALUE_DELIMITER, "|");

    converter.setSchema(schema);

    SObject sObject = new SObject("AggregateResult");

    sObject.addField("Id", "12345");
    sObject.addField("Name", "Qwerty");
    sObject.addField("Field_A", "42");
    sObject.addField("Field_B", "true");
    sObject.addField("Field_C", dateFormat.parse("2017-06-15T18:26:34.000Z"));
    sObject.addField("Field_D", 10200.45);

    SObject sObject1 = new SObject();
    sObject1.addField("Value", 245);
    sObject.addField("Field_E", sObject1);
    SObject sObject2 = new SObject();
    sObject2.addField("Value", 542);
    sObject.addField("Field_E", sObject2);

    sObject.addField("Field_F", new BigDecimal("20000000000000000000000000.123456789"));
    sObject.addField("Field_G", 899.5f);
    sObject.addField("Field_H", Boolean.TRUE);
    sObject.addField("Field_I", new byte[]{0x0a, 0x0b, 0x0c, 0x0d});
    sObject.addField("Field_J", 102030405060708090L);
    sObject.addField("Field_K", SalesforceRuntime.convertDateToCalendar(
            dateFormat.parse("2017-06-16T10:45:02.000Z"),false));
    sObject.addField("Field_L", null);

    IndexedRecord indexedRecord = converter.convertToAvro(sObject);
    assertNotNull(indexedRecord);
    assertNotNull(indexedRecord.getSchema());
    assertEquals(schema, indexedRecord.getSchema());

    assertEquals("12345", indexedRecord.get(0));
    assertEquals("Qwerty", indexedRecord.get(1));
    assertEquals(Integer.valueOf(42), indexedRecord.get(2));
    assertEquals(Boolean.TRUE, indexedRecord.get(3));
    assertEquals(dateFormat.parse("2017-06-15T18:26:34.000Z").getTime(), indexedRecord.get(4));
    assertEquals(Double.valueOf(10200.45), indexedRecord.get(5));
    assertEquals("245|542", indexedRecord.get(6));
    assertEquals("20000000000000000000000000.123456789", indexedRecord.get(7));
    assertEquals("899.5", indexedRecord.get(8));
    assertEquals("true", indexedRecord.get(9));
    assertEquals("CgsMDQ==", indexedRecord.get(10));
    assertEquals("102030405060708090", indexedRecord.get(11));
    assertEquals("2017-06-16T07:45:02.000Z", indexedRecord.get(12));
    assertNull(indexedRecord.get(13));
}
 
Example 16
Source Project: iceberg   File: AvroTestHelpers.java    License: Apache License 2.0 4 votes vote down vote up
static Schema addValueId(int id, Schema schema) {
  schema.addProp(AvroSchemaUtil.VALUE_ID_PROP, id);
  return schema;
}
 
Example 17
Source Project: iceberg   File: TestSchemaConversions.java    License: Apache License 2.0 4 votes vote down vote up
private Schema addAdjustToUtc(Schema schema, boolean adjustToUTC) {
  schema.addProp(AvroSchemaUtil.ADJUST_TO_UTC_PROP, adjustToUTC);
  return schema;
}
 
Example 18
private static Schema traverse(Map.Entry<String, HiveTypeInfo> node)
    throws StageException {
  switch(node.getValue().getHiveType()){
    case STRING:
      return Schema.create(Schema.Type.STRING);

    case BOOLEAN:
      return Schema.create(Schema.Type.BOOLEAN);

    case INT:
      return Schema.create(Schema.Type.INT);

    case BIGINT:
      return Schema.create(Schema.Type.LONG);

    case FLOAT:
      return Schema.create(Schema.Type.FLOAT);

    case DOUBLE:
      return Schema.create(Schema.Type.DOUBLE);

    case BINARY:
      return Schema.create(Schema.Type.BYTES);

    case TIMESTAMP:
      Schema timestampSchema = Schema.create(Schema.Type.LONG);
      timestampSchema.addProp(AvroTypeUtil.LOGICAL_TYPE, AvroTypeUtil.LOGICAL_TYPE_TIMESTAMP_MILLIS);
      return timestampSchema;

    case DATE:
      Schema dateSchema = Schema.create(Schema.Type.INT);
      dateSchema.addProp(AvroTypeUtil.LOGICAL_TYPE, AvroTypeUtil.LOGICAL_TYPE_DATE);
      return dateSchema;

    case DECIMAL:
      Utils.checkArgument(node.getValue() instanceof DecimalTypeInfo, "Invalid type used in HiveTypeInfo");
      DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo)node.getValue();
      Schema schema = Schema.create(Schema.Type.BYTES);
      schema.addProp(AvroTypeUtil.LOGICAL_TYPE, AvroTypeUtil.LOGICAL_TYPE_DECIMAL);
      schema.addProp(AvroTypeUtil.LOGICAL_TYPE_ATTR_PRECISION, getJsonNode(decimalTypeInfo.getPrecision()));
      schema.addProp(AvroTypeUtil.LOGICAL_TYPE_ATTR_SCALE, getJsonNode(decimalTypeInfo.getScale()));
      return schema;

    default:
      // Accessing Unsupported Type (Map, Union, Array, Enum, NULL, Record)
      throw new StageException(Errors.HIVE_24, node.getValue().getHiveType());
  }
}
 
Example 19
@Test
public void testConvertToAvroForNestedObjects() throws Exception {

    Schema schema = SchemaBuilder.builder().record("Schema").fields() //
            .name("Id").prop(SchemaConstants.TALEND_COLUMN_IS_KEY, "true").type().stringType().noDefault() //
            .name("Name").type(AvroUtils._string()).noDefault() //
            .name("FieldA").type(AvroUtils._int()).noDefault() //
            .name("Member_Name").type(AvroUtils._string()).noDefault() //
            .name("Member_FieldA").type(AvroUtils._string()).noDefault() //
            .name("Member_Contact_Name").type(AvroUtils._string()).noDefault() //
            .name("Member_Contact_FieldA").type(AvroUtils._string()).noDefault() //
            .endRecord();
    schema.addProp(SalesforceSchemaConstants.COLUMNNAME_DELIMTER, "_");
    schema.addProp(SalesforceSchemaConstants.VALUE_DELIMITER, "|");

    converter.setSchema(schema);

    SObject sObject = new SObject("Campaign");
    sObject.addField("Id", "12345");
    sObject.addField("Name", "Qwerty");

    // Intentionally add another value to check that it is skipped by converter
    sObject.addField("FieldA", 300);
    sObject.addField("FieldA", 400);

    SObject sObject1 = new SObject("CampaignMember");
    sObject1.addField("Id", "10001");
    sObject1.addField("Name", "Member A");
    sObject1.addField("FieldA", 10);
    sObject.addField("Member", sObject1);

    SObject sObject11 = new SObject("Contact");
    sObject11.addField("Id", "20001");
    sObject11.addField("Name", "Contact A");
    sObject11.addField("FieldA", "foo");
    sObject1.addField("Contact", sObject11);

    SObject sObject2 = new SObject("CampaignMember");
    sObject2.addField("Id", "10002");
    sObject2.addField("Name", "Member B");
    sObject2.addField("FieldA", 20);
    sObject.addField("Member", sObject2);

    SObject sObject21 = new SObject("Contact");
    sObject21.addField("Id", "20002");
    sObject21.addField("Name", "Contact B");
    sObject21.addField("FieldA", "bar");
    sObject2.addField("Contact", sObject21);

    IndexedRecord indexedRecord = converter.convertToAvro(sObject);
    assertNotNull(indexedRecord);
    assertNotNull(indexedRecord.getSchema());
    assertEquals(schema, indexedRecord.getSchema());

    assertEquals("12345", indexedRecord.get(0));
    assertEquals("Qwerty", indexedRecord.get(1));
    assertEquals(Integer.valueOf(300), indexedRecord.get(2));
    assertEquals("Member A|Member B", indexedRecord.get(3));
    assertEquals("10|20", indexedRecord.get(4));
    assertEquals("Contact A|Contact B", indexedRecord.get(5));
    assertEquals("foo|bar", indexedRecord.get(6));
}
 
Example 20
Source Project: incubator-gobblin   File: AvroUtils.java    License: Apache License 2.0 4 votes vote down vote up
public static Schema addSchemaCreationTime(Schema inputSchema, Schema outputSchema) {
  if (inputSchema.getProp(SCHEMA_CREATION_TIME_KEY) != null && outputSchema.getProp(SCHEMA_CREATION_TIME_KEY) == null) {
    outputSchema.addProp(SCHEMA_CREATION_TIME_KEY, inputSchema.getProp(SCHEMA_CREATION_TIME_KEY));
  }
  return outputSchema;
}