Java Code Examples for org.apache.avro.Schema#Field

The following examples show how to use org.apache.avro.Schema#Field . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: pulsar   File: JSONSchemaTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testNotAllowNullSchema() throws JSONException {
    JSONSchema<Foo> jsonSchema = JSONSchema.of(SchemaDefinition.<Foo>builder().withPojo(Foo.class).withAlwaysAllowNull(false).build());
    Assert.assertEquals(jsonSchema.getSchemaInfo().getType(), SchemaType.JSON);
    Schema.Parser parser = new Schema.Parser();
    String schemaJson = new String(jsonSchema.getSchemaInfo().getSchema());
    assertJSONEqual(schemaJson, SCHEMA_JSON_NOT_ALLOW_NULL);
    Schema schema = parser.parse(schemaJson);

    for (String fieldName : FOO_FIELDS) {
        Schema.Field field = schema.getField(fieldName);
        Assert.assertNotNull(field);

        if (field.name().equals("field4")) {
            Assert.assertNotNull(field.schema().getTypes().get(1).getField("field1"));
        }
        if (field.name().equals("fieldUnableNull")) {
            Assert.assertNotNull(field.schema().getType());
        }
    }
}
 
Example 2
@Test
public void testSetupSchema_serialization() {
    PythonRowProperties properties = new PythonRowProperties("test");
    properties.init();
    AvroRegistry registry = new AvroRegistry();
    Schema stringSchema = registry.getConverter(String.class).getSchema();
    Schema.Field inputValue1Field = new Schema.Field("inputValue1", stringSchema, null, null, Order.ASCENDING);
    Schema.Field inputValue2Field = new Schema.Field("inputValue2", stringSchema, null, null, Order.ASCENDING);
    Schema inputSchema = Schema.createRecord("inputSchema", null, null, false,
            Arrays.asList(inputValue1Field, inputValue2Field));
    properties.main.schema.setValue(inputSchema);

    properties = Helper.fromSerializedPersistent(properties.toSerialized(), PythonRowProperties.class).object;

    assertThat(inputSchema, equalTo(properties.main.schema.getValue()));
}
 
Example 3
@Test
public void testInferSchemaWithReferenceField() {
    Field referenceField = new Field();
    referenceField.setName("reference");
    referenceField.setType(FieldType.string);
    referenceField.setReferenceTo(new String[]{"SomeRecord"});
    referenceField.setRelationshipName("relationship");

    DescribeSObjectResult dsor = new DescribeSObjectResult();
    dsor.setName("MySObjectRecord");
    dsor.setFields(new Field[] { referenceField });

    Schema schema = SalesforceAvroRegistryString.get().inferSchema(dsor);

    Schema.Field field = schema.getField("reference");

    assertThat(field.schema().getType(), is(Schema.Type.STRING));
    assertThat(field.getProp(SalesforceSchemaConstants.REF_MODULE_NAME), is("SomeRecord"));
    assertThat(field.getProp(SalesforceSchemaConstants.REF_FIELD_NAME), is("relationship"));
}
 
Example 4
Source Project: pulsar   File: AvroSchemaTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testAllowNullSchema() throws JSONException {
    AvroSchema<Foo> avroSchema = AvroSchema.of(SchemaDefinition.<Foo>builder().withPojo(Foo.class).build());
    assertEquals(avroSchema.getSchemaInfo().getType(), SchemaType.AVRO);
    Schema.Parser parser = new Schema.Parser();
    parser.setValidateDefaults(false);
    String schemaJson = new String(avroSchema.getSchemaInfo().getSchema());
    assertJSONEquals(schemaJson, SCHEMA_AVRO_ALLOW_NULL);
    Schema schema = parser.parse(schemaJson);

    for (String fieldName : FOO_FIELDS) {
        Schema.Field field = schema.getField(fieldName);
        Assert.assertNotNull(field);

        if (field.name().equals("field4")) {
            Assert.assertNotNull(field.schema().getTypes().get(1).getField("field1"));
        }
        if (field.name().equals("fieldUnableNull")) {
            Assert.assertNotNull(field.schema().getType());
        }
    }
}
 
Example 5
Source Project: components   File: SnowflakeWriter.java    License: Apache License 2.0 6 votes vote down vote up
protected void populateRowData(IndexedRecord input,
        List<Schema.Field> recordFields, List<Schema.Field> remoteFields) {
    for (int i = 0, j = 0; i < row.length && j < remoteFields.size(); j++) {
        Field f = recordFields.get(j);
        Field remoteTableField = remoteFields.get(j);
        if (f == null) {
            if (Boolean.valueOf(remoteTableField.getProp(SnowflakeAvroRegistry.TALEND_FIELD_AUTOINCREMENTED))) {
                continue;
            }
            Object defaultValue = remoteTableField.defaultVal();
            row[i] = StringUtils.EMPTY.equals(defaultValue) ? null : defaultValue;
        } else {
            Object inputValue = input.get(f.pos());
            row[i] = getFieldValue(inputValue, remoteTableField);
        }
        i++;
    }

    loader.submitRow(row);
}
 
Example 6
Source Project: xml-avro   File: ConverterTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void nestedRecursiveRecords() {
    String xsd =
            "<xs:schema xmlns:xs='http://www.w3.org/2001/XMLSchema'>" +
            "  <xs:complexType name='type'>" +
            "    <xs:sequence>" +
            "      <xs:element name='node' type='type' minOccurs='0'/>" +
            "    </xs:sequence>" +
            "  </xs:complexType>" +
            "  <xs:element name='root' type='type'/>" +
            "</xs:schema>";

    Schema schema = Converter.createSchema(xsd);

    Schema.Field field = schema.getField("node");
    Schema subSchema = field.schema();
    assertSame(schema, subSchema.getTypes().get(1));

    String xml = "<root><node></node></root>";
    GenericData.Record record = Converter.createDatum(schema, xml);

    GenericData.Record child = (GenericData.Record) record.get("node");
    assertEquals(record.getSchema(), child.getSchema());

    assertNull(child.get("node"));
}
 
Example 7
Source Project: components   File: SchemaGeneratorUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Create a KV-SChema from a schema and a list of keys
 * 
 * @param inputSchema a Schema
 * @param keyPaths the list of key
 * @return a hierarchical KV-Schema
 */
public static Schema extractKeyValues(Schema inputSchema, List<String> keyPaths) {
    List<Schema.Field> fieldList = new ArrayList<>();

    Schema keySchema = extractKeys(inputSchema, keyPaths);
    fieldList.add(new Field(RECORD_KEY_PREFIX, keySchema, "", ""));
    Schema valueSchema = extractValues(inputSchema, keyPaths);
    fieldList.add(new Field(RECORD_VALUE_PREFIX, valueSchema, "", ""));
    return Schema.createRecord(RECORD_KEYVALUE_PREFIX, null, null, false, fieldList);
}
 
Example 8
Source Project: components   File: TFilterRowSink.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public ValidationResult validate(RuntimeContainer arg0) {
    for (FilterDescriptor descriptor : filterDescriptors) {
        Schema.Field f = inputSchema.getField(descriptor.getColumnName());
        ValidationResult filteringValidationResult = new FilterPrerequisitesValidator().validate(descriptor.getFunctionType(),
                descriptor.getOperatorType(), AvroUtils.unwrapIfNullable(f.schema()).getType(),
                descriptor.getPredefinedValue());
        if (filteringValidationResult.getStatus() != Result.OK) {
            return filteringValidationResult;
        }
    }
    return ValidationResult.OK;
}
 
Example 9
Source Project: parquet-mr   File: RecordBuilder.java    License: Apache License 2.0 5 votes vote down vote up
private void fillReflect(Object record, String[] data) {
  for (int i = 0; i < indexes.length; i += 1) {
    Schema.Field field = fields[i];
    int index = indexes[i];
    Object value = makeValue(index < data.length ? data[index] : null, field);
    ReflectData.get().setField(record, field.name(), i, value);
  }
}
 
Example 10
@Test
public void testSearch() throws Exception {
    NetSuiteInputProperties properties = new NetSuiteInputProperties("test");
    properties.init();
    properties.connection.referencedComponent.componentInstanceId.setValue(CONNECTION_COMPONENT_ID);
    properties.connection.referencedComponent.setReference(connectionProperties);
    properties.module.moduleName.setValue("Account");

    NetSuiteRuntimeImpl runtime = new NetSuiteRuntimeImpl();
    NetSuiteDatasetRuntime dataSetRuntime = runtime.getDatasetRuntime(CONTAINER, properties);
    Schema schema = dataSetRuntime.getSchema(properties.module.moduleName.getValue());
    properties.module.main.schema.setValue(schema);

    properties.module.searchQuery.field.setValue(Arrays.asList("Type"));
    properties.module.searchQuery.operator.setValue(Arrays.asList("List.anyOf"));
    properties.module.searchQuery.value1.setValue(Arrays.<Object>asList("Bank"));
    properties.module.searchQuery.value2.setValue(Arrays.<Object>asList((String) null));

    NetSuiteSource source = new NetSuiteSourceImpl();
    source.initialize(CONTAINER, properties);

    NetSuiteSearchInputReader reader = (NetSuiteSearchInputReader) source.createReader(CONTAINER);
    boolean started = reader.start();
    assertTrue(started);

    IndexedRecord record = reader.getCurrent();
    assertNotNull(record);

    List<Schema.Field> fields = record.getSchema().getFields();
    for (int i = 0; i < fields.size(); i++) {
        Schema.Field typeField = getFieldByName(fields, "AcctType");
        Object value = record.get(typeField.pos());
        assertNotNull(value);
        assertEquals(AccountType.BANK.value(), value);
    }
}
 
Example 11
Source Project: iceberg   File: AvroSchemaVisitor.java    License: Apache License 2.0 5 votes vote down vote up
public static <T> T visit(Schema schema, AvroSchemaVisitor<T> visitor) {
  switch (schema.getType()) {
    case RECORD:
      // check to make sure this hasn't been visited before
      String name = schema.getFullName();
      Preconditions.checkState(!visitor.recordLevels.contains(name),
          "Cannot process recursive Avro record %s", name);

      visitor.recordLevels.push(name);

      List<Schema.Field> fields = schema.getFields();
      List<String> names = Lists.newArrayListWithExpectedSize(fields.size());
      List<T> results = Lists.newArrayListWithExpectedSize(fields.size());
      for (Schema.Field field : schema.getFields()) {
        names.add(field.name());
        results.add(visit(field.schema(), visitor));
      }

      visitor.recordLevels.pop();

      return visitor.record(schema, names, results);

    case UNION:
      List<Schema> types = schema.getTypes();
      List<T> options = Lists.newArrayListWithExpectedSize(types.size());
      for (Schema type : types) {
        options.add(visit(type, visitor));
      }
      return visitor.union(schema, options);

    case ARRAY:
      return visitor.array(schema, visit(schema.getElementType(), visitor));

    case MAP:
      return visitor.map(schema, visit(schema.getValueType(), visitor));

    default:
      return visitor.primitive(schema);
  }
}
 
Example 12
private static HiveAttribute convertAvroSchemaFieldToHiveAttribute(Schema.Field field) {
  String avroFieldType = field.schema().getType().toString();
  if (avroFieldType.equalsIgnoreCase("UNION")) {
    avroFieldType = extractAvroTypeFromUnion(field);
  }
  if (HiveAttribute.fromAvroType(avroFieldType) == null) {
    throw new RuntimeException("Hive does not support attribute type '" + avroFieldType + "'");
  }
  return new HiveAttribute(field.name(), HiveAttribute.fromAvroType(avroFieldType));
}
 
Example 13
private boolean isLegacySchema( Schema schema ) {
  if ( schema.getFields().size() > 0 ) {
    Schema.Field field = schema.getFields().get( 0 );
    return field != null && field.name() != null && field.name()
      .contains( PentahoAvroInputFormat.FieldName.FIELDNAME_DELIMITER );
  } else {
    return false;
  }
}
 
Example 14
@Test
public void testSyncLeadSOAPFail() throws Exception {
    props = getSOAPProperties();
    List<Field> fields = new ArrayList<>();
    Field field = new Schema.Field("Id", Schema.create(Type.INT), null, (Object) null);
    fields.add(field);
    field = new Schema.Field("Email", Schema.create(Schema.Type.STRING), null, (Object) null);
    fields.add(field);
    Schema s = MarketoUtils.newSchema(MarketoConstants.getEmptySchema(), "leadAttribute", fields);
    props.schemaInput.schema.setValue(s);
    props.updateOutputSchemas();
    props.beforeMappingInput();
    IndexedRecord record = new GenericData.Record(s);
    record.put(0, 12345);
    record.put(1, "[email protected]");
    props.outputOperation.setValue(OutputOperation.syncLead);
    try {
        writer = getWriter(props);
        writer.open("testDieOnError");
        writer.write(record);
    } catch (IOException e) {
        assertNotNull(e);
        assertTrue(e.getMessage().contains("20103"));
    }
    props.dieOnError.setValue(false);
    writer = getWriter(props);
    writer.open("test");
    writer.write(record);
    MarketoResult result = (MarketoResult) writer.close();
    assertEquals(1, result.getApiCalls());
    assertEquals(0, result.getSuccessCount());
    assertEquals(1, result.getRejectCount());
    assertEquals("failed", writer.getRejectedWrites().get(0).get(2).toString());
    assertEquals(Collections.emptyList(), writer.getSuccessfulWrites());
}
 
Example 15
Source Project: components   File: CommonUtils.java    License: Apache License 2.0 5 votes vote down vote up
public static Schema.Field getField(Schema schema, String fieldName) {
    if (schema == null) {
        return null;
    }

    for (Schema.Field outField : schema.getFields()) {
        if (outField.name().equals(fieldName)) {
            return outField;
        }
    }

    return null;
}
 
Example 16
@Test
public void simple() {
  DdlToAvroSchemaConverter converter = new DdlToAvroSchemaConverter("spannertest", "booleans");
  Ddl ddl = Ddl.builder()
      .createTable("Users")
      .column("id").int64().notNull().endColumn()
      .column("first_name").string().size(10).endColumn()
      .column("last_name").type(Type.string()).max().endColumn()
      .primaryKey().asc("id").desc("last_name").end()
      .indexes(ImmutableList.of("CREATE INDEX `UsersByFirstName` ON `Users` (`first_name`)"))
      .foreignKeys(
          ImmutableList.of(
              "ALTER TABLE `Users` ADD CONSTRAINT `fk` FOREIGN KEY (`first_name`)"
                  + " REFERENCES `AllowedNames` (`first_name`)"))
      .endTable()
      .build();

  Collection<Schema> result = converter.convert(ddl);
  assertThat(result, hasSize(1));
  Schema avroSchema = result.iterator().next();

  assertThat(avroSchema.getNamespace(), equalTo("spannertest"));
  assertThat(avroSchema.getProp("googleFormatVersion"), equalTo("booleans"));
  assertThat(avroSchema.getProp("googleStorage"), equalTo("CloudSpanner"));

  assertThat(avroSchema.getName(), equalTo("Users"));

  List<Schema.Field> fields = avroSchema.getFields();

  assertThat(fields, hasSize(3));

  assertThat(fields.get(0).name(), equalTo("id"));
  // Not null
  assertThat(fields.get(0).schema().getType(), equalTo(Schema.Type.LONG));
  assertThat(fields.get(0).getProp("sqlType"), equalTo("INT64"));

  assertThat(fields.get(1).name(), equalTo("first_name"));
  assertThat(fields.get(1).schema(), equalTo(nullableUnion(Schema.Type.STRING)));
  assertThat(fields.get(1).getProp("sqlType"), equalTo("STRING(10)"));

  assertThat(fields.get(2).name(), equalTo("last_name"));
  assertThat(fields.get(2).schema(), equalTo(nullableUnion(Schema.Type.STRING)));
  assertThat(fields.get(2).getProp("sqlType"), equalTo("STRING(MAX)"));

  // spanner pk
  assertThat(avroSchema.getProp("spannerPrimaryKey_0"), equalTo("`id` ASC"));
  assertThat(avroSchema.getProp("spannerPrimaryKey_1"), equalTo("`last_name` DESC"));
  assertThat(avroSchema.getProp("spannerParent"), nullValue());
  assertThat(avroSchema.getProp("spannerOnDeleteAction"), nullValue());

  assertThat(
      avroSchema.getProp("spannerIndex_0"),
      equalTo("CREATE INDEX `UsersByFirstName` ON `Users` (`first_name`)"));
  assertThat(
      avroSchema.getProp("spannerForeignKey_0"),
      equalTo(
          "ALTER TABLE `Users` ADD CONSTRAINT `fk` FOREIGN KEY (`first_name`)"
              + " REFERENCES `AllowedNames` (`first_name`)"));

  System.out.println(avroSchema.toString(true));
}
 
Example 17
Source Project: spork   File: AvroSchema2Pig.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Convert a schema with field name to a pig schema
 */
 private static ResourceFieldSchema inconvert(Schema in, String fieldName, Set<Schema> visitedRecords)
         throws IOException {

    AvroStorageLog.details("InConvert avro schema with field name " + fieldName);

    Schema.Type avroType = in.getType();
    ResourceFieldSchema fieldSchema = new ResourceFieldSchema();
    fieldSchema.setName(fieldName);

    if (avroType.equals(Schema.Type.RECORD)) {

        AvroStorageLog.details("convert to a pig tuple");

        if (visitedRecords.contains(in)) {
            fieldSchema.setType(DataType.BYTEARRAY);
        } else {
            visitedRecords.add(in);
            fieldSchema.setType(DataType.TUPLE);
            ResourceSchema tupleSchema = new ResourceSchema();
            List<Schema.Field> fields = in.getFields();
            ResourceFieldSchema[] childFields = new ResourceFieldSchema[fields.size()];
            int index = 0;
            for (Schema.Field field : fields) {
                childFields[index++] = inconvert(field.schema(), field.name(), visitedRecords);
            }

            tupleSchema.setFields(childFields);
            fieldSchema.setSchema(tupleSchema);
            visitedRecords.remove(in);
        }

    } else if (avroType.equals(Schema.Type.ARRAY)) {

        AvroStorageLog.details("convert array to a pig bag");
        fieldSchema.setType(DataType.BAG);
        Schema elemSchema = in.getElementType();
        ResourceFieldSchema subFieldSchema = inconvert(elemSchema, ARRAY_FIELD, visitedRecords);
        add2BagSchema(fieldSchema, subFieldSchema);

    } else if (avroType.equals(Schema.Type.MAP)) {

        AvroStorageLog.details("convert map to a pig map");
        fieldSchema.setType(DataType.MAP);

    } else if (avroType.equals(Schema.Type.UNION)) {

        if (AvroStorageUtils.isAcceptableUnion(in)) {
            Schema acceptSchema = AvroStorageUtils.getAcceptedType(in);
            ResourceFieldSchema realFieldSchema = inconvert(acceptSchema, null, visitedRecords);
            fieldSchema.setType(realFieldSchema.getType());
            fieldSchema.setSchema(realFieldSchema.getSchema());
        } else
            throw new IOException("Do not support generic union:" + in);

    } else if (avroType.equals(Schema.Type.FIXED)) {
         fieldSchema.setType(DataType.BYTEARRAY);
    } else if (avroType.equals(Schema.Type.BOOLEAN)) {
        fieldSchema.setType(DataType.BOOLEAN);
    } else if (avroType.equals(Schema.Type.BYTES)) {
        fieldSchema.setType(DataType.BYTEARRAY);
    } else if (avroType.equals(Schema.Type.DOUBLE)) {
        fieldSchema.setType(DataType.DOUBLE);
    } else if (avroType.equals(Schema.Type.ENUM)) {
        fieldSchema.setType(DataType.CHARARRAY);
    } else if (avroType.equals(Schema.Type.FLOAT)) {
        fieldSchema.setType(DataType.FLOAT);
    } else if (avroType.equals(Schema.Type.INT)) {
        fieldSchema.setType(DataType.INTEGER);
    } else if (avroType.equals(Schema.Type.LONG)) {
        fieldSchema.setType(DataType.LONG);
    } else if (avroType.equals(Schema.Type.STRING)) {
        fieldSchema.setType(DataType.CHARARRAY);
    } else if (avroType.equals(Schema.Type.NULL)) {
        // value of NULL is always NULL
        fieldSchema.setType(DataType.INTEGER);
    } else {
        throw new IOException("Unsupported avro type:" + avroType);
    }
    return fieldSchema;
}
 
Example 18
private AvroSpec.DataType findActualDataType( Schema.Field field ) {
  AvroSpec.DataType avroDataType = null;
  LogicalType logicalType = null;
  Schema.Type primitiveAvroType = null;

  if ( field.schema().getType().equals( Schema.Type.UNION ) ) {
    for ( Schema typeSchema : field.schema().getTypes() ) {
      if ( !typeSchema.getType().equals( Schema.Type.NULL ) ) {
        logicalType = typeSchema.getLogicalType();
        primitiveAvroType = typeSchema.getType();
        break;
      }
    }
  } else {
    logicalType = field.schema().getLogicalType();
    primitiveAvroType = field.schema().getType();
  }

  if ( logicalType != null ) {
    for ( AvroSpec.DataType tmpType : AvroSpec.DataType.values() ) {
      if ( !tmpType.isPrimitiveType() && tmpType.getType().equals( logicalType.getName() ) ) {
        avroDataType = tmpType;
        break;
      }
    }
  } else {
    switch ( primitiveAvroType ) {
      case INT:
        avroDataType = AvroSpec.DataType.INTEGER;
        break;
      case LONG:
        avroDataType = AvroSpec.DataType.LONG;
        break;
      case BYTES:
        avroDataType = AvroSpec.DataType.BYTES;
        break;
      case FLOAT:
        avroDataType = AvroSpec.DataType.FLOAT;
        break;
      case DOUBLE:
        avroDataType = AvroSpec.DataType.DOUBLE;
        break;
      case STRING:
        avroDataType = AvroSpec.DataType.STRING;
        break;
      case BOOLEAN:
        avroDataType = AvroSpec.DataType.BOOLEAN;
        break;
    }
  }

  return avroDataType;
}
 
Example 19
BooleanConverter(Schema.Field field) {
    super(field);
}
 
Example 20
Object getSpecificDefaultValue(Schema.Field field);