org.apache.avro.Schema.Type Java Examples

The following examples show how to use org.apache.avro.Schema.Type. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: AvroSerializer.java    From envelope with Apache License 2.0 6 votes vote down vote up
private void validateSchemaIsSupported(Schema schema) {
  for (Field field : schema.getFields()) {
    Type type = field.schema().getType();

    if (type.equals(Type.UNION)) {
      List<Schema> types = field.schema().getTypes();

      if (types.size() != 2) {
        throw new RuntimeException("Union type in Avro serializer schema must only contain two types");
      }

      if (types.get(0).getType().equals(Type.NULL)) {
        type = types.get(1).getType();
      }
      else {
        type = types.get(0).getType();
      }
    }

    if (!supportedTypes.contains(type)) {
      throw new RuntimeException("Avro serializer for Kafka output does not support Avro schema type: " + type);
    }
  }
}
 
Example #2
Source File: KeyValueUtils.java    From components with Apache License 2.0 6 votes vote down vote up
/**
 * Use a Schema to generate a hierarchical GenericRecord that contains only null values.
 *
 * @param schema the parent schema of the field to set as null
 * @param fieldName the name of the field to set as null
 * @return if fieldName is a Record of the schema, the method will return a GenericRecord with any leaf set as null,
 * otherwise return null
 */
public static IndexedRecord generateEmptyRecord(Schema schema, String fieldName) {
    if (schema.getType().equals(Type.RECORD)) {
        Schema unwrappedSchema = getUnwrappedSchema(schema.getField(fieldName));
        if (unwrappedSchema.getType().equals(Type.RECORD)) {
            GenericRecordBuilder outputRecord = new GenericRecordBuilder(unwrappedSchema);
            for (Field field : unwrappedSchema.getFields()) {
                IndexedRecord value = generateEmptyRecord(unwrappedSchema, field.name());
                outputRecord.set(field.name(), value);
            }
            return outputRecord.build();
        } else {
            return null;
        }
    } else {
        return null;
    }
}
 
Example #3
Source File: AvroKeyEntitySchemaParserTest.java    From kite with Apache License 2.0 6 votes vote down vote up
@Test
public void testGoodSchema() {
  AvroEntitySchema avroEntitySchema = parser.parseEntitySchema(entitySchema);
  ColumnMapping descriptor = avroEntitySchema
      .getColumnMappingDescriptor();
  assertEquals(9, descriptor.getFieldMappings().size());
  assertEquals(FieldMapping.MappingType.COLUMN, descriptor.getFieldMapping("field1")
      .getMappingType());
  assertEquals(FieldMapping.MappingType.KEY_AS_COLUMN, descriptor
      .getFieldMapping("field4").getMappingType());
  assertEquals(FieldMapping.MappingType.OCC_VERSION, descriptor.getFieldMapping("version")
      .getMappingType());

  AvroKeySchema avroKeySchema = parser.parseKeySchema(entitySchema);
  assertEquals(Type.STRING, avroKeySchema.getAvroSchema()
      .getField("keyPart1_copy").schema().getType());
  assertEquals(Type.STRING, avroKeySchema.getAvroSchema()
      .getField("keyPart2_copy").schema().getType());
  assertEquals(2, Accessor.getDefault().getFieldPartitioners(avroKeySchema
      .getPartitionStrategy())
      .size());
}
 
Example #4
Source File: AvroRecordHelper.java    From attic-apex-malhar with Apache License 2.0 6 votes vote down vote up
/**
 * Convert a passed String value to the given type for the key as per Schema
 */
public static Object convertValueStringToAvroKeyType(Schema schema, String key, String value) throws ParseException
{
  Type type = null;

  if (schema.getField(key) != null) {
    type = schema.getField(key).schema().getType();
  } else {
    return value;
  }

  Object convertedValue = null;

  if (type == Type.UNION) {
    convertedValue = convertAndResolveUnionToPrimitive(schema, key, value);
  } else {
    convertedValue = convertValueToAvroPrimitive(type, key, value);
  }

  return convertedValue;

}
 
Example #5
Source File: AvroEntitySerDe.java    From kite with Apache License 2.0 6 votes vote down vote up
@Override
public CharSequence deserializeKeyAsColumnKeyFromBytes(String fieldName,
    byte[] columnKeyBytes) {
  Field field = avroSchema.getAvroSchema().getField(fieldName);
  if (field == null) {
    throw new ValidationException("Invalid field name " + fieldName
        + " for schema " + avroSchema.toString());
  }

  Schema.Type schemaType = field.schema().getType();
  if (schemaType == Schema.Type.MAP) {
    String stringProp = field.schema().getProp("avro.java.string");
    if (stringProp != null && stringProp.equals("String")) {
      return new String(columnKeyBytes);
    } else {
      return new Utf8(columnKeyBytes);
    }
  } else if (schemaType == Schema.Type.RECORD) {
    return new String(columnKeyBytes);
  } else {
    throw new ValidationException("Unsupported type for keyAsColumn: "
        + schemaType);
  }
}
 
Example #6
Source File: KeyPathValidatorTest.java    From data-highway with Apache License 2.0 6 votes vote down vote up
@Test(expected = InvalidKeyPathException.class)
public void arrayFail() throws InvalidKeyPathException {
  Path path = KeyPathParser.parse("$.a.b.c");
  Schema schema = SchemaBuilder
      .record("X")
      .fields()
      .name("a")
      .type(SchemaBuilder.array().items(
          SchemaBuilder.record("Z").fields().name("c").type(Schema.create(Type.STRING)).noDefault().endRecord()))
      .noDefault()
      .endRecord();
  try {
    new KeyPathValidator(path, schema).validate();
  } catch (Exception e) {
    assertThat(e.getMessage(), is("Element '$.a' not a traversable type (found 'ARRAY'), in path: $.a.b.c"));
    throw e;
  }
}
 
Example #7
Source File: KeyPathValidatorTest.java    From data-highway with Apache License 2.0 6 votes vote down vote up
@Test(expected = InvalidKeyPathException.class)
public void mapFail() throws InvalidKeyPathException {
  Path path = KeyPathParser.parse("$.a.b[\"c\"]");
  Schema schema = SchemaBuilder
      .record("X")
      .fields()
      .name("a")
      .type(SchemaBuilder.map().values(
          SchemaBuilder.record("Z").fields().name("c").type(Schema.create(Type.STRING)).noDefault().endRecord()))
      .noDefault()
      .endRecord();
  try {
    new KeyPathValidator(path, schema).validate();
  } catch (Exception e) {
    assertThat(e.getMessage(), is("Element '$.a' not a traversable type (found 'MAP'), in path: $.a.b[\"c\"]"));
    throw e;
  }
}
 
Example #8
Source File: CSVUtils.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
public static GenericRecord read(InputStream record, char delimiter, Schema schema, char quoteChar) {
    Record avroRecord = new GenericData.Record(schema);
    String[] parsedRecord = parseFields(convertInputStreamToString(record), delimiter, quoteChar);
    List<Field> fields = schema.getFields();
    if (parsedRecord.length != fields.size()) {
        throw new IllegalStateException("Incompatible schema. Parsed fields count does not match the count of fields from schema. "
                + "Schema: " + schema.toString(true) + "\n Record: " + record);
    }

    for (int i = 0; i < fields.size(); i++) {
        Field field = fields.get(i);
        Type type = field.schema().getType();
        updateRecord(field, type, parsedRecord[i], avroRecord);
    }
    return avroRecord;
}
 
Example #9
Source File: KeyPathValidatorTest.java    From data-highway with Apache License 2.0 6 votes vote down vote up
@Test(expected = InvalidKeyPathException.class)
public void nestedFail() throws InvalidKeyPathException {
  Path path = KeyPathParser.parse("$[\"a\"].b[\"c\"]");
  Schema schema = SchemaBuilder
      .record("X")
      .fields()
      .name("a")
      .type(SchemaBuilder.record("Y").fields().name("b").type(Schema.create(Type.STRING)).noDefault().endRecord())
      .noDefault()
      .endRecord();
  try {
    new KeyPathValidator(path, schema).validate();
  } catch (Exception e) {
    assertThat(e.getMessage(),
        is("Element '$[\"a\"].b' not a traversable type (found 'STRING'), in path: $[\"a\"].b[\"c\"]"));
    throw e;
  }
}
 
Example #10
Source File: RecordBuilderBase.java    From avro-util with BSD 2-Clause "Simplified" License 6 votes vote down vote up
protected static boolean isValidValue(Field f, Object value) {
  if (value != null) {
    return true;
  } else {
    Schema schema = f.schema();
    Type type = schema.getType();
    if (type == Type.NULL) {
      return true;
    } else {
      if (type == Type.UNION) {
        Iterator i$ = schema.getTypes().iterator();

        while(i$.hasNext()) {
          Schema s = (Schema)i$.next();
          if (s.getType() == Type.NULL) {
            return true;
          }
        }
      }

      return false;
    }
  }
}
 
Example #11
Source File: AvroFieldsPickConverter.java    From incubator-gobblin with Apache License 2.0 6 votes vote down vote up
/**
 * For the schema that is a UNION type with NULL and Record type, it provides Records type.
 * @param inputSchema
 * @return
 */
private static Schema getActualRecord(Schema inputSchema) {
  if (Type.RECORD.equals(inputSchema.getType())) {
    return inputSchema;
  }

  Preconditions.checkArgument(Type.UNION.equals(inputSchema.getType()), "Nested schema is only support with either record or union type of null with record");
  Preconditions.checkArgument(inputSchema.getTypes().size() <= 2,
      "For union type in nested record, it should only have NULL and Record type");

  for (Schema inner : inputSchema.getTypes()) {
    if (Type.NULL.equals(inner.getType())) {
      continue;
    }
    Preconditions.checkArgument(Type.RECORD.equals(inner.getType()), "For union type in nested record, it should only have NULL and Record type");
    return inner;

  }
  throw new IllegalArgumentException(inputSchema + " is not supported.");
}
 
Example #12
Source File: AvroToJdbcEntryConverter.java    From incubator-gobblin with Apache License 2.0 6 votes vote down vote up
private static Schema determineType(Schema schema) throws SchemaConversionException {
  if (!AVRO_SUPPORTED_TYPES.contains(schema.getType())) {
    throw new SchemaConversionException(schema.getType() + " is not supported");
  }

  if (!Type.UNION.equals(schema.getType())) {
    return schema;
  }

  //For UNION, only supported avro type with NULL is allowed.
  List<Schema> schemas = schema.getTypes();
  if (schemas.size() > 2) {
    throw new SchemaConversionException("More than two types are not supported " + schemas);
  }

  for (Schema s : schemas) {
    if (Type.NULL.equals(s.getType())) {
      continue;
    }
    return s;
  }

  throw new SchemaConversionException("Cannot determine type of " + schema);
}
 
Example #13
Source File: AvroUtils.java    From incubator-gobblin with Apache License 2.0 6 votes vote down vote up
private static Optional<Schema> removeUncomparableFieldsFromRecord(Schema record, Map<Schema, Optional<Schema>> processed) {
  Preconditions.checkArgument(record.getType() == Schema.Type.RECORD);

  Optional<Schema> result = processed.get(record);
  if (null != result) {
    return result;
  }

  List<Field> fields = Lists.newArrayList();
  for (Field field : record.getFields()) {
    Optional<Schema> newFieldSchema = removeUncomparableFields(field.schema(), processed);
    if (newFieldSchema.isPresent()) {
      fields.add(new Field(field.name(), newFieldSchema.get(), field.doc(), field.defaultValue()));
    }
  }

  Schema newSchema = Schema.createRecord(record.getName(), record.getDoc(), record.getNamespace(), false);
  newSchema.setFields(fields);
  result = Optional.of(newSchema);
  processed.put(record, result);

  return result;
}
 
Example #14
Source File: AvroUtils.java    From incubator-gobblin with Apache License 2.0 6 votes vote down vote up
/**
 * Helper method that does the actual work for {@link #getFieldSchema(Schema, String)}
 * @param schema passed from {@link #getFieldSchema(Schema, String)}
 * @param pathList passed from {@link #getFieldSchema(Schema, String)}
 * @param field keeps track of the index used to access the list pathList
 * @return the schema of the field
 */
private static Optional<Schema> getFieldSchemaHelper(Schema schema, List<String> pathList, int field) {
  if (schema.getType() == Type.RECORD && schema.getField(pathList.get(field)) == null) {
    return Optional.absent();
  }
  switch (schema.getType()) {
    case UNION:
      if (AvroSerdeUtils.isNullableType(schema)) {
        return AvroUtils.getFieldSchemaHelper(AvroSerdeUtils.getOtherTypeFromNullableType(schema), pathList, field);
      }
      throw new AvroRuntimeException("Union of complex types cannot be handled : " + schema);
    case MAP:
      if ((field + 1) == pathList.size()) {
        return Optional.fromNullable(schema.getValueType());
      }
      return AvroUtils.getFieldSchemaHelper(schema.getValueType(), pathList, ++field);
    case RECORD:
      if ((field + 1) == pathList.size()) {
        return Optional.fromNullable(schema.getField(pathList.get(field)).schema());
      }
      return AvroUtils.getFieldSchemaHelper(schema.getField(pathList.get(field)).schema(), pathList, ++field);
    default:
      throw new AvroRuntimeException("Invalid type in schema : " + schema);
  }
}
 
Example #15
Source File: AvroUtils.java    From Cubert with Apache License 2.0 6 votes vote down vote up
private static Type convertToAvroType(DataType colType)
{
    final Type subType;
    if (colType == DataType.TUPLE)
    {
        /* Pig converts RECORD to TUPLE. Converting it back. */
        subType = Type.RECORD;
    }
    else if (colType == DataType.BAG)
    {
        subType = Type.ARRAY;
    }
    else if (colType == DataType.MAP)
    {
      subType = Type.MAP;
    }
    else
    {
        subType = Type.valueOf(colType.toString().toUpperCase());
    }
    return subType;
}
 
Example #16
Source File: GenerateDictionary.java    From Cubert with Apache License 2.0 6 votes vote down vote up
public static Schema getSchema()
{
    Field[] fields =
            {
                    new Schema.Field("colname",
                                     Schema.create(Type.STRING),
                                     null,
                                     null),
                    new Schema.Field("colvalue",
                                     Schema.create(Type.STRING),
                                     null,
                                     null),
                    new Schema.Field("code", Schema.create(Type.INT), null, null) };

    Schema schema = Schema.createRecord("dictionary", null, null, false);
    schema.setFields(Arrays.asList(fields));

    return schema;
}
 
Example #17
Source File: SegmentTestUtils.java    From incubator-pinot with Apache License 2.0 6 votes vote down vote up
public static DataType getColumnType(Field field) {
  org.apache.avro.Schema fieldSchema = field.schema();
  fieldSchema = extractSchemaFromUnionIfNeeded(fieldSchema);

  final Type type = fieldSchema.getType();
  if (type == Type.ARRAY) {
    org.apache.avro.Schema elementSchema = extractSchemaFromUnionIfNeeded(fieldSchema.getElementType());
    if (elementSchema.getType() == Type.RECORD) {
      if (elementSchema.getFields().size() == 1) {
        elementSchema = elementSchema.getFields().get(0).schema();
      } else {
        throw new RuntimeException("More than one schema in Multi-value column!");
      }
      elementSchema = extractSchemaFromUnionIfNeeded(elementSchema);
    }
    return AvroSchemaUtil.valueOf(elementSchema.getType());
  } else {
    return AvroSchemaUtil.valueOf(type);
  }
}
 
Example #18
Source File: AvroStorageDataConversionUtilities.java    From spork with Apache License 2.0 6 votes vote down vote up
/**
 * Packs a Pig DataBag into an Avro array.
 * @param db the Pig databad to pack into the avro array
 * @param s The avro schema for which to determine the type
 * @return the avro array corresponding to the input bag
 * @throws IOException
 */
public static GenericData.Array<Object> packIntoAvro(
    final DataBag db, final Schema s) throws IOException {

  try {
    GenericData.Array<Object> array
      = new GenericData.Array<Object>(new Long(db.size()).intValue(), s);
    for (Tuple t : db) {
      if (s.getElementType() != null
          && s.getElementType().getType() == Type.RECORD) {
        array.add(packIntoAvro(t, s.getElementType()));
      } else if (t.size() == 1) {
        array.add(t.get(0));
      } else {
        throw new IOException(
            "AvroStorageDataConversionUtilities.packIntoAvro: Can't pack "
                + t + " into schema " + s);
      }
    }
    return array;
  } catch (Exception e) {
    throw new IOException(
        "exception in AvroStorageDataConversionUtilities.packIntoAvro", e);
  }
}
 
Example #19
Source File: TestAvroImport.java    From aliyun-maxcompute-data-collectors with Apache License 2.0 6 votes vote down vote up
public void testNonIdentCharactersInColumnName() throws IOException {
  String [] names = { "test_a-v+r/o" };
  String [] types = { "INT" };
  String [] vals = { "2015" };
  createTableWithColTypesAndNames(names, types, vals);

  runImport(getOutputArgv(true, null));

  Path outputFile = new Path(getTablePath(), "part-m-00000.avro");
  DataFileReader<GenericRecord> reader = read(outputFile);
  Schema schema = reader.getSchema();
  assertEquals(Schema.Type.RECORD, schema.getType());
  List<Field> fields = schema.getFields();
  assertEquals(types.length, fields.size());

  checkField(fields.get(0), "TEST_A_V_R_O", Type.INT);

  GenericRecord record1 = reader.next();
  assertEquals("TEST_A_V_R_O", 2015, record1.get("TEST_A_V_R_O"));
}
 
Example #20
Source File: TestAvroImport.java    From aliyun-maxcompute-data-collectors with Apache License 2.0 6 votes vote down vote up
public void testNonstandardCharactersInColumnName() throws IOException {
  String [] names = { "avro\uC3A11" };
  String [] types = { "INT" };
  String [] vals = { "1987" };
  createTableWithColTypesAndNames(names, types, vals);

  runImport(getOutputArgv(true, null));

  Path outputFile = new Path(getTablePath(), "part-m-00000.avro");
  DataFileReader<GenericRecord> reader = read(outputFile);
  Schema schema = reader.getSchema();
  assertEquals(Schema.Type.RECORD, schema.getType());
  List<Field> fields = schema.getFields();
  assertEquals(types.length, fields.size());

  checkField(fields.get(0), "AVRO1", Type.INT);

  GenericRecord record1 = reader.next();
  assertEquals("AVRO1", 1987, record1.get("AVRO1"));
}
 
Example #21
Source File: TestAvroImport.java    From aliyun-maxcompute-data-collectors with Apache License 2.0 6 votes vote down vote up
public void testFirstUnderscoreInColumnName() throws IOException {
  String [] names = { "_NAME" };
  String [] types = { "INT" };
  String [] vals = { "1987" };
  createTableWithColTypesAndNames(names, types, vals);

  runImport(getOutputArgv(true, null));

  Path outputFile = new Path(getTablePath(), "part-m-00000.avro");
  DataFileReader<GenericRecord> reader = read(outputFile);
  Schema schema = reader.getSchema();
  assertEquals(Schema.Type.RECORD, schema.getType());
  List<Field> fields = schema.getFields();
  assertEquals(types.length, fields.size());

  checkField(fields.get(0), "__NAME", Type.INT);

  GenericRecord record1 = reader.next();
  assertEquals("__NAME", 1987, record1.get("__NAME"));
}
 
Example #22
Source File: AvroTupleWrapper.java    From spork with Apache License 2.0 6 votes vote down vote up
@Override
public void append(final Object o) {
  List<Field> fields = avroObject.getSchema().getFields();
  avroObject.put(fields.size(), o);
  Schema fieldSchema = null;
  if (o instanceof String) {
    fieldSchema = Schema.create(Type.STRING);
  } else if (o instanceof Integer) {
    fieldSchema = Schema.create(Type.INT);
  } else if (o instanceof Long) {
    fieldSchema = Schema.create(Type.LONG);
  } else if (o instanceof Double) {
    fieldSchema = Schema.create(Type.DOUBLE);
  } else if (o instanceof Float) {
    fieldSchema = Schema.create(Type.FLOAT);
  } else if (o == null) {
    fieldSchema = Schema.create(Type.NULL);
  } else if (o instanceof Boolean) {
    fieldSchema = Schema.create(Type.BOOLEAN);
  } else if (o instanceof Map) {
    fieldSchema = Schema.create(Type.MAP);
  }
  Field newField = new Field("FIELD_" + fields.size(), fieldSchema, "", null);
  fields.add(newField);
  avroObject.getSchema().setFields(fields);
}
 
Example #23
Source File: TestParquetImport.java    From aliyun-maxcompute-data-collectors with Apache License 2.0 6 votes vote down vote up
public void testFirstUnderscoreInColumnName() throws IOException {
  String [] names = { "_NAME" };
  String [] types = { "INT" };
  String [] vals = { "1987" };
  createTableWithColTypesAndNames(names, types, vals);

  runImport(getOutputArgv(true, null));

  Schema schema = getSchema();
  assertEquals(Type.RECORD, schema.getType());
  List<Field> fields = schema.getFields();
  assertEquals(types.length, fields.size());
  checkField(fields.get(0), "__NAME", Type.INT);

  DatasetReader<GenericRecord> reader = getReader();
  try {
    assertTrue(reader.hasNext());
    GenericRecord record1 = reader.next();
    assertEquals("__NAME", 1987, record1.get("__NAME"));
    assertFalse(reader.hasNext());
  } finally {
    reader.close();
  }
}
 
Example #24
Source File: AvroSchemaGenerator.java    From aliyun-maxcompute-data-collectors with Apache License 2.0 6 votes vote down vote up
private Type toAvroType(String columnName, int sqlType) {
  Properties mapping = options.getMapColumnJava();

  if (mapping.containsKey(columnName)) {
    String type = mapping.getProperty(columnName);
    if (LOG.isDebugEnabled()) {
      LOG.info("Overriding type of column " + columnName + " to " + type);
    }

    if (type.equalsIgnoreCase("INTEGER")) { return Type.INT; }
    if (type.equalsIgnoreCase("LONG")) { return Type.LONG; }
    if (type.equalsIgnoreCase("BOOLEAN")) { return Type.BOOLEAN; }
    if (type.equalsIgnoreCase("FLOAT")) { return Type.FLOAT; }
    if (type.equalsIgnoreCase("DOUBLE")) { return Type.DOUBLE; }
    if (type.equalsIgnoreCase("STRING")) { return Type.STRING; }
    if (type.equalsIgnoreCase("BYTES")) { return Type.BYTES; }

    // Mapping was not found
    throw new IllegalArgumentException("Cannot convert to AVRO type " + type);
  }

  return connManager.toAvroType(tableName, columnName, sqlType);
}
 
Example #25
Source File: AvroUtilsTest.java    From beam with Apache License 2.0 5 votes vote down vote up
@Test
public void testNullableArrayFieldToBeamArrayField() {
  org.apache.avro.Schema.Field avroField =
      new org.apache.avro.Schema.Field(
          "arrayField",
          ReflectData.makeNullable(
              org.apache.avro.Schema.createArray((org.apache.avro.Schema.create(Type.INT)))),
          "",
          null);

  Field expectedBeamField = Field.nullable("arrayField", FieldType.array(FieldType.INT32));

  Field beamField = AvroUtils.toBeamField(avroField);
  assertEquals(expectedBeamField, beamField);
}
 
Example #26
Source File: AvroToJdbcEntryConverter.java    From incubator-gobblin with Apache License 2.0 5 votes vote down vote up
/**
 * Flattens Avro's (possibly recursive) structure and provides field name and type.
 * It assumes that the leaf level field name has unique name.
 * @param schema
 * @return
 * @throws SchemaConversionException if there's duplicate name in leaf level of Avro Schema
 */
private static Map<String, Type> flatten(Schema schema) throws SchemaConversionException {
  Map<String, Type> flattened = new LinkedHashMap<>();
  Schema recordSchema = determineType(schema);

  Preconditions.checkArgument(Type.RECORD.equals(recordSchema.getType()), "%s is expected. Schema: %s",
      Type.RECORD, recordSchema);

  for (Field f : recordSchema.getFields()) {
    produceFlattenedHelper(f, flattened);
  }
  return flattened;
}
 
Example #27
Source File: KeyValueUtils.java    From components with Apache License 2.0 5 votes vote down vote up
/**
 * Generate a new Index Record which is the filtered result of the input record.
 *
 * The user can freely remove column, add empty column or change the place of column in the same hierarchical level.
 *
 * @return the new record
 */
public static IndexedRecord extractIndexedRecord(IndexedRecord inputRecord, Schema outputSchema) {
    GenericRecordBuilder outputRecord = new GenericRecordBuilder(outputSchema);
    Schema inputSchema = getUnwrappedSchema(inputRecord);
    for (Field field : outputSchema.getFields()) {
        if (inputSchema.getField(field.name()) != null) {
            // The column was existing on the input record, we forward it to the output record.
            Object inputValue = inputRecord.get(inputSchema.getField(field.name()).pos());

            // The current column can be a Record (an hierarchical sub-object) or directly a value.
            // If we are on a record, we need to recursively do the process
            // if we are on a object, we save it to the output.
            if (inputValue instanceof Record) {
                // The sub-schema at this level is a union of "empty" and a record,
                // so we need to get the true sub-schema
                Schema inputChildSchema = getUnwrappedSchema(inputSchema.getField(field.name()));
                Schema outputChildSchema = getUnwrappedSchema(outputSchema.getField(field.name()));
                if (inputChildSchema.getType().equals(Type.RECORD)
                        && outputChildSchema.getType().equals(Type.RECORD)) {
                    Object childRecord = extractIndexedRecord((IndexedRecord) inputValue, outputChildSchema);
                    outputRecord.set(field.name(), childRecord);
                }
            } else {
                outputRecord.set(field.name(), inputValue);
            }
        } else {
            // element not found => set to the value and its hierarchy to null
            outputRecord.set(field.name(), KeyValueUtils.generateEmptyRecord(outputSchema, field.name()));
        }
    }
    return outputRecord.build();
}
 
Example #28
Source File: AvroUtils.java    From beam with Apache License 2.0 5 votes vote down vote up
@Override
protected java.lang.reflect.Type convertDefault(TypeDescriptor<?> type) {
  if (type.isSubtypeOf(TypeDescriptor.of(GenericFixed.class))) {
    return byte[].class;
  } else {
    return super.convertDefault(type);
  }
}
 
Example #29
Source File: MarketoClientUtilsTest.java    From components with Apache License 2.0 5 votes vote down vote up
@Test
public void testIsDateTypeField() throws Exception {
    assertTrue(MarketoClientUtils.isDateTypeField(fValidDate));
    assertFalse(MarketoClientUtils.isDateTypeField(null));
    assertFalse(MarketoClientUtils.isDateTypeField(fInvalidDate));
    assertFalse(MarketoClientUtils.isDateTypeField(generateFieldType(Type.LONG)));
}
 
Example #30
Source File: AvroRecordInputFormat.java    From stratosphere with Apache License 2.0 5 votes vote down vote up
private final Value convertAvroPrimitiveToValue(Type type, Object avroRecord) {
	switch (type) {
	case STRING:
		sString.setValue((CharSequence) avroRecord);
		return sString;
	case INT:
		sInt.setValue((Integer) avroRecord);
		return sInt;
	case BOOLEAN:
		sBool.setValue((Boolean) avroRecord);
		return sBool;
	case DOUBLE:
		sDouble.setValue((Double) avroRecord);
		return sDouble;
	case FLOAT:
		sFloat.setValue((Float) avroRecord);
		return sFloat;
	case LONG:
		sLong.setValue((Long) avroRecord);
		return sLong;
	case NULL:
		return NullValue.getInstance();
	default:
		throw new RuntimeException(
				"Type "
						+ type
						+ " for AvroInputFormat is not implemented. Open an issue on GitHub.");
	}
}