Java Code Examples for org.apache.avro.Schema.Type#RECORD

private static Stream<TableFieldSchema> mapTableFieldSchema(
    TableFieldSchema fieldSchema, Schema avroSchema) {
  Field avroFieldSchema = avroSchema.getField(fieldSchema.getName());
  if (avroFieldSchema == null) {
    return Stream.empty();
  } else if (avroFieldSchema.schema().getType() != Type.RECORD) {
    return Stream.of(fieldSchema);
  }

  List<TableFieldSchema> subSchemas =
      fieldSchema.getFields().stream()
          .flatMap(subSchema -> mapTableFieldSchema(subSchema, avroFieldSchema.schema()))
          .collect(Collectors.toList());

  TableFieldSchema output =
      new TableFieldSchema()
          .setCategories(fieldSchema.getCategories())
          .setDescription(fieldSchema.getDescription())
          .setFields(subSchemas)
          .setMode(fieldSchema.getMode())
          .setName(fieldSchema.getName())
          .setType(fieldSchema.getType());

  return Stream.of(output);
}
 

/**
 * Checks whether given schema is Root schema
 * It checks names of this schema and its 2 child fields
 * Also it checks schema type
 * This schema name is supposed to be "Root"
 * Field name are supposed to be "Main" and "OutOfBand"
 * However this method doesn't check field schemas types
 * 
 * @param avro record schema
 * @return true is given schema is Root; false otherwise
 */
public static boolean isRootSchema(final Schema schema) {
    if (schema == null) {
        return false;
    }

    String schemaName = schema.getName();
    if (!"Root".equals(schemaName)) {
        return false;
    }
    if (Type.RECORD != schema.getType()) {
        return false;
    }
    Field main = schema.getField(MAIN_FIELD_NAME);
    Field outOfBand = schema.getField(OUTOFBAND_FIELD_NAME);
    if (main == null || outOfBand == null) {
        return false;
    }
    return true;
}
 

private static Type convertToAvroType(DataType colType)
{
    final Type subType;
    if (colType == DataType.TUPLE)
    {
        /* Pig converts RECORD to TUPLE. Converting it back. */
        subType = Type.RECORD;
    }
    else if (colType == DataType.BAG)
    {
        subType = Type.ARRAY;
    }
    else if (colType == DataType.MAP)
    {
      subType = Type.MAP;
    }
    else
    {
        subType = Type.valueOf(colType.toString().toUpperCase());
    }
    return subType;
}
 

/**
 * Helper method that does the actual work for {@link #getFieldSchema(Schema, String)}
 * @param schema passed from {@link #getFieldSchema(Schema, String)}
 * @param pathList passed from {@link #getFieldSchema(Schema, String)}
 * @param field keeps track of the index used to access the list pathList
 * @return the schema of the field
 */
private static Optional<Schema> getFieldSchemaHelper(Schema schema, List<String> pathList, int field) {
  if (schema.getType() == Type.RECORD && schema.getField(pathList.get(field)) == null) {
    return Optional.absent();
  }
  switch (schema.getType()) {
    case UNION:
      if (AvroSerdeUtils.isNullableType(schema)) {
        return AvroUtils.getFieldSchemaHelper(AvroSerdeUtils.getOtherTypeFromNullableType(schema), pathList, field);
      }
      throw new AvroRuntimeException("Union of complex types cannot be handled : " + schema);
    case MAP:
      if ((field + 1) == pathList.size()) {
        return Optional.fromNullable(schema.getValueType());
      }
      return AvroUtils.getFieldSchemaHelper(schema.getValueType(), pathList, ++field);
    case RECORD:
      if ((field + 1) == pathList.size()) {
        return Optional.fromNullable(schema.getField(pathList.get(field)).schema());
      }
      return AvroUtils.getFieldSchemaHelper(schema.getField(pathList.get(field)).schema(), pathList, ++field);
    default:
      throw new AvroRuntimeException("Invalid type in schema : " + schema);
  }
}
 

/**
 * Packs a Pig DataBag into an Avro array.
 * @param db the Pig databad to pack into the avro array
 * @param s The avro schema for which to determine the type
 * @return the avro array corresponding to the input bag
 * @throws IOException
 */
public static GenericData.Array<Object> packIntoAvro(
    final DataBag db, final Schema s) throws IOException {

  try {
    GenericData.Array<Object> array
      = new GenericData.Array<Object>(new Long(db.size()).intValue(), s);
    for (Tuple t : db) {
      if (s.getElementType() != null
          && s.getElementType().getType() == Type.RECORD) {
        array.add(packIntoAvro(t, s.getElementType()));
      } else if (t.size() == 1) {
        array.add(t.get(0));
      } else {
        throw new IOException(
            "AvroStorageDataConversionUtilities.packIntoAvro: Can't pack "
                + t + " into schema " + s);
      }
    }
    return array;
  } catch (Exception e) {
    throw new IOException(
        "exception in AvroStorageDataConversionUtilities.packIntoAvro", e);
  }
}
 

public static DataType getColumnType(Field field) {
  org.apache.avro.Schema fieldSchema = field.schema();
  fieldSchema = extractSchemaFromUnionIfNeeded(fieldSchema);

  final Type type = fieldSchema.getType();
  if (type == Type.ARRAY) {
    org.apache.avro.Schema elementSchema = extractSchemaFromUnionIfNeeded(fieldSchema.getElementType());
    if (elementSchema.getType() == Type.RECORD) {
      if (elementSchema.getFields().size() == 1) {
        elementSchema = elementSchema.getFields().get(0).schema();
      } else {
        throw new RuntimeException("More than one schema in Multi-value column!");
      }
      elementSchema = extractSchemaFromUnionIfNeeded(elementSchema);
    }
    return AvroSchemaUtil.valueOf(elementSchema.getType());
  } else {
    return AvroSchemaUtil.valueOf(type);
  }
}
 

private static Field convertField(TableFieldSchema bigQueryField) {
  ImmutableCollection<Type> avroTypes = BIG_QUERY_TO_AVRO_TYPES.get(bigQueryField.getType());
  if (avroTypes.isEmpty()) {
    throw new IllegalArgumentException(
        "Unable to map BigQuery field type " + bigQueryField.getType() + " to avro type.");
  }

  Type avroType = avroTypes.iterator().next();
  Schema elementSchema;
  if (avroType == Type.RECORD) {
    elementSchema = toGenericAvroSchema(bigQueryField.getName(), bigQueryField.getFields());
  } else {
    elementSchema = Schema.create(avroType);
  }
  Schema fieldSchema;
  if (bigQueryField.getMode() == null || "NULLABLE".equals(bigQueryField.getMode())) {
    fieldSchema = Schema.createUnion(Schema.create(Type.NULL), elementSchema);
  } else if ("REQUIRED".equals(bigQueryField.getMode())) {
    fieldSchema = elementSchema;
  } else if ("REPEATED".equals(bigQueryField.getMode())) {
    fieldSchema = Schema.createArray(elementSchema);
  } else {
    throw new IllegalArgumentException(
        String.format("Unknown BigQuery Field Mode: %s", bigQueryField.getMode()));
  }
  return new Field(
      bigQueryField.getName(),
      fieldSchema,
      bigQueryField.getDescription(),
      (Object) null /* Cast to avoid deprecated JsonNode constructor. */);
}
 

private final Type checkTypeConstraintsAndGetType(final Schema schema) {
	final Type type = schema.getType();
	if (type == Type.RECORD) {
		throw new RuntimeException("The given Avro file contains complex data types which are not supported right now");
	}

	if (type == Type.UNION) {
		List<Schema> types = schema.getTypes();
		if (types.size() > 2) {
			throw new RuntimeException("The given Avro file contains a union that has more than two elements");
		}
		if (types.size() == 1 && types.get(0).getType() != Type.UNION) {
			return types.get(0).getType();
		}
		if (types.get(0).getType() == Type.UNION || types.get(1).getType() == Type.UNION) {
			throw new RuntimeException("The given Avro file contains a nested union");
		}
		if (types.get(0).getType() == Type.NULL) {
			return types.get(1).getType();
		} else {
			if (types.get(1).getType() != Type.NULL) {
				throw new RuntimeException("The given Avro file is contains a union with two non-null types.");
			}
			return types.get(0).getType();
		}
	}
	return type;
}
 

private static void validateIsRecord(Schema schema) throws SchemaValidationException {
  if (schema.getType() != Type.RECORD) {
    throw new SchemaValidationException(
        String.format("Unexpected schema root type '%s', expected '%s'", schema.getType(), Type.RECORD));
  }
}
 

/**
 *
 */
private static void updateRecord(Field field, Type type, String providedValue, Record avroRecord) {
    if (Type.NULL != type) {
        Object value;
        if (Type.INT == type) {
            value = null == providedValue ? possiblyGetDefaultValue(field, IntNode.class).getIntValue()
                    : Integer.parseInt(providedValue);
            avroRecord.put(field.name(), value);
        } else if (Type.BOOLEAN == type) {
            value = null == providedValue
                    ? possiblyGetDefaultValue(field, BooleanNode.class).getBooleanValue()
                    : Boolean.parseBoolean(providedValue);
            avroRecord.put(field.name(), value);
        } else if (Type.DOUBLE == type) {
            value = null == providedValue ? possiblyGetDefaultValue(field, DoubleNode.class).getDoubleValue()
                    : Double.parseDouble(providedValue);
            avroRecord.put(field.name(), value);
        } else if (Type.FLOAT == type) {
            value = null == providedValue ? possiblyGetDefaultValue(field, DoubleNode.class).getDoubleValue()
                    : Float.parseFloat(providedValue);
            avroRecord.put(field.name(), value);
        } else if (Type.LONG == type) {
            value = null == providedValue ? possiblyGetDefaultValue(field, LongNode.class).getLongValue()
                    : Long.parseLong(providedValue);
            avroRecord.put(field.name(), value);
        } else if (Type.STRING == type) {
            value = null == providedValue ? possiblyGetDefaultValue(field, TextNode.class).getTextValue()
                    : providedValue;
            avroRecord.put(field.name(), value);
        } else if (Type.BYTES == type) {
            value = encodeLogicalType(field, providedValue);
            avroRecord.put(field.name(), value);
        } else if (Type.UNION == type) {
            field.schema().getTypes()
                    .forEach(schema -> updateRecord(field, schema.getType(), providedValue, avroRecord));
        } else if (Type.ARRAY == type || Type.ENUM == type || Type.FIXED == type || Type.MAP == type
                || Type.NULL == type || Type.RECORD == type) {
            throw new IllegalArgumentException("The field type '" + type + "' is not supported at the moment");
        } else {
            avroRecord.put(field.name(), providedValue);
        }
    }
}