Java Code Examples for org.apache.avro.Schema#createUnion()

/**
 * Generate schema for given field and optionally wrap it in union with null if configured.
 */
private Schema.Field schemaFieldForType(
    String fieldPath,
    Record record,
    String fieldName,
    Field field
) throws OnRecordErrorException {
  Schema simpleSchema = simpleSchemaForType(fieldPath, record, field);
  Schema finalSchema = simpleSchema;

  // If Nullable check box was selected, wrap the whole schema in union with null
  if(getConfig().avroNullableFields) {
    finalSchema = Schema.createUnion(ImmutableList.of(
      Schema.create(Schema.Type.NULL),
      simpleSchema
    ));
  }

  return new Schema.Field(
    fieldName,
    finalSchema,
    null,
    getDefaultValue(simpleSchema)
  );
}
 

@Test
public void testUnionOfTwoTypes() throws Exception {
  Schema schema = Schema.createRecord("record2", null, null, false);
  Schema multipleTypes = Schema.createUnion(Arrays.asList(Schema.create(Schema.Type
          .NULL),
      Schema.create(INT),
      Schema.create(Schema.Type.FLOAT)));
  schema.setFields(Arrays.asList(
      new Schema.Field("myunion", multipleTypes, null, JsonProperties.NULL_VALUE)));

  // Avro union is modelled using optional data members of the different
  // types. This does not translate back into an Avro union
  testAvroToParquetConversion(
      schema,
      "message record2 {\n" +
          "  optional group myunion {\n" +
          "    optional int32 member0;\n" +
          "    optional float member1;\n" +
          "  }\n" +
          "}\n");
}
 

/** Create and return a union of the null schema and the provided schema. */
public static Schema makeNullableSchema(Schema schema) {
  if (schema.getType() == Schema.Type.UNION) {
    // check to see if the union already contains NULL
    for (Schema subType : schema.getTypes()) {
      if (subType.getType() == Schema.Type.NULL) {
        return schema;
      }
    }
    // add null as the first type in a new union
    List<Schema> withNull = new ArrayList<Schema>();
    withNull.add(Schema.create(Schema.Type.NULL));
    withNull.addAll(schema.getTypes());
    return Schema.createUnion(withNull);
  } else {
    // create a union with null
    return Schema.createUnion(Arrays.asList(Schema.create(Schema.Type.NULL),
        schema));
  }
}
 

@Override
public Schema integerB(IList<String> path, String name) {
    return Schema.createUnion(
            Schema.create(Schema.Type.INT),
            Schema.create(Schema.Type.NULL)
    );
}
 

@Override
public Object encode(
        CodecCoreEx<WithSchema, Object, Config> core,
        boolean[] value,
        Object out
) {
    return Schema.createUnion(
            Schema.createArray(Schema.create(Schema.Type.BOOLEAN)),
            Schema.create(Schema.Type.NULL)
    );
}
 

@Override
public Schema longB(IList<String> path, String name) {
    return Schema.createUnion(
            Schema.create(Schema.Type.LONG),
            Schema.create(Schema.Type.NULL)
    );
}
 

@Override
public Schema generate(SourceOfRandomness random, GenerationStatus status) {
  Map<String, Schema> schemaMap =
      IntStream.range(0, random.nextInt(0, status.size()) + 1)
          .mapToObj(
              i -> {
                // deterministically avoid collisions in record names
                branchPush(status, String.valueOf(i));
                Schema schema =
                    SchemaGenerator.INSTANCE
                        // nested unions aren't supported in AVRO
                        .filter(x -> x.getType() != Schema.Type.UNION)
                        .generate(random, status);
                branchPop(status);
                return schema;
              })
          // AVRO requires uniqueness by full name
          .collect(Collectors.toMap(Schema::getFullName, Function.identity(), (x, y) -> x));

  List<Schema> schemas = new ArrayList<>(schemaMap.values());

  if (random.nextBoolean()) {
    org.apache.avro.Schema nullSchema = org.apache.avro.Schema.create(Schema.Type.NULL);
    schemas.add(nullSchema);
    Collections.shuffle(schemas, random.toJDKRandom());
  }

  return Schema.createUnion(schemas);
}
 

@Override
public Object encode(
        CodecCoreEx<WithSchema, Object, Config> core,
        int[] value,
        Object out
) {
    return Schema.createUnion(
            Schema.createArray(Schema.create(Schema.Type.INT)),
            Schema.create(Schema.Type.NULL)
    );
}
 

/**
 * avro schema for the converted type
 * @return
 */
public Schema getSchema() {
  if (isNullable()) {
    List<Schema> list = new ArrayList<>();
    list.add(Schema.create(Schema.Type.NULL));
    list.add(schema());
    return Schema.createUnion(list);
  }
  return schema();
}
 

@Override
public Schema stringMap(IList<String> path, String name, Schema value) {
    return Schema.createUnion(
            Schema.createMap(value),
            Schema.create(Schema.Type.NULL)
    );
}
 

@Override
public Schema byteB(IList<String> path, String name) {
    return Schema.createUnion(
            Schema.createFixed(toName(path, name), null, null, 1),
            Schema.create(Schema.Type.NULL)
    );
}
 

/**
 * Wrap an avro schema as a nullable union if needed.
 * For instance, wrap schema "int" as ["null", "int"]
 */
public static Schema wrapAsUnion(Schema schema, boolean nullable) {
    if (nullable) {
        /* if schema is an acceptable union, then return itself */
        if (schema.getType().equals(Schema.Type.UNION)
                && isAcceptableUnion(schema))
            return schema;
        else
            return Schema.createUnion(Arrays.asList(NullSchema, schema));
    } else
        /*do not wrap it if not */
        return schema;
}
 

@Override
public Schema charB(IList<String> path, String name) {
    return Schema.createUnion(
            Schema.create(Schema.Type.STRING),
            Schema.create(Schema.Type.NULL)
    );
}
 

private static Schema optional(Schema original) {
  // null is first in the union because Parquet's default is always null
  return Schema.createUnion(Arrays.asList(
      Schema.create(Schema.Type.NULL),
      original)
  );
}
 

@Override
public Schema shortArr(IList<String> path, String name) {
    return Schema.createUnion(
            Schema.createArray(Schema.create(Schema.Type.INT)),
            Schema.create(Schema.Type.NULL)
    );
}
 

/**
 * Return the given schema wrapped in a nullable union.
 */
private static Schema toNullableSchema(final Schema schema) {
    return Schema.createUnion(Arrays.asList(Schema.create(Schema.Type.NULL), schema));
}
 

public static DataSet<Tuple2<Void, Person>> readAvro(ExecutionEnvironment env, String inputPath) throws
	IOException {
	Job job = Job.getInstance();

	HadoopInputFormat hadoopInputFormat = new HadoopInputFormat(new AvroParquetInputFormat(), Void.class, Person
		.class, job);

	FileInputFormat.addInputPath(job, new Path(inputPath));

	// schema projection: don't read type of phonenumber     
	Schema phone = Schema.createRecord("PhoneNumber", null, null, false);
	phone.setFields(Arrays.asList(
		new Schema.Field("number", Schema.create(Schema.Type.BYTES), null, null)));

	Schema array = Schema.createArray(phone);
	Schema union = Schema.createUnion(Lists.newArrayList(Schema.create(Schema.Type.BYTES), Schema.create(Schema
		.Type
		.NULL)));


	Schema projection = Schema.createRecord("Person", null, null, false);
	projection.setFields(
		Arrays.asList(
			new Schema.Field("name", Schema.create(Schema.Type.BYTES), null, null),
			new Schema.Field("id", Schema.create(Schema.Type.INT), null, null),
			new Schema.Field("email", union, null, null),
			new Schema.Field("phone", array, null, null)
		)
	);

	AvroParquetInputFormat.setRequestedProjection(job, projection);

	// push down predicates: get all persons with name = "Felix"
	BinaryColumn name = binaryColumn("name");
	FilterPredicate namePred = eq(name, Binary.fromString("Felix"));
	ParquetInputFormat.setFilterPredicate(job.getConfiguration(), namePred);

	DataSet<Tuple2<Void, Person>> data = env.createInput(hadoopInputFormat);

	return data;
}
 

protected Schema buildUnionIfNullable(Schema schema) {
  if (this.isNullable()) {
    return Schema.createUnion(Arrays.asList(Schema.create(Schema.Type.NULL), schema));
  }
  return schema;
}
 

@Override
public Schema booleanB(IList<String> path, String name) {
    return Schema.createUnion(Schema.create(Schema.Type.BOOLEAN), Schema.create(Schema.Type.NULL));
}
 

/**
 * Will create union, because each type is assumed to be nullable.
 *
 * @param sqlType Original SQL type (might be overridden by user)
 * @param columnName Column name from the query
 * @return Schema
 */
public Schema toAvroSchema(int sqlType, String columnName) {
  List<Schema> childSchemas = new ArrayList<Schema>();
  childSchemas.add(Schema.create(Schema.Type.NULL));
  childSchemas.add(Schema.create(toAvroType(columnName, sqlType)));
  return Schema.createUnion(childSchemas);
}