Java Code Examples for org.apache.avro.Schema#createUnion()

The following examples show how to use org.apache.avro.Schema#createUnion() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
/**
 * Generate schema for given field and optionally wrap it in union with null if configured.
 */
private Schema.Field schemaFieldForType(
    String fieldPath,
    Record record,
    String fieldName,
    Field field
) throws OnRecordErrorException {
  Schema simpleSchema = simpleSchemaForType(fieldPath, record, field);
  Schema finalSchema = simpleSchema;

  // If Nullable check box was selected, wrap the whole schema in union with null
  if(getConfig().avroNullableFields) {
    finalSchema = Schema.createUnion(ImmutableList.of(
      Schema.create(Schema.Type.NULL),
      simpleSchema
    ));
  }

  return new Schema.Field(
    fieldName,
    finalSchema,
    null,
    getDefaultValue(simpleSchema)
  );
}
 
Example 2
@Test
public void testUnionOfTwoTypes() throws Exception {
  Schema schema = Schema.createRecord("record2", null, null, false);
  Schema multipleTypes = Schema.createUnion(Arrays.asList(Schema.create(Schema.Type
          .NULL),
      Schema.create(INT),
      Schema.create(Schema.Type.FLOAT)));
  schema.setFields(Arrays.asList(
      new Schema.Field("myunion", multipleTypes, null, JsonProperties.NULL_VALUE)));

  // Avro union is modelled using optional data members of the different
  // types. This does not translate back into an Avro union
  testAvroToParquetConversion(
      schema,
      "message record2 {\n" +
          "  optional group myunion {\n" +
          "    optional int32 member0;\n" +
          "    optional float member1;\n" +
          "  }\n" +
          "}\n");
}
 
Example 3
Source Project: kite   File: DataModelUtil.java    License: Apache License 2.0 6 votes vote down vote up
/** Create and return a union of the null schema and the provided schema. */
public static Schema makeNullableSchema(Schema schema) {
  if (schema.getType() == Schema.Type.UNION) {
    // check to see if the union already contains NULL
    for (Schema subType : schema.getTypes()) {
      if (subType.getType() == Schema.Type.NULL) {
        return schema;
      }
    }
    // add null as the first type in a new union
    List<Schema> withNull = new ArrayList<Schema>();
    withNull.add(Schema.create(Schema.Type.NULL));
    withNull.addAll(schema.getTypes());
    return Schema.createUnion(withNull);
  } else {
    // create a union with null
    return Schema.createUnion(Arrays.asList(Schema.create(Schema.Type.NULL),
        schema));
  }
}
 
Example 4
Source Project: funcj   File: GenerateSchema.java    License: MIT License 5 votes vote down vote up
@Override
public Schema shortArr(IList<String> path, String name) {
    return Schema.createUnion(
            Schema.createArray(Schema.create(Schema.Type.INT)),
            Schema.create(Schema.Type.NULL)
    );
}
 
Example 5
private static Schema optional(Schema original) {
  // null is first in the union because Parquet's default is always null
  return Schema.createUnion(Arrays.asList(
      Schema.create(Schema.Type.NULL),
      original)
  );
}
 
Example 6
Source Project: funcj   File: GenerateSchema.java    License: MIT License 5 votes vote down vote up
@Override
public Schema charB(IList<String> path, String name) {
    return Schema.createUnion(
            Schema.create(Schema.Type.STRING),
            Schema.create(Schema.Type.NULL)
    );
}
 
Example 7
Source Project: spork   File: AvroStorageUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Wrap an avro schema as a nullable union if needed.
 * For instance, wrap schema "int" as ["null", "int"]
 */
public static Schema wrapAsUnion(Schema schema, boolean nullable) {
    if (nullable) {
        /* if schema is an acceptable union, then return itself */
        if (schema.getType().equals(Schema.Type.UNION)
                && isAcceptableUnion(schema))
            return schema;
        else
            return Schema.createUnion(Arrays.asList(NullSchema, schema));
    } else
        /*do not wrap it if not */
        return schema;
}
 
Example 8
Source Project: funcj   File: GenerateSchema.java    License: MIT License 5 votes vote down vote up
@Override
public Schema byteB(IList<String> path, String name) {
    return Schema.createUnion(
            Schema.createFixed(toName(path, name), null, null, 1),
            Schema.create(Schema.Type.NULL)
    );
}
 
Example 9
Source Project: funcj   File: GenerateSchema.java    License: MIT License 5 votes vote down vote up
@Override
public Schema stringMap(IList<String> path, String name, Schema value) {
    return Schema.createUnion(
            Schema.createMap(value),
            Schema.create(Schema.Type.NULL)
    );
}
 
Example 10
/**
 * avro schema for the converted type
 * @return
 */
public Schema getSchema() {
  if (isNullable()) {
    List<Schema> list = new ArrayList<>();
    list.add(Schema.create(Schema.Type.NULL));
    list.add(schema());
    return Schema.createUnion(list);
  }
  return schema();
}
 
Example 11
Source Project: funcj   File: AvroSchemaCodecFormat.java    License: MIT License 5 votes vote down vote up
@Override
public Object encode(
        CodecCoreEx<WithSchema, Object, Config> core,
        int[] value,
        Object out
) {
    return Schema.createUnion(
            Schema.createArray(Schema.create(Schema.Type.INT)),
            Schema.create(Schema.Type.NULL)
    );
}
 
Example 12
Source Project: beam   File: AvroGenerators.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Schema generate(SourceOfRandomness random, GenerationStatus status) {
  Map<String, Schema> schemaMap =
      IntStream.range(0, random.nextInt(0, status.size()) + 1)
          .mapToObj(
              i -> {
                // deterministically avoid collisions in record names
                branchPush(status, String.valueOf(i));
                Schema schema =
                    SchemaGenerator.INSTANCE
                        // nested unions aren't supported in AVRO
                        .filter(x -> x.getType() != Schema.Type.UNION)
                        .generate(random, status);
                branchPop(status);
                return schema;
              })
          // AVRO requires uniqueness by full name
          .collect(Collectors.toMap(Schema::getFullName, Function.identity(), (x, y) -> x));

  List<Schema> schemas = new ArrayList<>(schemaMap.values());

  if (random.nextBoolean()) {
    org.apache.avro.Schema nullSchema = org.apache.avro.Schema.create(Schema.Type.NULL);
    schemas.add(nullSchema);
    Collections.shuffle(schemas, random.toJDKRandom());
  }

  return Schema.createUnion(schemas);
}
 
Example 13
Source Project: funcj   File: GenerateSchema.java    License: MIT License 5 votes vote down vote up
@Override
public Schema longB(IList<String> path, String name) {
    return Schema.createUnion(
            Schema.create(Schema.Type.LONG),
            Schema.create(Schema.Type.NULL)
    );
}
 
Example 14
Source Project: funcj   File: AvroSchemaCodecFormat.java    License: MIT License 5 votes vote down vote up
@Override
public Object encode(
        CodecCoreEx<WithSchema, Object, Config> core,
        boolean[] value,
        Object out
) {
    return Schema.createUnion(
            Schema.createArray(Schema.create(Schema.Type.BOOLEAN)),
            Schema.create(Schema.Type.NULL)
    );
}
 
Example 15
Source Project: funcj   File: GenerateSchema.java    License: MIT License 5 votes vote down vote up
@Override
public Schema integerB(IList<String> path, String name) {
    return Schema.createUnion(
            Schema.create(Schema.Type.INT),
            Schema.create(Schema.Type.NULL)
    );
}
 
Example 16
Source Project: funcj   File: GenerateSchema.java    License: MIT License 4 votes vote down vote up
@Override
public Schema booleanB(IList<String> path, String name) {
    return Schema.createUnion(Schema.create(Schema.Type.BOOLEAN), Schema.create(Schema.Type.NULL));
}
 
Example 17
protected Schema buildUnionIfNullable(Schema schema) {
  if (this.isNullable()) {
    return Schema.createUnion(Arrays.asList(Schema.create(Schema.Type.NULL), schema));
  }
  return schema;
}
 
Example 18
public static DataSet<Tuple2<Void, Person>> readAvro(ExecutionEnvironment env, String inputPath) throws
	IOException {
	Job job = Job.getInstance();

	HadoopInputFormat hadoopInputFormat = new HadoopInputFormat(new AvroParquetInputFormat(), Void.class, Person
		.class, job);

	FileInputFormat.addInputPath(job, new Path(inputPath));

	// schema projection: don't read type of phonenumber     
	Schema phone = Schema.createRecord("PhoneNumber", null, null, false);
	phone.setFields(Arrays.asList(
		new Schema.Field("number", Schema.create(Schema.Type.BYTES), null, null)));

	Schema array = Schema.createArray(phone);
	Schema union = Schema.createUnion(Lists.newArrayList(Schema.create(Schema.Type.BYTES), Schema.create(Schema
		.Type
		.NULL)));


	Schema projection = Schema.createRecord("Person", null, null, false);
	projection.setFields(
		Arrays.asList(
			new Schema.Field("name", Schema.create(Schema.Type.BYTES), null, null),
			new Schema.Field("id", Schema.create(Schema.Type.INT), null, null),
			new Schema.Field("email", union, null, null),
			new Schema.Field("phone", array, null, null)
		)
	);

	AvroParquetInputFormat.setRequestedProjection(job, projection);

	// push down predicates: get all persons with name = "Felix"
	BinaryColumn name = binaryColumn("name");
	FilterPredicate namePred = eq(name, Binary.fromString("Felix"));
	ParquetInputFormat.setFilterPredicate(job.getConfiguration(), namePred);

	DataSet<Tuple2<Void, Person>> data = env.createInput(hadoopInputFormat);

	return data;
}
 
Example 19
Source Project: simplesource   File: AvroGenericUtils.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Return the given schema wrapped in a nullable union.
 */
private static Schema toNullableSchema(final Schema schema) {
    return Schema.createUnion(Arrays.asList(Schema.create(Schema.Type.NULL), schema));
}
 
Example 20
/**
 * Will create union, because each type is assumed to be nullable.
 *
 * @param sqlType Original SQL type (might be overridden by user)
 * @param columnName Column name from the query
 * @return Schema
 */
public Schema toAvroSchema(int sqlType, String columnName) {
  List<Schema> childSchemas = new ArrayList<Schema>();
  childSchemas.add(Schema.create(Schema.Type.NULL));
  childSchemas.add(Schema.create(toAvroType(columnName, sqlType)));
  return Schema.createUnion(childSchemas);
}