Java Code Examples for org.apache.avro.SchemaBuilder#RecordBuilder
The following examples show how to use
org.apache.avro.SchemaBuilder#RecordBuilder .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: AvroUtils.java From envelope with Apache License 2.0 | 6 votes |
|
private static Schema schemaFor(StructType structType, String record, String namespace, int recordCount) {
// Increment after using the recordCount
record = (null == record) ? "record" + recordCount++ : record;
LOG.debug("Converting {} to Avro Record schema [{}:{}]", structType, record, namespace);
SchemaBuilder.RecordBuilder<Schema> schema = SchemaBuilder.record(record);
if (null != namespace) {
schema.namespace(namespace);
}
schema.doc("Auto-generated from Spark DataFrame");
SchemaBuilder.FieldAssembler<Schema> assembler = schema.fields();
StructField[] structFields = structType.fields();
for (StructField f : structFields) {
assembler.name(f.name()).type(typeFor(f.dataType(), f.nullable(), recordCount)).noDefault();
}
return assembler.endRecord();
}
Example 2
| Source File: ImportFromAvroTest.java From DataflowTemplates with Apache License 2.0 | 5 votes |
|
@Test
public void booleans() throws Exception {
SchemaBuilder.RecordBuilder<Schema> record = SchemaBuilder.record("booleans");
SchemaBuilder.FieldAssembler<Schema> fieldAssembler = record.fields();
fieldAssembler
// Primary key.
.requiredLong("id")
// Boolean columns.
.optionalBoolean("optional_boolean")
.requiredBoolean("required_boolean")
.optionalString("optional_string_boolean")
.requiredString("required_string_boolean");
Schema schema = fieldAssembler.endRecord();
String spannerSchema =
"CREATE TABLE `AvroTable` (" + "`id` INT64 NOT NULL,"
+ "`optional_boolean` BOOL,"
+ "`required_boolean` BOOL NOT NULL,"
+ "`optional_string_boolean` BOOL,"
+ "`required_string_boolean` BOOL NOT NULL,"
+ ") PRIMARY KEY (`id`)";
runTest(schema, spannerSchema, Arrays.asList(new GenericRecordBuilder(schema)
.set("id", 1L)
.set("required_boolean", true)
.set("optional_boolean", false)
.set("required_string_boolean", "FALSE")
.set("optional_string_boolean", "TRUE")
.build(), new GenericRecordBuilder(schema)
.set("id", 2L)
.set("required_boolean", false)
.set("optional_boolean", true)
.set("required_string_boolean", "true")
.set("optional_string_boolean", "f")
.build()));
}
Example 3
| Source File: DdlToAvroSchemaConverter.java From DataflowTemplates with Apache License 2.0 | 4 votes |
|
public Collection<Schema> convert(Ddl ddl) {
Collection<Schema> schemas = new ArrayList<>();
for (Table table : ddl.allTables()) {
SchemaBuilder.RecordBuilder<Schema> recordBuilder =
SchemaBuilder.record(table.name()).namespace(this.namespace);
recordBuilder.prop("googleFormatVersion", version);
recordBuilder.prop("googleStorage", "CloudSpanner");
if (table.interleaveInParent() != null) {
recordBuilder.prop("spannerParent", table.interleaveInParent());
recordBuilder.prop(
"spannerOnDeleteAction", table.onDeleteCascade() ? "cascade" : "no action");
}
if (table.primaryKeys() != null) {
String encodedPk =
table.primaryKeys().stream()
.map(IndexColumn::prettyPrint)
.collect(Collectors.joining(","));
recordBuilder.prop("spannerPrimaryKey", encodedPk);
}
for (int i = 0; i < table.primaryKeys().size(); i++) {
recordBuilder.prop("spannerPrimaryKey_" + i, table.primaryKeys().get(i).prettyPrint());
}
for (int i = 0; i < table.indexes().size(); i++) {
recordBuilder.prop("spannerIndex_" + i, table.indexes().get(i));
}
for (int i = 0; i < table.foreignKeys().size(); i++) {
recordBuilder.prop("spannerForeignKey_" + i, table.foreignKeys().get(i));
}
SchemaBuilder.FieldAssembler<Schema> fieldsAssembler = recordBuilder.fields();
for (Column cm : table.columns()) {
SchemaBuilder.FieldBuilder<Schema> fieldBuilder = fieldsAssembler.name(cm.name());
fieldBuilder.prop("sqlType", cm.typeString());
for (int i = 0; i < cm.columnOptions().size(); i++) {
fieldBuilder.prop("spannerOption_" + i, cm.columnOptions().get(i));
}
Schema avroType = avroType(cm.type());
if (!cm.notNull()) {
avroType = wrapAsNullable(avroType);
}
fieldBuilder.type(avroType).noDefault();
}
Schema schema = fieldsAssembler.endRecord();
schemas.add(schema);
}
return schemas;
}
Example 4
| Source File: ImportFromAvroTest.java From DataflowTemplates with Apache License 2.0 | 4 votes |
|
@Test
public void integers() throws Exception {
SchemaBuilder.RecordBuilder<Schema> record = SchemaBuilder.record("integers");
SchemaBuilder.FieldAssembler<Schema> fieldAssembler = record.fields();
fieldAssembler
// Primary key.
.requiredLong("id")
// Integer columns.
.optionalInt("optional_int")
.requiredInt("required_int")
.requiredLong("required_long")
.optionalLong("optional_long")
.optionalString("optional_string_long")
.requiredString("required_string_long");
Schema schema = fieldAssembler.endRecord();
String spannerSchema =
"CREATE TABLE `AvroTable` (" + "`id` INT64 NOT NULL,"
+ "`optional_int` INT64,"
+ "`required_int` INT64 NOT NULL,"
+ "`optional_long` INT64,"
+ "`required_long` INT64 NOT NULL,"
+ "`optional_string_long` INT64,"
+ "`required_string_long` INT64 NOT NULL,"
+ ") PRIMARY KEY (`id`)";
runTest(schema, spannerSchema, Arrays.asList(new GenericRecordBuilder(schema)
.set("id", 1L)
.set("optional_int", 1)
.set("optional_long", 2L)
.set("required_long", 3L)
.set("required_int", 4)
.set("optional_string_long", "1000")
.set("required_string_long", "5000")
.build(), new GenericRecordBuilder(schema)
.set("id", 2L)
.set("optional_int", 10)
.set("optional_long", 20L)
.set("required_long", 30L)
.set("required_int", 40)
.set("optional_string_long", "10000")
.set("required_string_long", "50000")
.build()));
}
Example 5
| Source File: ImportFromAvroTest.java From DataflowTemplates with Apache License 2.0 | 4 votes |
|
@Test
public void floats() throws Exception {
SchemaBuilder.RecordBuilder<Schema> record = SchemaBuilder.record("floats");
SchemaBuilder.FieldAssembler<Schema> fieldAssembler = record.fields();
fieldAssembler
// Primary key.
.requiredLong("id")
// Integer columns.
.optionalInt("optional_int")
.requiredInt("required_int")
.requiredLong("required_long")
.optionalLong("optional_long")
// Floating columns
.optionalFloat("optional_float")
.requiredFloat("required_float")
.requiredDouble("required_double")
.optionalDouble("optional_double")
.optionalString("optional_string_double")
.requiredString("required_string_double");
Schema schema = fieldAssembler.endRecord();
String spannerSchema =
"CREATE TABLE `AvroTable` (" + "`id` INT64 NOT NULL,"
+ "`optional_int` FLOAT64,"
+ "`required_int` FLOAT64 NOT NULL,"
+ "`optional_long` FLOAT64,"
+ "`required_long` FLOAT64 NOT NULL,"
+ "`optional_float` FLOAT64,"
+ "`required_float` FLOAT64 NOT NULL,"
+ "`optional_double` FLOAT64,"
+ "`required_double` FLOAT64 NOT NULL,"
+ "`optional_string_double` FLOAT64,"
+ "`required_string_double` FLOAT64 NOT NULL,"
+ ") PRIMARY KEY (`id`)";
runTest(schema, spannerSchema, Arrays.asList(new GenericRecordBuilder(schema)
.set("id", 1L)
.set("optional_int", 1)
.set("required_int", 4)
.set("optional_long", 2L)
.set("required_long", 3L)
.set("optional_float", 2.3f)
.set("required_float", 3.4f)
.set("optional_double", 2.5)
.set("required_double", 3.6)
.set("optional_string_double", "100.30")
.set("required_string_double", "0.1e-3")
.build(), new GenericRecordBuilder(schema)
.set("id", 2L)
.set("optional_int", 10)
.set("required_int", 40)
.set("optional_long", 20L)
.set("required_long", 30L)
.set("optional_float", 2.03f)
.set("required_float", 3.14f)
.set("optional_double", 2.05)
.set("required_double", 3.16)
.set("optional_string_double", "100.301")
.set("required_string_double", "1.1e-3")
.build()));
}
Example 6
| Source File: ImportFromAvroTest.java From DataflowTemplates with Apache License 2.0 | 4 votes |
|
@Test
public void strings() throws Exception {
SchemaBuilder.RecordBuilder<Schema> record = SchemaBuilder.record("strings");
SchemaBuilder.FieldAssembler<Schema> fieldAssembler = record.fields();
fieldAssembler
// Primary key.
.requiredLong("id")
// Integer columns.
.optionalInt("optional_int")
.requiredInt("required_int")
.requiredLong("required_long")
.optionalLong("optional_long")
// Floating columns
.optionalFloat("optional_float")
.requiredFloat("required_float")
.requiredDouble("required_double")
.optionalDouble("optional_double")
// String columns
.optionalString("optional_string")
.requiredString("required_string");
Schema schema = fieldAssembler.endRecord();
String spannerSchema =
"CREATE TABLE `AvroTable` (" + "`id` INT64 NOT NULL,"
+ "`optional_int` STRING(10),"
+ "`required_int` STRING(MAX) NOT NULL,"
+ "`optional_long` STRING(MAX),"
+ "`required_long` STRING(MAX) NOT NULL,"
+ "`optional_float` STRING(MAX),"
+ "`required_float` STRING(MAX) NOT NULL,"
+ "`optional_double` STRING(MAX),"
+ "`required_double` STRING(MAX) NOT NULL,"
+ "`optional_string` STRING(MAX),"
+ "`required_string` STRING(30) NOT NULL,"
+ ") PRIMARY KEY (`id`)";
runTest(schema, spannerSchema, Arrays.asList(new GenericRecordBuilder(schema)
.set("id", 1L)
.set("optional_int", 1)
.set("required_int", 4)
.set("optional_long", 2L)
.set("required_long", 3L)
.set("optional_float", 2.3f)
.set("required_float", 3.4f)
.set("optional_double", 2.5)
.set("required_double", 3.6)
.set("optional_string", "ONE STRING")
.set("required_string", "TWO STRING")
.build(), new GenericRecordBuilder(schema)
.set("id", 2L)
.set("optional_int", 10)
.set("required_int", 40)
.set("optional_long", 20L)
.set("required_long", 30L)
.set("optional_float", 2.03f)
.set("required_float", 3.14f)
.set("optional_double", 2.05)
.set("required_double", 3.16)
.set("optional_string", null)
.set("required_string", "THE STRING")
.build()));
}
Example 7
| Source File: ImportFromAvroTest.java From DataflowTemplates with Apache License 2.0 | 4 votes |
|
@Test
public void timestamps() throws Exception {
SchemaBuilder.RecordBuilder<Schema> record = SchemaBuilder.record("timestamps");
SchemaBuilder.FieldAssembler<Schema> fieldAssembler = record.fields();
fieldAssembler
// Primary key.
.requiredLong("id")
// Long columns.
.requiredLong("required_long")
.optionalLong("optional_long")
// String columns
.optionalString("optional_string")
.requiredString("required_string");
Schema schema = fieldAssembler.endRecord();
String spannerSchema =
"CREATE TABLE `AvroTable` ("
+ "`id` INT64 NOT NULL,"
+ "`optional_long` TIMESTAMP,"
+ "`required_long` TIMESTAMP NOT NULL,"
+ "`optional_string` TIMESTAMP,"
+ "`required_string` TIMESTAMP NOT NULL,"
+ ") PRIMARY KEY (`id`)";
runTest(
schema,
spannerSchema,
Arrays.asList(
new GenericRecordBuilder(schema)
.set("id", 1L)
.set("optional_long", 5000000L)
.set("required_long", 6000000L)
.set("optional_string", "2018-06-06T21:00:35.312000000Z")
.set("required_string", "2018-06-06T21:00:35.312000000Z")
.build(),
new GenericRecordBuilder(schema)
.set("id", 2L)
.set("optional_long", 500000330L)
.set("required_long", 6000020000L)
.set("optional_string", "2017-06-06T21:00:35.312000000Z")
.set("required_string", "2017-06-06T21:00:35.312000000Z")
.build(),
new GenericRecordBuilder(schema)
.set("id", 3L)
.set("optional_long", null)
.set("required_long", 6000020000L)
.set("optional_string", null)
.set("required_string", "0001-01-01T00:00:00Z")
.build()));
}
