Java Code Examples for org.apache.avro.generic.GenericRecordBuilder

The following examples show how to use org.apache.avro.generic.GenericRecordBuilder. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: data-highway   Source File: AvroRecordWriterTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void typical() throws Exception {
  Schema schema = SchemaBuilder
      .builder()
      .record("record")
      .fields()
      .requiredLong("id")
      .requiredString("name")
      .endRecord();
  Record value = new GenericRecordBuilder(schema).set("id", 1L).set("name", "hello").build();
  ByteArrayOutputStream output = new ByteArrayOutputStream();

  Factory factory = new Factory(CodecFactory.nullCodec());
  RecordWriter writer = factory.create(schema, output);
  writer.write(value);
  writer.close();

  SeekableInput input = new SeekableByteArrayInput(output.toByteArray());
  DatumReader<Record> datumReader = new GenericDatumReader<>(schema);
  DataFileReader<Record> dataFileReader = new DataFileReader<>(input, datumReader);
  assertThat(dataFileReader.next(), is(value));
  assertThat(dataFileReader.hasNext(), is(false));
  dataFileReader.close();
}
 
Example 2
Source Project: components   Source File: KeyValueUtilsTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * From the input: {"a": "a", "b": "b", "c": "c"}
 * 
 * Extract elements: "a", "b" and "c"
 * 
 * The result should be:
 * 
 * key: {"a": "a", "b": "b", "c": "c"}
 * 
 * value: null
 */
@Test
public void test_EverythingIsAKey() throws Exception {
    GenericRecord inputRecord = new GenericRecordBuilder(inputSimpleSchema) //
            .set("a", "a") //
            .set("b", "b") //
            .set("c", "c") //
            .build();

    List<String> keyList = Arrays.asList("a", "b", "c");

    String transformedIndexedRecord = ("{'key': {'a': 'a', 'b': 'b', 'c': 'c'}, " + "'value': {}}").replaceAll("\\'", "\"");
    IndexedRecord outputRecord = KeyValueUtils.transformToKV(inputRecord,
            SchemaGeneratorUtils.extractKeyValues(inputRecord.getSchema(), keyList));
    assertEquals(transformedIndexedRecord, outputRecord.toString());

    Schema kvSchema = SchemaGeneratorUtils.mergeKeyValues(outputRecord.getSchema());
    String mergedRecord = ("{'a': 'a', 'b': 'b', 'c': 'c'}").replaceAll("\\'", "\"");
    assertEquals(mergedRecord, KeyValueUtils.transformFromKV(outputRecord, kvSchema).toString());
}
 
Example 3
Source Project: components   Source File: KeyValueUtilsTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * From the input: {"name": "testdata", "data": {"a": "a", "b": "b", "c": "c"}}
 * 
 * No extracted element.
 * 
 * The result should be:
 * 
 * key: empty
 * 
 * value: {"name": "testdata", "data": {"a": "a", "b": "b", "c": "c"}}
 */
@Test
public void test_Hierarchical_EverythingIsAValue() throws Exception {
    GenericRecord inputRecord = new GenericRecordBuilder(inputHierarchicalSchema) //
            .set("name", "testdata") //
            .build();
    inputRecord.put("data",
            new GenericRecordBuilder(inputSimpleSchema) //
                    .set("a", "a") //
                    .set("b", "b") //
                    .set("c", "c") //
                    .build());

    String transformedIndexedRecord = ("{'key': {}, "
            + "'value': {'name': 'testdata', 'data': {'a': 'a', 'b': 'b', 'c': 'c'}}}").replaceAll("\\'", "\"");
    IndexedRecord outputRecord = KeyValueUtils.transformToKV(inputRecord,
            SchemaGeneratorUtils.extractKeyValues(inputRecord.getSchema(), new ArrayList<String>()));
    assertEquals(transformedIndexedRecord, outputRecord.toString());

    Schema kvSchema = SchemaGeneratorUtils.mergeKeyValues(outputRecord.getSchema());
    String mergedRecord = ("{'name': 'testdata', 'data': {'a': 'a', 'b': 'b', 'c': 'c'}}").replaceAll("\\'", "\"");
    assertEquals(mergedRecord, KeyValueUtils.transformFromKV(outputRecord, kvSchema).toString());
}
 
Example 4
Source Project: components   Source File: KeyValueUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Use a Schema to generate a hierarchical GenericRecord that contains only null values.
 *
 * @param schema the parent schema of the field to set as null
 * @param fieldName the name of the field to set as null
 * @return if fieldName is a Record of the schema, the method will return a GenericRecord with any leaf set as null,
 * otherwise return null
 */
public static IndexedRecord generateEmptyRecord(Schema schema, String fieldName) {
    if (schema.getType().equals(Type.RECORD)) {
        Schema unwrappedSchema = getUnwrappedSchema(schema.getField(fieldName));
        if (unwrappedSchema.getType().equals(Type.RECORD)) {
            GenericRecordBuilder outputRecord = new GenericRecordBuilder(unwrappedSchema);
            for (Field field : unwrappedSchema.getFields()) {
                IndexedRecord value = generateEmptyRecord(unwrappedSchema, field.name());
                outputRecord.set(field.name(), value);
            }
            return outputRecord.build();
        } else {
            return null;
        }
    } else {
        return null;
    }
}
 
Example 5
Source Project: components   Source File: KeyValueUtilsTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * From the input: {"name": "testdata", "data": {"a": "a", "b": "b", "c": "c"}}
 * 
 * Extract elements: "name", "data.a", "data.b" and "data.c"
 * 
 * The result should be:
 * 
 * key: {"name": "testdata", "data": {"a": "a", "b": "b", "c": "c"}}
 * 
 * value: null
 */
@Test
public void test_Hierarchical_EverythingIsAKey() throws Exception {
    GenericRecord inputRecord = new GenericRecordBuilder(inputHierarchicalSchema) //
            .set("name", "testdata") //
            .build();
    inputRecord.put("data",
            new GenericRecordBuilder(inputSimpleSchema) //
                    .set("a", "a") //
                    .set("b", "b") //
                    .set("c", "c") //
                    .build());

    List<String> keyList = Arrays.asList("name", "data");

    String transformedIndexedRecord = ("{'key': {'name': 'testdata', 'data': {'a': 'a', 'b': 'b', 'c': 'c'}}, "
            + "'value': {}}").replaceAll("\\'", "\"");
    IndexedRecord outputRecord = KeyValueUtils.transformToKV(inputRecord,
            SchemaGeneratorUtils.extractKeyValues(inputRecord.getSchema(), keyList));
    assertEquals(transformedIndexedRecord, outputRecord.toString());

    Schema kvSchema = SchemaGeneratorUtils.mergeKeyValues(outputRecord.getSchema());
    String mergedRecord = ("{'name': 'testdata', 'data': {'a': 'a', 'b': 'b', 'c': 'c'}}").replaceAll("\\'", "\"");
    assertEquals(mergedRecord, KeyValueUtils.transformFromKV(outputRecord, kvSchema).toString());
}
 
Example 6
@Test
public void testConfluentSerDes() throws Exception {

    org.apache.avro.Schema schema = new org.apache.avro.Schema.Parser().parse(GENERIC_TEST_RECORD_SCHEMA);
    GenericRecord record = new GenericRecordBuilder(schema).set("field1", "some value").set("field2", "some other value").build();

    Map<String, Object> config = new HashMap<>();
    config.put(AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, rootTarget.getUri().toString());

    KafkaAvroSerializer kafkaAvroSerializer = new KafkaAvroSerializer();
    kafkaAvroSerializer.configure(config, false);
    byte[] bytes = kafkaAvroSerializer.serialize("topic", record);

    KafkaAvroDeserializer kafkaAvroDeserializer = new KafkaAvroDeserializer();
    kafkaAvroDeserializer.configure(config, false);

    GenericRecord result = (GenericRecord) kafkaAvroDeserializer.deserialize("topic", bytes);
    LOG.info(result.toString());
}
 
Example 7
Source Project: registry   Source File: KafkaAvroSerdesTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testGenericSerializedGenericDeserialized() {
    String topic = "topic";
    Map<String, Object> config = new HashMap<>();
    config.put(AvroSnapshotDeserializer.SPECIFIC_AVRO_READER, false);
    KafkaAvroDeserializer kafkaAvroDeserializer = new KafkaAvroDeserializer(schemaRegistryClient);
    kafkaAvroDeserializer.configure(config, false);

    KafkaAvroSerializer kafkaAvroSerializer = new KafkaAvroSerializer(schemaRegistryClient);
    kafkaAvroSerializer.configure(config, false);

    GenericRecord record = new GenericRecordBuilder(schema).set("field1", "some value").set("field2", "some other value").build();

    byte[] bytes = kafkaAvroSerializer.serialize(topic , record);
    Object o = kafkaAvroDeserializer.deserialize(topic, bytes);
    checkGenericSerializedGenericDeserializedEquals(record, o);

    Headers headers = new RecordHeaders();
    bytes = kafkaAvroSerializer.serialize(topic, headers, record);
    o = kafkaAvroDeserializer.deserialize(topic, headers, bytes);
    checkGenericSerializedGenericDeserializedEquals(record, o);
}
 
Example 8
Source Project: avro-fastserde   Source File: FastDatumWriterTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
@SuppressWarnings("unchecked")
public void shouldCreateGenericDatumReader() throws IOException {
    Schema recordSchema = createRecord("TestSchema",
            createPrimitiveUnionFieldSchema("test", Schema.Type.STRING));
    FastGenericDatumWriter<GenericRecord> fastGenericDatumReader = new FastGenericDatumWriter<>(
            recordSchema, cache);

    GenericRecordBuilder recordBuilder = new GenericRecordBuilder(recordSchema);
    recordBuilder.set("test", "test");

    // when
    fastGenericDatumReader.write(recordBuilder.build(),
            EncoderFactory.get().directBinaryEncoder(new ByteArrayOutputStream(), null));

    // then
    FastSerializer<GenericRecord> fastGenericSerializer = (FastSerializer<GenericRecord>) cache
            .getFastGenericSerializer(recordSchema);

    fastGenericSerializer = (FastSerializer<GenericRecord>) cache.getFastGenericSerializer(recordSchema);

    Assert.assertNotNull(fastGenericSerializer);
    Assert.assertNotEquals(2, fastGenericSerializer.getClass().getDeclaredMethods().length);
}
 
Example 9
@Test
@SuppressWarnings("unchecked")
public void shouldWriteEnum() {
    // given
    Schema enumSchema = createEnumSchema("testEnum", new String[]{"A", "B"});
    Schema recordSchema = createRecord("testRecord", createField("testEnum", enumSchema),
            createUnionField("testEnumUnion", enumSchema), createArrayFieldSchema("testEnumArray", enumSchema),
            createArrayFieldSchema("testEnumUnionArray", createUnionSchema(enumSchema)));

    GenericRecordBuilder builder = new GenericRecordBuilder(recordSchema);
    builder.set("testEnum", new GenericData.EnumSymbol(enumSchema, "A"));
    builder.set("testEnumUnion", new GenericData.EnumSymbol(enumSchema, "A"));
    builder.set("testEnumArray", Arrays.asList(new GenericData.EnumSymbol(enumSchema, "A")));
    builder.set("testEnumUnionArray", Arrays.asList(new GenericData.EnumSymbol(enumSchema, "A")));

    // when
    GenericRecord record = deserializeGeneric(recordSchema, serializeGenericFast(builder.build()));

    // then
    Assert.assertEquals("A", record.get("testEnum").toString());
    Assert.assertEquals("A", record.get("testEnumUnion").toString());
    Assert.assertEquals("A", ((List<GenericData.EnumSymbol>) record.get("testEnumArray")).get(0).toString());
    Assert.assertEquals("A", ((List<GenericData.EnumSymbol>) record.get("testEnumUnionArray")).get(0).toString());
}
 
Example 10
@Test
@SuppressWarnings("unchecked")
public void shouldReadEnum() {
    // given
    Schema enumSchema = createEnumSchema("testEnum", new String[]{"A", "B"});
    Schema recordSchema = createRecord("testRecord", createField("testEnum", enumSchema),
            createUnionField("testEnumUnion", enumSchema), createArrayFieldSchema("testEnumArray", enumSchema),
            createArrayFieldSchema("testEnumUnionArray", createUnionSchema(enumSchema)));

    GenericRecordBuilder builder = new GenericRecordBuilder(recordSchema);
    builder.set("testEnum", new GenericData.EnumSymbol(enumSchema, "A"));
    builder.set("testEnumUnion", new GenericData.EnumSymbol(enumSchema, "A"));
    builder.set("testEnumArray", Arrays.asList(new GenericData.EnumSymbol(enumSchema, "A")));
    builder.set("testEnumUnionArray", Arrays.asList(new GenericData.EnumSymbol(enumSchema, "A")));

    // when
    GenericRecord record = deserializeGenericFast(recordSchema, recordSchema, serializeGeneric(builder.build()));

    // then
    Assert.assertEquals("A", record.get("testEnum").toString());
    Assert.assertEquals("A", record.get("testEnumUnion").toString());
    Assert.assertEquals("A", ((List<GenericData.EnumSymbol>) record.get("testEnumArray")).get(0).toString());
    Assert.assertEquals("A", ((List<GenericData.EnumSymbol>) record.get("testEnumUnionArray")).get(0).toString());
}
 
Example 11
@Test
public void shouldReadJavaStringKeyedMapOfRecords() {
    // given
    Schema recordSchema = createRecord("record",
            createPrimitiveUnionFieldSchema("field", Schema.Type.STRING));

    Schema mapRecordSchema = Schema.createMap(recordSchema);
    GenericData.setStringType(mapRecordSchema, GenericData.StringType.String);

    GenericRecordBuilder subRecordBuilder = new GenericRecordBuilder(recordSchema);
    subRecordBuilder.set("field", "abc");

    Map<String, GenericData.Record> recordsMap = new HashMap<>();
    recordsMap.put("1", subRecordBuilder.build());
    recordsMap.put("2", subRecordBuilder.build());

    // when
    Map<String, GenericRecord> mapWithStringKeys = deserializeGenericFast(mapRecordSchema, mapRecordSchema, serializeGeneric(recordsMap, mapRecordSchema));

    // then
    Assert.assertEquals(2, mapWithStringKeys.size());
    Assert.assertEquals("abc", mapWithStringKeys.get("1").get("field").toString());
    Assert.assertEquals("abc", mapWithStringKeys.get("2").get("field").toString());
}
 
Example 12
Source Project: apicurio-registry   Source File: AvroData.java    License: Apache License 2.0 5 votes vote down vote up
private static Object maybeWrapSchemaless(Schema schema, Object value, String typeField) {
    if (schema != null) {
        return value;
    }

    GenericRecordBuilder builder = new GenericRecordBuilder(ANYTHING_SCHEMA);
    if (value != null) {
        builder.set(typeField, value);
    }
    return builder.build();
}
 
Example 13
Source Project: components   Source File: TypeConverterUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Copy fields value from inputRecord to outputRecordBuilder
 *
 * @param inputRecord
 * @param outputRecordBuilder
 */
public static void copyFieldsValues(IndexedRecord inputRecord, GenericRecordBuilder outputRecordBuilder) {
    List<Schema.Field> fields = inputRecord.getSchema().getFields();
    for (Schema.Field field : fields) {
        outputRecordBuilder.set(field.name(), inputRecord.get(field.pos()));
    }
}
 
Example 14
@Test
public void testSpecificRecordReadMoreFieldsThanWereWritten() throws IOException {
	Schema smallerUserSchema = new Schema.Parser().parse(
		"{\"namespace\": \"org.apache.flink.formats.avro.generated\",\n" +
			" \"type\": \"record\",\n" +
			" \"name\": \"SimpleRecord\",\n" +
			" \"fields\": [\n" +
			"     {\"name\": \"name\", \"type\": \"string\"}" +
			" ]\n" +
			"}]");
	RegistryAvroDeserializationSchema<SimpleRecord> deserializer = new RegistryAvroDeserializationSchema<>(
		SimpleRecord.class,
		null,
		() -> in -> smallerUserSchema
	);

	GenericData.Record smallUser = new GenericRecordBuilder(smallerUserSchema)
		.set("name", "someName")
		.build();

	SimpleRecord simpleRecord = deserializer.deserialize(writeRecord(
		smallUser,
		smallerUserSchema));

	assertEquals("someName", simpleRecord.getName().toString());
	assertNull(simpleRecord.getOptionalField());
}
 
Example 15
@Test
public void testSpecificRecordReadMoreFieldsThanWereWritten() throws IOException {
	Schema smallerUserSchema = new Schema.Parser().parse(
		"{\"namespace\": \"org.apache.flink.formats.avro.generated\",\n" +
			" \"type\": \"record\",\n" +
			" \"name\": \"SimpleRecord\",\n" +
			" \"fields\": [\n" +
			"     {\"name\": \"name\", \"type\": \"string\"}" +
			" ]\n" +
			"}]");
	RegistryAvroDeserializationSchema<SimpleRecord> deserializer = new RegistryAvroDeserializationSchema<>(
		SimpleRecord.class,
		null,
		() -> in -> smallerUserSchema
	);

	GenericData.Record smallUser = new GenericRecordBuilder(smallerUserSchema)
		.set("name", "someName")
		.build();

	SimpleRecord simpleRecord = deserializer.deserialize(writeRecord(
		smallUser,
		smallerUserSchema));

	assertEquals("someName", simpleRecord.getName().toString());
	assertNull(simpleRecord.getOptionalField());
}
 
Example 16
Source Project: components   Source File: FilterRowPredicateTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void test_FilterSimple_utf8() throws Exception {
    FilterRowProperties properties = new FilterRowProperties("test");
    properties.init();
    FilterRowCriteriaProperties filterProp = new FilterRowCriteriaProperties("filter");
    filterProp.init();
    properties.schemaListener.afterSchema();
    filterProp.columnName.setValue("a");
    filterProp.value.setValue("aaa");
    properties.filters.addRow(filterProp);

    Schema inputSimpleSchema = SchemaBuilder.record("inputRow") //
            .fields() //
            .name("a").type().optional().stringType() //
            .name("b").type().optional().stringType() //
            .name("c").type().optional().stringType() //
            .endRecord();

    GenericRecord inputSimpleRecord = new GenericRecordBuilder(inputSimpleSchema) //
            .set("a", new Utf8("aaa")) //
            .set("b", new Utf8("BBB")) //
            .set("c", new Utf8("Ccc")) //
            .build();

    FilterRowPredicate predicate = new FilterRowPredicate(properties);
    assertThat(predicate.apply(inputSimpleRecord), is(Boolean.TRUE));

    predicate = new FilterRowPredicate.Negate(properties);
    assertThat(predicate.apply(inputSimpleRecord), is(Boolean.FALSE));
}
 
Example 17
Source Project: flink   Source File: ParquetRecordReaderTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testMapGroup() throws IOException {
	Preconditions.checkState(unWrapSchema(NESTED_SCHEMA.getField("spamMap").schema())
		.getType().equals(Schema.Type.MAP));
	ImmutableMap.Builder<String, String> map = ImmutableMap.builder();
	map.put("testKey", "testValue");

	GenericRecord record = new GenericRecordBuilder(NESTED_SCHEMA)
		.set("foo", 32L)
		.set("spamMap", map.build())
		.build();

	Path path = createTempParquetFile(tempRoot.getRoot(), NESTED_SCHEMA, Collections.singletonList(record));
	MessageType readSchema = (new AvroSchemaConverter()).convert(NESTED_SCHEMA);
	ParquetRecordReader<Row> rowReader = new ParquetRecordReader<>(new RowReadSupport(), readSchema);

	InputFile inputFile =
		HadoopInputFile.fromPath(new org.apache.hadoop.fs.Path(path.toUri()), testConfig);
	ParquetReadOptions options = ParquetReadOptions.builder().build();
	ParquetFileReader fileReader = new ParquetFileReader(inputFile, options);

	rowReader.initialize(fileReader, testConfig);
	assertFalse(rowReader.reachEnd());

	Row row = rowReader.nextRecord();
	assertEquals(7, row.getArity());

	assertEquals(32L, row.getField(0));
	Map<?, ?> result = (Map<?, ?>) row.getField(1);
	assertEquals(result.get("testKey").toString(), "testValue");
	assertTrue(rowReader.reachEnd());
}
 
Example 18
Source Project: flink   Source File: ParquetRecordReaderTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testNestedMapGroup() throws IOException {
	Schema nestedMapSchema = unWrapSchema(NESTED_SCHEMA.getField("nestedMap").schema());
	Preconditions.checkState(nestedMapSchema.getType().equals(Schema.Type.MAP));

	Schema mapValueSchema = nestedMapSchema.getValueType();
	GenericRecord mapValue = new GenericRecordBuilder(mapValueSchema)
		.set("type", "nested")
		.set("value", "nested_value").build();

	ImmutableMap.Builder<String, GenericRecord> map = ImmutableMap.builder();
	map.put("testKey", mapValue);

	GenericRecord record = new GenericRecordBuilder(NESTED_SCHEMA)
		.set("nestedMap", map.build())
		.set("foo", 34L).build();

	Path path = createTempParquetFile(tempRoot.getRoot(), NESTED_SCHEMA, Collections.singletonList(record));
	MessageType readSchema = (new AvroSchemaConverter()).convert(NESTED_SCHEMA);
	ParquetRecordReader<Row> rowReader = new ParquetRecordReader<>(new RowReadSupport(), readSchema);

	InputFile inputFile =
		HadoopInputFile.fromPath(new org.apache.hadoop.fs.Path(path.toUri()), testConfig);
	ParquetReadOptions options = ParquetReadOptions.builder().build();
	ParquetFileReader fileReader = new ParquetFileReader(inputFile, options);

	rowReader.initialize(fileReader, testConfig);
	assertFalse(rowReader.reachEnd());

	Row row = rowReader.nextRecord();
	assertEquals(7, row.getArity());

	assertEquals(34L, row.getField(0));
	Map result = (Map) row.getField(5);

	Row nestedRow = (Row) result.get("testKey");
	assertEquals("nested", nestedRow.getField(0));
	assertEquals("nested_value", nestedRow.getField(1));
}
 
Example 19
@SuppressWarnings("unchecked")
private Object wrapOption(Schema schema, Object option) {
  if (schema.getType() == Schema.Type.BYTES && option instanceof String) {
    option = ByteBuffer.wrap(((String) option).getBytes(Charset.defaultCharset()));
  } else if (schema.getType() == Schema.Type.FLOAT && option instanceof Double) {
    option = ((Double) option).floatValue();
  } else if (schema.getType() == Schema.Type.LONG && option instanceof Integer) {
    option = ((Integer) option).longValue();
  } else if (schema.getType() == Schema.Type.ARRAY && option instanceof Collection) {
    option = new GenericData.Array(schema, (Collection) option);
  } else if (schema.getType() == Schema.Type.ENUM && option instanceof String) {
    option = new GenericData.EnumSymbol(schema, (String) option);
  } else if (schema.getType() == Schema.Type.FIXED && option instanceof String) {
    option =
        new GenericData.Fixed(schema, ((String) option).getBytes(Charset.defaultCharset()));
  } else if (schema.getType() == Schema.Type.RECORD && option instanceof Map) {
    Map optionMap = (Map) option;
    GenericRecordBuilder optionBuilder = new GenericRecordBuilder(schema);
    for (Schema.Field field : schema.getFields()) {
      if (optionMap.containsKey(field.name())) {
        optionBuilder.set(field, optionMap.get(field.name()));
      }
    }
    option = optionBuilder.build();
  }
  return option;
}
 
Example 20
private GenericRecord generateRecord(Schema schema) {
  GenericRecordBuilder builder = new GenericRecordBuilder(schema);
  for (Schema.Field field : schema.getFields()) {
    builder.set(field, generateObject(field.schema(), field.name()));
  }
  return builder.build();
}
 
Example 21
Source Project: components   Source File: SalesforceBulkFileWriterTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testGetValuesForDeleteAction() {
    IndexedRecord record = new GenericRecordBuilder(schema).set("Id", "123456").set("Name", "Talend").build();
    List<String> expected = new ArrayList<>();
    expected.add("123456");
    writer.getBulkProperties().outputAction.setValue(OutputAction.DELETE);
    List<String> actual = writer.getValues(record);
    Assert.assertThat(actual, CoreMatchers.is(expected));
}
 
Example 22
/** Generates the records using {@link GenericRecordBuilder}. */
@ProcessElement
public void processElement(ProcessContext c) {

  KV<String, String> message = c.element();
  String attributeKey = message.getKey();
  String attributeValue = message.getValue();

  Map<String, String> attributeMap;

  if (attributeValue != null) {
    if (attributeKey != null) {
      attributeMap = Collections.singletonMap(attributeKey, attributeValue);
    } else {
      attributeMap = Collections.singletonMap("", attributeValue);
    }
  } else {
    attributeMap = Collections.EMPTY_MAP;
  }

  c.output(
      new GenericRecordBuilder(SCHEMA)
          .set("message", attributeValue)
          .set("attributes", attributeMap)
          .set("timestamp", c.timestamp().getMillis())
          .build());
}
 
Example 23
Source Project: DataflowTemplates   Source File: BigQueryConvertersTest.java    License: Apache License 2.0 5 votes vote down vote up
/** Generates an Avro record with a single field. */
private Record generateSingleFieldAvroRecord(
    String name, String type, String description, Object value) {
  Schema avroSchema =
      new Schema.Parser()
          .parse(
              String.format(
                  AVRO_SCHEMA_TEMPLATE,
                  new StringBuilder()
                      .append(String.format(avroFieldTemplate, name, type, description))
                      .toString()));
  GenericRecordBuilder builder = new GenericRecordBuilder(avroSchema);
  builder.set(name, value);
  return builder.build();
}
 
Example 24
Source Project: DataflowTemplates   Source File: BigQueryConvertersTest.java    License: Apache License 2.0 5 votes vote down vote up
/** Generates an Avro record with a record field type. */
static Record generateNestedAvroRecord() {
  String avroRecordFieldSchema =
      new StringBuilder()
          .append("{")
          .append("  \"name\" : \"address\",")
          .append("  \"type\" :")
          .append("  {")
          .append("    \"type\" : \"record\",")
          .append("    \"name\" : \"address\",")
          .append("    \"namespace\"  : \"nothing\",")
          .append("    \"fields\" : ")
          .append("    [")
          .append("      {\"name\" : \"street_number\", \"type\" : \"int\"},")
          .append("      {\"name\" : \"street_name\", \"type\" : \"string\"}")
          .append("    ]")
          .append("  }")
          .append("}")
          .toString();
  Schema avroSchema =
      new Schema.Parser().parse(String.format(AVRO_SCHEMA_TEMPLATE, avroRecordFieldSchema));
  GenericRecordBuilder addressBuilder =
      new GenericRecordBuilder(avroSchema.getField("address").schema());
  addressBuilder.set("street_number", 12);
  addressBuilder.set("street_name", "Magnolia street");
  GenericRecordBuilder builder = new GenericRecordBuilder(avroSchema);
  builder.set("address", addressBuilder.build());
  return builder.build();
}
 
Example 25
@SuppressWarnings("unchecked")
private Object wrapOption(Schema schema, Object option) {
  if (schema.getType() == Schema.Type.BYTES && option instanceof String) {
    option = ByteBuffer.wrap(((String) option).getBytes(Charset.defaultCharset()));
  } else if (schema.getType() == Schema.Type.FLOAT && option instanceof Double) {
    option = ((Double) option).floatValue();
  } else if (schema.getType() == Schema.Type.LONG && option instanceof Integer) {
    option = ((Integer) option).longValue();
  } else if (schema.getType() == Schema.Type.ARRAY && option instanceof Collection) {
    option = new GenericData.Array(schema, (Collection) option);
  } else if (schema.getType() == Schema.Type.ENUM && option instanceof String) {
    option = new GenericData.EnumSymbol(schema, (String) option);
  } else if (schema.getType() == Schema.Type.FIXED && option instanceof String) {
    option =
        new GenericData.Fixed(schema, ((String) option).getBytes(Charset.defaultCharset()));
  } else if (schema.getType() == Schema.Type.RECORD && option instanceof Map) {
    Map optionMap = (Map) option;
    GenericRecordBuilder optionBuilder = new GenericRecordBuilder(schema);
    for (Schema.Field field : schema.getFields()) {
      if (optionMap.containsKey(field.name())) {
        optionBuilder.set(field, optionMap.get(field.name()));
      }
    }
    option = optionBuilder.build();
  }
  return option;
}
 
Example 26
Source Project: DataflowTemplates   Source File: BigQueryConvertersTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests that {@link BigQueryConverters.AvroToEntity} creates an Entity without a key when the
 * unique name column exceeds the maximum size allowed of 1500 bytes.
 */
@Test
public void testAvroToEntityTooLongIdColumn() throws Exception {
  // Create test data
  List<TableFieldSchema> fields = new ArrayList<>();
  fields.add(new TableFieldSchema().setName(idField).setType("STRING"));
  fields.add(new TableFieldSchema().setName(shortStringField).setType("STRING"));
  TableSchema bqSchema = new TableSchema().setFields(fields);
  Schema avroSchema =
      new Schema.Parser()
          .parse(
              String.format(
                  avroSchemaTemplate,
                  new StringBuilder()
                      .append(String.format(avroFieldTemplate, idField, "string", idFieldDesc))
                      .append(",")
                      .append(generateShortStringField())
                      .toString()));
  GenericRecordBuilder builder = new GenericRecordBuilder(avroSchema);
  builder.set(idField, longStringFieldValue);
  builder.set(shortStringField, shortStringFieldValue);
  Record record = builder.build();
  SchemaAndRecord inputBqData = new SchemaAndRecord(record, bqSchema);
  // Run the test
  Entity outputEntity = converter.apply(inputBqData);
  assertTrue(!outputEntity.hasKey());
}
 
Example 27
Source Project: DataflowTemplates   Source File: BigQueryConvertersTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests that {@link BigQueryConverters.AvroToEntity} creates an Entity with a valid key when the
 * unique name column is integer.
 */
@Test
public void testAvroToEntityIntegerIdColumn() throws Exception {
  // Create test data
  List<TableFieldSchema> fields = new ArrayList<>();
  fields.add(new TableFieldSchema().setName(idField).setType("INTEGER"));
  fields.add(new TableFieldSchema().setName(shortStringField).setType("STRING"));
  TableSchema bqSchema = new TableSchema().setFields(fields);
  Schema avroSchema =
      new Schema.Parser()
          .parse(
              String.format(
                  avroSchemaTemplate,
                  new StringBuilder()
                      .append(String.format(avroFieldTemplate, idField, "int", idFieldDesc))
                      .append(",")
                      .append(generateShortStringField())
                      .toString()));
  GenericRecordBuilder builder = new GenericRecordBuilder(avroSchema);
  builder.set(idField, idFieldValueInt);
  builder.set(shortStringField, shortStringFieldValue);
  Record record = builder.build();
  SchemaAndRecord inputBqData = new SchemaAndRecord(record, bqSchema);
  // Run the test
  Entity outputEntity = converter.apply(inputBqData);
  assertTrue(outputEntity.hasKey());
  assertEquals(idFieldValueStr, outputEntity.getKey().getPath(0).getName());
  validateMetadata(outputEntity);
}
 
Example 28
Source Project: DataflowTemplates   Source File: BigQueryConvertersTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests that {@link BigQueryConverters.AvroToEntity} creates an Entity with a default namespace
 * when the namespace is not specified.
 */
@Test
public void testAvroToEntityDefaultNamespace() throws Exception {
  // Create test data
  List<TableFieldSchema> fields = new ArrayList<>();
  fields.add(new TableFieldSchema().setName(idField).setType("STRING"));
  fields.add(new TableFieldSchema().setName(shortStringField).setType("STRING"));
  TableSchema bqSchema = new TableSchema().setFields(fields);
  Schema avroSchema =
      new Schema.Parser()
          .parse(
              String.format(
                  avroSchemaTemplate,
                  new StringBuilder()
                      .append(String.format(avroFieldTemplate, idField, "int", idFieldDesc))
                      .append(",")
                      .append(generateShortStringField())
                      .toString()));
  GenericRecordBuilder builder = new GenericRecordBuilder(avroSchema);
  builder.set(idField, 1);
  builder.set(shortStringField, shortStringFieldValue);
  Record record = builder.build();
  SchemaAndRecord inputBqData = new SchemaAndRecord(record, bqSchema);
  // Run the test
  AvroToEntity noNamespaceConverter =
      AvroToEntity.newBuilder()
          .setEntityKind(entityKind)
          .setUniqueNameColumn(uniqueNameColumn)
          .build();
  Entity outputEntity = noNamespaceConverter.apply(inputBqData);
  // Assess results
  assertTrue(outputEntity.hasKey());
  assertEquals("", outputEntity.getKey().getPartitionId().getNamespaceId());
}
 
Example 29
Source Project: DataflowTemplates   Source File: BigQueryConvertersTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests that {@link BigQueryConverters.AvroToEntity} creates an Entity without a valid key when a
 * Timestamp field is invalid.
 */
@Test
public void testAvroToEntityInvalidTimestampField() throws Exception {
  // Create test data
  List<TableFieldSchema> fields = new ArrayList<>();
  fields.add(new TableFieldSchema().setName(idField).setType("STRING"));
  fields.add(new TableFieldSchema().setName(invalidTimestampField).setType("TIMESTAMP"));
  TableSchema bqSchema = new TableSchema().setFields(fields);
  Schema avroSchema =
      new Schema.Parser()
          .parse(
              String.format(
                  avroSchemaTemplate,
                  new StringBuilder()
                      .append(String.format(avroFieldTemplate, idField, "string", idFieldDesc))
                      .append(",")
                      .append(
                          String.format(
                              avroFieldTemplate,
                              invalidTimestampField,
                              "long",
                              invalidTimestampFieldDesc))
                      .toString()));
  GenericRecordBuilder builder = new GenericRecordBuilder(avroSchema);
  builder.set(idField, idFieldValueStr);
  builder.set(invalidTimestampField, invalidTimestampFieldValueNanos);
  Record record = builder.build();
  SchemaAndRecord inputBqData = new SchemaAndRecord(record, bqSchema);
  // Run the test
  Entity outputEntity = converter.apply(inputBqData);
  // Assess results
  assertTrue(!outputEntity.hasKey());
  assertTrue(
      outputEntity
          .getPropertiesMap()
          .get("cause")
          .getStringValue()
          .startsWith("Timestamp is not valid"));
  assertEquals(record.toString(), outputEntity.getPropertiesMap().get("row").getStringValue());
}
 
Example 30
Source Project: DataflowTemplates   Source File: BigQueryConvertersTest.java    License: Apache License 2.0 5 votes vote down vote up
/** Generates an Avro record with a single field. */
private Record generateSingleFieldAvroRecord(
    String name, String type, String description, Object value) {
  Schema avroSchema =
      new Schema.Parser()
          .parse(
              String.format(
                  avroSchemaTemplate,
                  new StringBuilder()
                      .append(String.format(avroFieldTemplate, name, type, description))
                      .toString()));
  GenericRecordBuilder builder = new GenericRecordBuilder(avroSchema);
  builder.set(name, value);
  return builder.build();
}