Java Code Examples for org.apache.avro.generic.GenericData

The following examples show how to use org.apache.avro.generic.GenericData. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: components   Source File: JiraUpdateWriterTestIT.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Checks {@link JiraUpdateWriter#write()} throws {@link IOException} which message contains
 * "Reason: user is not authenticated. Record wasn't updated"
 * in case server responses with 401 Unauthorized status code
 * 
 * @throws IOException
 */
@Test
public void testWriteUnauthorized() throws IOException {
    IndexedRecord badJsonRecord = new GenericData.Record(UPDATE_SCHEMA);
    String badProject = "{\"name\":\"Updated Integration Test Project\"\"assigneeType\":\"PROJECT_LEAD\"}";
    badJsonRecord.put(0, "TP");
    badJsonRecord.put(1, badProject);

    JiraWriter updateProjectWriter = JiraTestsHelper.createWriter(HOST_PORT, WRONG_USER, PASS, Resource.PROJECT,
            Action.UPDATE);

    thrown.expect(IOException.class);
    thrown.expectMessage("Reason: user is not authenticated. Record wasn't updated");
    thrown.expectMessage("Record: " + badProject);
    thrown.expectMessage("Error: ");

    updateProjectWriter.open("upd");
    try {
        updateProjectWriter.write(badJsonRecord);
    } finally {
        updateProjectWriter.close();
    }
}
 
Example 2
public void validateAvroFile(File file) throws IOException {
  // read the events back using GenericRecord
  DatumReader<GenericRecord> reader = new GenericDatumReader<GenericRecord>();
  DataFileReader<GenericRecord> fileReader =
      new DataFileReader<GenericRecord>(file, reader);
  GenericRecord record = new GenericData.Record(fileReader.getSchema());
  int numEvents = 0;
  while (fileReader.hasNext()) {
    fileReader.next(record);
    ByteBuffer body = (ByteBuffer) record.get("body");
    CharsetDecoder decoder = Charsets.UTF_8.newDecoder();
    String bodyStr = decoder.decode(body).toString();
    System.out.println(bodyStr);
    numEvents++;
  }
  fileReader.close();
  Assert.assertEquals("Should have found a total of 3 events", 3, numEvents);
}
 
Example 3
Source Project: presto   Source File: TestAvroDecoder.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testNestedLongArrayWithNulls()
{
    DecoderTestColumnHandle row = new DecoderTestColumnHandle(0, "row", new ArrayType(new ArrayType(BIGINT)), "array_field", null, null, false, false, false);
    Schema schema = SchemaBuilder.array().items().nullable().array().items().nullable().longType();
    List<List<Long>> data = Arrays.asList(
            ImmutableList.of(12L, 15L, 17L),
            ImmutableList.of(22L, 25L, 27L, 29L),
            null,
            Arrays.asList(3L, 5L, null, 6L));

    GenericArray<List<Long>> list = new GenericData.Array<>(schema, data);
    Map<DecoderColumnHandle, FieldValueProvider> decodedRow = buildAndDecodeColumn(row, "array_field", schema.toString(), list);

    checkArrayValue(decodedRow, row, list);
}
 
Example 4
Source Project: components   Source File: JDBCRowWriter.java    License: Apache License 2.0 6 votes vote down vote up
private void handleSuccess(IndexedRecord input) {
    successCount++;

    if (outSchema == null || outSchema.getFields().size() == 0) {
        return;
    }

    IndexedRecord output = new GenericData.Record(outSchema);
    for (Schema.Field outField : output.getSchema().getFields()) {
        Object outValue = null;

        if (propagateQueryResultSet && outField.name().equals(setting.getUseColumn())) {
            output.put(outField.pos(), resultSet);
        } else {
            Schema.Field inField = input.getSchema().getField(outField.name());
            if (inField != null) {
                outValue = input.get(inField.pos());
            }
            output.put(outField.pos(), outValue);
        }
    }

    successfulWrites.add(output);
}
 
Example 5
Source Project: flink   Source File: AvroSerializationSchema.java    License: Apache License 2.0 6 votes vote down vote up
protected void checkAvroInitialized() {
	if (datumWriter != null) {
		return;
	}
	ClassLoader cl = Thread.currentThread().getContextClassLoader();
		if (SpecificRecord.class.isAssignableFrom(recordClazz)) {
		Schema schema = SpecificData.get().getSchema(recordClazz);
		this.datumWriter = new SpecificDatumWriter<>(schema);
		this.schema = schema;
	} else {
		this.schema = new Schema.Parser().parse(this.schemaString);
		GenericData genericData = new GenericData(cl);

		this.datumWriter = new GenericDatumWriter<>(schema, genericData);
	}
	this.arrayOutputStream = new ByteArrayOutputStream();
	this.encoder = EncoderFactory.get().directBinaryEncoder(arrayOutputStream, null);
}
 
Example 6
Source Project: presto   Source File: TestAvroDecoder.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testArrayOfMapsWithNulls()
{
    Schema schema = SchemaBuilder.array()
            .items()
            .nullable().map()
            .values()
            .nullable().floatType();
    List<Map<String, Float>> data = Arrays.asList(
            buildMapFromKeysAndValues(ImmutableList.of("key1", "key2", "key3"), ImmutableList.of(1.3F, 2.3F, -.5F)),
            null,
            buildMapFromKeysAndValues(ImmutableList.of("key10", "key20", "key30"), ImmutableList.of(11.3F, 12.3F, -1.5F)),
            buildMapFromKeysAndValues(ImmutableList.of("key100", "key200", "key300"), Arrays.asList(111.3F, null, -11.5F)));

    DecoderTestColumnHandle row = new DecoderTestColumnHandle(0, "row", new ArrayType(REAL_MAP_TYPE), "array_field", null, null, false, false, false);
    GenericArray<Map<String, Float>> list = new GenericData.Array<>(schema, data);
    Map<DecoderColumnHandle, FieldValueProvider> decodedRow = buildAndDecodeColumn(row, "array_field", schema.toString(), list);
    checkArrayValues(getBlock(decodedRow, row), row.getType(), data);
}
 
Example 7
Source Project: iow-hadoop-streaming   Source File: GenericDataTSVTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testToString2() throws Exception {

    Schema s2 = p.parse(sc2);
    GenericData.Record r2 = new GenericData.Record(s2);
    r2.put("x",0.345621f);
    r2.put("y",Double.NaN);
    r2.put("x2", Float.POSITIVE_INFINITY);
    r2.put("y2", 1.0);
    r2.put("x3", Float.NaN);
    r2.put("y3", Double.POSITIVE_INFINITY);
    r2.put("x4", Float.NaN);
    r2.put("y4", Double.NEGATIVE_INFINITY);

    String tsv = gd.toString(r2);
    Assert.assertNotNull(tsv);
    Assert.assertEquals("0.345621\tNaN\tInfinity\t1.0\tNaN\tInfinity\tNaN\t-Infinity",tsv);
}
 
Example 8
@Test
public void testWriteSimpleMessage() throws Throwable {
    queue.clear();
    //
    TAzureStorageQueueOutputProperties properties = new TAzureStorageQueueOutputProperties("tests");
    properties = (TAzureStorageQueueOutputProperties) setupConnectionProperties(
            (AzureStorageProvideConnectionProperties) properties);
    properties.setupProperties();
    properties.queueName.setValue(TEST_QUEUE_NAME);
    Writer<?> writer = createWriter(properties);
    writer.open("test-uid");
    for (String m : messages) {
        IndexedRecord entity = new GenericData.Record(properties.schema.schema.getValue());
        entity.put(0, m + "SIMPLE");
        writer.write(entity);
    }
    writer.close();
    queue.downloadAttributes();
    assertEquals(3, queue.getApproximateMessageCount());
    for (CloudQueueMessage msg : queue.retrieveMessages(3)) {
        assertNotNull(msg.getMessageContentAsString());
        assertTrue(msg.getMessageContentAsString().indexOf("SIMPLE") > 0);
    }
}
 
Example 9
Source Project: components   Source File: AggregateCombineFnTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void MaxDoubleAccumulatorFnTest() {
    List<GenericData.Record> testData = genRecords(Arrays.asList(1.1, 2.2, 3.3, 10.10, 5.5), AvroUtils._double());

    AggregateCombineFn.MaxDoubleAccumulatorFn fn1 = new AggregateCombineFn.MaxDoubleAccumulatorFn();
    fn1.createAccumulator();
    AggregateCombineFn.MaxDoubleAccumulatorFn fn2 = new AggregateCombineFn.MaxDoubleAccumulatorFn();
    fn2.createAccumulator();
    AggregateCombineFn.MaxDoubleAccumulatorFn fn3 = new AggregateCombineFn.MaxDoubleAccumulatorFn();
    fn3.createAccumulator();
    double delta = 0.0;
    fn1.addInput(testData.get(0));
    Assert.assertEquals(1.1, fn1.extractOutput(), delta);
    fn1.addInput(testData.get(1));
    Assert.assertEquals(2.2, fn1.getAccumulators(), delta);

    fn2.addInput(testData.get(2));
    fn2.addInput(testData.get(3));
    Assert.assertEquals(10.10, fn2.extractOutput(), delta);

    fn3.addInput(testData.get(4));
    Assert.assertEquals(5.5, fn3.extractOutput(), delta);

    fn1.mergeAccumulators(Arrays.asList(fn2.getAccumulators(), fn3.getAccumulators()));
    Assert.assertEquals(10.10, fn1.extractOutput(), delta);
}
 
Example 10
Source Project: Flink-CEPplus   Source File: AvroKryoSerializerUtils.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void addAvroSerializersIfRequired(ExecutionConfig reg, Class<?> type) {
	if (org.apache.avro.specific.SpecificRecordBase.class.isAssignableFrom(type) ||
		org.apache.avro.generic.GenericData.Record.class.isAssignableFrom(type)) {

		// Avro POJOs contain java.util.List which have GenericData.Array as their runtime type
		// because Kryo is not able to serialize them properly, we use this serializer for them
		reg.registerTypeWithKryoSerializer(GenericData.Array.class, Serializers.SpecificInstanceCollectionSerializerForArrayList.class);

		// We register this serializer for users who want to use untyped Avro records (GenericData.Record).
		// Kryo is able to serialize everything in there, except for the Schema.
		// This serializer is very slow, but using the GenericData.Records of Kryo is in general a bad idea.
		// we add the serializer as a default serializer because Avro is using a private sub-type at runtime.
		reg.addDefaultKryoSerializer(Schema.class, AvroSchemaSerializer.class);
	}
}
 
Example 11
Source Project: attic-apex-malhar   Source File: AvroToPojoTest.java    License: Apache License 2.0 6 votes vote down vote up
private void createReaderInput()
{
  int cnt = 3;

  recordList = Lists.newArrayList();

  while (cnt > 0) {
    GenericRecord rec = new GenericData.Record(new Schema.Parser().parse(AVRO_SCHEMA));
    rec.put("orderId", cnt * 1L);
    rec.put("customerId", cnt * 2);
    rec.put("total", cnt * 1.5);
    rec.put("customerName", "*" + cnt + "*");
    cnt--;
    recordList.add(rec);
  }
}
 
Example 12
Source Project: components   Source File: JiraDeleteWriterTestIT.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Checks {@link JiraDeleteWriter#write()} throws {@link IOException} which message contains
 * "User is not authenticated. Record wasn't deleted"
 * in case server responses with 401 Unauthorized status code
 * 
 * @throws IOException
 */
@Test
public void testWriteUnauthorized() throws IOException {
    String expectedError = "User is not authenticated. Record wasn't deleted";

    IndexedRecord badJsonRecord = new GenericData.Record(DELETE_SCHEMA);
    badJsonRecord.put(0, "TP");

    thrown.expect(IOException.class);
    thrown.expectMessage("Reason: user is not authenticated. Record wasn't deleted");
    thrown.expectMessage("Record: TP");
    thrown.expectMessage("Error: ");

    JiraWriter deleteProjectWriter = JiraTestsHelper.createWriter(HOST_PORT, WRONG_USER, PASS, Resource.PROJECT,
            Action.DELETE);
    deleteProjectWriter.open("del");
    try {
        deleteProjectWriter.write(badJsonRecord);
        fail();
    } catch (DataRejectException e) {
        String rejectError = e.getRejectInfo().get("error").toString();
        assertEquals(expectedError, rejectError);
    } finally {
        deleteProjectWriter.close();
    }
}
 
Example 13
Source Project: incubator-pinot   Source File: TopkPhaseTest.java    License: Apache License 2.0 6 votes vote down vote up
private List<GenericRecord> generateTestMapperData() throws Exception {
  List<GenericRecord> inputRecords = new ArrayList<GenericRecord>();

  GenericRecord input = new GenericData.Record(inputSchema);
  input.put("d1", "abc1");
  input.put("d2", 501L);
  input.put("d3", "xyz1");
  input.put("hoursSinceEpoch", generateRandomHoursSinceEpoch());
  input.put("m1", 100);
  input.put("m2", 20);
  inputRecords.add(input);

  input = new GenericData.Record(inputSchema);
  input.put("d1", "abc2");
  input.put("d2", 502L);
  input.put("d3", "xyz2");
  input.put("hoursSinceEpoch", generateRandomHoursSinceEpoch());
  input.put("m1", 10);
  input.put("m2", 20);
  inputRecords.add(input);

  return inputRecords;
}
 
Example 14
Source Project: components   Source File: JiraInsertWriterTestIT.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Checks {@link JiraInsertWriter#write()} throws {@link IOException} which error message contains:
 * "Reason: record is invalid"
 * 
 * @throws IOException
 */
@Test
public void testWriteErrorMessage() throws IOException {
    IndexedRecord badIssueTypeRecord = new GenericData.Record(INSERT_SCHEMA);
    String insertIssue1 = "{\"fields\":{\"project\":{\"key\":\"TP\"},\"summary\":\"Integration test issue 1\",\"issuetype\":{\"id\":\"12345\"}}}";
    badIssueTypeRecord.put(0, insertIssue1);
    
    thrown.expect(IOException.class);
    thrown.expectMessage("Reason: record is invalid");
    thrown.expectMessage("Record: " + insertIssue1);
    thrown.expectMessage("Error: ");
    thrown.expectMessage("{\"errorMessages\":[],\"errors\":{\"issuetype\":\"valid issue type is required\"}}");

    JiraWriter insertIssueWriter = JiraTestsHelper.createWriter(HOST_PORT, USER, PASS, Resource.ISSUE, Action.INSERT);

    insertIssueWriter.open("ins");
    try {
        insertIssueWriter.write(badIssueTypeRecord);
    } finally {
        insertIssueWriter.close();
    }
}
 
Example 15
Source Project: flink   Source File: AvroKeyValueSinkWriter.java    License: Apache License 2.0 6 votes vote down vote up
AvroKeyValueWriter(Schema keySchema, Schema valueSchema,
		CodecFactory compressionCodec, OutputStream outputStream,
		int syncInterval) throws IOException {
	// Create the generic record schema for the key/value pair.
	mKeyValuePairSchema = AvroKeyValue
			.getSchema(keySchema, valueSchema);

	// Create an Avro container file and a writer to it.
	DatumWriter<GenericRecord> genericDatumWriter = new GenericDatumWriter<GenericRecord>(
			mKeyValuePairSchema);
	mAvroFileWriter = new DataFileWriter<GenericRecord>(
			genericDatumWriter);
	mAvroFileWriter.setCodec(compressionCodec);
	mAvroFileWriter.setSyncInterval(syncInterval);
	mAvroFileWriter.create(mKeyValuePairSchema, outputStream);

	// Create a reusable output record.
	mOutputRecord = new AvroKeyValue<Object, Object>(
			new GenericData.Record(mKeyValuePairSchema));
}
 
Example 16
Source Project: datacollector   Source File: AvroDataStreamParser.java    License: Apache License 2.0 6 votes vote down vote up
public AvroDataStreamParser(
    ProtoConfigurableEntity.Context context,
    Schema schema,
    String streamName,
    InputStream inputStream,
    long recordCount,
    int maxObjectLength,
    boolean skipAvroUnionIndexes
) throws IOException {
  this.context = context;
  avroSchema = schema;
  this.streamName = streamName;
  this.recordCount = recordCount;
  datumReader = new GenericDatumReader<>(avroSchema, avroSchema, GenericData.get()); //Reader schema argument is optional
  overrunInputStream = new OverrunInputStream(inputStream, maxObjectLength, true);
  dataFileStream = new DataFileStream<>(overrunInputStream, datumReader);
  seekToOffset();
  this.skipAvroUnionIndexes = skipAvroUnionIndexes;
}
 
Example 17
@Test(groups = {"serializationTest"})
@SuppressWarnings("unchecked")
public void shouldCreateGenericDatumReader() throws IOException, InterruptedException {
  Schema recordSchema = createRecord("TestSchema", createPrimitiveUnionFieldSchema("test", Schema.Type.STRING));
  FastGenericDatumWriter<GenericRecord> fastGenericDatumReader = new FastGenericDatumWriter<>(recordSchema, cache);

  GenericRecord record = new GenericData.Record(recordSchema);
  record.put("test", "test");

  // when
  fastGenericDatumReader.write(record, AvroCompatibilityHelper.newBinaryEncoder(new ByteArrayOutputStream(), true, null));

  // then
  FastSerializer<GenericRecord> fastGenericSerializer =
      (FastSerializer<GenericRecord>) cache.getFastGenericSerializer(recordSchema);

  fastGenericSerializer = (FastSerializer<GenericRecord>) cache.getFastGenericSerializer(recordSchema);

  Assert.assertNotNull(fastGenericSerializer);
  Assert.assertNotEquals(2, fastGenericSerializer.getClass().getDeclaredMethods().length);
}
 
Example 18
Source Project: schema-evolution-samples   Source File: AvroCodecTests.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void genericEncoderV1GenericDecoderV1() throws Exception{
	Schema schema = load("users_v1.schema");
	SchemaRegistryClient client = mock(SchemaRegistryClient.class);
	AvroCodec codec = new AvroCodec();
	codec.setSchemaRegistryClient(client);
	when(client.register(any())).thenReturn(1);
	when(client.fetch(eq(1))).thenReturn(schema);
	GenericRecord record = new GenericData.Record(schema);
	record.put("name","joe");
	record.put("favoriteNumber",42);
	record.put("favoriteColor","blue");
	byte[] results = codec.encode(record);
	GenericRecord decoded = codec.decode(results,GenericRecord.class);
	Assert.assertEquals(record.get("name").toString(),decoded.get("name").toString());
}
 
Example 19
Source Project: nifi   Source File: TestConvertAvroToJSON.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSingleAvroMessage_noContainer() throws IOException {
    final TestRunner runner = TestRunners.newTestRunner(new ConvertAvroToJSON());
    runner.setProperty(ConvertAvroToJSON.CONTAINER_OPTIONS, ConvertAvroToJSON.CONTAINER_NONE);
    final Schema schema = new Schema.Parser().parse(new File("src/test/resources/user.avsc"));

    final GenericRecord user1 = new GenericData.Record(schema);
    user1.put("name", "Alyssa");
    user1.put("favorite_number", 256);

    final DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<>(schema);
    final ByteArrayOutputStream out1 = AvroTestUtil.serializeAvroRecord(schema, datumWriter, user1);
    runner.enqueue(out1.toByteArray());

    runner.run();

    runner.assertAllFlowFilesTransferred(ConvertAvroToJSON.REL_SUCCESS, 1);
    final MockFlowFile out = runner.getFlowFilesForRelationship(ConvertAvroToJSON.REL_SUCCESS).get(0);
    out.assertContentEquals("{\"name\": \"Alyssa\", \"favorite_number\": 256, \"favorite_color\": null}");
}
 
Example 20
Source Project: parquet-mr   Source File: TestReflectReadWrite.java    License: Apache License 2.0 5 votes vote down vote up
private GenericRecord getGenericPojoUtf8() {
  Schema schema = ReflectData.get().getSchema(Pojo.class);
  GenericData.Record record = new GenericData.Record(schema);
  record.put("myboolean", true);
  record.put("mybyte", 1);
  record.put("myshort", 1);
  record.put("myint", 1);
  record.put("mylong", 2L);
  record.put("myfloat", 3.1f);
  record.put("mydouble", 4.1);
  record.put("mybytes", ByteBuffer.wrap(new byte[] { 1, 2, 3, 4 }));
  record.put("mystring", new Utf8("Hello"));
  record.put("myenum", new GenericData.EnumSymbol(
      schema.getField("myenum").schema(), "A"));
  Map<CharSequence, CharSequence> map = new HashMap<CharSequence, CharSequence>();
  map.put(new Utf8("a"), new Utf8("1"));
  map.put(new Utf8("b"), new Utf8("2"));
  record.put("mymap", map);
  record.put("myshortarray", new GenericData.Array<Integer>(
      schema.getField("myshortarray").schema(), Lists.newArrayList(1, 2)));
  record.put("myintarray", new GenericData.Array<Integer>(
      schema.getField("myintarray").schema(), Lists.newArrayList(1, 2)));
  record.put("mystringarray", new GenericData.Array<Utf8>(
      schema.getField("mystringarray").schema(), Lists.newArrayList(new Utf8("a"), new Utf8("b"))));
  record.put("mylist", new GenericData.Array<Utf8>(
      schema.getField("mylist").schema(), Lists.newArrayList(new Utf8("a"), new Utf8("b"), new Utf8("c"))));
  record.put("mystringable", new StringableObj("blah blah"));
  return record;
}
 
Example 21
Source Project: parquet-mr   Source File: AvroRecordConverter.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
private Collection<Object> newContainer() {
  if (containerClass == null) {
    return new GenericData.Array<Object>(0, avroSchema);
  } else if (containerClass.isAssignableFrom(ArrayList.class)) {
    return new ArrayList<Object>();
  } else {
    // not need to use the data model to instantiate because it resolved
    // the class, which used the correct ClassLoader
    return (Collection<Object>) ReflectData.newInstance(containerClass, avroSchema);
  }
}
 
Example 22
Source Project: components   Source File: DBTestUtils.java    License: Apache License 2.0 5 votes vote down vote up
private static IndexedRecord copyValueFrom(IndexedRecord record) {
    Schema schema = record.getSchema();
    IndexedRecord result = new GenericData.Record(schema);
    List<Schema.Field> fields = schema.getFields();
    for (int i = 0; i < fields.size(); i++) {
        result.put(i, record.get(i));
    }

    return result;
}
 
Example 23
Source Project: streamline   Source File: StreamlineEventSerializerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testRecordWithFixed () {
    List<Schema.Field> fields = new ArrayList<>();
    Map<String, Object> data = new HashMap<>();
    for (int i = 0; i < PRIMITIVE_VALUES.length; ++i) {
        Schema.Field field = new Schema.Field(NAMES[i], Schema.create(SCHEMA_TYPES[i]), null, null);
        fields.add(field);
        data.put(NAMES[i], PRIMITIVE_VALUES[i]);
    }

    //add fixed to test case
    fields.add(new Schema.Field("fixed", Schema.createFixed("fixedSchema", null, null, 10), null, null));
    data.put("fixed", "bytes".getBytes());

    //add array to test case
    fields.add(new Schema.Field("array", Schema.createArray(Schema.create(Schema.Type.INT)), null, null));
    List<Integer> integerList = new ArrayList<>();
    integerList.add(1);
    integerList.add(2);
    data.put("array", integerList);

    Schema schema = Schema.createRecord(fields);
    GenericRecord expected = new GenericData.Record(schema);
    for (int i = 0; i < PRIMITIVE_VALUES.length; ++i) {
        expected.put(NAMES[i], PRIMITIVE_VALUES[i]);
    }
    expected.put("fixed", new GenericData.Fixed(Schema.createFixed("fixedSchema", null, null, 10), "bytes".getBytes()));
    expected.put("array", new GenericData.Array<Integer>(Schema.createArray(Schema.create(Schema.Type.INT)), integerList));
    StreamlineEvent streamlineEvent = StreamlineEventImpl.builder().fieldsAndValues(data).dataSourceId("dataSourceId").build();
    Assert.assertEquals(expected, StreamlineEventSerializer.getAvroRecord(streamlineEvent, schema));
}
 
Example 24
Source Project: flink   Source File: AvroRowSerializationSchema.java    License: Apache License 2.0 5 votes vote down vote up
private GenericRecord convertRowToAvroRecord(Schema schema, Row row) {
	final List<Schema.Field> fields = schema.getFields();
	final int length = fields.size();
	final GenericRecord record = new GenericData.Record(schema);
	for (int i = 0; i < length; i++) {
		final Schema.Field field = fields.get(i);
		record.put(i, convertFlinkType(field.schema(), row.getField(i)));
	}
	return record;
}
 
Example 25
Source Project: samza   Source File: TestAvroRelConversion.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testComplexRecordConversion() throws IOException {
  GenericData.Record record = new GenericData.Record(ComplexRecord.SCHEMA$);

  record.put("id", id);
  record.put("bool_value", boolValue);
  record.put("double_value", doubleValue);
  record.put("float_value0", floatValue);
  record.put("string_value", testStrValue);
  record.put("bytes_value", testBytes);
  record.put("fixed_value", fixedBytes);
  record.put("long_value", longValue);
  record.put("array_values", arrayValue);
  record.put("map_values", mapValue);
  record.put("union_value", testStrValue);

  ComplexRecord complexRecord = new ComplexRecord();
  complexRecord.id = id;
  complexRecord.bool_value = boolValue;
  complexRecord.double_value = doubleValue;
  complexRecord.float_value0 = floatValue;
  complexRecord.string_value = testStrValue;
  complexRecord.bytes_value = testBytes;
  complexRecord.fixed_value = fixedBytes;
  complexRecord.long_value = longValue;
  complexRecord.array_values = new ArrayList<>();
  complexRecord.array_values.addAll(arrayValue);
  complexRecord.map_values = new HashMap<>();
  complexRecord.map_values.putAll(mapValue);
  complexRecord.union_value = testStrValue;


  byte[] serializedData = bytesFromGenericRecord(record);
  validateAvroSerializedData(serializedData, testStrValue);

  serializedData = encodeAvroSpecificRecord(ComplexRecord.class, complexRecord);
  validateAvroSerializedData(serializedData, testStrValue);
}
 
Example 26
Source Project: DBus   Source File: OracleGenericMessage.java    License: Apache License 2.0 5 votes vote down vote up
public GenericRecord generateRecord(Schema genericSchema) {
    GenericRecord record = new GenericData.Record(genericSchema);

    record.put(NAMESAPCE, this.nameSpace);
    record.put(SCHEMA_HASH, this.schemaHash);
    record.put(PAYLOAD, ByteBuffer.wrap(this.payload));

    return record;
}
 
Example 27
Source Project: spork   Source File: AvroStorage.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * @see org.apache.pig.LoadFunc#getInputFormat()
 */
@Override
public InputFormat<NullWritable, GenericData.Record> getInputFormat()
    throws IOException {

  return new org.apache.pig.backend.hadoop.executionengine.mapReduceLayer
      .PigFileInputFormat<NullWritable, GenericData.Record>() {

    @Override
    public RecordReader<NullWritable, GenericData.Record>
      createRecordReader(final InputSplit is, final TaskAttemptContext tc)
        throws IOException, InterruptedException {
      Schema s = getInputAvroSchema();
      RecordReader<NullWritable, GenericData.Record> rr = null;
      if (s.getType() == Type.ARRAY) {
        rr = new AvroArrayReader(s);
      } else {
        rr = new AvroRecordReader(s);
      }
      try {
          rr.initialize(is, tc);
      } finally {
          rr.close();
      }
      tc.setStatus(is.toString());
      return rr;
    }
  };

}
 
Example 28
Source Project: kite   Source File: TestDataModelUtil.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testResolveTypeGenericToGeneric() {
  Class<GenericData.Record> type = GenericData.Record.class;
  Schema schema = StandardEvent.getClassSchema();
  Class expResult = type;
  Class result = DataModelUtil.resolveType(type, schema);
  assertEquals(expResult, result);
}
 
Example 29
Source Project: incubator-pinot   Source File: ParquetUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Returns a ParquetReader with the given path.
 */
public static ParquetReader<GenericRecord> getParquetReader(Path path)
    throws IOException {
  //noinspection unchecked
  return AvroParquetReader.<GenericRecord>builder(path).disableCompatibility().withDataModel(GenericData.get())
      .withConf(getConfiguration()).build();
}
 
Example 30
Source Project: localization_nifi   Source File: TestNiFiOrcUtils.java    License: Apache License 2.0 5 votes vote down vote up
public static GenericData.Record buildComplexAvroRecord(Integer i, Map<String, Double> m, String e, Object unionVal, List<Integer> intArray) {
    Schema schema = buildComplexAvroSchema();
    GenericData.Record row = new GenericData.Record(schema);
    row.put("myInt", i);
    row.put("myMap", m);
    row.put("myEnum", e);
    row.put("myLongOrFloat", unionVal);
    row.put("myIntList", intArray);
    return row;
}