org.apache.flink.formats.avro.generated.User Java Examples

The following examples show how to use org.apache.flink.formats.avro.generated.User. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: AvroSerializerSnapshotTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void recordSerializedShouldBeDeserializeWithTheResortedSerializer() throws IOException {
	// user is an avro generated test object.
	final User user = TestDataGenerator.generateRandomUser(new Random());
	final AvroSerializer<User> originalSerializer = new AvroSerializer<>(User.class);
	//
	// first serialize the record
	//
	ByteBuffer serializedUser = serialize(originalSerializer, user);
	//
	// then restore a serializer from the snapshot
	//
	TypeSerializer<User> restoredSerializer = originalSerializer.snapshotConfiguration().restoreSerializer();
	//
	// now deserialize the user with the resorted serializer.
	//
	User restoredUser = deserialize(restoredSerializer, serializedUser);

	assertThat(user, is(restoredUser));
}
 
Example #2
Source File: TestDataGenerator.java    From flink with Apache License 2.0 6 votes vote down vote up
public static User generateRandomUser(Random rnd) {
	return new User(
			generateRandomString(rnd, 50),
			rnd.nextBoolean() ? null : rnd.nextInt(),
			rnd.nextBoolean() ? null : generateRandomString(rnd, 6),
			rnd.nextBoolean() ? null : rnd.nextLong(),
			rnd.nextDouble(),
			null,
			rnd.nextBoolean(),
			generateRandomStringList(rnd, 20, 30),
			generateRandomBooleanList(rnd, 20),
			rnd.nextBoolean() ? null : generateRandomStringList(rnd, 20, 20),
			generateRandomColor(rnd),
			new HashMap<>(),
			generateRandomFixed16(rnd),
			generateRandomUnion(rnd),
			generateRandomAddress(rnd),
			generateRandomBytes(rnd),
			LocalDate.parse("2014-03-01"),
			LocalTime.parse("12:12:12"),
			123456,
			DateTime.parse("2014-03-01T12:12:12.321Z"),
			123456L,
			ByteBuffer.wrap(BigDecimal.valueOf(2000, 2).unscaledValue().toByteArray()),
			new Fixed2(BigDecimal.valueOf(2000, 2).unscaledValue().toByteArray()));
}
 
Example #3
Source File: AvroTypeExtractionTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testKeySelection() throws Exception {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.getConfig().enableObjectReuse();
	Path in = new Path(inFile.getAbsoluteFile().toURI());

	AvroInputFormat<User> users = new AvroInputFormat<>(in, User.class);
	DataSet<User> usersDS = env.createInput(users);

	DataSet<Tuple2<String, Integer>> res = usersDS
		.groupBy("name")
		.reduceGroup((GroupReduceFunction<User, Tuple2<String, Integer>>) (values, out) -> {
			for (User u : values) {
				out.collect(new Tuple2<>(u.getName().toString(), 1));
			}
		})
		.returns(Types.TUPLE(Types.STRING, Types.INT));
	res.writeAsText(resultPath);
	env.execute("Avro Key selection");

	expected = "(Alyssa,1)\n(Charlie,1)\n";
}
 
Example #4
Source File: AvroTypeExtractionTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testWithKryoGenericSer() throws Exception {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.getConfig().enableForceKryo();
	Path in = new Path(inFile.getAbsoluteFile().toURI());

	AvroInputFormat<User> users = new AvroInputFormat<>(in, User.class);
	DataSet<User> usersDS = env.createInput(users);

	DataSet<Tuple2<String, Integer>> res = usersDS
		.groupBy((KeySelector<User, String>) value -> String.valueOf(value.getName()))
		.reduceGroup((GroupReduceFunction<User, Tuple2<String, Integer>>) (values, out) -> {
			for (User u : values) {
				out.collect(new Tuple2<>(u.getName().toString(), 1));
			}
		})
		.returns(Types.TUPLE(Types.STRING, Types.INT));

	res.writeAsText(resultPath);
	env.execute("Avro Key selection");

	expected = "(Charlie,1)\n(Alyssa,1)\n";
}
 
Example #5
Source File: AvroTypeExtractionTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testWithAvroGenericSer() throws Exception {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.getConfig().enableForceAvro();
	Path in = new Path(inFile.getAbsoluteFile().toURI());

	AvroInputFormat<User> users = new AvroInputFormat<>(in, User.class);
	DataSet<User> usersDS = env.createInput(users);

	DataSet<Tuple2<String, Integer>> res = usersDS
		.groupBy((KeySelector<User, String>) value -> String.valueOf(value.getName()))
		.reduceGroup((GroupReduceFunction<User, Tuple2<String, Integer>>) (values, out) -> {
			for (User u : values) {
				out.collect(new Tuple2<>(u.getName().toString(), 1));
			}
		})
		.returns(Types.TUPLE(Types.STRING, Types.INT));

	res.writeAsText(resultPath);
	env.execute("Avro Key selection");

	expected = "(Charlie,1)\n(Alyssa,1)\n";
}
 
Example #6
Source File: AvroTypeExtractionTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testKeySelection() throws Exception {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.getConfig().enableObjectReuse();
	Path in = new Path(inFile.getAbsoluteFile().toURI());

	AvroInputFormat<User> users = new AvroInputFormat<>(in, User.class);
	DataSet<User> usersDS = env.createInput(users);

	DataSet<Tuple2<String, Integer>> res = usersDS
		.groupBy("name")
		.reduceGroup((GroupReduceFunction<User, Tuple2<String, Integer>>) (values, out) -> {
			for (User u : values) {
				out.collect(new Tuple2<>(u.getName().toString(), 1));
			}
		})
		.returns(Types.TUPLE(Types.STRING, Types.INT));
	res.writeAsText(resultPath);
	env.execute("Avro Key selection");

	expected = "(Alyssa,1)\n(Charlie,1)\n";
}
 
Example #7
Source File: AvroOutputFormatTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testCompression() throws Exception {
	// given
	final Path outputPath = new Path(File.createTempFile("avro-output-file", "avro").getAbsolutePath());
	final AvroOutputFormat<User> outputFormat = new AvroOutputFormat<>(outputPath, User.class);
	outputFormat.setWriteMode(FileSystem.WriteMode.OVERWRITE);

	final Path compressedOutputPath = new Path(File.createTempFile("avro-output-file", "compressed.avro").getAbsolutePath());
	final AvroOutputFormat<User> compressedOutputFormat = new AvroOutputFormat<>(compressedOutputPath, User.class);
	compressedOutputFormat.setWriteMode(FileSystem.WriteMode.OVERWRITE);
	compressedOutputFormat.setCodec(AvroOutputFormat.Codec.SNAPPY);

	// when
	output(outputFormat);
	output(compressedOutputFormat);

	// then
	assertTrue(fileSize(outputPath) > fileSize(compressedOutputPath));

	// cleanup
	FileSystem fs = FileSystem.getLocalFileSystem();
	fs.delete(outputPath, false);
	fs.delete(compressedOutputPath, false);
}
 
Example #8
Source File: AvroOutputFormatTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testCompression() throws Exception {
	// given
	final Path outputPath = new Path(File.createTempFile("avro-output-file", "avro").getAbsolutePath());
	final AvroOutputFormat<User> outputFormat = new AvroOutputFormat<>(outputPath, User.class);
	outputFormat.setWriteMode(FileSystem.WriteMode.OVERWRITE);

	final Path compressedOutputPath = new Path(File.createTempFile("avro-output-file", "compressed.avro").getAbsolutePath());
	final AvroOutputFormat<User> compressedOutputFormat = new AvroOutputFormat<>(compressedOutputPath, User.class);
	compressedOutputFormat.setWriteMode(FileSystem.WriteMode.OVERWRITE);
	compressedOutputFormat.setCodec(AvroOutputFormat.Codec.SNAPPY);

	// when
	output(outputFormat);
	output(compressedOutputFormat);

	// then
	assertTrue(fileSize(outputPath) > fileSize(compressedOutputPath));

	// cleanup
	FileSystem fs = FileSystem.getLocalFileSystem();
	fs.delete(outputPath, false);
	fs.delete(compressedOutputPath, false);
}
 
Example #9
Source File: AvroOutputFormatITCase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public User map(Tuple3<String, Integer, String> value) {
	User user = new User();
	user.setName(value.f0);
	user.setFavoriteNumber(value.f1);
	user.setFavoriteColor(value.f2);
	user.setTypeBoolTest(true);
	user.setTypeArrayString(Collections.emptyList());
	user.setTypeArrayBoolean(Collections.emptyList());
	user.setTypeEnum(Colors.BLUE);
	user.setTypeMap(Collections.emptyMap());
	user.setTypeBytes(ByteBuffer.allocate(10));
	user.setTypeDate(LocalDate.parse("2014-03-01"));
	user.setTypeTimeMillis(LocalTime.parse("12:12:12"));
	user.setTypeTimeMicros(123456);
	user.setTypeTimestampMillis(DateTime.parse("2014-03-01T12:12:12.321Z"));
	user.setTypeTimestampMicros(123456L);
	// 20.00
	user.setTypeDecimalBytes(ByteBuffer.wrap(BigDecimal.valueOf(2000, 2).unscaledValue().toByteArray()));
	// 20.00
	user.setTypeDecimalFixed(new Fixed2(BigDecimal.valueOf(2000, 2).unscaledValue().toByteArray()));
	return user;
}
 
Example #10
Source File: AvroTypeExtractionTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testWithKryoGenericSer() throws Exception {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.getConfig().enableForceKryo();
	Path in = new Path(inFile.getAbsoluteFile().toURI());

	AvroInputFormat<User> users = new AvroInputFormat<>(in, User.class);
	DataSet<User> usersDS = env.createInput(users);

	DataSet<Tuple2<String, Integer>> res = usersDS
		.groupBy((KeySelector<User, String>) value -> String.valueOf(value.getName()))
		.reduceGroup((GroupReduceFunction<User, Tuple2<String, Integer>>) (values, out) -> {
			for (User u : values) {
				out.collect(new Tuple2<>(u.getName().toString(), 1));
			}
		})
		.returns(Types.TUPLE(Types.STRING, Types.INT));

	res.writeAsText(resultPath);
	env.execute("Avro Key selection");

	expected = "(Charlie,1)\n(Alyssa,1)\n";
}
 
Example #11
Source File: AvroOutputFormatTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testSetCodecError() {
	// given
	boolean error = false;
	final AvroOutputFormat<User> outputFormat = new AvroOutputFormat<>(User.class);

	// when
	try {
		outputFormat.setCodec(null);
	} catch (Exception ex) {
		error = true;
	}

	// then
	assertTrue(error);
}
 
Example #12
Source File: TestDataGenerator.java    From flink with Apache License 2.0 6 votes vote down vote up
public static User generateRandomUser(Random rnd) {
	return new User(
			generateRandomString(rnd, 50),
			rnd.nextBoolean() ? null : rnd.nextInt(),
			rnd.nextBoolean() ? null : generateRandomString(rnd, 6),
			rnd.nextBoolean() ? null : rnd.nextLong(),
			rnd.nextDouble(),
			null,
			rnd.nextBoolean(),
			generateRandomStringList(rnd, 20, 30),
			generateRandomBooleanList(rnd, 20),
			rnd.nextBoolean() ? null : generateRandomStringList(rnd, 20, 20),
			generateRandomColor(rnd),
			new HashMap<>(),
			generateRandomFixed16(rnd),
			generateRandomUnion(rnd),
			generateRandomAddress(rnd),
			generateRandomBytes(rnd),
			LocalDate.parse("2014-03-01"),
			LocalTime.parse("12:12:12"),
			123456,
			DateTime.parse("2014-03-01T12:12:12.321Z"),
			123456L,
			ByteBuffer.wrap(BigDecimal.valueOf(2000, 2).unscaledValue().toByteArray()),
			new Fixed2(BigDecimal.valueOf(2000, 2).unscaledValue().toByteArray()));
}
 
Example #13
Source File: AvroOutputFormatITCase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
protected void testProgram() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple3<String, Integer, String>> input = env.readCsvFile(inputPath)
		.fieldDelimiter("|")
		.types(String.class, Integer.class, String.class);

	//output the data with AvroOutputFormat for specific user type
	DataSet<User> specificUser = input.map(new ConvertToUser());
	AvroOutputFormat<User> avroOutputFormat = new AvroOutputFormat<>(User.class);
	avroOutputFormat.setCodec(Codec.SNAPPY); // FLINK-4771: use a codec
	avroOutputFormat.setSchema(User.SCHEMA$); //FLINK-3304: Ensure the OF is properly serializing the schema
	specificUser.write(avroOutputFormat, outputPath1);

	//output the data with AvroOutputFormat for reflect user type
	DataSet<ReflectiveUser> reflectiveUser = specificUser.map(new ConvertToReflective());
	reflectiveUser.write(new AvroOutputFormat<>(ReflectiveUser.class), outputPath2);

	env.execute();
}
 
Example #14
Source File: AvroOutputFormatITCase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Override
protected void testProgram() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple3<String, Integer, String>> input = env.readCsvFile(inputPath)
		.fieldDelimiter("|")
		.types(String.class, Integer.class, String.class);

	//output the data with AvroOutputFormat for specific user type
	DataSet<User> specificUser = input.map(new ConvertToUser());
	AvroOutputFormat<User> avroOutputFormat = new AvroOutputFormat<>(User.class);
	avroOutputFormat.setCodec(Codec.SNAPPY); // FLINK-4771: use a codec
	avroOutputFormat.setSchema(User.SCHEMA$); //FLINK-3304: Ensure the OF is properly serializing the schema
	specificUser.write(avroOutputFormat, outputPath1);

	//output the data with AvroOutputFormat for reflect user type
	DataSet<ReflectiveUser> reflectiveUser = specificUser.map(new ConvertToReflective());
	reflectiveUser.write(new AvroOutputFormat<>(ReflectiveUser.class), outputPath2);

	env.execute();
}
 
Example #15
Source File: AvroOutputFormatTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testSetCodecError() {
	// given
	boolean error = false;
	final AvroOutputFormat<User> outputFormat = new AvroOutputFormat<>(User.class);

	// when
	try {
		outputFormat.setCodec(null);
	} catch (Exception ex) {
		error = true;
	}

	// then
	assertTrue(error);
}
 
Example #16
Source File: AvroOutputFormatITCase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public User map(Tuple3<String, Integer, String> value) {
	User user = new User();
	user.setName(value.f0);
	user.setFavoriteNumber(value.f1);
	user.setFavoriteColor(value.f2);
	user.setTypeBoolTest(true);
	user.setTypeArrayString(Collections.emptyList());
	user.setTypeArrayBoolean(Collections.emptyList());
	user.setTypeEnum(Colors.BLUE);
	user.setTypeMap(Collections.emptyMap());
	user.setTypeBytes(ByteBuffer.allocate(10));
	user.setTypeDate(LocalDate.parse("2014-03-01"));
	user.setTypeTimeMillis(LocalTime.parse("12:12:12"));
	user.setTypeTimeMicros(123456);
	user.setTypeTimestampMillis(DateTime.parse("2014-03-01T12:12:12.321Z"));
	user.setTypeTimestampMicros(123456L);
	// 20.00
	user.setTypeDecimalBytes(ByteBuffer.wrap(BigDecimal.valueOf(2000, 2).unscaledValue().toByteArray()));
	// 20.00
	user.setTypeDecimalFixed(new Fixed2(BigDecimal.valueOf(2000, 2).unscaledValue().toByteArray()));
	return user;
}
 
Example #17
Source File: AvroTypeExtractionTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testWithAvroGenericSer() throws Exception {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.getConfig().enableForceAvro();
	Path in = new Path(inFile.getAbsoluteFile().toURI());

	AvroInputFormat<User> users = new AvroInputFormat<>(in, User.class);
	DataSet<User> usersDS = env.createInput(users);

	DataSet<Tuple2<String, Integer>> res = usersDS
		.groupBy((KeySelector<User, String>) value -> String.valueOf(value.getName()))
		.reduceGroup((GroupReduceFunction<User, Tuple2<String, Integer>>) (values, out) -> {
			for (User u : values) {
				out.collect(new Tuple2<>(u.getName().toString(), 1));
			}
		})
		.returns(Types.TUPLE(Types.STRING, Types.INT));

	res.writeAsText(resultPath);
	env.execute("Avro Key selection");

	expected = "(Charlie,1)\n(Alyssa,1)\n";
}
 
Example #18
Source File: AvroTypeExtractionTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testWithKryoGenericSer() throws Exception {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.getConfig().enableForceKryo();
	Path in = new Path(inFile.getAbsoluteFile().toURI());

	AvroInputFormat<User> users = new AvroInputFormat<>(in, User.class);
	DataSet<User> usersDS = env.createInput(users);

	DataSet<Tuple2<String, Integer>> res = usersDS
		.groupBy((KeySelector<User, String>) value -> String.valueOf(value.getName()))
		.reduceGroup((GroupReduceFunction<User, Tuple2<String, Integer>>) (values, out) -> {
			for (User u : values) {
				out.collect(new Tuple2<>(u.getName().toString(), 1));
			}
		})
		.returns(Types.TUPLE(Types.STRING, Types.INT));

	res.writeAsText(resultPath);
	env.execute("Avro Key selection");

	expected = "(Charlie,1)\n(Alyssa,1)\n";
}
 
Example #19
Source File: AvroOutputFormatTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testSetCodecError() {
	// given
	boolean error = false;
	final AvroOutputFormat<User> outputFormat = new AvroOutputFormat<>(User.class);

	// when
	try {
		outputFormat.setCodec(null);
	} catch (Exception ex) {
		error = true;
	}

	// then
	assertTrue(error);
}
 
Example #20
Source File: AvroOutputFormatTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testSerialization() throws Exception {

	serializeAndDeserialize(null, null);
	serializeAndDeserialize(null, User.SCHEMA$);
	for (final AvroOutputFormat.Codec codec : AvroOutputFormat.Codec.values()) {
		serializeAndDeserialize(codec, null);
		serializeAndDeserialize(codec, User.SCHEMA$);
	}
}
 
Example #21
Source File: AvroOutputFormatTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testSerialization() throws Exception {

	serializeAndDeserialize(null, null);
	serializeAndDeserialize(null, User.SCHEMA$);
	for (final AvroOutputFormat.Codec codec : AvroOutputFormat.Codec.values()) {
		serializeAndDeserialize(codec, null);
		serializeAndDeserialize(codec, User.SCHEMA$);
	}
}
 
Example #22
Source File: AvroOutputFormatTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private void serializeAndDeserialize(final AvroOutputFormat.Codec codec, final Schema schema) throws IOException, ClassNotFoundException {
	// given
	final AvroOutputFormat<User> outputFormat = new AvroOutputFormat<>(User.class);
	if (codec != null) {
		outputFormat.setCodec(codec);
	}
	if (schema != null) {
		outputFormat.setSchema(schema);
	}

	final ByteArrayOutputStream bos = new ByteArrayOutputStream();

	// when
	try (final ObjectOutputStream oos = new ObjectOutputStream(bos)) {
		oos.writeObject(outputFormat);
	}
	try (final ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(bos.toByteArray()))) {
		// then
		Object o = ois.readObject();
		assertTrue(o instanceof AvroOutputFormat);
		@SuppressWarnings("unchecked")
		final AvroOutputFormat<User> restored = (AvroOutputFormat<User>) o;
		final AvroOutputFormat.Codec restoredCodec = (AvroOutputFormat.Codec) Whitebox.getInternalState(restored, "codec");
		final Schema restoredSchema = (Schema) Whitebox.getInternalState(restored, "userDefinedSchema");

		assertTrue(codec != null ? restoredCodec == codec : restoredCodec == null);
		assertTrue(schema != null ? restoredSchema.equals(schema) : restoredSchema == null);
	}
}
 
Example #23
Source File: AvroTypesITCase.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testAvroToAvro() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tEnv = BatchTableEnvironment.create(env, config());

	Table t = tEnv.fromDataSet(testData(env));
	Table result = t.select($("*"));

	List<User> results = tEnv.toDataSet(result, Types.POJO(User.class)).collect();
	List<User> expected = Arrays.asList(USER_1, USER_2, USER_3);
	assertEquals(expected, results);
}
 
Example #24
Source File: AvroSplittableInputFormatTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testSplittedIF() throws IOException {
	Configuration parameters = new Configuration();

	AvroInputFormat<User> format = new AvroInputFormat<>(new Path(testFile.getAbsolutePath()), User.class);

	format.configure(parameters);
	FileInputSplit[] splits = format.createInputSplits(4);
	assertEquals(splits.length, 4);
	int elements = 0;
	int[] elementsPerSplit = new int[4];
	for (int i = 0; i < splits.length; i++) {
		format.open(splits[i]);
		while (!format.reachedEnd()) {
			User u = format.nextRecord(null);
			Assert.assertTrue(u.getName().toString().startsWith(TEST_NAME));
			elements++;
			elementsPerSplit[i]++;
		}
		format.close();
	}

	Assert.assertEquals(1604, elementsPerSplit[0]);
	Assert.assertEquals(1203, elementsPerSplit[1]);
	Assert.assertEquals(1203, elementsPerSplit[2]);
	Assert.assertEquals(990, elementsPerSplit[3]);
	Assert.assertEquals(NUM_RECORDS, elements);
	format.close();
}
 
Example #25
Source File: EncoderDecoderTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testGeneratedObjectWithNullableFields() {
	List<CharSequence> strings = Arrays.asList(new CharSequence[] { "These", "strings", "should", "be", "recognizable", "as", "a", "meaningful", "sequence" });
	List<Boolean> bools = Arrays.asList(true, true, false, false, true, false, true, true);
	Map<CharSequence, Long> map = new HashMap<>();
	map.put("1", 1L);
	map.put("2", 2L);
	map.put("3", 3L);

	byte[] b = new byte[16];
	new Random().nextBytes(b);
	Fixed16 f = new Fixed16(b);
	Address addr = new Address(239, "6th Main", "Bangalore", "Karnataka", "560075");
	User user = new User(
		"Freudenreich",
		1337,
		"macintosh gray",
		1234567890L,
		3.1415926,
		null,
		true,
		strings,
		bools,
		null,
		Colors.GREEN,
		map,
		f,
		Boolean.TRUE,
		addr,
		ByteBuffer.wrap(b),
		LocalDate.parse("2014-03-01"),
		LocalTime.parse("12:12:12"),
		123456,
		DateTime.parse("2014-03-01T12:12:12.321Z"),
		123456L,
		ByteBuffer.wrap(BigDecimal.valueOf(2000, 2).unscaledValue().toByteArray()), // 20.00
		new Fixed2(BigDecimal.valueOf(2000, 2).unscaledValue().toByteArray())); // 20.00

	testObjectSerialization(user);
}
 
Example #26
Source File: AvroSplittableInputFormatTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testSplittedIF() throws IOException {
	Configuration parameters = new Configuration();

	AvroInputFormat<User> format = new AvroInputFormat<>(new Path(testFile.getAbsolutePath()), User.class);

	format.configure(parameters);
	FileInputSplit[] splits = format.createInputSplits(4);
	assertEquals(splits.length, 4);
	int elements = 0;
	int[] elementsPerSplit = new int[4];
	for (int i = 0; i < splits.length; i++) {
		format.open(splits[i]);
		while (!format.reachedEnd()) {
			User u = format.nextRecord(null);
			Assert.assertTrue(u.getName().toString().startsWith(TEST_NAME));
			elements++;
			elementsPerSplit[i]++;
		}
		format.close();
	}

	Assert.assertEquals(1604, elementsPerSplit[0]);
	Assert.assertEquals(1203, elementsPerSplit[1]);
	Assert.assertEquals(1203, elementsPerSplit[2]);
	Assert.assertEquals(990, elementsPerSplit[3]);
	Assert.assertEquals(NUM_RECORDS, elements);
	format.close();
}
 
Example #27
Source File: AvroTypesITCase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private DataSet<User> testData(ExecutionEnvironment env) {
	List<User> data = new ArrayList<>(3);
	data.add(USER_1);
	data.add(USER_2);
	data.add(USER_3);
	return env.fromCollection(data);
}
 
Example #28
Source File: AvroTypesITCase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testAvroToAvro() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tEnv = BatchTableEnvironment.create(env, config());

	Table t = tEnv.fromDataSet(testData(env));
	Table result = t.select("*");

	List<User> results = tEnv.toDataSet(result, Types.POJO(User.class)).collect();
	List<User> expected = Arrays.asList(USER_1, USER_2, USER_3);
	assertEquals(expected, results);
}
 
Example #29
Source File: AvroSerializerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
protected User[] getTestData() {
	final Random rnd = new Random();
	final User[] users = new User[20];

	for (int i = 0; i < users.length; i++) {
		users[i] = TestDataGenerator.generateRandomUser(rnd);
	}

	return users;
}
 
Example #30
Source File: AvroTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
public List<Descriptor> descriptors() {
	final Descriptor desc1 = new Avro().recordClass(User.class);

	final Descriptor desc2 = new Avro().avroSchema("{...}");

	return Arrays.asList(desc1, desc2);
}