Java Code Examples for org.apache.flink.core.memory.ByteArrayOutputStreamWithPos#toByteArray()

The following examples show how to use org.apache.flink.core.memory.ByteArrayOutputStreamWithPos#toByteArray() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: InternalPriorityQueueTestBase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Override
public int compare(TestElement o1, TestElement o2) {

	ByteArrayOutputStreamWithPos os = new ByteArrayOutputStreamWithPos();
	DataOutputViewStreamWrapper ow = new DataOutputViewStreamWrapper(os);
	try {
		TestElementSerializer.INSTANCE.serialize(o1, ow);
		byte[] a1 = os.toByteArray();
		os.reset();
		TestElementSerializer.INSTANCE.serialize(o2, ow);
		byte[] a2 = os.toByteArray();
		return UnsignedBytes.lexicographicalComparator().compare(a1, a2);
	} catch (Exception e) {
		throw new RuntimeException(e);
	}
}
Example 2
Source File: InternalPriorityQueueTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
public int compare(TestElement o1, TestElement o2) {

	ByteArrayOutputStreamWithPos os = new ByteArrayOutputStreamWithPos();
	DataOutputViewStreamWrapper ow = new DataOutputViewStreamWrapper(os);
	try {
		TestElementSerializer.INSTANCE.serialize(o1, ow);
		byte[] a1 = os.toByteArray();
		os.reset();
		TestElementSerializer.INSTANCE.serialize(o2, ow);
		byte[] a2 = os.toByteArray();
		return UnsignedBytes.lexicographicalComparator().compare(a1, a2);
	} catch (Exception e) {
		throw new RuntimeException(e);
	}
}
Example 3
Source File: InternalPriorityQueueTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
public int compare(TestElement o1, TestElement o2) {

	ByteArrayOutputStreamWithPos os = new ByteArrayOutputStreamWithPos();
	DataOutputViewStreamWrapper ow = new DataOutputViewStreamWrapper(os);
	try {
		TestElementSerializer.INSTANCE.serialize(o1, ow);
		byte[] a1 = os.toByteArray();
		os.reset();
		TestElementSerializer.INSTANCE.serialize(o2, ow);
		byte[] a2 = os.toByteArray();
		return UnsignedBytes.lexicographicalComparator().compare(a1, a2);
	} catch (Exception e) {
		throw new RuntimeException(e);
	}
}
Example 4
Source File: SavepointV1SerializerTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
/**
 * Test serialization of {@link SavepointV1} instance.
 */
@Test
public void testSerializeDeserializeV1() throws Exception {
	final Random r = new Random(42);

	for (int i = 0; i < 50; ++i) {
		SavepointV1 expected =
				new SavepointV1(i+ 123123, CheckpointTestUtils.createTaskStates(r, 1 + r.nextInt(64), 1 + r.nextInt(64)));

		SavepointV1Serializer serializer = SavepointV1Serializer.INSTANCE;

		// Serialize
		ByteArrayOutputStreamWithPos baos = new ByteArrayOutputStreamWithPos();
		serializer.serializeOld(expected, new DataOutputViewStreamWrapper(baos));
		byte[] bytes = baos.toByteArray();

		// Deserialize
		ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
		SavepointV2 actual = serializer.deserialize(
				new DataInputViewStreamWrapper(bais),
				Thread.currentThread().getContextClassLoader());


		assertEquals(expected.getCheckpointId(), actual.getCheckpointId());
		assertEquals(expected.getTaskStates(), actual.getTaskStates());
		assertTrue(actual.getMasterStates().isEmpty());
	}
}
Example 5
Source File: SavepointV2SerializerTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
private void testCheckpointSerialization(
		long checkpointId,
		Collection<OperatorState> operatorStates,
		Collection<MasterState> masterStates) throws IOException {

	SavepointV2Serializer serializer = SavepointV2Serializer.INSTANCE;

	ByteArrayOutputStreamWithPos baos = new ByteArrayOutputStreamWithPos();
	DataOutputStream out = new DataOutputViewStreamWrapper(baos);

	serializer.serialize(new SavepointV2(checkpointId, operatorStates, masterStates), out);
	out.close();

	byte[] bytes = baos.toByteArray();

	DataInputStream in = new DataInputViewStreamWrapper(new ByteArrayInputStreamWithPos(bytes));
	SavepointV2 deserialized = serializer.deserialize(in, getClass().getClassLoader());

	assertEquals(checkpointId, deserialized.getCheckpointId());
	assertEquals(operatorStates, deserialized.getOperatorStates());

	assertEquals(masterStates.size(), deserialized.getMasterStates().size());
	for (Iterator<MasterState> a = masterStates.iterator(), b = deserialized.getMasterStates().iterator();
			a.hasNext();)
	{
		CheckpointTestUtils.assertMasterStateEquality(a.next(), b.next());
	}
}
Example 6
Source File: SavepointV1SerializerTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Test serialization of {@link SavepointV1} instance.
 */
@Test
public void testSerializeDeserializeV1() throws Exception {
	final Random r = new Random(42);

	for (int i = 0; i < 50; ++i) {
		SavepointV1 expected =
				new SavepointV1(i+ 123123, CheckpointTestUtils.createTaskStates(r, 1 + r.nextInt(64), 1 + r.nextInt(64)));

		SavepointV1Serializer serializer = SavepointV1Serializer.INSTANCE;

		// Serialize
		ByteArrayOutputStreamWithPos baos = new ByteArrayOutputStreamWithPos();
		serializer.serializeOld(expected, new DataOutputViewStreamWrapper(baos));
		byte[] bytes = baos.toByteArray();

		// Deserialize
		ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
		SavepointV2 actual = serializer.deserialize(
				new DataInputViewStreamWrapper(bais),
				Thread.currentThread().getContextClassLoader());


		assertEquals(expected.getCheckpointId(), actual.getCheckpointId());
		assertEquals(expected.getTaskStates(), actual.getTaskStates());
		assertTrue(actual.getMasterStates().isEmpty());
	}
}
Example 7
Source File: SavepointV2SerializerTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
private void testCheckpointSerialization(
		long checkpointId,
		Collection<OperatorState> operatorStates,
		Collection<MasterState> masterStates) throws IOException {

	SavepointV2Serializer serializer = SavepointV2Serializer.INSTANCE;

	ByteArrayOutputStreamWithPos baos = new ByteArrayOutputStreamWithPos();
	DataOutputStream out = new DataOutputViewStreamWrapper(baos);

	serializer.serialize(new SavepointV2(checkpointId, operatorStates, masterStates), out);
	out.close();

	byte[] bytes = baos.toByteArray();

	DataInputStream in = new DataInputViewStreamWrapper(new ByteArrayInputStreamWithPos(bytes));
	SavepointV2 deserialized = serializer.deserialize(in, getClass().getClassLoader());

	assertEquals(checkpointId, deserialized.getCheckpointId());
	assertEquals(operatorStates, deserialized.getOperatorStates());

	assertEquals(masterStates.size(), deserialized.getMasterStates().size());
	for (Iterator<MasterState> a = masterStates.iterator(), b = deserialized.getMasterStates().iterator();
			a.hasNext();)
	{
		CheckpointTestUtils.assertMasterStateEquality(a.next(), b.next());
	}
}
Example 8
Source File: ValueStateToKeyedStateRow.java    From bravo with Apache License 2.0 5 votes vote down vote up 
@Override
public KeyedStateRow map(Tuple2<K, V> t) throws Exception {
	int keyGroup = KeyGroupRangeAssignment.assignToKeyGroup(t.f0, maxParallelism);
	ByteArrayOutputStreamWithPos os = new ByteArrayOutputStreamWithPos();
	DataOutputViewStreamWrapper ov = new DataOutputViewStreamWrapper(os);

	RocksDBUtils.writeKeyGroup(keyGroup, keygroupPrefixBytes, ov);
	RocksDBUtils.writeKey(t.f0, keySerializer, os, ov, false);
	RocksDBUtils.writeNameSpace(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, os,
			ov, false);

	os.close();
	return new KeyedStateRow(stateName, os.toByteArray(),
			InstantiationUtil.serializeToByteArray(valueSerializer, t.f1));
}
Example 9
Source File: MetadataV3SerializerTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 * Test checkpoint metadata (de)serialization.
 *
 * @param checkpointId The given checkpointId will write into the metadata.
 * @param operatorStates the given states for all the operators.
 * @param masterStates the masterStates of the given checkpoint/savepoint.
 */
private void testCheckpointSerialization(
		long checkpointId,
		Collection<OperatorState> operatorStates,
		Collection<MasterState> masterStates,
		@Nullable String basePath) throws IOException {

	MetadataV3Serializer serializer = MetadataV3Serializer.INSTANCE;

	ByteArrayOutputStreamWithPos baos = new ByteArrayOutputStreamWithPos();
	DataOutputStream out = new DataOutputViewStreamWrapper(baos);

	CheckpointMetadata metadata = new CheckpointMetadata(checkpointId, operatorStates, masterStates);
	MetadataV3Serializer.serialize(metadata, out);
	out.close();

	// The relative pointer resolution in MetadataV2V3SerializerBase currently runs the same
	// code as the file system checkpoint location resolution. Because of that, it needs the
	// a "_metadata" file present. we could change the code to resolve the pointer without doing
	// file I/O, but it is somewhat delicate to reproduce that logic without I/O and the same guarantees
	// to differentiate between the supported options of directory addressing and metadata file addressing.
	// So, better safe than sorry, we do actually do the file system operations in the serializer for now,
	// even if it makes the tests a a tad bit more clumsy
	if (basePath != null) {
		final Path metaPath = new Path(basePath, "_metadata");
		// this is in the temp folder, so it will get automatically deleted
		FileSystem.getLocalFileSystem().create(metaPath, FileSystem.WriteMode.OVERWRITE).close();
	}

	byte[] bytes = baos.toByteArray();

	DataInputStream in = new DataInputViewStreamWrapper(new ByteArrayInputStreamWithPos(bytes));
	CheckpointMetadata deserialized = serializer.deserialize(in, getClass().getClassLoader(), basePath);
	assertEquals(checkpointId, deserialized.getCheckpointId());
	assertEquals(operatorStates, deserialized.getOperatorStates());

	assertEquals(masterStates.size(), deserialized.getMasterStates().size());
	for (Iterator<MasterState> a = masterStates.iterator(), b = deserialized.getMasterStates().iterator();
			a.hasNext();) {
		CheckpointTestUtils.assertMasterStateEquality(a.next(), b.next());
	}
}