Java Code Examples for org.apache.flink.core.memory.ByteArrayOutputStreamWithPos#toByteArray()
The following examples show how to use
org.apache.flink.core.memory.ByteArrayOutputStreamWithPos#toByteArray() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: InternalPriorityQueueTestBase.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Override public int compare(TestElement o1, TestElement o2) { ByteArrayOutputStreamWithPos os = new ByteArrayOutputStreamWithPos(); DataOutputViewStreamWrapper ow = new DataOutputViewStreamWrapper(os); try { TestElementSerializer.INSTANCE.serialize(o1, ow); byte[] a1 = os.toByteArray(); os.reset(); TestElementSerializer.INSTANCE.serialize(o2, ow); byte[] a2 = os.toByteArray(); return UnsignedBytes.lexicographicalComparator().compare(a1, a2); } catch (Exception e) { throw new RuntimeException(e); } }
Example 2
Source File: InternalPriorityQueueTestBase.java From flink with Apache License 2.0 | 6 votes |
@Override public int compare(TestElement o1, TestElement o2) { ByteArrayOutputStreamWithPos os = new ByteArrayOutputStreamWithPos(); DataOutputViewStreamWrapper ow = new DataOutputViewStreamWrapper(os); try { TestElementSerializer.INSTANCE.serialize(o1, ow); byte[] a1 = os.toByteArray(); os.reset(); TestElementSerializer.INSTANCE.serialize(o2, ow); byte[] a2 = os.toByteArray(); return UnsignedBytes.lexicographicalComparator().compare(a1, a2); } catch (Exception e) { throw new RuntimeException(e); } }
Example 3
Source File: InternalPriorityQueueTestBase.java From flink with Apache License 2.0 | 6 votes |
@Override public int compare(TestElement o1, TestElement o2) { ByteArrayOutputStreamWithPos os = new ByteArrayOutputStreamWithPos(); DataOutputViewStreamWrapper ow = new DataOutputViewStreamWrapper(os); try { TestElementSerializer.INSTANCE.serialize(o1, ow); byte[] a1 = os.toByteArray(); os.reset(); TestElementSerializer.INSTANCE.serialize(o2, ow); byte[] a2 = os.toByteArray(); return UnsignedBytes.lexicographicalComparator().compare(a1, a2); } catch (Exception e) { throw new RuntimeException(e); } }
Example 4
Source File: SavepointV1SerializerTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
/** * Test serialization of {@link SavepointV1} instance. */ @Test public void testSerializeDeserializeV1() throws Exception { final Random r = new Random(42); for (int i = 0; i < 50; ++i) { SavepointV1 expected = new SavepointV1(i+ 123123, CheckpointTestUtils.createTaskStates(r, 1 + r.nextInt(64), 1 + r.nextInt(64))); SavepointV1Serializer serializer = SavepointV1Serializer.INSTANCE; // Serialize ByteArrayOutputStreamWithPos baos = new ByteArrayOutputStreamWithPos(); serializer.serializeOld(expected, new DataOutputViewStreamWrapper(baos)); byte[] bytes = baos.toByteArray(); // Deserialize ByteArrayInputStream bais = new ByteArrayInputStream(bytes); SavepointV2 actual = serializer.deserialize( new DataInputViewStreamWrapper(bais), Thread.currentThread().getContextClassLoader()); assertEquals(expected.getCheckpointId(), actual.getCheckpointId()); assertEquals(expected.getTaskStates(), actual.getTaskStates()); assertTrue(actual.getMasterStates().isEmpty()); } }
Example 5
Source File: SavepointV2SerializerTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
private void testCheckpointSerialization( long checkpointId, Collection<OperatorState> operatorStates, Collection<MasterState> masterStates) throws IOException { SavepointV2Serializer serializer = SavepointV2Serializer.INSTANCE; ByteArrayOutputStreamWithPos baos = new ByteArrayOutputStreamWithPos(); DataOutputStream out = new DataOutputViewStreamWrapper(baos); serializer.serialize(new SavepointV2(checkpointId, operatorStates, masterStates), out); out.close(); byte[] bytes = baos.toByteArray(); DataInputStream in = new DataInputViewStreamWrapper(new ByteArrayInputStreamWithPos(bytes)); SavepointV2 deserialized = serializer.deserialize(in, getClass().getClassLoader()); assertEquals(checkpointId, deserialized.getCheckpointId()); assertEquals(operatorStates, deserialized.getOperatorStates()); assertEquals(masterStates.size(), deserialized.getMasterStates().size()); for (Iterator<MasterState> a = masterStates.iterator(), b = deserialized.getMasterStates().iterator(); a.hasNext();) { CheckpointTestUtils.assertMasterStateEquality(a.next(), b.next()); } }
Example 6
Source File: SavepointV1SerializerTest.java From flink with Apache License 2.0 | 5 votes |
/** * Test serialization of {@link SavepointV1} instance. */ @Test public void testSerializeDeserializeV1() throws Exception { final Random r = new Random(42); for (int i = 0; i < 50; ++i) { SavepointV1 expected = new SavepointV1(i+ 123123, CheckpointTestUtils.createTaskStates(r, 1 + r.nextInt(64), 1 + r.nextInt(64))); SavepointV1Serializer serializer = SavepointV1Serializer.INSTANCE; // Serialize ByteArrayOutputStreamWithPos baos = new ByteArrayOutputStreamWithPos(); serializer.serializeOld(expected, new DataOutputViewStreamWrapper(baos)); byte[] bytes = baos.toByteArray(); // Deserialize ByteArrayInputStream bais = new ByteArrayInputStream(bytes); SavepointV2 actual = serializer.deserialize( new DataInputViewStreamWrapper(bais), Thread.currentThread().getContextClassLoader()); assertEquals(expected.getCheckpointId(), actual.getCheckpointId()); assertEquals(expected.getTaskStates(), actual.getTaskStates()); assertTrue(actual.getMasterStates().isEmpty()); } }
Example 7
Source File: SavepointV2SerializerTest.java From flink with Apache License 2.0 | 5 votes |
private void testCheckpointSerialization( long checkpointId, Collection<OperatorState> operatorStates, Collection<MasterState> masterStates) throws IOException { SavepointV2Serializer serializer = SavepointV2Serializer.INSTANCE; ByteArrayOutputStreamWithPos baos = new ByteArrayOutputStreamWithPos(); DataOutputStream out = new DataOutputViewStreamWrapper(baos); serializer.serialize(new SavepointV2(checkpointId, operatorStates, masterStates), out); out.close(); byte[] bytes = baos.toByteArray(); DataInputStream in = new DataInputViewStreamWrapper(new ByteArrayInputStreamWithPos(bytes)); SavepointV2 deserialized = serializer.deserialize(in, getClass().getClassLoader()); assertEquals(checkpointId, deserialized.getCheckpointId()); assertEquals(operatorStates, deserialized.getOperatorStates()); assertEquals(masterStates.size(), deserialized.getMasterStates().size()); for (Iterator<MasterState> a = masterStates.iterator(), b = deserialized.getMasterStates().iterator(); a.hasNext();) { CheckpointTestUtils.assertMasterStateEquality(a.next(), b.next()); } }
Example 8
Source File: ValueStateToKeyedStateRow.java From bravo with Apache License 2.0 | 5 votes |
@Override public KeyedStateRow map(Tuple2<K, V> t) throws Exception { int keyGroup = KeyGroupRangeAssignment.assignToKeyGroup(t.f0, maxParallelism); ByteArrayOutputStreamWithPos os = new ByteArrayOutputStreamWithPos(); DataOutputViewStreamWrapper ov = new DataOutputViewStreamWrapper(os); RocksDBUtils.writeKeyGroup(keyGroup, keygroupPrefixBytes, ov); RocksDBUtils.writeKey(t.f0, keySerializer, os, ov, false); RocksDBUtils.writeNameSpace(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, os, ov, false); os.close(); return new KeyedStateRow(stateName, os.toByteArray(), InstantiationUtil.serializeToByteArray(valueSerializer, t.f1)); }
Example 9
Source File: MetadataV3SerializerTest.java From flink with Apache License 2.0 | 4 votes |
/** * Test checkpoint metadata (de)serialization. * * @param checkpointId The given checkpointId will write into the metadata. * @param operatorStates the given states for all the operators. * @param masterStates the masterStates of the given checkpoint/savepoint. */ private void testCheckpointSerialization( long checkpointId, Collection<OperatorState> operatorStates, Collection<MasterState> masterStates, @Nullable String basePath) throws IOException { MetadataV3Serializer serializer = MetadataV3Serializer.INSTANCE; ByteArrayOutputStreamWithPos baos = new ByteArrayOutputStreamWithPos(); DataOutputStream out = new DataOutputViewStreamWrapper(baos); CheckpointMetadata metadata = new CheckpointMetadata(checkpointId, operatorStates, masterStates); MetadataV3Serializer.serialize(metadata, out); out.close(); // The relative pointer resolution in MetadataV2V3SerializerBase currently runs the same // code as the file system checkpoint location resolution. Because of that, it needs the // a "_metadata" file present. we could change the code to resolve the pointer without doing // file I/O, but it is somewhat delicate to reproduce that logic without I/O and the same guarantees // to differentiate between the supported options of directory addressing and metadata file addressing. // So, better safe than sorry, we do actually do the file system operations in the serializer for now, // even if it makes the tests a a tad bit more clumsy if (basePath != null) { final Path metaPath = new Path(basePath, "_metadata"); // this is in the temp folder, so it will get automatically deleted FileSystem.getLocalFileSystem().create(metaPath, FileSystem.WriteMode.OVERWRITE).close(); } byte[] bytes = baos.toByteArray(); DataInputStream in = new DataInputViewStreamWrapper(new ByteArrayInputStreamWithPos(bytes)); CheckpointMetadata deserialized = serializer.deserialize(in, getClass().getClassLoader(), basePath); assertEquals(checkpointId, deserialized.getCheckpointId()); assertEquals(operatorStates, deserialized.getOperatorStates()); assertEquals(masterStates.size(), deserialized.getMasterStates().size()); for (Iterator<MasterState> a = masterStates.iterator(), b = deserialized.getMasterStates().iterator(); a.hasNext();) { CheckpointTestUtils.assertMasterStateEquality(a.next(), b.next()); } }