org.apache.flink.core.io.SimpleVersionedSerialization Java Examples

The following examples show how to use org.apache.flink.core.io.SimpleVersionedSerialization. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: SourceOperatorTest.java From flink with Apache License 2.0

6 votes

private StateInitializationContext getStateContext() throws Exception {
	// Create a mock split.
	byte[] serializedSplitWithVersion = SimpleVersionedSerialization
			.writeVersionAndSerialize(new MockSourceSplitSerializer(), MOCK_SPLIT);

	// Crate the state context.
	OperatorStateStore operatorStateStore = createOperatorStateStore();
	StateInitializationContext stateContext = new StateInitializationContextImpl(
			false,
			operatorStateStore,
			null,
			null,
			null);

	// Update the context.
	stateContext.getOperatorStateStore()
				.getListState(SourceOperator.SPLITS_STATE_DESC)
				.update(Collections.singletonList(serializedSplitWithVersion));

	return stateContext;
}

Example #2

Source File: Buckets.java From Flink-CEPplus with Apache License 2.0

6 votes

private void snapshotActiveBuckets(
		final long checkpointId,
		final ListState<byte[]> bucketStatesContainer) throws Exception {

	for (Bucket<IN, BucketID> bucket : activeBuckets.values()) {
		final BucketState<BucketID> bucketState = bucket.onReceptionOfCheckpoint(checkpointId);

		final byte[] serializedBucketState = SimpleVersionedSerialization
				.writeVersionAndSerialize(bucketStateSerializer, bucketState);

		bucketStatesContainer.add(serializedBucketState);

		if (LOG.isDebugEnabled()) {
			LOG.debug("Subtask {} checkpointing: {}", subtaskIndex, bucketState);
		}
	}
}

Example #3

Source File: BucketStateSerializerTest.java From Flink-CEPplus with Apache License 2.0

6 votes

@Test
public void testSerializationEmpty() throws IOException {
	final File testFolder = tempFolder.newFolder();
	final FileSystem fs = FileSystem.get(testFolder.toURI());
	final RecoverableWriter writer = fs.createRecoverableWriter();

	final Path testBucket = new Path(testFolder.getPath(), "test");

	final BucketState<String> bucketState = new BucketState<>(
			"test", testBucket, Long.MAX_VALUE, null, new HashMap<>());

	final SimpleVersionedSerializer<BucketState<String>> serializer =
			new BucketStateSerializer<>(
					writer.getResumeRecoverableSerializer(),
					writer.getCommitRecoverableSerializer(),
					SimpleVersionedStringSerializer.INSTANCE
			);

	byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(serializer, bucketState);
	final BucketState<String> recoveredState =  SimpleVersionedSerialization.readVersionAndDeSerialize(serializer, bytes);

	Assert.assertEquals(testBucket, recoveredState.getBucketPath());
	Assert.assertNull(recoveredState.getInProgressResumableFile());
	Assert.assertTrue(recoveredState.getCommittableFilesPerCheckpoint().isEmpty());
}

Example #4

Source File: BucketStateSerializerTest.java From flink with Apache License 2.0

6 votes

@Test
@Ignore
public void prepareDeserializationEmpty() throws IOException {

	final String scenarioName = "empty";
	final java.nio.file.Path scenarioPath = getResourcePath(scenarioName, CURRENT_VERSION);

	FileUtils.deleteDirectory(scenarioPath.toFile());
	Files.createDirectories(scenarioPath);

	final java.nio.file.Path outputPath = getOutputPath(scenarioName, CURRENT_VERSION);
	final Path testBucketPath = new Path(outputPath.resolve(BUCKET_ID).toString());

	final Bucket<String, String> bucket =
		createNewBucket(testBucketPath);

	final BucketState<String> bucketState = bucket.onReceptionOfCheckpoint(0);

	byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(
		bucketStateSerializer(),
		bucketState);
	Files.write(getSnapshotPath(scenarioName, CURRENT_VERSION), bytes);
}

Example #5

Source File: BucketStateSerializerTest.java From flink with Apache License 2.0

6 votes

@Test
@Ignore
public void prepareDeserializationOnlyInProgress() throws IOException {

	final String scenarioName = "only-in-progress";
	final java.nio.file.Path scenarioPath = getResourcePath(scenarioName, CURRENT_VERSION);
	FileUtils.deleteDirectory(scenarioPath.toFile());
	Files.createDirectories(scenarioPath);

	final java.nio.file.Path outputPath = getOutputPath(scenarioName, CURRENT_VERSION);
	final Path testBucketPath = new Path(outputPath.resolve(BUCKET_ID).toString());

	final Bucket<String, String> bucket =
		createNewBucket(testBucketPath);

	bucket.write(IN_PROGRESS_CONTENT, System.currentTimeMillis());

	final BucketState<String> bucketState = bucket.onReceptionOfCheckpoint(0);

	final byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(
		bucketStateSerializer(), bucketState);

	Files.write(getSnapshotPath(scenarioName, CURRENT_VERSION), bytes);
}

Example #6

Source File: Buckets.java From flink with Apache License 2.0

6 votes

private void snapshotActiveBuckets(
		final long checkpointId,
		final ListState<byte[]> bucketStatesContainer) throws Exception {

	for (Bucket<IN, BucketID> bucket : activeBuckets.values()) {
		final BucketState<BucketID> bucketState = bucket.onReceptionOfCheckpoint(checkpointId);

		final byte[] serializedBucketState = SimpleVersionedSerialization
				.writeVersionAndSerialize(bucketStateSerializer, bucketState);

		bucketStatesContainer.add(serializedBucketState);

		if (LOG.isDebugEnabled()) {
			LOG.debug("Subtask {} checkpointing: {}", subtaskIndex, bucketState);
		}
	}
}

Example #7

Source File: Buckets.java From flink with Apache License 2.0

6 votes

private void snapshotActiveBuckets(
		final long checkpointId,
		final ListState<byte[]> bucketStatesContainer) throws Exception {

	for (Bucket<IN, BucketID> bucket : activeBuckets.values()) {
		final BucketState<BucketID> bucketState = bucket.onReceptionOfCheckpoint(checkpointId);

		final byte[] serializedBucketState = SimpleVersionedSerialization
				.writeVersionAndSerialize(bucketStateSerializer, bucketState);

		bucketStatesContainer.add(serializedBucketState);

		if (LOG.isDebugEnabled()) {
			LOG.debug("Subtask {} checkpointing: {}", subtaskIndex, bucketState);
		}
	}
}

Example #8

Source File: BucketStateSerializerTest.java From flink with Apache License 2.0

6 votes

@Test
public void testSerializationEmpty() throws IOException {
	final File testFolder = tempFolder.newFolder();
	final FileSystem fs = FileSystem.get(testFolder.toURI());
	final RecoverableWriter writer = fs.createRecoverableWriter();

	final Path testBucket = new Path(testFolder.getPath(), "test");

	final BucketState<String> bucketState = new BucketState<>(
			"test", testBucket, Long.MAX_VALUE, null, new HashMap<>());

	final SimpleVersionedSerializer<BucketState<String>> serializer =
			new BucketStateSerializer<>(
					writer.getResumeRecoverableSerializer(),
					writer.getCommitRecoverableSerializer(),
					SimpleVersionedStringSerializer.INSTANCE
			);

	byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(serializer, bucketState);
	final BucketState<String> recoveredState =  SimpleVersionedSerialization.readVersionAndDeSerialize(serializer, bytes);

	Assert.assertEquals(testBucket, recoveredState.getBucketPath());
	Assert.assertNull(recoveredState.getInProgressResumableFile());
	Assert.assertTrue(recoveredState.getCommittableFilesPerCheckpoint().isEmpty());
}

Example #9

Source File: BucketStateSerializer.java From Flink-CEPplus with Apache License 2.0

5 votes

@VisibleForTesting
BucketState<BucketID> deserializeV1(DataInputView in) throws IOException {
	final BucketID bucketId = SimpleVersionedSerialization.readVersionAndDeSerialize(bucketIdSerializer, in);
	final String bucketPathStr = in.readUTF();
	final long creationTime = in.readLong();

	// then get the current resumable stream
	RecoverableWriter.ResumeRecoverable current = null;
	if (in.readBoolean()) {
		current = SimpleVersionedSerialization.readVersionAndDeSerialize(resumableSerializer, in);
	}

	final int committableVersion = in.readInt();
	final int numCheckpoints = in.readInt();
	final HashMap<Long, List<RecoverableWriter.CommitRecoverable>> resumablesPerCheckpoint = new HashMap<>(numCheckpoints);

	for (int i = 0; i < numCheckpoints; i++) {
		final long checkpointId = in.readLong();
		final int noOfResumables = in.readInt();

		final List<RecoverableWriter.CommitRecoverable> resumables = new ArrayList<>(noOfResumables);
		for (int j = 0; j < noOfResumables; j++) {
			final byte[] bytes = new byte[in.readInt()];
			in.readFully(bytes);
			resumables.add(commitableSerializer.deserialize(committableVersion, bytes));
		}
		resumablesPerCheckpoint.put(checkpointId, resumables);
	}

	return new BucketState<>(
			bucketId,
			new Path(bucketPathStr),
			creationTime,
			current,
			resumablesPerCheckpoint);
}

Example #10

Source File: Buckets.java From flink with Apache License 2.0

5 votes

private void initializeActiveBuckets(final ListState<byte[]> bucketStates) throws Exception {
	for (byte[] serializedRecoveredState : bucketStates.get()) {
		final BucketState<BucketID> recoveredState =
				SimpleVersionedSerialization.readVersionAndDeSerialize(
						bucketStateSerializer, serializedRecoveredState);
		handleRestoredBucketState(recoveredState);
	}
}

Example #11

Source File: BucketStateSerializer.java From flink with Apache License 2.0

5 votes

private BucketState<BucketID> deserializeV2(DataInputView dataInputView) throws IOException {
	final BucketID bucketId = SimpleVersionedSerialization.readVersionAndDeSerialize(bucketIdSerializer, dataInputView);
	final String bucketPathStr = dataInputView.readUTF();
	final long creationTime = dataInputView.readLong();

	// then get the current resumable stream
	InProgressFileWriter.InProgressFileRecoverable current = null;
	if (dataInputView.readBoolean()) {
		current = SimpleVersionedSerialization.readVersionAndDeSerialize(inProgressFileRecoverableSerializer, dataInputView);
	}

	final int pendingFileRecoverableSerializerVersion = dataInputView.readInt();
	final int numCheckpoints = dataInputView.readInt();
	final HashMap<Long, List<InProgressFileWriter.PendingFileRecoverable>> pendingFileRecoverablesPerCheckpoint = new HashMap<>(numCheckpoints);

	for (int i = 0; i < numCheckpoints; i++) {
		final long checkpointId = dataInputView.readLong();
		final int numOfPendingFileRecoverables = dataInputView.readInt();

		final List<InProgressFileWriter.PendingFileRecoverable> pendingFileRecoverables = new ArrayList<>(numOfPendingFileRecoverables);
		for (int j = 0; j < numOfPendingFileRecoverables; j++) {
			final byte[] bytes = new byte[dataInputView.readInt()];
			dataInputView.readFully(bytes);
			pendingFileRecoverables.add(pendingFileRecoverableSerializer.deserialize(pendingFileRecoverableSerializerVersion, bytes));
		}
		pendingFileRecoverablesPerCheckpoint.put(checkpointId, pendingFileRecoverables);
	}

	return new BucketState<>(bucketId, new Path(bucketPathStr), creationTime, current, pendingFileRecoverablesPerCheckpoint);
}

Example #12

Source File: BucketStateSerializerTest.java From flink with Apache License 2.0

5 votes

private static void prepareDeserializationFull(final boolean withInProgress, final String scenarioName) throws IOException {

		final java.nio.file.Path scenarioPath = getResourcePath(scenarioName, CURRENT_VERSION);
		FileUtils.deleteDirectory(Paths.get(scenarioPath.toString() + "-template").toFile());
		Files.createDirectories(scenarioPath);

		final int noOfPendingCheckpoints = 5;

		final java.nio.file.Path outputPath = getOutputPath(scenarioName, CURRENT_VERSION);

		final Path testBucketPath = new Path(outputPath.resolve(BUCKET_ID).toString());

		final Bucket<String, String> bucket = createNewBucket(testBucketPath);

		BucketState<String> bucketState = null;
		// pending for checkpoints
		for (int i = 0; i < noOfPendingCheckpoints; i++) {
			// write 10 bytes to the in progress file
			bucket.write(PENDING_CONTENT, System.currentTimeMillis());
			bucket.write(PENDING_CONTENT, System.currentTimeMillis());
			// every checkpoint would produce a pending file
			bucketState = bucket.onReceptionOfCheckpoint(i);
		}

		if (withInProgress) {
			// create a in progress file
			bucket.write(IN_PROGRESS_CONTENT, System.currentTimeMillis());

			// 5 pending files and 1 in progress file
			bucketState = bucket.onReceptionOfCheckpoint(noOfPendingCheckpoints);
		}

		final byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(bucketStateSerializer(), bucketState);

		Files.write(getSnapshotPath(scenarioName, CURRENT_VERSION), bytes);

		// copy the scenario file to a template directory.
		// it is because that the test `testSerializationFull` would change the in progress file to pending files.
		moveToTemplateDirectory(scenarioPath);
	}

Example #13

Source File: BucketStateSerializer.java From flink with Apache License 2.0

5 votes

private void serializeV2(BucketState<BucketID> state, DataOutputView dataOutputView) throws IOException {
	SimpleVersionedSerialization.writeVersionAndSerialize(bucketIdSerializer, state.getBucketId(), dataOutputView);
	dataOutputView.writeUTF(state.getBucketPath().toString());
	dataOutputView.writeLong(state.getInProgressFileCreationTime());

	// put the current open part file
	if (state.hasInProgressFileRecoverable()) {
		final InProgressFileWriter.InProgressFileRecoverable inProgressFileRecoverable = state.getInProgressFileRecoverable();
		dataOutputView.writeBoolean(true);
		SimpleVersionedSerialization.writeVersionAndSerialize(inProgressFileRecoverableSerializer, inProgressFileRecoverable, dataOutputView);
	} else {
		dataOutputView.writeBoolean(false);
	}

	// put the map of pending files per checkpoint
	final Map<Long, List<InProgressFileWriter.PendingFileRecoverable>> pendingFileRecoverables = state.getPendingFileRecoverablesPerCheckpoint();

	dataOutputView.writeInt(pendingFileRecoverableSerializer.getVersion());

	dataOutputView.writeInt(pendingFileRecoverables.size());

	for (Entry<Long, List<InProgressFileWriter.PendingFileRecoverable>> pendingFilesForCheckpoint : pendingFileRecoverables.entrySet()) {
		final List<InProgressFileWriter.PendingFileRecoverable> pendingFileRecoverableList = pendingFilesForCheckpoint.getValue();

		dataOutputView.writeLong(pendingFilesForCheckpoint.getKey());
		dataOutputView.writeInt(pendingFileRecoverableList.size());

		for (InProgressFileWriter.PendingFileRecoverable pendingFileRecoverable : pendingFileRecoverableList) {
			byte[] serialized = pendingFileRecoverableSerializer.serialize(pendingFileRecoverable);
			dataOutputView.writeInt(serialized.length);
			dataOutputView.write(serialized);
		}
	}
}

Example #14

Source File: SimpleVersionedListState.java From flink with Apache License 2.0

5 votes

@Override
public T next() {
	final byte[] bytes = rawIterator.next();
	try {
		return SimpleVersionedSerialization.readVersionAndDeSerialize(serializer, bytes);
	}
	catch (IOException e) {
		throw new FlinkRuntimeException("Failed to deserialize value", e);
	}
}

Example #15

Source File: BucketStateSerializerTest.java From flink with Apache License 2.0

5 votes

@Test
public void testSerializationOnlyInProgress() throws IOException {
	final File testFolder = tempFolder.newFolder();
	final FileSystem fs = FileSystem.get(testFolder.toURI());

	final Path testBucket = new Path(testFolder.getPath(), "test");

	final RecoverableWriter writer = fs.createRecoverableWriter();
	final RecoverableFsDataOutputStream stream = writer.open(testBucket);
	stream.write(IN_PROGRESS_CONTENT.getBytes(Charset.forName("UTF-8")));

	final RecoverableWriter.ResumeRecoverable current = stream.persist();

	final BucketState<String> bucketState = new BucketState<>(
			"test", testBucket, Long.MAX_VALUE, current, new HashMap<>());

	final SimpleVersionedSerializer<BucketState<String>> serializer =
			new BucketStateSerializer<>(
					writer.getResumeRecoverableSerializer(),
					writer.getCommitRecoverableSerializer(),
					SimpleVersionedStringSerializer.INSTANCE
			);

	final byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(serializer, bucketState);

	// to simulate that everything is over for file.
	stream.close();

	final BucketState<String> recoveredState =  SimpleVersionedSerialization.readVersionAndDeSerialize(serializer, bytes);

	Assert.assertEquals(testBucket, recoveredState.getBucketPath());

	FileStatus[] statuses = fs.listStatus(testBucket.getParent());
	Assert.assertEquals(1L, statuses.length);
	Assert.assertTrue(
			statuses[0].getPath().getPath().startsWith(
					(new Path(testBucket.getParent(), ".test.inprogress")).getPath())
	);
}

Example #16

Source File: Buckets.java From flink with Apache License 2.0

5 votes

private void initializeActiveBuckets(final ListState<byte[]> bucketStates) throws Exception {
	for (byte[] serializedRecoveredState : bucketStates.get()) {
		final BucketState<BucketID> recoveredState =
				SimpleVersionedSerialization.readVersionAndDeSerialize(
						bucketStateSerializer, serializedRecoveredState);
		handleRestoredBucketState(recoveredState);
	}
}

Example #17

Source File: BucketStateSerializer.java From flink with Apache License 2.0

5 votes

@VisibleForTesting
BucketState<BucketID> deserializeV1(DataInputView in) throws IOException {
	final BucketID bucketId = SimpleVersionedSerialization.readVersionAndDeSerialize(bucketIdSerializer, in);
	final String bucketPathStr = in.readUTF();
	final long creationTime = in.readLong();

	// then get the current resumable stream
	RecoverableWriter.ResumeRecoverable current = null;
	if (in.readBoolean()) {
		current = SimpleVersionedSerialization.readVersionAndDeSerialize(resumableSerializer, in);
	}

	final int committableVersion = in.readInt();
	final int numCheckpoints = in.readInt();
	final HashMap<Long, List<RecoverableWriter.CommitRecoverable>> resumablesPerCheckpoint = new HashMap<>(numCheckpoints);

	for (int i = 0; i < numCheckpoints; i++) {
		final long checkpointId = in.readLong();
		final int noOfResumables = in.readInt();

		final List<RecoverableWriter.CommitRecoverable> resumables = new ArrayList<>(noOfResumables);
		for (int j = 0; j < noOfResumables; j++) {
			final byte[] bytes = new byte[in.readInt()];
			in.readFully(bytes);
			resumables.add(commitableSerializer.deserialize(committableVersion, bytes));
		}
		resumablesPerCheckpoint.put(checkpointId, resumables);
	}

	return new BucketState<>(
			bucketId,
			new Path(bucketPathStr),
			creationTime,
			current,
			resumablesPerCheckpoint);
}

Example #18

Source File: BucketStateSerializerTest.java From Flink-CEPplus with Apache License 2.0

5 votes

@Test
public void testSerializationOnlyInProgress() throws IOException {
	final File testFolder = tempFolder.newFolder();
	final FileSystem fs = FileSystem.get(testFolder.toURI());

	final Path testBucket = new Path(testFolder.getPath(), "test");

	final RecoverableWriter writer = fs.createRecoverableWriter();
	final RecoverableFsDataOutputStream stream = writer.open(testBucket);
	stream.write(IN_PROGRESS_CONTENT.getBytes(Charset.forName("UTF-8")));

	final RecoverableWriter.ResumeRecoverable current = stream.persist();

	final BucketState<String> bucketState = new BucketState<>(
			"test", testBucket, Long.MAX_VALUE, current, new HashMap<>());

	final SimpleVersionedSerializer<BucketState<String>> serializer =
			new BucketStateSerializer<>(
					writer.getResumeRecoverableSerializer(),
					writer.getCommitRecoverableSerializer(),
					SimpleVersionedStringSerializer.INSTANCE
			);

	final byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(serializer, bucketState);

	// to simulate that everything is over for file.
	stream.close();

	final BucketState<String> recoveredState =  SimpleVersionedSerialization.readVersionAndDeSerialize(serializer, bytes);

	Assert.assertEquals(testBucket, recoveredState.getBucketPath());

	FileStatus[] statuses = fs.listStatus(testBucket.getParent());
	Assert.assertEquals(1L, statuses.length);
	Assert.assertTrue(
			statuses[0].getPath().getPath().startsWith(
					(new Path(testBucket.getParent(), ".test.inprogress")).toString())
	);
}

Example #19

Source File: Buckets.java From Flink-CEPplus with Apache License 2.0

5 votes

private void initializeActiveBuckets(final ListState<byte[]> bucketStates) throws Exception {
	for (byte[] serializedRecoveredState : bucketStates.get()) {
		final BucketState<BucketID> recoveredState =
				SimpleVersionedSerialization.readVersionAndDeSerialize(
						bucketStateSerializer, serializedRecoveredState);
		handleRestoredBucketState(recoveredState);
	}
}

Example #20

Source File: BucketStateSerializerTest.java From flink with Apache License 2.0

4 votes

private static BucketState<String> readBucketState(final String scenarioName, final int version) throws IOException {
	byte[] bytes = Files.readAllBytes(getSnapshotPath(scenarioName, version));
	return SimpleVersionedSerialization.readVersionAndDeSerialize(bucketStateSerializer(), bytes);
}

Example #21

Source File: OutputStreamBasedPartFileWriter.java From flink with Apache License 2.0

4 votes

private OutputStreamBasedPendingFileRecoverable deserializeV1(final DataInputView dataInputView) throws IOException {
	return new OutputStreamBasedPendingFileRecoverable(SimpleVersionedSerialization.readVersionAndDeSerialize(commitSerializer, dataInputView));
}

Example #22

Source File: OutputStreamBasedPartFileWriter.java From flink with Apache License 2.0

4 votes

private void serializeV1(final OutputStreamBasedPendingFileRecoverable outputStreamBasedPendingFileRecoverable, final DataOutputView dataOutputView) throws IOException {
	SimpleVersionedSerialization.writeVersionAndSerialize(commitSerializer, outputStreamBasedPendingFileRecoverable.getCommitRecoverable(), dataOutputView);
}

Example #23

Source File: OutputStreamBasedPartFileWriter.java From flink with Apache License 2.0

4 votes

private OutputStreamBasedInProgressFileRecoverable deserializeV1(final DataInputView dataInputView) throws IOException {
	return new OutputStreamBasedInProgressFileRecoverable(SimpleVersionedSerialization.readVersionAndDeSerialize(resumeSerializer, dataInputView));
}

Example #24

Source File: OutputStreamBasedPartFileWriter.java From flink with Apache License 2.0

4 votes

private void serializeV1(final OutputStreamBasedInProgressFileRecoverable outputStreamBasedInProgressRecoverable, final DataOutputView dataOutputView) throws IOException {
	SimpleVersionedSerialization.writeVersionAndSerialize(resumeSerializer, outputStreamBasedInProgressRecoverable.getResumeRecoverable(), dataOutputView);
}

Example #25

Source File: BucketStateSerializer.java From flink with Apache License 2.0

4 votes

private BucketState<BucketID> deserializeV1(DataInputView in) throws IOException {

		final SimpleVersionedSerializer<RecoverableWriter.CommitRecoverable> commitableSerializer = getCommitableSerializer();
		final SimpleVersionedSerializer<RecoverableWriter.ResumeRecoverable> resumableSerializer = getResumableSerializer();

		final BucketID bucketId = SimpleVersionedSerialization.readVersionAndDeSerialize(bucketIdSerializer, in);
		final String bucketPathStr = in.readUTF();
		final long creationTime = in.readLong();

		// then get the current resumable stream
		InProgressFileWriter.InProgressFileRecoverable current = null;
		if (in.readBoolean()) {
			current =
				new OutputStreamBasedPartFileWriter.OutputStreamBasedInProgressFileRecoverable(
					SimpleVersionedSerialization.readVersionAndDeSerialize(resumableSerializer, in));
		}

		final int committableVersion = in.readInt();
		final int numCheckpoints = in.readInt();
		final HashMap<Long, List<InProgressFileWriter.PendingFileRecoverable>> pendingFileRecoverablePerCheckpoint = new HashMap<>(numCheckpoints);

		for (int i = 0; i < numCheckpoints; i++) {
			final long checkpointId = in.readLong();
			final int noOfResumables = in.readInt();

			final List<InProgressFileWriter.PendingFileRecoverable> pendingFileRecoverables = new ArrayList<>(noOfResumables);
			for (int j = 0; j < noOfResumables; j++) {
				final byte[] bytes = new byte[in.readInt()];
				in.readFully(bytes);
				pendingFileRecoverables.add(
					new OutputStreamBasedPartFileWriter.OutputStreamBasedPendingFileRecoverable(commitableSerializer.deserialize(committableVersion, bytes)));
			}
			pendingFileRecoverablePerCheckpoint.put(checkpointId, pendingFileRecoverables);
		}

		return new BucketState<>(
			bucketId,
			new Path(bucketPathStr),
			creationTime,
			current,
			pendingFileRecoverablePerCheckpoint);
	}

Example #26

Source File: SimpleVersionedListState.java From flink with Apache License 2.0

4 votes

private byte[] serialize(T value) throws IOException {
	return SimpleVersionedSerialization.writeVersionAndSerialize(serializer, value);
}