org.apache.flink.core.io.SimpleVersionedSerialization Java Examples

private StateInitializationContext getStateContext() throws Exception {
	// Create a mock split.
	byte[] serializedSplitWithVersion = SimpleVersionedSerialization
			.writeVersionAndSerialize(new MockSourceSplitSerializer(), MOCK_SPLIT);

	// Crate the state context.
	OperatorStateStore operatorStateStore = createOperatorStateStore();
	StateInitializationContext stateContext = new StateInitializationContextImpl(
			false,
			operatorStateStore,
			null,
			null,
			null);

	// Update the context.
	stateContext.getOperatorStateStore()
				.getListState(SourceOperator.SPLITS_STATE_DESC)
				.update(Collections.singletonList(serializedSplitWithVersion));

	return stateContext;
}
 

private void snapshotActiveBuckets(
		final long checkpointId,
		final ListState<byte[]> bucketStatesContainer) throws Exception {

	for (Bucket<IN, BucketID> bucket : activeBuckets.values()) {
		final BucketState<BucketID> bucketState = bucket.onReceptionOfCheckpoint(checkpointId);

		final byte[] serializedBucketState = SimpleVersionedSerialization
				.writeVersionAndSerialize(bucketStateSerializer, bucketState);

		bucketStatesContainer.add(serializedBucketState);

		if (LOG.isDebugEnabled()) {
			LOG.debug("Subtask {} checkpointing: {}", subtaskIndex, bucketState);
		}
	}
}
 

@Test
public void testSerializationEmpty() throws IOException {
	final File testFolder = tempFolder.newFolder();
	final FileSystem fs = FileSystem.get(testFolder.toURI());
	final RecoverableWriter writer = fs.createRecoverableWriter();

	final Path testBucket = new Path(testFolder.getPath(), "test");

	final BucketState<String> bucketState = new BucketState<>(
			"test", testBucket, Long.MAX_VALUE, null, new HashMap<>());

	final SimpleVersionedSerializer<BucketState<String>> serializer =
			new BucketStateSerializer<>(
					writer.getResumeRecoverableSerializer(),
					writer.getCommitRecoverableSerializer(),
					SimpleVersionedStringSerializer.INSTANCE
			);

	byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(serializer, bucketState);
	final BucketState<String> recoveredState =  SimpleVersionedSerialization.readVersionAndDeSerialize(serializer, bytes);

	Assert.assertEquals(testBucket, recoveredState.getBucketPath());
	Assert.assertNull(recoveredState.getInProgressResumableFile());
	Assert.assertTrue(recoveredState.getCommittableFilesPerCheckpoint().isEmpty());
}
 

@Test
@Ignore
public void prepareDeserializationEmpty() throws IOException {

	final String scenarioName = "empty";
	final java.nio.file.Path scenarioPath = getResourcePath(scenarioName, CURRENT_VERSION);

	FileUtils.deleteDirectory(scenarioPath.toFile());
	Files.createDirectories(scenarioPath);

	final java.nio.file.Path outputPath = getOutputPath(scenarioName, CURRENT_VERSION);
	final Path testBucketPath = new Path(outputPath.resolve(BUCKET_ID).toString());

	final Bucket<String, String> bucket =
		createNewBucket(testBucketPath);

	final BucketState<String> bucketState = bucket.onReceptionOfCheckpoint(0);

	byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(
		bucketStateSerializer(),
		bucketState);
	Files.write(getSnapshotPath(scenarioName, CURRENT_VERSION), bytes);
}
 

@Test
@Ignore
public void prepareDeserializationOnlyInProgress() throws IOException {

	final String scenarioName = "only-in-progress";
	final java.nio.file.Path scenarioPath = getResourcePath(scenarioName, CURRENT_VERSION);
	FileUtils.deleteDirectory(scenarioPath.toFile());
	Files.createDirectories(scenarioPath);

	final java.nio.file.Path outputPath = getOutputPath(scenarioName, CURRENT_VERSION);
	final Path testBucketPath = new Path(outputPath.resolve(BUCKET_ID).toString());

	final Bucket<String, String> bucket =
		createNewBucket(testBucketPath);

	bucket.write(IN_PROGRESS_CONTENT, System.currentTimeMillis());

	final BucketState<String> bucketState = bucket.onReceptionOfCheckpoint(0);

	final byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(
		bucketStateSerializer(), bucketState);

	Files.write(getSnapshotPath(scenarioName, CURRENT_VERSION), bytes);
}
 

private void snapshotActiveBuckets(
		final long checkpointId,
		final ListState<byte[]> bucketStatesContainer) throws Exception {

	for (Bucket<IN, BucketID> bucket : activeBuckets.values()) {
		final BucketState<BucketID> bucketState = bucket.onReceptionOfCheckpoint(checkpointId);

		final byte[] serializedBucketState = SimpleVersionedSerialization
				.writeVersionAndSerialize(bucketStateSerializer, bucketState);

		bucketStatesContainer.add(serializedBucketState);

		if (LOG.isDebugEnabled()) {
			LOG.debug("Subtask {} checkpointing: {}", subtaskIndex, bucketState);
		}
	}
}
 

private void snapshotActiveBuckets(
		final long checkpointId,
		final ListState<byte[]> bucketStatesContainer) throws Exception {

	for (Bucket<IN, BucketID> bucket : activeBuckets.values()) {
		final BucketState<BucketID> bucketState = bucket.onReceptionOfCheckpoint(checkpointId);

		final byte[] serializedBucketState = SimpleVersionedSerialization
				.writeVersionAndSerialize(bucketStateSerializer, bucketState);

		bucketStatesContainer.add(serializedBucketState);

		if (LOG.isDebugEnabled()) {
			LOG.debug("Subtask {} checkpointing: {}", subtaskIndex, bucketState);
		}
	}
}
 

@Test
public void testSerializationEmpty() throws IOException {
	final File testFolder = tempFolder.newFolder();
	final FileSystem fs = FileSystem.get(testFolder.toURI());
	final RecoverableWriter writer = fs.createRecoverableWriter();

	final Path testBucket = new Path(testFolder.getPath(), "test");

	final BucketState<String> bucketState = new BucketState<>(
			"test", testBucket, Long.MAX_VALUE, null, new HashMap<>());

	final SimpleVersionedSerializer<BucketState<String>> serializer =
			new BucketStateSerializer<>(
					writer.getResumeRecoverableSerializer(),
					writer.getCommitRecoverableSerializer(),
					SimpleVersionedStringSerializer.INSTANCE
			);

	byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(serializer, bucketState);
	final BucketState<String> recoveredState =  SimpleVersionedSerialization.readVersionAndDeSerialize(serializer, bytes);

	Assert.assertEquals(testBucket, recoveredState.getBucketPath());
	Assert.assertNull(recoveredState.getInProgressResumableFile());
	Assert.assertTrue(recoveredState.getCommittableFilesPerCheckpoint().isEmpty());
}
 

@VisibleForTesting
BucketState<BucketID> deserializeV1(DataInputView in) throws IOException {
	final BucketID bucketId = SimpleVersionedSerialization.readVersionAndDeSerialize(bucketIdSerializer, in);
	final String bucketPathStr = in.readUTF();
	final long creationTime = in.readLong();

	// then get the current resumable stream
	RecoverableWriter.ResumeRecoverable current = null;
	if (in.readBoolean()) {
		current = SimpleVersionedSerialization.readVersionAndDeSerialize(resumableSerializer, in);
	}

	final int committableVersion = in.readInt();
	final int numCheckpoints = in.readInt();
	final HashMap<Long, List<RecoverableWriter.CommitRecoverable>> resumablesPerCheckpoint = new HashMap<>(numCheckpoints);

	for (int i = 0; i < numCheckpoints; i++) {
		final long checkpointId = in.readLong();
		final int noOfResumables = in.readInt();

		final List<RecoverableWriter.CommitRecoverable> resumables = new ArrayList<>(noOfResumables);
		for (int j = 0; j < noOfResumables; j++) {
			final byte[] bytes = new byte[in.readInt()];
			in.readFully(bytes);
			resumables.add(commitableSerializer.deserialize(committableVersion, bytes));
		}
		resumablesPerCheckpoint.put(checkpointId, resumables);
	}

	return new BucketState<>(
			bucketId,
			new Path(bucketPathStr),
			creationTime,
			current,
			resumablesPerCheckpoint);
}
 

private void initializeActiveBuckets(final ListState<byte[]> bucketStates) throws Exception {
	for (byte[] serializedRecoveredState : bucketStates.get()) {
		final BucketState<BucketID> recoveredState =
				SimpleVersionedSerialization.readVersionAndDeSerialize(
						bucketStateSerializer, serializedRecoveredState);
		handleRestoredBucketState(recoveredState);
	}
}
 

private BucketState<BucketID> deserializeV2(DataInputView dataInputView) throws IOException {
	final BucketID bucketId = SimpleVersionedSerialization.readVersionAndDeSerialize(bucketIdSerializer, dataInputView);
	final String bucketPathStr = dataInputView.readUTF();
	final long creationTime = dataInputView.readLong();

	// then get the current resumable stream
	InProgressFileWriter.InProgressFileRecoverable current = null;
	if (dataInputView.readBoolean()) {
		current = SimpleVersionedSerialization.readVersionAndDeSerialize(inProgressFileRecoverableSerializer, dataInputView);
	}

	final int pendingFileRecoverableSerializerVersion = dataInputView.readInt();
	final int numCheckpoints = dataInputView.readInt();
	final HashMap<Long, List<InProgressFileWriter.PendingFileRecoverable>> pendingFileRecoverablesPerCheckpoint = new HashMap<>(numCheckpoints);

	for (int i = 0; i < numCheckpoints; i++) {
		final long checkpointId = dataInputView.readLong();
		final int numOfPendingFileRecoverables = dataInputView.readInt();

		final List<InProgressFileWriter.PendingFileRecoverable> pendingFileRecoverables = new ArrayList<>(numOfPendingFileRecoverables);
		for (int j = 0; j < numOfPendingFileRecoverables; j++) {
			final byte[] bytes = new byte[dataInputView.readInt()];
			dataInputView.readFully(bytes);
			pendingFileRecoverables.add(pendingFileRecoverableSerializer.deserialize(pendingFileRecoverableSerializerVersion, bytes));
		}
		pendingFileRecoverablesPerCheckpoint.put(checkpointId, pendingFileRecoverables);
	}

	return new BucketState<>(bucketId, new Path(bucketPathStr), creationTime, current, pendingFileRecoverablesPerCheckpoint);
}
 

private static void prepareDeserializationFull(final boolean withInProgress, final String scenarioName) throws IOException {

		final java.nio.file.Path scenarioPath = getResourcePath(scenarioName, CURRENT_VERSION);
		FileUtils.deleteDirectory(Paths.get(scenarioPath.toString() + "-template").toFile());
		Files.createDirectories(scenarioPath);

		final int noOfPendingCheckpoints = 5;

		final java.nio.file.Path outputPath = getOutputPath(scenarioName, CURRENT_VERSION);

		final Path testBucketPath = new Path(outputPath.resolve(BUCKET_ID).toString());

		final Bucket<String, String> bucket = createNewBucket(testBucketPath);

		BucketState<String> bucketState = null;
		// pending for checkpoints
		for (int i = 0; i < noOfPendingCheckpoints; i++) {
			// write 10 bytes to the in progress file
			bucket.write(PENDING_CONTENT, System.currentTimeMillis());
			bucket.write(PENDING_CONTENT, System.currentTimeMillis());
			// every checkpoint would produce a pending file
			bucketState = bucket.onReceptionOfCheckpoint(i);
		}

		if (withInProgress) {
			// create a in progress file
			bucket.write(IN_PROGRESS_CONTENT, System.currentTimeMillis());

			// 5 pending files and 1 in progress file
			bucketState = bucket.onReceptionOfCheckpoint(noOfPendingCheckpoints);
		}

		final byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(bucketStateSerializer(), bucketState);

		Files.write(getSnapshotPath(scenarioName, CURRENT_VERSION), bytes);

		// copy the scenario file to a template directory.
		// it is because that the test `testSerializationFull` would change the in progress file to pending files.
		moveToTemplateDirectory(scenarioPath);
	}
 

private void serializeV2(BucketState<BucketID> state, DataOutputView dataOutputView) throws IOException {
	SimpleVersionedSerialization.writeVersionAndSerialize(bucketIdSerializer, state.getBucketId(), dataOutputView);
	dataOutputView.writeUTF(state.getBucketPath().toString());
	dataOutputView.writeLong(state.getInProgressFileCreationTime());

	// put the current open part file
	if (state.hasInProgressFileRecoverable()) {
		final InProgressFileWriter.InProgressFileRecoverable inProgressFileRecoverable = state.getInProgressFileRecoverable();
		dataOutputView.writeBoolean(true);
		SimpleVersionedSerialization.writeVersionAndSerialize(inProgressFileRecoverableSerializer, inProgressFileRecoverable, dataOutputView);
	} else {
		dataOutputView.writeBoolean(false);
	}

	// put the map of pending files per checkpoint
	final Map<Long, List<InProgressFileWriter.PendingFileRecoverable>> pendingFileRecoverables = state.getPendingFileRecoverablesPerCheckpoint();

	dataOutputView.writeInt(pendingFileRecoverableSerializer.getVersion());

	dataOutputView.writeInt(pendingFileRecoverables.size());

	for (Entry<Long, List<InProgressFileWriter.PendingFileRecoverable>> pendingFilesForCheckpoint : pendingFileRecoverables.entrySet()) {
		final List<InProgressFileWriter.PendingFileRecoverable> pendingFileRecoverableList = pendingFilesForCheckpoint.getValue();

		dataOutputView.writeLong(pendingFilesForCheckpoint.getKey());
		dataOutputView.writeInt(pendingFileRecoverableList.size());

		for (InProgressFileWriter.PendingFileRecoverable pendingFileRecoverable : pendingFileRecoverableList) {
			byte[] serialized = pendingFileRecoverableSerializer.serialize(pendingFileRecoverable);
			dataOutputView.writeInt(serialized.length);
			dataOutputView.write(serialized);
		}
	}
}
 

@Override
public T next() {
	final byte[] bytes = rawIterator.next();
	try {
		return SimpleVersionedSerialization.readVersionAndDeSerialize(serializer, bytes);
	}
	catch (IOException e) {
		throw new FlinkRuntimeException("Failed to deserialize value", e);
	}
}
 

@Test
public void testSerializationOnlyInProgress() throws IOException {
	final File testFolder = tempFolder.newFolder();
	final FileSystem fs = FileSystem.get(testFolder.toURI());

	final Path testBucket = new Path(testFolder.getPath(), "test");

	final RecoverableWriter writer = fs.createRecoverableWriter();
	final RecoverableFsDataOutputStream stream = writer.open(testBucket);
	stream.write(IN_PROGRESS_CONTENT.getBytes(Charset.forName("UTF-8")));

	final RecoverableWriter.ResumeRecoverable current = stream.persist();

	final BucketState<String> bucketState = new BucketState<>(
			"test", testBucket, Long.MAX_VALUE, current, new HashMap<>());

	final SimpleVersionedSerializer<BucketState<String>> serializer =
			new BucketStateSerializer<>(
					writer.getResumeRecoverableSerializer(),
					writer.getCommitRecoverableSerializer(),
					SimpleVersionedStringSerializer.INSTANCE
			);

	final byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(serializer, bucketState);

	// to simulate that everything is over for file.
	stream.close();

	final BucketState<String> recoveredState =  SimpleVersionedSerialization.readVersionAndDeSerialize(serializer, bytes);

	Assert.assertEquals(testBucket, recoveredState.getBucketPath());

	FileStatus[] statuses = fs.listStatus(testBucket.getParent());
	Assert.assertEquals(1L, statuses.length);
	Assert.assertTrue(
			statuses[0].getPath().getPath().startsWith(
					(new Path(testBucket.getParent(), ".test.inprogress")).getPath())
	);
}
 

private void initializeActiveBuckets(final ListState<byte[]> bucketStates) throws Exception {
	for (byte[] serializedRecoveredState : bucketStates.get()) {
		final BucketState<BucketID> recoveredState =
				SimpleVersionedSerialization.readVersionAndDeSerialize(
						bucketStateSerializer, serializedRecoveredState);
		handleRestoredBucketState(recoveredState);
	}
}
 

@VisibleForTesting
BucketState<BucketID> deserializeV1(DataInputView in) throws IOException {
	final BucketID bucketId = SimpleVersionedSerialization.readVersionAndDeSerialize(bucketIdSerializer, in);
	final String bucketPathStr = in.readUTF();
	final long creationTime = in.readLong();

	// then get the current resumable stream
	RecoverableWriter.ResumeRecoverable current = null;
	if (in.readBoolean()) {
		current = SimpleVersionedSerialization.readVersionAndDeSerialize(resumableSerializer, in);
	}

	final int committableVersion = in.readInt();
	final int numCheckpoints = in.readInt();
	final HashMap<Long, List<RecoverableWriter.CommitRecoverable>> resumablesPerCheckpoint = new HashMap<>(numCheckpoints);

	for (int i = 0; i < numCheckpoints; i++) {
		final long checkpointId = in.readLong();
		final int noOfResumables = in.readInt();

		final List<RecoverableWriter.CommitRecoverable> resumables = new ArrayList<>(noOfResumables);
		for (int j = 0; j < noOfResumables; j++) {
			final byte[] bytes = new byte[in.readInt()];
			in.readFully(bytes);
			resumables.add(commitableSerializer.deserialize(committableVersion, bytes));
		}
		resumablesPerCheckpoint.put(checkpointId, resumables);
	}

	return new BucketState<>(
			bucketId,
			new Path(bucketPathStr),
			creationTime,
			current,
			resumablesPerCheckpoint);
}
 

@Test
public void testSerializationOnlyInProgress() throws IOException {
	final File testFolder = tempFolder.newFolder();
	final FileSystem fs = FileSystem.get(testFolder.toURI());

	final Path testBucket = new Path(testFolder.getPath(), "test");

	final RecoverableWriter writer = fs.createRecoverableWriter();
	final RecoverableFsDataOutputStream stream = writer.open(testBucket);
	stream.write(IN_PROGRESS_CONTENT.getBytes(Charset.forName("UTF-8")));

	final RecoverableWriter.ResumeRecoverable current = stream.persist();

	final BucketState<String> bucketState = new BucketState<>(
			"test", testBucket, Long.MAX_VALUE, current, new HashMap<>());

	final SimpleVersionedSerializer<BucketState<String>> serializer =
			new BucketStateSerializer<>(
					writer.getResumeRecoverableSerializer(),
					writer.getCommitRecoverableSerializer(),
					SimpleVersionedStringSerializer.INSTANCE
			);

	final byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(serializer, bucketState);

	// to simulate that everything is over for file.
	stream.close();

	final BucketState<String> recoveredState =  SimpleVersionedSerialization.readVersionAndDeSerialize(serializer, bytes);

	Assert.assertEquals(testBucket, recoveredState.getBucketPath());

	FileStatus[] statuses = fs.listStatus(testBucket.getParent());
	Assert.assertEquals(1L, statuses.length);
	Assert.assertTrue(
			statuses[0].getPath().getPath().startsWith(
					(new Path(testBucket.getParent(), ".test.inprogress")).toString())
	);
}
 

private void initializeActiveBuckets(final ListState<byte[]> bucketStates) throws Exception {
	for (byte[] serializedRecoveredState : bucketStates.get()) {
		final BucketState<BucketID> recoveredState =
				SimpleVersionedSerialization.readVersionAndDeSerialize(
						bucketStateSerializer, serializedRecoveredState);
		handleRestoredBucketState(recoveredState);
	}
}
 

private static BucketState<String> readBucketState(final String scenarioName, final int version) throws IOException {
	byte[] bytes = Files.readAllBytes(getSnapshotPath(scenarioName, version));
	return SimpleVersionedSerialization.readVersionAndDeSerialize(bucketStateSerializer(), bytes);
}
 

private OutputStreamBasedPendingFileRecoverable deserializeV1(final DataInputView dataInputView) throws IOException {
	return new OutputStreamBasedPendingFileRecoverable(SimpleVersionedSerialization.readVersionAndDeSerialize(commitSerializer, dataInputView));
}
 

private void serializeV1(final OutputStreamBasedPendingFileRecoverable outputStreamBasedPendingFileRecoverable, final DataOutputView dataOutputView) throws IOException {
	SimpleVersionedSerialization.writeVersionAndSerialize(commitSerializer, outputStreamBasedPendingFileRecoverable.getCommitRecoverable(), dataOutputView);
}
 

private OutputStreamBasedInProgressFileRecoverable deserializeV1(final DataInputView dataInputView) throws IOException {
	return new OutputStreamBasedInProgressFileRecoverable(SimpleVersionedSerialization.readVersionAndDeSerialize(resumeSerializer, dataInputView));
}
 

private void serializeV1(final OutputStreamBasedInProgressFileRecoverable outputStreamBasedInProgressRecoverable, final DataOutputView dataOutputView) throws IOException {
	SimpleVersionedSerialization.writeVersionAndSerialize(resumeSerializer, outputStreamBasedInProgressRecoverable.getResumeRecoverable(), dataOutputView);
}
 

private BucketState<BucketID> deserializeV1(DataInputView in) throws IOException {

		final SimpleVersionedSerializer<RecoverableWriter.CommitRecoverable> commitableSerializer = getCommitableSerializer();
		final SimpleVersionedSerializer<RecoverableWriter.ResumeRecoverable> resumableSerializer = getResumableSerializer();

		final BucketID bucketId = SimpleVersionedSerialization.readVersionAndDeSerialize(bucketIdSerializer, in);
		final String bucketPathStr = in.readUTF();
		final long creationTime = in.readLong();

		// then get the current resumable stream
		InProgressFileWriter.InProgressFileRecoverable current = null;
		if (in.readBoolean()) {
			current =
				new OutputStreamBasedPartFileWriter.OutputStreamBasedInProgressFileRecoverable(
					SimpleVersionedSerialization.readVersionAndDeSerialize(resumableSerializer, in));
		}

		final int committableVersion = in.readInt();
		final int numCheckpoints = in.readInt();
		final HashMap<Long, List<InProgressFileWriter.PendingFileRecoverable>> pendingFileRecoverablePerCheckpoint = new HashMap<>(numCheckpoints);

		for (int i = 0; i < numCheckpoints; i++) {
			final long checkpointId = in.readLong();
			final int noOfResumables = in.readInt();

			final List<InProgressFileWriter.PendingFileRecoverable> pendingFileRecoverables = new ArrayList<>(noOfResumables);
			for (int j = 0; j < noOfResumables; j++) {
				final byte[] bytes = new byte[in.readInt()];
				in.readFully(bytes);
				pendingFileRecoverables.add(
					new OutputStreamBasedPartFileWriter.OutputStreamBasedPendingFileRecoverable(commitableSerializer.deserialize(committableVersion, bytes)));
			}
			pendingFileRecoverablePerCheckpoint.put(checkpointId, pendingFileRecoverables);
		}

		return new BucketState<>(
			bucketId,
			new Path(bucketPathStr),
			creationTime,
			current,
			pendingFileRecoverablePerCheckpoint);
	}
 

private byte[] serialize(T value) throws IOException {
	return SimpleVersionedSerialization.writeVersionAndSerialize(serializer, value);
}