Java Code Examples for org.apache.flink.core.io.SimpleVersionedSerialization#writeVersionAndSerialize()

The following examples show how to use org.apache.flink.core.io.SimpleVersionedSerialization#writeVersionAndSerialize() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: Buckets.java From Flink-CEPplus with Apache License 2.0

6 votes

private void snapshotActiveBuckets(
		final long checkpointId,
		final ListState<byte[]> bucketStatesContainer) throws Exception {

	for (Bucket<IN, BucketID> bucket : activeBuckets.values()) {
		final BucketState<BucketID> bucketState = bucket.onReceptionOfCheckpoint(checkpointId);

		final byte[] serializedBucketState = SimpleVersionedSerialization
				.writeVersionAndSerialize(bucketStateSerializer, bucketState);

		bucketStatesContainer.add(serializedBucketState);

		if (LOG.isDebugEnabled()) {
			LOG.debug("Subtask {} checkpointing: {}", subtaskIndex, bucketState);
		}
	}
}

Example 2

Source File: BucketStateSerializerTest.java From Flink-CEPplus with Apache License 2.0

6 votes

@Test
public void testSerializationEmpty() throws IOException {
	final File testFolder = tempFolder.newFolder();
	final FileSystem fs = FileSystem.get(testFolder.toURI());
	final RecoverableWriter writer = fs.createRecoverableWriter();

	final Path testBucket = new Path(testFolder.getPath(), "test");

	final BucketState<String> bucketState = new BucketState<>(
			"test", testBucket, Long.MAX_VALUE, null, new HashMap<>());

	final SimpleVersionedSerializer<BucketState<String>> serializer =
			new BucketStateSerializer<>(
					writer.getResumeRecoverableSerializer(),
					writer.getCommitRecoverableSerializer(),
					SimpleVersionedStringSerializer.INSTANCE
			);

	byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(serializer, bucketState);
	final BucketState<String> recoveredState =  SimpleVersionedSerialization.readVersionAndDeSerialize(serializer, bytes);

	Assert.assertEquals(testBucket, recoveredState.getBucketPath());
	Assert.assertNull(recoveredState.getInProgressResumableFile());
	Assert.assertTrue(recoveredState.getCommittableFilesPerCheckpoint().isEmpty());
}

Example 3

Source File: Buckets.java From flink with Apache License 2.0

6 votes

private void snapshotActiveBuckets(
		final long checkpointId,
		final ListState<byte[]> bucketStatesContainer) throws Exception {

	for (Bucket<IN, BucketID> bucket : activeBuckets.values()) {
		final BucketState<BucketID> bucketState = bucket.onReceptionOfCheckpoint(checkpointId);

		final byte[] serializedBucketState = SimpleVersionedSerialization
				.writeVersionAndSerialize(bucketStateSerializer, bucketState);

		bucketStatesContainer.add(serializedBucketState);

		if (LOG.isDebugEnabled()) {
			LOG.debug("Subtask {} checkpointing: {}", subtaskIndex, bucketState);
		}
	}
}

Example 4

Source File: BucketStateSerializerTest.java From flink with Apache License 2.0

6 votes

@Test
public void testSerializationEmpty() throws IOException {
	final File testFolder = tempFolder.newFolder();
	final FileSystem fs = FileSystem.get(testFolder.toURI());
	final RecoverableWriter writer = fs.createRecoverableWriter();

	final Path testBucket = new Path(testFolder.getPath(), "test");

	final BucketState<String> bucketState = new BucketState<>(
			"test", testBucket, Long.MAX_VALUE, null, new HashMap<>());

	final SimpleVersionedSerializer<BucketState<String>> serializer =
			new BucketStateSerializer<>(
					writer.getResumeRecoverableSerializer(),
					writer.getCommitRecoverableSerializer(),
					SimpleVersionedStringSerializer.INSTANCE
			);

	byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(serializer, bucketState);
	final BucketState<String> recoveredState =  SimpleVersionedSerialization.readVersionAndDeSerialize(serializer, bytes);

	Assert.assertEquals(testBucket, recoveredState.getBucketPath());
	Assert.assertNull(recoveredState.getInProgressResumableFile());
	Assert.assertTrue(recoveredState.getCommittableFilesPerCheckpoint().isEmpty());
}

Example 5

Source File: Buckets.java From flink with Apache License 2.0

6 votes

private void snapshotActiveBuckets(
		final long checkpointId,
		final ListState<byte[]> bucketStatesContainer) throws Exception {

	for (Bucket<IN, BucketID> bucket : activeBuckets.values()) {
		final BucketState<BucketID> bucketState = bucket.onReceptionOfCheckpoint(checkpointId);

		final byte[] serializedBucketState = SimpleVersionedSerialization
				.writeVersionAndSerialize(bucketStateSerializer, bucketState);

		bucketStatesContainer.add(serializedBucketState);

		if (LOG.isDebugEnabled()) {
			LOG.debug("Subtask {} checkpointing: {}", subtaskIndex, bucketState);
		}
	}
}

Example 6

Source File: SourceOperatorTest.java From flink with Apache License 2.0

6 votes

private StateInitializationContext getStateContext() throws Exception {
	// Create a mock split.
	byte[] serializedSplitWithVersion = SimpleVersionedSerialization
			.writeVersionAndSerialize(new MockSourceSplitSerializer(), MOCK_SPLIT);

	// Crate the state context.
	OperatorStateStore operatorStateStore = createOperatorStateStore();
	StateInitializationContext stateContext = new StateInitializationContextImpl(
			false,
			operatorStateStore,
			null,
			null,
			null);

	// Update the context.
	stateContext.getOperatorStateStore()
				.getListState(SourceOperator.SPLITS_STATE_DESC)
				.update(Collections.singletonList(serializedSplitWithVersion));

	return stateContext;
}

Example 7

Source File: BucketStateSerializerTest.java From flink with Apache License 2.0

6 votes

@Test
@Ignore
public void prepareDeserializationEmpty() throws IOException {

	final String scenarioName = "empty";
	final java.nio.file.Path scenarioPath = getResourcePath(scenarioName, CURRENT_VERSION);

	FileUtils.deleteDirectory(scenarioPath.toFile());
	Files.createDirectories(scenarioPath);

	final java.nio.file.Path outputPath = getOutputPath(scenarioName, CURRENT_VERSION);
	final Path testBucketPath = new Path(outputPath.resolve(BUCKET_ID).toString());

	final Bucket<String, String> bucket =
		createNewBucket(testBucketPath);

	final BucketState<String> bucketState = bucket.onReceptionOfCheckpoint(0);

	byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(
		bucketStateSerializer(),
		bucketState);
	Files.write(getSnapshotPath(scenarioName, CURRENT_VERSION), bytes);
}

Example 8

Source File: BucketStateSerializerTest.java From flink with Apache License 2.0

6 votes

@Test
@Ignore
public void prepareDeserializationOnlyInProgress() throws IOException {

	final String scenarioName = "only-in-progress";
	final java.nio.file.Path scenarioPath = getResourcePath(scenarioName, CURRENT_VERSION);
	FileUtils.deleteDirectory(scenarioPath.toFile());
	Files.createDirectories(scenarioPath);

	final java.nio.file.Path outputPath = getOutputPath(scenarioName, CURRENT_VERSION);
	final Path testBucketPath = new Path(outputPath.resolve(BUCKET_ID).toString());

	final Bucket<String, String> bucket =
		createNewBucket(testBucketPath);

	bucket.write(IN_PROGRESS_CONTENT, System.currentTimeMillis());

	final BucketState<String> bucketState = bucket.onReceptionOfCheckpoint(0);

	final byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(
		bucketStateSerializer(), bucketState);

	Files.write(getSnapshotPath(scenarioName, CURRENT_VERSION), bytes);
}

Example 9

Source File: BucketStateSerializerTest.java From Flink-CEPplus with Apache License 2.0

5 votes

@Test
public void testSerializationOnlyInProgress() throws IOException {
	final File testFolder = tempFolder.newFolder();
	final FileSystem fs = FileSystem.get(testFolder.toURI());

	final Path testBucket = new Path(testFolder.getPath(), "test");

	final RecoverableWriter writer = fs.createRecoverableWriter();
	final RecoverableFsDataOutputStream stream = writer.open(testBucket);
	stream.write(IN_PROGRESS_CONTENT.getBytes(Charset.forName("UTF-8")));

	final RecoverableWriter.ResumeRecoverable current = stream.persist();

	final BucketState<String> bucketState = new BucketState<>(
			"test", testBucket, Long.MAX_VALUE, current, new HashMap<>());

	final SimpleVersionedSerializer<BucketState<String>> serializer =
			new BucketStateSerializer<>(
					writer.getResumeRecoverableSerializer(),
					writer.getCommitRecoverableSerializer(),
					SimpleVersionedStringSerializer.INSTANCE
			);

	final byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(serializer, bucketState);

	// to simulate that everything is over for file.
	stream.close();

	final BucketState<String> recoveredState =  SimpleVersionedSerialization.readVersionAndDeSerialize(serializer, bytes);

	Assert.assertEquals(testBucket, recoveredState.getBucketPath());

	FileStatus[] statuses = fs.listStatus(testBucket.getParent());
	Assert.assertEquals(1L, statuses.length);
	Assert.assertTrue(
			statuses[0].getPath().getPath().startsWith(
					(new Path(testBucket.getParent(), ".test.inprogress")).toString())
	);
}

Example 10

Source File: BucketStateSerializerTest.java From flink with Apache License 2.0

5 votes

@Test
public void testSerializationOnlyInProgress() throws IOException {
	final File testFolder = tempFolder.newFolder();
	final FileSystem fs = FileSystem.get(testFolder.toURI());

	final Path testBucket = new Path(testFolder.getPath(), "test");

	final RecoverableWriter writer = fs.createRecoverableWriter();
	final RecoverableFsDataOutputStream stream = writer.open(testBucket);
	stream.write(IN_PROGRESS_CONTENT.getBytes(Charset.forName("UTF-8")));

	final RecoverableWriter.ResumeRecoverable current = stream.persist();

	final BucketState<String> bucketState = new BucketState<>(
			"test", testBucket, Long.MAX_VALUE, current, new HashMap<>());

	final SimpleVersionedSerializer<BucketState<String>> serializer =
			new BucketStateSerializer<>(
					writer.getResumeRecoverableSerializer(),
					writer.getCommitRecoverableSerializer(),
					SimpleVersionedStringSerializer.INSTANCE
			);

	final byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(serializer, bucketState);

	// to simulate that everything is over for file.
	stream.close();

	final BucketState<String> recoveredState =  SimpleVersionedSerialization.readVersionAndDeSerialize(serializer, bytes);

	Assert.assertEquals(testBucket, recoveredState.getBucketPath());

	FileStatus[] statuses = fs.listStatus(testBucket.getParent());
	Assert.assertEquals(1L, statuses.length);
	Assert.assertTrue(
			statuses[0].getPath().getPath().startsWith(
					(new Path(testBucket.getParent(), ".test.inprogress")).getPath())
	);
}

Example 11

Source File: BucketStateSerializer.java From flink with Apache License 2.0

5 votes

private void serializeV2(BucketState<BucketID> state, DataOutputView dataOutputView) throws IOException {
	SimpleVersionedSerialization.writeVersionAndSerialize(bucketIdSerializer, state.getBucketId(), dataOutputView);
	dataOutputView.writeUTF(state.getBucketPath().toString());
	dataOutputView.writeLong(state.getInProgressFileCreationTime());

	// put the current open part file
	if (state.hasInProgressFileRecoverable()) {
		final InProgressFileWriter.InProgressFileRecoverable inProgressFileRecoverable = state.getInProgressFileRecoverable();
		dataOutputView.writeBoolean(true);
		SimpleVersionedSerialization.writeVersionAndSerialize(inProgressFileRecoverableSerializer, inProgressFileRecoverable, dataOutputView);
	} else {
		dataOutputView.writeBoolean(false);
	}

	// put the map of pending files per checkpoint
	final Map<Long, List<InProgressFileWriter.PendingFileRecoverable>> pendingFileRecoverables = state.getPendingFileRecoverablesPerCheckpoint();

	dataOutputView.writeInt(pendingFileRecoverableSerializer.getVersion());

	dataOutputView.writeInt(pendingFileRecoverables.size());

	for (Entry<Long, List<InProgressFileWriter.PendingFileRecoverable>> pendingFilesForCheckpoint : pendingFileRecoverables.entrySet()) {
		final List<InProgressFileWriter.PendingFileRecoverable> pendingFileRecoverableList = pendingFilesForCheckpoint.getValue();

		dataOutputView.writeLong(pendingFilesForCheckpoint.getKey());
		dataOutputView.writeInt(pendingFileRecoverableList.size());

		for (InProgressFileWriter.PendingFileRecoverable pendingFileRecoverable : pendingFileRecoverableList) {
			byte[] serialized = pendingFileRecoverableSerializer.serialize(pendingFileRecoverable);
			dataOutputView.writeInt(serialized.length);
			dataOutputView.write(serialized);
		}
	}
}

Example 12

Source File: BucketStateSerializerTest.java From flink with Apache License 2.0

5 votes

private static void prepareDeserializationFull(final boolean withInProgress, final String scenarioName) throws IOException {

		final java.nio.file.Path scenarioPath = getResourcePath(scenarioName, CURRENT_VERSION);
		FileUtils.deleteDirectory(Paths.get(scenarioPath.toString() + "-template").toFile());
		Files.createDirectories(scenarioPath);

		final int noOfPendingCheckpoints = 5;

		final java.nio.file.Path outputPath = getOutputPath(scenarioName, CURRENT_VERSION);

		final Path testBucketPath = new Path(outputPath.resolve(BUCKET_ID).toString());

		final Bucket<String, String> bucket = createNewBucket(testBucketPath);

		BucketState<String> bucketState = null;
		// pending for checkpoints
		for (int i = 0; i < noOfPendingCheckpoints; i++) {
			// write 10 bytes to the in progress file
			bucket.write(PENDING_CONTENT, System.currentTimeMillis());
			bucket.write(PENDING_CONTENT, System.currentTimeMillis());
			// every checkpoint would produce a pending file
			bucketState = bucket.onReceptionOfCheckpoint(i);
		}

		if (withInProgress) {
			// create a in progress file
			bucket.write(IN_PROGRESS_CONTENT, System.currentTimeMillis());

			// 5 pending files and 1 in progress file
			bucketState = bucket.onReceptionOfCheckpoint(noOfPendingCheckpoints);
		}

		final byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(bucketStateSerializer(), bucketState);

		Files.write(getSnapshotPath(scenarioName, CURRENT_VERSION), bytes);

		// copy the scenario file to a template directory.
		// it is because that the test `testSerializationFull` would change the in progress file to pending files.
		moveToTemplateDirectory(scenarioPath);
	}

Example 13

Source File: SimpleVersionedListState.java From flink with Apache License 2.0

4 votes

private byte[] serialize(T value) throws IOException {
	return SimpleVersionedSerialization.writeVersionAndSerialize(serializer, value);
}

Example 14

Source File: OutputStreamBasedPartFileWriter.java From flink with Apache License 2.0

4 votes

private void serializeV1(final OutputStreamBasedInProgressFileRecoverable outputStreamBasedInProgressRecoverable, final DataOutputView dataOutputView) throws IOException {
	SimpleVersionedSerialization.writeVersionAndSerialize(resumeSerializer, outputStreamBasedInProgressRecoverable.getResumeRecoverable(), dataOutputView);
}

Example 15

Source File: OutputStreamBasedPartFileWriter.java From flink with Apache License 2.0

4 votes

private void serializeV1(final OutputStreamBasedPendingFileRecoverable outputStreamBasedPendingFileRecoverable, final DataOutputView dataOutputView) throws IOException {
	SimpleVersionedSerialization.writeVersionAndSerialize(commitSerializer, outputStreamBasedPendingFileRecoverable.getCommitRecoverable(), dataOutputView);
}