org.apache.flink.core.io.SimpleVersionedSerializer Java Examples

SourceCoordinatorContext(
		ExecutorService coordinatorExecutor,
		SourceCoordinatorProvider.CoordinatorExecutorThreadFactory coordinatorThreadFactory,
		int numWorkerThreads,
		OperatorCoordinator.Context operatorCoordinatorContext,
		SimpleVersionedSerializer<SplitT> splitSerializer,
		SplitAssignmentTracker<SplitT> splitAssignmentTracker) {
	this.coordinatorExecutor = coordinatorExecutor;
	this.coordinatorThreadFactory = coordinatorThreadFactory;
	this.operatorCoordinatorContext = operatorCoordinatorContext;
	this.splitSerializer = splitSerializer;
	this.registeredReaders = new ConcurrentHashMap<>();
	this.assignmentTracker = splitAssignmentTracker;
	this.coordinatorThreadName = coordinatorThreadFactory.getCoordinatorThreadName();
	this.notifier = new ExecutorNotifier(
			Executors.newScheduledThreadPool(numWorkerThreads, new ThreadFactory() {
				private int index = 0;
				@Override
				public Thread newThread(Runnable r) {
					return new Thread(r, coordinatorThreadName + "-worker-" + index++);
				}
			}),
			coordinatorExecutor);
}
 

private static <T> T deserializeState(MasterState state, MasterTriggerRestoreHook<?> hook) throws FlinkException {
	@SuppressWarnings("unchecked")
	final MasterTriggerRestoreHook<T> typedHook = (MasterTriggerRestoreHook<T>) hook;
	final String id = hook.getIdentifier();

	try {
		final SimpleVersionedSerializer<T> deserializer = typedHook.createCheckpointDataSerializer();
		if (deserializer == null) {
			throw new FlinkException("null serializer for state of hook " + hook.getIdentifier());
		}

		return deserializer.deserialize(state.version(), state.bytes());
	}
	catch (Throwable t) {
		throw new FlinkException("Cannot deserialize state for master hook '" + id + '\'', t);
	}
}
 

private static <T> T deserializeState(MasterState state, MasterTriggerRestoreHook<?> hook) throws FlinkException {
	@SuppressWarnings("unchecked")
	final MasterTriggerRestoreHook<T> typedHook = (MasterTriggerRestoreHook<T>) hook;
	final String id = hook.getIdentifier();

	try {
		final SimpleVersionedSerializer<T> deserializer = typedHook.createCheckpointDataSerializer();
		if (deserializer == null) {
			throw new FlinkException("null serializer for state of hook " + hook.getIdentifier());
		}

		return deserializer.deserialize(state.version(), state.bytes());
	}
	catch (Throwable t) {
		throw new FlinkException("Cannot deserialize state for master hook '" + id + '\'', t);
	}
}
 

@Test
public void testSerializationEmpty() throws IOException {
	final File testFolder = tempFolder.newFolder();
	final FileSystem fs = FileSystem.get(testFolder.toURI());
	final RecoverableWriter writer = fs.createRecoverableWriter();

	final Path testBucket = new Path(testFolder.getPath(), "test");

	final BucketState<String> bucketState = new BucketState<>(
			"test", testBucket, Long.MAX_VALUE, null, new HashMap<>());

	final SimpleVersionedSerializer<BucketState<String>> serializer =
			new BucketStateSerializer<>(
					writer.getResumeRecoverableSerializer(),
					writer.getCommitRecoverableSerializer(),
					SimpleVersionedStringSerializer.INSTANCE
			);

	byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(serializer, bucketState);
	final BucketState<String> recoveredState =  SimpleVersionedSerialization.readVersionAndDeSerialize(serializer, bytes);

	Assert.assertEquals(testBucket, recoveredState.getBucketPath());
	Assert.assertNull(recoveredState.getInProgressResumableFile());
	Assert.assertTrue(recoveredState.getCommittableFilesPerCheckpoint().isEmpty());
}
 

@Test
public void testSerializationEmpty() throws IOException {
	final File testFolder = tempFolder.newFolder();
	final FileSystem fs = FileSystem.get(testFolder.toURI());
	final RecoverableWriter writer = fs.createRecoverableWriter();

	final Path testBucket = new Path(testFolder.getPath(), "test");

	final BucketState<String> bucketState = new BucketState<>(
			"test", testBucket, Long.MAX_VALUE, null, new HashMap<>());

	final SimpleVersionedSerializer<BucketState<String>> serializer =
			new BucketStateSerializer<>(
					writer.getResumeRecoverableSerializer(),
					writer.getCommitRecoverableSerializer(),
					SimpleVersionedStringSerializer.INSTANCE
			);

	byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(serializer, bucketState);
	final BucketState<String> recoveredState =  SimpleVersionedSerialization.readVersionAndDeSerialize(serializer, bytes);

	Assert.assertEquals(testBucket, recoveredState.getBucketPath());
	Assert.assertNull(recoveredState.getInProgressResumableFile());
	Assert.assertTrue(recoveredState.getCommittableFilesPerCheckpoint().isEmpty());
}
 

private static <T> T deserializeState(MasterState state, MasterTriggerRestoreHook<?> hook) throws FlinkException {
	@SuppressWarnings("unchecked")
	final MasterTriggerRestoreHook<T> typedHook = (MasterTriggerRestoreHook<T>) hook;
	final String id = hook.getIdentifier();

	try {
		final SimpleVersionedSerializer<T> deserializer = typedHook.createCheckpointDataSerializer();
		if (deserializer == null) {
			throw new FlinkException("null serializer for state of hook " + hook.getIdentifier());
		}

		return deserializer.deserialize(state.version(), state.bytes());
	}
	catch (Throwable t) {
		throw new FlinkException("Cannot deserialize state for master hook '" + id + '\'', t);
	}
}
 

@Override
public SimpleVersionedSerializer<CommitRecoverable> getCommitRecoverableSerializer() {
	@SuppressWarnings("unchecked")
	SimpleVersionedSerializer<CommitRecoverable> typedSerializer = (SimpleVersionedSerializer<CommitRecoverable>)
			(SimpleVersionedSerializer<?>) HadoopRecoverableSerializer.INSTANCE;

	return typedSerializer;
}
 

@Override
public SimpleVersionedSerializer<ResumeRecoverable> getResumeRecoverableSerializer() {
	@SuppressWarnings("unchecked")
	SimpleVersionedSerializer<ResumeRecoverable> typedSerializer = (SimpleVersionedSerializer<ResumeRecoverable>)
			(SimpleVersionedSerializer<?>) LocalRecoverableSerializer.INSTANCE;

	return typedSerializer;
}
 

BucketStateSerializer(
		final SimpleVersionedSerializer<RecoverableWriter.ResumeRecoverable> resumableSerializer,
		final SimpleVersionedSerializer<RecoverableWriter.CommitRecoverable> commitableSerializer,
		final SimpleVersionedSerializer<BucketID> bucketIdSerializer
) {
	this.resumableSerializer = Preconditions.checkNotNull(resumableSerializer);
	this.commitableSerializer = Preconditions.checkNotNull(commitableSerializer);
	this.bucketIdSerializer = Preconditions.checkNotNull(bucketIdSerializer);
}
 

@Test
public void testCommitAfterRecovery() throws Exception {
	final Path testDir = getBasePathForTest();
	final Path path = new Path(testDir, "part-0");

	final RecoverableWriter initWriter = getNewFileSystemWriter();

	final RecoverableWriter.CommitRecoverable recoverable;
	try (final RecoverableFsDataOutputStream stream = initWriter.open(path)) {
		stream.write(testData1.getBytes(StandardCharsets.UTF_8));

		stream.persist();
		stream.persist();

		// and write some more data
		stream.write(testData2.getBytes(StandardCharsets.UTF_8));

		recoverable = stream.closeForCommit().getRecoverable();
	}

	final byte[] serializedRecoverable = initWriter.getCommitRecoverableSerializer().serialize(recoverable);

	// get a new serializer from a new writer to make sure that no pre-initialized state leaks in.
	final RecoverableWriter newWriter = getNewFileSystemWriter();
	final SimpleVersionedSerializer<RecoverableWriter.CommitRecoverable> deserializer = newWriter.getCommitRecoverableSerializer();
	final RecoverableWriter.CommitRecoverable recoveredRecoverable = deserializer.deserialize(deserializer.getVersion(), serializedRecoverable);

	final RecoverableFsDataOutputStream.Committer committer = newWriter.recoverForCommit(recoveredRecoverable);
	committer.commitAfterRecovery();

	Map<Path, String> files = getFileContentByPath(testDir);
	Assert.assertEquals(1L, files.size());

	for (Map.Entry<Path, String> fileContents : files.entrySet()) {
		Assert.assertEquals("part-0", fileContents.getKey().getName());
		Assert.assertEquals(testData1 + testData2, fileContents.getValue());
	}
}
 

@Override
public SimpleVersionedSerializer<ResumeRecoverable> getResumeRecoverableSerializer() {
	@SuppressWarnings("unchecked")
	SimpleVersionedSerializer<ResumeRecoverable> typedSerializer = (SimpleVersionedSerializer<ResumeRecoverable>)
			(SimpleVersionedSerializer<?>) HadoopRecoverableSerializer.INSTANCE;

	return typedSerializer;
}
 

@Override
public SimpleVersionedSerializer<ResumeRecoverable> getResumeRecoverableSerializer() {
	@SuppressWarnings("unchecked")
	SimpleVersionedSerializer<ResumeRecoverable> typedSerializer = (SimpleVersionedSerializer<ResumeRecoverable>)
			(SimpleVersionedSerializer<?>) HadoopRecoverableSerializer.INSTANCE;

	return typedSerializer;
}
 

@Override
public SimpleVersionedSerializer<CommitRecoverable> getCommitRecoverableSerializer() {
	@SuppressWarnings("unchecked")
	SimpleVersionedSerializer<CommitRecoverable> typedSerializer = (SimpleVersionedSerializer<CommitRecoverable>)
			(SimpleVersionedSerializer<?>) HadoopRecoverableSerializer.INSTANCE;

	return typedSerializer;
}
 

private static SimpleVersionedSerializer<BucketState<String>> bucketStateSerializer() throws IOException {
	final RowWiseBucketWriter bucketWriter = createBucketWriter();
	return new BucketStateSerializer<>(
		bucketWriter.getProperties().getInProgressFileRecoverableSerializer(),
		bucketWriter.getProperties().getPendingFileRecoverableSerializer(),
		SimpleVersionedStringSerializer.INSTANCE);
}
 

public AddSplitEvent(List<SplitT> splits, SimpleVersionedSerializer<SplitT> splitSerializer) throws IOException {
	this.splits = new ArrayList<>(splits.size());
	this.serializerVersion = splitSerializer.getVersion();
	for (SplitT split : splits) {
		this.splits.add(splitSerializer.serialize(split));
	}
}
 

/**
 * Restore the state of the context.
 * @param splitSerializer the serializer for the SourceSplits.
 * @param in the input from which the states are read.
 * @throws Exception when the restoration failed.
 */
@SuppressWarnings("unchecked")
void restoreState(
		SimpleVersionedSerializer<SplitT> splitSerializer,
		DataInputStream in) throws Exception {
	Map<Integer, ReaderInfo> readers = readRegisteredReaders(in);
	registeredReaders.clear();
	registeredReaders.putAll(readers);
	assignmentTracker.restoreState(splitSerializer, in);
}
 

public List<SplitT> splits(SimpleVersionedSerializer<SplitT> splitSerializer) throws IOException {
	List<SplitT> result = new ArrayList<>(splits.size());
	for (byte[] serializedSplit : splits) {
		result.add(splitSerializer.deserialize(serializerVersion, serializedSplit));
	}
	return result;
}
 

@Override
public SimpleVersionedSerializer<ResumeRecoverable> getResumeRecoverableSerializer() {
	@SuppressWarnings("unchecked")
	SimpleVersionedSerializer<ResumeRecoverable> typedSerializer = (SimpleVersionedSerializer<ResumeRecoverable>)
			(SimpleVersionedSerializer<?>) LocalRecoverableSerializer.INSTANCE;

	return typedSerializer;
}
 

@Override
public SimpleVersionedSerializer<CommitRecoverable> getCommitRecoverableSerializer() {
	@SuppressWarnings("unchecked")
	SimpleVersionedSerializer<CommitRecoverable> typedSerializer = (SimpleVersionedSerializer<CommitRecoverable>)
			(SimpleVersionedSerializer<?>) LocalRecoverableSerializer.INSTANCE;

	return typedSerializer;
}
 

private File prepareCompressedFile(CompressWriterFactory<String> writer, List<String> lines) throws Exception {
	final File outDir = TEMPORARY_FOLDER.newFolder();

	final BucketAssigner<String, String> assigner = new BucketAssigner<String, String> () {
		@Override
		public String getBucketId(String element, BucketAssigner.Context context) {
			return "bucket";
		}

		@Override
		public SimpleVersionedSerializer<String> getSerializer() {
			return SimpleVersionedStringSerializer.INSTANCE;
		}
	};

	StreamingFileSink<String> sink = StreamingFileSink
		.forBulkFormat(new Path(outDir.toURI()), writer)
		.withBucketAssigner(assigner)
		.build();

	try (
		OneInputStreamOperatorTestHarness<String, Object> testHarness = new OneInputStreamOperatorTestHarness<>(new StreamSink<>(sink), 1, 1, 0)
	) {
		testHarness.setup();
		testHarness.open();

		int time = 0;
		for (String line: lines) {
			testHarness.processElement(new StreamRecord<>(line, ++time));
		}

		testHarness.snapshot(1, ++time);
		testHarness.notifyOfCompletedCheckpoint(1);
	}

	return outDir;
}
 

@Test
public void testSerializationOnlyInProgress() throws IOException {
	final File testFolder = tempFolder.newFolder();
	final FileSystem fs = FileSystem.get(testFolder.toURI());

	final Path testBucket = new Path(testFolder.getPath(), "test");

	final RecoverableWriter writer = fs.createRecoverableWriter();
	final RecoverableFsDataOutputStream stream = writer.open(testBucket);
	stream.write(IN_PROGRESS_CONTENT.getBytes(Charset.forName("UTF-8")));

	final RecoverableWriter.ResumeRecoverable current = stream.persist();

	final BucketState<String> bucketState = new BucketState<>(
			"test", testBucket, Long.MAX_VALUE, current, new HashMap<>());

	final SimpleVersionedSerializer<BucketState<String>> serializer =
			new BucketStateSerializer<>(
					writer.getResumeRecoverableSerializer(),
					writer.getCommitRecoverableSerializer(),
					SimpleVersionedStringSerializer.INSTANCE
			);

	final byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(serializer, bucketState);

	// to simulate that everything is over for file.
	stream.close();

	final BucketState<String> recoveredState =  SimpleVersionedSerialization.readVersionAndDeSerialize(serializer, bytes);

	Assert.assertEquals(testBucket, recoveredState.getBucketPath());

	FileStatus[] statuses = fs.listStatus(testBucket.getParent());
	Assert.assertEquals(1L, statuses.length);
	Assert.assertTrue(
			statuses[0].getPath().getPath().startsWith(
					(new Path(testBucket.getParent(), ".test.inprogress")).toString())
	);
}
 

/**
 * Deserialize the assignment by checkpoint ids.
 */
static <SplitT> Map<Long, Map<Integer, LinkedHashSet<SplitT>>> readAssignmentsByCheckpointId(
		DataInputStream in,
		SimpleVersionedSerializer<SplitT> splitSerializer) throws IOException {
	int splitSerializerVersion = in.readInt();
	int numCheckpoints = in.readInt();
	Map<Long, Map<Integer, LinkedHashSet<SplitT>>> assignmentsByCheckpointIds = new HashMap<>(numCheckpoints);
	for (int i = 0; i < numCheckpoints; i++) {
		long checkpointId = in.readLong();
		int numSubtasks = in.readInt();
		Map<Integer, LinkedHashSet<SplitT>> assignments = new HashMap<>();
		assignmentsByCheckpointIds.put(checkpointId, assignments);
		for (int j = 0; j < numSubtasks; j++) {
			int subtaskId = in.readInt();
			int numAssignedSplits = in.readInt();
			LinkedHashSet<SplitT> splits = new LinkedHashSet<>(numAssignedSplits);
			assignments.put(subtaskId, splits);
			for (int k = 0; k < numAssignedSplits; k++) {
				int serializedSplitSize = in.readInt();
				byte[] serializedSplit = readBytes(in, serializedSplitSize);
				SplitT split = splitSerializer.deserialize(splitSerializerVersion, serializedSplit);
				splits.add(split);
			}
		}
	}
	return assignmentsByCheckpointIds;
}
 

@Override
public SimpleVersionedSerializer<CommitRecoverable> getCommitRecoverableSerializer() {
	@SuppressWarnings("unchecked")
	SimpleVersionedSerializer<CommitRecoverable> typedSerializer = (SimpleVersionedSerializer<CommitRecoverable>)
			(SimpleVersionedSerializer<?>) LocalRecoverableSerializer.INSTANCE;

	return typedSerializer;
}
 

@Override
public SimpleVersionedSerializer<ResumeRecoverable> getResumeRecoverableSerializer() {
	@SuppressWarnings("unchecked")
	SimpleVersionedSerializer<ResumeRecoverable> typedSerializer = (SimpleVersionedSerializer<ResumeRecoverable>)
			(SimpleVersionedSerializer<?>) LocalRecoverableSerializer.INSTANCE;

	return typedSerializer;
}
 

@Test
public void testSerializationOnlyInProgress() throws IOException {
	final File testFolder = tempFolder.newFolder();
	final FileSystem fs = FileSystem.get(testFolder.toURI());

	final Path testBucket = new Path(testFolder.getPath(), "test");

	final RecoverableWriter writer = fs.createRecoverableWriter();
	final RecoverableFsDataOutputStream stream = writer.open(testBucket);
	stream.write(IN_PROGRESS_CONTENT.getBytes(Charset.forName("UTF-8")));

	final RecoverableWriter.ResumeRecoverable current = stream.persist();

	final BucketState<String> bucketState = new BucketState<>(
			"test", testBucket, Long.MAX_VALUE, current, new HashMap<>());

	final SimpleVersionedSerializer<BucketState<String>> serializer =
			new BucketStateSerializer<>(
					writer.getResumeRecoverableSerializer(),
					writer.getCommitRecoverableSerializer(),
					SimpleVersionedStringSerializer.INSTANCE
			);

	final byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(serializer, bucketState);

	// to simulate that everything is over for file.
	stream.close();

	final BucketState<String> recoveredState =  SimpleVersionedSerialization.readVersionAndDeSerialize(serializer, bytes);

	Assert.assertEquals(testBucket, recoveredState.getBucketPath());

	FileStatus[] statuses = fs.listStatus(testBucket.getParent());
	Assert.assertEquals(1L, statuses.length);
	Assert.assertTrue(
			statuses[0].getPath().getPath().startsWith(
					(new Path(testBucket.getParent(), ".test.inprogress")).getPath())
	);
}
 

private void testResumeAfterMultiplePersist(
		final String persistName,
		final String expectedFinalContents,
		final String firstItemToWrite,
		final String secondItemToWrite,
		final String thirdItemToWrite) throws Exception {

	final Path path = new Path(basePathForTest, "part-0");
	final RecoverableWriter initWriter = getRecoverableWriter();

	final Map<String, RecoverableWriter.ResumeRecoverable> recoverables = new HashMap<>(4);
	try (final RecoverableFsDataOutputStream stream = initWriter.open(path)) {
		recoverables.put(INIT_EMPTY_PERSIST, stream.persist());

		stream.write(bytesOf(firstItemToWrite));

		recoverables.put(INTERM_WITH_STATE_PERSIST, stream.persist());
		recoverables.put(INTERM_WITH_NO_ADDITIONAL_STATE_PERSIST, stream.persist());

		// and write some more data
		stream.write(bytesOf(secondItemToWrite));

		recoverables.put(FINAL_WITH_EXTRA_STATE, stream.persist());
	}

	final SimpleVersionedSerializer<RecoverableWriter.ResumeRecoverable> serializer = initWriter.getResumeRecoverableSerializer();
	final byte[] serializedRecoverable = serializer.serialize(recoverables.get(persistName));

	// get a new serializer from a new writer to make sure that no pre-initialized state leaks in.
	final RecoverableWriter newWriter = getRecoverableWriter();
	final SimpleVersionedSerializer<RecoverableWriter.ResumeRecoverable> deserializer = newWriter.getResumeRecoverableSerializer();
	final RecoverableWriter.ResumeRecoverable recoveredRecoverable =
			deserializer.deserialize(serializer.getVersion(), serializedRecoverable);

	final RecoverableFsDataOutputStream recoveredStream = newWriter.recover(recoveredRecoverable);
	recoveredStream.write(bytesOf(thirdItemToWrite));
	recoveredStream.closeForCommit().commit();

	Assert.assertEquals(expectedFinalContents, getContentsOfFile(path));
}
 

/**
 * Take a snapshot of this SourceCoordinatorContext.
 *
 * @param checkpointId The id of the ongoing checkpoint.
 * @param splitSerializer The serializer of the splits.
 * @param out An ObjectOutput that can be used to
 */
void snapshotState(
		long checkpointId,
		SimpleVersionedSerializer<SplitT> splitSerializer,
		DataOutputStream out) throws Exception {
	writeRegisteredReaders(registeredReaders, out);
	assignmentTracker.snapshotState(checkpointId, splitSerializer, out);
}
 

@Override
public SimpleVersionedSerializer<ResumeRecoverable> getResumeRecoverableSerializer() {
	@SuppressWarnings("unchecked")
	SimpleVersionedSerializer<ResumeRecoverable> typedSerializer = (SimpleVersionedSerializer<ResumeRecoverable>)
			(SimpleVersionedSerializer<?>) HadoopRecoverableSerializer.INSTANCE;

	return typedSerializer;
}
 

@Override
public SimpleVersionedSerializer<CommitRecoverable> getCommitRecoverableSerializer() {
	@SuppressWarnings("unchecked")
	SimpleVersionedSerializer<CommitRecoverable> typedSerializer = (SimpleVersionedSerializer<CommitRecoverable>)
			(SimpleVersionedSerializer<?>) LocalRecoverableSerializer.INSTANCE;

	return typedSerializer;
}
 

BucketStateSerializer(
		final SimpleVersionedSerializer<RecoverableWriter.ResumeRecoverable> resumableSerializer,
		final SimpleVersionedSerializer<RecoverableWriter.CommitRecoverable> commitableSerializer,
		final SimpleVersionedSerializer<BucketID> bucketIdSerializer
) {
	this.resumableSerializer = Preconditions.checkNotNull(resumableSerializer);
	this.commitableSerializer = Preconditions.checkNotNull(commitableSerializer);
	this.bucketIdSerializer = Preconditions.checkNotNull(bucketIdSerializer);
}