org.apache.flink.streaming.api.checkpoint.CheckpointedFunction Java Examples

The following examples show how to use org.apache.flink.streaming.api.checkpoint.CheckpointedFunction. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: AbstractUdfStreamOperator.java From Flink-CEPplus with Apache License 2.0

5 votes

private void checkUdfCheckpointingPreconditions() {

		if (userFunction instanceof CheckpointedFunction
			&& userFunction instanceof ListCheckpointed) {

			throw new IllegalStateException("User functions are not allowed to implement " +
				"CheckpointedFunction AND ListCheckpointed.");
		}
	}

Example #2

Source File: StreamingFunctionUtils.java From Flink-CEPplus with Apache License 2.0

5 votes

private static boolean trySnapshotFunctionState(
		StateSnapshotContext context,
		OperatorStateBackend backend,
		Function userFunction) throws Exception {

	if (userFunction instanceof CheckpointedFunction) {
		((CheckpointedFunction) userFunction).snapshotState(context);

		return true;
	}

	if (userFunction instanceof ListCheckpointed) {
		@SuppressWarnings("unchecked")
		List<Serializable> partitionableState = ((ListCheckpointed<Serializable>) userFunction).
				snapshotState(context.getCheckpointId(), context.getCheckpointTimestamp());

		ListState<Serializable> listState = backend.
				getSerializableListState(DefaultOperatorStateBackend.DEFAULT_OPERATOR_STATE_NAME);

		listState.clear();

		if (null != partitionableState) {
			try {
				for (Serializable statePartition : partitionableState) {
					listState.add(statePartition);
				}
			} catch (Exception e) {
				listState.clear();

				throw new Exception("Could not write partitionable state to operator " +
					"state backend.", e);
			}
		}

		return true;
	}

	return false;
}

Example #3

Source File: StreamingFunctionUtils.java From Flink-CEPplus with Apache License 2.0

5 votes

private static boolean tryRestoreFunction(
		StateInitializationContext context,
		Function userFunction) throws Exception {

	if (userFunction instanceof CheckpointedFunction) {
		((CheckpointedFunction) userFunction).initializeState(context);

		return true;
	}

	if (context.isRestored() && userFunction instanceof ListCheckpointed) {
		@SuppressWarnings("unchecked")
		ListCheckpointed<Serializable> listCheckpointedFun = (ListCheckpointed<Serializable>) userFunction;

		ListState<Serializable> listState = context.getOperatorStateStore().
				getSerializableListState(DefaultOperatorStateBackend.DEFAULT_OPERATOR_STATE_NAME);

		List<Serializable> list = new ArrayList<>();

		for (Serializable serializable : listState.get()) {
			list.add(serializable);
		}

		try {
			listCheckpointedFun.restoreState(list);
		} catch (Exception e) {

			throw new Exception("Failed to restore state to function: " + e.getMessage(), e);
		}

		return true;
	}

	return false;
}

Example #4

Source File: AbstractUdfStreamOperator.java From flink with Apache License 2.0

5 votes

private void checkUdfCheckpointingPreconditions() {

		if (userFunction instanceof CheckpointedFunction
			&& userFunction instanceof ListCheckpointed) {

			throw new IllegalStateException("User functions are not allowed to implement " +
				"CheckpointedFunction AND ListCheckpointed.");
		}
	}

Example #5

Source File: StreamingFunctionUtils.java From flink with Apache License 2.0

5 votes

private static boolean trySnapshotFunctionState(
		StateSnapshotContext context,
		OperatorStateBackend backend,
		Function userFunction) throws Exception {

	if (userFunction instanceof CheckpointedFunction) {
		((CheckpointedFunction) userFunction).snapshotState(context);

		return true;
	}

	if (userFunction instanceof ListCheckpointed) {
		@SuppressWarnings("unchecked")
		List<Serializable> partitionableState = ((ListCheckpointed<Serializable>) userFunction).
				snapshotState(context.getCheckpointId(), context.getCheckpointTimestamp());

		ListState<Serializable> listState = backend.
				getSerializableListState(DefaultOperatorStateBackend.DEFAULT_OPERATOR_STATE_NAME);

		listState.clear();

		if (null != partitionableState) {
			try {
				for (Serializable statePartition : partitionableState) {
					listState.add(statePartition);
				}
			} catch (Exception e) {
				listState.clear();

				throw new Exception("Could not write partitionable state to operator " +
					"state backend.", e);
			}
		}

		return true;
	}

	return false;
}

Example #6

Source File: StreamingFunctionUtils.java From flink with Apache License 2.0

5 votes

private static boolean tryRestoreFunction(
		StateInitializationContext context,
		Function userFunction) throws Exception {

	if (userFunction instanceof CheckpointedFunction) {
		((CheckpointedFunction) userFunction).initializeState(context);

		return true;
	}

	if (context.isRestored() && userFunction instanceof ListCheckpointed) {
		@SuppressWarnings("unchecked")
		ListCheckpointed<Serializable> listCheckpointedFun = (ListCheckpointed<Serializable>) userFunction;

		ListState<Serializable> listState = context.getOperatorStateStore().
				getSerializableListState(DefaultOperatorStateBackend.DEFAULT_OPERATOR_STATE_NAME);

		List<Serializable> list = new ArrayList<>();

		for (Serializable serializable : listState.get()) {
			list.add(serializable);
		}

		try {
			listCheckpointedFun.restoreState(list);
		} catch (Exception e) {

			throw new Exception("Failed to restore state to function: " + e.getMessage(), e);
		}

		return true;
	}

	return false;
}

Example #7

Source File: HdfsSink2.java From sylph with Apache License 2.0

5 votes

@Override
public void snapshotState(FunctionSnapshotContext context)
        throws Exception
{
    for (RichSinkFunction<T> sink : sinks) {
        if (sink instanceof CheckpointedFunction) {
            ((CheckpointedFunction) sink).snapshotState(context);
        }
    }
}

Example #8

Source File: HdfsSink2.java From sylph with Apache License 2.0

5 votes

@Override
public void initializeState(FunctionInitializationContext context)
        throws Exception
{
    for (RichSinkFunction<T> sink : sinks) {
        if (sink instanceof CheckpointedFunction) {
            ((CheckpointedFunction) sink).initializeState(context);
        }
    }
}

Example #9

Source File: AbstractUdfStreamOperator.java From flink with Apache License 2.0

5 votes

private void checkUdfCheckpointingPreconditions() {

		if (userFunction instanceof CheckpointedFunction
			&& userFunction instanceof ListCheckpointed) {

			throw new IllegalStateException("User functions are not allowed to implement " +
				"CheckpointedFunction AND ListCheckpointed.");
		}
	}

Example #10

Source File: StreamingFunctionUtils.java From flink with Apache License 2.0

5 votes

private static boolean tryRestoreFunction(
		StateInitializationContext context,
		Function userFunction) throws Exception {

	if (userFunction instanceof CheckpointedFunction) {
		((CheckpointedFunction) userFunction).initializeState(context);

		return true;
	}

	if (context.isRestored() && userFunction instanceof ListCheckpointed) {
		@SuppressWarnings("unchecked")
		ListCheckpointed<Serializable> listCheckpointedFun = (ListCheckpointed<Serializable>) userFunction;

		// We are using JavaSerializer from the flink-runtime module here. This is very naughty and
		// we shouldn't be doing it because ideally nothing in the API modules/connector depends
		// directly on flink-runtime. We are doing it here because we need to maintain backwards
		// compatibility with old state and because we will have to rework/remove this code soon.
		ListStateDescriptor<Serializable> listStateDescriptor =
			new ListStateDescriptor<>(DefaultOperatorStateBackend.DEFAULT_OPERATOR_STATE_NAME, new JavaSerializer<>());
		ListState<Serializable> listState = context.getOperatorStateStore().getListState(listStateDescriptor);

		List<Serializable> list = new ArrayList<>();

		for (Serializable serializable : listState.get()) {
			list.add(serializable);
		}

		try {
			listCheckpointedFun.restoreState(list);
		} catch (Exception e) {

			throw new Exception("Failed to restore state to function: " + e.getMessage(), e);
		}

		return true;
	}

	return false;
}

Example #11

Source File: StreamingFunctionUtils.java From flink with Apache License 2.0

4 votes

private static boolean trySnapshotFunctionState(
		StateSnapshotContext context,
		OperatorStateBackend backend,
		Function userFunction) throws Exception {

	if (userFunction instanceof CheckpointedFunction) {
		((CheckpointedFunction) userFunction).snapshotState(context);

		return true;
	}

	if (userFunction instanceof ListCheckpointed) {
		@SuppressWarnings("unchecked")
		List<Serializable> partitionableState = ((ListCheckpointed<Serializable>) userFunction).
				snapshotState(context.getCheckpointId(), context.getCheckpointTimestamp());

		// We are using JavaSerializer from the flink-runtime module here. This is very naughty and
		// we shouldn't be doing it because ideally nothing in the API modules/connector depends
		// directly on flink-runtime. We are doing it here because we need to maintain backwards
		// compatibility with old state and because we will have to rework/remove this code soon.
		ListStateDescriptor<Serializable> listStateDescriptor =
			new ListStateDescriptor<>(DefaultOperatorStateBackend.DEFAULT_OPERATOR_STATE_NAME, new JavaSerializer<>());
		ListState<Serializable> listState = backend.getListState(listStateDescriptor);

		listState.clear();

		if (null != partitionableState) {
			try {
				for (Serializable statePartition : partitionableState) {
					listState.add(statePartition);
				}
			} catch (Exception e) {
				listState.clear();

				throw new Exception("Could not write partitionable state to operator " +
					"state backend.", e);
			}
		}

		return true;
	}

	return false;
}