org.apache.flink.runtime.state.StateSnapshotContext Java Examples

The following examples show how to use org.apache.flink.runtime.state.StateSnapshotContext. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: AsyncWaitOperator.java From Flink-CEPplus with Apache License 2.0

6 votes

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
	super.snapshotState(context);

	ListState<StreamElement> partitionableState =
		getOperatorStateBackend().getListState(new ListStateDescriptor<>(STATE_NAME, inStreamElementSerializer));
	partitionableState.clear();

	Collection<StreamElementQueueEntry<?>> values = queue.values();

	try {
		for (StreamElementQueueEntry<?> value : values) {
			partitionableState.add(value.getStreamElement());
		}

		// add the pending stream element queue entry if the stream element queue is currently full
		if (pendingStreamElementQueueEntry != null) {
			partitionableState.add(pendingStreamElementQueueEntry.getStreamElement());
		}
	} catch (Exception e) {
		partitionableState.clear();

		throw new Exception("Could not add stream element queue entries to operator state " +
			"backend of operator " + getOperatorName() + '.', e);
	}
}

Example #2

Source File: StreamingFunctionUtils.java From Flink-CEPplus with Apache License 2.0

6 votes

public static void snapshotFunctionState(
		StateSnapshotContext context,
		OperatorStateBackend backend,
		Function userFunction) throws Exception {

	Preconditions.checkNotNull(context);
	Preconditions.checkNotNull(backend);

	while (true) {

		if (trySnapshotFunctionState(context, backend, userFunction)) {
			break;
		}

		// inspect if the user function is wrapped, then unwrap and try again if we can snapshot the inner function
		if (userFunction instanceof WrappingFunction) {
			userFunction = ((WrappingFunction<?>) userFunction).getWrappedFunction();
		} else {
			break;
		}
	}
}

Example #3

Source File: AsyncWaitOperator.java From flink with Apache License 2.0

6 votes

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
	super.snapshotState(context);

	ListState<StreamElement> partitionableState =
		getOperatorStateBackend().getListState(new ListStateDescriptor<>(STATE_NAME, inStreamElementSerializer));
	partitionableState.clear();

	try {
		partitionableState.addAll(queue.values());
	} catch (Exception e) {
		partitionableState.clear();

		throw new Exception("Could not add stream element queue entries to operator state " +
			"backend of operator " + getOperatorName() + '.', e);
	}
}

Example #4

Source File: StreamingFunctionUtils.java From flink with Apache License 2.0

6 votes

public static void snapshotFunctionState(
		StateSnapshotContext context,
		OperatorStateBackend backend,
		Function userFunction) throws Exception {

	Preconditions.checkNotNull(context);
	Preconditions.checkNotNull(backend);

	while (true) {

		if (trySnapshotFunctionState(context, backend, userFunction)) {
			break;
		}

		// inspect if the user function is wrapped, then unwrap and try again if we can snapshot the inner function
		if (userFunction instanceof WrappingFunction) {
			userFunction = ((WrappingFunction<?>) userFunction).getWrappedFunction();
		} else {
			break;
		}
	}
}

Example #5

Source File: StreamingFunctionUtils.java From flink with Apache License 2.0

6 votes

public static void snapshotFunctionState(
		StateSnapshotContext context,
		OperatorStateBackend backend,
		Function userFunction) throws Exception {

	Preconditions.checkNotNull(context);
	Preconditions.checkNotNull(backend);

	while (true) {

		if (trySnapshotFunctionState(context, backend, userFunction)) {
			break;
		}

		// inspect if the user function is wrapped, then unwrap and try again if we can snapshot the inner function
		if (userFunction instanceof WrappingFunction) {
			userFunction = ((WrappingFunction<?>) userFunction).getWrappedFunction();
		} else {
			break;
		}
	}
}

Example #6

Source File: AsyncWaitOperator.java From flink with Apache License 2.0

6 votes

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
	super.snapshotState(context);

	ListState<StreamElement> partitionableState =
		getOperatorStateBackend().getListState(new ListStateDescriptor<>(STATE_NAME, inStreamElementSerializer));
	partitionableState.clear();

	Collection<StreamElementQueueEntry<?>> values = queue.values();

	try {
		for (StreamElementQueueEntry<?> value : values) {
			partitionableState.add(value.getStreamElement());
		}

		// add the pending stream element queue entry if the stream element queue is currently full
		if (pendingStreamElementQueueEntry != null) {
			partitionableState.add(pendingStreamElementQueueEntry.getStreamElement());
		}
	} catch (Exception e) {
		partitionableState.clear();

		throw new Exception("Could not add stream element queue entries to operator state " +
			"backend of operator " + getOperatorName() + '.', e);
	}
}

Example #7

Source File: ContinuousFileReaderOperator.java From flink with Apache License 2.0

5 votes

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
	super.snapshotState(context);

	checkState(checkpointedState != null,
		"The operator state has not been properly initialized.");

	int subtaskIdx = getRuntimeContext().getIndexOfThisSubtask();

	checkpointedState.clear();

	List<TimestampedFileInputSplit> readerState = reader.getReaderState();

	try {
		for (TimestampedFileInputSplit split : readerState) {
			// create a new partition for each entry.
			checkpointedState.add(split);
		}
	} catch (Exception e) {
		checkpointedState.clear();

		throw new Exception("Could not add timestamped file input splits to to operator " +
			"state backend of operator " + getOperatorName() + '.', e);
	}

	if (LOG.isDebugEnabled()) {
		LOG.debug("{} (taskIdx={}) checkpointed {} splits: {}.",
			getClass().getSimpleName(), subtaskIdx, readerState.size(), readerState);
	}
}

Example #8

Source File: StreamSortOperator.java From flink with Apache License 2.0

5 votes

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
	super.snapshotState(context);
	// clear state first
	bufferState.clear();

	List<Tuple2<RowData, Long>> dataToFlush = new ArrayList<>(inputBuffer.size());
	inputBuffer.forEach((key, value) -> dataToFlush.add(Tuple2.of(key, value)));

	// batch put
	bufferState.addAll(dataToFlush);
}

Example #9

Source File: DoFnOperator.java From beam with Apache License 2.0

5 votes

@Override
public final void snapshotState(StateSnapshotContext context) throws Exception {
  if (checkpointStats != null) {
    checkpointStats.snapshotStart(context.getCheckpointId());
  }

  if (requiresStableInput) {
    // We notify the BufferingDoFnRunner to associate buffered state with this
    // snapshot id and start a new buffer for elements arriving after this snapshot.
    bufferingDoFnRunner.checkpoint(context.getCheckpointId());
  }

  try {
    outputManager.openBuffer();
    // Ensure that no new bundle gets started as part of finishing a bundle
    while (bundleStarted) {
      invokeFinishBundle();
    }
    outputManager.closeBuffer();
  } catch (Exception e) {
    // https://jira.apache.org/jira/browse/FLINK-14653
    // Any regular exception during checkpointing will be tolerated by Flink because those
    // typically do not affect the execution flow. We need to fail hard here because errors
    // in bundle execution are application errors which are not related to checkpointing.
    throw new Error("Checkpointing failed because bundle failed to finalize.", e);
  }

  super.snapshotState(context);
}

Example #10

Source File: NotifyCheckpointAbortedITCase.java From flink with Apache License 2.0

5 votes

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
	if (context.getCheckpointId() == DECLINE_CHECKPOINT_ID) {
		DeclineSink.waitLatch.await();
	}
	super.snapshotState(context);
}

Example #11

Source File: TaskCheckpointingBehaviourTest.java From flink with Apache License 2.0

5 votes

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
	OperatorStateCheckpointOutputStream outStream = context.getRawOperatorStateOutput();

	IN_CHECKPOINT_LATCH.trigger();

	// this should lock
	outStream.write(1);
}

Example #12

Source File: OneInputStreamTaskTest.java From flink with Apache License 2.0

5 votes

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
	ListState<Integer> partitionableState =
		getOperatorStateBackend().getListState(TEST_DESCRIPTOR);
	partitionableState.clear();

	partitionableState.add(42);
	partitionableState.add(4711);

	++numberSnapshotCalls;
}

Example #13

Source File: ContinuousFileReaderOperator.java From flink with Apache License 2.0

5 votes

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
	super.snapshotState(context);

	checkState(checkpointedState != null,
		"The operator state has not been properly initialized.");

	int subtaskIdx = getRuntimeContext().getIndexOfThisSubtask();

	checkpointedState.clear();

	List<T> readerState = getReaderState();

	try {
		for (T split : readerState) {
			checkpointedState.add(split);
		}
	} catch (Exception e) {
		checkpointedState.clear();

		throw new Exception("Could not add timestamped file input splits to to operator " +
				"state backend of operator " + getOperatorName() + '.', e);
	}

	if (LOG.isDebugEnabled()) {
		LOG.debug("{} (taskIdx={}) checkpointed {} splits: {}.",
				getClass().getSimpleName(), subtaskIdx, readerState.size(), readerState);
	}
}

Example #14

Source File: GenericWriteAheadSink.java From flink with Apache License 2.0

5 votes

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
	super.snapshotState(context);

	Preconditions.checkState(this.checkpointedState != null,
		"The operator state has not been properly initialized.");

	saveHandleInState(context.getCheckpointId(), context.getCheckpointTimestamp());

	this.checkpointedState.clear();

	try {
		for (PendingCheckpoint pendingCheckpoint : pendingCheckpoints) {
			// create a new partition for each entry.
			this.checkpointedState.add(pendingCheckpoint);
		}
	} catch (Exception e) {
		checkpointedState.clear();

		throw new Exception("Could not add panding checkpoints to operator state " +
			"backend of operator " + getOperatorName() + '.', e);
	}

	int subtaskIdx = getRuntimeContext().getIndexOfThisSubtask();
	if (LOG.isDebugEnabled()) {
		LOG.debug("{} (taskIdx= {}) checkpointed {}.", getClass().getSimpleName(), subtaskIdx, this.pendingCheckpoints);
	}
}

Example #15

Source File: StreamingFunctionUtils.java From flink with Apache License 2.0

5 votes

private static boolean trySnapshotFunctionState(
		StateSnapshotContext context,
		OperatorStateBackend backend,
		Function userFunction) throws Exception {

	if (userFunction instanceof CheckpointedFunction) {
		((CheckpointedFunction) userFunction).snapshotState(context);

		return true;
	}

	if (userFunction instanceof ListCheckpointed) {
		@SuppressWarnings("unchecked")
		List<Serializable> partitionableState = ((ListCheckpointed<Serializable>) userFunction).
				snapshotState(context.getCheckpointId(), context.getCheckpointTimestamp());

		ListState<Serializable> listState = backend.
				getSerializableListState(DefaultOperatorStateBackend.DEFAULT_OPERATOR_STATE_NAME);

		listState.clear();

		if (null != partitionableState) {
			try {
				for (Serializable statePartition : partitionableState) {
					listState.add(statePartition);
				}
			} catch (Exception e) {
				listState.clear();

				throw new Exception("Could not write partitionable state to operator " +
					"state backend.", e);
			}
		}

		return true;
	}

	return false;
}

Example #16

Source File: GenericWriteAheadSink.java From flink with Apache License 2.0

5 votes

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
	super.snapshotState(context);

	Preconditions.checkState(this.checkpointedState != null,
		"The operator state has not been properly initialized.");

	saveHandleInState(context.getCheckpointId(), context.getCheckpointTimestamp());

	this.checkpointedState.clear();

	try {
		for (PendingCheckpoint pendingCheckpoint : pendingCheckpoints) {
			// create a new partition for each entry.
			this.checkpointedState.add(pendingCheckpoint);
		}
	} catch (Exception e) {
		checkpointedState.clear();

		throw new Exception("Could not add panding checkpoints to operator state " +
			"backend of operator " + getOperatorName() + '.', e);
	}

	int subtaskIdx = getRuntimeContext().getIndexOfThisSubtask();
	if (LOG.isDebugEnabled()) {
		LOG.debug("{} (taskIdx= {}) checkpointed {}.", getClass().getSimpleName(), subtaskIdx, this.pendingCheckpoints);
	}
}

Example #17

Source File: StreamSortOperator.java From flink with Apache License 2.0

5 votes

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
	super.snapshotState(context);
	// clear state first
	bufferState.clear();

	List<Tuple2<BaseRow, Long>> dataToFlush = new ArrayList<>(inputBuffer.size());
	inputBuffer.forEach((key, value) -> dataToFlush.add(Tuple2.of(key, value)));

	// batch put
	bufferState.addAll(dataToFlush);
}

Example #18

Source File: TaskCheckpointingBehaviourTest.java From Flink-CEPplus with Apache License 2.0

5 votes

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
	OperatorStateCheckpointOutputStream outStream = context.getRawOperatorStateOutput();

	IN_CHECKPOINT_LATCH.trigger();

	// this should lock
	outStream.write(1);
}

Example #19

Source File: OneInputStreamTaskTest.java From Flink-CEPplus with Apache License 2.0

5 votes

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
	ListState<Integer> partitionableState =
		getOperatorStateBackend().getListState(TEST_DESCRIPTOR);
	partitionableState.clear();

	partitionableState.add(42);
	partitionableState.add(4711);

	++numberSnapshotCalls;
}

Example #20

Source File: OneInputStreamTaskTest.java From flink with Apache License 2.0

5 votes

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
	ListState<Integer> partitionableState =
		getOperatorStateBackend().getListState(TEST_DESCRIPTOR);
	partitionableState.clear();

	partitionableState.add(42);
	partitionableState.add(4711);

	++numberSnapshotCalls;
}

Example #21

Source File: StreamingFunctionUtils.java From Flink-CEPplus with Apache License 2.0

5 votes

private static boolean trySnapshotFunctionState(
		StateSnapshotContext context,
		OperatorStateBackend backend,
		Function userFunction) throws Exception {

	if (userFunction instanceof CheckpointedFunction) {
		((CheckpointedFunction) userFunction).snapshotState(context);

		return true;
	}

	if (userFunction instanceof ListCheckpointed) {
		@SuppressWarnings("unchecked")
		List<Serializable> partitionableState = ((ListCheckpointed<Serializable>) userFunction).
				snapshotState(context.getCheckpointId(), context.getCheckpointTimestamp());

		ListState<Serializable> listState = backend.
				getSerializableListState(DefaultOperatorStateBackend.DEFAULT_OPERATOR_STATE_NAME);

		listState.clear();

		if (null != partitionableState) {
			try {
				for (Serializable statePartition : partitionableState) {
					listState.add(statePartition);
				}
			} catch (Exception e) {
				listState.clear();

				throw new Exception("Could not write partitionable state to operator " +
					"state backend.", e);
			}
		}

		return true;
	}

	return false;
}

Example #22

Source File: GenericWriteAheadSink.java From Flink-CEPplus with Apache License 2.0

5 votes

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
	super.snapshotState(context);

	Preconditions.checkState(this.checkpointedState != null,
		"The operator state has not been properly initialized.");

	saveHandleInState(context.getCheckpointId(), context.getCheckpointTimestamp());

	this.checkpointedState.clear();

	try {
		for (PendingCheckpoint pendingCheckpoint : pendingCheckpoints) {
			// create a new partition for each entry.
			this.checkpointedState.add(pendingCheckpoint);
		}
	} catch (Exception e) {
		checkpointedState.clear();

		throw new Exception("Could not add panding checkpoints to operator state " +
			"backend of operator " + getOperatorName() + '.', e);
	}

	int subtaskIdx = getRuntimeContext().getIndexOfThisSubtask();
	if (LOG.isDebugEnabled()) {
		LOG.debug("{} (taskIdx= {}) checkpointed {}.", getClass().getSimpleName(), subtaskIdx, this.pendingCheckpoints);
	}
}

Example #23

Source File: ContinuousFileReaderOperator.java From Flink-CEPplus with Apache License 2.0

5 votes

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
	super.snapshotState(context);

	checkState(checkpointedState != null,
		"The operator state has not been properly initialized.");

	int subtaskIdx = getRuntimeContext().getIndexOfThisSubtask();

	checkpointedState.clear();

	List<TimestampedFileInputSplit> readerState = reader.getReaderState();

	try {
		for (TimestampedFileInputSplit split : readerState) {
			// create a new partition for each entry.
			checkpointedState.add(split);
		}
	} catch (Exception e) {
		checkpointedState.clear();

		throw new Exception("Could not add timestamped file input splits to to operator " +
			"state backend of operator " + getOperatorName() + '.', e);
	}

	if (LOG.isDebugEnabled()) {
		LOG.debug("{} (taskIdx={}) checkpointed {} splits: {}.",
			getClass().getSimpleName(), subtaskIdx, readerState.size(), readerState);
	}
}

Example #24

Source File: TaskCheckpointingBehaviourTest.java From flink with Apache License 2.0

5 votes

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
	OperatorStateCheckpointOutputStream outStream = context.getRawOperatorStateOutput();

	IN_CHECKPOINT_LATCH.trigger();

	// this should lock
	outStream.write(1);
}

Example #25

Source File: FeedbackUnionOperator.java From stateful-functions with Apache License 2.0

4 votes

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
  super.snapshotState(context);
  this.feedbackLogger.startLogging(context.getRawKeyedOperatorStateOutput());
}

Example #26

Source File: StreamingFunctionUtils.java From flink with Apache License 2.0

4 votes

private static boolean trySnapshotFunctionState(
		StateSnapshotContext context,
		OperatorStateBackend backend,
		Function userFunction) throws Exception {

	if (userFunction instanceof CheckpointedFunction) {
		((CheckpointedFunction) userFunction).snapshotState(context);

		return true;
	}

	if (userFunction instanceof ListCheckpointed) {
		@SuppressWarnings("unchecked")
		List<Serializable> partitionableState = ((ListCheckpointed<Serializable>) userFunction).
				snapshotState(context.getCheckpointId(), context.getCheckpointTimestamp());

		// We are using JavaSerializer from the flink-runtime module here. This is very naughty and
		// we shouldn't be doing it because ideally nothing in the API modules/connector depends
		// directly on flink-runtime. We are doing it here because we need to maintain backwards
		// compatibility with old state and because we will have to rework/remove this code soon.
		ListStateDescriptor<Serializable> listStateDescriptor =
			new ListStateDescriptor<>(DefaultOperatorStateBackend.DEFAULT_OPERATOR_STATE_NAME, new JavaSerializer<>());
		ListState<Serializable> listState = backend.getListState(listStateDescriptor);

		listState.clear();

		if (null != partitionableState) {
			try {
				for (Serializable statePartition : partitionableState) {
					listState.add(statePartition);
				}
			} catch (Exception e) {
				listState.clear();

				throw new Exception("Could not write partitionable state to operator " +
					"state backend.", e);
			}
		}

		return true;
	}

	return false;
}

Example #27

Source File: AbstractUdfStreamOperatorLifecycleTest.java From flink with Apache License 2.0

4 votes

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
	ACTUAL_ORDER_TRACKING.add("OPERATOR::snapshotState");
	super.snapshotState(context);
}

Example #28

Source File: RestoreStreamTaskTest.java From flink with Apache License 2.0

4 votes

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
	counterState.add(counter);
}

Example #29

Source File: RestoreStreamTaskTest.java From flink with Apache License 2.0

4 votes

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
}

Example #30

Source File: SourceOperator.java From flink with Apache License 2.0

4 votes

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
	LOG.debug("Taking a snapshot for checkpoint {}", context.getCheckpointId());
	readerState.update(sourceReader.snapshotState());
}