Java Code Examples for org.apache.flink.api.common.state.ListState#clear()

The following examples show how to use org.apache.flink.api.common.state.ListState#clear() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: StateBackendTestBase.java From Flink-CEPplus with Apache License 2.0

6 votes

/**
 * Verify that an empty {@code ListState} yields {@code null}.
 */
@Test
public void testListStateDefaultValue() throws Exception {
	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);

	ListStateDescriptor<String> kvId = new ListStateDescriptor<>("id", String.class);

	ListState<String> state = backend.getPartitionedState(
			VoidNamespace.INSTANCE,
			VoidNamespaceSerializer.INSTANCE, kvId);

	backend.setCurrentKey(1);
	assertNull(state.get());

	state.update(Arrays.asList("Ciao", "Bello"));
	assertThat(state.get(), containsInAnyOrder("Ciao", "Bello"));

	state.clear();
	assertNull(state.get());

	backend.dispose();
}

Example 2

Source File: FlinkBroadcastStateInternals.java From beam with Apache License 2.0

6 votes

/** Update map(namespce->T) from index 0. */
void updateMap(Map<String, T> map) throws Exception {
  if (indexInSubtaskGroup == 0) {
    ListState<Map<String, T>> state = flinkStateBackend.getUnionListState(flinkStateDescriptor);
    state.clear();
    if (map.size() > 0) {
      state.add(map);
    }
  } else {
    if (map.isEmpty()) {
      stateForNonZeroOperator.remove(name);
      // updateMap is always behind getMap,
      // getMap will clear map in BroadcastOperatorState,
      // we don't need clear here.
    } else {
      stateForNonZeroOperator.put(name, map);
    }
  }
}

Example 3

Source File: Buckets.java From flink with Apache License 2.0

6 votes

void snapshotState(
		final long checkpointId,
		final ListState<byte[]> bucketStatesContainer,
		final ListState<Long> partCounterStateContainer) throws Exception {

	Preconditions.checkState(
			fsWriter != null && bucketStateSerializer != null,
			"sink has not been initialized");

	LOG.info("Subtask {} checkpointing for checkpoint with id={} (max part counter={}).",
			subtaskIndex, checkpointId, maxPartCounter);

	bucketStatesContainer.clear();
	partCounterStateContainer.clear();

	snapshotActiveBuckets(checkpointId, bucketStatesContainer);
	partCounterStateContainer.add(maxPartCounter);
}

Example 4

Source File: AsyncWaitOperator.java From flink with Apache License 2.0

6 votes

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
	super.snapshotState(context);

	ListState<StreamElement> partitionableState =
		getOperatorStateBackend().getListState(new ListStateDescriptor<>(STATE_NAME, inStreamElementSerializer));
	partitionableState.clear();

	Collection<StreamElementQueueEntry<?>> values = queue.values();

	try {
		for (StreamElementQueueEntry<?> value : values) {
			partitionableState.add(value.getStreamElement());
		}

		// add the pending stream element queue entry if the stream element queue is currently full
		if (pendingStreamElementQueueEntry != null) {
			partitionableState.add(pendingStreamElementQueueEntry.getStreamElement());
		}
	} catch (Exception e) {
		partitionableState.clear();

		throw new Exception("Could not add stream element queue entries to operator state " +
			"backend of operator " + getOperatorName() + '.', e);
	}
}

Example 5

Source File: StateBackendTestBase.java From flink with Apache License 2.0

6 votes

/**
 * Verify that an empty {@code ListState} yields {@code null}.
 */
@Test
public void testListStateDefaultValue() throws Exception {
	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);

	ListStateDescriptor<String> kvId = new ListStateDescriptor<>("id", String.class);

	ListState<String> state = backend.getPartitionedState(
			VoidNamespace.INSTANCE,
			VoidNamespaceSerializer.INSTANCE, kvId);

	backend.setCurrentKey(1);
	assertNull(state.get());

	state.update(Arrays.asList("Ciao", "Bello"));
	assertThat(state.get(), containsInAnyOrder("Ciao", "Bello"));

	state.clear();
	assertNull(state.get());

	backend.dispose();
}

Example 6

Source File: StateBackendTestBase.java From flink with Apache License 2.0

6 votes

/**
 * Verify that an empty {@code ListState} yields {@code null}.
 */
@Test
public void testListStateDefaultValue() throws Exception {
	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);

	ListStateDescriptor<String> kvId = new ListStateDescriptor<>("id", String.class);

	ListState<String> state = backend.getPartitionedState(
			VoidNamespace.INSTANCE,
			VoidNamespaceSerializer.INSTANCE, kvId);

	backend.setCurrentKey(1);
	assertNull(state.get());

	state.update(Arrays.asList("Ciao", "Bello"));
	assertThat(state.get(), containsInAnyOrder("Ciao", "Bello"));

	state.clear();
	assertNull(state.get());

	backend.dispose();
}

Example 7

Source File: Buckets.java From flink with Apache License 2.0

6 votes

public void snapshotState(
		final long checkpointId,
		final ListState<byte[]> bucketStatesContainer,
		final ListState<Long> partCounterStateContainer) throws Exception {

	Preconditions.checkState(
		bucketWriter != null && bucketStateSerializer != null,
			"sink has not been initialized");

	LOG.info("Subtask {} checkpointing for checkpoint with id={} (max part counter={}).",
			subtaskIndex, checkpointId, maxPartCounter);

	bucketStatesContainer.clear();
	partCounterStateContainer.clear();

	snapshotActiveBuckets(checkpointId, bucketStatesContainer);
	partCounterStateContainer.add(maxPartCounter);
}

Example 8

Source File: Buckets.java From Flink-CEPplus with Apache License 2.0

6 votes

void snapshotState(
		final long checkpointId,
		final ListState<byte[]> bucketStatesContainer,
		final ListState<Long> partCounterStateContainer) throws Exception {

	Preconditions.checkState(
			fsWriter != null && bucketStateSerializer != null,
			"sink has not been initialized");

	LOG.info("Subtask {} checkpointing for checkpoint with id={} (max part counter={}).",
			subtaskIndex, checkpointId, maxPartCounter);

	bucketStatesContainer.clear();
	partCounterStateContainer.clear();

	snapshotActiveBuckets(checkpointId, bucketStatesContainer);
	partCounterStateContainer.add(maxPartCounter);
}

Example 9

Source File: AsyncWaitOperator.java From flink with Apache License 2.0

6 votes

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
	super.snapshotState(context);

	ListState<StreamElement> partitionableState =
		getOperatorStateBackend().getListState(new ListStateDescriptor<>(STATE_NAME, inStreamElementSerializer));
	partitionableState.clear();

	try {
		partitionableState.addAll(queue.values());
	} catch (Exception e) {
		partitionableState.clear();

		throw new Exception("Could not add stream element queue entries to operator state " +
			"backend of operator " + getOperatorName() + '.', e);
	}
}

Example 10

Source File: FlinkKafkaProducer.java From flink with Apache License 2.0

5 votes

private void migrateNextTransactionalIdHindState(FunctionInitializationContext context) throws Exception {
	ListState<NextTransactionalIdHint> oldNextTransactionalIdHintState = context.getOperatorStateStore().getUnionListState(
		NEXT_TRANSACTIONAL_ID_HINT_DESCRIPTOR);
	nextTransactionalIdHintState = context.getOperatorStateStore().getUnionListState(NEXT_TRANSACTIONAL_ID_HINT_DESCRIPTOR_V2);

	ArrayList<NextTransactionalIdHint> oldTransactionalIdHints = Lists.newArrayList(oldNextTransactionalIdHintState.get());
	if (!oldTransactionalIdHints.isEmpty()) {
		nextTransactionalIdHintState.addAll(oldTransactionalIdHints);
		//clear old state
		oldNextTransactionalIdHintState.clear();
	}
}

Example 11

Source File: StreamingFunctionUtils.java From Flink-CEPplus with Apache License 2.0

5 votes

private static boolean trySnapshotFunctionState(
		StateSnapshotContext context,
		OperatorStateBackend backend,
		Function userFunction) throws Exception {

	if (userFunction instanceof CheckpointedFunction) {
		((CheckpointedFunction) userFunction).snapshotState(context);

		return true;
	}

	if (userFunction instanceof ListCheckpointed) {
		@SuppressWarnings("unchecked")
		List<Serializable> partitionableState = ((ListCheckpointed<Serializable>) userFunction).
				snapshotState(context.getCheckpointId(), context.getCheckpointTimestamp());

		ListState<Serializable> listState = backend.
				getSerializableListState(DefaultOperatorStateBackend.DEFAULT_OPERATOR_STATE_NAME);

		listState.clear();

		if (null != partitionableState) {
			try {
				for (Serializable statePartition : partitionableState) {
					listState.add(statePartition);
				}
			} catch (Exception e) {
				listState.clear();

				throw new Exception("Could not write partitionable state to operator " +
					"state backend.", e);
			}
		}

		return true;
	}

	return false;
}

Example 12

Source File: EvictingWindowOperator.java From Flink-CEPplus with Apache License 2.0

5 votes

private void clearAllState(
		W window,
		ListState<StreamRecord<IN>> windowState,
		MergingWindowSet<W> mergingWindows) throws Exception {
	windowState.clear();
	triggerContext.clear();
	processContext.window = window;
	processContext.clear();
	if (mergingWindows != null) {
		mergingWindows.retireWindow(window);
		mergingWindows.persist();
	}
}

Example 13

Source File: EvictingWindowOperator.java From flink with Apache License 2.0

5 votes

private void clearAllState(
		W window,
		ListState<StreamRecord<IN>> windowState,
		MergingWindowSet<W> mergingWindows) throws Exception {
	windowState.clear();
	triggerContext.clear();
	processContext.window = window;
	processContext.clear();
	if (mergingWindows != null) {
		mergingWindows.retireWindow(window);
		mergingWindows.persist();
	}
}

Example 14

Source File: OneInputStreamTaskTest.java From flink with Apache License 2.0

5 votes

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
	ListState<Integer> partitionableState =
		getOperatorStateBackend().getListState(TEST_DESCRIPTOR);
	partitionableState.clear();

	partitionableState.add(42);
	partitionableState.add(4711);

	++numberSnapshotCalls;
}

Example 15

Source File: OneInputStreamTaskTest.java From flink with Apache License 2.0

5 votes

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
	ListState<Integer> partitionableState =
		getOperatorStateBackend().getListState(TEST_DESCRIPTOR);
	partitionableState.clear();

	partitionableState.add(42);
	partitionableState.add(4711);

	++numberSnapshotCalls;
}

Example 16

Source File: FlinkKafkaProducer.java From flink with Apache License 2.0

5 votes

private void migrateNextTransactionalIdHindState(FunctionInitializationContext context) throws Exception {
	ListState<NextTransactionalIdHint> oldNextTransactionalIdHintState = context.getOperatorStateStore().getUnionListState(
		NEXT_TRANSACTIONAL_ID_HINT_DESCRIPTOR);
	nextTransactionalIdHintState = context.getOperatorStateStore().getUnionListState(NEXT_TRANSACTIONAL_ID_HINT_DESCRIPTOR_V2);

	ArrayList<NextTransactionalIdHint> oldTransactionalIdHints = Lists.newArrayList(oldNextTransactionalIdHintState.get());
	if (!oldTransactionalIdHints.isEmpty()) {
		nextTransactionalIdHintState.addAll(oldTransactionalIdHints);
		//clear old state
		oldNextTransactionalIdHintState.clear();
	}
}

Example 17

Source File: FlinkKafkaProducer011.java From flink with Apache License 2.0

5 votes

private void migrateNextTransactionalIdHindState(FunctionInitializationContext context) throws Exception {
	ListState<NextTransactionalIdHint> oldNextTransactionalIdHintState = context.getOperatorStateStore().getUnionListState(
		NEXT_TRANSACTIONAL_ID_HINT_DESCRIPTOR);
	nextTransactionalIdHintState = context.getOperatorStateStore().getUnionListState(NEXT_TRANSACTIONAL_ID_HINT_DESCRIPTOR_V2);

	ArrayList<NextTransactionalIdHint> oldTransactionalIdHints = Lists.newArrayList(oldNextTransactionalIdHintState.get());
	if (!oldTransactionalIdHints.isEmpty()) {
		nextTransactionalIdHintState.addAll(oldTransactionalIdHints);
		//clear old state
		oldNextTransactionalIdHintState.clear();
	}
}

Example 18

Source File: StreamingFunctionUtils.java From flink with Apache License 2.0

4 votes

private static boolean trySnapshotFunctionState(
		StateSnapshotContext context,
		OperatorStateBackend backend,
		Function userFunction) throws Exception {

	if (userFunction instanceof CheckpointedFunction) {
		((CheckpointedFunction) userFunction).snapshotState(context);

		return true;
	}

	if (userFunction instanceof ListCheckpointed) {
		@SuppressWarnings("unchecked")
		List<Serializable> partitionableState = ((ListCheckpointed<Serializable>) userFunction).
				snapshotState(context.getCheckpointId(), context.getCheckpointTimestamp());

		// We are using JavaSerializer from the flink-runtime module here. This is very naughty and
		// we shouldn't be doing it because ideally nothing in the API modules/connector depends
		// directly on flink-runtime. We are doing it here because we need to maintain backwards
		// compatibility with old state and because we will have to rework/remove this code soon.
		ListStateDescriptor<Serializable> listStateDescriptor =
			new ListStateDescriptor<>(DefaultOperatorStateBackend.DEFAULT_OPERATOR_STATE_NAME, new JavaSerializer<>());
		ListState<Serializable> listState = backend.getListState(listStateDescriptor);

		listState.clear();

		if (null != partitionableState) {
			try {
				for (Serializable statePartition : partitionableState) {
					listState.add(statePartition);
				}
			} catch (Exception e) {
				listState.clear();

				throw new Exception("Could not write partitionable state to operator " +
					"state backend.", e);
			}
		}

		return true;
	}

	return false;
}

Example 19

Source File: StateBackendTestBase.java From Flink-CEPplus with Apache License 2.0

4 votes

@Test
public void testListStateAPIs() throws Exception {

	AbstractKeyedStateBackend<String> keyedBackend = createKeyedBackend(StringSerializer.INSTANCE);

	final ListStateDescriptor<Long> stateDescr = new ListStateDescriptor<>("my-state", Long.class);

	try {
		ListState<Long> state =
			keyedBackend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, stateDescr);

		keyedBackend.setCurrentKey("def");
		assertNull(state.get());
		state.add(17L);
		state.add(11L);
		assertThat(state.get(), containsInAnyOrder(17L, 11L));
		// update(emptyList) should remain the value null
		state.update(Collections.emptyList());
		assertNull(state.get());
		state.update(Arrays.asList(10L, 16L));
		assertThat(state.get(), containsInAnyOrder(16L, 10L));
		assertThat(state.get(), containsInAnyOrder(16L, 10L));

		keyedBackend.setCurrentKey("abc");
		assertNull(state.get());

		keyedBackend.setCurrentKey("g");
		assertNull(state.get());
		assertNull(state.get());
		state.addAll(Collections.emptyList());
		assertNull(state.get());
		state.addAll(Arrays.asList(3L, 4L));
		assertThat(state.get(), containsInAnyOrder(3L, 4L));
		assertThat(state.get(), containsInAnyOrder(3L, 4L));
		state.addAll(new ArrayList<>());
		assertThat(state.get(), containsInAnyOrder(3L, 4L));
		state.addAll(Arrays.asList(5L, 6L));
		assertThat(state.get(), containsInAnyOrder(3L, 4L, 5L, 6L));
		state.addAll(new ArrayList<>());
		assertThat(state.get(), containsInAnyOrder(3L, 4L, 5L, 6L));

		assertThat(state.get(), containsInAnyOrder(3L, 4L, 5L, 6L));
		state.update(Arrays.asList(1L, 2L));
		assertThat(state.get(), containsInAnyOrder(1L, 2L));

		keyedBackend.setCurrentKey("def");
		assertThat(state.get(), containsInAnyOrder(10L, 16L));
		state.clear();
		assertNull(state.get());

		keyedBackend.setCurrentKey("g");
		state.add(3L);
		state.add(2L);
		state.add(1L);

		keyedBackend.setCurrentKey("def");
		assertNull(state.get());

		keyedBackend.setCurrentKey("g");
		assertThat(state.get(), containsInAnyOrder(1L, 2L, 3L, 2L, 1L));
		state.update(Arrays.asList(5L, 6L));
		assertThat(state.get(), containsInAnyOrder(5L, 6L));
		state.clear();

		// make sure all lists / maps are cleared
		assertThat("State backend is not empty.", keyedBackend.numKeyValueStateEntries(), is(0));
	} finally {
		keyedBackend.close();
		keyedBackend.dispose();
	}
}

Example 20

Source File: StateBackendTestBase.java From flink with Apache License 2.0

4 votes

@Test
public void testListStateAPIs() throws Exception {

	AbstractKeyedStateBackend<String> keyedBackend = createKeyedBackend(StringSerializer.INSTANCE);

	final ListStateDescriptor<Long> stateDescr = new ListStateDescriptor<>("my-state", Long.class);

	try {
		ListState<Long> state =
			keyedBackend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, stateDescr);

		keyedBackend.setCurrentKey("def");
		assertNull(state.get());
		state.add(17L);
		state.add(11L);
		assertThat(state.get(), containsInAnyOrder(17L, 11L));
		// update(emptyList) should remain the value null
		state.update(Collections.emptyList());
		assertNull(state.get());
		state.update(Arrays.asList(10L, 16L));
		assertThat(state.get(), containsInAnyOrder(16L, 10L));
		assertThat(state.get(), containsInAnyOrder(16L, 10L));

		keyedBackend.setCurrentKey("abc");
		assertNull(state.get());

		keyedBackend.setCurrentKey("g");
		assertNull(state.get());
		assertNull(state.get());
		state.addAll(Collections.emptyList());
		assertNull(state.get());
		state.addAll(Arrays.asList(3L, 4L));
		assertThat(state.get(), containsInAnyOrder(3L, 4L));
		assertThat(state.get(), containsInAnyOrder(3L, 4L));
		state.addAll(new ArrayList<>());
		assertThat(state.get(), containsInAnyOrder(3L, 4L));
		state.addAll(Arrays.asList(5L, 6L));
		assertThat(state.get(), containsInAnyOrder(3L, 4L, 5L, 6L));
		state.addAll(new ArrayList<>());
		assertThat(state.get(), containsInAnyOrder(3L, 4L, 5L, 6L));

		assertThat(state.get(), containsInAnyOrder(3L, 4L, 5L, 6L));
		state.update(Arrays.asList(1L, 2L));
		assertThat(state.get(), containsInAnyOrder(1L, 2L));

		keyedBackend.setCurrentKey("def");
		assertThat(state.get(), containsInAnyOrder(10L, 16L));
		state.clear();
		assertNull(state.get());

		keyedBackend.setCurrentKey("g");
		state.add(3L);
		state.add(2L);
		state.add(1L);

		keyedBackend.setCurrentKey("def");
		assertNull(state.get());

		keyedBackend.setCurrentKey("g");
		assertThat(state.get(), containsInAnyOrder(1L, 2L, 3L, 2L, 1L));
		state.update(Arrays.asList(5L, 6L));
		assertThat(state.get(), containsInAnyOrder(5L, 6L));
		state.clear();

		// make sure all lists / maps are cleared
		assertThat("State backend is not empty.", keyedBackend.numKeyValueStateEntries(), is(0));
	} finally {
		keyedBackend.close();
		keyedBackend.dispose();
	}
}