Java Code Examples for org.apache.flink.api.common.state.StateDescriptor

The following examples show how to use org.apache.flink.api.common.state.StateDescriptor. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: flink   Source File: HeapKeyedStateBackend.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public <N, S extends State, T> void applyToAllKeys(
	final N namespace,
	final TypeSerializer<N> namespaceSerializer,
	final StateDescriptor<S, T> stateDescriptor,
	final KeyedStateFunction<K, S> function) throws Exception {

	try (Stream<K> keyStream = getKeys(stateDescriptor.getName(), namespace)) {

		// we copy the keys into list to avoid the concurrency problem
		// when state.clear() is invoked in function.process().
		final List<K> keys = keyStream.collect(Collectors.toList());

		final S state = getPartitionedState(
			namespace,
			namespaceSerializer,
			stateDescriptor);

		for (K key : keys) {
			setCurrentKey(key);
			function.process(key, state);
		}
	}
}
 
Example 2
Source Project: flink   Source File: MockKeyedStateBackend.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
private <SV, SEV> StateSnapshotTransformer<SV> getStateSnapshotTransformer(
	StateDescriptor<?, SV> stateDesc,
	StateSnapshotTransformFactory<SEV> snapshotTransformFactory) {
	Optional<StateSnapshotTransformer<SEV>> original = snapshotTransformFactory.createForDeserializedState();
	if (original.isPresent()) {
		if (stateDesc instanceof ListStateDescriptor) {
			return (StateSnapshotTransformer<SV>) new StateSnapshotTransformers.ListStateSnapshotTransformer<>(original.get());
		} else if (stateDesc instanceof MapStateDescriptor) {
			return (StateSnapshotTransformer<SV>) new StateSnapshotTransformers.MapStateSnapshotTransformer<>(original.get());
		} else {
			return (StateSnapshotTransformer<SV>) original.get();
		}
	} else {
		return null;
	}
}
 
Example 3
Source Project: flink   Source File: SerializationProxiesTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testKeyedStateMetaInfoSerialization() throws Exception {

	String name = "test";
	TypeSerializer<?> namespaceSerializer = LongSerializer.INSTANCE;
	TypeSerializer<?> stateSerializer = DoubleSerializer.INSTANCE;

	StateMetaInfoSnapshot metaInfo = new RegisteredKeyValueStateBackendMetaInfo<>(
		StateDescriptor.Type.VALUE, name, namespaceSerializer, stateSerializer).snapshot();

	byte[] serialized;
	try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
		StateMetaInfoSnapshotReadersWriters.getWriter().
			writeStateMetaInfoSnapshot(metaInfo, new DataOutputViewStreamWrapper(out));
		serialized = out.toByteArray();
	}

	try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
		final StateMetaInfoReader reader = StateMetaInfoSnapshotReadersWriters.getReader(
			CURRENT_STATE_META_INFO_SNAPSHOT_VERSION, StateMetaInfoSnapshotReadersWriters.StateTypeHint.KEYED_STATE);
		metaInfo = reader.readStateMetaInfoSnapshot(
			new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader());
	}

	Assert.assertEquals(name, metaInfo.getName());
}
 
Example 4
Source Project: flink   Source File: MultiStateKeyIterator.java    License: Apache License 2.0 6 votes vote down vote up
/** Removes the current key from <b>ALL</b> known states in the state backend. */
@Override
public void remove() {
	if (currentKey == null) {
		return;
	}

	for (StateDescriptor<?, ?> descriptor : descriptors) {
		try {
			State state = backend.getPartitionedState(
				VoidNamespace.INSTANCE,
				VoidNamespaceSerializer.INSTANCE,
				descriptor);

			state.clear();
		} catch (Exception e) {
			throw new RuntimeException("Failed to drop partitioned state from state backend", e);
		}
	}
}
 
Example 5
Source Project: Flink-CEPplus   Source File: HeapKeyedStateBackend.java    License: Apache License 2.0 6 votes vote down vote up
@Override
@Nonnull
public <N, SV, SEV, S extends State, IS extends S> IS createInternalState(
	@Nonnull TypeSerializer<N> namespaceSerializer,
	@Nonnull StateDescriptor<S, SV> stateDesc,
	@Nonnull StateSnapshotTransformFactory<SEV> snapshotTransformFactory) throws Exception {
	StateFactory stateFactory = STATE_FACTORIES.get(stateDesc.getClass());
	if (stateFactory == null) {
		String message = String.format("State %s is not supported by %s",
			stateDesc.getClass(), this.getClass());
		throw new FlinkRuntimeException(message);
	}
	StateTable<K, N, SV> stateTable = tryRegisterStateTable(
		namespaceSerializer, stateDesc, getStateSnapshotTransformFactory(stateDesc, snapshotTransformFactory));
	return stateFactory.createState(stateDesc, stateTable, getKeySerializer());
}
 
Example 6
Source Project: flink   Source File: MultiStateKeyIteratorTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testIteratorRemovesFromAllDescriptors() throws Exception {
	AbstractKeyedStateBackend<Integer> keyedStateBackend = createKeyedStateBackend();

	setKey(keyedStateBackend, descriptors.get(0), 1);
	setKey(keyedStateBackend, descriptors.get(1), 1);

	MultiStateKeyIterator<Integer> iterator =
		new MultiStateKeyIterator<>(descriptors, keyedStateBackend);

	int key = iterator.next();
	Assert.assertEquals("Unexpected keys pulled from state backend", 1, key);

	iterator.remove();
	Assert.assertFalse("Failed to drop key from all descriptors in state backend", iterator.hasNext());

	for (StateDescriptor<?, ?> descriptor : descriptors) {
		Assert.assertEquals(
			"Failed to drop key for state descriptor",
			0,
			keyedStateBackend.getKeys(descriptor.getName(), VoidNamespace.INSTANCE).count());
	}
}
 
Example 7
Source Project: flink   Source File: StateDescriptorPassingTest.java    License: Apache License 2.0 6 votes vote down vote up
private void validateListStateDescriptorConfigured(SingleOutputStreamOperator<?> result) {
	OneInputTransformation<?, ?> transform = (OneInputTransformation<?, ?>) result.getTransformation();
	WindowOperator<?, ?, ?, ?, ?> op = (WindowOperator<?, ?, ?, ?, ?>) transform.getOperator();
	StateDescriptor<?, ?> descr = op.getStateDescriptor();

	assertTrue(descr instanceof ListStateDescriptor);

	ListStateDescriptor<?> listDescr = (ListStateDescriptor<?>) descr;

	// this would be the first statement to fail if state descriptors were not properly initialized
	TypeSerializer<?> serializer = listDescr.getSerializer();
	assertTrue(serializer instanceof ListSerializer);

	TypeSerializer<?> elementSerializer = listDescr.getElementSerializer();
	assertTrue(elementSerializer instanceof KryoSerializer);

	Kryo kryo = ((KryoSerializer<?>) elementSerializer).getKryo();

	assertTrue("serializer registration was not properly passed on",
			kryo.getSerializer(File.class) instanceof JavaSerializer);
}
 
Example 8
Source Project: flink   Source File: WindowOperator.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public <S extends MergingState<?, ?>> void mergePartitionedState(
		StateDescriptor<S, ?> stateDescriptor) {
	if (mergedWindows != null && mergedWindows.size() > 0) {
		try {
			State state =
					WindowOperator.this.getOrCreateKeyedState(
							windowSerializer,
							stateDescriptor);
			if (state instanceof InternalMergingState) {
				((InternalMergingState<K, W, ?, ?, ?>) state).mergeNamespaces(window, mergedWindows);
			} else {
				throw new IllegalArgumentException(
						"The given state descriptor does not refer to a mergeable state (MergingState)");
			}
		}
		catch (Exception e) {
			throw new RuntimeException("Error while merging state.", e);
		}
	}
}
 
Example 9
Source Project: flink   Source File: RocksDBKeyedStateBackend.java    License: Apache License 2.0 6 votes vote down vote up
@Override
@Nonnull
public <N, SV, SEV, S extends State, IS extends S> IS createInternalState(
	@Nonnull TypeSerializer<N> namespaceSerializer,
	@Nonnull StateDescriptor<S, SV> stateDesc,
	@Nonnull StateSnapshotTransformFactory<SEV> snapshotTransformFactory) throws Exception {
	StateFactory stateFactory = STATE_FACTORIES.get(stateDesc.getClass());
	if (stateFactory == null) {
		String message = String.format("State %s is not supported by %s",
			stateDesc.getClass(), this.getClass());
		throw new FlinkRuntimeException(message);
	}
	Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> registerResult = tryRegisterKvStateInformation(
		stateDesc, namespaceSerializer, snapshotTransformFactory);
	return stateFactory.createState(stateDesc, registerResult, RocksDBKeyedStateBackend.this);
}
 
Example 10
Source Project: flink   Source File: AbstractQueryableStateTestBase.java    License: Apache License 2.0 6 votes vote down vote up
private static <K, S extends State, V> CompletableFuture<S> getKvState(
		final Deadline deadline,
		final QueryableStateClient client,
		final JobID jobId,
		final String queryName,
		final K key,
		final TypeInformation<K> keyTypeInfo,
		final StateDescriptor<S, V> stateDescriptor,
		final boolean failForUnknownKeyOrNamespace,
		final ScheduledExecutor executor) {

	final CompletableFuture<S> resultFuture = new CompletableFuture<>();
	getKvStateIgnoringCertainExceptions(
			deadline, resultFuture, client, jobId, queryName, key, keyTypeInfo,
			stateDescriptor, failForUnknownKeyOrNamespace, executor);
	return resultFuture;
}
 
Example 11
Source Project: Flink-CEPplus   Source File: EvictingWindowOperator.java    License: Apache License 2.0 6 votes vote down vote up
public EvictingWindowOperator(WindowAssigner<? super IN, W> windowAssigner,
		TypeSerializer<W> windowSerializer,
		KeySelector<IN, K> keySelector,
		TypeSerializer<K> keySerializer,
		StateDescriptor<? extends ListState<StreamRecord<IN>>, ?> windowStateDescriptor,
		InternalWindowFunction<Iterable<IN>, OUT, K, W> windowFunction,
		Trigger<? super IN, ? super W> trigger,
		Evictor<? super IN, ? super W> evictor,
		long allowedLateness,
		OutputTag<IN> lateDataOutputTag) {

	super(windowAssigner, windowSerializer, keySelector,
		keySerializer, null, windowFunction, trigger, allowedLateness, lateDataOutputTag);

	this.evictor = checkNotNull(evictor);
	this.evictingWindowStateDescriptor = checkNotNull(windowStateDescriptor);
}
 
Example 12
Source Project: Flink-CEPplus   Source File: MockKeyedStateBackend.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
private <SV, SEV> StateSnapshotTransformer<SV> getStateSnapshotTransformer(
	StateDescriptor<?, SV> stateDesc,
	StateSnapshotTransformFactory<SEV> snapshotTransformFactory) {
	Optional<StateSnapshotTransformer<SEV>> original = snapshotTransformFactory.createForDeserializedState();
	if (original.isPresent()) {
		if (stateDesc instanceof ListStateDescriptor) {
			return (StateSnapshotTransformer<SV>) new StateSnapshotTransformers.ListStateSnapshotTransformer<>(original.get());
		} else if (stateDesc instanceof MapStateDescriptor) {
			return (StateSnapshotTransformer<SV>) new StateSnapshotTransformers.MapStateSnapshotTransformer<>(original.get());
		} else {
			return (StateSnapshotTransformer<SV>) original.get();
		}
	} else {
		return null;
	}
}
 
Example 13
Source Project: flink   Source File: MockKeyedStateBackend.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
private <SV, SEV> StateSnapshotTransformer<SV> getStateSnapshotTransformer(
	StateDescriptor<?, SV> stateDesc,
	StateSnapshotTransformFactory<SEV> snapshotTransformFactory) {
	Optional<StateSnapshotTransformer<SEV>> original = snapshotTransformFactory.createForDeserializedState();
	if (original.isPresent()) {
		if (stateDesc instanceof ListStateDescriptor) {
			return (StateSnapshotTransformer<SV>) new StateSnapshotTransformers.ListStateSnapshotTransformer<>(original.get());
		} else if (stateDesc instanceof MapStateDescriptor) {
			return (StateSnapshotTransformer<SV>) new StateSnapshotTransformers.MapStateSnapshotTransformer<>(original.get());
		} else {
			return (StateSnapshotTransformer<SV>) original.get();
		}
	} else {
		return null;
	}
}
 
Example 14
Source Project: flink   Source File: AbstractKeyedStateBackend.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * @see KeyedStateBackend
 */
@Override
@SuppressWarnings("unchecked")
public <N, S extends State, V> S getOrCreateKeyedState(
		final TypeSerializer<N> namespaceSerializer,
		StateDescriptor<S, V> stateDescriptor) throws Exception {
	checkNotNull(namespaceSerializer, "Namespace serializer");
	checkNotNull(keySerializer, "State key serializer has not been configured in the config. " +
			"This operation cannot use partitioned state.");

	InternalKvState<K, ?, ?> kvState = keyValueStatesByName.get(stateDescriptor.getName());
	if (kvState == null) {
		if (!stateDescriptor.isSerializerInitialized()) {
			stateDescriptor.initializeSerializerUnlessSet(executionConfig);
		}
		kvState = TtlStateFactory.createStateAndWrapWithTtlIfEnabled(
			namespaceSerializer, stateDescriptor, this, ttlTimeProvider);
		keyValueStatesByName.put(stateDescriptor.getName(), kvState);
		publishQueryableStateIfEnabled(stateDescriptor, kvState);
	}
	return (S) kvState;
}
 
Example 15
@SuppressWarnings("unchecked")
static <SV, SEV> StateSnapshotTransformFactory<SV> wrapStateSnapshotTransformFactory(
	StateDescriptor<?, SV> stateDesc,
	StateSnapshotTransformFactory<SEV> snapshotTransformFactory,
	TypeSerializer<SV> stateSerializer) {
	if (stateDesc instanceof ListStateDescriptor) {
		TypeSerializer<SEV> elementSerializer = ((ListSerializer<SEV>) stateSerializer).getElementSerializer();
		return new RocksDBListStateSnapshotTransformFactory<>(snapshotTransformFactory, elementSerializer);
	} else if (stateDesc instanceof MapStateDescriptor) {
		return new RocksDBMapStateSnapshotTransformFactory<>(snapshotTransformFactory);
	} else {
		return new RocksDBValueStateSnapshotTransformFactory<>(snapshotTransformFactory);
	}
}
 
Example 16
Source Project: Flink-CEPplus   Source File: RocksDBMapState.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
static <UK, UV, K, N, SV, S extends State, IS extends S> IS create(
	StateDescriptor<S, SV> stateDesc,
	Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> registerResult,
	RocksDBKeyedStateBackend<K> backend) {
	return (IS) new RocksDBMapState<>(
		registerResult.f0,
		registerResult.f1.getNamespaceSerializer(),
		(TypeSerializer<Map<UK, UV>>) registerResult.f1.getStateSerializer(),
		(Map<UK, UV>) stateDesc.getDefaultValue(),
		backend);
}
 
Example 17
Source Project: Flink-CEPplus   Source File: RocksDBListState.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
static <E, K, N, SV, S extends State, IS extends S> IS create(
	StateDescriptor<S, SV> stateDesc,
	Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> registerResult,
	RocksDBKeyedStateBackend<K> backend) {
	return (IS) new RocksDBListState<>(
		registerResult.f0,
		registerResult.f1.getNamespaceSerializer(),
		(TypeSerializer<List<E>>) registerResult.f1.getStateSerializer(),
		(List<E>) stateDesc.getDefaultValue(),
		backend);
}
 
Example 18
Source Project: flink   Source File: StreamingRuntimeContextTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testReducingStateInstantiation() throws Exception {

	final ExecutionConfig config = new ExecutionConfig();
	config.registerKryoType(Path.class);

	final AtomicReference<Object> descriptorCapture = new AtomicReference<>();

	StreamingRuntimeContext context = new StreamingRuntimeContext(
			createDescriptorCapturingMockOp(descriptorCapture, config),
			createMockEnvironment(),
			Collections.<String, Accumulator<?, ?>>emptyMap());

	@SuppressWarnings("unchecked")
	ReduceFunction<TaskInfo> reducer = (ReduceFunction<TaskInfo>) mock(ReduceFunction.class);

	ReducingStateDescriptor<TaskInfo> descr =
			new ReducingStateDescriptor<>("name", reducer, TaskInfo.class);

	context.getReducingState(descr);

	StateDescriptor<?, ?> descrIntercepted = (StateDescriptor<?, ?>) descriptorCapture.get();
	TypeSerializer<?> serializer = descrIntercepted.getSerializer();

	// check that the Path class is really registered, i.e., the execution config was applied
	assertTrue(serializer instanceof KryoSerializer);
	assertTrue(((KryoSerializer<?>) serializer).getKryo().getRegistration(Path.class).getId() > 0);
}
 
Example 19
Source Project: flink   Source File: RocksDbTtlCompactFiltersManager.java    License: Apache License 2.0 5 votes vote down vote up
public void configCompactFilter(
		@Nonnull StateDescriptor<?, ?> stateDesc,
		TypeSerializer<?> stateSerializer) {
	StateTtlConfig ttlConfig = stateDesc.getTtlConfig();
	if (ttlConfig.isEnabled() && ttlConfig.getCleanupStrategies().inRocksdbCompactFilter()) {
		if (!enableTtlCompactionFilter) {
			LOG.warn("Cannot configure RocksDB TTL compaction filter for state <{}>: " +
				"feature is disabled for the state backend.", stateDesc.getName());
			return;
		}
		FlinkCompactionFilterFactory compactionFilterFactory = compactionFilterFactories.get(stateDesc.getName());
		Preconditions.checkNotNull(compactionFilterFactory);
		long ttl = ttlConfig.getTtl().toMilliseconds();

		StateTtlConfig.RocksdbCompactFilterCleanupStrategy rocksdbCompactFilterCleanupStrategy =
			ttlConfig.getCleanupStrategies().getRocksdbCompactFilterCleanupStrategy();
		Preconditions.checkNotNull(rocksdbCompactFilterCleanupStrategy);
		long queryTimeAfterNumEntries =
			rocksdbCompactFilterCleanupStrategy.getQueryTimeAfterNumEntries();

		FlinkCompactionFilter.Config config;
		if (stateDesc instanceof ListStateDescriptor) {
			TypeSerializer<?> elemSerializer = ((ListSerializer<?>) stateSerializer).getElementSerializer();
			int len = elemSerializer.getLength();
			if (len > 0) {
				config = FlinkCompactionFilter.Config.createForFixedElementList(
					ttl, queryTimeAfterNumEntries, len + 1); // plus one byte for list element delimiter
			} else {
				config = FlinkCompactionFilter.Config.createForList(
					ttl, queryTimeAfterNumEntries,
					new ListElementFilterFactory<>(elemSerializer.duplicate()));
			}
		} else if (stateDesc instanceof MapStateDescriptor) {
			config = FlinkCompactionFilter.Config.createForMap(ttl, queryTimeAfterNumEntries);
		} else {
			config = FlinkCompactionFilter.Config.createForValue(ttl, queryTimeAfterNumEntries);
		}
		compactionFilterFactory.configure(config);
	}
}
 
Example 20
Source Project: flink   Source File: WindowOperator.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public <S extends State> S getPartitionedState(StateDescriptor<S, ?> stateDescriptor) {
	try {
		return WindowOperator.this.getPartitionedState(window, windowSerializer, stateDescriptor);
	} catch (Exception e) {
		throw new RuntimeException("Could not retrieve state", e);
	}
}
 
Example 21
Source Project: flink   Source File: RocksDBValueState.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
static <K, N, SV, S extends State, IS extends S> IS create(
	StateDescriptor<S, SV> stateDesc,
	Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> registerResult,
	RocksDBKeyedStateBackend<K> backend) {
	return (IS) new RocksDBValueState<>(
		registerResult.f0,
		registerResult.f1.getNamespaceSerializer(),
		registerResult.f1.getStateSerializer(),
		stateDesc.getDefaultValue(),
		backend);
}
 
Example 22
Source Project: Flink-CEPplus   Source File: HeapKeyedStateBackend.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
private <SV, SEV> StateSnapshotTransformFactory<SV> getStateSnapshotTransformFactory(
	StateDescriptor<?, SV> stateDesc,
	StateSnapshotTransformFactory<SEV> snapshotTransformFactory) {
	if (stateDesc instanceof ListStateDescriptor) {
		return (StateSnapshotTransformFactory<SV>) new StateSnapshotTransformers.ListStateSnapshotTransformFactory<>(snapshotTransformFactory);
	} else if (stateDesc instanceof MapStateDescriptor) {
		return (StateSnapshotTransformFactory<SV>) new StateSnapshotTransformers.MapStateSnapshotTransformFactory<>(snapshotTransformFactory);
	} else {
		return (StateSnapshotTransformFactory<SV>) snapshotTransformFactory;
	}
}
 
Example 23
Source Project: Flink-CEPplus   Source File: HeapValueState.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
static <K, N, SV, S extends State, IS extends S> IS create(
	StateDescriptor<S, SV> stateDesc,
	StateTable<K, N, SV> stateTable,
	TypeSerializer<K> keySerializer) {
	return (IS) new HeapValueState<>(
		stateTable,
		keySerializer,
		stateTable.getStateSerializer(),
		stateTable.getNamespaceSerializer(),
		stateDesc.getDefaultValue());
}
 
Example 24
Source Project: flink   Source File: ImmutableMapState.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
public static <K, V, T, S extends State> S createState(
	StateDescriptor<S, T> stateDescriptor,
	byte[] serializedState) throws IOException {
	MapStateDescriptor<K, V> mapStateDescriptor = (MapStateDescriptor<K, V>) stateDescriptor;
	final Map<K, V> state = KvStateSerializer.deserializeMap(
		serializedState,
		mapStateDescriptor.getKeySerializer(),
		mapStateDescriptor.getValueSerializer());
	return (S) new ImmutableMapState<>(state);
}
 
Example 25
Source Project: flink   Source File: HeapListState.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
static <E, K, N, SV, S extends State, IS extends S> IS create(
	StateDescriptor<S, SV> stateDesc,
	StateTable<K, N, SV> stateTable,
	TypeSerializer<K> keySerializer) {
	return (IS) new HeapListState<>(
		(StateTable<K, N, List<E>>) stateTable,
		keySerializer,
		(TypeSerializer<List<E>>) stateTable.getStateSerializer(),
		stateTable.getNamespaceSerializer(),
		(List<E>) stateDesc.getDefaultValue());
}
 
Example 26
Source Project: Flink-CEPplus   Source File: HeapListState.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
static <E, K, N, SV, S extends State, IS extends S> IS create(
	StateDescriptor<S, SV> stateDesc,
	StateTable<K, N, SV> stateTable,
	TypeSerializer<K> keySerializer) {
	return (IS) new HeapListState<>(
		(StateTable<K, N, List<E>>) stateTable,
		keySerializer,
		(TypeSerializer<List<E>>) stateTable.getStateSerializer(),
		stateTable.getNamespaceSerializer(),
		(List<E>) stateDesc.getDefaultValue());
}
 
Example 27
Source Project: flink   Source File: WindowOperator.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates a new {@code WindowOperator} based on the given policies and user functions.
 */
public WindowOperator(
		WindowAssigner<? super IN, W> windowAssigner,
		TypeSerializer<W> windowSerializer,
		KeySelector<IN, K> keySelector,
		TypeSerializer<K> keySerializer,
		StateDescriptor<? extends AppendingState<IN, ACC>, ?> windowStateDescriptor,
		InternalWindowFunction<ACC, OUT, K, W> windowFunction,
		Trigger<? super IN, ? super W> trigger,
		long allowedLateness,
		OutputTag<IN> lateDataOutputTag) {

	super(windowFunction);

	checkArgument(!(windowAssigner instanceof BaseAlignedWindowAssigner),
		"The " + windowAssigner.getClass().getSimpleName() + " cannot be used with a WindowOperator. " +
			"This assigner is only used with the AccumulatingProcessingTimeWindowOperator and " +
			"the AggregatingProcessingTimeWindowOperator");

	checkArgument(allowedLateness >= 0);

	checkArgument(windowStateDescriptor == null || windowStateDescriptor.isSerializerInitialized(),
			"window state serializer is not properly initialized");

	this.windowAssigner = checkNotNull(windowAssigner);
	this.windowSerializer = checkNotNull(windowSerializer);
	this.keySelector = checkNotNull(keySelector);
	this.keySerializer = checkNotNull(keySerializer);
	this.windowStateDescriptor = windowStateDescriptor;
	this.trigger = checkNotNull(trigger);
	this.allowedLateness = allowedLateness;
	this.lateDataOutputTag = lateDataOutputTag;

	setChainingStrategy(ChainingStrategy.ALWAYS);
}
 
Example 28
Source Project: Flink-CEPplus   Source File: AbstractKeyedStateBackend.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * TODO: NOTE: This method does a lot of work caching / retrieving states just to update the namespace.
 *       This method should be removed for the sake of namespaces being lazily fetched from the keyed
 *       state backend, or being set on the state directly.
 *
 * @see KeyedStateBackend
 */
@SuppressWarnings("unchecked")
@Override
public <N, S extends State> S getPartitionedState(
		final N namespace,
		final TypeSerializer<N> namespaceSerializer,
		final StateDescriptor<S, ?> stateDescriptor) throws Exception {

	checkNotNull(namespace, "Namespace");

	if (lastName != null && lastName.equals(stateDescriptor.getName())) {
		lastState.setCurrentNamespace(namespace);
		return (S) lastState;
	}

	InternalKvState<K, ?, ?> previous = keyValueStatesByName.get(stateDescriptor.getName());
	if (previous != null) {
		lastState = previous;
		lastState.setCurrentNamespace(namespace);
		lastName = stateDescriptor.getName();
		return (S) previous;
	}

	final S state = getOrCreateKeyedState(namespaceSerializer, stateDescriptor);
	final InternalKvState<K, N, ?> kvState = (InternalKvState<K, N, ?>) state;

	lastName = stateDescriptor.getName();
	lastState = kvState;
	kvState.setCurrentNamespace(namespace);

	return state;
}
 
Example 29
Source Project: bravo   Source File: OperatorStateWriter.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Defines/redefines a value state with the given name and type. This can be
 * used to create new states of an operator or change the type of an already
 * existing state.
 * <p>
 * When redefining a pre-existing state make sure you haven't added that as
 * keyed state rows before.
 * 
 * @param stateName
 * @param newState
 * @param valueSerializer
 */
public <K, V> void createNewValueState(String stateName, DataSet<Tuple2<K, V>> newState,
		TypeSerializer<V> valueSerializer) {

	metaSnapshots.put(stateName, new RegisteredKeyValueStateBackendMetaInfo<>(StateDescriptor.Type.VALUE, stateName,
			VoidNamespaceSerializer.INSTANCE, valueSerializer).snapshot());

	updateProxy();
	addKeyedStateRows(newState
			.map(new ValueStateToKeyedStateRow<K, V>(stateName,
					getKeySerializer(),
					valueSerializer,
					baseOpState.getMaxParallelism())));
}
 
Example 30
Source Project: Flink-CEPplus   Source File: TtlStateFactory.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("deprecation")
private Map<Class<? extends StateDescriptor>, SupplierWithException<IS, Exception>> createStateFactories() {
	return Stream.of(
		Tuple2.of(ValueStateDescriptor.class, (SupplierWithException<IS, Exception>) this::createValueState),
		Tuple2.of(ListStateDescriptor.class, (SupplierWithException<IS, Exception>) this::createListState),
		Tuple2.of(MapStateDescriptor.class, (SupplierWithException<IS, Exception>) this::createMapState),
		Tuple2.of(ReducingStateDescriptor.class, (SupplierWithException<IS, Exception>) this::createReducingState),
		Tuple2.of(AggregatingStateDescriptor.class, (SupplierWithException<IS, Exception>) this::createAggregatingState),
		Tuple2.of(FoldingStateDescriptor.class, (SupplierWithException<IS, Exception>) this::createFoldingState)
	).collect(Collectors.toMap(t -> t.f0, t -> t.f1));
}