org.apache.flink.api.common.typeutils.base.ListSerializer Java Examples

The following examples show how to use org.apache.flink.api.common.typeutils.base.ListSerializer. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: CepOperator.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public void initializeState(StateInitializationContext context) throws Exception {
	super.initializeState(context);

	// initializeState through the provided context
	computationStates = context.getKeyedStateStore().getState(
		new ValueStateDescriptor<>(
			NFA_STATE_NAME,
			new NFAStateSerializer()));

	partialMatches = new SharedBuffer<>(context.getKeyedStateStore(), inputSerializer);

	elementQueueState = context.getKeyedStateStore().getMapState(
			new MapStateDescriptor<>(
					EVENT_QUEUE_STATE_NAME,
					LongSerializer.INSTANCE,
					new ListSerializer<>(inputSerializer)));

	migrateOldState();
}
 
Example #2
Source File: TtlListState.java    From flink with Apache License 2.0 6 votes vote down vote up
@Nullable
@Override
public List<TtlValue<T>> getUnexpiredOrNull(@Nonnull List<TtlValue<T>> ttlValues) {
	long currentTimestamp = timeProvider.currentTimestamp();
	List<TtlValue<T>> unexpired = new ArrayList<>(ttlValues.size());
	TypeSerializer<TtlValue<T>> elementSerializer =
		((ListSerializer<TtlValue<T>>) original.getValueSerializer()).getElementSerializer();
	for (TtlValue<T> ttlValue : ttlValues) {
		if (!TtlUtils.expired(ttlValue, ttl, currentTimestamp)) {
			// we have to do the defensive copy to update the value
			unexpired.add(elementSerializer.copy(ttlValue));
		}
	}
	if (!unexpired.isEmpty()) {
		return unexpired;
	} else {
		return ttlValues.size() == unexpired.size() ? ttlValues : unexpired;
	}
}
 
Example #3
Source File: IntervalJoinOperator.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public void initializeState(StateInitializationContext context) throws Exception {
	super.initializeState(context);

	this.leftBuffer = context.getKeyedStateStore().getMapState(new MapStateDescriptor<>(
		LEFT_BUFFER,
		LongSerializer.INSTANCE,
		new ListSerializer<>(new BufferEntrySerializer<>(leftTypeSerializer))
	));

	this.rightBuffer = context.getKeyedStateStore().getMapState(new MapStateDescriptor<>(
		RIGHT_BUFFER,
		LongSerializer.INSTANCE,
		new ListSerializer<>(new BufferEntrySerializer<>(rightTypeSerializer))
	));
}
 
Example #4
Source File: CepOperator.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Override
public void initializeState(StateInitializationContext context) throws Exception {
	super.initializeState(context);

	// initializeState through the provided context
	computationStates = context.getKeyedStateStore().getState(
		new ValueStateDescriptor<>(
			NFA_STATE_NAME,
			new NFAStateSerializer()));

	partialMatches = new SharedBuffer<>(context.getKeyedStateStore(), inputSerializer);

	elementQueueState = context.getKeyedStateStore().getMapState(
			new MapStateDescriptor<>(
					EVENT_QUEUE_STATE_NAME,
					LongSerializer.INSTANCE,
					new ListSerializer<>(inputSerializer)));

	migrateOldState();
}
 
Example #5
Source File: ListStateDescriptorTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testListStateDescriptor() throws Exception {

	TypeSerializer<String> serializer = new KryoSerializer<>(String.class, new ExecutionConfig());

	ListStateDescriptor<String> descr =
			new ListStateDescriptor<>("testName", serializer);

	assertEquals("testName", descr.getName());
	assertNotNull(descr.getSerializer());
	assertTrue(descr.getSerializer() instanceof ListSerializer);
	assertNotNull(descr.getElementSerializer());
	assertEquals(serializer, descr.getElementSerializer());

	ListStateDescriptor<String> copy = CommonTestUtils.createCopySerializable(descr);

	assertEquals("testName", copy.getName());
	assertNotNull(copy.getSerializer());
	assertTrue(copy.getSerializer() instanceof ListSerializer);

	assertNotNull(copy.getElementSerializer());
	assertEquals(serializer, copy.getElementSerializer());
}
 
Example #6
Source File: StateDescriptorPassingTest.java    From flink with Apache License 2.0 6 votes vote down vote up
private void validateListStateDescriptorConfigured(SingleOutputStreamOperator<?> result) {
	OneInputTransformation<?, ?> transform = (OneInputTransformation<?, ?>) result.getTransformation();
	WindowOperator<?, ?, ?, ?, ?> op = (WindowOperator<?, ?, ?, ?, ?>) transform.getOperator();
	StateDescriptor<?, ?> descr = op.getStateDescriptor();

	assertTrue(descr instanceof ListStateDescriptor);

	ListStateDescriptor<?> listDescr = (ListStateDescriptor<?>) descr;

	// this would be the first statement to fail if state descriptors were not properly initialized
	TypeSerializer<?> serializer = listDescr.getSerializer();
	assertTrue(serializer instanceof ListSerializer);

	TypeSerializer<?> elementSerializer = listDescr.getElementSerializer();
	assertTrue(elementSerializer instanceof KryoSerializer);

	Kryo kryo = ((KryoSerializer<?>) elementSerializer).getKryo();

	assertTrue("serializer registration was not properly passed on",
			kryo.getSerializer(File.class) instanceof JavaSerializer);
}
 
Example #7
Source File: StreamingRuntimeContextTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testListStateInstantiation() throws Exception {

	final ExecutionConfig config = new ExecutionConfig();
	config.registerKryoType(Path.class);

	final AtomicReference<Object> descriptorCapture = new AtomicReference<>();

	StreamingRuntimeContext context = createRuntimeContext(descriptorCapture, config);

	ListStateDescriptor<TaskInfo> descr = new ListStateDescriptor<>("name", TaskInfo.class);
	context.getListState(descr);

	ListStateDescriptor<?> descrIntercepted = (ListStateDescriptor<?>) descriptorCapture.get();
	TypeSerializer<?> serializer = descrIntercepted.getSerializer();

	// check that the Path class is really registered, i.e., the execution config was applied
	assertTrue(serializer instanceof ListSerializer);

	TypeSerializer<?> elementSerializer = descrIntercepted.getElementSerializer();
	assertTrue(elementSerializer instanceof KryoSerializer);
	assertTrue(((KryoSerializer<?>) elementSerializer).getKryo().getRegistration(Path.class).getId() > 0);
}
 
Example #8
Source File: TtlListState.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Nullable
@Override
public List<TtlValue<T>> getUnexpiredOrNull(@Nonnull List<TtlValue<T>> ttlValues) {
	long currentTimestamp = timeProvider.currentTimestamp();
	List<TtlValue<T>> unexpired = new ArrayList<>(ttlValues.size());
	TypeSerializer<TtlValue<T>> elementSerializer =
		((ListSerializer<TtlValue<T>>) original.getValueSerializer()).getElementSerializer();
	for (TtlValue<T> ttlValue : ttlValues) {
		if (!TtlUtils.expired(ttlValue, ttl, currentTimestamp)) {
			// we have to do the defensive copy to update the value
			unexpired.add(elementSerializer.copy(ttlValue));
		}
	}
	if (!unexpired.isEmpty()) {
		return unexpired;
	} else {
		return ttlValues.size() == unexpired.size() ? ttlValues : unexpired;
	}
}
 
Example #9
Source File: IntervalJoinOperator.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Override
public void initializeState(StateInitializationContext context) throws Exception {
	super.initializeState(context);

	this.leftBuffer = context.getKeyedStateStore().getMapState(new MapStateDescriptor<>(
		LEFT_BUFFER,
		LongSerializer.INSTANCE,
		new ListSerializer<>(new BufferEntrySerializer<>(leftTypeSerializer))
	));

	this.rightBuffer = context.getKeyedStateStore().getMapState(new MapStateDescriptor<>(
		RIGHT_BUFFER,
		LongSerializer.INSTANCE,
		new ListSerializer<>(new BufferEntrySerializer<>(rightTypeSerializer))
	));
}
 
Example #10
Source File: StateDescriptorPassingTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
private void validateListStateDescriptorConfigured(SingleOutputStreamOperator<?> result) {
	OneInputTransformation<?, ?> transform = (OneInputTransformation<?, ?>) result.getTransformation();
	WindowOperator<?, ?, ?, ?, ?> op = (WindowOperator<?, ?, ?, ?, ?>) transform.getOperator();
	StateDescriptor<?, ?> descr = op.getStateDescriptor();

	assertTrue(descr instanceof ListStateDescriptor);

	ListStateDescriptor<?> listDescr = (ListStateDescriptor<?>) descr;

	// this would be the first statement to fail if state descriptors were not properly initialized
	TypeSerializer<?> serializer = listDescr.getSerializer();
	assertTrue(serializer instanceof ListSerializer);

	TypeSerializer<?> elementSerializer = listDescr.getElementSerializer();
	assertTrue(elementSerializer instanceof KryoSerializer);

	Kryo kryo = ((KryoSerializer<?>) elementSerializer).getKryo();

	assertTrue("serializer registration was not properly passed on",
			kryo.getSerializer(File.class) instanceof JavaSerializer);
}
 
Example #11
Source File: StreamingRuntimeContextTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testListStateInstantiation() throws Exception {

	final ExecutionConfig config = new ExecutionConfig();
	config.registerKryoType(Path.class);

	final AtomicReference<Object> descriptorCapture = new AtomicReference<>();

	StreamingRuntimeContext context = new StreamingRuntimeContext(
			createDescriptorCapturingMockOp(descriptorCapture, config),
			createMockEnvironment(),
			Collections.<String, Accumulator<?, ?>>emptyMap());

	ListStateDescriptor<TaskInfo> descr = new ListStateDescriptor<>("name", TaskInfo.class);
	context.getListState(descr);

	ListStateDescriptor<?> descrIntercepted = (ListStateDescriptor<?>) descriptorCapture.get();
	TypeSerializer<?> serializer = descrIntercepted.getSerializer();

	// check that the Path class is really registered, i.e., the execution config was applied
	assertTrue(serializer instanceof ListSerializer);

	TypeSerializer<?> elementSerializer = descrIntercepted.getElementSerializer();
	assertTrue(elementSerializer instanceof KryoSerializer);
	assertTrue(((KryoSerializer<?>) elementSerializer).getKryo().getRegistration(Path.class).getId() > 0);
}
 
Example #12
Source File: ListStateDescriptorTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testListStateDescriptor() throws Exception {

	TypeSerializer<String> serializer = new KryoSerializer<>(String.class, new ExecutionConfig());

	ListStateDescriptor<String> descr =
			new ListStateDescriptor<>("testName", serializer);

	assertEquals("testName", descr.getName());
	assertNotNull(descr.getSerializer());
	assertTrue(descr.getSerializer() instanceof ListSerializer);
	assertNotNull(descr.getElementSerializer());
	assertEquals(serializer, descr.getElementSerializer());

	ListStateDescriptor<String> copy = CommonTestUtils.createCopySerializable(descr);

	assertEquals("testName", copy.getName());
	assertNotNull(copy.getSerializer());
	assertTrue(copy.getSerializer() instanceof ListSerializer);

	assertNotNull(copy.getElementSerializer());
	assertEquals(serializer, copy.getElementSerializer());
}
 
Example #13
Source File: StreamingRuntimeContextTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testListStateInstantiation() throws Exception {

	final ExecutionConfig config = new ExecutionConfig();
	config.registerKryoType(Path.class);

	final AtomicReference<Object> descriptorCapture = new AtomicReference<>();

	StreamingRuntimeContext context = new StreamingRuntimeContext(
			createDescriptorCapturingMockOp(descriptorCapture, config),
			createMockEnvironment(),
			Collections.<String, Accumulator<?, ?>>emptyMap());

	ListStateDescriptor<TaskInfo> descr = new ListStateDescriptor<>("name", TaskInfo.class);
	context.getListState(descr);

	ListStateDescriptor<?> descrIntercepted = (ListStateDescriptor<?>) descriptorCapture.get();
	TypeSerializer<?> serializer = descrIntercepted.getSerializer();

	// check that the Path class is really registered, i.e., the execution config was applied
	assertTrue(serializer instanceof ListSerializer);

	TypeSerializer<?> elementSerializer = descrIntercepted.getElementSerializer();
	assertTrue(elementSerializer instanceof KryoSerializer);
	assertTrue(((KryoSerializer<?>) elementSerializer).getKryo().getRegistration(Path.class).getId() > 0);
}
 
Example #14
Source File: StateDescriptorPassingTest.java    From flink with Apache License 2.0 6 votes vote down vote up
private void validateListStateDescriptorConfigured(SingleOutputStreamOperator<?> result) {
	OneInputTransformation<?, ?> transform = (OneInputTransformation<?, ?>) result.getTransformation();
	WindowOperator<?, ?, ?, ?, ?> op = (WindowOperator<?, ?, ?, ?, ?>) transform.getOperator();
	StateDescriptor<?, ?> descr = op.getStateDescriptor();

	assertTrue(descr instanceof ListStateDescriptor);

	ListStateDescriptor<?> listDescr = (ListStateDescriptor<?>) descr;

	// this would be the first statement to fail if state descriptors were not properly initialized
	TypeSerializer<?> serializer = listDescr.getSerializer();
	assertTrue(serializer instanceof ListSerializer);

	TypeSerializer<?> elementSerializer = listDescr.getElementSerializer();
	assertTrue(elementSerializer instanceof KryoSerializer);

	Kryo kryo = ((KryoSerializer<?>) elementSerializer).getKryo();

	assertTrue("serializer registration was not properly passed on",
			kryo.getSerializer(File.class) instanceof JavaSerializer);
}
 
Example #15
Source File: IntervalJoinOperator.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public void initializeState(StateInitializationContext context) throws Exception {
	super.initializeState(context);

	this.leftBuffer = context.getKeyedStateStore().getMapState(new MapStateDescriptor<>(
		LEFT_BUFFER,
		LongSerializer.INSTANCE,
		new ListSerializer<>(new BufferEntrySerializer<>(leftTypeSerializer))
	));

	this.rightBuffer = context.getKeyedStateStore().getMapState(new MapStateDescriptor<>(
		RIGHT_BUFFER,
		LongSerializer.INSTANCE,
		new ListSerializer<>(new BufferEntrySerializer<>(rightTypeSerializer))
	));
}
 
Example #16
Source File: TtlListState.java    From flink with Apache License 2.0 6 votes vote down vote up
@Nullable
@Override
public List<TtlValue<T>> getUnexpiredOrNull(@Nonnull List<TtlValue<T>> ttlValues) {
	long currentTimestamp = timeProvider.currentTimestamp();
	List<TtlValue<T>> unexpired = new ArrayList<>(ttlValues.size());
	TypeSerializer<TtlValue<T>> elementSerializer =
		((ListSerializer<TtlValue<T>>) original.getValueSerializer()).getElementSerializer();
	for (TtlValue<T> ttlValue : ttlValues) {
		if (!TtlUtils.expired(ttlValue, ttl, currentTimestamp)) {
			// we have to do the defensive copy to update the value
			unexpired.add(elementSerializer.copy(ttlValue));
		}
	}
	if (!unexpired.isEmpty()) {
		return unexpired;
	} else {
		return ttlValues.size() == unexpired.size() ? ttlValues : unexpired;
	}
}
 
Example #17
Source File: ListStateDescriptorTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testListStateDescriptor() throws Exception {

	TypeSerializer<String> serializer = new KryoSerializer<>(String.class, new ExecutionConfig());

	ListStateDescriptor<String> descr =
			new ListStateDescriptor<>("testName", serializer);

	assertEquals("testName", descr.getName());
	assertNotNull(descr.getSerializer());
	assertTrue(descr.getSerializer() instanceof ListSerializer);
	assertNotNull(descr.getElementSerializer());
	assertEquals(serializer, descr.getElementSerializer());

	ListStateDescriptor<String> copy = CommonTestUtils.createCopySerializable(descr);

	assertEquals("testName", copy.getName());
	assertNotNull(copy.getSerializer());
	assertTrue(copy.getSerializer() instanceof ListSerializer);

	assertNotNull(copy.getElementSerializer());
	assertEquals(serializer, copy.getElementSerializer());
}
 
Example #18
Source File: CepOperator.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public void initializeState(StateInitializationContext context) throws Exception {
	super.initializeState(context);

	// initializeState through the provided context
	computationStates = context.getKeyedStateStore().getState(
		new ValueStateDescriptor<>(
			NFA_STATE_NAME,
			new NFAStateSerializer()));

	partialMatches = new SharedBuffer<>(context.getKeyedStateStore(), inputSerializer);

	elementQueueState = context.getKeyedStateStore().getMapState(
			new MapStateDescriptor<>(
					EVENT_QUEUE_STATE_NAME,
					LongSerializer.INSTANCE,
					new ListSerializer<>(inputSerializer)));

	migrateOldState();
}
 
Example #19
Source File: ListViewTypeInfo.java    From flink with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
public TypeSerializer<ListView<T>> createSerializer(ExecutionConfig config) {
	if (nullSerializer) {
		return (TypeSerializer<ListView<T>>) (TypeSerializer<?>) NullSerializer.INSTANCE;
	} else {
		TypeSerializer<T> elementSerializer = elementType.createSerializer(config);
		return new ListViewSerializer<>(new ListSerializer<>(elementSerializer));
	}
}
 
Example #20
Source File: HeapListState.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public byte[] getSerializedValue(
		final byte[] serializedKeyAndNamespace,
		final TypeSerializer<K> safeKeySerializer,
		final TypeSerializer<N> safeNamespaceSerializer,
		final TypeSerializer<List<V>> safeValueSerializer) throws Exception {

	Preconditions.checkNotNull(serializedKeyAndNamespace);
	Preconditions.checkNotNull(safeKeySerializer);
	Preconditions.checkNotNull(safeNamespaceSerializer);
	Preconditions.checkNotNull(safeValueSerializer);

	Tuple2<K, N> keyAndNamespace = KvStateSerializer.deserializeKeyAndNamespace(
			serializedKeyAndNamespace, safeKeySerializer, safeNamespaceSerializer);

	List<V> result = stateTable.get(keyAndNamespace.f0, keyAndNamespace.f1);

	if (result == null) {
		return null;
	}

	final TypeSerializer<V> dupSerializer = ((ListSerializer<V>) safeValueSerializer).getElementSerializer();

	ByteArrayOutputStream baos = new ByteArrayOutputStream();
	DataOutputViewStreamWrapper view = new DataOutputViewStreamWrapper(baos);

	// write the same as RocksDB writes lists, with one ',' separator
	for (int i = 0; i < result.size(); i++) {
		dupSerializer.serialize(result.get(i), view);
		if (i < result.size() -1) {
			view.writeByte(',');
		}
	}
	view.flush();

	return baos.toByteArray();
}
 
Example #21
Source File: RocksDbTtlCompactFiltersManager.java    From flink with Apache License 2.0 5 votes vote down vote up
public void configCompactFilter(
		@Nonnull StateDescriptor<?, ?> stateDesc,
		TypeSerializer<?> stateSerializer) {
	StateTtlConfig ttlConfig = stateDesc.getTtlConfig();
	if (ttlConfig.isEnabled() && ttlConfig.getCleanupStrategies().inRocksdbCompactFilter()) {
		FlinkCompactionFilterFactory compactionFilterFactory = compactionFilterFactories.get(stateDesc.getName());
		Preconditions.checkNotNull(compactionFilterFactory);
		long ttl = ttlConfig.getTtl().toMilliseconds();

		StateTtlConfig.RocksdbCompactFilterCleanupStrategy rocksdbCompactFilterCleanupStrategy =
			ttlConfig.getCleanupStrategies().getRocksdbCompactFilterCleanupStrategy();
		Preconditions.checkNotNull(rocksdbCompactFilterCleanupStrategy);
		long queryTimeAfterNumEntries =
			rocksdbCompactFilterCleanupStrategy.getQueryTimeAfterNumEntries();

		FlinkCompactionFilter.Config config;
		if (stateDesc instanceof ListStateDescriptor) {
			TypeSerializer<?> elemSerializer = ((ListSerializer<?>) stateSerializer).getElementSerializer();
			int len = elemSerializer.getLength();
			if (len > 0) {
				config = FlinkCompactionFilter.Config.createForFixedElementList(
					ttl, queryTimeAfterNumEntries, len + 1); // plus one byte for list element delimiter
			} else {
				config = FlinkCompactionFilter.Config.createForList(
					ttl, queryTimeAfterNumEntries,
					new ListElementFilterFactory<>(elemSerializer.duplicate()));
			}
		} else if (stateDesc instanceof MapStateDescriptor) {
			config = FlinkCompactionFilter.Config.createForMap(ttl, queryTimeAfterNumEntries);
		} else {
			config = FlinkCompactionFilter.Config.createForValue(ttl, queryTimeAfterNumEntries);
		}
		compactionFilterFactory.configure(config);
	}
}
 
Example #22
Source File: RocksDBListState.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void migrateSerializedValue(
		DataInputDeserializer serializedOldValueInput,
		DataOutputSerializer serializedMigratedValueOutput,
		TypeSerializer<List<V>> priorSerializer,
		TypeSerializer<List<V>> newSerializer) throws StateMigrationException {

	Preconditions.checkArgument(priorSerializer instanceof ListSerializer);
	Preconditions.checkArgument(newSerializer instanceof ListSerializer);

	TypeSerializer<V> priorElementSerializer =
		((ListSerializer<V>) priorSerializer).getElementSerializer();

	TypeSerializer<V> newElementSerializer =
		((ListSerializer<V>) newSerializer).getElementSerializer();

	try {
		while (serializedOldValueInput.available() > 0) {
			V element = deserializeNextElement(serializedOldValueInput, priorElementSerializer);
			newElementSerializer.serialize(element, serializedMigratedValueOutput);
			if (serializedOldValueInput.available() > 0) {
				serializedMigratedValueOutput.write(DELIMITER);
			}
		}
	} catch (Exception e) {
		throw new StateMigrationException("Error while trying to migrate RocksDB list state.", e);
	}
}
 
Example #23
Source File: RocksDBListState.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a new {@code RocksDBListState}.
 *
 * @param columnFamily The RocksDB column family that this state is associated to.
 * @param namespaceSerializer The serializer for the namespace.
 * @param valueSerializer The serializer for the state.
 * @param defaultValue The default value for the state.
 * @param backend The backend for which this state is bind to.
 */
private RocksDBListState(
		ColumnFamilyHandle columnFamily,
		TypeSerializer<N> namespaceSerializer,
		TypeSerializer<List<V>> valueSerializer,
		List<V> defaultValue,
		RocksDBKeyedStateBackend<K> backend) {

	super(columnFamily, namespaceSerializer, valueSerializer, defaultValue, backend);

	ListSerializer<V> castedListSerializer = (ListSerializer<V>) valueSerializer;
	this.elementSerializer = castedListSerializer.getElementSerializer();
}
 
Example #24
Source File: TtlStateFactory.java    From flink with Apache License 2.0 5 votes vote down vote up
public static boolean isTtlStateSerializer(TypeSerializer<?> typeSerializer) {
	boolean ttlSerializer = typeSerializer instanceof TtlStateFactory.TtlSerializer;
	boolean ttlListSerializer = typeSerializer instanceof ListSerializer &&
		((ListSerializer) typeSerializer).getElementSerializer() instanceof TtlStateFactory.TtlSerializer;
	boolean ttlMapSerializer = typeSerializer instanceof MapSerializer &&
		((MapSerializer) typeSerializer).getValueSerializer() instanceof TtlStateFactory.TtlSerializer;
	return ttlSerializer || ttlListSerializer || ttlMapSerializer;
}
 
Example #25
Source File: RocksDbTtlCompactFiltersManager.java    From flink with Apache License 2.0 5 votes vote down vote up
public void configCompactFilter(
		@Nonnull StateDescriptor<?, ?> stateDesc,
		TypeSerializer<?> stateSerializer) {
	StateTtlConfig ttlConfig = stateDesc.getTtlConfig();
	if (ttlConfig.isEnabled() && ttlConfig.getCleanupStrategies().inRocksdbCompactFilter()) {
		if (!enableTtlCompactionFilter) {
			LOG.warn("Cannot configure RocksDB TTL compaction filter for state <{}>: " +
				"feature is disabled for the state backend.", stateDesc.getName());
			return;
		}
		FlinkCompactionFilterFactory compactionFilterFactory = compactionFilterFactories.get(stateDesc.getName());
		Preconditions.checkNotNull(compactionFilterFactory);
		long ttl = ttlConfig.getTtl().toMilliseconds();

		StateTtlConfig.RocksdbCompactFilterCleanupStrategy rocksdbCompactFilterCleanupStrategy =
			ttlConfig.getCleanupStrategies().getRocksdbCompactFilterCleanupStrategy();
		Preconditions.checkNotNull(rocksdbCompactFilterCleanupStrategy);
		long queryTimeAfterNumEntries =
			rocksdbCompactFilterCleanupStrategy.getQueryTimeAfterNumEntries();

		FlinkCompactionFilter.Config config;
		if (stateDesc instanceof ListStateDescriptor) {
			TypeSerializer<?> elemSerializer = ((ListSerializer<?>) stateSerializer).getElementSerializer();
			int len = elemSerializer.getLength();
			if (len > 0) {
				config = FlinkCompactionFilter.Config.createForFixedElementList(
					ttl, queryTimeAfterNumEntries, len + 1); // plus one byte for list element delimiter
			} else {
				config = FlinkCompactionFilter.Config.createForList(
					ttl, queryTimeAfterNumEntries,
					new ListElementFilterFactory<>(elemSerializer.duplicate()));
			}
		} else if (stateDesc instanceof MapStateDescriptor) {
			config = FlinkCompactionFilter.Config.createForMap(ttl, queryTimeAfterNumEntries);
		} else {
			config = FlinkCompactionFilter.Config.createForValue(ttl, queryTimeAfterNumEntries);
		}
		compactionFilterFactory.configure(config);
	}
}
 
Example #26
Source File: TtlStateFactory.java    From flink with Apache License 2.0 5 votes vote down vote up
public static boolean isTtlStateSerializer(TypeSerializer<?> typeSerializer) {
	boolean ttlSerializer = typeSerializer instanceof TtlStateFactory.TtlSerializer;
	boolean ttlListSerializer = typeSerializer instanceof ListSerializer &&
		((ListSerializer) typeSerializer).getElementSerializer() instanceof TtlStateFactory.TtlSerializer;
	boolean ttlMapSerializer = typeSerializer instanceof MapSerializer &&
		((MapSerializer) typeSerializer).getValueSerializer() instanceof TtlStateFactory.TtlSerializer;
	return ttlSerializer || ttlListSerializer || ttlMapSerializer;
}
 
Example #27
Source File: HeapListState.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public byte[] getSerializedValue(
		final byte[] serializedKeyAndNamespace,
		final TypeSerializer<K> safeKeySerializer,
		final TypeSerializer<N> safeNamespaceSerializer,
		final TypeSerializer<List<V>> safeValueSerializer) throws Exception {

	Preconditions.checkNotNull(serializedKeyAndNamespace);
	Preconditions.checkNotNull(safeKeySerializer);
	Preconditions.checkNotNull(safeNamespaceSerializer);
	Preconditions.checkNotNull(safeValueSerializer);

	Tuple2<K, N> keyAndNamespace = KvStateSerializer.deserializeKeyAndNamespace(
			serializedKeyAndNamespace, safeKeySerializer, safeNamespaceSerializer);

	List<V> result = stateTable.get(keyAndNamespace.f0, keyAndNamespace.f1);

	if (result == null) {
		return null;
	}

	final TypeSerializer<V> dupSerializer = ((ListSerializer<V>) safeValueSerializer).getElementSerializer();

	ByteArrayOutputStream baos = new ByteArrayOutputStream();
	DataOutputViewStreamWrapper view = new DataOutputViewStreamWrapper(baos);

	// write the same as RocksDB writes lists, with one ',' separator
	for (int i = 0; i < result.size(); i++) {
		dupSerializer.serialize(result.get(i), view);
		if (i < result.size() -1) {
			view.writeByte(',');
		}
	}
	view.flush();

	return baos.toByteArray();
}
 
Example #28
Source File: RocksDBListState.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void migrateSerializedValue(
		DataInputDeserializer serializedOldValueInput,
		DataOutputSerializer serializedMigratedValueOutput,
		TypeSerializer<List<V>> priorSerializer,
		TypeSerializer<List<V>> newSerializer) throws StateMigrationException {

	Preconditions.checkArgument(priorSerializer instanceof ListSerializer);
	Preconditions.checkArgument(newSerializer instanceof ListSerializer);

	TypeSerializer<V> priorElementSerializer =
		((ListSerializer<V>) priorSerializer).getElementSerializer();

	TypeSerializer<V> newElementSerializer =
		((ListSerializer<V>) newSerializer).getElementSerializer();

	try {
		while (serializedOldValueInput.available() > 0) {
			V element = deserializeNextElement(serializedOldValueInput, priorElementSerializer);
			newElementSerializer.serialize(element, serializedMigratedValueOutput);
			if (serializedOldValueInput.available() > 0) {
				serializedMigratedValueOutput.write(DELIMITER);
			}
		}
	} catch (Exception e) {
		throw new StateMigrationException("Error while trying to migrate RocksDB list state.", e);
	}
}
 
Example #29
Source File: RocksDBSnapshotTransformFactoryAdaptor.java    From flink with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
static <SV, SEV> StateSnapshotTransformFactory<SV> wrapStateSnapshotTransformFactory(
	StateDescriptor<?, SV> stateDesc,
	StateSnapshotTransformFactory<SEV> snapshotTransformFactory,
	TypeSerializer<SV> stateSerializer) {
	if (stateDesc instanceof ListStateDescriptor) {
		TypeSerializer<SEV> elementSerializer = ((ListSerializer<SEV>) stateSerializer).getElementSerializer();
		return new RocksDBListStateSnapshotTransformFactory<>(snapshotTransformFactory, elementSerializer);
	} else if (stateDesc instanceof MapStateDescriptor) {
		return new RocksDBMapStateSnapshotTransformFactory<>(snapshotTransformFactory);
	} else {
		return new RocksDBValueStateSnapshotTransformFactory<>(snapshotTransformFactory);
	}
}
 
Example #30
Source File: RocksDBListState.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a new {@code RocksDBListState}.
 *
 * @param columnFamily The RocksDB column family that this state is associated to.
 * @param namespaceSerializer The serializer for the namespace.
 * @param valueSerializer The serializer for the state.
 * @param defaultValue The default value for the state.
 * @param backend The backend for which this state is bind to.
 */
private RocksDBListState(
		ColumnFamilyHandle columnFamily,
		TypeSerializer<N> namespaceSerializer,
		TypeSerializer<List<V>> valueSerializer,
		List<V> defaultValue,
		RocksDBKeyedStateBackend<K> backend) {

	super(columnFamily, namespaceSerializer, valueSerializer, defaultValue, backend);

	ListSerializer<V> castedListSerializer = (ListSerializer<V>) valueSerializer;
	this.elementSerializer = castedListSerializer.getElementSerializer();
}