Java Code Examples for org.apache.flink.api.common.typeutils.base.StringSerializer#INSTANCE

@Before
public void before() {
	str = StringData.fromString("haha");
	generic = RawValueData.fromObject("haha");
	genericSerializer = new RawValueDataSerializer<>(StringSerializer.INSTANCE);
	decimal1 = DecimalData.fromUnscaledLong(10, 5, 0);
	decimal2 = DecimalData.fromBigDecimal(new BigDecimal(11), 20, 0);
	array = new BinaryArrayData();
	{
		BinaryArrayWriter arrayWriter = new BinaryArrayWriter(array, 2, 4);
		arrayWriter.writeInt(0, 15);
		arrayWriter.writeInt(1, 16);
		arrayWriter.complete();
	}
	map = BinaryMapData.valueOf(array, array);
	underRow = new BinaryRowData(2);
	{
		BinaryRowWriter writer = new BinaryRowWriter(underRow);
		writer.writeInt(0, 15);
		writer.writeInt(1, 16);
		writer.complete();
	}
	bytes = new byte[] {1, 5, 6};
	timestamp1 = TimestampData.fromEpochMillis(123L);
	timestamp2 = TimestampData.fromLocalDateTime(LocalDateTime.of(1969, 1, 1, 0, 0, 0, 123456789));
}
 

@Test
public void testHashCodeAndEquals() throws Exception {
	final String name = "testName";

	TestStateDescriptor<String> original = new TestStateDescriptor<>(name, String.class);
	TestStateDescriptor<String> same = new TestStateDescriptor<>(name, String.class);
	TestStateDescriptor<String> sameBySerializer = new TestStateDescriptor<>(name, StringSerializer.INSTANCE);

	// test that hashCode() works on state descriptors with initialized and uninitialized serializers
	assertEquals(original.hashCode(), same.hashCode());
	assertEquals(original.hashCode(), sameBySerializer.hashCode());

	assertEquals(original, same);
	assertEquals(original, sameBySerializer);

	// equality with a clone
	TestStateDescriptor<String> clone = CommonTestUtils.createCopySerializable(original);
	assertEquals(original, clone);

	// equality with an initialized
	clone.initializeSerializerUnlessSet(new ExecutionConfig());
	assertEquals(original, clone);

	original.initializeSerializerUnlessSet(new ExecutionConfig());
	assertEquals(original, same);
}
 

@Test
public void testApplyToAllKeysLambdaFunction() throws Exception {
	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);

	try {
		ListStateDescriptor<String> listStateDescriptor =
			new ListStateDescriptor<>("foo", StringSerializer.INSTANCE);

		ListState<String> listState =
			backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, listStateDescriptor);

		for (int i = 0; i < 100; ++i) {
			backend.setCurrentKey(i);
			listState.add("Hello" + i);
		}

		// valid state value via applyToAllKeys().
		backend.applyToAllKeys(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, listStateDescriptor,
			(Integer key, ListState<String> state) -> assertEquals("Hello" + key, state.get().iterator().next())
		);
	}
	finally {
		IOUtils.closeQuietly(backend);
		backend.dispose();
	}
}
 

@Test
public void testBroadcastStateMetaInfoSerialization() throws Exception {

	String name = "test";
	TypeSerializer<?> keySerializer = DoubleSerializer.INSTANCE;
	TypeSerializer<?> valueSerializer = StringSerializer.INSTANCE;

	StateMetaInfoSnapshot snapshot =
		new RegisteredBroadcastStateBackendMetaInfo<>(
			name, OperatorStateHandle.Mode.BROADCAST, keySerializer, valueSerializer).snapshot();

	byte[] serialized;
	try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
		StateMetaInfoSnapshotReadersWriters.getWriter().
			writeStateMetaInfoSnapshot(snapshot, new DataOutputViewStreamWrapper(out));

		serialized = out.toByteArray();
	}

	try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
		final StateMetaInfoReader reader = StateMetaInfoSnapshotReadersWriters.getReader(
			CURRENT_STATE_META_INFO_SNAPSHOT_VERSION, StateMetaInfoSnapshotReadersWriters.StateTypeHint.OPERATOR_STATE);
		snapshot = reader.readStateMetaInfoSnapshot(
			new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader());
	}

	RegisteredBroadcastStateBackendMetaInfo<?, ?> restoredMetaInfo =
		new RegisteredBroadcastStateBackendMetaInfo<>(snapshot);

	Assert.assertEquals(name, restoredMetaInfo.getName());
	Assert.assertEquals(
		OperatorStateHandle.Mode.BROADCAST,
		restoredMetaInfo.getAssignmentMode());
	Assert.assertEquals(keySerializer, restoredMetaInfo.getKeySerializer());
	Assert.assertEquals(valueSerializer, restoredMetaInfo.getValueSerializer());
}
 

@Test
public void testHashCodeEquals() throws Exception {
	final String name = "testName";

	MapStateDescriptor<String, String> original = new MapStateDescriptor<>(name, String.class, String.class);
	MapStateDescriptor<String, String> same = new MapStateDescriptor<>(name, String.class, String.class);
	MapStateDescriptor<String, String> sameBySerializer =
			new MapStateDescriptor<>(name, StringSerializer.INSTANCE, StringSerializer.INSTANCE);

	// test that hashCode() works on state descriptors with initialized and uninitialized serializers
	assertEquals(original.hashCode(), same.hashCode());
	assertEquals(original.hashCode(), sameBySerializer.hashCode());

	assertEquals(original, same);
	assertEquals(original, sameBySerializer);

	// equality with a clone
	MapStateDescriptor<String, String> clone = CommonTestUtils.createCopySerializable(original);
	assertEquals(original, clone);

	// equality with an initialized
	clone.initializeSerializerUnlessSet(new ExecutionConfig());
	assertEquals(original, clone);

	original.initializeSerializerUnlessSet(new ExecutionConfig());
	assertEquals(original, same);
}
 

/**
 * Skips bytes corresponding to serialized states. In flink 1.6+ the states are no longer kept in state.
 */
static void skipSerializedStates(DataInputView in) throws IOException {
	TypeSerializer<String> nameSerializer = StringSerializer.INSTANCE;
	TypeSerializer<State.StateType> stateTypeSerializer = new EnumSerializer<>(State.StateType.class);
	TypeSerializer<StateTransitionAction> actionSerializer = new EnumSerializer<>(StateTransitionAction.class);

	final int noOfStates = in.readInt();

	for (int i = 0; i < noOfStates; i++) {
		nameSerializer.deserialize(in);
		stateTypeSerializer.deserialize(in);
	}

	for (int i = 0; i < noOfStates; i++) {
		String srcName = nameSerializer.deserialize(in);

		int noOfTransitions = in.readInt();
		for (int j = 0; j < noOfTransitions; j++) {
			String src = nameSerializer.deserialize(in);
			Preconditions.checkState(src.equals(srcName),
				"Source Edge names do not match (" + srcName + " - " + src + ").");

			nameSerializer.deserialize(in);
			actionSerializer.deserialize(in);

			try {
				skipCondition(in);
			} catch (ClassNotFoundException e) {
				e.printStackTrace();
			}
		}
	}
}
 

@Override
public InternalTimerService<VoidNamespace> createTimerService(
    Triggerable<String, VoidNamespace> triggerable) {
  final TimerSerializer<String, VoidNamespace> timerSerializer =
      new TimerSerializer<>(StringSerializer.INSTANCE, VoidNamespaceSerializer.INSTANCE);

  return timeServiceManager.getInternalTimerService(
      DELAYED_MSG_TIMER_SERVICE_NAME, timerSerializer, triggerable);
}
 

@Test
public void testCorrectMergeOperatorSet() throws Exception {
	prepareRocksDB();
	final ColumnFamilyOptions columnFamilyOptions = spy(new ColumnFamilyOptions());
	RocksDBKeyedStateBackend<Integer> test = null;

	try {
		test = RocksDBTestUtils.builderForTestDB(
			tempFolder.newFolder(),
			IntSerializer.INSTANCE,
			db,
			defaultCFHandle,
			columnFamilyOptions)
			.setEnableIncrementalCheckpointing(enableIncrementalCheckpointing)
			.build();

		ValueStateDescriptor<String> stubState1 =
			new ValueStateDescriptor<>("StubState-1", StringSerializer.INSTANCE);
		test.createInternalState(StringSerializer.INSTANCE, stubState1);
		ValueStateDescriptor<String> stubState2 =
			new ValueStateDescriptor<>("StubState-2", StringSerializer.INSTANCE);
		test.createInternalState(StringSerializer.INSTANCE, stubState2);

		// The default CF is pre-created so sum up to 2 times (once for each stub state)
		verify(columnFamilyOptions, Mockito.times(2))
			.setMergeOperatorName(RocksDBKeyedStateBackend.MERGE_OPERATOR_NAME);
	} finally {
		if (test != null) {
			IOUtils.closeQuietly(test);
			test.dispose();
		}
		columnFamilyOptions.close();
	}
}
 

@Test
public void testHashCodeEquals() throws Exception {
	final String name = "testName";
	final ReduceFunction<String> reducer = (a, b) -> a;

	ReducingStateDescriptor<String> original = new ReducingStateDescriptor<>(name, reducer, String.class);
	ReducingStateDescriptor<String> same = new ReducingStateDescriptor<>(name, reducer, String.class);
	ReducingStateDescriptor<String> sameBySerializer = new ReducingStateDescriptor<>(name, reducer, StringSerializer.INSTANCE);

	// test that hashCode() works on state descriptors with initialized and uninitialized serializers
	assertEquals(original.hashCode(), same.hashCode());
	assertEquals(original.hashCode(), sameBySerializer.hashCode());

	assertEquals(original, same);
	assertEquals(original, sameBySerializer);

	// equality with a clone
	ReducingStateDescriptor<String> clone = CommonTestUtils.createCopySerializable(original);
	assertEquals(original, clone);

	// equality with an initialized
	clone.initializeSerializerUnlessSet(new ExecutionConfig());
	assertEquals(original, clone);

	original.initializeSerializerUnlessSet(new ExecutionConfig());
	assertEquals(original, same);
}
 

@Test
public void testHashCodeEquals() throws Exception {
	final String name = "testName";

	MapStateDescriptor<String, String> original = new MapStateDescriptor<>(name, String.class, String.class);
	MapStateDescriptor<String, String> same = new MapStateDescriptor<>(name, String.class, String.class);
	MapStateDescriptor<String, String> sameBySerializer =
			new MapStateDescriptor<>(name, StringSerializer.INSTANCE, StringSerializer.INSTANCE);

	// test that hashCode() works on state descriptors with initialized and uninitialized serializers
	assertEquals(original.hashCode(), same.hashCode());
	assertEquals(original.hashCode(), sameBySerializer.hashCode());

	assertEquals(original, same);
	assertEquals(original, sameBySerializer);

	// equality with a clone
	MapStateDescriptor<String, String> clone = CommonTestUtils.createCopySerializable(original);
	assertEquals(original, clone);

	// equality with an initialized
	clone.initializeSerializerUnlessSet(new ExecutionConfig());
	assertEquals(original, clone);

	original.initializeSerializerUnlessSet(new ExecutionConfig());
	assertEquals(original, same);
}
 

@SuppressWarnings("unchecked")
@Override
public <US extends State, SV> StateDescriptor<US, SV> createStateDescriptor() {
	return (StateDescriptor<US, SV>) new MapStateDescriptor<>(
		getName(), IntSerializer.INSTANCE, StringSerializer.INSTANCE);
}
 

TtlNonFixedLenElemListStateTestContext() {
	super(StringSerializer.INSTANCE);
}
 

@Override
public TypeSerializer<Option<String>> createUpgradedSerializer() {
	return new OptionSerializer<>(StringSerializer.INSTANCE);
}
 

@SuppressWarnings({"unchecked", "rawtypes"})
@Override
public TypeSerializer<BufferEntry<String>> createPriorSerializer() {
	return new BufferEntrySerializer(StringSerializer.INSTANCE);
}
 

@Test
public void testCorrectMergeOperatorSet() throws Exception {
	prepareRocksDB();
	final ColumnFamilyOptions columnFamilyOptions = spy(new ColumnFamilyOptions());
	RocksDBKeyedStateBackend<Integer> test = null;
	try (DBOptions options = new DBOptions().setCreateIfMissing(true)) {
		ExecutionConfig executionConfig = new ExecutionConfig();
		test = new RocksDBKeyedStateBackendBuilder<>(
			"test",
			Thread.currentThread().getContextClassLoader(),
			tempFolder.newFolder(),
			options,
			stateName -> columnFamilyOptions,
			mock(TaskKvStateRegistry.class),
			IntSerializer.INSTANCE,
			1,
			new KeyGroupRange(0, 0),
			executionConfig,
			mock(LocalRecoveryConfig.class),
			RocksDBStateBackend.PriorityQueueStateType.HEAP,
			TtlTimeProvider.DEFAULT,
			new UnregisteredMetricsGroup(),
			Collections.emptyList(),
			AbstractStateBackend.getCompressionDecorator(executionConfig),
			db,
			defaultCFHandle,
			new CloseableRegistry())
			.setEnableIncrementalCheckpointing(enableIncrementalCheckpointing)
			.build();
		ValueStateDescriptor<String> stubState1 =
			new ValueStateDescriptor<>("StubState-1", StringSerializer.INSTANCE);
		test.createInternalState(StringSerializer.INSTANCE, stubState1);
		ValueStateDescriptor<String> stubState2 =
			new ValueStateDescriptor<>("StubState-2", StringSerializer.INSTANCE);
		test.createInternalState(StringSerializer.INSTANCE, stubState2);

		// The default CF is pre-created so sum up to 2 times (once for each stub state)
		verify(columnFamilyOptions, Mockito.times(2))
			.setMergeOperatorName(RocksDBKeyedStateBackend.MERGE_OPERATOR_NAME);
	} finally {
		if (test != null) {
			IOUtils.closeQuietly(test);
			test.dispose();
		}
		columnFamilyOptions.close();
	}
}
 

@Override
public TypeSerializer<Either<String, Integer>> createPriorSerializer() {
	return new EitherSerializer<>(StringSerializer.INSTANCE, IntSerializer.INSTANCE);
}
 

@Override
protected TypeSerializer<Either<String, Integer>> createSerializer() {
	return new EitherSerializer<>(StringSerializer.INSTANCE, IntSerializer.INSTANCE);
}
 

@Override
public TypeSerializer<TaggedUnion<String, Long>> createPriorSerializer() {
	return new UnionSerializer<>(StringSerializer.INSTANCE, LongSerializer.INSTANCE);
}
 

@Override
public TypeSerializer<Lockable<String>> createPriorSerializer() {
	return new Lockable.LockableTypeSerializer<>(StringSerializer.INSTANCE);
}
 

@Test
@SuppressWarnings("unchecked")
public void testMultipleValueStates() throws Exception {
	CheckpointStreamFactory streamFactory = createStreamFactory();
	SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(
			IntSerializer.INSTANCE,
			1,
			new KeyGroupRange(0, 0),
			new DummyEnvironment());

	ValueStateDescriptor<String> desc1 = new ValueStateDescriptor<>("a-string", StringSerializer.INSTANCE);
	ValueStateDescriptor<Integer> desc2 = new ValueStateDescriptor<>("an-integer", IntSerializer.INSTANCE);

	ValueState<String> state1 = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc1);
	ValueState<Integer> state2 = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc2);

	// some modifications to the state
	backend.setCurrentKey(1);
	assertNull(state1.value());
	assertNull(state2.value());
	state1.update("1");

	// state2 should still have nothing
	assertEquals("1", state1.value());
	assertNull(state2.value());
	state2.update(13);

	// both have some state now
	assertEquals("1", state1.value());
	assertEquals(13, (int) state2.value());

	// draw a snapshot
	KeyedStateHandle snapshot1 =
		runSnapshot(
			backend.snapshot(682375462378L, 2, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()),
			sharedStateRegistry);

	backend.dispose();
	backend = restoreKeyedBackend(
			IntSerializer.INSTANCE,
			1,
			new KeyGroupRange(0, 0),
			Collections.singletonList(snapshot1),
			new DummyEnvironment());

	snapshot1.discardState();

	backend.setCurrentKey(1);

	state1 = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc1);
	state2 = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc2);

	// verify that they are still the same
	assertEquals("1", state1.value());
	assertEquals(13, (int) state2.value());

	backend.dispose();
}