Java Code Examples for org.apache.flink.api.common.state.MapState#put()

The following examples show how to use org.apache.flink.api.common.state.MapState#put() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: StateBackendTestBase.java From Flink-CEPplus with Apache License 2.0

6 votes

/**
 * Verify that an empty {@code MapState} yields {@code null}.
 */
@Test
public void testMapStateDefaultValue() throws Exception {
	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);

	MapStateDescriptor<String, String> kvId = new MapStateDescriptor<>("id", String.class, String.class);

	MapState<String, String> state = backend.getPartitionedState(
			VoidNamespace.INSTANCE,
			VoidNamespaceSerializer.INSTANCE, kvId);

	backend.setCurrentKey(1);
	assertNull(state.entries());

	state.put("Ciao", "Hello");
	state.put("Bello", "Nice");

	assertNotNull(state.entries());
	assertEquals(state.get("Ciao"), "Hello");
	assertEquals(state.get("Bello"), "Nice");

	state.clear();
	assertNull(state.entries());

	backend.dispose();
}

Example 2

Source File: StateBackendTestBase.java From flink with Apache License 2.0

6 votes

/**
 * Verify that an empty {@code MapState} yields {@code null}.
 */
@Test
public void testMapStateDefaultValue() throws Exception {
	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);

	MapStateDescriptor<String, String> kvId = new MapStateDescriptor<>("id", String.class, String.class);

	MapState<String, String> state = backend.getPartitionedState(
			VoidNamespace.INSTANCE,
			VoidNamespaceSerializer.INSTANCE, kvId);

	backend.setCurrentKey(1);
	assertNull(state.entries());

	state.put("Ciao", "Hello");
	state.put("Bello", "Nice");

	assertNotNull(state.entries());
	assertEquals(state.get("Ciao"), "Hello");
	assertEquals(state.get("Bello"), "Nice");

	state.clear();
	assertNull(state.entries());

	backend.dispose();
}

Example 3

Source File: IntervalJoinOperator.java From flink with Apache License 2.0

5 votes

private static <T> void addToBuffer(
		final MapState<Long, List<IntervalJoinOperator.BufferEntry<T>>> buffer,
		final T value,
		final long timestamp) throws Exception {
	List<BufferEntry<T>> elemsInBucket = buffer.get(timestamp);
	if (elemsInBucket == null) {
		elemsInBucket = new ArrayList<>();
	}
	elemsInBucket.add(new BufferEntry<>(value, false));
	buffer.put(timestamp, elemsInBucket);
}

Example 4

Source File: IntervalJoinOperator.java From Flink-CEPplus with Apache License 2.0

5 votes

private static <T> void addToBuffer(
		final MapState<Long, List<IntervalJoinOperator.BufferEntry<T>>> buffer,
		final T value,
		final long timestamp) throws Exception {
	List<BufferEntry<T>> elemsInBucket = buffer.get(timestamp);
	if (elemsInBucket == null) {
		elemsInBucket = new ArrayList<>();
	}
	elemsInBucket.add(new BufferEntry<>(value, false));
	buffer.put(timestamp, elemsInBucket);
}

Example 5

Source File: StateBackendTestBase.java From flink with Apache License 2.0

5 votes

/**
 * Verify that an empty {@code MapState} yields {@code null}.
 */
@Test
public void testMapStateDefaultValue() throws Exception {
	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);

	MapStateDescriptor<String, String> kvId = new MapStateDescriptor<>("id", String.class, String.class);

	MapState<String, String> state = backend.getPartitionedState(
			VoidNamespace.INSTANCE,
			VoidNamespaceSerializer.INSTANCE, kvId);

	backend.setCurrentKey(1);
	assertNotNull(state.entries());
	assertFalse(state.entries().iterator().hasNext());

	state.put("Ciao", "Hello");
	state.put("Bello", "Nice");

	assertNotNull(state.entries());
	assertEquals(state.get("Ciao"), "Hello");
	assertEquals(state.get("Bello"), "Nice");

	state.clear();
	assertNotNull(state.entries());
	assertFalse(state.entries().iterator().hasNext());

	backend.dispose();
}

Example 6

Source File: StateBackendTestBase.java From flink with Apache License 2.0

5 votes

@Test
public void testMapStateIsEmpty() throws Exception {
	MapStateDescriptor<Integer, Long> kvId = new MapStateDescriptor<>("id", Integer.class, Long.class);

	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);

	try {
		MapState<Integer, Long> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
		backend.setCurrentKey(1);
		assertTrue(state.isEmpty());

		int stateSize = 1024;
		for (int i = 0; i < stateSize; i++) {
			state.put(i, i * 2L);
			assertFalse(state.isEmpty());
		}

		for (int i = 0; i < stateSize; i++) {
			assertFalse(state.isEmpty());
			state.remove(i);
		}
		assertTrue(state.isEmpty());

	} finally {
		backend.dispose();
	}
}

Example 7

Source File: RocksDBStateMisuseOptionTest.java From flink with Apache License 2.0

5 votes

/**
 * Tests to cover case when user misuse optimizeForPointLookup with iterator interfaces on map state.
 *
 * <p>The option {@link ColumnFamilyOptions#optimizeForPointLookup(long)} would lead to iterator.seek with prefix bytes invalid.
 */
@Test
public void testMisuseOptimizePointLookupWithMapState() throws Exception {
	RocksDBStateBackend rocksDBStateBackend = createStateBackendWithOptimizePointLookup();
	RocksDBKeyedStateBackend<Integer> keyedStateBackend =
		createKeyedStateBackend(rocksDBStateBackend, new MockEnvironmentBuilder().build(), IntSerializer.INSTANCE);
	try {
		MapStateDescriptor<Integer, Long> stateDescriptor = new MapStateDescriptor<>("map", IntSerializer.INSTANCE, LongSerializer.INSTANCE);
		MapState<Integer, Long> mapState = keyedStateBackend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, stateDescriptor);

		keyedStateBackend.setCurrentKey(1);
		Map<Integer, Long> expectedResult = new HashMap<>();
		for (int i = 0; i < 100; i++) {
			long uv = ThreadLocalRandom.current().nextLong();
			mapState.put(i, uv);
			expectedResult.put(i, uv);
		}

		Iterator<Map.Entry<Integer, Long>> iterator = mapState.entries().iterator();
		while (iterator.hasNext()) {
			Map.Entry<Integer, Long> entry = iterator.next();
			assertEquals(entry.getValue(), expectedResult.remove(entry.getKey()));
			iterator.remove();
		}
		assertTrue(expectedResult.isEmpty());
		assertTrue(mapState.isEmpty());
	} finally {
		keyedStateBackend.dispose();
	}
}

Example 8

Source File: BroadcastState.java From flink-training-exercises with Apache License 2.0

5 votes

@Override
public void processElement(Item nextItem, ReadOnlyContext ctx, Collector<String> out) throws Exception {

	final MapState<String, List<Item>> partialMatches = getRuntimeContext().getMapState(matchStateDesc);
	final Shape shapeOfNextItem = nextItem.getShape();

	System.out.println("SAW: " + nextItem);
	for (Map.Entry<String, Tuple2<Shape, Shape>> entry: ctx.getBroadcastState(broadcastStateDescriptor).immutableEntries()) {
		final String ruleName = entry.getKey();
		final Tuple2<Shape, Shape> rule = entry.getValue();

		List<Item> partialsForThisRule = partialMatches.get(ruleName);
		if (partialsForThisRule == null) {
			partialsForThisRule = new ArrayList<>();
		}

		if (shapeOfNextItem == rule.f1 && !partialsForThisRule.isEmpty()) {
			for (Item i : partialsForThisRule) {
				out.collect("MATCH: " + i + " - " + nextItem);
			}
			partialsForThisRule.clear();
		}

		if (shapeOfNextItem == rule.f0) {
			partialsForThisRule.add(nextItem);
		}

		if (partialsForThisRule.isEmpty()) {
			partialMatches.remove(ruleName);
		} else {
			partialMatches.put(ruleName, partialsForThisRule);
		}
	}
}

Example 9

Source File: IntervalJoinOperator.java From flink with Apache License 2.0

5 votes

private static <T> void addToBuffer(
		final MapState<Long, List<IntervalJoinOperator.BufferEntry<T>>> buffer,
		final T value,
		final long timestamp) throws Exception {
	List<BufferEntry<T>> elemsInBucket = buffer.get(timestamp);
	if (elemsInBucket == null) {
		elemsInBucket = new ArrayList<>();
	}
	elemsInBucket.add(new BufferEntry<>(value, false));
	buffer.put(timestamp, elemsInBucket);
}

Example 10

Source File: StateBackendTestBase.java From flink with Apache License 2.0

4 votes

@Test
@SuppressWarnings("unchecked")
public void testMapStateRestoreWithWrongSerializers() throws Exception {
	CheckpointStreamFactory streamFactory = createStreamFactory();
	SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);

	try {
		MapStateDescriptor<String, String> kvId = new MapStateDescriptor<>("id", StringSerializer.INSTANCE, StringSerializer.INSTANCE);
		MapState<String, String> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);

		backend.setCurrentKey(1);
		state.put("1", "First");
		backend.setCurrentKey(2);
		state.put("2", "Second");

		// draw a snapshot
		KeyedStateHandle snapshot1 = runSnapshot(
			backend.snapshot(682375462378L, 2, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()),
			sharedStateRegistry);

		backend.dispose();
		// restore the first snapshot and validate it
		backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot1);
		snapshot1.discardState();

		@SuppressWarnings("unchecked")
		TypeSerializer<String> fakeStringSerializer =
			(TypeSerializer<String>) (TypeSerializer<?>) FloatSerializer.INSTANCE;

		try {
			kvId = new MapStateDescriptor<>("id", fakeStringSerializer, StringSerializer.INSTANCE);

			state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);

			state.entries();

			fail("should recognize wrong serializers");
		} catch (StateMigrationException ignored) {
			// expected
		}
		backend.dispose();
	} finally {
		backend.dispose();
	}
}

Example 11

Source File: TtlMapStateVerifier.java From flink with Apache License 2.0

4 votes

@Override
void updateInternal(@Nonnull MapState<String, String> state, Tuple2<String, String> update) throws Exception {
	state.put(update.f0, update.f1);
}

Example 12

Source File: StateBackendMigrationTestBase.java From flink with Apache License 2.0

4 votes

private void testKeyedMapStateUpgrade(
	MapStateDescriptor<Integer, TestType> initialAccessDescriptor,
	MapStateDescriptor<Integer, TestType> newAccessDescriptorAfterRestore) throws Exception {
	CheckpointStreamFactory streamFactory = createStreamFactory();
	SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();

	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);

	try {
		MapState<Integer, TestType> mapState = backend.getPartitionedState(
			VoidNamespace.INSTANCE,
			CustomVoidNamespaceSerializer.INSTANCE,
			initialAccessDescriptor);

		backend.setCurrentKey(1);
		mapState.put(1, new TestType("key-1", 1));
		mapState.put(2, new TestType("key-1", 2));
		mapState.put(3, new TestType("key-1", 3));

		backend.setCurrentKey(2);
		mapState.put(1, new TestType("key-2", 1));

		backend.setCurrentKey(3);
		mapState.put(1, new TestType("key-3", 1));
		mapState.put(2, new TestType("key-3", 2));

		KeyedStateHandle snapshot = runSnapshot(
			backend.snapshot(1L, 2L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()),
			sharedStateRegistry);
		backend.dispose();

		backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot);

		mapState = backend.getPartitionedState(
			VoidNamespace.INSTANCE,
			CustomVoidNamespaceSerializer.INSTANCE,
			newAccessDescriptorAfterRestore);

		// make sure that reading and writing each key state works with the new serializer
		backend.setCurrentKey(1);
		Iterator<Map.Entry<Integer, TestType>> iterable1 = mapState.iterator();
		Map.Entry<Integer, TestType> actual = iterable1.next();
		assertEquals((Integer) 1, actual.getKey());
		assertEquals(new TestType("key-1", 1), actual.getValue());

		actual = iterable1.next();
		assertEquals((Integer) 2, actual.getKey());
		assertEquals(new TestType("key-1", 2), actual.getValue());

		actual = iterable1.next();
		assertEquals((Integer) 3, actual.getKey());
		assertEquals(new TestType("key-1", 3), actual.getValue());

		Assert.assertFalse(iterable1.hasNext());

		mapState.put(123, new TestType("new-key-1", 123));

		backend.setCurrentKey(2);
		Iterator<Map.Entry<Integer, TestType>> iterable2 = mapState.iterator();

		actual = iterable2.next();
		assertEquals((Integer) 1, actual.getKey());
		assertEquals(new TestType("key-2", 1), actual.getValue());
		Assert.assertFalse(iterable2.hasNext());

		mapState.put(456, new TestType("new-key-2", 456));

		backend.setCurrentKey(3);
		Iterator<Map.Entry<Integer, TestType>> iterable3 = mapState.iterator();

		actual = iterable3.next();
		assertEquals((Integer) 1, actual.getKey());
		assertEquals(new TestType("key-3", 1), actual.getValue());

		actual = iterable3.next();
		assertEquals((Integer) 2, actual.getKey());
		assertEquals(new TestType("key-3", 2), actual.getValue());

		Assert.assertFalse(iterable3.hasNext());
		mapState.put(777, new TestType("new-key-3", 777));

		// do another snapshot to verify the snapshot logic after migration
		snapshot = runSnapshot(
			backend.snapshot(2L, 3L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()),
			sharedStateRegistry);
		snapshot.discardState();

	} finally {
		backend.dispose();
	}
}

Example 13

Source File: StateBackendTestBase.java From flink with Apache License 2.0

4 votes

/**
 * Verify iterator of {@link MapState} supporting arbitrary access, see [FLINK-10267] to know more details.
 */
@Test
public void testMapStateIteratorArbitraryAccess() throws Exception {
	MapStateDescriptor<Integer, Long> kvId = new MapStateDescriptor<>("id", Integer.class, Long.class);

	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);

	try {
		MapState<Integer, Long> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
		backend.setCurrentKey(1);
		int stateSize = 4096;
		for (int i = 0; i < stateSize; i++) {
			state.put(i, i * 2L);
		}
		Iterator<Map.Entry<Integer, Long>> iterator = state.iterator();
		int iteratorCount = 0;
		while (iterator.hasNext()) {
			Map.Entry<Integer, Long> entry = iterator.next();
			assertEquals(iteratorCount, (int) entry.getKey());
			switch (ThreadLocalRandom.current().nextInt() % 3) {
				case 0: // remove twice
					iterator.remove();
					try {
						iterator.remove();
						fail();
					} catch (IllegalStateException e) {
						// ignore expected exception
					}
					break;
				case 1: // hasNext -> remove
					iterator.hasNext();
					iterator.remove();
					break;
				case 2: // nothing to do
					break;
			}
			iteratorCount++;
		}
		assertEquals(stateSize, iteratorCount);
	} finally {
		backend.dispose();
	}
}

Example 14

Source File: TtlMapStateVerifier.java From flink with Apache License 2.0

4 votes

@Override
void updateInternal(@Nonnull MapState<String, String> state, Tuple2<String, String> update) throws Exception {
	state.put(update.f0, update.f1);
}

Example 15

Source File: StateBackendMigrationTestBase.java From flink with Apache License 2.0

4 votes

private void testKeyedMapStateUpgrade(
	MapStateDescriptor<Integer, TestType> initialAccessDescriptor,
	MapStateDescriptor<Integer, TestType> newAccessDescriptorAfterRestore) throws Exception {
	CheckpointStreamFactory streamFactory = createStreamFactory();
	SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();

	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);

	try {
		MapState<Integer, TestType> mapState = backend.getPartitionedState(
			VoidNamespace.INSTANCE,
			CustomVoidNamespaceSerializer.INSTANCE,
			initialAccessDescriptor);

		backend.setCurrentKey(1);
		mapState.put(1, new TestType("key-1", 1));
		mapState.put(2, new TestType("key-1", 2));
		mapState.put(3, new TestType("key-1", 3));

		backend.setCurrentKey(2);
		mapState.put(1, new TestType("key-2", 1));

		backend.setCurrentKey(3);
		mapState.put(1, new TestType("key-3", 1));
		mapState.put(2, new TestType("key-3", 2));

		KeyedStateHandle snapshot = runSnapshot(
			backend.snapshot(1L, 2L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()),
			sharedStateRegistry);
		backend.dispose();

		backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot);

		mapState = backend.getPartitionedState(
			VoidNamespace.INSTANCE,
			CustomVoidNamespaceSerializer.INSTANCE,
			newAccessDescriptorAfterRestore);

		// make sure that reading and writing each key state works with the new serializer
		backend.setCurrentKey(1);
		Iterator<Map.Entry<Integer, TestType>> iterable1 = mapState.iterator();
		Map.Entry<Integer, TestType> actual = iterable1.next();
		assertEquals((Integer) 1, actual.getKey());
		assertEquals(new TestType("key-1", 1), actual.getValue());

		actual = iterable1.next();
		assertEquals((Integer) 2, actual.getKey());
		assertEquals(new TestType("key-1", 2), actual.getValue());

		actual = iterable1.next();
		assertEquals((Integer) 3, actual.getKey());
		assertEquals(new TestType("key-1", 3), actual.getValue());

		Assert.assertFalse(iterable1.hasNext());

		mapState.put(123, new TestType("new-key-1", 123));

		backend.setCurrentKey(2);
		Iterator<Map.Entry<Integer, TestType>> iterable2 = mapState.iterator();

		actual = iterable2.next();
		assertEquals((Integer) 1, actual.getKey());
		assertEquals(new TestType("key-2", 1), actual.getValue());
		Assert.assertFalse(iterable2.hasNext());

		mapState.put(456, new TestType("new-key-2", 456));

		backend.setCurrentKey(3);
		Iterator<Map.Entry<Integer, TestType>> iterable3 = mapState.iterator();

		actual = iterable3.next();
		assertEquals((Integer) 1, actual.getKey());
		assertEquals(new TestType("key-3", 1), actual.getValue());

		actual = iterable3.next();
		assertEquals((Integer) 2, actual.getKey());
		assertEquals(new TestType("key-3", 2), actual.getValue());

		Assert.assertFalse(iterable3.hasNext());
		mapState.put(777, new TestType("new-key-3", 777));

		// do another snapshot to verify the snapshot logic after migration
		snapshot = runSnapshot(
			backend.snapshot(2L, 3L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()),
			sharedStateRegistry);
		snapshot.discardState();

	} finally {
		backend.dispose();
	}
}

Example 16

Source File: StateBackendTestBase.java From flink with Apache License 2.0

4 votes

@Test
@SuppressWarnings("unchecked")
public void testMapStateRestoreWithWrongSerializers() throws Exception {
	CheckpointStreamFactory streamFactory = createStreamFactory();
	SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);

	try {
		MapStateDescriptor<String, String> kvId = new MapStateDescriptor<>("id", StringSerializer.INSTANCE, StringSerializer.INSTANCE);
		MapState<String, String> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);

		backend.setCurrentKey(1);
		state.put("1", "First");
		backend.setCurrentKey(2);
		state.put("2", "Second");

		// draw a snapshot
		KeyedStateHandle snapshot1 = runSnapshot(
			backend.snapshot(682375462378L, 2, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()),
			sharedStateRegistry);

		backend.dispose();
		// restore the first snapshot and validate it
		backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot1);
		snapshot1.discardState();

		@SuppressWarnings("unchecked")
		TypeSerializer<String> fakeStringSerializer =
			(TypeSerializer<String>) (TypeSerializer<?>) FloatSerializer.INSTANCE;

		try {
			kvId = new MapStateDescriptor<>("id", fakeStringSerializer, StringSerializer.INSTANCE);

			state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);

			state.entries();

			fail("should recognize wrong serializers");
		} catch (StateMigrationException ignored) {
			// expected
		}
		backend.dispose();
	} finally {
		backend.dispose();
	}
}

Example 17

Source File: StateBackendTestBase.java From flink with Apache License 2.0

4 votes

@Test
public void testSnapshotNonAccessedState() throws Exception {
	CheckpointStreamFactory streamFactory = createStreamFactory();
	SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
	AbstractKeyedStateBackend<String> backend = createKeyedBackend(StringSerializer.INSTANCE);

	final String stateName = "test-name";
	try {
		MapStateDescriptor<Integer, String> kvId = new MapStateDescriptor<>(stateName, Integer.class, String.class);
		MapState<Integer, String> mapState = backend
			.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);

		// write some state to be snapshotted
		backend.setCurrentKey("1");
		mapState.put(11, "foo");
		backend.setCurrentKey("2");
		mapState.put(8, "bar");
		backend.setCurrentKey("3");
		mapState.put(91, "hello world");

		// take a snapshot, and then restore backend with snapshot
		KeyedStateHandle snapshot = runSnapshot(
			backend.snapshot(1L, 2L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()),
			sharedStateRegistry);
		backend.dispose();

		backend = restoreKeyedBackend(StringSerializer.INSTANCE, snapshot);

		// now take a snapshot again without accessing the state
		snapshot = runSnapshot(
			backend.snapshot(2L, 3L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()),
			sharedStateRegistry);
		backend.dispose();

		// we restore again and try to access previous state
		backend = restoreKeyedBackend(StringSerializer.INSTANCE, snapshot);
		mapState = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);

		backend.setCurrentKey("1");
		assertEquals("foo", mapState.get(11));
		backend.setCurrentKey("2");
		assertEquals("bar", mapState.get(8));
		backend.setCurrentKey("3");
		assertEquals("hello world", mapState.get(91));

		snapshot.discardState();
	} finally {
		backend.dispose();
	}
}

Example 18

Source File: StateBackendTestBase.java From flink with Apache License 2.0

4 votes

/**
 * Verify iterator of {@link MapState} supporting arbitrary access, see [FLINK-10267] to know more details.
 */
@Test
public void testMapStateIteratorArbitraryAccess() throws Exception {
	MapStateDescriptor<Integer, Long> kvId = new MapStateDescriptor<>("id", Integer.class, Long.class);

	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);

	try {
		MapState<Integer, Long> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
		backend.setCurrentKey(1);
		int stateSize = 4096;
		for (int i = 0; i < stateSize; i++) {
			state.put(i, i * 2L);
		}
		Iterator<Map.Entry<Integer, Long>> iterator = state.iterator();
		int iteratorCount = 0;
		while (iterator.hasNext()) {
			Map.Entry<Integer, Long> entry = iterator.next();
			assertEquals(iteratorCount, (int) entry.getKey());
			switch (ThreadLocalRandom.current().nextInt() % 3) {
				case 0: // remove twice
					iterator.remove();
					try {
						iterator.remove();
						fail();
					} catch (IllegalStateException e) {
						// ignore expected exception
					}
					break;
				case 1: // hasNext -> remove
					iterator.hasNext();
					iterator.remove();
					break;
				case 2: // nothing to do
					break;
			}
			iteratorCount++;
		}
		assertEquals(stateSize, iteratorCount);
	} finally {
		backend.dispose();
	}
}

Example 19

Source File: TtlMapStateVerifier.java From Flink-CEPplus with Apache License 2.0

4 votes

@Override
void updateInternal(@Nonnull MapState<String, String> state, Tuple2<String, String> update) throws Exception {
	state.put(update.f0, update.f1);
}

Example 20

Source File: StateBackendTestBase.java From Flink-CEPplus with Apache License 2.0

4 votes

@Test
public void testSnapshotNonAccessedState() throws Exception {
	CheckpointStreamFactory streamFactory = createStreamFactory();
	SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
	AbstractKeyedStateBackend<String> backend = createKeyedBackend(StringSerializer.INSTANCE);

	final String stateName = "test-name";
	try {
		MapStateDescriptor<Integer, String> kvId = new MapStateDescriptor<>(stateName, Integer.class, String.class);
		MapState<Integer, String> mapState = backend
			.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);

		// write some state to be snapshotted
		backend.setCurrentKey("1");
		mapState.put(11, "foo");
		backend.setCurrentKey("2");
		mapState.put(8, "bar");
		backend.setCurrentKey("3");
		mapState.put(91, "hello world");

		// take a snapshot, and then restore backend with snapshot
		KeyedStateHandle snapshot = runSnapshot(
			backend.snapshot(1L, 2L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()),
			sharedStateRegistry);
		backend.dispose();

		backend = restoreKeyedBackend(StringSerializer.INSTANCE, snapshot);

		// now take a snapshot again without accessing the state
		snapshot = runSnapshot(
			backend.snapshot(2L, 3L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()),
			sharedStateRegistry);
		backend.dispose();

		// we restore again and try to access previous state
		backend = restoreKeyedBackend(StringSerializer.INSTANCE, snapshot);
		mapState = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);

		backend.setCurrentKey("1");
		assertEquals("foo", mapState.get(11));
		backend.setCurrentKey("2");
		assertEquals("bar", mapState.get(8));
		backend.setCurrentKey("3");
		assertEquals("hello world", mapState.get(91));

		snapshot.discardState();
	} finally {
		backend.dispose();
	}
}