Java Code Examples for org.apache.flink.api.common.state.ListState

The following examples show how to use org.apache.flink.api.common.state.ListState. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Flink-CEPplus   Source File: MigrationTestUtils.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
	ListState<String> unionListState = context.getOperatorStateStore().getListState(
			CheckpointingNonParallelSourceWithListState.STATE_DESCRIPTOR);

	if (context.isRestored()) {
		assertThat(unionListState.get(),
				containsInAnyOrder(
						CheckpointingNonParallelSourceWithListState.CHECKPOINTED_STRING,
						CheckpointingNonParallelSourceWithListState.CHECKPOINTED_STRING_1,
						CheckpointingNonParallelSourceWithListState.CHECKPOINTED_STRING_2,
						CheckpointingNonParallelSourceWithListState.CHECKPOINTED_STRING_3));

		getRuntimeContext().addAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, new IntCounter());
		getRuntimeContext().getAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR).add(1);
	} else {
		throw new RuntimeException(
				"This source should always be restored because it's only used when restoring from a savepoint.");
	}
}
 
Example 2
Source Project: Flink-CEPplus   Source File: MigrationTestUtils.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
	ListState<String> unionListState = context.getOperatorStateStore().getUnionListState(
			CheckpointingNonParallelSourceWithListState.STATE_DESCRIPTOR);

	if (context.isRestored()) {
		assertThat(unionListState.get(),
				containsInAnyOrder(CheckpointingParallelSourceWithUnionListState.CHECKPOINTED_STRINGS));

		getRuntimeContext().addAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, new IntCounter());
		getRuntimeContext().getAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR).add(1);
	} else {
		throw new RuntimeException(
				"This source should always be restored because it's only used when restoring from a savepoint.");
	}
}
 
Example 3
Source Project: flink   Source File: StateBackendTestBase.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Verify that an empty {@code ListState} yields {@code null}.
 */
@Test
public void testListStateDefaultValue() throws Exception {
	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);

	ListStateDescriptor<String> kvId = new ListStateDescriptor<>("id", String.class);

	ListState<String> state = backend.getPartitionedState(
			VoidNamespace.INSTANCE,
			VoidNamespaceSerializer.INSTANCE, kvId);

	backend.setCurrentKey(1);
	assertNull(state.get());

	state.update(Arrays.asList("Ciao", "Bello"));
	assertThat(state.get(), containsInAnyOrder("Ciao", "Bello"));

	state.clear();
	assertNull(state.get());

	backend.dispose();
}
 
Example 4
Source Project: flink   Source File: StateBackendBenchmarkUtilsTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testApplyToAllKeys() throws Exception {
	KeyedStateBackend<Long> backend = createKeyedStateBackend(backendType);
	ListState<Long> listState = getListState(backend, listStateDescriptor);
	for (long i = 0; i < 10; i++) {
		backend.setCurrentKey(i);
		listState.add(i);
	}
	applyToAllKeys(
		backend,
		listStateDescriptor,
		(k, state) -> {
			backend.setCurrentKey(k);
			state.clear();
		});
	for (long i = 0; i < 10; i++) {
		backend.setCurrentKey(i);
		Assert.assertNull(listState.get());
	}
	cleanUp(backend);
}
 
Example 5
Source Project: flink   Source File: MergingWindowSet.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Restores a {@link MergingWindowSet} from the given state.
 */
public MergingWindowSet(MergingWindowAssigner<?, W> windowAssigner, ListState<Tuple2<W, W>> state) throws Exception {
	this.windowAssigner = windowAssigner;
	mapping = new HashMap<>();

	Iterable<Tuple2<W, W>> windowState = state.get();
	if (windowState != null) {
		for (Tuple2<W, W> window: windowState) {
			mapping.put(window.f0, window.f1);
		}
	}

	this.state = state;

	initialMapping = new HashMap<>();
	initialMapping.putAll(mapping);
}
 
Example 6
Source Project: beam   Source File: BufferingDoFnRunnerTest.java    License: Apache License 2.0 6 votes vote down vote up
private static BufferingDoFnRunner createBufferingDoFnRunner(
    int concurrentCheckpoints,
    List<BufferingDoFnRunner.CheckpointIdentifier> notYetAcknowledgeCheckpoints)
    throws Exception {
  DoFnRunner doFnRunner = Mockito.mock(DoFnRunner.class);
  OperatorStateBackend operatorStateBackend = Mockito.mock(OperatorStateBackend.class);

  // Setup not yet acknowledged checkpoint union list state
  ListState unionListState = Mockito.mock(ListState.class);
  Mockito.when(operatorStateBackend.getUnionListState(Mockito.any())).thenReturn(unionListState);
  Mockito.when(unionListState.get()).thenReturn(notYetAcknowledgeCheckpoints);

  // Setup buffer list state
  Mockito.when(operatorStateBackend.getListState(Mockito.any()))
      .thenReturn(Mockito.mock(ListState.class));

  return BufferingDoFnRunner.create(
      doFnRunner,
      "stable-input",
      StringUtf8Coder.of(),
      WindowedValue.getFullCoder(VarIntCoder.of(), GlobalWindow.Coder.INSTANCE),
      operatorStateBackend,
      null,
      concurrentCheckpoints);
}
 
Example 7
Source Project: flink   Source File: BucketsTest.java    License: Apache License 2.0 6 votes vote down vote up
private static Buckets<String, String> restoreBuckets(
		final Path basePath,
		final RollingPolicy<String, String> rollingPolicy,
		final BucketLifeCycleListener<String, String> bucketLifeCycleListener,
		final int subtaskIdx,
		final ListState<byte[]> bucketState,
		final ListState<Long> partCounterState,
		final OutputFileConfig outputFileConfig) throws Exception {
	final Buckets<String, String> restoredBuckets = createBuckets(
		basePath,
		rollingPolicy,
		bucketLifeCycleListener,
		subtaskIdx,
		outputFileConfig);
	restoredBuckets.initializeState(bucketState, partCounterState);
	return restoredBuckets;
}
 
Example 8
Source Project: flink   Source File: StateBackendTestBase.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * This test verifies that all ListState implementations are consistent in not allowing
 * adding {@code null}.
 */
@Test
public void testListStateAddNull() throws Exception {
	AbstractKeyedStateBackend<String> keyedBackend = createKeyedBackend(StringSerializer.INSTANCE);

	final ListStateDescriptor<Long> stateDescr = new ListStateDescriptor<>("my-state", Long.class);

	try {
		ListState<Long> state =
			keyedBackend.getPartitionedState(
				VoidNamespace.INSTANCE,
				VoidNamespaceSerializer.INSTANCE,
				stateDescr);

		keyedBackend.setCurrentKey("abc");
		assertNull(state.get());

		expectedException.expect(NullPointerException.class);
		state.add(null);
	} finally {
		keyedBackend.close();
		keyedBackend.dispose();
	}
}
 
Example 9
Source Project: Flink-CEPplus   Source File: StateBackendTestBase.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * This test verifies that all ListState implementations are consistent in not allowing
 * {@link ListState#addAll(List)} to be called with {@code null}.
 */
@Test
public void testListStateUpdateNull() throws Exception {
	AbstractKeyedStateBackend<String> keyedBackend = createKeyedBackend(StringSerializer.INSTANCE);

	final ListStateDescriptor<Long> stateDescr = new ListStateDescriptor<>("my-state", Long.class);

	try {
		ListState<Long> state =
			keyedBackend.getPartitionedState(
				VoidNamespace.INSTANCE,
				VoidNamespaceSerializer.INSTANCE,
				stateDescr);

		keyedBackend.setCurrentKey("abc");
		assertNull(state.get());

		expectedException.expect(NullPointerException.class);
		state.update(null);
	} finally {
		keyedBackend.close();
		keyedBackend.dispose();
	}
}
 
Example 10
Source Project: Flink-CEPplus   Source File: StateBackendTestBase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testApplyToAllKeysLambdaFunction() throws Exception {
	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);

	try {
		ListStateDescriptor<String> listStateDescriptor =
			new ListStateDescriptor<>("foo", StringSerializer.INSTANCE);

		ListState<String> listState =
			backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, listStateDescriptor);

		for (int i = 0; i < 100; ++i) {
			backend.setCurrentKey(i);
			listState.add("Hello" + i);
		}

		// valid state value via applyToAllKeys().
		backend.applyToAllKeys(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, listStateDescriptor,
			(Integer key, ListState<String> state) -> assertEquals("Hello" + key, state.get().iterator().next())
		);
	}
	finally {
		IOUtils.closeQuietly(backend);
		backend.dispose();
	}
}
 
Example 11
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {

	readyToFail = false;

	if (context.isRestored()) {
		isRunning = false;
	} else {
		isRunning = true;

		OperatorStateStore operatorStateStore = context.getOperatorStateStore();
		for (int i = 0; i < numListStates; ++i) {

			ListStateDescriptor<String> listStateDescriptor =
				new ListStateDescriptor<>("test-list-state-" + i, String.class);

			ListState<String> unionListState =
				operatorStateStore.getUnionListState(listStateDescriptor);

			for (int j = 0; j < numPartitionsPerListState; ++j) {
				unionListState.add(String.valueOf(j));
			}
		}
	}
}
 
Example 12
Source Project: flink   Source File: Buckets.java    License: Apache License 2.0 6 votes vote down vote up
void snapshotState(
		final long checkpointId,
		final ListState<byte[]> bucketStatesContainer,
		final ListState<Long> partCounterStateContainer) throws Exception {

	Preconditions.checkState(
			fsWriter != null && bucketStateSerializer != null,
			"sink has not been initialized");

	LOG.info("Subtask {} checkpointing for checkpoint with id={} (max part counter={}).",
			subtaskIndex, checkpointId, maxPartCounter);

	bucketStatesContainer.clear();
	partCounterStateContainer.clear();

	snapshotActiveBuckets(checkpointId, bucketStatesContainer);
	partCounterStateContainer.add(maxPartCounter);
}
 
Example 13
Source Project: flink   Source File: StateBackendTestBase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testApplyToAllKeysLambdaFunction() throws Exception {
	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);

	try {
		ListStateDescriptor<String> listStateDescriptor =
			new ListStateDescriptor<>("foo", StringSerializer.INSTANCE);

		ListState<String> listState =
			backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, listStateDescriptor);

		for (int i = 0; i < 100; ++i) {
			backend.setCurrentKey(i);
			listState.add("Hello" + i);
		}

		// valid state value via applyToAllKeys().
		backend.applyToAllKeys(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, listStateDescriptor,
			(Integer key, ListState<String> state) -> assertEquals("Hello" + key, state.get().iterator().next())
		);
	}
	finally {
		IOUtils.closeQuietly(backend);
		backend.dispose();
	}
}
 
Example 14
Source Project: Flink-CEPplus   Source File: Buckets.java    License: Apache License 2.0 6 votes vote down vote up
void snapshotState(
		final long checkpointId,
		final ListState<byte[]> bucketStatesContainer,
		final ListState<Long> partCounterStateContainer) throws Exception {

	Preconditions.checkState(
			fsWriter != null && bucketStateSerializer != null,
			"sink has not been initialized");

	LOG.info("Subtask {} checkpointing for checkpoint with id={} (max part counter={}).",
			subtaskIndex, checkpointId, maxPartCounter);

	bucketStatesContainer.clear();
	partCounterStateContainer.clear();

	snapshotActiveBuckets(checkpointId, bucketStatesContainer);
	partCounterStateContainer.add(maxPartCounter);
}
 
Example 15
Source Project: Flink-CEPplus   Source File: EvictingWindowOperator.java    License: Apache License 2.0 6 votes vote down vote up
public EvictingWindowOperator(WindowAssigner<? super IN, W> windowAssigner,
		TypeSerializer<W> windowSerializer,
		KeySelector<IN, K> keySelector,
		TypeSerializer<K> keySerializer,
		StateDescriptor<? extends ListState<StreamRecord<IN>>, ?> windowStateDescriptor,
		InternalWindowFunction<Iterable<IN>, OUT, K, W> windowFunction,
		Trigger<? super IN, ? super W> trigger,
		Evictor<? super IN, ? super W> evictor,
		long allowedLateness,
		OutputTag<IN> lateDataOutputTag) {

	super(windowAssigner, windowSerializer, keySelector,
		keySerializer, null, windowFunction, trigger, allowedLateness, lateDataOutputTag);

	this.evictor = checkNotNull(evictor);
	this.evictingWindowStateDescriptor = checkNotNull(windowStateDescriptor);
}
 
Example 16
Source Project: flink   Source File: AsyncWaitOperator.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
	super.snapshotState(context);

	ListState<StreamElement> partitionableState =
		getOperatorStateBackend().getListState(new ListStateDescriptor<>(STATE_NAME, inStreamElementSerializer));
	partitionableState.clear();

	try {
		partitionableState.addAll(queue.values());
	} catch (Exception e) {
		partitionableState.clear();

		throw new Exception("Could not add stream element queue entries to operator state " +
			"backend of operator " + getOperatorName() + '.', e);
	}
}
 
Example 17
Source Project: Flink-CEPplus   Source File: MergingWindowSet.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Restores a {@link MergingWindowSet} from the given state.
 */
public MergingWindowSet(MergingWindowAssigner<?, W> windowAssigner, ListState<Tuple2<W, W>> state) throws Exception {
	this.windowAssigner = windowAssigner;
	mapping = new HashMap<>();

	Iterable<Tuple2<W, W>> windowState = state.get();
	if (windowState != null) {
		for (Tuple2<W, W> window: windowState) {
			mapping.put(window.f0, window.f1);
		}
	}

	this.state = state;

	initialMapping = new HashMap<>();
	initialMapping.putAll(mapping);
}
 
Example 18
@Override
public void processBroadcastElement(Integer value, Context ctx, Collector<String> out) throws Exception {
	// put an element in the broadcast state
	ctx.applyToKeyedState(
			listStateDesc,
			new KeyedStateFunction<String, ListState<String>>() {
				@Override
				public void process(String key, ListState<String> state) throws Exception {
					final Iterator<String> it = state.get().iterator();

					final List<String> list = new ArrayList<>();
					while (it.hasNext()) {
						list.add(it.next());
					}
					assertEquals(expectedKeyedStates.get(key), list);
				}
			});
}
 
Example 19
Source Project: flink   Source File: MergingWindowSetTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Test merging of a large new window that covers one existing windows.
 */
@Test
public void testMergeLargeWindowCoveringSingleWindow() throws Exception {
	@SuppressWarnings("unchecked")
	ListState<Tuple2<TimeWindow, TimeWindow>> mockState = mock(ListState.class);

	MergingWindowSet<TimeWindow> windowSet = new MergingWindowSet<>(EventTimeSessionWindows.withGap(Time.milliseconds(3)), mockState);

	TestingMergeFunction mergeFunction = new TestingMergeFunction();

	// add an initial small window

	mergeFunction.reset();
	assertEquals(new TimeWindow(1, 2), windowSet.addWindow(new TimeWindow(1, 2), mergeFunction));
	assertFalse(mergeFunction.hasMerged());
	assertEquals(new TimeWindow(1, 2), windowSet.getStateWindow(new TimeWindow(1, 2)));

	// add a new window that completely covers the existing window

	mergeFunction.reset();
	assertEquals(new TimeWindow(0, 3), windowSet.addWindow(new TimeWindow(0, 3), mergeFunction));
	assertTrue(mergeFunction.hasMerged());
	assertEquals(new TimeWindow(1, 2), windowSet.getStateWindow(new TimeWindow(0, 3)));
}
 
Example 20
Source Project: flink   Source File: MergingWindowSetTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Test adding a new window that is identical to an existing window. This should not cause
 * a merge.
 */
@Test
public void testAddingIdenticalWindows() throws Exception {
	@SuppressWarnings("unchecked")
	ListState<Tuple2<TimeWindow, TimeWindow>> mockState = mock(ListState.class);

	MergingWindowSet<TimeWindow> windowSet = new MergingWindowSet<>(EventTimeSessionWindows.withGap(Time.milliseconds(3)), mockState);

	TestingMergeFunction mergeFunction = new TestingMergeFunction();

	mergeFunction.reset();
	assertEquals(new TimeWindow(1, 2), windowSet.addWindow(new TimeWindow(1, 2), mergeFunction));
	assertFalse(mergeFunction.hasMerged());
	assertEquals(new TimeWindow(1, 2), windowSet.getStateWindow(new TimeWindow(1, 2)));

	mergeFunction.reset();
	assertEquals(new TimeWindow(1, 2), windowSet.addWindow(new TimeWindow(1, 2), mergeFunction));
	assertFalse(mergeFunction.hasMerged());
	assertEquals(new TimeWindow(1, 2), windowSet.getStateWindow(new TimeWindow(1, 2)));
}
 
Example 21
Source Project: Flink-CEPplus   Source File: MergingWindowSetTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Test adding a new window that is identical to an existing window. This should not cause
 * a merge.
 */
@Test
public void testAddingIdenticalWindows() throws Exception {
	@SuppressWarnings("unchecked")
	ListState<Tuple2<TimeWindow, TimeWindow>> mockState = mock(ListState.class);

	MergingWindowSet<TimeWindow> windowSet = new MergingWindowSet<>(EventTimeSessionWindows.withGap(Time.milliseconds(3)), mockState);

	TestingMergeFunction mergeFunction = new TestingMergeFunction();

	mergeFunction.reset();
	assertEquals(new TimeWindow(1, 2), windowSet.addWindow(new TimeWindow(1, 2), mergeFunction));
	assertFalse(mergeFunction.hasMerged());
	assertEquals(new TimeWindow(1, 2), windowSet.getStateWindow(new TimeWindow(1, 2)));

	mergeFunction.reset();
	assertEquals(new TimeWindow(1, 2), windowSet.addWindow(new TimeWindow(1, 2), mergeFunction));
	assertFalse(mergeFunction.hasMerged());
	assertEquals(new TimeWindow(1, 2), windowSet.getStateWindow(new TimeWindow(1, 2)));
}
 
Example 22
Source Project: flink   Source File: MigrationTestUtils.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
	ListState<String> unionListState = context.getOperatorStateStore().getUnionListState(
			CheckpointingNonParallelSourceWithListState.STATE_DESCRIPTOR);

	if (context.isRestored()) {
		assertThat(unionListState.get(),
				containsInAnyOrder(CheckpointingParallelSourceWithUnionListState.CHECKPOINTED_STRINGS));

		getRuntimeContext().addAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, new IntCounter());
		getRuntimeContext().getAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR).add(1);
	} else {
		throw new RuntimeException(
				"This source should always be restored because it's only used when restoring from a savepoint.");
	}
}
 
Example 23
Source Project: Flink-CEPplus   Source File: MergingWindowSetTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testPersistOnlyIfHaveUpdates() throws Exception {
	@SuppressWarnings("unchecked")
	ListState<Tuple2<TimeWindow, TimeWindow>> mockState = mock(ListState.class);
	when(mockState.get()).thenReturn(Lists.newArrayList(
			new Tuple2<>(new TimeWindow(17, 42), new TimeWindow(42, 17)),
			new Tuple2<>(new TimeWindow(1, 2), new TimeWindow(3, 4))
	));

	MergingWindowSet<TimeWindow> windowSet = new MergingWindowSet<>(EventTimeSessionWindows.withGap(Time.milliseconds(3)), mockState);

	assertEquals(new TimeWindow(42, 17), windowSet.getStateWindow(new TimeWindow(17, 42)));
	assertEquals(new TimeWindow(3, 4), windowSet.getStateWindow(new TimeWindow(1, 2)));

	windowSet.persist();

	verify(mockState, times(0)).add(Matchers.<Tuple2<TimeWindow, TimeWindow>>anyObject());

}
 
Example 24
Source Project: beam   Source File: FlinkStateInternals.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void add(T input) {
  try {
    ListState<T> partitionedState =
        flinkStateBackend.getPartitionedState(
            namespace.stringKey(), StringSerializer.INSTANCE, flinkStateDescriptor);
    if (storesVoidValues) {
      Preconditions.checkState(input == null, "Expected to a null value but was: %s", input);
      // Flink does not allow storing null values
      // If we have null values, we use the structural null value
      input = (T) VoidCoder.of().structuralValue((Void) input);
    }
    partitionedState.add(input);
  } catch (Exception e) {
    throw new RuntimeException("Error adding to bag state.", e);
  }
}
 
Example 25
Source Project: flink   Source File: MergingWindowSetTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Test merging of a large new window that covers multiple existing windows.
 */
@Test
public void testMergeLargeWindowCoveringMultipleWindows() throws Exception {
	@SuppressWarnings("unchecked")
	ListState<Tuple2<TimeWindow, TimeWindow>> mockState = mock(ListState.class);

	MergingWindowSet<TimeWindow> windowSet = new MergingWindowSet<>(EventTimeSessionWindows.withGap(Time.milliseconds(3)), mockState);

	TestingMergeFunction mergeFunction = new TestingMergeFunction();

	// add several non-overlapping initial windows

	mergeFunction.reset();
	assertEquals(new TimeWindow(1, 3), windowSet.addWindow(new TimeWindow(1, 3), mergeFunction));
	assertFalse(mergeFunction.hasMerged());
	assertEquals(new TimeWindow(1, 3), windowSet.getStateWindow(new TimeWindow(1, 3)));

	mergeFunction.reset();
	assertEquals(new TimeWindow(5, 8), windowSet.addWindow(new TimeWindow(5, 8), mergeFunction));
	assertFalse(mergeFunction.hasMerged());
	assertEquals(new TimeWindow(5, 8), windowSet.getStateWindow(new TimeWindow(5, 8)));

	mergeFunction.reset();
	assertEquals(new TimeWindow(10, 13), windowSet.addWindow(new TimeWindow(10, 13), mergeFunction));
	assertFalse(mergeFunction.hasMerged());
	assertEquals(new TimeWindow(10, 13), windowSet.getStateWindow(new TimeWindow(10, 13)));

	// add a new window that completely covers the existing windows

	mergeFunction.reset();
	assertEquals(new TimeWindow(0, 13), windowSet.addWindow(new TimeWindow(0, 13), mergeFunction));
	assertTrue(mergeFunction.hasMerged());
	assertThat(mergeFunction.mergedStateWindows(), anyOf(
			containsInAnyOrder(new TimeWindow(0, 3), new TimeWindow(5, 8)),
			containsInAnyOrder(new TimeWindow(0, 3), new TimeWindow(10, 13)),
			containsInAnyOrder(new TimeWindow(5, 8), new TimeWindow(10, 13))));
	assertThat(windowSet.getStateWindow(new TimeWindow(0, 13)), anyOf(is(new TimeWindow(1, 3)), is(new TimeWindow(5, 8)), is(new TimeWindow(10, 13))));
}
 
Example 26
Source Project: flink   Source File: StreamingFunctionUtils.java    License: Apache License 2.0 5 votes vote down vote up
private static boolean tryRestoreFunction(
		StateInitializationContext context,
		Function userFunction) throws Exception {

	if (userFunction instanceof CheckpointedFunction) {
		((CheckpointedFunction) userFunction).initializeState(context);

		return true;
	}

	if (context.isRestored() && userFunction instanceof ListCheckpointed) {
		@SuppressWarnings("unchecked")
		ListCheckpointed<Serializable> listCheckpointedFun = (ListCheckpointed<Serializable>) userFunction;

		ListState<Serializable> listState = context.getOperatorStateStore().
				getSerializableListState(DefaultOperatorStateBackend.DEFAULT_OPERATOR_STATE_NAME);

		List<Serializable> list = new ArrayList<>();

		for (Serializable serializable : listState.get()) {
			list.add(serializable);
		}

		try {
			listCheckpointedFun.restoreState(list);
		} catch (Exception e) {

			throw new Exception("Failed to restore state to function: " + e.getMessage(), e);
		}

		return true;
	}

	return false;
}
 
Example 27
Source Project: beam   Source File: FlinkBroadcastStateInternals.java    License: Apache License 2.0 5 votes vote down vote up
Map<String, T> getMapFromBroadcastState() throws Exception {
  ListState<Map<String, T>> state = flinkStateBackend.getUnionListState(flinkStateDescriptor);
  Iterable<Map<String, T>> iterable = state.get();
  Map<String, T> ret = null;
  if (iterable != null) {
    // just use index 0
    Iterator<Map<String, T>> iterator = iterable.iterator();
    if (iterator.hasNext()) {
      ret = iterator.next();
    }
  }
  return ret;
}
 
Example 28
Source Project: flink   Source File: StreamingRuntimeContextTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testListStateReturnsEmptyListByDefault() throws Exception {

	StreamingRuntimeContext context = new StreamingRuntimeContext(
			createListPlainMockOp(),
			createMockEnvironment(),
			Collections.<String, Accumulator<?, ?>>emptyMap());

	ListStateDescriptor<String> descr = new ListStateDescriptor<>("name", String.class);
	ListState<String> state = context.getListState(descr);

	Iterable<String> value = state.get();
	assertNotNull(value);
	assertFalse(value.iterator().hasNext());
}
 
Example 29
Source Project: flink   Source File: DefaultKeyedStateStore.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public <T> ListState<T> getListState(ListStateDescriptor<T> stateProperties) {
	requireNonNull(stateProperties, "The state properties must not be null");
	try {
		stateProperties.initializeSerializerUnlessSet(executionConfig);
		ListState<T> originalState = getPartitionedState(stateProperties);
		return new UserFacingListState<>(originalState);
	} catch (Exception e) {
		throw new RuntimeException("Error while getting state", e);
	}
}
 
Example 30
Source Project: flink   Source File: MergingWindowSetTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * This test uses a special (misbehaving) {@code MergingWindowAssigner} that produces cases
 * where windows that don't overlap with the newly added window are being merged. We verify
 * that the merging window set is nevertheless correct and contains all added windows.
 */
@Test
public void testNonEagerMerging() throws Exception {
	@SuppressWarnings("unchecked")
	ListState<Tuple2<TimeWindow, TimeWindow>> mockState = mock(ListState.class);

	MergingWindowSet<TimeWindow> windowSet =
			new MergingWindowSet<>(new NonEagerlyMergingWindowAssigner(3000), mockState);

	TestingMergeFunction mergeFunction = new TestingMergeFunction();

	TimeWindow result;

	mergeFunction.reset();
	result = windowSet.addWindow(new TimeWindow(0, 2), mergeFunction);
	assertNotNull(windowSet.getStateWindow(result));

	mergeFunction.reset();
	result = windowSet.addWindow(new TimeWindow(2, 5), mergeFunction);
	assertNotNull(windowSet.getStateWindow(result));

	mergeFunction.reset();
	result = windowSet.addWindow(new TimeWindow(1, 2), mergeFunction);
	assertNotNull(windowSet.getStateWindow(result));

	mergeFunction.reset();
	result = windowSet.addWindow(new TimeWindow(10, 12), mergeFunction);
	assertNotNull(windowSet.getStateWindow(result));
}