Java Code Examples for org.apache.flink.runtime.state.FunctionInitializationContext

The following examples show how to use org.apache.flink.runtime.state.FunctionInitializationContext. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: flink   Source File: FlinkKafkaConsumerBase.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public final void initializeState(FunctionInitializationContext context) throws Exception {

	OperatorStateStore stateStore = context.getOperatorStateStore();

	this.unionOffsetStates = stateStore.getUnionListState(new ListStateDescriptor<>(OFFSETS_STATE_NAME,
		createStateSerializer(getRuntimeContext().getExecutionConfig())));

	if (context.isRestored()) {
		restoredState = new TreeMap<>(new KafkaTopicPartition.Comparator());

		// populate actual holder for restored state
		for (Tuple2<KafkaTopicPartition, Long> kafkaOffset : unionOffsetStates.get()) {
			restoredState.put(kafkaOffset.f0, kafkaOffset.f1);
		}

		LOG.info("Consumer subtask {} restored state: {}.", getRuntimeContext().getIndexOfThisSubtask(), restoredState);
	} else {
		LOG.info("Consumer subtask {} has no restore state.", getRuntimeContext().getIndexOfThisSubtask());
	}
}
 
Example 2
Source Project: flink   Source File: MigrationTestUtils.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
	ListState<String> unionListState = context.getOperatorStateStore().getListState(
			CheckpointingNonParallelSourceWithListState.STATE_DESCRIPTOR);

	if (context.isRestored()) {
		assertThat(unionListState.get(),
				containsInAnyOrder(
						CheckpointingNonParallelSourceWithListState.CHECKPOINTED_STRING,
						CheckpointingNonParallelSourceWithListState.CHECKPOINTED_STRING_1,
						CheckpointingNonParallelSourceWithListState.CHECKPOINTED_STRING_2,
						CheckpointingNonParallelSourceWithListState.CHECKPOINTED_STRING_3));

		getRuntimeContext().addAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, new IntCounter());
		getRuntimeContext().getAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR).add(1);
	} else {
		throw new RuntimeException(
				"This source should always be restored because it's only used when restoring from a savepoint.");
	}
}
 
Example 3
Source Project: flink   Source File: RescalingITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {

	if (broadcast) {
		this.counterPartitions = context
				.getOperatorStateStore()
				.getUnionListState(new ListStateDescriptor<>("counter_partitions", IntSerializer.INSTANCE));
	} else {
		this.counterPartitions = context
				.getOperatorStateStore()
				.getListState(new ListStateDescriptor<>("counter_partitions", IntSerializer.INSTANCE));
	}

	if (context.isRestored()) {
		for (int v : counterPartitions.get()) {
			counter += v;
		}
		checkCorrectRestore[getRuntimeContext().getIndexOfThisSubtask()] = counter;
	}
}
 
Example 4
Source Project: Flink-CEPplus   Source File: MigrationTestUtils.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
	ListState<String> unionListState = context.getOperatorStateStore().getUnionListState(
			CheckpointingNonParallelSourceWithListState.STATE_DESCRIPTOR);

	if (context.isRestored()) {
		assertThat(unionListState.get(),
				containsInAnyOrder(CheckpointingParallelSourceWithUnionListState.CHECKPOINTED_STRINGS));

		getRuntimeContext().addAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, new IntCounter());
		getRuntimeContext().getAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR).add(1);
	} else {
		throw new RuntimeException(
				"This source should always be restored because it's only used when restoring from a savepoint.");
	}
}
 
Example 5
Source Project: flink   Source File: HeavyDeploymentStressTestProgram.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {

	readyToFail = false;

	if (context.isRestored()) {
		isRunning = false;
	} else {
		isRunning = true;

		OperatorStateStore operatorStateStore = context.getOperatorStateStore();
		for (int i = 0; i < numListStates; ++i) {

			ListStateDescriptor<String> listStateDescriptor =
				new ListStateDescriptor<>("test-list-state-" + i, String.class);

			ListState<String> unionListState =
				operatorStateStore.getUnionListState(listStateDescriptor);

			for (int j = 0; j < numPartitionsPerListState; ++j) {
				unionListState.add(String.valueOf(j));
			}
		}
	}
}
 
Example 6
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {

	readyToFail = false;

	if (context.isRestored()) {
		isRunning = false;
	} else {
		isRunning = true;

		OperatorStateStore operatorStateStore = context.getOperatorStateStore();
		for (int i = 0; i < numListStates; ++i) {

			ListStateDescriptor<String> listStateDescriptor =
				new ListStateDescriptor<>("test-list-state-" + i, String.class);

			ListState<String> unionListState =
				operatorStateStore.getUnionListState(listStateDescriptor);

			for (int j = 0; j < numPartitionsPerListState; ++j) {
				unionListState.add(String.valueOf(j));
			}
		}
	}
}
 
Example 7
Source Project: Flink-CEPplus   Source File: TtlVerifyUpdateFunction.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void initializeState(FunctionInitializationContext context) {
	states = TtlStateVerifier.VERIFIERS.stream()
		.collect(Collectors.toMap(TtlStateVerifier::getId, v -> v.createState(context, ttlConfig)));
	prevUpdatesByVerifierId = TtlStateVerifier.VERIFIERS.stream()
		.collect(Collectors.toMap(TtlStateVerifier::getId, v -> {
			checkNotNull(v);
			final TypeSerializer<ValueWithTs<?>> typeSerializer = new ValueWithTs.Serializer(
				v.getUpdateSerializer(),
				LongSerializer.INSTANCE);

			ListStateDescriptor<ValueWithTs<?>> stateDesc = new ListStateDescriptor<>(
				"TtlPrevValueState_" + v.getId(), typeSerializer);
			KeyedStateStore store = context.getKeyedStateStore();
			return store.getListState(stateDesc);
		}));
}
 
Example 8
Source Project: Flink-CEPplus   Source File: FromElementsFunction.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
	Preconditions.checkState(this.checkpointedState == null,
		"The " + getClass().getSimpleName() + " has already been initialized.");

	this.checkpointedState = context.getOperatorStateStore().getListState(
		new ListStateDescriptor<>(
			"from-elements-state",
			IntSerializer.INSTANCE
		)
	);

	if (context.isRestored()) {
		List<Integer> retrievedStates = new ArrayList<>();
		for (Integer entry : this.checkpointedState.get()) {
			retrievedStates.add(entry);
		}

		// given that the parallelism of the function is 1, we can only have 1 state
		Preconditions.checkArgument(retrievedStates.size() == 1,
			getClass().getSimpleName() + " retrieved invalid state.");

		this.numElementsToSkip = retrievedStates.get(0);
	}
}
 
Example 9
Source Project: pulsar-flink   Source File: FlinkPulsarSource.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
    OperatorStateStore stateStore = context.getOperatorStateStore();

    unionOffsetStates = stateStore.getUnionListState(
            new ListStateDescriptor<>(
                    OFFSETS_STATE_NAME,
                    TypeInformation.of(new TypeHint<Tuple2<String, MessageId>>() {
                    })));

    if (context.isRestored()) {
        restoredState = new TreeMap<>();
        unionOffsetStates.get().forEach(e -> restoredState.put(e.f0, e.f1));
        log.info("Source subtask {} restored state {}",
                taskIndex,
                StringUtils.join(restoredState.entrySet()));
    } else {
        log.info("Source subtask {} has no restore state", taskIndex);
    }
}
 
Example 10
Source Project: flink   Source File: RMQSourceTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testOverrideConnection() throws Exception {
	final Connection mockConnection = Mockito.mock(Connection.class);
	Channel channel = Mockito.mock(Channel.class);
	Mockito.when(mockConnection.createChannel()).thenReturn(channel);

	RMQMockedRuntimeTestSource source = new RMQMockedRuntimeTestSource() {
		@Override
		protected Connection setupConnection() throws Exception {
			return mockConnection;
		}
	};

	FunctionInitializationContext mockContext = getMockContext();
	source.initializeState(mockContext);
	source.open(new Configuration());

	Mockito.verify(mockConnection, Mockito.times(1)).createChannel();
}
 
Example 11
Source Project: flink   Source File: FromElementsFunction.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
	Preconditions.checkState(this.checkpointedState == null,
		"The " + getClass().getSimpleName() + " has already been initialized.");

	this.checkpointedState = context.getOperatorStateStore().getListState(
		new ListStateDescriptor<>(
			"from-elements-state",
			IntSerializer.INSTANCE
		)
	);

	if (context.isRestored()) {
		List<Integer> retrievedStates = new ArrayList<>();
		for (Integer entry : this.checkpointedState.get()) {
			retrievedStates.add(entry);
		}

		// given that the parallelism of the function is 1, we can only have 1 state
		Preconditions.checkArgument(retrievedStates.size() == 1,
			getClass().getSimpleName() + " retrieved invalid state.");

		this.numElementsToSkip = retrievedStates.get(0);
	}
}
 
Example 12
Source Project: flink   Source File: MigrationTestUtils.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
	ListState<String> unionListState = context.getOperatorStateStore().getUnionListState(
			CheckpointingNonParallelSourceWithListState.STATE_DESCRIPTOR);

	if (context.isRestored()) {
		assertThat(unionListState.get(),
				containsInAnyOrder(CheckpointingParallelSourceWithUnionListState.CHECKPOINTED_STRINGS));

		getRuntimeContext().addAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, new IntCounter());
		getRuntimeContext().getAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR).add(1);
	} else {
		throw new RuntimeException(
				"This source should always be restored because it's only used when restoring from a savepoint.");
	}
}
 
Example 13
Source Project: flink   Source File: RMQSourceTest.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void beforeTest() throws Exception {

	OperatorStateStore mockStore = Mockito.mock(OperatorStateStore.class);
	FunctionInitializationContext mockContext = Mockito.mock(FunctionInitializationContext.class);
	Mockito.when(mockContext.getOperatorStateStore()).thenReturn(mockStore);
	Mockito.when(mockStore.getSerializableListState(any(String.class))).thenReturn(null);

	source = new RMQTestSource();
	source.initializeState(mockContext);
	source.open(config);

	messageId = 0;
	generateCorrelationIds = true;

	sourceThread = new Thread(new Runnable() {
		@Override
		public void run() {
			try {
				source.run(new DummySourceContext());
			} catch (Exception e) {
				exception = e;
			}
		}
	});
}
 
Example 14
Source Project: beam   Source File: ImpulseSourceFunctionTest.java    License: Apache License 2.0 5 votes vote down vote up
private static <T> FunctionInitializationContext getInitializationContext(ListState<T> listState)
    throws Exception {
  FunctionInitializationContext mock = Mockito.mock(FunctionInitializationContext.class);
  OperatorStateStore mockOperatorState = getMockOperatorState(listState);
  when(mock.getOperatorStateStore()).thenReturn(mockOperatorState);
  return mock;
}
 
Example 15
Source Project: Flink-CEPplus   Source File: FlinkKafkaProducer.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
	if (semantic != FlinkKafkaProducer.Semantic.NONE && !((StreamingRuntimeContext) this.getRuntimeContext()).isCheckpointingEnabled()) {
		LOG.warn("Using {} semantic, but checkpointing is not enabled. Switching to {} semantic.", semantic, FlinkKafkaProducer.Semantic.NONE);
		semantic = FlinkKafkaProducer.Semantic.NONE;
	}

	nextTransactionalIdHintState = context.getOperatorStateStore().getUnionListState(
		NEXT_TRANSACTIONAL_ID_HINT_DESCRIPTOR);
	transactionalIdsGenerator = new TransactionalIdsGenerator(
		getRuntimeContext().getTaskName() + "-" + ((StreamingRuntimeContext) getRuntimeContext()).getOperatorUniqueID(),
		getRuntimeContext().getIndexOfThisSubtask(),
		getRuntimeContext().getNumberOfParallelSubtasks(),
		kafkaProducersPoolSize,
		SAFE_SCALE_DOWN_FACTOR);

	if (semantic != FlinkKafkaProducer.Semantic.EXACTLY_ONCE) {
		nextTransactionalIdHint = null;
	} else {
		ArrayList<FlinkKafkaProducer.NextTransactionalIdHint> transactionalIdHints = Lists.newArrayList(nextTransactionalIdHintState.get());
		if (transactionalIdHints.size() > 1) {
			throw new IllegalStateException(
				"There should be at most one next transactional id hint written by the first subtask");
		} else if (transactionalIdHints.size() == 0) {
			nextTransactionalIdHint = new FlinkKafkaProducer.NextTransactionalIdHint(0, 0);

			// this means that this is either:
			// (1) the first execution of this application
			// (2) previous execution has failed before first checkpoint completed
			//
			// in case of (2) we have to abort all previous transactions
			abortTransactions(transactionalIdsGenerator.generateIdsToAbort());
		} else {
			nextTransactionalIdHint = transactionalIdHints.get(0);
		}
	}

	super.initializeState(context);
}
 
Example 16
Source Project: flink   Source File: AbstractArrowSourceFunction.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
	Preconditions.checkState(this.checkpointedState == null,
		"The " + getClass().getSimpleName() + " has already been initialized.");

	this.checkpointedState = context.getOperatorStateStore().getListState(
		new ListStateDescriptor<>(
			"arrow-source-state",
			new TupleSerializer<>(
				(Class<Tuple2<Integer, Integer>>) (Class<?>) Tuple2.class,
				new TypeSerializer[]{IntSerializer.INSTANCE, IntSerializer.INSTANCE})
		)
	);

	this.indexesToEmit = new ArrayDeque<>();
	if (context.isRestored()) {
		// upon restoring
		for (Tuple2<Integer, Integer> v : this.checkpointedState.get()) {
			this.indexesToEmit.add(v);
		}
		LOG.info("Subtask {} restored state: {}.", getRuntimeContext().getIndexOfThisSubtask(), indexesToEmit);
	} else {
		// the first time the job is executed
		final int stepSize = getRuntimeContext().getNumberOfParallelSubtasks();
		final int taskIdx = getRuntimeContext().getIndexOfThisSubtask();

		for (int i = taskIdx; i < arrowData.length; i += stepSize) {
			this.indexesToEmit.add(Tuple2.of(i, 0));
		}
		LOG.info("Subtask {} has no restore state, initialized with {}.", taskIdx, indexesToEmit);
	}
}
 
Example 17
Source Project: flink   Source File: PojoSerializerUpgradeTest.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
	pojoClass = getRuntimeContext().getUserCodeClassLoader().loadClass(POJO_NAME);

	fieldA = pojoClass.getDeclaredField("a");
	fieldA.setAccessible(true);

	if (hasBField) {
		fieldB = pojoClass.getDeclaredField("b");
		fieldB.setAccessible(true);
	}

	if (keyed) {
		keyedValueState = context.getKeyedStateStore().getState(
			new ValueStateDescriptor<>("keyedValueState", (Class<Object>) pojoClass));
		keyedListState = context.getKeyedStateStore().getListState(
			new ListStateDescriptor<>("keyedListState", (Class<Object>) pojoClass));

		ReduceFunction<Object> reduceFunction = new FirstValueReducer<>();
		keyedReducingState = context.getKeyedStateStore().getReducingState(
			new ReducingStateDescriptor<>("keyedReducingState", reduceFunction, (Class<Object>) pojoClass));
	} else {
		partitionableListState = context.getOperatorStateStore().getListState(
			new ListStateDescriptor<>("partitionableListState", (Class<Object>) pojoClass));
		unionListState = context.getOperatorStateStore().getUnionListState(
			new ListStateDescriptor<>("unionListState", (Class<Object>) pojoClass));
	}
}
 
Example 18
Source Project: Flink-CEPplus   Source File: PojoSerializerUpgradeTest.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
	pojoClass = getRuntimeContext().getUserCodeClassLoader().loadClass(POJO_NAME);

	fieldA = pojoClass.getDeclaredField("a");
	fieldA.setAccessible(true);

	if (hasBField) {
		fieldB = pojoClass.getDeclaredField("b");
		fieldB.setAccessible(true);
	}

	if (keyed) {
		keyedValueState = context.getKeyedStateStore().getState(
			new ValueStateDescriptor<>("keyedValueState", (Class<Object>) pojoClass));
		keyedListState = context.getKeyedStateStore().getListState(
			new ListStateDescriptor<>("keyedListState", (Class<Object>) pojoClass));

		ReduceFunction<Object> reduceFunction = new FirstValueReducer<>();
		keyedReducingState = context.getKeyedStateStore().getReducingState(
			new ReducingStateDescriptor<>("keyedReducingState", reduceFunction, (Class<Object>) pojoClass));
	} else {
		partitionableListState = context.getOperatorStateStore().getListState(
			new ListStateDescriptor<>("partitionableListState", (Class<Object>) pojoClass));
		unionListState = context.getOperatorStateStore().getUnionListState(
			new ListStateDescriptor<>("unionListState", (Class<Object>) pojoClass));
	}
}
 
Example 19
Source Project: flink   Source File: MessageAcknowledgingSourceBase.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
	Preconditions.checkState(this.checkpointedState == null,
		"The " + getClass().getSimpleName() + " has already been initialized.");

	this.checkpointedState = context
		.getOperatorStateStore()
		.getSerializableListState("message-acknowledging-source-state");

	this.idsForCurrentCheckpoint = new HashSet<>(64);
	this.pendingCheckpoints = new ArrayDeque<>();
	this.idsProcessedButNotAcknowledged = new HashSet<>();

	if (context.isRestored()) {
		LOG.info("Restoring state for the {}.", getClass().getSimpleName());

		List<SerializedCheckpointData[]> retrievedStates = new ArrayList<>();
		for (SerializedCheckpointData[] entry : this.checkpointedState.get()) {
			retrievedStates.add(entry);
		}

		// given that the parallelism of the function is 1, we can only have at most 1 state
		Preconditions.checkArgument(retrievedStates.size() == 1,
			getClass().getSimpleName() + " retrieved invalid state.");

		pendingCheckpoints = SerializedCheckpointData.toDeque(retrievedStates.get(0), idSerializer);
		// build a set which contains all processed ids. It may be used to check if we have
		// already processed an incoming message.
		for (Tuple2<Long, Set<UId>> checkpoint : pendingCheckpoints) {
			idsProcessedButNotAcknowledged.addAll(checkpoint.f1);
		}
	} else {
		LOG.info("No state to restore for the {}.", getClass().getSimpleName());
	}
}
 
Example 20
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
	sumState = context.getOperatorStateStore().getListState(
		new ListStateDescriptor<>("sumState", Integer.class));

	if (context.isRestored()) {
		for (int value : sumState.get()) {
			runningSum += value;
		}
	}
}
 
Example 21
Source Project: flink   Source File: StreamSQLTestProgram.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
	state = context.getOperatorStateStore().getListState(
			new ListStateDescriptor<Integer>("state", IntSerializer.INSTANCE));

	for (Integer i : state.get()) {
		saveRecordCnt += i;
	}
}
 
Example 22
Source Project: flink   Source File: RegionFailoverITCase.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
	int indexOfThisSubtask = getRuntimeContext().getIndexOfThisSubtask();
	if (context.isRestored()) {
		restoredState = true;

		unionListState = context.getOperatorStateStore().getUnionListState(unionStateDescriptor);
		Set<Integer> actualIndices = StreamSupport.stream(unionListState.get().spliterator(), false).collect(Collectors.toSet());
		if (getRuntimeContext().getTaskName().contains(SINGLE_REGION_SOURCE_NAME)) {
			Assert.assertTrue(CollectionUtils.isEqualCollection(EXPECTED_INDICES_SINGLE_REGION, actualIndices));
		} else {
			Assert.assertTrue(CollectionUtils.isEqualCollection(EXPECTED_INDICES_MULTI_REGION, actualIndices));
		}

		if (indexOfThisSubtask == 0) {
			listState = context.getOperatorStateStore().getListState(stateDescriptor);
			Assert.assertTrue("list state should be empty for subtask-0",
				((List<Integer>) listState.get()).isEmpty());
		} else {
			listState = context.getOperatorStateStore().getListState(stateDescriptor);
			Assert.assertTrue("list state should not be empty for subtask-" + indexOfThisSubtask,
				((List<Integer>) listState.get()).size() > 0);

			if (indexOfThisSubtask == NUM_OF_REGIONS - 1) {
				index = listState.get().iterator().next();
				if (index != snapshotIndicesOfSubTask.get(lastCompletedCheckpointId.get())) {
					throw new RuntimeException("Test failed due to unexpected recovered index: " + index +
						", while last completed checkpoint record index: " + snapshotIndicesOfSubTask.get(lastCompletedCheckpointId.get()));
				}
			}
		}
	} else {
		unionListState = context.getOperatorStateStore().getUnionListState(unionStateDescriptor);

		if (indexOfThisSubtask != 0) {
			listState = context.getOperatorStateStore().getListState(stateDescriptor);
		}
	}

}
 
Example 23
Source Project: Flink-CEPplus   Source File: StreamingFileSink.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
	final int subtaskIndex = getRuntimeContext().getIndexOfThisSubtask();
	this.buckets = bucketsBuilder.createBuckets(subtaskIndex);

	final OperatorStateStore stateStore = context.getOperatorStateStore();
	bucketStates = stateStore.getListState(BUCKET_STATE_DESC);
	maxPartCountersState = stateStore.getUnionListState(MAX_PART_COUNTER_STATE_DESC);

	if (context.isRestored()) {
		buckets.initializeState(bucketStates, maxPartCountersState);
	}
}
 
Example 24
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
	Preconditions.checkState(this.checkpointedState == null,
		"The " + getClass().getSimpleName() + " has already been initialized.");

	this.checkpointedState = context
		.getOperatorStateStore()
		.getSerializableListState("message-acknowledging-source-state");

	this.idsForCurrentCheckpoint = new HashSet<>(64);
	this.pendingCheckpoints = new ArrayDeque<>();
	this.idsProcessedButNotAcknowledged = new HashSet<>();

	if (context.isRestored()) {
		LOG.info("Restoring state for the {}.", getClass().getSimpleName());

		List<SerializedCheckpointData[]> retrievedStates = new ArrayList<>();
		for (SerializedCheckpointData[] entry : this.checkpointedState.get()) {
			retrievedStates.add(entry);
		}

		// given that the parallelism of the function is 1, we can only have at most 1 state
		Preconditions.checkArgument(retrievedStates.size() == 1,
			getClass().getSimpleName() + " retrieved invalid state.");

		pendingCheckpoints = SerializedCheckpointData.toDeque(retrievedStates.get(0), idSerializer);
		// build a set which contains all processed ids. It may be used to check if we have
		// already processed an incoming message.
		for (Tuple2<Long, Set<UId>> checkpoint : pendingCheckpoints) {
			idsProcessedButNotAcknowledged.addAll(checkpoint.f1);
		}
	} else {
		LOG.info("No state to restore for the {}.", getClass().getSimpleName());
	}
}
 
Example 25
Source Project: flink   Source File: WrappingFunctionSnapshotRestoreTest.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
	serializableListState = context
			.getOperatorStateStore()
			.getListState(new ListStateDescriptor<>("test-state", IntSerializer.INSTANCE));
	if (context.isRestored()) {
		Iterator<Integer> integers = serializableListState.get().iterator();
		int act = integers.next();
		Assert.assertEquals(42, act);
		Assert.assertFalse(integers.hasNext());
		wasRestored = true;
	}
}
 
Example 26
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
	sumState = context.getOperatorStateStore().getListState(
		new ListStateDescriptor<>("sumState", Integer.class));

	if (context.isRestored()) {
		for (int value : sumState.get()) {
			runningSum += value;
		}
	}
}
 
Example 27
Source Project: flink   Source File: BucketingSink.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
	Preconditions.checkArgument(this.restoredBucketStates == null, "The operator has already been initialized.");

	try {
		initFileSystem();
	} catch (IOException e) {
		LOG.error("Error while creating FileSystem when initializing the state of the BucketingSink.", e);
		throw new RuntimeException("Error while creating FileSystem when initializing the state of the BucketingSink.", e);
	}

	if (this.refTruncate == null) {
		this.refTruncate = reflectTruncate(fs);
	}

	// We are using JavaSerializer from the flink-runtime module here. This is very naughty and
	// we shouldn't be doing it because ideally nothing in the API modules/connector depends
	// directly on flink-runtime. We are doing it here because we need to maintain backwards
	// compatibility with old state and because we will have to rework/remove this code soon.
	OperatorStateStore stateStore = context.getOperatorStateStore();
	this.restoredBucketStates = stateStore.getListState(new ListStateDescriptor<>("bucket-states", new JavaSerializer<>()));

	int subtaskIndex = getRuntimeContext().getIndexOfThisSubtask();
	if (context.isRestored()) {
		LOG.info("Restoring state for the {} (taskIdx={}).", getClass().getSimpleName(), subtaskIndex);

		for (State<T> recoveredState : restoredBucketStates.get()) {
			handleRestoredBucketState(recoveredState);
			if (LOG.isDebugEnabled()) {
				LOG.debug("{} idx {} restored {}", getClass().getSimpleName(), subtaskIndex, recoveredState);
			}
		}
	} else {
		LOG.info("No state to restore for the {} (taskIdx={}).", getClass().getSimpleName(), subtaskIndex);
	}
}
 
Example 28
Source Project: flink   Source File: BucketingSinkTestProgram.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
	state = context.getOperatorStateStore().getListState(
			new ListStateDescriptor<Long>("state", LongSerializer.INSTANCE));

	for (Long l : state.get()) {
		ms += l;
	}
}
 
Example 29
Source Project: flink   Source File: StreamingFileSinkProgram.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
	state = context.getOperatorStateStore().getListState(
			new ListStateDescriptor<Integer>("state", IntSerializer.INSTANCE));

	for (Integer i : state.get()) {
		numRecordsEmitted += i;
	}
}
 
Example 30
Source Project: flink   Source File: StreamSQLTestProgram.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
	state = context.getOperatorStateStore().getListState(
			new ListStateDescriptor<Long>("state", LongSerializer.INSTANCE));

	for (Long l : state.get()) {
		ms += l;
	}
}