org.apache.flink.runtime.state.VoidNamespaceSerializer Java Examples

private void migrateOldState() throws Exception {
	getKeyedStateBackend().applyToAllKeys(
		VoidNamespace.INSTANCE,
		VoidNamespaceSerializer.INSTANCE,
		new ValueStateDescriptor<>(
			"nfaOperatorStateName",
			new NFA.NFASerializer<>(inputSerializer)
		),
		new KeyedStateFunction<Object, ValueState<MigratedNFA<IN>>>() {
			@Override
			public void process(Object key, ValueState<MigratedNFA<IN>> state) throws Exception {
				MigratedNFA<IN> oldState = state.value();
				computationStates.update(new NFAState(oldState.getComputationStates()));
				org.apache.flink.cep.nfa.SharedBuffer<IN> sharedBuffer = oldState.getSharedBuffer();
				partialMatches.init(sharedBuffer.getEventsBuffer(), sharedBuffer.getPages());
				state.clear();
			}
		}
	);
}
 

@Override
public void open() throws Exception {
	super.open();

	InternalTimerService<VoidNamespace> internalTimerService =
			getInternalTimerService("user-timers", VoidNamespaceSerializer.INSTANCE, this);

	TimerService timerService = new SimpleTimerService(internalTimerService);

	collector = new TimestampedCollector<>(output);

	this.broadcastStates = new HashMap<>(broadcastStateDescriptors.size());
	for (MapStateDescriptor<?, ?> descriptor: broadcastStateDescriptors) {
		broadcastStates.put(descriptor, getOperatorStateBackend().getBroadcastState(descriptor));
	}

	rwContext = new ReadWriteContextImpl(getExecutionConfig(), getKeyedStateBackend(), userFunction, broadcastStates, timerService);
	rContext = new ReadOnlyContextImpl(getExecutionConfig(), userFunction, broadcastStates, timerService);
	onTimerContext = new OnTimerContextImpl(getExecutionConfig(), userFunction, broadcastStates, timerService);
}
 

@Override
public void open() throws Exception {
	joinCondition = generatedJoinCondition.newInstance(getRuntimeContext().getUserCodeClassLoader());
	joinCondition.setRuntimeContext(getRuntimeContext());
	joinCondition.open(new Configuration());

	nextLeftIndex = getRuntimeContext().getState(
		new ValueStateDescriptor<>(NEXT_LEFT_INDEX_STATE_NAME, Types.LONG));
	leftState = getRuntimeContext().getMapState(
		new MapStateDescriptor<>(LEFT_STATE_NAME, Types.LONG, leftType));
	rightState = getRuntimeContext().getMapState(
		new MapStateDescriptor<>(RIGHT_STATE_NAME, Types.LONG, rightType));
	registeredTimer = getRuntimeContext().getState(
		new ValueStateDescriptor<>(REGISTERED_TIMER_STATE_NAME, Types.LONG));

	timerService = getInternalTimerService(
		TIMERS_STATE_NAME, VoidNamespaceSerializer.INSTANCE, this);
	collector = new TimestampedCollector<>(output);
	outRow = new JoinedRow();
	outRow.setHeader(BaseRowUtil.ACCUMULATE_MSG);
}
 

private void migrateOldState() throws Exception {
	getKeyedStateBackend().applyToAllKeys(
		VoidNamespace.INSTANCE,
		VoidNamespaceSerializer.INSTANCE,
		new ValueStateDescriptor<>(
			"nfaOperatorStateName",
			new NFA.NFASerializer<>(inputSerializer)
		),
		new KeyedStateFunction<Object, ValueState<MigratedNFA<IN>>>() {
			@Override
			public void process(Object key, ValueState<MigratedNFA<IN>> state) throws Exception {
				MigratedNFA<IN> oldState = state.value();
				computationStates.update(new NFAState(oldState.getComputationStates()));
				org.apache.flink.cep.nfa.SharedBuffer<IN> sharedBuffer = oldState.getSharedBuffer();
				partialMatches.init(sharedBuffer.getEventsBuffer(), sharedBuffer.getPages());
				state.clear();
			}
		}
	);
}
 

/** Removes the current key from <b>ALL</b> known states in the state backend. */
@Override
public void remove() {
	if (currentKey == null) {
		return;
	}

	for (StateDescriptor<?, ?> descriptor : descriptors) {
		try {
			State state = backend.getPartitionedState(
				VoidNamespace.INSTANCE,
				VoidNamespaceSerializer.INSTANCE,
				descriptor);

			state.clear();
		} catch (Exception e) {
			throw new RuntimeException("Failed to drop partitioned state from state backend", e);
		}
	}
}
 

static void fireExpiredAsyncOperations(
    MapStateDescriptor<Long, Message> asyncOperationStateDescriptor,
    Reductions reductions,
    MapState<Long, Message> asyncOperationState,
    KeyedStateBackend<String> keyedStateBackend)
    throws Exception {

  AsyncOperationFailureNotifier asyncOperationFailureNotifier =
      new AsyncOperationFailureNotifier(reductions, asyncOperationState);

  keyedStateBackend.applyToAllKeys(
      VoidNamespace.get(),
      VoidNamespaceSerializer.INSTANCE,
      asyncOperationStateDescriptor,
      asyncOperationFailureNotifier);

  if (asyncOperationFailureNotifier.enqueued()) {
    reductions.processEnvelopes();
  }
}
 

static void fireExpiredAsyncOperations(
    MapStateDescriptor<Long, Message> asyncOperationStateDescriptor,
    Reductions reductions,
    KeyedStateBackend<String> keyedStateBackend)
    throws Exception {

  AsyncOperationFailureNotifier asyncOperationFailureNotifier =
      new AsyncOperationFailureNotifier(reductions);

  keyedStateBackend.applyToAllKeys(
      VoidNamespace.get(),
      VoidNamespaceSerializer.INSTANCE,
      asyncOperationStateDescriptor,
      asyncOperationFailureNotifier);

  if (asyncOperationFailureNotifier.enqueued()) {
    reductions.processEnvelopes();
  }
}
 

public FlinkWatermarkHoldState(
    KeyedStateBackend<ByteBuffer> flinkStateBackend,
    MapStateDescriptor<String, Instant> watermarkHoldStateDescriptor,
    String stateId,
    StateNamespace namespace,
    TimestampCombiner timestampCombiner) {
  this.timestampCombiner = timestampCombiner;
  // Combines StateNamespace and stateId to generate a unique namespace for
  // watermarkHoldsState. We do not want to use Flink's namespacing to be
  // able to recover watermark holds efficiently during recovery.
  this.namespaceString = namespace.stringKey() + stateId;
  try {
    this.watermarkHoldsState =
        flinkStateBackend.getPartitionedState(
            VoidNamespace.INSTANCE,
            VoidNamespaceSerializer.INSTANCE,
            watermarkHoldStateDescriptor);
  } catch (Exception e) {
    throw new RuntimeException("Could not access state for watermark partition view");
  }
}
 

@Override
public void open() throws Exception {
	joinCondition = generatedJoinCondition.newInstance(getRuntimeContext().getUserCodeClassLoader());
	joinCondition.setRuntimeContext(getRuntimeContext());
	joinCondition.open(new Configuration());

	nextLeftIndex = getRuntimeContext().getState(
		new ValueStateDescriptor<>(NEXT_LEFT_INDEX_STATE_NAME, Types.LONG));
	leftState = getRuntimeContext().getMapState(
		new MapStateDescriptor<>(LEFT_STATE_NAME, Types.LONG, leftType));
	rightState = getRuntimeContext().getMapState(
		new MapStateDescriptor<>(RIGHT_STATE_NAME, Types.LONG, rightType));
	registeredTimer = getRuntimeContext().getState(
		new ValueStateDescriptor<>(REGISTERED_TIMER_STATE_NAME, Types.LONG));

	timerService = getInternalTimerService(
		TIMERS_STATE_NAME, VoidNamespaceSerializer.INSTANCE, this);
	collector = new TimestampedCollector<>(output);
	outRow = new JoinedRowData();
	// all the output records should be INSERT only,
	// because current temporal join only supports INSERT only left stream
	outRow.setRowKind(RowKind.INSERT);
}
 

private void migrateOldState() throws Exception {
	getKeyedStateBackend().applyToAllKeys(
		VoidNamespace.INSTANCE,
		VoidNamespaceSerializer.INSTANCE,
		new ValueStateDescriptor<>(
			"nfaOperatorStateName",
			new NFA.NFASerializer<>(inputSerializer)
		),
		new KeyedStateFunction<Object, ValueState<MigratedNFA<IN>>>() {
			@Override
			public void process(Object key, ValueState<MigratedNFA<IN>> state) throws Exception {
				MigratedNFA<IN> oldState = state.value();
				computationStates.update(new NFAState(oldState.getComputationStates()));
				org.apache.flink.cep.nfa.SharedBuffer<IN> sharedBuffer = oldState.getSharedBuffer();
				partialMatches.init(sharedBuffer.getEventsBuffer(), sharedBuffer.getPages());
				state.clear();
			}
		}
	);
}
 

@Override
public void open() throws Exception {
	super.open();
	timerService = getInternalTimerService(
			"watermark-callbacks",
			VoidNamespaceSerializer.INSTANCE,
			this);

	nfa = nfaFactory.createNFA();
	nfa.open(cepRuntimeContext, new Configuration());

	context = new ContextFunctionImpl();
	collector = new TimestampedCollector<>(output);
	cepTimerService = new TimerServiceImpl();

	// metrics
	this.numLateRecordsDropped = metrics.counter(LATE_ELEMENTS_DROPPED_METRIC_NAME);
}
 

/** Removes the current key from <b>ALL</b> known states in the state backend. */
@Override
public void remove() {
	if (currentKey == null) {
		return;
	}

	for (StateDescriptor<?, ?> descriptor : descriptors) {
		try {
			State state = backend.getPartitionedState(
				VoidNamespace.INSTANCE,
				VoidNamespaceSerializer.INSTANCE,
				descriptor);

			state.clear();
		} catch (Exception e) {
			throw new RuntimeException("Failed to drop partitioned state from state backend", e);
		}
	}
}
 

@Override
public void open() throws Exception {
	super.open();

	InternalTimerService<VoidNamespace> internalTimerService =
			getInternalTimerService("user-timers", VoidNamespaceSerializer.INSTANCE, this);

	TimerService timerService = new SimpleTimerService(internalTimerService);

	collector = new TimestampedCollector<>(output);

	this.broadcastStates = new HashMap<>(broadcastStateDescriptors.size());
	for (MapStateDescriptor<?, ?> descriptor: broadcastStateDescriptors) {
		broadcastStates.put(descriptor, getOperatorStateBackend().getBroadcastState(descriptor));
	}

	rwContext = new ReadWriteContextImpl(getExecutionConfig(), getKeyedStateBackend(), userFunction, broadcastStates, timerService);
	rContext = new ReadOnlyContextImpl(getExecutionConfig(), userFunction, broadcastStates, timerService);
	onTimerContext = new OnTimerContextImpl(getExecutionConfig(), userFunction, broadcastStates, timerService);
}
 

@Override
public void open() throws Exception {
	super.open();
	timerService = getInternalTimerService(
			"watermark-callbacks",
			VoidNamespaceSerializer.INSTANCE,
			this);

	nfa = nfaFactory.createNFA();
	nfa.open(cepRuntimeContext, new Configuration());

	context = new ContextFunctionImpl();
	collector = new TimestampedCollector<>(output);
	cepTimerService = new TimerServiceImpl();
}
 

@Test
public void testDisposeDeletesAllDirectories() throws Exception {
	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
	Collection<File> allFilesInDbDir =
		FileUtils.listFilesAndDirs(new File(dbPath), new AcceptAllFilter(), new AcceptAllFilter());
	try {
		ValueStateDescriptor<String> kvId =
			new ValueStateDescriptor<>("id", String.class, null);

		kvId.initializeSerializerUnlessSet(new ExecutionConfig());

		ValueState<String> state =
			backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);

		backend.setCurrentKey(1);
		state.update("Hello");

		// more than just the root directory
		assertTrue(allFilesInDbDir.size() > 1);
	} finally {
		IOUtils.closeQuietly(backend);
		backend.dispose();
	}
	allFilesInDbDir =
		FileUtils.listFilesAndDirs(new File(dbPath), new AcceptAllFilter(), new AcceptAllFilter());

	// just the root directory left
	assertEquals(1, allFilesInDbDir.size());
}
 

@Override
public void open() throws Exception {
	super.open();

	// also get the state in open, this way we are sure that it was created before
	// we trigger the test checkpoint
	ValueState<String> state = getPartitionedState(
			VoidNamespace.INSTANCE,
			VoidNamespaceSerializer.INSTANCE,
			new ValueStateDescriptor<>("count", StringSerializer.INSTANCE));

}
 

@Override
public void processElement(StreamRecord<String> element) throws Exception {
	// we also don't care

	ValueState<String> state = getPartitionedState(
			VoidNamespace.INSTANCE,
			VoidNamespaceSerializer.INSTANCE,
			new ValueStateDescriptor<>("count", StringSerializer.INSTANCE));

	state.update(element.getValue());
}
 

@Override
public void open() throws Exception {
	super.open();
	collector = new TimestampedCollector<>(output);

	InternalTimerService<VoidNamespace> internalTimerService =
			getInternalTimerService("user-timers", VoidNamespaceSerializer.INSTANCE, this);

	TimerService timerService = new SimpleTimerService(internalTimerService);

	context = new ContextImpl(userFunction, timerService);
	onTimerContext = new OnTimerContextImpl(userFunction, timerService);
}
 

@Override
public void open() throws Exception {
	super.open();
	collector = new TimestampedCollector<>(output);

	InternalTimerService<VoidNamespace> internalTimerService =
			getInternalTimerService("user-timers", VoidNamespaceSerializer.INSTANCE, this);

	TimerService timerService = new SimpleTimerService(internalTimerService);

	context = new ContextImpl<>(userFunction, timerService);
	onTimerContext = new OnTimerContextImpl<>(userFunction, timerService);
}
 

@Override
public <VS, S extends State> void applyToKeyedState(
		final StateDescriptor<S, VS> stateDescriptor,
		final KeyedStateFunction<KS, S> function) throws Exception {

	keyedStateBackend.applyToAllKeys(
			VoidNamespace.INSTANCE,
			VoidNamespaceSerializer.INSTANCE,
			Preconditions.checkNotNull(stateDescriptor),
			Preconditions.checkNotNull(function));
}
 

@Override
public void open() throws Exception {
	super.open();
	collector = new TimestampedCollector<>(output);

	InternalTimerService<VoidNamespace> internalTimerService =
			getInternalTimerService("user-timers", VoidNamespaceSerializer.INSTANCE, this);

	TimerService timerService = new SimpleTimerService(internalTimerService);

	context = new ContextImpl(userFunction, timerService);
	onTimerContext = new OnTimerContextImpl(userFunction, timerService);
}
 

@Override
public void open() throws Exception {
	super.open();

	this.timerService = getInternalTimerService(
			"test-timers",
			VoidNamespaceSerializer.INSTANCE,
			this);
}
 

@SuppressWarnings("unchecked")
private static AbstractStreamOperator<?> createListPlainMockOp() throws Exception {

	AbstractStreamOperator<?> operatorMock = mock(AbstractStreamOperator.class);
	ExecutionConfig config = new ExecutionConfig();

	KeyedStateBackend keyedStateBackend = mock(KeyedStateBackend.class);

	DefaultKeyedStateStore keyedStateStore = new DefaultKeyedStateStore(keyedStateBackend, config);

	when(operatorMock.getExecutionConfig()).thenReturn(config);

	doAnswer(new Answer<ListState<String>>() {

		@Override
		public ListState<String> answer(InvocationOnMock invocationOnMock) throws Throwable {
			ListStateDescriptor<String> descr =
					(ListStateDescriptor<String>) invocationOnMock.getArguments()[2];

			AbstractKeyedStateBackend<Integer> backend = new MemoryStateBackend().createKeyedStateBackend(
				new DummyEnvironment("test_task", 1, 0),
				new JobID(),
				"test_op",
				IntSerializer.INSTANCE,
				1,
				new KeyGroupRange(0, 0),
				new KvStateRegistry().createTaskRegistry(new JobID(), new JobVertexID()),
				TtlTimeProvider.DEFAULT,
				new UnregisteredMetricsGroup(),
				Collections.emptyList(),
				new CloseableRegistry());
			backend.setCurrentKey(0);
			return backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, descr);
		}
	}).when(keyedStateBackend).getPartitionedState(Matchers.any(), any(TypeSerializer.class), any(ListStateDescriptor.class));

	when(operatorMock.getKeyedStateStore()).thenReturn(keyedStateStore);
	when(operatorMock.getOperatorID()).thenReturn(new OperatorID());
	return operatorMock;
}
 

@SuppressWarnings("unchecked")
private static AbstractStreamOperator<?> createMapPlainMockOp() throws Exception {

	AbstractStreamOperator<?> operatorMock = mock(AbstractStreamOperator.class);
	ExecutionConfig config = new ExecutionConfig();

	KeyedStateBackend keyedStateBackend = mock(KeyedStateBackend.class);

	DefaultKeyedStateStore keyedStateStore = new DefaultKeyedStateStore(keyedStateBackend, config);

	when(operatorMock.getExecutionConfig()).thenReturn(config);

	doAnswer(new Answer<MapState<Integer, String>>() {

		@Override
		public MapState<Integer, String> answer(InvocationOnMock invocationOnMock) throws Throwable {
			MapStateDescriptor<Integer, String> descr =
					(MapStateDescriptor<Integer, String>) invocationOnMock.getArguments()[2];

			AbstractKeyedStateBackend<Integer> backend = new MemoryStateBackend().createKeyedStateBackend(
				new DummyEnvironment("test_task", 1, 0),
				new JobID(),
				"test_op",
				IntSerializer.INSTANCE,
				1,
				new KeyGroupRange(0, 0),
				new KvStateRegistry().createTaskRegistry(new JobID(), new JobVertexID()),
				TtlTimeProvider.DEFAULT,
				new UnregisteredMetricsGroup(),
				Collections.emptyList(),
				new CloseableRegistry());
			backend.setCurrentKey(0);
			return backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, descr);
		}
	}).when(keyedStateBackend).getPartitionedState(Matchers.any(), any(TypeSerializer.class), any(MapStateDescriptor.class));

	when(operatorMock.getKeyedStateStore()).thenReturn(keyedStateStore);
	when(operatorMock.getOperatorID()).thenReturn(new OperatorID());
	return operatorMock;
}
 

@Override
public void open() throws Exception {
	InternalTimerService<VoidNamespace> internalTimerService = getInternalTimerService("user-timers",
			VoidNamespaceSerializer.INSTANCE,
			this);
	timerService = new SimpleTimerService(internalTimerService);
	collector = new TimestampedCollector<>(output);
}
 

private void initializeTimerService() {
	InternalTimerService<VoidNamespace> internalTimerService = getInternalTimerService(
		TIMERS_STATE_NAME,
		VoidNamespaceSerializer.INSTANCE,
		this);

	timerService = new SimpleTimerService(internalTimerService);
}
 

@Override
public void open() throws Exception {
	super.open();

	this.timerService = getInternalTimerService(
		"test-timers",
		VoidNamespaceSerializer.INSTANCE,
		this);
}
 

@Override
public void open() throws Exception {
	super.open();

	InternalTimerService<VoidNamespace> internalTimerService = getInternalTimerService(
		"user-timers",
		VoidNamespaceSerializer.INSTANCE,
		VoidTriggerable.instance());

	TimerService timerService = new SimpleTimerService(internalTimerService);

	context = new KeyedStateBootstrapOperator<K, IN>.ContextImpl(userFunction, timerService);
}
 

private static void setKey(
	AbstractKeyedStateBackend<Integer> backend,
	ValueStateDescriptor<Integer> descriptor,
	int key) throws Exception {
	backend.setCurrentKey(key);
	backend
		.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, descriptor)
		.update(0);
}
 

@Test
public void testDisposeDeletesAllDirectories() throws Exception {
	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
	Collection<File> allFilesInDbDir =
		FileUtils.listFilesAndDirs(new File(dbPath), new AcceptAllFilter(), new AcceptAllFilter());
	try {
		ValueStateDescriptor<String> kvId =
			new ValueStateDescriptor<>("id", String.class, null);

		kvId.initializeSerializerUnlessSet(new ExecutionConfig());

		ValueState<String> state =
			backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);

		backend.setCurrentKey(1);
		state.update("Hello");

		// more than just the root directory
		assertTrue(allFilesInDbDir.size() > 1);
	} finally {
		IOUtils.closeQuietly(backend);
		backend.dispose();
	}
	allFilesInDbDir =
		FileUtils.listFilesAndDirs(new File(dbPath), new AcceptAllFilter(), new AcceptAllFilter());

	// just the root directory left
	assertEquals(1, allFilesInDbDir.size());
}