org.apache.flink.streaming.api.operators.AbstractStreamOperator Java Examples

The following examples show how to use org.apache.flink.streaming.api.operators.AbstractStreamOperator. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: StreamTaskTestHarness.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Users of the test harness can call this utility method to setup the stream config
 * if there will only be a single operator to be tested. The method will setup the
 * outgoing network connection for the operator.
 *
 * <p>For more advanced test cases such as testing chains of multiple operators with the harness,
 * please manually configure the stream config.
 */
public void setupOutputForSingletonOperatorChain() {
	Preconditions.checkState(!setupCalled, "This harness was already setup.");
	setupCalled = true;
	streamConfig.setChainStart();
	streamConfig.setTimeCharacteristic(TimeCharacteristic.EventTime);
	streamConfig.setOutputSelectors(Collections.emptyList());
	streamConfig.setNumberOfOutputs(1);
	streamConfig.setTypeSerializerOut(outputSerializer);
	streamConfig.setVertexID(0);
	streamConfig.setOperatorID(new OperatorID(4711L, 123L));

	StreamOperator<OUT> dummyOperator = new AbstractStreamOperator<OUT>() {
		private static final long serialVersionUID = 1L;
	};

	List<StreamEdge> outEdgesInOrder = new LinkedList<>();
	StreamNode sourceVertexDummy = new StreamNode(0, "group", null, dummyOperator, "source dummy", new LinkedList<>(), SourceStreamTask.class);
	StreamNode targetVertexDummy = new StreamNode(1, "group", null, dummyOperator, "target dummy", new LinkedList<>(), SourceStreamTask.class);

	outEdgesInOrder.add(new StreamEdge(sourceVertexDummy, targetVertexDummy, 0, new LinkedList<>(), new BroadcastPartitioner<>(), null /* output tag */));

	streamConfig.setOutEdgesInOrder(outEdgesInOrder);
	streamConfig.setNonChainedOutputs(outEdgesInOrder);
}
 
Example #2
Source File: StreamOperatorWrapperTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testClosingOperatorWithException() {
	AbstractStreamOperator streamOperator = new AbstractStreamOperator<Void>() {
		@Override
		public void close() throws Exception {
			throw new Exception("test exception at closing");
		}
	};

	StreamOperatorWrapper<?, ?> operatorWrapper = new StreamOperatorWrapper<>(
		streamOperator,
		Optional.ofNullable(streamOperator.getProcessingTimeService()),
		containingTask.getMailboxExecutorFactory().createExecutor(Integer.MAX_VALUE - 1));

	try {
		operatorWrapper.close(containingTask.getActionExecutor());
		fail("should throw an exception");
	} catch (Throwable t) {
		Optional<Throwable> optional = ExceptionUtils.findThrowableWithMessage(t, "test exception at closing");
		assertTrue(optional.isPresent());
	}
}
 
Example #3
Source File: AbstractStreamOperatorTestHarness.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Calls {@link SetupableStreamOperator#setup(StreamTask, StreamConfig, Output)} ()}.
 */
public void setup(TypeSerializer<OUT> outputSerializer) {
	if (!setupCalled) {
		streamTaskStateInitializer =
			createStreamTaskStateManager(environment, stateBackend, ttlTimeProvider);
		mockTask.setStreamTaskStateInitializer(streamTaskStateInitializer);

		if (operator == null) {
			this.operator = StreamOperatorFactoryUtil.createOperator(factory, mockTask, config,
					new MockOutput(outputSerializer), null).f0;
		} else {
			if (operator instanceof AbstractStreamOperator) {
				((AbstractStreamOperator) operator).setProcessingTimeService(processingTimeService);
			}
			if (operator instanceof SetupableStreamOperator) {
				((SetupableStreamOperator) operator).setup(mockTask, config, new MockOutput(outputSerializer));
			}
		}
		setupCalled = true;
		this.mockTask.init();
	}
}
 
Example #4
Source File: SourceFunctionUtil.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
private static <T extends Serializable> List<T> runRichSourceFunction(SourceFunction<T> sourceFunction) throws Exception {
	try (MockEnvironment environment =
			new MockEnvironmentBuilder()
				.setTaskName("MockTask")
				.setMemorySize(3 * 1024 * 1024)
				.setInputSplitProvider(new MockInputSplitProvider())
				.setBufferSize(1024)
				.build()) {

		AbstractStreamOperator<?> operator = mock(AbstractStreamOperator.class);
		when(operator.getExecutionConfig()).thenReturn(new ExecutionConfig());

		RuntimeContext runtimeContext = new StreamingRuntimeContext(
			operator,
			environment,
			new HashMap<>());
		((RichFunction) sourceFunction).setRuntimeContext(runtimeContext);
		((RichFunction) sourceFunction).open(new Configuration());

		return runNonRichSourceFunction(sourceFunction);
	}
}
 
Example #5
Source File: AbstractStreamOperatorTestHarnessTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testInitializeAfterOpenning() throws Throwable {
	expectedException.expect(IllegalStateException.class);
	expectedException.expectMessage(containsString("TestHarness has already been initialized."));

	AbstractStreamOperatorTestHarness<Integer> result;
	result =
		new AbstractStreamOperatorTestHarness<>(
			new AbstractStreamOperator<Integer>() {
			},
			1,
			1,
			0);
	result.setup();
	result.open();
	result.initializeState(new OperatorSubtaskState());
}
 
Example #6
Source File: AbstractStreamOperatorTestHarnessTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testInitializeAfterOpenning() throws Throwable {
	expectedException.expect(IllegalStateException.class);
	expectedException.expectMessage(containsString("TestHarness has already been initialized."));

	AbstractStreamOperatorTestHarness<Integer> result;
	result =
		new AbstractStreamOperatorTestHarness<>(
			new AbstractStreamOperator<Integer>() {
			},
			1,
			1,
			0);
	result.setup();
	result.open();
	result.initializeState(new OperatorSubtaskState());
}
 
Example #7
Source File: SourceFunctionUtil.java    From flink with Apache License 2.0 6 votes vote down vote up
private static <T extends Serializable> List<T> runRichSourceFunction(SourceFunction<T> sourceFunction) throws Exception {
	try (MockEnvironment environment =
			new MockEnvironmentBuilder()
				.setTaskName("MockTask")
				.setMemorySize(3 * 1024 * 1024)
				.setInputSplitProvider(new MockInputSplitProvider())
				.setBufferSize(1024)
				.build()) {

		AbstractStreamOperator<?> operator = mock(AbstractStreamOperator.class);
		when(operator.getExecutionConfig()).thenReturn(new ExecutionConfig());

		RuntimeContext runtimeContext = new StreamingRuntimeContext(
			operator,
			environment,
			new HashMap<>());
		((RichFunction) sourceFunction).setRuntimeContext(runtimeContext);
		((RichFunction) sourceFunction).open(new Configuration());

		return runNonRichSourceFunction(sourceFunction);
	}
}
 
Example #8
Source File: MockStreamConfig.java    From flink with Apache License 2.0 6 votes vote down vote up
public MockStreamConfig(Configuration configuration, int numberOfOutputs) {
	super(configuration);

	setChainStart();
	setOutputSelectors(Collections.emptyList());
	setNumberOfOutputs(numberOfOutputs);
	setTypeSerializerOut(new StringSerializer());
	setVertexID(0);
	setStreamOperator(new TestSequentialReadingStreamOperator("test operator"));
	setOperatorID(new OperatorID());

	StreamOperator dummyOperator = new AbstractStreamOperator() {
		private static final long serialVersionUID = 1L;
	};

	StreamNode sourceVertex = new StreamNode(0, null, null, dummyOperator, "source", new ArrayList<>(), SourceStreamTask.class);
	StreamNode targetVertex = new StreamNode(1, null, null, dummyOperator, "target", new ArrayList<>(), SourceStreamTask.class);

	List<StreamEdge> outEdgesInOrder = new ArrayList<>(numberOfOutputs);
	for (int i = 0; i < numberOfOutputs; i++) {
		outEdgesInOrder.add(
			new StreamEdge(sourceVertex, targetVertex, numberOfOutputs, new ArrayList<>(), new BroadcastPartitioner<>(), null));
	}
	setOutEdgesInOrder(outEdgesInOrder);
	setNonChainedOutputs(outEdgesInOrder);
}
 
Example #9
Source File: SourceFunctionUtil.java    From flink with Apache License 2.0 6 votes vote down vote up
private static <T extends Serializable> List<T> runRichSourceFunction(SourceFunction<T> sourceFunction) throws Exception {
	try (MockEnvironment environment =
			new MockEnvironmentBuilder()
				.setTaskName("MockTask")
				.setManagedMemorySize(3 * 1024 * 1024)
				.setInputSplitProvider(new MockInputSplitProvider())
				.setBufferSize(1024)
				.build()) {

		AbstractStreamOperator<?> operator = mock(AbstractStreamOperator.class);
		when(operator.getExecutionConfig()).thenReturn(new ExecutionConfig());

		RuntimeContext runtimeContext = new StreamingRuntimeContext(
			operator,
			environment,
			new HashMap<>());
		((RichFunction) sourceFunction).setRuntimeContext(runtimeContext);
		((RichFunction) sourceFunction).open(new Configuration());

		return runNonRichSourceFunction(sourceFunction);
	}
}
 
Example #10
Source File: AbstractStreamOperatorTestHarnessTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testInitializeAfterOpenning() throws Throwable {
	expectedException.expect(IllegalStateException.class);
	expectedException.expectMessage(containsString("TestHarness has already been initialized."));

	AbstractStreamOperatorTestHarness<Integer> result;
	result =
		new AbstractStreamOperatorTestHarness<>(
			new AbstractStreamOperator<Integer>() {
			},
			1,
			1,
			0);
	result.setup();
	result.open();
	result.initializeState(new OperatorSubtaskState());
}
 
Example #11
Source File: StreamTaskTestHarness.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Users of the test harness can call this utility method to setup the stream config
 * if there will only be a single operator to be tested. The method will setup the
 * outgoing network connection for the operator.
 *
 * <p>For more advanced test cases such as testing chains of multiple operators with the harness,
 * please manually configure the stream config.
 */
public void setupOutputForSingletonOperatorChain() {
	Preconditions.checkState(!setupCalled, "This harness was already setup.");
	setupCalled = true;
	streamConfig.setChainStart();
	streamConfig.setBufferTimeout(0);
	streamConfig.setTimeCharacteristic(TimeCharacteristic.EventTime);
	streamConfig.setOutputSelectors(Collections.<OutputSelector<?>>emptyList());
	streamConfig.setNumberOfOutputs(1);
	streamConfig.setTypeSerializerOut(outputSerializer);
	streamConfig.setVertexID(0);
	streamConfig.setOperatorID(new OperatorID(4711L, 123L));

	StreamOperator<OUT> dummyOperator = new AbstractStreamOperator<OUT>() {
		private static final long serialVersionUID = 1L;
	};

	List<StreamEdge> outEdgesInOrder = new LinkedList<StreamEdge>();
	StreamNode sourceVertexDummy = new StreamNode(0, "group", null, dummyOperator, "source dummy", new LinkedList<OutputSelector<?>>(), SourceStreamTask.class);
	StreamNode targetVertexDummy = new StreamNode(1, "group", null, dummyOperator, "target dummy", new LinkedList<OutputSelector<?>>(), SourceStreamTask.class);

	outEdgesInOrder.add(new StreamEdge(sourceVertexDummy, targetVertexDummy, 0, new LinkedList<String>(), new BroadcastPartitioner<Object>(), null /* output tag */));

	streamConfig.setOutEdgesInOrder(outEdgesInOrder);
	streamConfig.setNonChainedOutputs(outEdgesInOrder);
}
 
Example #12
Source File: StreamTaskTestHarness.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Users of the test harness can call this utility method to setup the stream config
 * if there will only be a single operator to be tested. The method will setup the
 * outgoing network connection for the operator.
 *
 * <p>For more advanced test cases such as testing chains of multiple operators with the harness,
 * please manually configure the stream config.
 */
public void setupOutputForSingletonOperatorChain() {
	Preconditions.checkState(!setupCalled, "This harness was already setup.");
	setupCalled = true;
	streamConfig.setChainStart();
	streamConfig.setBufferTimeout(0);
	streamConfig.setTimeCharacteristic(TimeCharacteristic.EventTime);
	streamConfig.setOutputSelectors(Collections.<OutputSelector<?>>emptyList());
	streamConfig.setNumberOfOutputs(1);
	streamConfig.setTypeSerializerOut(outputSerializer);
	streamConfig.setVertexID(0);
	streamConfig.setOperatorID(new OperatorID(4711L, 123L));

	StreamOperator<OUT> dummyOperator = new AbstractStreamOperator<OUT>() {
		private static final long serialVersionUID = 1L;
	};

	List<StreamEdge> outEdgesInOrder = new LinkedList<StreamEdge>();
	StreamNode sourceVertexDummy = new StreamNode(null, 0, "group", null, dummyOperator, "source dummy", new LinkedList<OutputSelector<?>>(), SourceStreamTask.class);
	StreamNode targetVertexDummy = new StreamNode(null, 1, "group", null, dummyOperator, "target dummy", new LinkedList<OutputSelector<?>>(), SourceStreamTask.class);

	outEdgesInOrder.add(new StreamEdge(sourceVertexDummy, targetVertexDummy, 0, new LinkedList<String>(), new BroadcastPartitioner<Object>(), null /* output tag */));

	streamConfig.setOutEdgesInOrder(outEdgesInOrder);
	streamConfig.setNonChainedOutputs(outEdgesInOrder);
}
 
Example #13
Source File: KeyedOneInputStreamOperatorTestHarness.java    From flink with Apache License 2.0 5 votes vote down vote up
public <N> int numKeyedStateEntries(N namespace) {
	AbstractStreamOperator<?> abstractStreamOperator = (AbstractStreamOperator<?>) operator;
	KeyedStateBackend<Object> keyedStateBackend = abstractStreamOperator.getKeyedStateBackend();
	if (keyedStateBackend instanceof HeapKeyedStateBackend) {
		return ((HeapKeyedStateBackend) keyedStateBackend).numKeyValueStateEntries(namespace);
	} else {
		throw new UnsupportedOperationException();
	}
}
 
Example #14
Source File: StreamTaskTest.java    From flink with Apache License 2.0 5 votes vote down vote up
MockStreamTask(
		Environment env,
		OperatorChain<String, AbstractStreamOperator<String>> operatorChain,
		Thread.UncaughtExceptionHandler uncaughtExceptionHandler) throws Exception {
	super(env, null, uncaughtExceptionHandler);
	this.overrideOperatorChain = operatorChain;
}
 
Example #15
Source File: AbstractStreamOperatorTestHarness.java    From flink with Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
public int numProcessingTimeTimers() {
	if (operator instanceof AbstractStreamOperator) {
		return ((AbstractStreamOperator) operator).numProcessingTimeTimers();
	} else {
		throw new UnsupportedOperationException();
	}
}
 
Example #16
Source File: AbstractStreamOperatorTestHarness.java    From flink with Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
public int numEventTimeTimers() {
	if (operator instanceof AbstractStreamOperator) {
		return ((AbstractStreamOperator) operator).numEventTimeTimers();
	} else {
		throw new UnsupportedOperationException();
	}
}
 
Example #17
Source File: MultipleInputStreamTaskTestHarnessBuilder.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
protected void initializeInputs(StreamMockEnvironment streamMockEnvironment) {
	inputGates = new StreamTestSingleInputGate[inputSerializers.size()];
	List<StreamEdge> inPhysicalEdges = new LinkedList<>();

	StreamOperator<?> dummyOperator = new AbstractStreamOperator<Object>() {
		private static final long serialVersionUID = 1L;
	};

	StreamNode sourceVertexDummy = new StreamNode(0, "default group", null, dummyOperator, "source dummy", new LinkedList<>(), SourceStreamTask.class);
	StreamNode targetVertexDummy = new StreamNode(1, "default group", null, dummyOperator, "target dummy", new LinkedList<>(), SourceStreamTask.class);

	for (int i = 0; i < inputSerializers.size(); i++) {
		TypeSerializer<?> inputSerializer = inputSerializers.get(i);
		inputGates[i] = new StreamTestSingleInputGate<>(
			inputChannelsPerGate.get(i),
			i,
			inputSerializer,
			bufferSize);

		StreamEdge streamEdge = new StreamEdge(
			sourceVertexDummy,
			targetVertexDummy,
			i + 1,
			new LinkedList<>(),
			new BroadcastPartitioner<>(),
			null);

		inPhysicalEdges.add(streamEdge);
		streamMockEnvironment.addInputGate(inputGates[i].getInputGate());
	}

	streamConfig.setInPhysicalEdges(inPhysicalEdges);
	streamConfig.setNumberOfInputs(inputGates.length);
	streamConfig.setTypeSerializersIn(inputSerializers.toArray(new TypeSerializer[inputSerializers.size()]));
}
 
Example #18
Source File: SubtaskCheckpointCoordinatorTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testNotifyCheckpointAbortedBeforeAsyncPhase() throws Exception {
	TestTaskStateManager stateManager = new TestTaskStateManager();
	MockEnvironment mockEnvironment = MockEnvironment.builder().setTaskStateManager(stateManager).build();
	SubtaskCheckpointCoordinatorImpl subtaskCheckpointCoordinator = (SubtaskCheckpointCoordinatorImpl) new MockSubtaskCheckpointCoordinatorBuilder()
		.setEnvironment(mockEnvironment)
		.setUnalignedCheckpointEnabled(true)
		.build();

	CheckpointOperator checkpointOperator = new CheckpointOperator(new OperatorSnapshotFutures());

	final OperatorChain<String, AbstractStreamOperator<String>> operatorChain = operatorChain(checkpointOperator);

	long checkpointId = 42L;
	// notify checkpoint aborted before execution.
	subtaskCheckpointCoordinator.notifyCheckpointAborted(checkpointId, operatorChain, () -> true);
	assertEquals(1, subtaskCheckpointCoordinator.getAbortedCheckpointSize());

	subtaskCheckpointCoordinator.getChannelStateWriter().start(checkpointId, CheckpointOptions.forCheckpointWithDefaultLocation());
	subtaskCheckpointCoordinator.checkpointState(
		new CheckpointMetaData(checkpointId, System.currentTimeMillis()),
		CheckpointOptions.forCheckpointWithDefaultLocation(),
		new CheckpointMetrics(),
		operatorChain,
		() -> true);
	assertFalse(checkpointOperator.isCheckpointed());
	assertEquals(-1, stateManager.getReportedCheckpointId());
	assertEquals(0, subtaskCheckpointCoordinator.getAbortedCheckpointSize());
	assertEquals(0, subtaskCheckpointCoordinator.getAsyncCheckpointRunnableSize());
}
 
Example #19
Source File: SubtaskCheckpointCoordinatorTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testNotifyCheckpointAbortedDuringAsyncPhase() throws Exception {
	MockEnvironment mockEnvironment = MockEnvironment.builder().build();
	SubtaskCheckpointCoordinatorImpl subtaskCheckpointCoordinator = (SubtaskCheckpointCoordinatorImpl) new MockSubtaskCheckpointCoordinatorBuilder()
		.setEnvironment(mockEnvironment)
		.setExecutor(Executors.newSingleThreadExecutor())
		.setUnalignedCheckpointEnabled(true)
		.build();

	final BlockingRunnableFuture rawKeyedStateHandleFuture = new BlockingRunnableFuture();
	OperatorSnapshotFutures operatorSnapshotResult = new OperatorSnapshotFutures(
		DoneFuture.of(SnapshotResult.empty()),
		rawKeyedStateHandleFuture,
		DoneFuture.of(SnapshotResult.empty()),
		DoneFuture.of(SnapshotResult.empty()),
		DoneFuture.of(SnapshotResult.empty()),
		DoneFuture.of(SnapshotResult.empty()));

	final OperatorChain<String, AbstractStreamOperator<String>> operatorChain = operatorChain(new CheckpointOperator(operatorSnapshotResult));

	long checkpointId = 42L;
	subtaskCheckpointCoordinator.getChannelStateWriter().start(checkpointId, CheckpointOptions.forCheckpointWithDefaultLocation());
	subtaskCheckpointCoordinator.checkpointState(
		new CheckpointMetaData(checkpointId, System.currentTimeMillis()),
		CheckpointOptions.forCheckpointWithDefaultLocation(),
		new CheckpointMetrics(),
		operatorChain,
		() -> true);
	rawKeyedStateHandleFuture.awaitRun();
	assertEquals(1, subtaskCheckpointCoordinator.getAsyncCheckpointRunnableSize());
	assertFalse(rawKeyedStateHandleFuture.isCancelled());

	subtaskCheckpointCoordinator.notifyCheckpointAborted(checkpointId, operatorChain, () -> true);
	assertTrue(rawKeyedStateHandleFuture.isCancelled());
	assertEquals(0, subtaskCheckpointCoordinator.getAsyncCheckpointRunnableSize());
}
 
Example #20
Source File: StreamTaskMailboxTestHarnessBuilder.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Users of the test harness can call this utility method to setup the stream config
 * if there will only be a single operator to be tested. The method will setup the
 * outgoing network connection for the operator.
 *
 * <p>For more advanced test cases such as testing chains of multiple operators with the harness,
 * please manually configure the stream config.
 */
public StreamTaskMailboxTestHarnessBuilder<OUT> setupOutputForSingletonOperatorChain(
		StreamOperatorFactory<?> factory,
		OperatorID operatorID) {
	checkState(!setupCalled, "This harness was already setup.");
	setupCalled = true;
	streamConfig.setChainStart();
	streamConfig.setTimeCharacteristic(TimeCharacteristic.EventTime);
	streamConfig.setOutputSelectors(Collections.<OutputSelector<?>>emptyList());
	streamConfig.setNumberOfOutputs(1);
	streamConfig.setTypeSerializerOut(outputSerializer);
	streamConfig.setVertexID(0);

	StreamOperator<OUT> dummyOperator = new AbstractStreamOperator<OUT>() {
		private static final long serialVersionUID = 1L;
	};

	List<StreamEdge> outEdgesInOrder = new LinkedList<StreamEdge>();
	StreamNode sourceVertexDummy = new StreamNode(0, "group", null, dummyOperator, "source dummy", new LinkedList<OutputSelector<?>>(), SourceStreamTask.class);
	StreamNode targetVertexDummy = new StreamNode(1, "group", null, dummyOperator, "target dummy", new LinkedList<OutputSelector<?>>(), SourceStreamTask.class);

	outEdgesInOrder.add(new StreamEdge(sourceVertexDummy, targetVertexDummy, 0, new LinkedList<String>(), new BroadcastPartitioner<Object>(), null /* output tag */));

	streamConfig.setOutEdgesInOrder(outEdgesInOrder);
	streamConfig.setNonChainedOutputs(outEdgesInOrder);

	streamConfig.setStreamOperatorFactory(factory);
	streamConfig.setOperatorID(operatorID);

	return this;
}
 
Example #21
Source File: KeyedOneInputStreamOperatorTestHarness.java    From flink with Apache License 2.0 5 votes vote down vote up
public int numKeyedStateEntries() {
	AbstractStreamOperator<?> abstractStreamOperator = (AbstractStreamOperator<?>) operator;
	KeyedStateBackend<Object> keyedStateBackend = abstractStreamOperator.getKeyedStateBackend();
	if (keyedStateBackend instanceof HeapKeyedStateBackend) {
		return ((HeapKeyedStateBackend) keyedStateBackend).numKeyValueStateEntries();
	} else {
		throw new UnsupportedOperationException();
	}
}
 
Example #22
Source File: KeyedTwoInputStreamOperatorTestHarness.java    From flink with Apache License 2.0 5 votes vote down vote up
public int numKeyedStateEntries() {
	AbstractStreamOperator<?> abstractStreamOperator = (AbstractStreamOperator<?>) operator;
	KeyedStateBackend<Object> keyedStateBackend = abstractStreamOperator.getKeyedStateBackend();
	if (keyedStateBackend instanceof HeapKeyedStateBackend) {
		return ((HeapKeyedStateBackend) keyedStateBackend).numKeyValueStateEntries();
	} else {
		throw new UnsupportedOperationException();
	}
}
 
Example #23
Source File: AbstractStreamOperatorTestHarnessTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testSetTtlTimeProvider() throws Exception {
	AbstractStreamOperator<Integer> operator = new AbstractStreamOperator<Integer>() {};
	try (AbstractStreamOperatorTestHarness<Integer> result = new AbstractStreamOperatorTestHarness<>(
			operator,
			1,
			1,
			0)) {

		result.config.setStateKeySerializer(IntSerializer.INSTANCE);

		Time timeToLive = Time.hours(1);
		result.initializeState(new OperatorSubtaskState());
		result.open();

		ValueStateDescriptor<Integer> stateDescriptor = new ValueStateDescriptor<>("test", IntSerializer.INSTANCE);
		stateDescriptor.enableTimeToLive(StateTtlConfig.newBuilder(timeToLive).build());
		KeyedStateBackend<Integer> keyedStateBackend = operator.getKeyedStateBackend();
		ValueState<Integer> state = keyedStateBackend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, stateDescriptor);

		int expectedValue = 42;
		keyedStateBackend.setCurrentKey(1);
		result.setStateTtlProcessingTime(0L);
		state.update(expectedValue);
		Assert.assertEquals(expectedValue, (int) state.value());
		result.setStateTtlProcessingTime(timeToLive.toMilliseconds() + 1);
		Assert.assertNull(state.value());
	}
}
 
Example #24
Source File: KeyedTwoInputStreamOperatorTestHarness.java    From flink with Apache License 2.0 5 votes vote down vote up
public int numKeyedStateEntries() {
	AbstractStreamOperator<?> abstractStreamOperator = (AbstractStreamOperator<?>) operator;
	KeyedStateBackend<Object> keyedStateBackend = abstractStreamOperator.getKeyedStateBackend();
	if (keyedStateBackend instanceof HeapKeyedStateBackend) {
		return ((HeapKeyedStateBackend) keyedStateBackend).numKeyValueStateEntries();
	} else {
		throw new UnsupportedOperationException();
	}
}
 
Example #25
Source File: KeyedOneInputStreamOperatorTestHarness.java    From flink with Apache License 2.0 5 votes vote down vote up
public int numKeyedStateEntries() {
	AbstractStreamOperator<?> abstractStreamOperator = (AbstractStreamOperator<?>) operator;
	KeyedStateBackend<Object> keyedStateBackend = abstractStreamOperator.getKeyedStateBackend();
	if (keyedStateBackend instanceof HeapKeyedStateBackend) {
		return ((HeapKeyedStateBackend) keyedStateBackend).numKeyValueStateEntries();
	} else {
		throw new UnsupportedOperationException();
	}
}
 
Example #26
Source File: KeyedOneInputStreamOperatorTestHarness.java    From flink with Apache License 2.0 5 votes vote down vote up
public <N> int numKeyedStateEntries(N namespace) {
	AbstractStreamOperator<?> abstractStreamOperator = (AbstractStreamOperator<?>) operator;
	KeyedStateBackend<Object> keyedStateBackend = abstractStreamOperator.getKeyedStateBackend();
	if (keyedStateBackend instanceof HeapKeyedStateBackend) {
		return ((HeapKeyedStateBackend) keyedStateBackend).numKeyValueStateEntries(namespace);
	} else {
		throw new UnsupportedOperationException();
	}
}
 
Example #27
Source File: AbstractStreamOperatorTestHarness.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
public int numEventTimeTimers() {
	if (operator instanceof AbstractStreamOperator) {
		return ((AbstractStreamOperator) operator).numEventTimeTimers();
	} else {
		throw new UnsupportedOperationException();
	}
}
 
Example #28
Source File: AbstractStreamOperatorTestHarness.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
public int numProcessingTimeTimers() {
	if (operator instanceof AbstractStreamOperator) {
		return ((AbstractStreamOperator) operator).numProcessingTimeTimers();
	} else {
		throw new UnsupportedOperationException();
	}
}
 
Example #29
Source File: KeyedOneInputStreamOperatorTestHarness.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
public <N> int numKeyedStateEntries(N namespace) {
	AbstractStreamOperator<?> abstractStreamOperator = (AbstractStreamOperator<?>) operator;
	KeyedStateBackend<Object> keyedStateBackend = abstractStreamOperator.getKeyedStateBackend();
	if (keyedStateBackend instanceof HeapKeyedStateBackend) {
		return ((HeapKeyedStateBackend) keyedStateBackend).numKeyValueStateEntries(namespace);
	} else {
		throw new UnsupportedOperationException();
	}
}
 
Example #30
Source File: KeyedOneInputStreamOperatorTestHarness.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
public int numKeyedStateEntries() {
	AbstractStreamOperator<?> abstractStreamOperator = (AbstractStreamOperator<?>) operator;
	KeyedStateBackend<Object> keyedStateBackend = abstractStreamOperator.getKeyedStateBackend();
	if (keyedStateBackend instanceof HeapKeyedStateBackend) {
		return ((HeapKeyedStateBackend) keyedStateBackend).numKeyValueStateEntries();
	} else {
		throw new UnsupportedOperationException();
	}
}