org.apache.flink.streaming.runtime.tasks.SourceStreamTask Java Examples

The following examples show how to use org.apache.flink.streaming.runtime.tasks.SourceStreamTask. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: AbstractUdfStreamOperatorLifecycleTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testLifeCycleFull() throws Exception {
	ACTUAL_ORDER_TRACKING.clear();

	Configuration taskManagerConfig = new Configuration();
	StreamConfig cfg = new StreamConfig(new Configuration());
	MockSourceFunction srcFun = new MockSourceFunction();

	cfg.setStreamOperator(new LifecycleTrackingStreamSource<>(srcFun, true));
	cfg.setOperatorID(new OperatorID());
	cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);

	Task task = StreamTaskTest.createTask(SourceStreamTask.class, cfg, taskManagerConfig);

	task.startTaskThread();

	LifecycleTrackingStreamSource.runStarted.await();

	// wait for clean termination
	task.getExecutingThread().join();
	assertEquals(ExecutionState.FINISHED, task.getExecutionState());
	assertEquals(EXPECTED_CALL_ORDER_FULL, ACTUAL_ORDER_TRACKING);
}
Example #2
Source File: AbstractUdfStreamOperatorLifecycleTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testLifeCycleCancel() throws Exception {
	ACTUAL_ORDER_TRACKING.clear();

	Configuration taskManagerConfig = new Configuration();
	StreamConfig cfg = new StreamConfig(new Configuration());
	MockSourceFunction srcFun = new MockSourceFunction();
	cfg.setStreamOperator(new LifecycleTrackingStreamSource<>(srcFun, false));
	cfg.setOperatorID(new OperatorID());
	cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);

	Task task = StreamTaskTest.createTask(SourceStreamTask.class, cfg, taskManagerConfig);

	task.startTaskThread();
	LifecycleTrackingStreamSource.runStarted.await();

	// this should cancel the task even though it is blocked on runFinished
	task.cancelExecution();

	// wait for clean termination
	task.getExecutingThread().join();
	assertEquals(ExecutionState.CANCELED, task.getExecutionState());
	assertEquals(EXPECTED_CALL_ORDER_CANCEL_RUNNING, ACTUAL_ORDER_TRACKING);
}
Example #3
Source File: AbstractUdfStreamOperatorLifecycleTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testLifeCycleFull() throws Exception {
	ACTUAL_ORDER_TRACKING.clear();

	Configuration taskManagerConfig = new Configuration();
	StreamConfig cfg = new StreamConfig(new Configuration());
	MockSourceFunction srcFun = new MockSourceFunction();

	cfg.setStreamOperator(new LifecycleTrackingStreamSource<>(srcFun, true));
	cfg.setOperatorID(new OperatorID());
	cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);

	Task task = StreamTaskTest.createTask(SourceStreamTask.class, cfg, taskManagerConfig);

	task.startTaskThread();

	LifecycleTrackingStreamSource.runStarted.await();

	// wait for clean termination
	task.getExecutingThread().join();
	assertEquals(ExecutionState.FINISHED, task.getExecutionState());
	assertEquals(EXPECTED_CALL_ORDER_FULL, ACTUAL_ORDER_TRACKING);
}
Example #4
Source File: AbstractUdfStreamOperatorLifecycleTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testLifeCycleCancel() throws Exception {
	ACTUAL_ORDER_TRACKING.clear();

	Configuration taskManagerConfig = new Configuration();
	StreamConfig cfg = new StreamConfig(new Configuration());
	MockSourceFunction srcFun = new MockSourceFunction();
	cfg.setStreamOperator(new LifecycleTrackingStreamSource<>(srcFun, false));
	cfg.setOperatorID(new OperatorID());
	cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);

	Task task = StreamTaskTest.createTask(SourceStreamTask.class, cfg, taskManagerConfig);

	task.startTaskThread();
	LifecycleTrackingStreamSource.runStarted.await();

	// this should cancel the task even though it is blocked on runFinished
	task.cancelExecution();

	// wait for clean termination
	task.getExecutingThread().join();
	assertEquals(ExecutionState.CANCELED, task.getExecutionState());
	assertEquals(EXPECTED_CALL_ORDER_CANCEL_RUNNING, ACTUAL_ORDER_TRACKING);
}
Example #5
Source File: AbstractUdfStreamOperatorLifecycleTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testLifeCycleFull() throws Exception {
	ACTUAL_ORDER_TRACKING.clear();

	Configuration taskManagerConfig = new Configuration();
	StreamConfig cfg = new StreamConfig(new Configuration());
	MockSourceFunction srcFun = new MockSourceFunction();

	cfg.setStreamOperator(new LifecycleTrackingStreamSource<>(srcFun, true));
	cfg.setOperatorID(new OperatorID());
	cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);

	try (ShuffleEnvironment shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build()) {
		Task task = StreamTaskTest.createTask(SourceStreamTask.class, shuffleEnvironment, cfg, taskManagerConfig);

		task.startTaskThread();

		LifecycleTrackingStreamSource.runStarted.await();

		// wait for clean termination
		task.getExecutingThread().join();
		assertEquals(ExecutionState.FINISHED, task.getExecutionState());
		assertEquals(EXPECTED_CALL_ORDER_FULL, ACTUAL_ORDER_TRACKING);
	}
}
Example #6
Source File: AbstractUdfStreamOperatorLifecycleTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testLifeCycleCancel() throws Exception {
	ACTUAL_ORDER_TRACKING.clear();

	Configuration taskManagerConfig = new Configuration();
	StreamConfig cfg = new StreamConfig(new Configuration());
	MockSourceFunction srcFun = new MockSourceFunction();
	cfg.setStreamOperator(new LifecycleTrackingStreamSource<>(srcFun, false));
	cfg.setOperatorID(new OperatorID());
	cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);

	try (ShuffleEnvironment shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build()) {
		Task task = StreamTaskTest.createTask(SourceStreamTask.class, shuffleEnvironment, cfg, taskManagerConfig);

		task.startTaskThread();
		LifecycleTrackingStreamSource.runStarted.await();

		// this should cancel the task even though it is blocked on runFinished
		task.cancelExecution();

		// wait for clean termination
		task.getExecutingThread().join();
		assertEquals(ExecutionState.CANCELED, task.getExecutionState());
		assertEquals(EXPECTED_CALL_ORDER_CANCEL_RUNNING, ACTUAL_ORDER_TRACKING);
	}
}
Example #7
Source File: MockStreamConfig.java    From flink with Apache License 2.0 6 votes vote down vote up 
public MockStreamConfig(Configuration configuration, int numberOfOutputs) {
	super(configuration);

	setChainStart();
	setOutputSelectors(Collections.emptyList());
	setNumberOfOutputs(numberOfOutputs);
	setTypeSerializerOut(new StringSerializer());
	setVertexID(0);
	setStreamOperator(new TestSequentialReadingStreamOperator("test operator"));
	setOperatorID(new OperatorID());

	StreamOperator dummyOperator = new AbstractStreamOperator() {
		private static final long serialVersionUID = 1L;
	};

	StreamNode sourceVertex = new StreamNode(0, null, null, dummyOperator, "source", new ArrayList<>(), SourceStreamTask.class);
	StreamNode targetVertex = new StreamNode(1, null, null, dummyOperator, "target", new ArrayList<>(), SourceStreamTask.class);

	List<StreamEdge> outEdgesInOrder = new ArrayList<>(numberOfOutputs);
	for (int i = 0; i < numberOfOutputs; i++) {
		outEdgesInOrder.add(
			new StreamEdge(sourceVertex, targetVertex, numberOfOutputs, new ArrayList<>(), new BroadcastPartitioner<>(), null));
	}
	setOutEdgesInOrder(outEdgesInOrder);
	setNonChainedOutputs(outEdgesInOrder);
}
Example #8
Source File: StreamGraph.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
public <IN, OUT> void addOperator(
		Integer vertexID,
		String slotSharingGroup,
		@Nullable String coLocationGroup,
		StreamOperator<OUT> operatorObject,
		TypeInformation<IN> inTypeInfo,
		TypeInformation<OUT> outTypeInfo,
		String operatorName) {

	if (operatorObject instanceof StoppableStreamSource) {
		addNode(vertexID, slotSharingGroup, coLocationGroup, StoppableSourceStreamTask.class, operatorObject, operatorName);
	} else if (operatorObject instanceof StreamSource) {
		addNode(vertexID, slotSharingGroup, coLocationGroup, SourceStreamTask.class, operatorObject, operatorName);
	} else {
		addNode(vertexID, slotSharingGroup, coLocationGroup, OneInputStreamTask.class, operatorObject, operatorName);
	}

	TypeSerializer<IN> inSerializer = inTypeInfo != null && !(inTypeInfo instanceof MissingTypeInfo) ? inTypeInfo.createSerializer(executionConfig) : null;

	TypeSerializer<OUT> outSerializer = outTypeInfo != null && !(outTypeInfo instanceof MissingTypeInfo) ? outTypeInfo.createSerializer(executionConfig) : null;

	setSerializers(vertexID, inSerializer, null, outSerializer);

	if (operatorObject instanceof OutputTypeConfigurable && outTypeInfo != null) {
		@SuppressWarnings("unchecked")
		OutputTypeConfigurable<OUT> outputTypeConfigurable = (OutputTypeConfigurable<OUT>) operatorObject;
		// sets the output type which must be know at StreamGraph creation time
		outputTypeConfigurable.setOutputType(outTypeInfo, executionConfig);
	}

	if (operatorObject instanceof InputTypeConfigurable) {
		InputTypeConfigurable inputTypeConfigurable = (InputTypeConfigurable) operatorObject;
		inputTypeConfigurable.setInputType(inTypeInfo, executionConfig);
	}

	if (LOG.isDebugEnabled()) {
		LOG.debug("Vertex: {}", vertexID);
	}
}
Example #9
Source File: StreamGraph.java    From flink with Apache License 2.0 5 votes vote down vote up 
public <IN, OUT> void addOperator(
		Integer vertexID,
		@Nullable String slotSharingGroup,
		@Nullable String coLocationGroup,
		StreamOperatorFactory<OUT> operatorFactory,
		TypeInformation<IN> inTypeInfo,
		TypeInformation<OUT> outTypeInfo,
		String operatorName) {

	if (operatorFactory.isStreamSource()) {
		addNode(vertexID, slotSharingGroup, coLocationGroup, SourceStreamTask.class, operatorFactory, operatorName);
	} else {
		addNode(vertexID, slotSharingGroup, coLocationGroup, OneInputStreamTask.class, operatorFactory, operatorName);
	}

	TypeSerializer<IN> inSerializer = inTypeInfo != null && !(inTypeInfo instanceof MissingTypeInfo) ? inTypeInfo.createSerializer(executionConfig) : null;

	TypeSerializer<OUT> outSerializer = outTypeInfo != null && !(outTypeInfo instanceof MissingTypeInfo) ? outTypeInfo.createSerializer(executionConfig) : null;

	setSerializers(vertexID, inSerializer, null, outSerializer);

	if (operatorFactory.isOutputTypeConfigurable() && outTypeInfo != null) {
		// sets the output type which must be know at StreamGraph creation time
		operatorFactory.setOutputType(outTypeInfo, executionConfig);
	}

	if (operatorFactory.isInputTypeConfigurable()) {
		operatorFactory.setInputType(inTypeInfo, executionConfig);
	}

	if (LOG.isDebugEnabled()) {
		LOG.debug("Vertex: {}", vertexID);
	}
}
Example #10
Source File: StreamGraph.java    From flink with Apache License 2.0 5 votes vote down vote up 
public <IN, OUT> void addOperator(
		Integer vertexID,
		@Nullable String slotSharingGroup,
		@Nullable String coLocationGroup,
		StreamOperatorFactory<OUT> operatorFactory,
		TypeInformation<IN> inTypeInfo,
		TypeInformation<OUT> outTypeInfo,
		String operatorName) {
	Class<? extends AbstractInvokable> invokableClass =
			operatorFactory.isStreamSource() ? SourceStreamTask.class : OneInputStreamTask.class;
	addOperator(vertexID, slotSharingGroup, coLocationGroup, operatorFactory, inTypeInfo,
			outTypeInfo, operatorName, invokableClass);
}
Example #11
Source File: StreamSourceOperatorWatermarksTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
private static <T> StreamTaskTestHarness<T> setupSourceStreamTask(
	StreamSource<T, ?> sourceOperator,
	TypeInformation<T> outputType,
	final boolean cancelImmediatelyAfterCreation) {

	final StreamTaskTestHarness<T> testHarness = new StreamTaskTestHarness<>(
		(env) -> {
			SourceStreamTask<T, ?, ?> sourceTask = new SourceStreamTask<>(env);
			if (cancelImmediatelyAfterCreation) {
				try {
					sourceTask.cancel();
				} catch (Exception e) {
					throw new RuntimeException(e);
				}
			}
			return sourceTask;
		},
		outputType);
	testHarness.setupOutputForSingletonOperatorChain();

	StreamConfig streamConfig = testHarness.getStreamConfig();
	streamConfig.setStreamOperator(sourceOperator);
	streamConfig.setOperatorID(new OperatorID());
	streamConfig.setTimeCharacteristic(TimeCharacteristic.EventTime);

	return testHarness;
}