Java Code Examples for org.apache.flink.streaming.api.operators.StreamSource#run()

The following examples show how to use org.apache.flink.streaming.api.operators.StreamSource#run() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: StreamSourceOperatorWatermarksTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testNoMaxWatermarkOnImmediateCancel() throws Exception {

	final List<StreamElement> output = new ArrayList<>();

	// regular stream source operator
	final StreamSource<String, InfiniteSource<String>> operator =
			new StreamSource<>(new InfiniteSource<String>());

	setupSourceOperator(operator, TimeCharacteristic.EventTime, 0);
	operator.cancel();

	// run and exit
	operator.run(new Object(), mock(StreamStatusMaintainer.class), new CollectorOutput<String>(output));

	assertTrue(output.isEmpty());
}
Example 2
Source File: StreamSourceOperatorWatermarksTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testEmitMaxWatermarkForFiniteSource() throws Exception {

	// regular stream source operator
	StreamSource<String, FiniteSource<String>> operator =
			new StreamSource<>(new FiniteSource<String>());

	final List<StreamElement> output = new ArrayList<>();

	setupSourceOperator(operator, TimeCharacteristic.EventTime, 0);
	OperatorChain<?, ?> operatorChain = createOperatorChain(operator);
	try {
		operator.run(new Object(), mock(StreamStatusMaintainer.class), new CollectorOutput<String>(output), operatorChain);
	} finally {
		operatorChain.releaseOutputs();
	}

	assertEquals(1, output.size());
	assertEquals(Watermark.MAX_WATERMARK, output.get(0));
}
Example 3
Source File: StreamSourceOperatorWatermarksTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testNoMaxWatermarkOnImmediateCancel() throws Exception {

	final List<StreamElement> output = new ArrayList<>();

	// regular stream source operator
	final StreamSource<String, InfiniteSource<String>> operator =
			new StreamSource<>(new InfiniteSource<String>());

	setupSourceOperator(operator, TimeCharacteristic.EventTime, 0);
	operator.cancel();

	// run and exit
	OperatorChain<?, ?> operatorChain = createOperatorChain(operator);
	try {
		operator.run(new Object(), mock(StreamStatusMaintainer.class), new CollectorOutput<String>(output), operatorChain);
	} finally {
		operatorChain.releaseOutputs();
	}

	assertTrue(output.isEmpty());
}
Example 4
Source File: StreamSourceOperatorWatermarksTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Test
public void testEmitMaxWatermarkForFiniteSource() throws Exception {

	// regular stream source operator
	StreamSource<String, FiniteSource<String>> operator =
			new StreamSource<>(new FiniteSource<String>());

	final List<StreamElement> output = new ArrayList<>();

	setupSourceOperator(operator, TimeCharacteristic.EventTime, 0);
	operator.run(new Object(), mock(StreamStatusMaintainer.class), new CollectorOutput<String>(output));

	assertEquals(1, output.size());
	assertEquals(Watermark.MAX_WATERMARK, output.get(0));
}
Example 5
Source File: StreamSources.java    From beam with Apache License 2.0 5 votes vote down vote up 
public static <OutT, SrcT extends SourceFunction<OutT>> void run(
    StreamSource<OutT, SrcT> streamSource,
    Object lockingObject,
    StreamStatusMaintainer streamStatusMaintainer,
    Output<StreamRecord<OutT>> collector)
    throws Exception {
  streamSource.run(lockingObject, streamStatusMaintainer, collector);
}
Example 6
Source File: StreamSources.java    From beam with Apache License 2.0 5 votes vote down vote up 
public static <OutT, SrcT extends SourceFunction<OutT>> void run(
    StreamSource<OutT, SrcT> streamSource,
    Object lockingObject,
    StreamStatusMaintainer streamStatusMaintainer,
    Output<StreamRecord<OutT>> collector)
    throws Exception {
  streamSource.run(
      lockingObject, streamStatusMaintainer, collector, createOperatorChain(streamSource));
}
Example 7
Source File: StreamSources.java    From beam with Apache License 2.0 5 votes vote down vote up 
public static <OutT, SrcT extends SourceFunction<OutT>> void run(
    StreamSource<OutT, SrcT> streamSource,
    Object lockingObject,
    StreamStatusMaintainer streamStatusMaintainer,
    Output<StreamRecord<OutT>> collector)
    throws Exception {
  streamSource.run(
      lockingObject, streamStatusMaintainer, collector, createOperatorChain(streamSource));
}
Example 8
Source File: StreamSourceOperatorLatencyMetricsTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
private void testLatencyMarkEmission(int numberLatencyMarkers, OperatorSetupOperation operatorSetup) throws Exception {
	final List<StreamElement> output = new ArrayList<>();

	final TestProcessingTimeService testProcessingTimeService = new TestProcessingTimeService();
	testProcessingTimeService.setCurrentTime(0L);
	final List<Long> processingTimes = Arrays.asList(1L, 10L, 11L, 21L, maxProcessingTime);

	// regular stream source operator
	final StreamSource<Long, ProcessingTimeServiceSource> operator =
		new StreamSource<>(new ProcessingTimeServiceSource(testProcessingTimeService, processingTimes));

	operatorSetup.setupSourceOperator(operator, testProcessingTimeService);

	// run and wait to be stopped
	operator.run(new Object(), mock(StreamStatusMaintainer.class), new CollectorOutput<Long>(output));

	assertEquals(
		numberLatencyMarkers + 1, // + 1 is the final watermark element
		output.size());

	long timestamp = 0L;
	int expectedLatencyIndex = 0;

	int i = 0;
	// verify that its only latency markers + a final watermark
	for (; i < numberLatencyMarkers; i++) {
		StreamElement se = output.get(i);
		Assert.assertTrue(se.isLatencyMarker());
		Assert.assertEquals(operator.getOperatorID(), se.asLatencyMarker().getOperatorId());
		Assert.assertEquals(0, se.asLatencyMarker().getSubtaskIndex());

		// determines the next latency mark that should've been emitted
		// latency marks are emitted once per latencyMarkInterval,
		// as a result of which we never emit both 10 and 11
		while (timestamp > processingTimes.get(expectedLatencyIndex)) {
			expectedLatencyIndex++;
		}
		Assert.assertEquals(processingTimes.get(expectedLatencyIndex).longValue(), se.asLatencyMarker().getMarkedTime());

		timestamp += latencyMarkInterval;
	}

	Assert.assertTrue(output.get(i).isWatermark());
}
Example 9
Source File: StreamSourceOperatorLatencyMetricsTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
private void testLatencyMarkEmission(int numberLatencyMarkers, OperatorSetupOperation operatorSetup) throws Exception {
	final List<StreamElement> output = new ArrayList<>();

	final TestProcessingTimeService testProcessingTimeService = new TestProcessingTimeService();
	testProcessingTimeService.setCurrentTime(0L);
	final List<Long> processingTimes = Arrays.asList(1L, 10L, 11L, 21L, maxProcessingTime);

	// regular stream source operator
	final StreamSource<Long, ProcessingTimeServiceSource> operator =
		new StreamSource<>(new ProcessingTimeServiceSource(testProcessingTimeService, processingTimes));

	operatorSetup.setupSourceOperator(operator, testProcessingTimeService);

	// run and wait to be stopped
	OperatorChain<?, ?> operatorChain = new OperatorChain<>(
		operator.getContainingTask(),
		StreamTask.createRecordWriters(operator.getOperatorConfig(), new MockEnvironmentBuilder().build()));
	try {
		operator.run(new Object(), mock(StreamStatusMaintainer.class), new CollectorOutput<Long>(output), operatorChain);
	} finally {
		operatorChain.releaseOutputs();
	}

	assertEquals(
		numberLatencyMarkers + 1, // + 1 is the final watermark element
		output.size());

	long timestamp = 0L;
	int expectedLatencyIndex = 0;

	int i = 0;
	// verify that its only latency markers + a final watermark
	for (; i < numberLatencyMarkers; i++) {
		StreamElement se = output.get(i);
		Assert.assertTrue(se.isLatencyMarker());
		Assert.assertEquals(operator.getOperatorID(), se.asLatencyMarker().getOperatorId());
		Assert.assertEquals(0, se.asLatencyMarker().getSubtaskIndex());

		// determines the next latency mark that should've been emitted
		// latency marks are emitted once per latencyMarkInterval,
		// as a result of which we never emit both 10 and 11
		while (timestamp > processingTimes.get(expectedLatencyIndex)) {
			expectedLatencyIndex++;
		}
		Assert.assertEquals(processingTimes.get(expectedLatencyIndex).longValue(), se.asLatencyMarker().getMarkedTime());

		timestamp += latencyMarkInterval;
	}

	Assert.assertTrue(output.get(i).isWatermark());
}
Example 10
Source File: StreamSourceOperatorLatencyMetricsTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
private void testLatencyMarkEmission(int numberLatencyMarkers, OperatorSetupOperation operatorSetup) throws Exception {
	final List<StreamElement> output = new ArrayList<>();

	final TestProcessingTimeService testProcessingTimeService = new TestProcessingTimeService();
	testProcessingTimeService.setCurrentTime(0L);
	final List<Long> processingTimes = Arrays.asList(1L, 10L, 11L, 21L, maxProcessingTime);

	// regular stream source operator
	final StreamSource<Long, ProcessingTimeServiceSource> operator =
		new StreamSource<>(new ProcessingTimeServiceSource(testProcessingTimeService, processingTimes));

	operatorSetup.setupSourceOperator(operator, testProcessingTimeService);

	// run and wait to be stopped
	OperatorChain<?, ?> operatorChain = new OperatorChain<>(
		operator.getContainingTask(),
		StreamTask.createRecordWriterDelegate(operator.getOperatorConfig(), new MockEnvironmentBuilder().build()));
	try {
		operator.run(new Object(), mock(StreamStatusMaintainer.class), new CollectorOutput<>(output), operatorChain);
		operator.close();
	} finally {
		operatorChain.releaseOutputs();
	}

	assertEquals(
		numberLatencyMarkers + 1, // + 1 is the final watermark element
		output.size());

	long timestamp = 0L;
	int expectedLatencyIndex = 0;

	int i = 0;
	// verify that its only latency markers + a final watermark
	for (; i < numberLatencyMarkers; i++) {
		StreamElement se = output.get(i);
		Assert.assertTrue(se.isLatencyMarker());
		Assert.assertEquals(operator.getOperatorID(), se.asLatencyMarker().getOperatorId());
		Assert.assertEquals(0, se.asLatencyMarker().getSubtaskIndex());

		// determines the next latency mark that should've been emitted
		// latency marks are emitted once per latencyMarkInterval,
		// as a result of which we never emit both 10 and 11
		while (timestamp > processingTimes.get(expectedLatencyIndex)) {
			expectedLatencyIndex++;
		}
		Assert.assertEquals(processingTimes.get(expectedLatencyIndex).longValue(), se.asLatencyMarker().getMarkedTime());

		timestamp += latencyMarkInterval;
	}

	Assert.assertTrue(output.get(i).isWatermark());
}