org.apache.flink.streaming.api.TimeCharacteristic Java Examples

The following examples show how to use org.apache.flink.streaming.api.TimeCharacteristic. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: AbstractUdfStreamOperatorLifecycleTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testLifeCycleCancel() throws Exception {
	ACTUAL_ORDER_TRACKING.clear();

	Configuration taskManagerConfig = new Configuration();
	StreamConfig cfg = new StreamConfig(new Configuration());
	MockSourceFunction srcFun = new MockSourceFunction();
	cfg.setStreamOperator(new LifecycleTrackingStreamSource<>(srcFun, false));
	cfg.setOperatorID(new OperatorID());
	cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);

	Task task = StreamTaskTest.createTask(SourceStreamTask.class, cfg, taskManagerConfig);

	task.startTaskThread();
	LifecycleTrackingStreamSource.runStarted.await();

	// this should cancel the task even though it is blocked on runFinished
	task.cancelExecution();

	// wait for clean termination
	task.getExecutingThread().join();
	assertEquals(ExecutionState.CANCELED, task.getExecutionState());
	assertEquals(EXPECTED_CALL_ORDER_CANCEL_RUNNING, ACTUAL_ORDER_TRACKING);
}
 
Example #2
Source File: AllWindowTranslationTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
@SuppressWarnings("rawtypes")
public void testFoldWithCustomTrigger() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);

	DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2));

	DataStream<Tuple3<String, String, Integer>> window1 = source
			.windowAll(SlidingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS), Time.of(100, TimeUnit.MILLISECONDS)))
			.trigger(CountTrigger.of(1))
			.fold(new Tuple3<>("", "", 1), new DummyFolder());

	OneInputTransformation<Tuple2<String, Integer>, Tuple3<String, String, Integer>> transform =
			(OneInputTransformation<Tuple2<String, Integer>, Tuple3<String, String, Integer>>) window1.getTransformation();
	OneInputStreamOperator<Tuple2<String, Integer>, Tuple3<String, String, Integer>> operator = transform.getOperator();
	Assert.assertTrue(operator instanceof WindowOperator);
	WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator;
	Assert.assertTrue(winOperator.getTrigger() instanceof CountTrigger);
	Assert.assertTrue(winOperator.getWindowAssigner() instanceof SlidingEventTimeWindows);
	Assert.assertTrue(winOperator.getStateDescriptor() instanceof FoldingStateDescriptor);

	processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1));
}
 
Example #3
Source File: FlinkPravegaReaderTest.java    From flink-connectors with Apache License 2.0 6 votes vote down vote up
/**
 * Tests the cancellation support.
 */
@Test
public void testCancellation() throws Exception {
    TestableFlinkPravegaReader<Integer> reader = createReader();

    try (StreamSourceOperatorTestHarness<Integer, TestableFlinkPravegaReader<Integer>> testHarness =
                 createTestHarness(reader, 1, 1, 0, TimeCharacteristic.ProcessingTime)) {
        testHarness.open();

        // prepare a sequence of events
        TestEventGenerator<Integer> evts = new TestEventGenerator<>();
        when(reader.eventStreamReader.readNextEvent(anyLong()))
                .thenAnswer(i -> {
                    testHarness.cancel();
                    return evts.idle();
                });

        // run the source, which should return upon cancellation
        testHarness.run();
        assertFalse(reader.running);
    }
}
 
Example #4
Source File: AllWindowTranslationTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
@SuppressWarnings("rawtypes")
public void testReduceWithEvictor() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);

	DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2));

	DummyReducer reducer = new DummyReducer();

	DataStream<Tuple2<String, Integer>> window1 = source
			.windowAll(SlidingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS), Time.of(100, TimeUnit.MILLISECONDS)))
			.evictor(CountEvictor.of(100))
			.reduce(reducer);

	OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window1.getTransformation();
	OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator = transform.getOperator();
	Assert.assertTrue(operator instanceof EvictingWindowOperator);
	EvictingWindowOperator<String, Tuple2<String, Integer>, ?, ?> winOperator = (EvictingWindowOperator<String, Tuple2<String, Integer>, ?, ?>) operator;
	Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger);
	Assert.assertTrue(winOperator.getWindowAssigner() instanceof SlidingEventTimeWindows);
	Assert.assertTrue(winOperator.getEvictor() instanceof CountEvictor);
	Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor);

	processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1));
}
 
Example #5
Source File: StateDescriptorPassingTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testReduceWindowState() throws Exception {
	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);
	env.registerTypeWithKryoSerializer(File.class, JavaSerializer.class);

	DataStream<File> src = env.fromElements(new File("/"));

	SingleOutputStreamOperator<?> result = src
			.keyBy(new KeySelector<File, String>() {
				@Override
				public String getKey(File value) {
					return null;
				}
			})
			.timeWindow(Time.milliseconds(1000))
			.reduce(new ReduceFunction<File>() {

				@Override
				public File reduce(File value1, File value2) {
					return null;
				}
			});

	validateStateDescriptorConfigured(result);
}
 
Example #6
Source File: AllWindowTranslationTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
@SuppressWarnings({"rawtypes", "unchecked"})
public void testFoldWithEvictor() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);

	DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2));

	DataStream<Tuple3<String, String, Integer>> window1 = source
			.windowAll(SlidingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS), Time.of(100, TimeUnit.MILLISECONDS)))
			.evictor(CountEvictor.of(100))
			.fold(new Tuple3<>("", "", 1), new DummyFolder());

	OneInputTransformation<Tuple2<String, Integer>, Tuple3<String, String, Integer>> transform =
			(OneInputTransformation<Tuple2<String, Integer>, Tuple3<String, String, Integer>>) window1.getTransformation();
	OneInputStreamOperator<Tuple2<String, Integer>, Tuple3<String, String, Integer>> operator = transform.getOperator();
	Assert.assertTrue(operator instanceof EvictingWindowOperator);
	EvictingWindowOperator<String, Tuple2<String, Integer>, ?, ?> winOperator = (EvictingWindowOperator<String, Tuple2<String, Integer>, ?, ?>) operator;
	Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger);
	Assert.assertTrue(winOperator.getWindowAssigner() instanceof SlidingEventTimeWindows);
	Assert.assertTrue(winOperator.getEvictor() instanceof CountEvictor);
	Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor);

	winOperator.setOutputType((TypeInformation) window1.getType(), new ExecutionConfig());
	processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1));
}
 
Example #7
Source File: ExecutionEntry.java    From flink with Apache License 2.0 6 votes vote down vote up
public TimeCharacteristic getTimeCharacteristic() {
	return properties.getOptionalString(EXECUTION_TIME_CHARACTERISTIC)
		.flatMap((v) -> {
			switch (v) {
				case EXECUTION_TIME_CHARACTERISTIC_VALUE_EVENT_TIME:
					return Optional.of(TimeCharacteristic.EventTime);
				case EXECUTION_TIME_CHARACTERISTIC_VALUE_PROCESSING_TIME:
					return Optional.of(TimeCharacteristic.ProcessingTime);
				default:
					return Optional.empty();
			}
		})
		.orElseGet(() ->
			useDefaultValue(
				EXECUTION_TIME_CHARACTERISTIC,
				TimeCharacteristic.EventTime,
				EXECUTION_TIME_CHARACTERISTIC_VALUE_EVENT_TIME));
}
 
Example #8
Source File: KeyedStream.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Specifies the time boundaries over which the join operation works, so that
 * <pre>leftElement.timestamp + lowerBound <= rightElement.timestamp <= leftElement.timestamp + upperBound</pre>
 * By default both the lower and the upper bound are inclusive. This can be configured
 * with {@link IntervalJoined#lowerBoundExclusive()} and
 * {@link IntervalJoined#upperBoundExclusive()}
 *
 * @param lowerBound The lower bound. Needs to be smaller than or equal to the upperBound
 * @param upperBound The upper bound. Needs to be bigger than or equal to the lowerBound
 */
@PublicEvolving
public IntervalJoined<T1, T2, KEY> between(Time lowerBound, Time upperBound) {

	TimeCharacteristic timeCharacteristic =
		streamOne.getExecutionEnvironment().getStreamTimeCharacteristic();

	if (timeCharacteristic != TimeCharacteristic.EventTime) {
		throw new UnsupportedTimeCharacteristicException("Time-bounded stream joins are only supported in event time");
	}

	checkNotNull(lowerBound, "A lower bound needs to be provided for a time-bounded join");
	checkNotNull(upperBound, "An upper bound needs to be provided for a time-bounded join");

	return new IntervalJoined<>(
		streamOne,
		streamTwo,
		lowerBound.toMilliseconds(),
		upperBound.toMilliseconds(),
		true,
		true
	);
}
 
Example #9
Source File: IntervalJoinITCase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test(expected = UnsupportedTimeCharacteristicException.class)
public void testExecutionFailsInProcessingTime() throws Exception {
	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);
	env.setParallelism(1);

	DataStream<Tuple2<String, Integer>> streamOne = env.fromElements(Tuple2.of("1", 1));
	DataStream<Tuple2<String, Integer>> streamTwo = env.fromElements(Tuple2.of("1", 1));

	streamOne.keyBy(new Tuple2KeyExtractor())
		.intervalJoin(streamTwo.keyBy(new Tuple2KeyExtractor()))
		.between(Time.milliseconds(0), Time.milliseconds(0))
		.process(new ProcessJoinFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, String>() {
			@Override
			public void processElement(Tuple2<String, Integer> left,
				Tuple2<String, Integer> right, Context ctx,
				Collector<String> out) throws Exception {
				out.collect(left + ":" + right);
			}
		});
}
 
Example #10
Source File: AllWindowTranslationTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
@SuppressWarnings("rawtypes")
public void testReduceEventTime() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);

	DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2));

	DataStream<Tuple2<String, Integer>> window1 = source
			.windowAll(SlidingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS), Time.of(100, TimeUnit.MILLISECONDS)))
			.reduce(new DummyReducer());

	OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window1.getTransformation();
	OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator = transform.getOperator();
	Assert.assertTrue(operator instanceof WindowOperator);
	WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator;
	Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger);
	Assert.assertTrue(winOperator.getWindowAssigner() instanceof SlidingEventTimeWindows);
	Assert.assertTrue(winOperator.getStateDescriptor() instanceof ReducingStateDescriptor);

	processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1));
}
 
Example #11
Source File: StreamSourceOperatorWatermarksTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testNoMaxWatermarkOnImmediateStop() throws Exception {

	final List<StreamElement> output = new ArrayList<>();

	// regular stream source operator
	final StoppableStreamSource<String, InfiniteSource<String>> operator =
			new StoppableStreamSource<>(new InfiniteSource<String>());

	setupSourceOperator(operator, TimeCharacteristic.EventTime, 0);
	operator.stop();

	// run and stop
	operator.run(new Object(), mock(StreamStatusMaintainer.class), new CollectorOutput<String>(output));

	assertTrue(output.isEmpty());
}
 
Example #12
Source File: KeyedStream.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Specifies the time boundaries over which the join operation works, so that
 * <pre>leftElement.timestamp + lowerBound <= rightElement.timestamp <= leftElement.timestamp + upperBound</pre>
 * By default both the lower and the upper bound are inclusive. This can be configured
 * with {@link IntervalJoined#lowerBoundExclusive()} and
 * {@link IntervalJoined#upperBoundExclusive()}
 *
 * @param lowerBound The lower bound. Needs to be smaller than or equal to the upperBound
 * @param upperBound The upper bound. Needs to be bigger than or equal to the lowerBound
 */
@PublicEvolving
public IntervalJoined<T1, T2, KEY> between(Time lowerBound, Time upperBound) {

	TimeCharacteristic timeCharacteristic =
		streamOne.getExecutionEnvironment().getStreamTimeCharacteristic();

	if (timeCharacteristic != TimeCharacteristic.EventTime) {
		throw new UnsupportedTimeCharacteristicException("Time-bounded stream joins are only supported in event time");
	}

	checkNotNull(lowerBound, "A lower bound needs to be provided for a time-bounded join");
	checkNotNull(upperBound, "An upper bound needs to be provided for a time-bounded join");

	return new IntervalJoined<>(
		streamOne,
		streamTwo,
		lowerBound.toMilliseconds(),
		upperBound.toMilliseconds(),
		true,
		true
	);
}
 
Example #13
Source File: AllWindowTranslationTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
@SuppressWarnings({"rawtypes", "unchecked"})
public void testFoldWithEvictor() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);

	DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2));

	DataStream<Tuple3<String, String, Integer>> window1 = source
			.windowAll(SlidingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS), Time.of(100, TimeUnit.MILLISECONDS)))
			.evictor(CountEvictor.of(100))
			.fold(new Tuple3<>("", "", 1), new DummyFolder());

	OneInputTransformation<Tuple2<String, Integer>, Tuple3<String, String, Integer>> transform =
			(OneInputTransformation<Tuple2<String, Integer>, Tuple3<String, String, Integer>>) window1.getTransformation();
	OneInputStreamOperator<Tuple2<String, Integer>, Tuple3<String, String, Integer>> operator = transform.getOperator();
	Assert.assertTrue(operator instanceof EvictingWindowOperator);
	EvictingWindowOperator<String, Tuple2<String, Integer>, ?, ?> winOperator = (EvictingWindowOperator<String, Tuple2<String, Integer>, ?, ?>) operator;
	Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger);
	Assert.assertTrue(winOperator.getWindowAssigner() instanceof SlidingEventTimeWindows);
	Assert.assertTrue(winOperator.getEvictor() instanceof CountEvictor);
	Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor);

	winOperator.setOutputType((TypeInformation) window1.getType(), new ExecutionConfig());
	processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1));
}
 
Example #14
Source File: ExecutionContext.java    From flink with Apache License 2.0 6 votes vote down vote up
private TableConfig createTableConfig() {
	final TableConfig config = new TableConfig();
	config.addConfiguration(flinkConfig);
	Configuration conf = config.getConfiguration();
	environment.getConfiguration().asMap().forEach(conf::setString);
	ExecutionEntry execution = environment.getExecution();
	config.setIdleStateRetentionTime(
			Time.milliseconds(execution.getMinStateRetention()),
			Time.milliseconds(execution.getMaxStateRetention()));

	conf.set(CoreOptions.DEFAULT_PARALLELISM, execution.getParallelism());
	conf.set(PipelineOptions.MAX_PARALLELISM, execution.getMaxParallelism());
	conf.set(StreamPipelineOptions.TIME_CHARACTERISTIC, execution.getTimeCharacteristic());
	if (execution.getTimeCharacteristic() == TimeCharacteristic.EventTime) {
		conf.set(PipelineOptions.AUTO_WATERMARK_INTERVAL,
				Duration.ofMillis(execution.getPeriodicWatermarksInterval()));
	}

	setRestartStrategy(conf);
	return config;
}
 
Example #15
Source File: AllWindowTranslationTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
@SuppressWarnings("rawtypes")
public void testApplyProcessingTimeTime() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);

	DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2));

	DataStream<Tuple2<String, Integer>> window1 = source
			.windowAll(TumblingProcessingTimeWindows.of(Time.of(1, TimeUnit.SECONDS)))
			.apply(new AllWindowFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void apply(
						TimeWindow window,
						Iterable<Tuple2<String, Integer>> values,
						Collector<Tuple2<String, Integer>> out) throws Exception {
					for (Tuple2<String, Integer> in : values) {
						out.collect(in);
					}
				}
			});

	OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window1.getTransformation();
	OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator = transform.getOperator();
	Assert.assertTrue(operator instanceof WindowOperator);
	WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator;
	Assert.assertTrue(winOperator.getTrigger() instanceof ProcessingTimeTrigger);
	Assert.assertTrue(winOperator.getWindowAssigner() instanceof TumblingProcessingTimeWindows);
	Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor);

	processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1));

}
 
Example #16
Source File: ExecutorUtils.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Sets batch properties for {@link StreamExecutionEnvironment}.
 */
public static void setBatchProperties(StreamExecutionEnvironment execEnv, TableConfig tableConfig) {
	ExecutionConfig executionConfig = execEnv.getConfig();
	executionConfig.enableObjectReuse();
	executionConfig.setLatencyTrackingInterval(-1);
	execEnv.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);
	execEnv.setBufferTimeout(-1);
	if (isShuffleModeAllBlocking(tableConfig)) {
		executionConfig.setDefaultInputDependencyConstraint(InputDependencyConstraint.ALL);
	}
}
 
Example #17
Source File: DataStreamAllroundTestJobFactory.java    From flink with Apache License 2.0 5 votes vote down vote up
public static void setupEnvironment(StreamExecutionEnvironment env, ParameterTool pt) throws Exception {
	setupCheckpointing(env, pt);
	setupParallelism(env, pt);
	setupRestartStrategy(env, pt);
	setupStateBackend(env, pt);

	env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);

	// make parameters available in the web interface
	env.getConfig().setGlobalJobParameters(pt);
}
 
Example #18
Source File: SourceStreamTaskTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testClosingAllOperatorsOnChainProperly() throws Exception {
	final StreamTaskTestHarness<String> testHarness = new StreamTaskTestHarness<>(
		SourceStreamTask::new,
		BasicTypeInfo.STRING_TYPE_INFO);

	testHarness
		.setupOperatorChain(
			new OperatorID(),
			new OutputRecordInCloseTestSource<>(
				"Source0", new FromElementsFunction<>(StringSerializer.INSTANCE, "Hello")))
		.chain(
			new OperatorID(),
			new TestBoundedOneInputStreamOperator("Operator1"),
			BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()))
		.finish();

	StreamConfig streamConfig = testHarness.getStreamConfig();
	streamConfig.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);

	testHarness.invoke();
	testHarness.waitForTaskCompletion();

	ArrayList<StreamRecord<String>> expected = new ArrayList<>();
	Collections.addAll(expected,
		new StreamRecord<>("Hello"),
		new StreamRecord<>("[Source0]: End of input"),
		new StreamRecord<>("[Source0]: Bye"),
		new StreamRecord<>("[Operator1]: End of input"),
		new StreamRecord<>("[Operator1]: Bye"));

	final Object[] output = testHarness.getOutput().toArray();
	assertArrayEquals("Output was not correct.", expected.toArray(), output);
}
 
Example #19
Source File: TimestampITCase.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testErrorOnEventTimeOverProcessingTime() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	env.setParallelism(2);
	env.getConfig().disableSysoutLogging();
	env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);

	DataStream<Tuple2<String, Integer>> source1 =
			env.fromElements(new Tuple2<>("a", 1), new Tuple2<>("b", 2));

	source1
			.keyBy(0)
			.window(TumblingEventTimeWindows.of(Time.seconds(5)))
			.reduce(new ReduceFunction<Tuple2<String, Integer>>() {
				@Override
				public Tuple2<String, Integer> reduce(Tuple2<String, Integer> value1, Tuple2<String, Integer> value2)  {
					return value1;
				}
			})
			.print();

	try {
		env.execute();
		fail("this should fail with an exception");
	} catch (Exception e) {
		// expected
	}
}
 
Example #20
Source File: StreamSourceContexts.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Depending on the {@link TimeCharacteristic}, this method will return the adequate
 * {@link org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext}. That is:
 * <ul>
 *     <li>{@link TimeCharacteristic#IngestionTime} = {@code AutomaticWatermarkContext}</li>
 *     <li>{@link TimeCharacteristic#ProcessingTime} = {@code NonTimestampContext}</li>
 *     <li>{@link TimeCharacteristic#EventTime} = {@code ManualWatermarkContext}</li>
 * </ul>
 * */
public static <OUT> SourceFunction.SourceContext<OUT> getSourceContext(
		TimeCharacteristic timeCharacteristic,
		ProcessingTimeService processingTimeService,
		Object checkpointLock,
		StreamStatusMaintainer streamStatusMaintainer,
		Output<StreamRecord<OUT>> output,
		long watermarkInterval,
		long idleTimeout) {

	final SourceFunction.SourceContext<OUT> ctx;
	switch (timeCharacteristic) {
		case EventTime:
			ctx = new ManualWatermarkContext<>(
				output,
				processingTimeService,
				checkpointLock,
				streamStatusMaintainer,
				idleTimeout);

			break;
		case IngestionTime:
			ctx = new AutomaticWatermarkContext<>(
				output,
				watermarkInterval,
				processingTimeService,
				checkpointLock,
				streamStatusMaintainer,
				idleTimeout);

			break;
		case ProcessingTime:
			ctx = new NonTimestampContext<>(checkpointLock, output);
			break;
		default:
			throw new IllegalArgumentException(String.valueOf(timeCharacteristic));
	}
	return ctx;
}
 
Example #21
Source File: WindowTranslationTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
@SuppressWarnings("rawtypes")
public void testFoldWithProcessWindowFunctionEventTime() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);

	DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2));

	DataStream<Tuple2<String, Integer>> window = source
			.keyBy(new TupleKeySelector())
			.window(TumblingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS)))
			.fold(new Tuple3<>("", "", 0), new DummyFolder(), new ProcessWindowFunction<Tuple3<String, String, Integer>, Tuple2<String, Integer>, String, TimeWindow>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void process(String key,
						Context ctx,
						Iterable<Tuple3<String, String, Integer>> values,
						Collector<Tuple2<String, Integer>> out) throws Exception {
					for (Tuple3<String, String, Integer> in : values) {
						out.collect(new Tuple2<>(in.f0, in.f2));
					}
				}
			});

	OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform =
			(OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window.getTransformation();
	OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator = transform.getOperator();
	Assert.assertTrue(operator instanceof WindowOperator);
	WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator;
	Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger);
	Assert.assertTrue(winOperator.getWindowAssigner() instanceof TumblingEventTimeWindows);
	Assert.assertTrue(winOperator.getStateDescriptor() instanceof FoldingStateDescriptor);

	processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1));
}
 
Example #22
Source File: WhyLate.java    From flink-training-exercises with Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) throws Exception {

		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
		env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);

		OutputTag<Event> lateDataTag = new OutputTag<Event>("late"){};

		// When PARTITIONS_PER_INSTANCE is greater than 1, there can be late events.
		DataStream<Event> events = env.addSource(new ParallelEventSource(PARTITIONS_PER_INSTANCE));

		// Count the number of events per user in one second event-time windows,
		// and capture late events on a side output.
		SingleOutputStreamOperator<Tuple2<String, Integer>> windowOperator = events
				.assignTimestampsAndWatermarks(new TimestampsAndWatermarks())
				.keyBy(e -> e.userId)
				.window(TumblingEventTimeWindows.of(Time.seconds(1)))
				.sideOutputLateData(lateDataTag)
				.process(new CountEventsPerUser());

		windowOperator.print();

		// Count the number of late events for every second of processing time.pri
		windowOperator.getSideOutput(lateDataTag)
				.windowAll(TumblingProcessingTimeWindows.of(Time.seconds(1)))
				.process(new CountLateEvents())
				.map(i -> new Tuple3<String, Integer, String>("LATE", i, Instant.now().toString()))
				.returns(Types.TUPLE(Types.STRING, Types.INT, Types.STRING))
				.print();

		env.execute();
	}
 
Example #23
Source File: OngoingRidesSolution.java    From flink-training-exercises with Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) throws Exception {

		ParameterTool params = ParameterTool.fromArgs(args);
		final String input = params.get("input", ExerciseBase.pathToRideData);

		final int maxEventDelay = 60;       	// events are out of order by at most 60 seconds
		final int servingSpeedFactor = 600; 	// 10 minutes worth of events are served every second

		// In this simple case we need a broadcast state descriptor, but aren't going to
		// use it to store anything.
		final MapStateDescriptor<Long, Long> dummyBroadcastState = new MapStateDescriptor<>(
				"dummy",
				BasicTypeInfo.LONG_TYPE_INFO,
				BasicTypeInfo.LONG_TYPE_INFO
		);

		// set up streaming execution environment
		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
		env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
		env.setParallelism(ExerciseBase.parallelism);

		DataStream<TaxiRide> rides = env.addSource(new TaxiRideSource(input, maxEventDelay, servingSpeedFactor));

		// add a socket source
		BroadcastStream<String> queryStream = env.socketTextStream("localhost", 9999)
				.assignTimestampsAndWatermarks(new QueryStreamAssigner())
				.broadcast(dummyBroadcastState);

		DataStream<TaxiRide> reports = rides
				.keyBy((TaxiRide ride) -> ride.taxiId)
				.connect(queryStream)
				.process(new QueryFunction());

		printOrTest(reports);

		env.execute("Ongoing Rides");
	}
 
Example #24
Source File: TaxiQueryExercise.java    From flink-training-exercises with Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) throws Exception {

		ParameterTool params = ParameterTool.fromArgs(args);
		final String input = params.get("input", ExerciseBase.pathToRideData);

		final int maxEventDelay = 60;       	// events are out of order by at most 60 seconds
		final int servingSpeedFactor = 1800; 	// 30 minutes worth of events are served every second

		// set up streaming execution environment
		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
		env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
		env.setParallelism(ExerciseBase.parallelism);

		// setup a stream of taxi rides
		DataStream<TaxiRide> rides = env.addSource(rideSourceOrTest(new TaxiRideSource(input, maxEventDelay, servingSpeedFactor)));

		// add a socket source for the query stream
		BroadcastStream<String> queryStream = env
				.addSource(stringSourceOrTest(new SocketTextStreamFunction("localhost", 9999, "\n", -1)))
				.broadcast(queryDescriptor);

		// connect the two streams and process queries
		DataStream<Tuple2<String, String>> results = rides
				.keyBy((TaxiRide ride) -> ride.taxiId)
				.connect(queryStream)
				.process(new QueryProcessor());

		printOrTest(results);

		env.execute("Taxi Query");
	}
 
Example #25
Source File: WindowTranslationTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
@SuppressWarnings("rawtypes")
public void testReduceWithProcesWindowFunctionEventTime() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);

	DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2));

	DummyReducer reducer = new DummyReducer();

	DataStream<Tuple3<String, String, Integer>> window = source
			.keyBy(new TupleKeySelector())
			.window(TumblingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS)))
			.reduce(reducer, new ProcessWindowFunction<Tuple2<String, Integer>, Tuple3<String, String, Integer>, String, TimeWindow>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void process(String key,
						Context ctx,
						Iterable<Tuple2<String, Integer>> values,
						Collector<Tuple3<String, String, Integer>> out) throws Exception {
					for (Tuple2<String, Integer> in : values) {
						out.collect(new Tuple3<>(in.f0, in.f0, in.f1));
					}
				}
			});

	OneInputTransformation<Tuple2<String, Integer>, Tuple3<String, String, Integer>> transform =
			(OneInputTransformation<Tuple2<String, Integer>, Tuple3<String, String, Integer>>) window.getTransformation();
	OneInputStreamOperator<Tuple2<String, Integer>, Tuple3<String, String, Integer>> operator = transform.getOperator();
	Assert.assertTrue(operator instanceof WindowOperator);
	WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator;
	Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger);
	Assert.assertTrue(winOperator.getWindowAssigner() instanceof TumblingEventTimeWindows);
	Assert.assertTrue(winOperator.getStateDescriptor() instanceof ReducingStateDescriptor);

	processElementAndEnsureOutput(operator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1));
}
 
Example #26
Source File: StreamTaskTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * This test checks that cancel calls that are issued before the operator is
 * instantiated still lead to proper canceling.
 */
@Test
public void testEarlyCanceling() throws Exception {
	final StreamConfig cfg = new StreamConfig(new Configuration());
	cfg.setOperatorID(new OperatorID(4711L, 42L));
	cfg.setStreamOperator(new SlowlyDeserializingOperator());
	cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);

	final TaskManagerActions taskManagerActions = spy(new NoOpTaskManagerActions());
	final Task task = createTask(SourceStreamTask.class, cfg, new Configuration(), taskManagerActions);

	final TaskExecutionState state = new TaskExecutionState(
		task.getJobID(), task.getExecutionId(), ExecutionState.RUNNING);

	task.startTaskThread();

	verify(taskManagerActions, timeout(2000L)).updateTaskExecutionState(eq(state));

	// send a cancel. because the operator takes a long time to deserialize, this should
	// hit the task before the operator is deserialized
	task.cancelExecution();

	task.getExecutingThread().join();

	assertFalse("Task did not cancel", task.getExecutingThread().isAlive());
	assertEquals(ExecutionState.CANCELED, task.getExecutionState());
}
 
Example #27
Source File: WorkingTimeMonitor.java    From infoworld-post with Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) throws Exception {

        // check parameter
        if (args.length != 1) {
            System.err.println("Please provide the path to the taxi rides file as a parameter");
        }
        String inputPath = args[0];

        // create execution environment
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        // configure event-time and watermarks
        env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
        env.getConfig().setAutoWatermarkInterval(1000L);

        // ingest stream of taxi rides.
        DataStream<TaxiRide> rides = TaxiRides.getRides(env, inputPath);

        DataStream<Tuple2<String, String>> notifications = rides
            // partition stream by the driver's license id
            .keyBy(r -> r.licenseId)
            // monitor ride events and generate notifications
            .process(new MonitorWorkTime());

        // print notifications
        notifications.print();

        // run the application
        env.execute();
    }
 
Example #28
Source File: IntervalJoinITCase.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testBoundsCanBeExclusive() throws Exception {
	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
	env.setParallelism(1);

	DataStream<Tuple2<String, Integer>> streamOne = env.fromElements(
		Tuple2.of("key", 0),
		Tuple2.of("key", 1),
		Tuple2.of("key", 2)
	).assignTimestampsAndWatermarks(new AscendingTuple2TimestampExtractor());

	DataStream<Tuple2<String, Integer>> streamTwo = env.fromElements(
		Tuple2.of("key", 0),
		Tuple2.of("key", 1),
		Tuple2.of("key", 2)
	).assignTimestampsAndWatermarks(new AscendingTuple2TimestampExtractor());

	streamOne.keyBy(new Tuple2KeyExtractor())
		.intervalJoin(streamTwo.keyBy(new Tuple2KeyExtractor()))
		.between(Time.milliseconds(0), Time.milliseconds(2))
		.upperBoundExclusive()
		.lowerBoundExclusive()
		.process(new CombineToStringJoinFunction())
		.addSink(new ResultSink());

	env.execute();

	expectInAnyOrder(
		"(key,0):(key,1)",
		"(key,1):(key,2)"
	);
}
 
Example #29
Source File: BravoTestPipeline.java    From bravo with Apache License 2.0 5 votes vote down vote up
private StreamExecutionEnvironment createJobGraph(int parallelism,
		Function<DataStream<String>, DataStream<String>> pipelinerBuilder) throws Exception {
	final Path checkpointDir = getCheckpointDir();
	final Path savepointRootDir = getSavepointDir();

	checkpointDir.getFileSystem().mkdirs(checkpointDir);
	savepointRootDir.getFileSystem().mkdirs(savepointRootDir);

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.getConfig().disableSysoutLogging();
	env.getCheckpointConfig().enableExternalizedCheckpoints(ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION);
	env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
	env.setBufferTimeout(0);
	env.setParallelism(parallelism);
	env.enableCheckpointing(500, CheckpointingMode.EXACTLY_ONCE);

	env.setStateBackend((StateBackend) new RocksDBStateBackend(checkpointDir.toString(), true));

	DataStream<String> sourceData = env
			.addSource(new TestPipelineSource())
			.uid("TestSource")
			.name("TestSource")
			.setParallelism(1);

	pipelinerBuilder.apply(sourceData)
			.addSink(new CollectingSink()).name("Output").uid("Output")
			.setParallelism(1);

	return env;
}
 
Example #30
Source File: AllWindowTranslationTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
@SuppressWarnings("rawtypes")
public void testProcessWithEvictor() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);

	DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2));

	DataStream<Tuple2<String, Integer>> window1 = source
			.windowAll(TumblingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS)))
			.trigger(CountTrigger.of(1))
			.evictor(TimeEvictor.of(Time.of(100, TimeUnit.MILLISECONDS)))
			.process(new ProcessAllWindowFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void process(
						Context ctx,
						Iterable<Tuple2<String, Integer>> values,
						Collector<Tuple2<String, Integer>> out) throws Exception {
					for (Tuple2<String, Integer> in : values) {
						out.collect(in);
					}
				}
			});

	OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window1.getTransformation();
	OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator = transform.getOperator();
	Assert.assertTrue(operator instanceof EvictingWindowOperator);
	EvictingWindowOperator<String, Tuple2<String, Integer>, ?, ?> winOperator = (EvictingWindowOperator<String, Tuple2<String, Integer>, ?, ?>) operator;
	Assert.assertTrue(winOperator.getTrigger() instanceof CountTrigger);
	Assert.assertTrue(winOperator.getEvictor() instanceof TimeEvictor);
	Assert.assertTrue(winOperator.getWindowAssigner() instanceof TumblingEventTimeWindows);
	Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor);

	processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1));
}