org.apache.flink.streaming.api.datastream.AsyncDataStream Java Examples

private void testTimeoutExceptionHandling(AsyncDataStream.OutputMode outputMode) throws Exception {
	OneInputStreamOperatorTestHarness<Integer, Integer> harness =
		createTestHarness(new NoOpAsyncFunction<>(), 10L, 2, outputMode);

	harness.getEnvironment().setExpectedExternalFailureCause(Throwable.class);
	harness.open();

	synchronized (harness.getCheckpointLock()) {
		harness.processElement(1, 1L);
	}

	harness.setProcessingTime(10L);

	synchronized (harness.getCheckpointLock()) {
		harness.close();
	}
}
 

private void testUserExceptionHandling(AsyncDataStream.OutputMode outputMode) throws Exception {
	OneInputStreamOperatorTestHarness<Integer, Integer> harness =
		createTestHarness(new UserExceptionAsyncFunction(), TIMEOUT, 2, outputMode);

	harness.getEnvironment().setExpectedExternalFailureCause(Throwable.class);
	harness.open();

	synchronized (harness.getCheckpointLock()) {
		harness.processElement(1, 1L);
	}

	synchronized (harness.getCheckpointLock()) {
		harness.close();
	}

	assertTrue(harness.getEnvironment().getActualExternalFailureCause().isPresent());
}
 

/**
 * FLINK-5652
 * Tests that registered timers are properly canceled upon completion of a
 * {@link StreamElement} in order to avoid resource leaks because TriggerTasks hold
 * a reference on the StreamRecordQueueEntry.
 */
@Test
public void testTimeoutCleanup() throws Exception {
	OneInputStreamOperatorTestHarness<Integer, Integer> harness =
		createTestHarness(new MyAsyncFunction(), TIMEOUT, 1, AsyncDataStream.OutputMode.UNORDERED);

	harness.open();

	synchronized (harness.getCheckpointLock()) {
		harness.processElement(42, 1L);
	}

	synchronized (harness.getCheckpointLock()) {
		harness.endInput();
		harness.close();
	}

	// check that we actually outputted the result of the single input
	assertEquals(Arrays.asList(new StreamRecord(42 * 2, 1L)), new ArrayList<>(harness.getOutput()));

	// check that we have cancelled our registered timeout
	assertEquals(0, harness.getProcessingTimeService().getNumActiveTimers());
}
 

public AsyncWaitOperator(
		@Nonnull AsyncFunction<IN, OUT> asyncFunction,
		long timeout,
		int capacity,
		@Nonnull AsyncDataStream.OutputMode outputMode,
		@Nonnull ProcessingTimeService processingTimeService,
		@Nonnull MailboxExecutor mailboxExecutor) {
	super(asyncFunction);

	setChainingStrategy(ChainingStrategy.ALWAYS);

	Preconditions.checkArgument(capacity > 0, "The number of concurrent async operation should be greater than 0.");
	this.capacity = capacity;

	this.outputMode = Preconditions.checkNotNull(outputMode, "outputMode");

	this.timeout = timeout;

	this.processingTimeService = Preconditions.checkNotNull(processingTimeService);

	this.mailboxExecutor = mailboxExecutor;
}
 

private DataStream<Long> createAsyncOperator(DataStreamSource<Long> source) {
	switch (outputMode) {
		case ORDERED:
			return AsyncDataStream.orderedWait(
					source,
					new BenchmarkAsyncFunctionExecutor(),
					0,
					TimeUnit.MILLISECONDS);
		case UNORDERED:
			return AsyncDataStream.unorderedWait(
					source,
					new BenchmarkAsyncFunctionExecutor(),
					0,
					TimeUnit.MILLISECONDS);
		default:
			throw new UnsupportedOperationException("Unknown mode");
	}
}
 

public AsyncWaitOperator(
		AsyncFunction<IN, OUT> asyncFunction,
		long timeout,
		int capacity,
		AsyncDataStream.OutputMode outputMode) {
	super(asyncFunction);

	// TODO this is a temporary fix for the problems described under FLINK-13063 at the cost of breaking chains for
	//  AsyncOperators.
	setChainingStrategy(ChainingStrategy.HEAD);

	Preconditions.checkArgument(capacity > 0, "The number of concurrent async operation should be greater than 0.");
	this.capacity = capacity;

	this.outputMode = Preconditions.checkNotNull(outputMode, "outputMode");

	this.timeout = timeout;
}
 

/**
 * Test for the temporary fix to FLINK-13063.
 */
@Test
public void testAsyncOperatorIsNeverChained() {
	StreamExecutionEnvironment chainEnv = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStream<Integer> input = chainEnv.fromElements(1);
	input = AsyncDataStream.orderedWait(
		input,
		new LazyAsyncFunction(),
		TIMEOUT,
		TimeUnit.MILLISECONDS,
		6).map((x) -> x);
	AsyncDataStream.unorderedWait(
		input,
		new MyAsyncFunction(),
		TIMEOUT,
		TimeUnit.MILLISECONDS,
		3).map((x) -> x).addSink(new DiscardingSink<>());

	final JobGraph jobGraph = chainEnv.getStreamGraph().getJobGraph();

	Assert.assertEquals(3, jobGraph.getVerticesSortedTopologicallyFromSources().size());
}
 

private static <OUT> OneInputStreamOperatorTestHarness<Integer, OUT> createTestHarness(
		AsyncFunction<Integer, OUT> function,
		long timeout,
		int capacity,
		AsyncDataStream.OutputMode outputMode) throws Exception {

	return new OneInputStreamOperatorTestHarness<>(
		new AsyncWaitOperatorFactory<>(function, timeout, capacity, outputMode),
		IntSerializer.INSTANCE);
}
 

private void testTimeoutExceptionHandling(AsyncDataStream.OutputMode outputMode) throws Exception {
	AsyncFunction<Integer, Integer> asyncFunction = new NoOpAsyncFunction<>();
	long timeout = 10L; // 1 milli second

	AsyncWaitOperator<Integer, Integer> asyncWaitOperator = new AsyncWaitOperator<>(
		asyncFunction,
		timeout,
		2,
		outputMode);

	final MockEnvironment mockEnvironment = createMockEnvironment();
	mockEnvironment.setExpectedExternalFailureCause(Throwable.class);

	OneInputStreamOperatorTestHarness<Integer, Integer> harness = new OneInputStreamOperatorTestHarness<>(
		asyncWaitOperator,
		IntSerializer.INSTANCE,
		mockEnvironment);

	harness.open();

	synchronized (harness.getCheckpointLock()) {
		harness.processElement(1, 1L);
	}

	harness.setProcessingTime(10L);

	synchronized (harness.getCheckpointLock()) {
		harness.close();
	}
}
 

public TestAsyncWaitOperator(
		AsyncFunction<IN, OUT> asyncFunction,
		long timeout,
		int capacity,
		AsyncDataStream.OutputMode outputMode,
		OneShotLatch closingLatch) {
	super(asyncFunction, timeout, capacity, outputMode);

	this.closingLatch = Preconditions.checkNotNull(closingLatch);
}
 

private void testUserExceptionHandling(AsyncDataStream.OutputMode outputMode) throws Exception {
	UserExceptionAsyncFunction asyncWaitFunction = new UserExceptionAsyncFunction();
	long timeout = 2000L;

	AsyncWaitOperator<Integer, Integer> asyncWaitOperator = new AsyncWaitOperator<>(
		asyncWaitFunction,
		TIMEOUT,
		2,
		outputMode);

	final MockEnvironment mockEnvironment = createMockEnvironment();
	mockEnvironment.setExpectedExternalFailureCause(Throwable.class);

	OneInputStreamOperatorTestHarness<Integer, Integer> harness = new OneInputStreamOperatorTestHarness<>(
		asyncWaitOperator,
		IntSerializer.INSTANCE,
		mockEnvironment);

	harness.open();

	synchronized (harness.getCheckpointLock()) {
		harness.processElement(1, 1L);
	}

	synchronized (harness.getCheckpointLock()) {
		harness.close();
	}

	assertTrue(harness.getEnvironment().getActualExternalFailureCause().isPresent());
}
 

@SuppressWarnings("unchecked")
private OneInputStreamOperatorTestHarness<BaseRow, BaseRow> createHarness(
		JoinType joinType,
		FilterOnTable filterOnTable) throws Exception {
	RichAsyncFunction<BaseRow, BaseRow> joinRunner;
	boolean isLeftJoin = joinType == JoinType.LEFT_JOIN;
	if (filterOnTable == FilterOnTable.WITHOUT_FILTER) {
		joinRunner = new AsyncLookupJoinRunner(
			new GeneratedFunctionWrapper(new TestingFetcherFunction()),
			new GeneratedResultFutureWrapper<>(new TestingFetcherResultFuture()),
			fetcherReturnType,
			rightRowTypeInfo,
			isLeftJoin,
			ASYNC_BUFFER_CAPACITY);
	} else {
		joinRunner = new AsyncLookupJoinWithCalcRunner(
			new GeneratedFunctionWrapper(new TestingFetcherFunction()),
			new GeneratedFunctionWrapper<>(new CalculateOnTemporalTable()),
			new GeneratedResultFutureWrapper<>(new TestingFetcherResultFuture()),
			fetcherReturnType,
			rightRowTypeInfo,
			isLeftJoin,
			ASYNC_BUFFER_CAPACITY);
	}

	AsyncWaitOperator<BaseRow, BaseRow> operator = new AsyncWaitOperator<>(
		joinRunner,
		ASYNC_TIMEOUT_MS,
		ASYNC_BUFFER_CAPACITY,
		AsyncDataStream.OutputMode.ORDERED);

	return new OneInputStreamOperatorTestHarness<>(
		operator,
		inSerializer);
}
 

@SuppressWarnings("unchecked")
public static void main(String[] args) throws Exception {
    final ParameterTool parameterTool = ExecutionEnvUtil.createParameterTool(args);
    StreamExecutionEnvironment env = ExecutionEnvUtil.prepare(parameterTool);

    Properties properties = KafkaConfigUtil.buildKafkaProps(parameterTool);
    FlinkKafkaConsumer011<MetricEvent> consumer = new FlinkKafkaConsumer011<>(
            parameterTool.get("metrics.topic"),
            new MetricSchema(),
            properties);
    SingleOutputStreamOperator<MetricEvent> machineData = env.addSource(consumer)
            .assignTimestampsAndWatermarks(new MetricWatermark());

    AsyncDataStream.unorderedWait(machineData, new AlertRuleAsyncIOFunction(), 10000, TimeUnit.MICROSECONDS, 100)
            .map(metricEvent -> {
                List<String> ma = (List<String>) metricEvent.getFields().get("xx");
                AlertEvent alertEvent = new AlertEvent();
                alertEvent.setType(metricEvent.getName());
                alertEvent.setTrigerTime(metricEvent.getTimestamp());
                alertEvent.setMetricEvent(metricEvent);
                if (metricEvent.getTags().get("recover") != null && Boolean.valueOf(metricEvent.getTags().get("recover"))) {
                    alertEvent.setRecover(true);
                    alertEvent.setRecoverTime(metricEvent.getTimestamp());
                } else {
                    alertEvent.setRecover(false);
                }
                return alertEvent;
            })
            .print();

    env.execute("Async IO get MySQL data");
}
 

public TestAsyncWaitOperator(
		AsyncFunction<IN, OUT> asyncFunction,
		long timeout,
		int capacity,
		AsyncDataStream.OutputMode outputMode,
		OneShotLatch closingLatch) {
	super(asyncFunction, timeout, capacity, outputMode);

	this.closingLatch = Preconditions.checkNotNull(closingLatch);
}
 

public AsyncWaitOperator(
		AsyncFunction<IN, OUT> asyncFunction,
		long timeout,
		int capacity,
		AsyncDataStream.OutputMode outputMode) {
	super(asyncFunction);
	chainingStrategy = ChainingStrategy.ALWAYS;

	Preconditions.checkArgument(capacity > 0, "The number of concurrent async operation should be greater than 0.");
	this.capacity = capacity;

	this.outputMode = Preconditions.checkNotNull(outputMode, "outputMode");

	this.timeout = timeout;
}
 

private void testTimeoutExceptionHandling(AsyncDataStream.OutputMode outputMode) throws Exception {
	AsyncFunction<Integer, Integer> asyncFunction = new NoOpAsyncFunction<>();
	long timeout = 10L; // 1 milli second

	AsyncWaitOperator<Integer, Integer> asyncWaitOperator = new AsyncWaitOperator<>(
		asyncFunction,
		timeout,
		2,
		outputMode);

	final MockEnvironment mockEnvironment = createMockEnvironment();
	mockEnvironment.setExpectedExternalFailureCause(Throwable.class);

	OneInputStreamOperatorTestHarness<Integer, Integer> harness = new OneInputStreamOperatorTestHarness<>(
		asyncWaitOperator,
		IntSerializer.INSTANCE,
		mockEnvironment);

	harness.open();

	synchronized (harness.getCheckpointLock()) {
		harness.processElement(1, 1L);
	}

	harness.setProcessingTime(10L);

	synchronized (harness.getCheckpointLock()) {
		harness.close();
	}
}
 

/**
 * This helper function is needed to check that the temporary fix for FLINK-13063 can be backwards compatible with
 * the old chaining behavior by setting the ChainingStrategy manually. TODO: remove after a proper fix for
 * FLINK-13063 is in place that allows chaining.
 */
private <IN, OUT> SingleOutputStreamOperator<OUT> addAsyncOperatorLegacyChained(
	DataStream<IN> in,
	AsyncFunction<IN, OUT> func,
	long timeout,
	int bufSize,
	AsyncDataStream.OutputMode mode) {

	TypeInformation<OUT> outTypeInfo = TypeExtractor.getUnaryOperatorReturnType(
		func,
		AsyncFunction.class,
		0,
		1,
		new int[]{1, 0},
		in.getType(),
		Utils.getCallLocationName(),
		true);

	// create transform
	AsyncWaitOperator<IN, OUT> operator = new AsyncWaitOperator<>(
		in.getExecutionEnvironment().clean(func),
		timeout,
		bufSize,
		mode);

	operator.setChainingStrategy(ChainingStrategy.ALWAYS);

	return in.transform("async wait operator", outTypeInfo, operator);
}
 

public static DataStream<Row> buildStream(StreamTableEnvironment env, SqlSelect sqlSelect, Alias leftAlias,
    Alias sideAlias, SourceDescriptor sideSource) throws Exception {
    SqlSelect leftSelect = SideParser.newSelect(sqlSelect, leftAlias.getTable(), leftAlias.getAlias(), true, false);
    // register leftTable
    Table leftTable = env.sqlQuery(leftSelect.toString());
    DataStream<Row> leftStream = env.toAppendStream(leftTable, Row.class);
    SqlSelect rightSelect
        = SideParser.newSelect(sqlSelect, sideAlias.getTable(), sideAlias.getAlias(), false, false);
    SqlJoin sqlJoin = (SqlJoin)sqlSelect.getFrom();
    List<String> equalFields = SideParser.findConditionFields(sqlJoin.getCondition(), leftAlias.getAlias());
    if (sideSource.getSide().isPartition()) {
        leftStream = leftStream.keyBy(equalFields.toArray(new String[equalFields.size()]));
    }
    RowTypeInfo sideType = createSideType(rightSelect.getSelectList(), sideSource.getSchema());
    RowTypeInfo returnType = createReturnType(leftTable.getSchema(), sideType);
    SideTable sideTable = createSideTable(leftTable.getSchema(), sideType, sqlJoin.getJoinType(), rightSelect,
        equalFields, sideAlias, sideSource.getSide());
    DataStream<Row> returnStream;
    if (sideSource.getSide().isAsync()) {
        AbstractAsyncSideFunction reqRow = sideSource.transform(sideTable);
        returnStream = AsyncDataStream.orderedWait(leftStream, reqRow, sideSource.getSide().getTimeout(),
            TimeUnit.MILLISECONDS, sideSource.getSide().getCapacity());
    } else {
        AbstractSyncSideFunction syncReqRow = sideSource.transform(sideTable);
        returnStream = leftStream.flatMap(syncReqRow);
    }
    returnStream.getTransformation().setOutputType(returnType);
    return returnStream;
}
 

@SuppressWarnings("unchecked")
public static void main(String[] args) throws Exception {
    final ParameterTool parameterTool = ExecutionEnvUtil.createParameterTool(args);
    StreamExecutionEnvironment env = ExecutionEnvUtil.prepare(parameterTool);

    Properties properties = KafkaConfigUtil.buildKafkaProps(parameterTool);
    FlinkKafkaConsumer011<MetricEvent> consumer = new FlinkKafkaConsumer011<>(
            parameterTool.get("metrics.topic"),
            new MetricSchema(),
            properties);
    SingleOutputStreamOperator<MetricEvent> machineData = env.addSource(consumer)
            .assignTimestampsAndWatermarks(new MetricWatermark());

    AsyncDataStream.unorderedWait(machineData, new AlertRuleAsyncIOFunction(), 10000, TimeUnit.MICROSECONDS, 100)
            .map(metricEvent -> {
                List<String> ma = (List<String>) metricEvent.getFields().get("xx");
                AlertEvent alertEvent = new AlertEvent();
                alertEvent.setType(metricEvent.getName());
                alertEvent.setTrigerTime(metricEvent.getTimestamp());
                alertEvent.setMetricEvent(metricEvent);
                if (metricEvent.getTags().get("recover") != null && Boolean.valueOf(metricEvent.getTags().get("recover"))) {
                    alertEvent.setRecover(true);
                    alertEvent.setRecoverTime(metricEvent.getTimestamp());
                } else {
                    alertEvent.setRecover(false);
                }
                return alertEvent;
            })
            .print();

    env.execute("Async IO get MySQL data");
}
 

public static DataStream<Row> translate(
        DataStream<Row> inputStream,
        RealTimeTransForm transForm)
{
    RowTypeInfo streamRowType = (RowTypeInfo) inputStream.getType();
    AsyncFunction<Row, Row> asyncFunction = new RichAsyncFunctionImpl(transForm, streamRowType);

    DataStream<Row> joinResultStream = AsyncDataStream.orderedWait(
            inputStream, asyncFunction,
            1000, TimeUnit.MILLISECONDS, // 超时时间
            100);  // 进行中的异步请求的最大数量

    return joinResultStream;
}
 

/**
 * Create a snippet of the topology that takes a RawUrl DataStream and applies an async lengthener, then a
 * normalizer and a validator. What we get out is a CrawlStateUrl DataStream.
 * 
 * @param rawUrls
 * @return
 */
private DataStream<CrawlStateUrl> cleanUrls(DataStream<RawUrl> rawUrls) {
    return AsyncDataStream
            .unorderedWait(rawUrls, new LengthenUrlsFunction(_urlLengthener),
                    _urlLengthener.getTimeoutInSeconds(), TimeUnit.SECONDS)
            .name("LengthenUrlsFunction")
            .flatMap(new NormalizeUrlsFunction(_urlNormalizer))
            .name("NormalizeUrlsFunction")
            .flatMap(new ValidUrlsFilter(_urlFilter))
            .name("ValidUrlsFilter");
}
 

@SuppressWarnings("unchecked")
private OneInputStreamOperatorTestHarness<RowData, RowData> createHarness(
		JoinType joinType,
		FilterOnTable filterOnTable) throws Exception {
	RichAsyncFunction<RowData, RowData> joinRunner;
	boolean isLeftJoin = joinType == JoinType.LEFT_JOIN;
	if (filterOnTable == FilterOnTable.WITHOUT_FILTER) {
		joinRunner = new AsyncLookupJoinRunner(
			new GeneratedFunctionWrapper(new TestingFetcherFunction()),
			new GeneratedResultFutureWrapper<>(new TestingFetcherResultFuture()),
			fetcherReturnType,
			rightRowTypeInfo,
			isLeftJoin,
			ASYNC_BUFFER_CAPACITY);
	} else {
		joinRunner = new AsyncLookupJoinWithCalcRunner(
			new GeneratedFunctionWrapper(new TestingFetcherFunction()),
			new GeneratedFunctionWrapper<>(new CalculateOnTemporalTable()),
			new GeneratedResultFutureWrapper<>(new TestingFetcherResultFuture()),
			fetcherReturnType,
			rightRowTypeInfo,
			isLeftJoin,
			ASYNC_BUFFER_CAPACITY);
	}

	return new OneInputStreamOperatorTestHarness<>(
		new AsyncWaitOperatorFactory<>(
			joinRunner,
			ASYNC_TIMEOUT_MS,
			ASYNC_BUFFER_CAPACITY,
			AsyncDataStream.OutputMode.ORDERED),
		inSerializer);
}
 

public AsyncWaitOperatorFactory(
		AsyncFunction<IN, OUT> asyncFunction,
		long timeout,
		int capacity,
		AsyncDataStream.OutputMode outputMode) {
	this.asyncFunction = asyncFunction;
	this.timeout = timeout;
	this.capacity = capacity;
	this.outputMode = outputMode;
	this.chainingStrategy = ChainingStrategy.ALWAYS;
}
 

/**
 * This helper function is needed to check that the temporary fix for FLINK-13063 can be backwards compatible with
 * the old chaining behavior by setting the ChainingStrategy manually. TODO: remove after a proper fix for
 * FLINK-13063 is in place that allows chaining.
 */
private <IN, OUT> SingleOutputStreamOperator<OUT> addAsyncOperatorLegacyChained(
	DataStream<IN> in,
	AsyncFunction<IN, OUT> func,
	long timeout,
	int bufSize,
	AsyncDataStream.OutputMode mode) {

	TypeInformation<OUT> outTypeInfo = TypeExtractor.getUnaryOperatorReturnType(
		func,
		AsyncFunction.class,
		0,
		1,
		new int[]{1, 0},
		in.getType(),
		Utils.getCallLocationName(),
		true);

	// create transform
	AsyncWaitOperatorFactory<IN, OUT> factory = new AsyncWaitOperatorFactory<>(
		in.getExecutionEnvironment().clean(func),
		timeout,
		bufSize,
		mode);

	factory.setChainingStrategy(ChainingStrategy.ALWAYS);

	return in.transform("async wait operator", outTypeInfo, factory);
}
 

private void testUserExceptionHandling(AsyncDataStream.OutputMode outputMode) throws Exception {
	UserExceptionAsyncFunction asyncWaitFunction = new UserExceptionAsyncFunction();
	long timeout = 2000L;

	AsyncWaitOperator<Integer, Integer> asyncWaitOperator = new AsyncWaitOperator<>(
		asyncWaitFunction,
		TIMEOUT,
		2,
		outputMode);

	final MockEnvironment mockEnvironment = createMockEnvironment();
	mockEnvironment.setExpectedExternalFailureCause(Throwable.class);

	OneInputStreamOperatorTestHarness<Integer, Integer> harness = new OneInputStreamOperatorTestHarness<>(
		asyncWaitOperator,
		IntSerializer.INSTANCE,
		mockEnvironment);

	harness.open();

	synchronized (harness.getCheckpointLock()) {
		harness.processElement(1, 1L);
	}

	synchronized (harness.getCheckpointLock()) {
		harness.close();
	}

	assertTrue(harness.getEnvironment().getActualExternalFailureCause().isPresent());
}
 

/**
 * Test the AsyncWaitOperator with ordered mode and event time.
 */
@Test
public void testEventTimeOrdered() throws Exception {
	testEventTime(AsyncDataStream.OutputMode.ORDERED);
}
 

/**
 * FLINK-5652
 * Tests that registered timers are properly canceled upon completion of a
 * {@link StreamRecordQueueEntry} in order to avoid resource leaks because TriggerTasks hold
 * a reference on the StreamRecordQueueEntry.
 */
@Test
public void testTimeoutCleanup() throws Exception {
	final Object lock = new Object();

	final long timeout = 100000L;
	final long timestamp = 1L;

	Environment environment = createMockEnvironment();

	ScheduledFuture<?> scheduledFuture = mock(ScheduledFuture.class);

	ProcessingTimeService processingTimeService = mock(ProcessingTimeService.class);
	when(processingTimeService.getCurrentProcessingTime()).thenReturn(timestamp);
	doReturn(scheduledFuture).when(processingTimeService).registerTimer(anyLong(), any(ProcessingTimeCallback.class));

	StreamTask<?, ?> containingTask = mock(StreamTask.class);
	when(containingTask.getEnvironment()).thenReturn(environment);
	when(containingTask.getCheckpointLock()).thenReturn(lock);
	when(containingTask.getProcessingTimeService()).thenReturn(processingTimeService);

	StreamConfig streamConfig = new MockStreamConfig();
	streamConfig.setTypeSerializerIn1(IntSerializer.INSTANCE);

	Output<StreamRecord<Integer>> output = mock(Output.class);

	AsyncWaitOperator<Integer, Integer> operator = new AsyncWaitOperator<>(
		new AsyncFunction<Integer, Integer>() {
			private static final long serialVersionUID = -3718276118074877073L;

			@Override
			public void asyncInvoke(Integer input, ResultFuture<Integer> resultFuture) throws Exception {
				resultFuture.complete(Collections.singletonList(input));
			}
		},
		timeout,
		1,
		AsyncDataStream.OutputMode.UNORDERED);

	operator.setup(
		containingTask,
		streamConfig,
		output);

	operator.open();

	final StreamRecord<Integer> streamRecord = new StreamRecord<>(42, timestamp);

	synchronized (lock) {
		// processing an element will register a timeout
		operator.processElement(streamRecord);
	}

	synchronized (lock) {
		// closing the operator waits until all inputs have been processed
		operator.close();
	}

	// check that we actually outputted the result of the single input
	verify(output).collect(eq(streamRecord));
	verify(processingTimeService).registerTimer(eq(processingTimeService.getCurrentProcessingTime() + timeout), any(ProcessingTimeCallback.class));

	// check that we have cancelled our registered timeout
	verify(scheduledFuture).cancel(eq(true));
}
 

public StreamElementQueueTest(AsyncDataStream.OutputMode outputMode) {
	this.outputMode = Preconditions.checkNotNull(outputMode);
}
 

@Parameterized.Parameters
public static Collection<AsyncDataStream.OutputMode> outputModes() {
	return Arrays.asList(AsyncDataStream.OutputMode.ORDERED, AsyncDataStream.OutputMode.UNORDERED);
}
 

/**
 * Test case for FLINK-5638: Tests that the async wait operator can be closed even if the
 * emitter is currently waiting on the checkpoint lock (e.g. in the case of two chained async
 * wait operators where the latter operator's queue is currently full).
 *
 * <p>Note that this test does not enforce the exact strict ordering because with the fix it is no
 * longer possible. However, it provokes the described situation without the fix.
 */
@Test(timeout = 10000L)
public void testClosingWithBlockedEmitter() throws Exception {
	final Object lock = new Object();

	ArgumentCaptor<Throwable> failureReason = ArgumentCaptor.forClass(Throwable.class);

	MockEnvironment environment = createMockEnvironment();

	StreamTask<?, ?> containingTask = mock(StreamTask.class);
	when(containingTask.getEnvironment()).thenReturn(environment);
	when(containingTask.getCheckpointLock()).thenReturn(lock);
	when(containingTask.getProcessingTimeService()).thenReturn(new TestProcessingTimeService());

	StreamConfig streamConfig = new MockStreamConfig();
	streamConfig.setTypeSerializerIn1(IntSerializer.INSTANCE);

	final OneShotLatch closingLatch = new OneShotLatch();
	final OneShotLatch outputLatch = new OneShotLatch();

	Output<StreamRecord<Integer>> output = mock(Output.class);
	doAnswer(new Answer() {
		@Override
		public Object answer(InvocationOnMock invocation) throws Throwable {
			assertTrue("Output should happen under the checkpoint lock.", Thread.currentThread().holdsLock(lock));

			outputLatch.trigger();

			// wait until we're in the closing method of the operator
			while (!closingLatch.isTriggered()) {
				lock.wait();
			}

			return null;
		}
	}).when(output).collect(any(StreamRecord.class));

	AsyncWaitOperator<Integer, Integer> operator = new TestAsyncWaitOperator<>(
		new MyAsyncFunction(),
		1000L,
		1,
		AsyncDataStream.OutputMode.ORDERED,
		closingLatch);

	operator.setup(
		containingTask,
		streamConfig,
		output);

	operator.open();

	synchronized (lock) {
		operator.processElement(new StreamRecord<>(42));
	}

	outputLatch.await();

	synchronized (lock) {
		operator.close();
	}
}