org.apache.flink.cep.nfa.aftermatch.AfterMatchSkipStrategy Java Examples

@Test
public void testSharedBufferClearing() throws Exception {
	Pattern<Event, ?> pattern = Pattern.<Event>begin("start").followedBy("end");

	Event a = new Event(40, "a", 1.0);
	Event b = new Event(41, "b", 2.0);

	NFA<Event> nfa = compile(pattern, false);
	TestTimerService timerService = new TestTimerService();
	try (SharedBufferAccessor<Event> accessor = Mockito.spy(sharedBuffer.getAccessor())) {
		nfa.process(accessor, nfa.createInitialNFAState(), a, 1, AfterMatchSkipStrategy.noSkip(),
			timerService);
		nfa.process(accessor, nfa.createInitialNFAState(), b, 2, AfterMatchSkipStrategy.noSkip(),
			timerService);
		Mockito.verify(accessor, Mockito.never()).advanceTime(anyLong());
		nfa.advanceTime(accessor, nfa.createInitialNFAState(), 2);
		Mockito.verify(accessor, Mockito.times(1)).advanceTime(2);
	}
}
 

public CepOperator(
		final TypeSerializer<IN> inputSerializer,
		final boolean isProcessingTime,
		final NFACompiler.NFAFactory<IN> nfaFactory,
		@Nullable final EventComparator<IN> comparator,
		@Nullable final AfterMatchSkipStrategy afterMatchSkipStrategy,
		final PatternProcessFunction<IN, OUT> function,
		@Nullable final OutputTag<IN> lateDataOutputTag) {
	super(function);

	this.inputSerializer = Preconditions.checkNotNull(inputSerializer);
	this.nfaFactory = Preconditions.checkNotNull(nfaFactory);

	this.isProcessingTime = isProcessingTime;
	this.comparator = comparator;
	this.lateDataOutputTag = lateDataOutputTag;

	if (afterMatchSkipStrategy == null) {
		this.afterMatchSkipStrategy = AfterMatchSkipStrategy.noSkip();
	} else {
		this.afterMatchSkipStrategy = afterMatchSkipStrategy;
	}
}
 

@Test
public void testSharedBufferIsProperlyCleared() throws Exception {
	List<StreamRecord<Event>> inputEvents = new ArrayList<>();

	for (int i = 0; i < 4; i++) {
		inputEvents.add(new StreamRecord<>(new Event(1, "a", 1.0), i));
	}

	SkipPastLastStrategy matchSkipStrategy = AfterMatchSkipStrategy.skipPastLastEvent();
	Pattern<Event, ?> pattern = Pattern.<Event>begin("start", matchSkipStrategy)
		.where(new SimpleCondition<Event>() {
			private static final long serialVersionUID = 5726188262756267490L;

			@Override
			public boolean filter(Event value) throws Exception {
				return true;
			}
		}).times(2);

	SharedBuffer<Event> sharedBuffer = TestSharedBuffer.createTestBuffer(Event.createTypeSerializer());
	NFATestHarness nfaTestHarness = NFATestHarness.forPattern(pattern).withSharedBuffer(sharedBuffer).build();

	nfaTestHarness.feedRecords(inputEvents);

	assertThat(sharedBuffer.isEmpty(), Matchers.is(true));
}
 

@Test
public void testProcessingTimestampisPassedToNFA() throws Exception {

	final NFA<Event> nfa = NFACompiler.compileFactory(Pattern.<Event>begin("begin"), true).createNFA();
	final NFA<Event> spyNFA = spy(nfa);

	try (
		OneInputStreamOperatorTestHarness<Event, Map<String, List<Event>>> harness =
			CepOperatorTestUtilities.getCepTestHarness(createOperatorForNFA(spyNFA).build())) {

		long timestamp = 5;
		harness.open();
		harness.setProcessingTime(timestamp);
		StreamRecord<Event> event = event().withTimestamp(3).asStreamRecord();
		harness.processElement(event);
		verify(spyNFA).process(
			any(SharedBufferAccessor.class),
			any(NFAState.class),
			eq(event.getValue()),
			eq(timestamp),
			any(AfterMatchSkipStrategy.class),
			any(TimerService.class));
	}
}
 

@Test
public void testProcessingTimestampisPassedToNFA() throws Exception {

	final NFA<Event> nfa = NFACompiler.compileFactory(Pattern.<Event>begin("begin"), true).createNFA();
	final NFA<Event> spyNFA = spy(nfa);

	try (
		OneInputStreamOperatorTestHarness<Event, Map<String, List<Event>>> harness =
			CepOperatorTestUtilities.getCepTestHarness(createOperatorForNFA(spyNFA).build())) {

		long timestamp = 5;
		harness.open();
		harness.setProcessingTime(timestamp);
		StreamRecord<Event> event = event().withTimestamp(3).asStreamRecord();
		harness.processElement(event);
		verify(spyNFA).process(
			any(SharedBufferAccessor.class),
			any(NFAState.class),
			eq(event.getValue()),
			eq(timestamp),
			any(AfterMatchSkipStrategy.class),
			any(TimerService.class));
	}
}
 

public CepOperator(
		final TypeSerializer<IN> inputSerializer,
		final boolean isProcessingTime,
		final NFACompiler.NFAFactory<IN> nfaFactory,
		@Nullable final EventComparator<IN> comparator,
		@Nullable final AfterMatchSkipStrategy afterMatchSkipStrategy,
		final PatternProcessFunction<IN, OUT> function,
		@Nullable final OutputTag<IN> lateDataOutputTag) {
	super(function);

	this.inputSerializer = Preconditions.checkNotNull(inputSerializer);
	this.nfaFactory = Preconditions.checkNotNull(nfaFactory);

	this.isProcessingTime = isProcessingTime;
	this.comparator = comparator;
	this.lateDataOutputTag = lateDataOutputTag;

	if (afterMatchSkipStrategy == null) {
		this.afterMatchSkipStrategy = AfterMatchSkipStrategy.noSkip();
	} else {
		this.afterMatchSkipStrategy = afterMatchSkipStrategy;
	}
}
 

@Test
public void testSharedBufferIsProperlyCleared() throws Exception {
	List<StreamRecord<Event>> inputEvents = new ArrayList<>();

	for (int i = 0; i < 4; i++) {
		inputEvents.add(new StreamRecord<>(new Event(1, "a", 1.0), i));
	}

	SkipPastLastStrategy matchSkipStrategy = AfterMatchSkipStrategy.skipPastLastEvent();
	Pattern<Event, ?> pattern = Pattern.<Event>begin("start", matchSkipStrategy)
		.where(new SimpleCondition<Event>() {
			private static final long serialVersionUID = 5726188262756267490L;

			@Override
			public boolean filter(Event value) throws Exception {
				return true;
			}
		}).times(2);

	SharedBuffer<Event> sharedBuffer = TestSharedBuffer.createTestBuffer(Event.createTypeSerializer());
	NFATestHarness nfaTestHarness = NFATestHarness.forPattern(pattern).withSharedBuffer(sharedBuffer).build();

	nfaTestHarness.feedRecords(inputEvents);

	assertThat(sharedBuffer.isEmpty(), Matchers.is(true));
}
 

public CepOperator(
		final TypeSerializer<IN> inputSerializer,
		final boolean isProcessingTime,
		final NFACompiler.NFAFactory<IN> nfaFactory,
		@Nullable final EventComparator<IN> comparator,
		@Nullable final AfterMatchSkipStrategy afterMatchSkipStrategy,
		final PatternProcessFunction<IN, OUT> function,
		@Nullable final OutputTag<IN> lateDataOutputTag) {
	super(function);

	this.inputSerializer = Preconditions.checkNotNull(inputSerializer);
	this.nfaFactory = Preconditions.checkNotNull(nfaFactory);

	this.isProcessingTime = isProcessingTime;
	this.comparator = comparator;
	this.lateDataOutputTag = lateDataOutputTag;

	if (afterMatchSkipStrategy == null) {
		this.afterMatchSkipStrategy = AfterMatchSkipStrategy.noSkip();
	} else {
		this.afterMatchSkipStrategy = afterMatchSkipStrategy;
	}
}
 

@Test
public void testSharedBufferIsProperlyCleared() throws Exception {
	List<StreamRecord<Event>> inputEvents = new ArrayList<>();

	for (int i = 0; i < 4; i++) {
		inputEvents.add(new StreamRecord<>(new Event(1, "a", 1.0), i));
	}

	SkipPastLastStrategy matchSkipStrategy = AfterMatchSkipStrategy.skipPastLastEvent();
	Pattern<Event, ?> pattern = Pattern.<Event>begin("start", matchSkipStrategy)
		.where(new SimpleCondition<Event>() {
			private static final long serialVersionUID = 5726188262756267490L;

			@Override
			public boolean filter(Event value) throws Exception {
				return true;
			}
		}).times(2);

	SharedBuffer<Event> sharedBuffer = TestSharedBuffer.createTestBuffer(Event.createTypeSerializer());
	NFATestHarness nfaTestHarness = NFATestHarness.forPattern(pattern).withSharedBuffer(sharedBuffer).build();

	nfaTestHarness.feedRecords(inputEvents);

	assertThat(sharedBuffer.isEmpty(), Matchers.is(true));
}
 

@Test
public void testProcessingTimestampisPassedToNFA() throws Exception {

	final NFA<Event> nfa = NFACompiler.compileFactory(Pattern.<Event>begin("begin"), true).createNFA();
	final NFA<Event> spyNFA = spy(nfa);

	try (
		OneInputStreamOperatorTestHarness<Event, Map<String, List<Event>>> harness =
			CepOperatorTestUtilities.getCepTestHarness(createOperatorForNFA(spyNFA).build())) {

		long timestamp = 5;
		harness.open();
		harness.setProcessingTime(timestamp);
		StreamRecord<Event> event = event().withTimestamp(3).asStreamRecord();
		harness.processElement(event);
		verify(spyNFA).process(
			any(SharedBufferAccessor.class),
			any(NFAState.class),
			eq(event.getValue()),
			eq(timestamp),
			any(AfterMatchSkipStrategy.class),
			any(TimerService.class));
	}
}
 

@Test
public void testSkipPastLast() throws Exception {
	List<StreamRecord<Event>> streamEvents = new ArrayList<>();

	Event a1 = new Event(1, "a", 0.0);
	Event a2 = new Event(2, "a", 0.0);
	Event a3 = new Event(3, "a", 0.0);
	Event a4 = new Event(4, "a", 0.0);
	Event a5 = new Event(5, "a", 0.0);
	Event a6 = new Event(6, "a", 0.0);

	streamEvents.add(new StreamRecord<Event>(a1));
	streamEvents.add(new StreamRecord<Event>(a2));
	streamEvents.add(new StreamRecord<Event>(a3));
	streamEvents.add(new StreamRecord<Event>(a4));
	streamEvents.add(new StreamRecord<Event>(a5));
	streamEvents.add(new StreamRecord<Event>(a6));

	Pattern<Event, ?> pattern = Pattern.<Event>begin("start", AfterMatchSkipStrategy.skipPastLastEvent())
		.where(new SimpleCondition<Event>() {

			@Override
			public boolean filter(Event value) throws Exception {
				return value.getName().equals("a");
			}
		}).times(3);

	NFATestHarness nfaTestHarness = NFATestHarness.forPattern(pattern).build();

	List<List<Event>> resultingPatterns = nfaTestHarness.feedRecords(streamEvents);

	compareMaps(resultingPatterns, Lists.newArrayList(
		Lists.newArrayList(a1, a2, a3),
		Lists.newArrayList(a4, a5, a6)
	));
}
 

GroupPattern(
	final Pattern<T, ? extends T> previous,
	final Pattern<T, ? extends T> groupPattern,
	final Quantifier.ConsumingStrategy consumingStrategy,
	final AfterMatchSkipStrategy afterMatchSkipStrategy) {
	super("GroupPattern", previous, consumingStrategy, afterMatchSkipStrategy);
	this.groupPattern = groupPattern;
}
 

@Test
public void testSkipToLastNonExistentPositionWithoutException() throws Exception {
	List<List<Event>> resultingPatterns = MissedSkipTo.compute(AfterMatchSkipStrategy.skipToFirst("b"));

	compareMaps(resultingPatterns, Collections.singletonList(
		Lists.newArrayList(MissedSkipTo.a, MissedSkipTo.c)
	));
}
 

@Test
public void testSkipPastLast() throws Exception {
	List<StreamRecord<Event>> streamEvents = new ArrayList<>();

	Event a1 = new Event(1, "a", 0.0);
	Event a2 = new Event(2, "a", 0.0);
	Event a3 = new Event(3, "a", 0.0);
	Event a4 = new Event(4, "a", 0.0);
	Event a5 = new Event(5, "a", 0.0);
	Event a6 = new Event(6, "a", 0.0);

	streamEvents.add(new StreamRecord<Event>(a1));
	streamEvents.add(new StreamRecord<Event>(a2));
	streamEvents.add(new StreamRecord<Event>(a3));
	streamEvents.add(new StreamRecord<Event>(a4));
	streamEvents.add(new StreamRecord<Event>(a5));
	streamEvents.add(new StreamRecord<Event>(a6));

	Pattern<Event, ?> pattern = Pattern.<Event>begin("start", AfterMatchSkipStrategy.skipPastLastEvent())
		.where(new SimpleCondition<Event>() {

			@Override
			public boolean filter(Event value) throws Exception {
				return value.getName().equals("a");
			}
		}).times(3);

	NFATestHarness nfaTestHarness = NFATestHarness.forPattern(pattern).build();

	List<List<Event>> resultingPatterns = nfaTestHarness.feedRecords(streamEvents);

	comparePatterns(resultingPatterns, Lists.newArrayList(
		Lists.newArrayList(a1, a2, a3),
		Lists.newArrayList(a4, a5, a6)
	));
}
 

protected Pattern(
	final String name,
	final Pattern<T, ? extends T> previous,
	final ConsumingStrategy consumingStrategy,
	final AfterMatchSkipStrategy afterMatchSkipStrategy) {
	this.name = name;
	this.previous = previous;
	this.quantifier = Quantifier.one(consumingStrategy);
	this.afterMatchSkipStrategy = afterMatchSkipStrategy;
}
 

GroupPattern(
	final Pattern<T, ? extends T> previous,
	final Pattern<T, ? extends T> groupPattern,
	final Quantifier.ConsumingStrategy consumingStrategy,
	final AfterMatchSkipStrategy afterMatchSkipStrategy) {
	super("GroupPattern", previous, consumingStrategy, afterMatchSkipStrategy);
	this.groupPattern = groupPattern;
}
 

@Test
public void testSkipToLastNonExistentPositionWithoutException() throws Exception {
	List<List<Event>> resultingPatterns = MissedSkipTo.compute(AfterMatchSkipStrategy.skipToFirst("b"));

	compareMaps(resultingPatterns, Collections.singletonList(
		Lists.newArrayList(MissedSkipTo.a, MissedSkipTo.c)
	));
}
 

@Test
public void testSkipToFirstNonExistentPositionWithoutException() throws Exception {
	List<List<Event>> resultingPatterns = MissedSkipTo.compute(AfterMatchSkipStrategy.skipToFirst("b"));

	compareMaps(resultingPatterns, Collections.singletonList(
		Lists.newArrayList(MissedSkipTo.a, MissedSkipTo.c)
	));
}
 

@Test
public void testNoSkip() throws Exception {
	List<StreamRecord<Event>> streamEvents = new ArrayList<>();

	Event a1 = new Event(1, "a", 0.0);
	Event a2 = new Event(2, "a", 0.0);
	Event a3 = new Event(3, "a", 0.0);
	Event a4 = new Event(4, "a", 0.0);
	Event a5 = new Event(5, "a", 0.0);
	Event a6 = new Event(6, "a", 0.0);

	streamEvents.add(new StreamRecord<Event>(a1));
	streamEvents.add(new StreamRecord<Event>(a2));
	streamEvents.add(new StreamRecord<Event>(a3));
	streamEvents.add(new StreamRecord<Event>(a4));
	streamEvents.add(new StreamRecord<Event>(a5));
	streamEvents.add(new StreamRecord<Event>(a6));

	Pattern<Event, ?> pattern = Pattern.<Event>begin("start", AfterMatchSkipStrategy.noSkip())
		.where(new SimpleCondition<Event>() {

			@Override
			public boolean filter(Event value) throws Exception {
				return value.getName().equals("a");
			}
		}).times(3);

	NFATestHarness nfaTestHarness = NFATestHarness.forPattern(pattern).build();

	List<List<Event>> resultingPatterns = nfaTestHarness.feedRecords(streamEvents);

	comparePatterns(resultingPatterns, Lists.newArrayList(
		Lists.newArrayList(a1, a2, a3),
		Lists.newArrayList(a2, a3, a4),
		Lists.newArrayList(a3, a4, a5),
		Lists.newArrayList(a4, a5, a6)
	));
}
 

@Test(expected = FlinkRuntimeException.class)
public void testSkipToFirstElementOfMatch() throws Exception {
	List<StreamRecord<Event>> streamEvents = new ArrayList<>();

	Event a1 = new Event(1, "a1", 0.0);

	streamEvents.add(new StreamRecord<Event>(a1));

	Pattern<Event, ?> pattern = Pattern.<Event>begin("a",
		AfterMatchSkipStrategy.skipToFirst("a").throwExceptionOnMiss()
	).where(
		new SimpleCondition<Event>() {

			@Override
			public boolean filter(Event value) throws Exception {
				return value.getName().contains("a");
			}
		}
	);
	NFATestHarness nfaTestHarness = NFATestHarness.forPattern(pattern).build();

	List<List<Event>> resultingPatterns = nfaTestHarness.feedRecords(streamEvents);

	//skip to first element of a match should throw exception if they are enabled,
	//this mode is used in MATCH RECOGNIZE which assumes that skipping to first element
	//would result in infinite loop. In CEP by default(with exceptions disabled), we use no skip
	//strategy in this case.
}
 

public CepOperatorBuilder<OUT> withSkipStrategy(AfterMatchSkipStrategy skipStrategy) {
	return new CepOperatorBuilder<>(
		false,
		nfaFactory,
		comparator,
		skipStrategy,
		function,
		lateDataOutputTag);
}
 

@Test
public void testNoSkipWithFollowedByAny() throws Exception {
	List<List<Event>> resultingPatterns = TwoVariablesFollowedByAny.compute(AfterMatchSkipStrategy.noSkip());

	comparePatterns(resultingPatterns, Lists.newArrayList(
		Lists.newArrayList(TwoVariablesFollowedByAny.a1, TwoVariablesFollowedByAny.b1),
		Lists.newArrayList(TwoVariablesFollowedByAny.a1, TwoVariablesFollowedByAny.b2),
		Lists.newArrayList(TwoVariablesFollowedByAny.a2, TwoVariablesFollowedByAny.b2)
	));
}
 

@Test
public void testSkipToNextWithQuantifierAtTheEnd() throws Exception {
	List<List<Event>> resultingPatterns = QuantifierAtEndOfPattern.compute(AfterMatchSkipStrategy.skipToNext());

	compareMaps(resultingPatterns, Lists.<List<Event>>newArrayList(
		Lists.newArrayList(QuantifierAtEndOfPattern.a1, QuantifierAtEndOfPattern.b1)
	));
}
 

private NFATestHarness(
		SharedBuffer<Event> sharedBuffer,
		NFA<Event> nfa,
		NFAState nfaState,
		AfterMatchSkipStrategy afterMatchSkipStrategy,
		TimerService timerService) {
	this.sharedBuffer = sharedBuffer;
	this.nfa = nfa;
	this.nfaState = nfaState;
	this.afterMatchSkipStrategy = afterMatchSkipStrategy;
	this.timerService = timerService;
}
 

@Test
public void testSkipToLastNonExistentPositionWithoutException() throws Exception {
	List<List<Event>> resultingPatterns = MissedSkipTo.compute(AfterMatchSkipStrategy.skipToFirst("b"));

	comparePatterns(resultingPatterns, Collections.singletonList(
		Lists.newArrayList(MissedSkipTo.a, MissedSkipTo.c)
	));
}
 

@Test
public void testSkipToNextWithFollowedByAny() throws Exception {
	List<List<Event>> resultingPatterns = TwoVariablesFollowedByAny.compute(AfterMatchSkipStrategy.skipToNext());

	compareMaps(resultingPatterns, Lists.newArrayList(
		Lists.newArrayList(TwoVariablesFollowedByAny.a1, TwoVariablesFollowedByAny.b1),
		Lists.newArrayList(TwoVariablesFollowedByAny.a2, TwoVariablesFollowedByAny.b2)
	));
}
 

@Test(expected = FlinkRuntimeException.class)
public void testSkipToFirstElementOfMatch() throws Exception {
	List<StreamRecord<Event>> streamEvents = new ArrayList<>();

	Event a1 = new Event(1, "a1", 0.0);

	streamEvents.add(new StreamRecord<Event>(a1));

	Pattern<Event, ?> pattern = Pattern.<Event>begin("a",
		AfterMatchSkipStrategy.skipToFirst("a").throwExceptionOnMiss()
	).where(
		new SimpleCondition<Event>() {

			@Override
			public boolean filter(Event value) throws Exception {
				return value.getName().contains("a");
			}
		}
	);
	NFATestHarness nfaTestHarness = NFATestHarness.forPattern(pattern).build();

	List<List<Event>> resultingPatterns = nfaTestHarness.feedRecords(streamEvents);

	//skip to first element of a match should throw exception if they are enabled,
	//this mode is used in MATCH RECOGNIZE which assumes that skipping to first element
	//would result in infinite loop. In CEP by default(with exceptions disabled), we use no skip
	//strategy in this case.
}
 

public CepOperatorBuilder<OUT> withSkipStrategy(AfterMatchSkipStrategy skipStrategy) {
	return new CepOperatorBuilder<>(
		false,
		nfaFactory,
		comparator,
		skipStrategy,
		function,
		lateDataOutputTag);
}
 

@Test
public void testNoSkip() throws Exception {
	List<StreamRecord<Event>> streamEvents = new ArrayList<>();

	Event a1 = new Event(1, "a", 0.0);
	Event a2 = new Event(2, "a", 0.0);
	Event a3 = new Event(3, "a", 0.0);
	Event a4 = new Event(4, "a", 0.0);
	Event a5 = new Event(5, "a", 0.0);
	Event a6 = new Event(6, "a", 0.0);

	streamEvents.add(new StreamRecord<Event>(a1));
	streamEvents.add(new StreamRecord<Event>(a2));
	streamEvents.add(new StreamRecord<Event>(a3));
	streamEvents.add(new StreamRecord<Event>(a4));
	streamEvents.add(new StreamRecord<Event>(a5));
	streamEvents.add(new StreamRecord<Event>(a6));

	Pattern<Event, ?> pattern = Pattern.<Event>begin("start", AfterMatchSkipStrategy.noSkip())
		.where(new SimpleCondition<Event>() {

			@Override
			public boolean filter(Event value) throws Exception {
				return value.getName().equals("a");
			}
		}).times(3);

	NFATestHarness nfaTestHarness = NFATestHarness.forPattern(pattern).build();

	List<List<Event>> resultingPatterns = nfaTestHarness.feedRecords(streamEvents);

	compareMaps(resultingPatterns, Lists.newArrayList(
		Lists.newArrayList(a1, a2, a3),
		Lists.newArrayList(a2, a3, a4),
		Lists.newArrayList(a3, a4, a5),
		Lists.newArrayList(a4, a5, a6)
	));
}
 

@Test
public void testNoSkipWithFollowedByAny() throws Exception {
	List<List<Event>> resultingPatterns = TwoVariablesFollowedByAny.compute(AfterMatchSkipStrategy.noSkip());

	compareMaps(resultingPatterns, Lists.newArrayList(
		Lists.newArrayList(TwoVariablesFollowedByAny.a1, TwoVariablesFollowedByAny.b1),
		Lists.newArrayList(TwoVariablesFollowedByAny.a1, TwoVariablesFollowedByAny.b2),
		Lists.newArrayList(TwoVariablesFollowedByAny.a2, TwoVariablesFollowedByAny.b2)
	));
}