Martijn Visser created FLINK-41553:
--------------------------------------

             Summary: CepOperator accepts late events after a restore and 
corrupts the SharedBuffer
                 Key: FLINK-41553
                 URL: https://issues.apache.org/jira/browse/FLINK-41553
             Project: Flink
          Issue Type: Bug
          Components: Library / CEP
    Affects Versions: 2.3.0, 2.4.0
            Reporter: Martijn Visser


After a restore, {{CepOperator}} uses {{timerService.currentWatermark()}} to 
decide whether an event is late. That watermark is not part of the snapshot and 
starts at {{{}Long.MIN_VALUE{}}}, so until the first watermark arrives, events 
older than the pre-restore watermark are buffered instead of dropped.

{{SharedBuffer#advanceTime}} already removed the event counter for those 
timestamps, while a partial match within {{within()}} can still reference an 
event at the same timestamp. {{SharedBuffer#registerEvent}} then hands out the 
same {{EventId}} again. Depending on the input this fails with an 
{{IllegalStateException}} in {{NFA#extractCurrentMatches}} 
({{{}checkState(paths.size() == 1){}}}) or with an NPE in 
{{{}SharedBufferAccessor#materializeMatch{}}}. Every restart restores the same 
state and replays the same input, so the job fails in a loop.

This test can be dropped it into 
{{flink-libraries/flink-cep/src/test/java/org/apache/flink/cep/operator as a 
reproducer}}
{code:java}
package org.apache.flink.cep.operator;

import org.apache.flink.api.common.serialization.SerializerConfigImpl;
import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.cep.functions.PatternProcessFunction;
import org.apache.flink.cep.nfa.aftermatch.AfterMatchSkipStrategy;
import org.apache.flink.cep.nfa.compiler.NFACompiler;
import org.apache.flink.cep.pattern.Pattern;
import org.apache.flink.cep.pattern.conditions.IterativeCondition;
import org.apache.flink.runtime.checkpoint.OperatorSubtaskState;
import org.apache.flink.state.rocksdb.EmbeddedRocksDBStateBackend;
import org.apache.flink.streaming.util.KeyedOneInputStreamOperatorTestHarness;
import org.apache.flink.util.Collector;

import org.junit.jupiter.api.Test;

import java.time.Duration;
import java.util.List;
import java.util.Map;

import static org.junit.jupiter.api.Assertions.assertEquals;

/**
 * After a restore the operator's watermark is Long.MIN_VALUE, so an event 
older than the
 * pre-restore watermark is not dropped as late. Its timestamp's event counter 
was already removed
 * by SharedBuffer#advanceTime, so it reuses the EventId of an event a partial 
match still
 * references.
 */
class CepOperatorRestoreLateEventTest {

    /** Input row. */
    public static class Event {
        public String key;
        public String kind;
        public int id;

        public Event() {}

        Event(String kind, int id) {
            this.key = "k";
            this.kind = kind;
            this.id = id;
        }

        @Override
        public String toString() {
            return kind + "#" + id;
        }
    }

    /** Like SQL DEFINE x AS x.kind = '...': the generated condition reads 
getEventsForPattern(x). */
    private static IterativeCondition<Event> kind(String var, String kind, 
boolean negate) {
        return new IterativeCondition<>() {
            @Override
            public boolean filter(Event value, Context<Event> ctx) throws 
Exception {
                ctx.getEventsForPattern(var).forEach(e -> {});
                return negate != kind.equals(value.kind);
            }
        };
    }

    private static KeyedOneInputStreamOperatorTestHarness<String, Event, 
String> harness()
            throws Exception {
        // PATTERN (V M* L) WITHIN INTERVAL '30' MINUTE AFTER MATCH SKIP PAST 
LAST ROW
        Pattern<Event, Event> pattern =
                Pattern.<Event>begin("V", 
AfterMatchSkipStrategy.skipPastLastEvent())
                        .where(kind("V", "start", false))
                        .next("M")
                        .where(kind("M", "end", true))
                        .oneOrMore()
                        .optional()
                        .consecutive()
                        .greedy()
                        .next("L")
                        .where(kind("L", "end", false))
                        .within(Duration.ofMinutes(30));
        CepOperator<Event, String, String> operator =
                new CepOperator<>(
                        TypeInformation.of(Event.class).createSerializer(new 
SerializerConfigImpl()),
                        false,
                        NFACompiler.compileFactory(pattern, false),
                        null,
                        pattern.getAfterMatchSkipStrategy(),
                        new PatternProcessFunction<Event, String>() {
                            @Override
                            public void processMatch(
                                    Map<String, List<Event>> match,
                                    Context ctx,
                                    Collector<String> out) {
                                out.collect(match.toString());
                            }
                        },
                        null);
        KeyedOneInputStreamOperatorTestHarness<String, Event, String> harness =
                new KeyedOneInputStreamOperatorTestHarness<>(
                        operator,
                        (KeySelector<Event, String>) e -> e.key,
                        BasicTypeInfo.STRING_TYPE_INFO);
        harness.setStateBackend(new EmbeddedRocksDBStateBackend());
        return harness;
    }

    private static void runRestoreWithLateEvents(Event... lateEvents) throws 
Exception {
        OperatorSubtaskState snapshot;
        try (KeyedOneInputStreamOperatorTestHarness<String, Event, String> 
harness = harness()) {
            harness.open();
            harness.processElement(new Event("start", 1), 1000L);
            harness.processElement(new Event("middle", 2), 2000L);
            harness.processWatermark(5000L);
            snapshot = harness.snapshot(1L, 1L);
        }

        try (KeyedOneInputStreamOperatorTestHarness<String, Event, String> 
harness = harness()) {
            harness.initializeState(snapshot);
            harness.open();
            for (Event late : lateEvents) {
                // older than the pre-restore watermark of 5000, so these are 
late
                harness.processElement(late, late.kind.equals("start") ? 1000L 
: 2000L);
            }
            harness.processElement(new Event("end", 9), 6000L);
            harness.processWatermark(7000L);

            assertEquals(
                    List.of("{V=[start#1], M=[middle#2], L=[end#9]}"),
                    harness.extractOutputValues());
        }
    }

    @Test
    void testLateEventAfterRestoreFailsCheckState() throws Exception {
        // IllegalStateException at NFA#extractCurrentMatches
        runRestoreWithLateEvents(new Event("start", 3), new Event("middle", 4));
    }

    @Test
    void testLateEventAfterRestoreFailsWithNpe() throws Exception {
        // NullPointerException, SharedBuffer#getEvent returns null in 
materializeMatch
        runRestoreWithLateEvents(new Event("middle", 3));
    }
}
{code}
{noformat}
java.lang.IllegalStateException
    at org.apache.flink.util.Preconditions.checkState(Preconditions.java:177)
    at org.apache.flink.cep.nfa.NFA.extractCurrentMatches(NFA.java:877)
    at 
org.apache.flink.cep.nfa.NFA$ConditionContext.getEventsForPattern(NFA.java:922)

java.lang.NullPointerException: Cannot invoke 
"org.apache.flink.cep.nfa.sharedbuffer.Lockable.getElement()" because the 
return value of 
"org.apache.flink.cep.nfa.sharedbuffer.SharedBuffer.getEvent(org.apache.flink.cep.nfa.sharedbuffer.EventId)"
 is null
    at 
org.apache.flink.cep.nfa.sharedbuffer.SharedBufferAccessor.materializeMatch(SharedBufferAccessor.java:215)
{noformat}



--
This message was sent by Atlassian Jira
(v10.3.26#10030026)

Reply via email to