twalthr commented on code in PR #26893: URL: https://github.com/apache/flink/pull/26893#discussion_r2272282744
########## flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/ChangelogNormalizeSemanticTestProgram.java: ########## @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.plan.nodes.exec.stream; + +import org.apache.flink.table.api.config.ExecutionConfigOptions; +import org.apache.flink.table.test.program.SinkTestStep; +import org.apache.flink.table.test.program.SourceTestStep; +import org.apache.flink.table.test.program.TableTestProgram; +import org.apache.flink.types.Row; +import org.apache.flink.types.RowKind; + +/** + * {@link TableTestProgram} definitions for semantic testing {@link StreamExecChangelogNormalize}. + */ +public class ChangelogNormalizeSemanticTestProgram { + + private static final String[] SINK_SCHEMA = { + "a VARCHAR", "b INT", "c VARCHAR", "PRIMARY KEY(a) NOT ENFORCED" Review Comment: nit: let's use `STRING` instead of unparameterized VARCHAR ########## flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/ChangelogNormalizeSemanticTestProgram.java: ########## @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.plan.nodes.exec.stream; + +import org.apache.flink.table.api.config.ExecutionConfigOptions; +import org.apache.flink.table.test.program.SinkTestStep; +import org.apache.flink.table.test.program.SourceTestStep; +import org.apache.flink.table.test.program.TableTestProgram; +import org.apache.flink.types.Row; +import org.apache.flink.types.RowKind; + +/** + * {@link TableTestProgram} definitions for semantic testing {@link StreamExecChangelogNormalize}. + */ +public class ChangelogNormalizeSemanticTestProgram { + + private static final String[] SINK_SCHEMA = { + "a VARCHAR", "b INT", "c VARCHAR", "PRIMARY KEY(a) NOT ENFORCED" + }; + private static final String[] SOURCE_SCHEMA = { + "a VARCHAR", "b INT", "c VARCHAR", "PRIMARY KEY(a) NOT ENFORCED" + }; + + static final TableTestProgram UPSERT_SOURCE_WITH_FILTER = + TableTestProgram.of( + "changelog-normalize-filter", + "validates changelog normalize with filter") + .setupConfig( + ExecutionConfigOptions.TABLE_EXEC_SOURCE_CDC_EVENTS_DUPLICATE, true) + .setupTableSource( + SourceTestStep.newBuilder("source_t") + .addOption("changelog-mode", "UA,D") + .addSchema(SOURCE_SCHEMA) + .producedValues( + Row.ofKind(RowKind.INSERT, "one", 1, "a"), + Row.ofKind(RowKind.UPDATE_AFTER, "one", 2, "bb"), + Row.ofKind(RowKind.UPDATE_AFTER, "three", 3, "ccc"), + Row.ofKind(RowKind.INSERT, "one", 4, "aaaa"), + Row.ofKind(RowKind.UPDATE_AFTER, "one", 4, "aaaa"), + Row.ofKind(RowKind.DELETE, "three", 3, "cc")) + .build()) + .setupTableSink( + SinkTestStep.newBuilder("sink_t") + .addSchema(SINK_SCHEMA) + .consumedValues( + "+I[one, 1, a]", + "+U[one, 2, bb]", + "+I[three, 3, ccc]", + "+U[one, 4, aaaa]", + "-D[three, 3, ccc]") + .build()) + .runSql("INSERT INTO sink_t SELECT a, b, c FROM source_t WHERE b < 10") Review Comment: add another WHERE condition that excludes a single value for column `a`. filters on upsert key trigger a whole new set of logic. ########## flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/deduplicate/utils/DeduplicateFunctionHelper.java: ########## @@ -292,6 +292,19 @@ public static boolean shouldKeepCurrentRow( } } + private static boolean areRowsWithSameContent( + RecordEqualiser equaliser, RowData prevRow, RowData currentRow) { + final RowKind currentRowKind = currentRow.getRowKind(); + if (currentRowKind == RowKind.UPDATE_AFTER) { + // setting row kind to prevRowKind to check whether the row content is the same + currentRow.setRowKind(RowKind.INSERT); + final boolean result = equaliser.equals(prevRow, currentRow); + currentRow.setRowKind(currentRowKind); + return result; + } + return equaliser.equals(prevRow, currentRow); Review Comment: when is this line important? ########## flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/ChangelogNormalizeSemanticTestProgram.java: ########## @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.plan.nodes.exec.stream; + +import org.apache.flink.table.api.config.ExecutionConfigOptions; +import org.apache.flink.table.test.program.SinkTestStep; +import org.apache.flink.table.test.program.SourceTestStep; +import org.apache.flink.table.test.program.TableTestProgram; +import org.apache.flink.types.Row; +import org.apache.flink.types.RowKind; + +/** + * {@link TableTestProgram} definitions for semantic testing {@link StreamExecChangelogNormalize}. + */ +public class ChangelogNormalizeSemanticTestProgram { + + private static final String[] SINK_SCHEMA = { + "a VARCHAR", "b INT", "c VARCHAR", "PRIMARY KEY(a) NOT ENFORCED" + }; + private static final String[] SOURCE_SCHEMA = { + "a VARCHAR", "b INT", "c VARCHAR", "PRIMARY KEY(a) NOT ENFORCED" + }; + + static final TableTestProgram UPSERT_SOURCE_WITH_FILTER = + TableTestProgram.of( + "changelog-normalize-filter", + "validates changelog normalize with filter") + .setupConfig( + ExecutionConfigOptions.TABLE_EXEC_SOURCE_CDC_EVENTS_DUPLICATE, true) + .setupTableSource( + SourceTestStep.newBuilder("source_t") + .addOption("changelog-mode", "UA,D") + .addSchema(SOURCE_SCHEMA) + .producedValues( + Row.ofKind(RowKind.INSERT, "one", 1, "a"), + Row.ofKind(RowKind.UPDATE_AFTER, "one", 2, "bb"), + Row.ofKind(RowKind.UPDATE_AFTER, "three", 3, "ccc"), + Row.ofKind(RowKind.INSERT, "one", 4, "aaaa"), + Row.ofKind(RowKind.UPDATE_AFTER, "one", 4, "aaaa"), + Row.ofKind(RowKind.DELETE, "three", 3, "cc")) + .build()) + .setupTableSink( + SinkTestStep.newBuilder("sink_t") + .addSchema(SINK_SCHEMA) Review Comment: in addition: we could keep this test for also testing the behavior with upsert sink materialize ########## flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/ChangelogNormalizeSemanticTestProgram.java: ########## @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.plan.nodes.exec.stream; + +import org.apache.flink.table.api.config.ExecutionConfigOptions; +import org.apache.flink.table.test.program.SinkTestStep; +import org.apache.flink.table.test.program.SourceTestStep; +import org.apache.flink.table.test.program.TableTestProgram; +import org.apache.flink.types.Row; +import org.apache.flink.types.RowKind; + +/** + * {@link TableTestProgram} definitions for semantic testing {@link StreamExecChangelogNormalize}. + */ +public class ChangelogNormalizeSemanticTestProgram { + + private static final String[] SINK_SCHEMA = { + "a VARCHAR", "b INT", "c VARCHAR", "PRIMARY KEY(a) NOT ENFORCED" + }; + private static final String[] SOURCE_SCHEMA = { + "a VARCHAR", "b INT", "c VARCHAR", "PRIMARY KEY(a) NOT ENFORCED" + }; + + static final TableTestProgram UPSERT_SOURCE_WITH_FILTER = + TableTestProgram.of( + "changelog-normalize-filter", + "validates changelog normalize with filter") + .setupConfig( + ExecutionConfigOptions.TABLE_EXEC_SOURCE_CDC_EVENTS_DUPLICATE, true) + .setupTableSource( + SourceTestStep.newBuilder("source_t") + .addOption("changelog-mode", "UA,D") + .addSchema(SOURCE_SCHEMA) + .producedValues( + Row.ofKind(RowKind.INSERT, "one", 1, "a"), + Row.ofKind(RowKind.UPDATE_AFTER, "one", 2, "bb"), + Row.ofKind(RowKind.UPDATE_AFTER, "three", 3, "ccc"), + Row.ofKind(RowKind.INSERT, "one", 4, "aaaa"), + Row.ofKind(RowKind.UPDATE_AFTER, "one", 4, "aaaa"), + Row.ofKind(RowKind.DELETE, "three", 3, "cc")) + .build()) + .setupTableSink( + SinkTestStep.newBuilder("sink_t") + .addSchema(SINK_SCHEMA) Review Comment: with the current configuration, an upsert sink materialize is added. in order to purely test ChangelogNormalize, switch the sink to retract: `sink-enforced-changelog-mode = 'I,UB,UA,D'` ########## flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/ChangelogNormalizeSemanticTestProgram.java: ########## @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.plan.nodes.exec.stream; + +import org.apache.flink.table.api.config.ExecutionConfigOptions; +import org.apache.flink.table.test.program.SinkTestStep; +import org.apache.flink.table.test.program.SourceTestStep; +import org.apache.flink.table.test.program.TableTestProgram; +import org.apache.flink.types.Row; +import org.apache.flink.types.RowKind; + +/** + * {@link TableTestProgram} definitions for semantic testing {@link StreamExecChangelogNormalize}. + */ +public class ChangelogNormalizeSemanticTestProgram { Review Comment: ```suggestion public class ChangelogNormalizeSemanticTestPrograms { ``` ########## flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/ChangelogNormalizeSemanticTestProgram.java: ########## @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.plan.nodes.exec.stream; + +import org.apache.flink.table.api.config.ExecutionConfigOptions; +import org.apache.flink.table.test.program.SinkTestStep; +import org.apache.flink.table.test.program.SourceTestStep; +import org.apache.flink.table.test.program.TableTestProgram; +import org.apache.flink.types.Row; +import org.apache.flink.types.RowKind; + +/** + * {@link TableTestProgram} definitions for semantic testing {@link StreamExecChangelogNormalize}. + */ +public class ChangelogNormalizeSemanticTestProgram { + + private static final String[] SINK_SCHEMA = { + "a VARCHAR", "b INT", "c VARCHAR", "PRIMARY KEY(a) NOT ENFORCED" + }; + private static final String[] SOURCE_SCHEMA = { + "a VARCHAR", "b INT", "c VARCHAR", "PRIMARY KEY(a) NOT ENFORCED" + }; + + static final TableTestProgram UPSERT_SOURCE_WITH_FILTER = + TableTestProgram.of( + "changelog-normalize-filter", + "validates changelog normalize with filter") + .setupConfig( + ExecutionConfigOptions.TABLE_EXEC_SOURCE_CDC_EVENTS_DUPLICATE, true) + .setupTableSource( + SourceTestStep.newBuilder("source_t") + .addOption("changelog-mode", "UA,D") + .addSchema(SOURCE_SCHEMA) + .producedValues( + Row.ofKind(RowKind.INSERT, "one", 1, "a"), + Row.ofKind(RowKind.UPDATE_AFTER, "one", 2, "bb"), + Row.ofKind(RowKind.UPDATE_AFTER, "three", 3, "ccc"), Review Comment: copy this line and insert a duplicate: ```suggestion Row.ofKind(RowKind.UPDATE_AFTER, "three", 3, "ccc"), ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
