This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/master/pr-20667-b2ee4f665b585d6c324ef110a04e7c6de675a516 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 0eb98b0051421125b9d1ca3ccb19072131d1e1a5 Author: Danny Chan <[email protected]> AuthorDate: Tue Nov 17 03:05:51 2026 +0000 fix(flink): avoid quadratic map array copying during schema evolution (#20667) --- .../hudi/util/SchemaEvolvingRowDataProjection.java | 6 +- .../java/org/apache/hudi/util/TypeConverters.java | 6 +- .../apache/hudi/util/TestMapDataConversion.java | 69 ++++++++++++++++++++++ 3 files changed, 77 insertions(+), 4 deletions(-) diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/SchemaEvolvingRowDataProjection.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/SchemaEvolvingRowDataProjection.java index ed6156274ebb..0012a482ee7e 100644 --- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/SchemaEvolvingRowDataProjection.java +++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/SchemaEvolvingRowDataProjection.java @@ -113,9 +113,11 @@ public class SchemaEvolvingRowDataProjection implements RowProjection { return obj -> { final Map<Object, Object> result = new HashMap<>(); MapData map = (MapData) obj; + final ArrayData keys = map.keyArray(); + final ArrayData values = map.valueArray(); for (int i = 0; i < map.size(); i++) { - Object keyObj = keyElementGetter.getElementOrNull(map.keyArray(), i); - Object fromObj = valueElementGetter.getElementOrNull(map.valueArray(), i); + Object keyObj = keyElementGetter.getElementOrNull(keys, i); + Object fromObj = valueElementGetter.getElementOrNull(values, i); // need to handle nulls to prevent NullPointerException in #getConversion() Object toObj = fromObj == null ? null : valueConverter.convert(fromObj); result.put(keyObj, toObj); diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/TypeConverters.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/TypeConverters.java index eaebf74ee795..ffd1bc07c004 100644 --- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/TypeConverters.java +++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/TypeConverters.java @@ -320,9 +320,11 @@ public class TypeConverters { public Object convert(Object val) { final Map<Object, Object> result = new HashMap<>(); MapData map = (MapData) val; + final ArrayData keys = map.keyArray(); + final ArrayData values = map.valueArray(); for (int i = 0; i < map.size(); i++) { - Object keyObj = keyElementGetter.getElementOrNull(map.keyArray(), i); - Object fromObj = valueElementGetter.getElementOrNull(map.valueArray(), i); + Object keyObj = keyElementGetter.getElementOrNull(keys, i); + Object fromObj = valueElementGetter.getElementOrNull(values, i); // need to handle nulls to prevent NullPointerException in #getConversion() Object toObj = fromObj != null ? converter.convert(fromObj) : null; result.put(keyObj, toObj); diff --git a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/util/TestMapDataConversion.java b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/util/TestMapDataConversion.java new file mode 100644 index 000000000000..85c6178a442e --- /dev/null +++ b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/util/TestMapDataConversion.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.util; + +import org.apache.flink.table.data.GenericMapData; +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.MapData; +import org.apache.flink.table.types.logical.BigIntType; +import org.apache.flink.table.types.logical.IntType; +import org.apache.flink.table.types.logical.MapType; +import org.apache.flink.table.types.logical.RowType; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; + +/** + * Tests that map conversions materialize the input arrays only once. + */ +public class TestMapDataConversion { + + @ParameterizedTest + @CsvSource({"true, 0", "true, 1", "true, 100", "false, 0", "false, 1", "false, 100"}) + public void testMapArraysMaterializedOnce(boolean projection, int size) { + Map<Integer, Integer> input = new HashMap<>(); + Map<Integer, Long> expected = new HashMap<>(); + for (int i = 0; i < size; i++) { + input.put(i, i % 2 == 0 ? i : null); + expected.put(i, i % 2 == 0 ? (long) i : null); + } + GenericMapData map = spy(new GenericMapData(input)); + MapType from = new MapType(new IntType(), new IntType()); + MapType to = new MapType(new IntType(), new BigIntType()); + MapData result; + if (projection) { + SchemaEvolvingRowDataProjection converter = SchemaEvolvingRowDataProjection.instance( + RowType.of(from), RowType.of(to), Collections.emptyMap()); + result = converter.project(GenericRowData.of(map)).getMap(0); + } else { + result = (MapData) TypeConverters.getInstance(from, to).convert(map); + } + + assertEquals(new GenericMapData(expected), result); + verify(map).keyArray(); + verify(map).valueArray(); + } +}
