This is an automated email from the ASF dual-hosted git repository. XiaoHongbo-Hope pushed a commit to branch release-2.1 in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 4bcbcec5ef9dc317abd2d894828e807c994dc652 Author: 玉升 <[email protected]> AuthorDate: Thu Sep 17 14:43:36 2026 +0800 [core] Preserve quoted empty and null keys/values in string-to-MAP cast (#9849) --- .../apache/paimon/casting/StringToMapCastRule.java | 81 ++++++------------- .../org/apache/paimon/casting/TokenSplitter.java | 62 ++++++++++++++- .../apache/paimon/casting/CastExecutorTest.java | 93 ++++++++++++++++++---- 3 files changed, 161 insertions(+), 75 deletions(-) diff --git a/paimon-common/src/main/java/org/apache/paimon/casting/StringToMapCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/StringToMapCastRule.java index d700e9c6d8..f336b790e6 100644 --- a/paimon-common/src/main/java/org/apache/paimon/casting/StringToMapCastRule.java +++ b/paimon-common/src/main/java/org/apache/paimon/casting/StringToMapCastRule.java @@ -26,15 +26,12 @@ import org.apache.paimon.types.DataTypeFamily; import org.apache.paimon.types.DataTypeRoot; import org.apache.paimon.types.MapType; import org.apache.paimon.types.VarCharType; -import org.apache.paimon.utils.StringUtils; import org.apache.paimon.shade.guava30.com.google.common.collect.Maps; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Stack; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -50,8 +47,6 @@ class StringToMapCastRule extends AbstractCastRule<BinaryString, InternalMap> { private static final Pattern FUNCTION_MAP_PATTERN = Pattern.compile("^\\s*MAP\\s*\\((.*)\\)\\s*$", Pattern.CASE_INSENSITIVE); - private static final Pattern ENTRY_PATTERN = Pattern.compile("(.+?)\\s*->\\s*(.+)"); - private StringToMapCastRule() { super( CastRulePredicate.builder() @@ -134,7 +129,7 @@ class StringToMapCastRule extends AbstractCastRule<BinaryString, InternalMap> { CastExecutor<BinaryString, Object> keyCastExecutor, CastExecutor<BinaryString, Object> valueCastExecutor) { - List<String> elements = splitMapEntries(content.trim()); + List<TokenSplitter.Token> elements = TokenSplitter.split(content); if (elements.size() % 2 != 0) { throw new RuntimeException("Invalid Function map format: odd number of elements"); } @@ -142,8 +137,8 @@ class StringToMapCastRule extends AbstractCastRule<BinaryString, InternalMap> { Map<Object, Object> mapContent = Maps.newHashMap(); for (int i = 0; i < elements.size(); i += 2) { mapContent.put( - parseValue(elements.get(i).trim(), keyCastExecutor), - parseValue(elements.get(i + 1).trim(), valueCastExecutor)); + parseValue(elements.get(i), keyCastExecutor), + parseValue(elements.get(i + 1), valueCastExecutor)); } return mapContent; } @@ -154,65 +149,35 @@ class StringToMapCastRule extends AbstractCastRule<BinaryString, InternalMap> { CastExecutor<BinaryString, Object> valueCastExecutor) { Map<Object, Object> mapContent = Maps.newHashMap(); - for (String entry : splitMapEntries(content)) { - Matcher entryMatcher = ENTRY_PATTERN.matcher(entry); - if (!entryMatcher.matches()) { + for (String entry : TokenSplitter.splitRaw(content, ",", 0)) { + if (entry.isEmpty()) { + continue; + } + List<String> keyValue = TokenSplitter.splitRaw(entry, "->", 2); + if (keyValue.size() != 2) { throw new RuntimeException("Invalid map entry format: " + entry); } mapContent.put( - parseValue(entryMatcher.group(1).trim(), keyCastExecutor), - parseValue(entryMatcher.group(2).trim(), valueCastExecutor)); + parseToken(keyValue.get(0), keyCastExecutor), + parseToken(keyValue.get(1), valueCastExecutor)); } return mapContent; } - private Object parseValue(String valueStr, CastExecutor<BinaryString, Object> castExecutor) { - return "null".equals(valueStr) - ? null - : castExecutor.cast(BinaryString.fromString(valueStr)); - } - - public List<String> splitMapEntries(String content) { - List<String> entries = new ArrayList<>(); - StringBuilder current = new StringBuilder(); - Stack<Character> bracketStack = new Stack<>(); - boolean inQuotes = false; - boolean escaped = false; - - for (char c : content.toCharArray()) { - if (escaped) { - // an escaped character is taken literally: it must survive in the - // entry instead of being dropped together with its backslash - escaped = false; - current.append(c); - continue; - } else if (c == '\\') { - escaped = true; - continue; - } else if (c == '"') { - inQuotes = !inQuotes; - continue; - } else if (!inQuotes) { - if (StringUtils.isOpenBracket(c)) { - bracketStack.push(c); - } else if (StringUtils.isCloseBracket(c) && !bracketStack.isEmpty()) { - bracketStack.pop(); - } else if (c == ',' && bracketStack.isEmpty()) { - addCurrentEntry(entries, current); - continue; - } - } - current.append(c); + private Object parseToken(String raw, CastExecutor<BinaryString, Object> castExecutor) { + List<TokenSplitter.Token> tokens = TokenSplitter.split(raw); + if (tokens.size() != 1) { + throw new RuntimeException("Invalid map entry format: " + raw); } - - addCurrentEntry(entries, current); - return entries; + return parseValue(tokens.get(0), castExecutor); } - private void addCurrentEntry(List<String> entries, StringBuilder current) { - if (current.length() > 0) { - entries.add(current.toString().trim()); - current.setLength(0); - } + private Object parseValue( + TokenSplitter.Token token, CastExecutor<BinaryString, Object> castExecutor) { + String value = token.value(); + // only an unquoted null is the null key or value; "null" is the four-character string + return !token.literal() && "null".equals(value) + ? null + : castExecutor.cast(BinaryString.fromString(value)); } } diff --git a/paimon-common/src/main/java/org/apache/paimon/casting/TokenSplitter.java b/paimon-common/src/main/java/org/apache/paimon/casting/TokenSplitter.java index 54059675db..c733654722 100644 --- a/paimon-common/src/main/java/org/apache/paimon/casting/TokenSplitter.java +++ b/paimon-common/src/main/java/org/apache/paimon/casting/TokenSplitter.java @@ -25,8 +25,8 @@ import java.util.List; import java.util.Stack; /** - * Splits the comma-separated body of an array or row literal into its tokens, honouring quotes, - * escapes and nesting. + * Splits the comma-separated body of an array, map or row literal into its tokens, honouring + * quotes, escapes and nesting. * * <p>A separator only separates outside quotes and at bracket depth zero, so {@code "a,b"} and * {@code [a, b]} each stay one token. @@ -64,6 +64,64 @@ class TokenSplitter { private TokenSplitter() {} + /** + * Splits {@code content} on {@code delimiter} at bracket depth zero and outside quotes, leaving + * quotes and escapes in place so the caller can split the result again before parsing it. + * + * <p>This is the two-level counterpart of {@link #split(String)}: a map first separates its + * entries on {@code ,} and then each entry on {@code ->}, and the quoting that groups a key or + * value must survive the first split to be understood by the second. + * + * <p>A positive {@code limit} performs at most {@code limit - 1} splits, so {@code 2} cuts at + * the first delimiter and keeps the rest together; a non-positive limit splits everywhere. + */ + static List<String> splitRaw(String content, String delimiter, int limit) { + List<String> tokens = new ArrayList<>(); + StringBuilder current = new StringBuilder(); + Stack<Character> bracketStack = new Stack<>(); + boolean inQuotes = false; + boolean escaped = false; + int splits = 0; + + for (int i = 0; i < content.length(); i++) { + char c = content.charAt(i); + if (escaped) { + escaped = false; + current.append(c); + continue; + } + if (c == '\\') { + escaped = true; + current.append(c); + continue; + } + if (c == '"') { + inQuotes = !inQuotes; + current.append(c); + continue; + } + if (!inQuotes) { + if (StringUtils.isOpenBracket(c)) { + bracketStack.push(c); + } else if (StringUtils.isCloseBracket(c) && !bracketStack.isEmpty()) { + bracketStack.pop(); + } else if (bracketStack.isEmpty() + && (limit <= 0 || splits < limit - 1) + && content.startsWith(delimiter, i)) { + tokens.add(current.toString().trim()); + current.setLength(0); + splits++; + i += delimiter.length() - 1; + continue; + } + } + current.append(c); + } + + tokens.add(current.toString().trim()); + return tokens; + } + static List<Token> split(String content) { List<Token> tokens = new ArrayList<>(); StringBuilder current = new StringBuilder(); diff --git a/paimon-common/src/test/java/org/apache/paimon/casting/CastExecutorTest.java b/paimon-common/src/test/java/org/apache/paimon/casting/CastExecutorTest.java index 8325687c21..bba5c0a30e 100644 --- a/paimon-common/src/test/java/org/apache/paimon/casting/CastExecutorTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/casting/CastExecutorTest.java @@ -53,7 +53,6 @@ import org.junit.jupiter.api.Test; import java.time.LocalDateTime; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.TimeZone; @@ -1145,23 +1144,78 @@ public class CastExecutorTest { } @Test - public void testSplitMapEntriesWithQuotes() { - String content = "1, \"abc\""; - List<String> result = StringToMapCastRule.INSTANCE.splitMapEntries(content); - assertThat(result).containsExactly("1", "abc"); + public void testStringToMapQuotingAndEscaping() { + CastExecutor<?, ?> cast = + CastExecutors.resolve( + VarCharType.STRING_TYPE, + new MapType(DataTypes.STRING(), DataTypes.STRING())); + + // quoting is how an empty key or value is written + Map<Object, Object> emptyKey = new HashMap<>(); + emptyKey.put(BinaryString.fromString(""), BinaryString.fromString("a")); + compareCastResult(cast, BinaryString.fromString("{\"\" -> a}"), new GenericMap(emptyKey)); + Map<Object, Object> emptyValue = new HashMap<>(); + emptyValue.put(BinaryString.fromString("k"), BinaryString.fromString("")); + compareCastResult(cast, BinaryString.fromString("{k -> \"\"}"), new GenericMap(emptyValue)); + + // an unquoted null is SQL NULL; a quoted one is the four-character string + Map<Object, Object> quotedNullValue = new HashMap<>(); + quotedNullValue.put(BinaryString.fromString("k"), BinaryString.fromString("null")); + compareCastResult( + cast, BinaryString.fromString("{k -> \"null\"}"), new GenericMap(quotedNullValue)); + Map<Object, Object> sqlNullValue = new HashMap<>(); + sqlNullValue.put(BinaryString.fromString("k"), null); + compareCastResult( + cast, BinaryString.fromString("{k -> null}"), new GenericMap(sqlNullValue)); + Map<Object, Object> quotedNullKey = new HashMap<>(); + quotedNullKey.put(BinaryString.fromString("null"), BinaryString.fromString("a")); + compareCastResult( + cast, BinaryString.fromString("{\"null\" -> a}"), new GenericMap(quotedNullKey)); + Map<Object, Object> sqlNullKey = new HashMap<>(); + sqlNullKey.put(null, BinaryString.fromString("a")); + compareCastResult(cast, BinaryString.fromString("{null -> a}"), new GenericMap(sqlNullKey)); + + // escaping, like quoting, writes the four-character string rather than SQL NULL + Map<Object, Object> escapedNullValue = new HashMap<>(); + escapedNullValue.put(BinaryString.fromString("k"), BinaryString.fromString("null")); + compareCastResult( + cast, BinaryString.fromString("{k -> \\null}"), new GenericMap(escapedNullValue)); + + // quotes group a token across the arrow and the entry separator + Map<Object, Object> quotedArrow = new HashMap<>(); + quotedArrow.put(BinaryString.fromString("a->b"), BinaryString.fromString("c")); + compareCastResult( + cast, BinaryString.fromString("{\"a->b\" -> c}"), new GenericMap(quotedArrow)); + Map<Object, Object> quotedComma = new HashMap<>(); + quotedComma.put(BinaryString.fromString("a,b"), BinaryString.fromString("c")); + compareCastResult( + cast, BinaryString.fromString("{\"a,b\" -> c}"), new GenericMap(quotedComma)); } @Test - public void testSplitMapEntriesWithEscapes() { - // the escaped separator must survive as a literal instead of vanishing - assertThat(StringToMapCastRule.INSTANCE.splitMapEntries("a\\,b, c")) - .containsExactly("a,b", "c"); - // an escaped backslash yields one literal backslash - assertThat(StringToMapCastRule.INSTANCE.splitMapEntries("x\\\\y, z")) - .containsExactly("x\\y", "z"); - // an escaped quote is a literal and does not toggle quote state - assertThat(StringToMapCastRule.INSTANCE.splitMapEntries("\"q\\\"z, w\"")) - .containsExactly("q\"z, w"); + public void testStringToMapFunctionQuoting() { + CastExecutor<?, ?> cast = + CastExecutors.resolve( + VarCharType.STRING_TYPE, + new MapType(DataTypes.STRING(), DataTypes.STRING())); + + Map<Object, Object> emptyKey = new HashMap<>(); + emptyKey.put(BinaryString.fromString(""), BinaryString.fromString("a")); + compareCastResult(cast, BinaryString.fromString("MAP(\"\", a)"), new GenericMap(emptyKey)); + + Map<Object, Object> emptyValue = new HashMap<>(); + emptyValue.put(BinaryString.fromString("k"), BinaryString.fromString("")); + compareCastResult( + cast, BinaryString.fromString("MAP(k, \"\")"), new GenericMap(emptyValue)); + + Map<Object, Object> quotedNullValue = new HashMap<>(); + quotedNullValue.put(BinaryString.fromString("k"), BinaryString.fromString("null")); + compareCastResult( + cast, BinaryString.fromString("MAP(k, \"null\")"), new GenericMap(quotedNullValue)); + Map<Object, Object> quotedNullKey = new HashMap<>(); + quotedNullKey.put(BinaryString.fromString("null"), BinaryString.fromString("a")); + compareCastResult( + cast, BinaryString.fromString("MAP(\"null\", a)"), new GenericMap(quotedNullKey)); } @Test @@ -1174,6 +1228,15 @@ public class CastExecutorTest { new MapType(DataTypes.STRING(), DataTypes.STRING())), BinaryString.fromString("{a\\,b -> v\\\\1}"), new GenericMap(expected)); + + Map<Object, Object> escapedQuote = new HashMap<>(); + escapedQuote.put(BinaryString.fromString("q\"z, w"), BinaryString.fromString("v")); + compareCastResult( + CastExecutors.resolve( + VarCharType.STRING_TYPE, + new MapType(DataTypes.STRING(), DataTypes.STRING())), + BinaryString.fromString("{\"q\\\"z, w\" -> v}"), + new GenericMap(escapedQuote)); } @SuppressWarnings("rawtypes")
