This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 3af4137f13 [core] Preserve quoted empty and null keys/values in
string-to-MAP cast (#9849)
3af4137f13 is described below
commit 3af4137f135355f39e30d85ba16dc97a6a46d25b
Author: 玉升 <[email protected]>
AuthorDate: Thu Sep 17 14:43:36 2026 +0800
[core] Preserve quoted empty and null keys/values in string-to-MAP cast
(#9849)
---
.../apache/paimon/casting/StringToMapCastRule.java | 81 ++++++-------------
.../org/apache/paimon/casting/TokenSplitter.java | 62 ++++++++++++++-
.../apache/paimon/casting/CastExecutorTest.java | 93 ++++++++++++++++++----
3 files changed, 161 insertions(+), 75 deletions(-)
diff --git
a/paimon-common/src/main/java/org/apache/paimon/casting/StringToMapCastRule.java
b/paimon-common/src/main/java/org/apache/paimon/casting/StringToMapCastRule.java
index d700e9c6d8..f336b790e6 100644
---
a/paimon-common/src/main/java/org/apache/paimon/casting/StringToMapCastRule.java
+++
b/paimon-common/src/main/java/org/apache/paimon/casting/StringToMapCastRule.java
@@ -26,15 +26,12 @@ import org.apache.paimon.types.DataTypeFamily;
import org.apache.paimon.types.DataTypeRoot;
import org.apache.paimon.types.MapType;
import org.apache.paimon.types.VarCharType;
-import org.apache.paimon.utils.StringUtils;
import org.apache.paimon.shade.guava30.com.google.common.collect.Maps;
-import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Stack;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -50,8 +47,6 @@ class StringToMapCastRule extends
AbstractCastRule<BinaryString, InternalMap> {
private static final Pattern FUNCTION_MAP_PATTERN =
Pattern.compile("^\\s*MAP\\s*\\((.*)\\)\\s*$",
Pattern.CASE_INSENSITIVE);
- private static final Pattern ENTRY_PATTERN =
Pattern.compile("(.+?)\\s*->\\s*(.+)");
-
private StringToMapCastRule() {
super(
CastRulePredicate.builder()
@@ -134,7 +129,7 @@ class StringToMapCastRule extends
AbstractCastRule<BinaryString, InternalMap> {
CastExecutor<BinaryString, Object> keyCastExecutor,
CastExecutor<BinaryString, Object> valueCastExecutor) {
- List<String> elements = splitMapEntries(content.trim());
+ List<TokenSplitter.Token> elements = TokenSplitter.split(content);
if (elements.size() % 2 != 0) {
throw new RuntimeException("Invalid Function map format: odd
number of elements");
}
@@ -142,8 +137,8 @@ class StringToMapCastRule extends
AbstractCastRule<BinaryString, InternalMap> {
Map<Object, Object> mapContent = Maps.newHashMap();
for (int i = 0; i < elements.size(); i += 2) {
mapContent.put(
- parseValue(elements.get(i).trim(), keyCastExecutor),
- parseValue(elements.get(i + 1).trim(), valueCastExecutor));
+ parseValue(elements.get(i), keyCastExecutor),
+ parseValue(elements.get(i + 1), valueCastExecutor));
}
return mapContent;
}
@@ -154,65 +149,35 @@ class StringToMapCastRule extends
AbstractCastRule<BinaryString, InternalMap> {
CastExecutor<BinaryString, Object> valueCastExecutor) {
Map<Object, Object> mapContent = Maps.newHashMap();
- for (String entry : splitMapEntries(content)) {
- Matcher entryMatcher = ENTRY_PATTERN.matcher(entry);
- if (!entryMatcher.matches()) {
+ for (String entry : TokenSplitter.splitRaw(content, ",", 0)) {
+ if (entry.isEmpty()) {
+ continue;
+ }
+ List<String> keyValue = TokenSplitter.splitRaw(entry, "->", 2);
+ if (keyValue.size() != 2) {
throw new RuntimeException("Invalid map entry format: " +
entry);
}
mapContent.put(
- parseValue(entryMatcher.group(1).trim(), keyCastExecutor),
- parseValue(entryMatcher.group(2).trim(),
valueCastExecutor));
+ parseToken(keyValue.get(0), keyCastExecutor),
+ parseToken(keyValue.get(1), valueCastExecutor));
}
return mapContent;
}
- private Object parseValue(String valueStr, CastExecutor<BinaryString,
Object> castExecutor) {
- return "null".equals(valueStr)
- ? null
- : castExecutor.cast(BinaryString.fromString(valueStr));
- }
-
- public List<String> splitMapEntries(String content) {
- List<String> entries = new ArrayList<>();
- StringBuilder current = new StringBuilder();
- Stack<Character> bracketStack = new Stack<>();
- boolean inQuotes = false;
- boolean escaped = false;
-
- for (char c : content.toCharArray()) {
- if (escaped) {
- // an escaped character is taken literally: it must survive in
the
- // entry instead of being dropped together with its backslash
- escaped = false;
- current.append(c);
- continue;
- } else if (c == '\\') {
- escaped = true;
- continue;
- } else if (c == '"') {
- inQuotes = !inQuotes;
- continue;
- } else if (!inQuotes) {
- if (StringUtils.isOpenBracket(c)) {
- bracketStack.push(c);
- } else if (StringUtils.isCloseBracket(c) &&
!bracketStack.isEmpty()) {
- bracketStack.pop();
- } else if (c == ',' && bracketStack.isEmpty()) {
- addCurrentEntry(entries, current);
- continue;
- }
- }
- current.append(c);
+ private Object parseToken(String raw, CastExecutor<BinaryString, Object>
castExecutor) {
+ List<TokenSplitter.Token> tokens = TokenSplitter.split(raw);
+ if (tokens.size() != 1) {
+ throw new RuntimeException("Invalid map entry format: " + raw);
}
-
- addCurrentEntry(entries, current);
- return entries;
+ return parseValue(tokens.get(0), castExecutor);
}
- private void addCurrentEntry(List<String> entries, StringBuilder current) {
- if (current.length() > 0) {
- entries.add(current.toString().trim());
- current.setLength(0);
- }
+ private Object parseValue(
+ TokenSplitter.Token token, CastExecutor<BinaryString, Object>
castExecutor) {
+ String value = token.value();
+ // only an unquoted null is the null key or value; "null" is the
four-character string
+ return !token.literal() && "null".equals(value)
+ ? null
+ : castExecutor.cast(BinaryString.fromString(value));
}
}
diff --git
a/paimon-common/src/main/java/org/apache/paimon/casting/TokenSplitter.java
b/paimon-common/src/main/java/org/apache/paimon/casting/TokenSplitter.java
index 54059675db..c733654722 100644
--- a/paimon-common/src/main/java/org/apache/paimon/casting/TokenSplitter.java
+++ b/paimon-common/src/main/java/org/apache/paimon/casting/TokenSplitter.java
@@ -25,8 +25,8 @@ import java.util.List;
import java.util.Stack;
/**
- * Splits the comma-separated body of an array or row literal into its tokens,
honouring quotes,
- * escapes and nesting.
+ * Splits the comma-separated body of an array, map or row literal into its
tokens, honouring
+ * quotes, escapes and nesting.
*
* <p>A separator only separates outside quotes and at bracket depth zero, so
{@code "a,b"} and
* {@code [a, b]} each stay one token.
@@ -64,6 +64,64 @@ class TokenSplitter {
private TokenSplitter() {}
+ /**
+ * Splits {@code content} on {@code delimiter} at bracket depth zero and
outside quotes, leaving
+ * quotes and escapes in place so the caller can split the result again
before parsing it.
+ *
+ * <p>This is the two-level counterpart of {@link #split(String)}: a map
first separates its
+ * entries on {@code ,} and then each entry on {@code ->}, and the quoting
that groups a key or
+ * value must survive the first split to be understood by the second.
+ *
+ * <p>A positive {@code limit} performs at most {@code limit - 1} splits,
so {@code 2} cuts at
+ * the first delimiter and keeps the rest together; a non-positive limit
splits everywhere.
+ */
+ static List<String> splitRaw(String content, String delimiter, int limit) {
+ List<String> tokens = new ArrayList<>();
+ StringBuilder current = new StringBuilder();
+ Stack<Character> bracketStack = new Stack<>();
+ boolean inQuotes = false;
+ boolean escaped = false;
+ int splits = 0;
+
+ for (int i = 0; i < content.length(); i++) {
+ char c = content.charAt(i);
+ if (escaped) {
+ escaped = false;
+ current.append(c);
+ continue;
+ }
+ if (c == '\\') {
+ escaped = true;
+ current.append(c);
+ continue;
+ }
+ if (c == '"') {
+ inQuotes = !inQuotes;
+ current.append(c);
+ continue;
+ }
+ if (!inQuotes) {
+ if (StringUtils.isOpenBracket(c)) {
+ bracketStack.push(c);
+ } else if (StringUtils.isCloseBracket(c) &&
!bracketStack.isEmpty()) {
+ bracketStack.pop();
+ } else if (bracketStack.isEmpty()
+ && (limit <= 0 || splits < limit - 1)
+ && content.startsWith(delimiter, i)) {
+ tokens.add(current.toString().trim());
+ current.setLength(0);
+ splits++;
+ i += delimiter.length() - 1;
+ continue;
+ }
+ }
+ current.append(c);
+ }
+
+ tokens.add(current.toString().trim());
+ return tokens;
+ }
+
static List<Token> split(String content) {
List<Token> tokens = new ArrayList<>();
StringBuilder current = new StringBuilder();
diff --git
a/paimon-common/src/test/java/org/apache/paimon/casting/CastExecutorTest.java
b/paimon-common/src/test/java/org/apache/paimon/casting/CastExecutorTest.java
index 8325687c21..bba5c0a30e 100644
---
a/paimon-common/src/test/java/org/apache/paimon/casting/CastExecutorTest.java
+++
b/paimon-common/src/test/java/org/apache/paimon/casting/CastExecutorTest.java
@@ -53,7 +53,6 @@ import org.junit.jupiter.api.Test;
import java.time.LocalDateTime;
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
import java.util.TimeZone;
@@ -1145,23 +1144,78 @@ public class CastExecutorTest {
}
@Test
- public void testSplitMapEntriesWithQuotes() {
- String content = "1, \"abc\"";
- List<String> result =
StringToMapCastRule.INSTANCE.splitMapEntries(content);
- assertThat(result).containsExactly("1", "abc");
+ public void testStringToMapQuotingAndEscaping() {
+ CastExecutor<?, ?> cast =
+ CastExecutors.resolve(
+ VarCharType.STRING_TYPE,
+ new MapType(DataTypes.STRING(), DataTypes.STRING()));
+
+ // quoting is how an empty key or value is written
+ Map<Object, Object> emptyKey = new HashMap<>();
+ emptyKey.put(BinaryString.fromString(""),
BinaryString.fromString("a"));
+ compareCastResult(cast, BinaryString.fromString("{\"\" -> a}"), new
GenericMap(emptyKey));
+ Map<Object, Object> emptyValue = new HashMap<>();
+ emptyValue.put(BinaryString.fromString("k"),
BinaryString.fromString(""));
+ compareCastResult(cast, BinaryString.fromString("{k -> \"\"}"), new
GenericMap(emptyValue));
+
+ // an unquoted null is SQL NULL; a quoted one is the four-character
string
+ Map<Object, Object> quotedNullValue = new HashMap<>();
+ quotedNullValue.put(BinaryString.fromString("k"),
BinaryString.fromString("null"));
+ compareCastResult(
+ cast, BinaryString.fromString("{k -> \"null\"}"), new
GenericMap(quotedNullValue));
+ Map<Object, Object> sqlNullValue = new HashMap<>();
+ sqlNullValue.put(BinaryString.fromString("k"), null);
+ compareCastResult(
+ cast, BinaryString.fromString("{k -> null}"), new
GenericMap(sqlNullValue));
+ Map<Object, Object> quotedNullKey = new HashMap<>();
+ quotedNullKey.put(BinaryString.fromString("null"),
BinaryString.fromString("a"));
+ compareCastResult(
+ cast, BinaryString.fromString("{\"null\" -> a}"), new
GenericMap(quotedNullKey));
+ Map<Object, Object> sqlNullKey = new HashMap<>();
+ sqlNullKey.put(null, BinaryString.fromString("a"));
+ compareCastResult(cast, BinaryString.fromString("{null -> a}"), new
GenericMap(sqlNullKey));
+
+ // escaping, like quoting, writes the four-character string rather
than SQL NULL
+ Map<Object, Object> escapedNullValue = new HashMap<>();
+ escapedNullValue.put(BinaryString.fromString("k"),
BinaryString.fromString("null"));
+ compareCastResult(
+ cast, BinaryString.fromString("{k -> \\null}"), new
GenericMap(escapedNullValue));
+
+ // quotes group a token across the arrow and the entry separator
+ Map<Object, Object> quotedArrow = new HashMap<>();
+ quotedArrow.put(BinaryString.fromString("a->b"),
BinaryString.fromString("c"));
+ compareCastResult(
+ cast, BinaryString.fromString("{\"a->b\" -> c}"), new
GenericMap(quotedArrow));
+ Map<Object, Object> quotedComma = new HashMap<>();
+ quotedComma.put(BinaryString.fromString("a,b"),
BinaryString.fromString("c"));
+ compareCastResult(
+ cast, BinaryString.fromString("{\"a,b\" -> c}"), new
GenericMap(quotedComma));
}
@Test
- public void testSplitMapEntriesWithEscapes() {
- // the escaped separator must survive as a literal instead of vanishing
- assertThat(StringToMapCastRule.INSTANCE.splitMapEntries("a\\,b, c"))
- .containsExactly("a,b", "c");
- // an escaped backslash yields one literal backslash
- assertThat(StringToMapCastRule.INSTANCE.splitMapEntries("x\\\\y, z"))
- .containsExactly("x\\y", "z");
- // an escaped quote is a literal and does not toggle quote state
- assertThat(StringToMapCastRule.INSTANCE.splitMapEntries("\"q\\\"z,
w\""))
- .containsExactly("q\"z, w");
+ public void testStringToMapFunctionQuoting() {
+ CastExecutor<?, ?> cast =
+ CastExecutors.resolve(
+ VarCharType.STRING_TYPE,
+ new MapType(DataTypes.STRING(), DataTypes.STRING()));
+
+ Map<Object, Object> emptyKey = new HashMap<>();
+ emptyKey.put(BinaryString.fromString(""),
BinaryString.fromString("a"));
+ compareCastResult(cast, BinaryString.fromString("MAP(\"\", a)"), new
GenericMap(emptyKey));
+
+ Map<Object, Object> emptyValue = new HashMap<>();
+ emptyValue.put(BinaryString.fromString("k"),
BinaryString.fromString(""));
+ compareCastResult(
+ cast, BinaryString.fromString("MAP(k, \"\")"), new
GenericMap(emptyValue));
+
+ Map<Object, Object> quotedNullValue = new HashMap<>();
+ quotedNullValue.put(BinaryString.fromString("k"),
BinaryString.fromString("null"));
+ compareCastResult(
+ cast, BinaryString.fromString("MAP(k, \"null\")"), new
GenericMap(quotedNullValue));
+ Map<Object, Object> quotedNullKey = new HashMap<>();
+ quotedNullKey.put(BinaryString.fromString("null"),
BinaryString.fromString("a"));
+ compareCastResult(
+ cast, BinaryString.fromString("MAP(\"null\", a)"), new
GenericMap(quotedNullKey));
}
@Test
@@ -1174,6 +1228,15 @@ public class CastExecutorTest {
new MapType(DataTypes.STRING(), DataTypes.STRING())),
BinaryString.fromString("{a\\,b -> v\\\\1}"),
new GenericMap(expected));
+
+ Map<Object, Object> escapedQuote = new HashMap<>();
+ escapedQuote.put(BinaryString.fromString("q\"z, w"),
BinaryString.fromString("v"));
+ compareCastResult(
+ CastExecutors.resolve(
+ VarCharType.STRING_TYPE,
+ new MapType(DataTypes.STRING(), DataTypes.STRING())),
+ BinaryString.fromString("{\"q\\\"z, w\" -> v}"),
+ new GenericMap(escapedQuote));
}
@SuppressWarnings("rawtypes")
