This is an automated email from the ASF dual-hosted git repository.
lgbo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 0b317a7518 imrove jsonpath support (#7556)
0b317a7518 is described below
commit 0b317a7518b217f000a85048b84bb91c56c4811f
Author: lgbo <[email protected]>
AuthorDate: Thu Oct 17 10:05:10 2024 +0800
imrove jsonpath support (#7556)
---
.../GlutenClickhouseFunctionSuite.scala | 31 ++++++
.../Functions/SparkFunctionGetJsonObject.cpp | 113 +++++++++++++++++++++
.../Functions/SparkFunctionGetJsonObject.h | 93 +++++------------
3 files changed, 167 insertions(+), 70 deletions(-)
diff --git
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/compatibility/GlutenClickhouseFunctionSuite.scala
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/compatibility/GlutenClickhouseFunctionSuite.scala
index 3d7b922e7b..3012be2e39 100644
---
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/compatibility/GlutenClickhouseFunctionSuite.scala
+++
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/compatibility/GlutenClickhouseFunctionSuite.scala
@@ -295,4 +295,35 @@ class GlutenClickhouseFunctionSuite extends
GlutenClickHouseTPCHAbstractSuite {
}
}
+ test("GLUTEN-7552 normalize json path") {
+ withTable("test_7552") {
+ sql("create table test_7552(a string) using parquet")
+ val insert_sql =
+ """
+ |insert into test_7552 values('{\'a\':\'1\'}')
+ |,('{"a":3}')
+ |,('{"3a":4}')
+ |,('{"a c":5}')
+ |,('{"3 d":6"}')
+ |,('{"a:b":7}')
+ |,('{"=a":8}')
+ |""".stripMargin
+ sql(insert_sql)
+ compareResultsAgainstVanillaSpark(
+ """
+ |select a
+ |, get_json_object(a, '$.a')
+ |, get_json_object(a, '$.3a')
+ |, get_json_object(a, '$.a c')
+ |, get_json_object(a, '$.3 d')
+ |, get_json_object(a, '$.a:b')
+ |, get_json_object(a, '$.=a')
+ |from test_7552
+ |""".stripMargin,
+ true,
+ { _ => }
+ )
+ }
+ }
+
}
diff --git a/cpp-ch/local-engine/Functions/SparkFunctionGetJsonObject.cpp
b/cpp-ch/local-engine/Functions/SparkFunctionGetJsonObject.cpp
index a124b178fc..869221ce77 100644
--- a/cpp-ch/local-engine/Functions/SparkFunctionGetJsonObject.cpp
+++ b/cpp-ch/local-engine/Functions/SparkFunctionGetJsonObject.cpp
@@ -21,6 +21,119 @@
namespace local_engine
{
+std::pair<DB::TokenType, StringRef>
JSONPathNormalizer::prevToken(DB::IParser::Pos & iter, size_t n)
+{
+ size_t i = 0;
+ for (; i < n && iter->type != DB::TokenType::DollarSign; ++i)
+ {
+ --iter;
+ }
+ std::pair<DB::TokenType, StringRef> res = {iter->type,
StringRef(iter->begin, iter->end - iter->begin)};
+ for (; i > 0; --i)
+ {
+ ++iter;
+ }
+ return res;
+}
+
+std::pair<DB::TokenType, StringRef>
JSONPathNormalizer::nextToken(DB::IParser::Pos & iter, size_t n)
+{
+ size_t i = 0;
+ for (; i < n && iter->type != DB::TokenType::EndOfStream; ++i)
+ {
+ ++iter;
+ }
+ std::pair<DB::TokenType, StringRef> res = {iter->type,
StringRef(iter->begin, iter->end - iter->begin)};
+ for (; i > 0; --i)
+ {
+ --iter;
+ }
+ return res;
+}
+
+bool JSONPathNormalizer::isSubPathBegin(DB::IParser::Pos & iter)
+{
+ if (iter->type == DB::TokenType::Dot || (iter->type ==
DB::TokenType::Number && *iter->begin == '.'))
+ {
+ return true;
+ }
+ return false;
+}
+
+
+void JSONPathNormalizer::normalizeOnNumber(DB::IParser::Pos & iter, String &
res)
+{
+ if (*iter->begin == '.')
+ {
+ res += ".\"";
+ res += String(iter->begin + 1, iter->end);
+ ++iter;
+ auto token_type = iter->type;
+ while (token_type != DB::TokenType::Dot && token_type !=
DB::TokenType::OpeningSquareBracket
+ && token_type != DB::TokenType::EndOfStream)
+ {
+ auto [_, prev_iter_str] = prevToken(iter);
+ // may contains spaces
+ if (prev_iter_str.data + prev_iter_str.size != iter->begin)
+ {
+ res += String(prev_iter_str.data + prev_iter_str.size,
iter->begin);
+ }
+ res += String(iter->begin, iter->end);
+ ++iter;
+ token_type = iter->type;
+ }
+ auto [_, prev_iter_str] = prevToken(iter);
+ res += String(prev_iter_str.data + prev_iter_str.size, iter->begin);
+ res += "\"";
+ }
+ else
+ normalizeOnOtherTokens(iter, res);
+}
+
+void JSONPathNormalizer::normalizeOnBareWord(DB::IParser::Pos & iter, String &
res)
+{
+ auto [prev_iter_type_2, _] = nextToken(iter);
+ /// e.g. $.data.forecast[?(@.aqi>65)]
+ if (prev_iter_type_2 == DB::TokenType::At)
+ {
+ normalizeOnOtherTokens(iter, res);
+ }
+ else
+ {
+ auto token_type = iter->type;
+ res += "\"";
+ size_t i = 0;
+ while (token_type != DB::TokenType::Dot && token_type !=
DB::TokenType::OpeningSquareBracket
+ && token_type != DB::TokenType::EndOfStream)
+ {
+ if (i)
+ {
+ auto [_, prev_iter_str] = prevToken(iter);
+ // may contains spaces
+ if (prev_iter_str.data + prev_iter_str.size != iter->begin)
+ {
+ res += String(prev_iter_str.data + prev_iter_str.size,
iter->begin);
+ }
+ }
+ res += String(iter->begin, iter->end);
+ ++iter;
+ i += 1;
+ token_type = iter->type;
+ }
+ auto [_, prev_iter_str] = prevToken(iter);
+ res += String(prev_iter_str.data + prev_iter_str.size, iter->begin);
+ res += "\"";
+ }
+}
+
+
+void JSONPathNormalizer::normalizeOnOtherTokens(DB::IParser::Pos & iter,
String & res)
+{
+ res += String(iter->begin, iter->end);
+ ++iter;
+}
+
+
REGISTER_FUNCTION(GetJsonObject)
{
factory.registerFunction<DB::FunctionSQLJSON<GetJsonObject,
GetJsonObjectImpl>>();
diff --git a/cpp-ch/local-engine/Functions/SparkFunctionGetJsonObject.h
b/cpp-ch/local-engine/Functions/SparkFunctionGetJsonObject.h
index b174cb4c7f..6ffe096b67 100644
--- a/cpp-ch/local-engine/Functions/SparkFunctionGetJsonObject.h
+++ b/cpp-ch/local-engine/Functions/SparkFunctionGetJsonObject.h
@@ -481,6 +481,7 @@ public:
}
};
+/// CH uses the lexer to parse the json path, it's not a good idea.
/// If a json field containt spaces, we wrap it by double quotes.
/// FIXME: If it contains \t, \n, simdjson cannot parse.
class JSONPathNormalizer
@@ -489,89 +490,41 @@ public:
static String normalize(const String & json_path_)
{
DB::Tokens tokens(json_path_.data(), json_path_.data() +
json_path_.size());
- DB::IParser::Pos pos(tokens, 0, 0);
+ DB::IParser::Pos iter(tokens, 0, 0);
String res;
- while (pos->type != DB::TokenType::EndOfStream)
+ while (iter->type != DB::TokenType::EndOfStream)
{
- if (pos->type == DB::TokenType::Number)
+ if (isSubPathBegin(iter))
{
- ++pos;
- // Two tokens are seperated by white spaces.
- if (pos->type == DB::TokenType::Number || pos->type ==
DB::TokenType::BareWord)
+ if (iter->type == DB::TokenType::Number)
{
- --pos;
- if (*pos->begin == '.')
- res += ".";
- ++pos;
- res += "\"";
-
- while (pos->type == DB::TokenType::Number || pos->type ==
DB::TokenType::BareWord)
- {
- --pos;
- const auto * last_end = pos->end;
- const auto * begin = *pos->begin == '.' ? pos->begin +
1 : pos->begin;
- res += String(begin, pos->end);
- ++pos;
- res += String(last_end, pos->begin);
- ++pos;
- }
- --pos;
- const auto * last_end = pos->end;
- res += String(pos->begin, pos->end);
- ++pos;
- res += String(last_end, pos->begin);
- res += "\"";
- }
- else if (
- pos->type == DB::TokenType::Dot || pos->type ==
DB::TokenType::OpeningSquareBracket
- || pos->type == DB::TokenType::EndOfStream)
- {
- --pos;
- if (*pos->begin == '.')
- res += ".";
- res += "\"";
- const auto * last_end = pos->end;
- const auto * begin = *pos->begin == '.' ? pos->begin + 1 :
pos->begin;
- res += String(begin, pos->end);
- ++pos;
- res += String(last_end, pos->begin);
- res += "\"";
+ normalizeOnNumber(iter, res);
}
else
{
- --pos;
- res += String(pos->begin, pos->end);
- ++pos;
- }
- }
- else if (pos->type == DB::TokenType::BareWord)
- {
- res += "\"";
- ++pos;
- while (pos->type == DB::TokenType::Number || pos->type ==
DB::TokenType::BareWord)
- {
- --pos;
- const auto * last_end = pos->end;
- res += String(pos->begin, pos->end);
- ++pos;
- res += String(last_end, pos->begin);
- ++pos;
+ // It may begins with '=', '==' and so on.
+ res += ".";
+ ++iter;
+ normalizeOnBareWord(iter, res);
}
- --pos;
- const auto * last_end = pos->end;
- res += String(pos->begin, pos->end);
- ++pos;
- res += String(last_end, pos->begin);
- res += "\"";
}
else
- {
- res += String(pos->begin, pos->end);
- ++pos;
- }
+ normalizeOnOtherTokens(iter, res);
}
return res;
}
+
+private:
+ static std::pair<DB::TokenType, StringRef> prevToken(DB::IParser::Pos &
iter, size_t n = 1);
+
+ static std::pair<DB::TokenType, StringRef> nextToken(DB::IParser::Pos &
iter, size_t n = 1);
+
+ static bool isSubPathBegin(DB::IParser::Pos & iter);
+
+ static void normalizeOnNumber(DB::IParser::Pos & iter, String & res);
+
+ static void normalizeOnBareWord(DB::IParser::Pos & iter, String & res);
+ static void normalizeOnOtherTokens(DB::IParser::Pos & iter, String & res);
};
/// Flatten a json string into a tuple.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
