anthonylouisbsb commented on code in PR #13497:
URL: https://github.com/apache/arrow/pull/13497#discussion_r914703374
##########
cpp/src/gandiva/gdv_function_stubs_test.cc:
##########
@@ -453,6 +453,90 @@ TEST(TestGdvFnStubs, TestCastVARCHARFromDouble) {
EXPECT_FALSE(ctx.has_error());
}
+TEST(TestGdvFnStubs, TestEncode) {
+ gandiva::ExecutionContext ctx;
+ int64_t ctx_ptr = reinterpret_cast<int64_t>(&ctx);
+ int32_t out_len = 0;
+
+ const char* out_str = gdv_fn_encode(ctx_ptr, "AbcDEfGh", 8, "UTF-16BE", 8 ,
&out_len);
+ ARROW_LOG(INFO) << " output: " << std::string(out_str, out_len) << " " <<
out_len;
+ EXPECT_EQ(std::string(out_str, out_len),
+ "00410062006300440045006600470068");
+ EXPECT_FALSE(ctx.has_error());
+
+ out_str = gdv_fn_encode(ctx_ptr, "asdfj", 5, "UTF-16BE", 8, &out_len);
+ ARROW_LOG(INFO) << " output: " << std::string(out_str, out_len) << " " <<
out_len;
+ EXPECT_EQ(std::string(out_str, out_len),
+ "0061007300640066006A");
+ EXPECT_FALSE(ctx.has_error());
+
+ out_str = gdv_fn_encode(ctx_ptr, "s;dcGS,jO!l", 11, "UTF-16BE", 8, &out_len);
+ ARROW_LOG(INFO) << " output: " << std::string(out_str, out_len) << " " <<
out_len;
+ EXPECT_EQ(std::string(out_str, out_len),
+ "0073003B0064006300470053002C006A004F0021006C");
+ EXPECT_FALSE(ctx.has_error());
+
+ out_str = gdv_fn_encode(ctx_ptr, "AbcDEfGh", 8, "UTF-16LE", 8, &out_len);
+ ARROW_LOG(INFO) << " output: " << std::string(out_str, out_len) << " " <<
out_len;
+ EXPECT_EQ(std::string(out_str, out_len),
+ "41006200630044004500660047006800");
+ EXPECT_FALSE(ctx.has_error());
+
+ out_str = gdv_fn_encode(ctx_ptr, "asdfj", 5,"UTF-16LE", 8, &out_len);
+ ARROW_LOG(INFO) << " output: " << std::string(out_str, out_len) << " " <<
out_len;
+ EXPECT_EQ(std::string(out_str, out_len),
+ "61007300640066006A00");
+ EXPECT_FALSE(ctx.has_error());
+
+ out_str = gdv_fn_encode(ctx_ptr, "s;dcGS,jO!l", 11, "UTF-16LE", 8,&out_len);
+ ARROW_LOG(INFO) << " output: " << std::string(out_str, out_len) << " " <<
out_len;
+ EXPECT_EQ(std::string(out_str, out_len),
+ "73003B0064006300470053002C006A004F0021006C00");
+ EXPECT_FALSE(ctx.has_error());
+}
+
+TEST(TestGdvFnStubs, TestDecode) {
+ gandiva::ExecutionContext ctx;
+ int64_t ctx_ptr = reinterpret_cast<int64_t>(&ctx);
+ int32_t out_len = 0;
+
+ const char* out_str = gdv_fn_decode(ctx_ptr,
"00410062006300440045006600470068",
+ 32, "UTF-16BE", 8, &out_len);
+ ARROW_LOG(INFO) << " output: " << std::string(out_str, out_len) << " " <<
out_len;
+ EXPECT_EQ(std::string(out_str, out_len), "AbcDEfGh");
+ EXPECT_FALSE(ctx.has_error());
+
+ out_str = gdv_fn_decode(ctx_ptr, "0061007300640066006A", 20, "UTF-16BE",
+ 8, &out_len);
Review Comment:
Just to cover that case, add inside these tests one test using an invalid
character. In the `upper` and `lower` tests I think you can find some examples
##########
cpp/src/gandiva/gdv_string_function_stubs.cc:
##########
@@ -264,6 +264,298 @@ const char* gdv_fn_lower_utf8(int64_t context, const
char* data, int32_t data_le
return out;
}
+// converts utf8 to utf16 encoding
+GANDIVA_EXPORT
+const char* gdv_fn_encode(int64_t context, const char* data, int32_t data_len,
+ const char* charset, int32_t charset_len, int32_t*
out_len) {
+ if (data_len == 0) {
+ *out_len = 0;
+ return "";
+ }
+
+ char* out =
+ reinterpret_cast<char*>(gdv_fn_context_arena_malloc(context, 32 *
data_len));
+ if (out == nullptr) {
+ gdv_fn_context_set_error_msg(context, "Could not allocate memory for
output string");
+ *out_len = 0;
+ return "";
+ }
+
+ int32_t char_len;
+ uint32_t char_codepoint;
+
+ if (memcmp(charset, "UTF-16BE", 8) == 0 || memcmp(charset, "utf-16be", 8) ==
0) {
Review Comment:
I think that case can still lead to incorrect behavior. Like if the user
passes Utf8-BE. I think we can create a string and uses boost::iequals
```cpp
std::string charset_str (charset, charset_len)
if (boost::iequals(charset_str, "UTF16BE){
// logic here
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
