SG011 commented on code in PR #13497:
URL: https://github.com/apache/arrow/pull/13497#discussion_r921072662
##########
cpp/src/gandiva/gdv_function_stubs_test.cc:
##########
@@ -453,6 +453,81 @@ TEST(TestGdvFnStubs, TestCastVARCHARFromDouble) {
EXPECT_FALSE(ctx.has_error());
}
+TEST(TestGdvFnStubs, TestEncode) {
+ gandiva::ExecutionContext ctx;
+ int64_t ctx_ptr = reinterpret_cast<int64_t>(&ctx);
+ int32_t out_len = 0;
+
+ const char* out_str = gdv_fn_encode(ctx_ptr, "AbcDEfGh", 8, "UTF-16BE", 8,
&out_len);
+ EXPECT_EQ(std::string(out_str, out_len), "00410062006300440045006600470068");
+ EXPECT_FALSE(ctx.has_error());
+
+ out_str = gdv_fn_encode(ctx_ptr, "asdfj", 5, "UTF-16BE", 8, &out_len);
+ EXPECT_EQ(std::string(out_str, out_len), "0061007300640066006A");
+ EXPECT_FALSE(ctx.has_error());
+
+ out_str = gdv_fn_encode(ctx_ptr, "s;dcGS,jO!l", 11, "UTF-16BE", 8, &out_len);
+ EXPECT_EQ(std::string(out_str, out_len),
+ "0073003B0064006300470053002C006A004F0021006C");
+ EXPECT_FALSE(ctx.has_error());
+
+ out_str = gdv_fn_encode(ctx_ptr, "AbcDEfGh", 8, "UTF-16LE", 8, &out_len);
+ EXPECT_EQ(std::string(out_str, out_len), "41006200630044004500660047006800");
+ EXPECT_FALSE(ctx.has_error());
+
+ out_str = gdv_fn_encode(ctx_ptr, "asdfj", 5, "UTF-16LE", 8, &out_len);
+ EXPECT_EQ(std::string(out_str, out_len), "61007300640066006A00");
+ EXPECT_FALSE(ctx.has_error());
+
+ out_str = gdv_fn_encode(ctx_ptr, "s;dcGS,jO!l", 11, "UTF-16LE", 8, &out_len);
+ EXPECT_EQ(std::string(out_str, out_len),
+ "73003B0064006300470053002C006A004F0021006C00");
+ EXPECT_FALSE(ctx.has_error());
+
+ std::string d("AbOJjÜoß\xc3");
+ out_str = gdv_fn_encode(ctx_ptr, d.data(), static_cast<int>(d.length()),
"UTF-16LE", 8,
+ &out_len);
+ EXPECT_EQ(std::string(out_str, out_len), "");
+ EXPECT_THAT(ctx.get_error(),
+ ::testing::HasSubstr(
+ "unexpected byte \\c3 encountered while decoding utf8
string"));
+ ctx.Reset();
+}
+
+TEST(TestGdvFnStubs, TestDecode) {
+ gandiva::ExecutionContext ctx;
+ int64_t ctx_ptr = reinterpret_cast<int64_t>(&ctx);
+ int32_t out_len = 0;
+
+ const char* out_str = gdv_fn_decode(ctx_ptr,
"00410062006300440045006600470068", 32,
+ "UTF-16BE", 8, &out_len);
+ EXPECT_EQ(std::string(out_str, out_len), "AbcDEfGh");
+ EXPECT_FALSE(ctx.has_error());
+
+ out_str = gdv_fn_decode(ctx_ptr, "0061007300640066006A", 20, "UTF-16BE", 8,
&out_len);
+ EXPECT_EQ(std::string(out_str, out_len), "asdfj");
+ EXPECT_FALSE(ctx.has_error());
+
+ out_str = gdv_fn_decode(ctx_ptr,
"0073003B0064006300470053002C006A004F0021006C", 44,
+ "UTF-16BE", 8, &out_len);
+ EXPECT_EQ(std::string(out_str, out_len), "s;dcGS,jO!l");
+ EXPECT_FALSE(ctx.has_error());
+
+ out_str = gdv_fn_decode(ctx_ptr, "41006200630044004500660047006800", 32,
"UTF-16LE", 8,
+ &out_len);
+ EXPECT_EQ(std::string(out_str, out_len), "AbcDEfGh");
+ EXPECT_FALSE(ctx.has_error());
+
+ out_str = gdv_fn_decode(ctx_ptr, "61007300640066006A00", 20, "UTF-16LE", 8,
&out_len);
+ EXPECT_EQ(std::string(out_str, out_len), "asdfj");
+ EXPECT_FALSE(ctx.has_error());
+
+ out_str = gdv_fn_decode(ctx_ptr,
"73003B0064006300470053002C006A004F0021006C00", 44,
+ "UTF-16LE", 8, &out_len);
+ EXPECT_EQ(std::string(out_str, out_len), "s;dcGS,jO!l");
+ EXPECT_FALSE(ctx.has_error());
+}
Review Comment:
I'll add those cases also. I think codepoints larger than 65535 don't get
decoded.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
