This is an automated email from the ASF dual-hosted git repository.
pitrou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new 374db363476 GH-50615: [C++] Reduce code generation for string kernels
(#50616)
374db363476 is described below
commit 374db36347664a38d074d26d28d81ddf1fbd7593
Author: Antoine Pitrou <[email protected]>
AuthorDate: Mon Jul 27 11:43:36 2026 +0200
GH-50615: [C++] Reduce code generation for string kernels (#50616)
### Rationale for this change
Some string kernels are generated twice for the corresponding binary and
string type (for example binary/utf8, large_binary, large_utf8). Most of the
time we can cut down on the code duplication to reduce compile times and binary
size slightly.
### What changes are included in this PR?
Only instantiate string kernels for base binary types, not the string
derived types.
This makes `libarrow_compute` somehow smaller on my system:
* before:
```console
$ size `find /build/ -name libarrow_compute.so.2600.0.0`
text data bss dec hex filename
13551567 113096 46432 13711095 d136f7
/build/build-release/relwithdebinfo/libarrow_compute.so.2600.0.0
```
* after:
```console
$ size `find /build/ -name libarrow_compute.so.2600.0.0`
text data bss dec hex filename
13363474 112128 44968 13520570 ce4eba
/build/build-release/relwithdebinfo/libarrow_compute.so.2600.0.0
```
### Are these changes tested?
Yes, including additional tests to check that utf8-ness of the input is
still propagated correctly.
### Are there any user-facing changes?
No.
* GitHub Issue: #50615
Authored-by: Antoine Pitrou <[email protected]>
Signed-off-by: Antoine Pitrou <[email protected]>
---
cpp/src/arrow/compute/kernels/codegen_internal.h | 48 +---
cpp/src/arrow/compute/kernels/scalar_compare.cc | 3 +-
cpp/src/arrow/compute/kernels/scalar_if_else.cc | 2 +-
.../arrow/compute/kernels/scalar_string_ascii.cc | 248 ++++++++++++---------
.../arrow/compute/kernels/scalar_string_internal.h | 45 ++--
.../arrow/compute/kernels/scalar_string_test.cc | 164 +++++++++++++-
.../arrow/compute/kernels/scalar_string_utf8.cc | 11 +-
cpp/src/arrow/compute/kernels/vector_array_sort.cc | 3 +-
cpp/src/arrow/compute/kernels/vector_replace.cc | 9 +-
9 files changed, 357 insertions(+), 176 deletions(-)
diff --git a/cpp/src/arrow/compute/kernels/codegen_internal.h
b/cpp/src/arrow/compute/kernels/codegen_internal.h
index 9d12d62d95f..fa89c86183f 100644
--- a/cpp/src/arrow/compute/kernels/codegen_internal.h
+++ b/cpp/src/arrow/compute/kernels/codegen_internal.h
@@ -1314,10 +1314,14 @@ KernelType
GenerateTypeAgnosticPrimitive(detail::GetTypeId get_id) {
}
}
-// similar to GenerateTypeAgnosticPrimitive, but for base variable binary types
-template <template <typename...> class Generator, typename KernelType =
ArrayKernelExec,
- typename... Args>
-KernelType GenerateTypeAgnosticVarBinaryBase(detail::GetTypeId get_id) {
+// Similar to GenerateTypeAgnosticPrimitive, but for base variable binary types
+//
+// Note that we don't offer to generate separate code for String types, because
+// the utf8-ness of a type can be retrieved and handled efficiently at runtime.
+// This helps cut down on code generation (see GH-50615).
+template <template <typename...> class Generator, typename... Args>
+auto GenerateTypeAgnosticVarBinaryBase(detail::GetTypeId get_id) {
+ using KernelType = decltype(&Generator<BinaryType, Args...>::Exec);
switch (get_id.id) {
case Type::BINARY:
case Type::STRING:
@@ -1331,24 +1335,6 @@ KernelType
GenerateTypeAgnosticVarBinaryBase(detail::GetTypeId get_id) {
}
}
-// Generate a kernel given a templated functor for binary and string types
-template <template <typename...> class Generator, typename... Args>
-ArrayKernelExec GenerateVarBinaryToVarBinary(detail::GetTypeId get_id) {
- switch (get_id.id) {
- case Type::BINARY:
- return Generator<BinaryType, Args...>::Exec;
- case Type::STRING:
- return Generator<StringType, Args...>::Exec;
- case Type::LARGE_BINARY:
- return Generator<LargeBinaryType, Args...>::Exec;
- case Type::LARGE_STRING:
- return Generator<LargeStringType, Args...>::Exec;
- default:
- ARROW_DCHECK(false);
- return nullptr;
- }
-}
-
// Generate a kernel given a templated functor for base binary types. Generates
// a single kernel for binary/string and large binary/large string. If your
kernel
// implementation needs access to the specific type at compile time, please use
@@ -1370,24 +1356,6 @@ ArrayKernelExec GenerateVarBinaryBase(detail::GetTypeId
get_id) {
}
}
-// See BaseBinary documentation
-template <template <typename...> class Generator, typename Type0, typename...
Args>
-ArrayKernelExec GenerateVarBinary(detail::GetTypeId get_id) {
- switch (get_id.id) {
- case Type::BINARY:
- return Generator<Type0, BinaryType, Args...>::Exec;
- case Type::STRING:
- return Generator<Type0, StringType, Args...>::Exec;
- case Type::LARGE_BINARY:
- return Generator<Type0, LargeBinaryType, Args...>::Exec;
- case Type::LARGE_STRING:
- return Generator<Type0, LargeStringType, Args...>::Exec;
- default:
- ARROW_DCHECK(false);
- return nullptr;
- }
-}
-
// Generate a kernel given a templated functor for binary-view types.
Generates a
// single kernel for binary/string-view.
//
diff --git a/cpp/src/arrow/compute/kernels/scalar_compare.cc
b/cpp/src/arrow/compute/kernels/scalar_compare.cc
index 3dfd66655e5..0f2c479b082 100644
--- a/cpp/src/arrow/compute/kernels/scalar_compare.cc
+++ b/cpp/src/arrow/compute/kernels/scalar_compare.cc
@@ -825,8 +825,7 @@ std::shared_ptr<ScalarFunction>
MakeScalarMinMax(std::string name, FunctionDoc d
DCHECK_OK(func->AddKernel(std::move(kernel)));
}
for (const auto& ty : BaseBinaryTypes()) {
- auto exec =
- GenerateTypeAgnosticVarBinaryBase<BinaryScalarMinMax, ArrayKernelExec,
Op>(ty);
+ auto exec = GenerateTypeAgnosticVarBinaryBase<BinaryScalarMinMax, Op>(ty);
ScalarKernel kernel{KernelSignature::Make({ty}, ty, /*is_varargs=*/true),
exec,
MinMaxState::Init};
kernel.null_handling = NullHandling::COMPUTED_NO_PREALLOCATE;
diff --git a/cpp/src/arrow/compute/kernels/scalar_if_else.cc
b/cpp/src/arrow/compute/kernels/scalar_if_else.cc
index 5618f24e5ce..26f880fc0c6 100644
--- a/cpp/src/arrow/compute/kernels/scalar_if_else.cc
+++ b/cpp/src/arrow/compute/kernels/scalar_if_else.cc
@@ -1328,7 +1328,7 @@ void AddBinaryIfElseKernels(const
std::shared_ptr<IfElseFunction>& scalar_functi
const std::vector<std::shared_ptr<DataType>>&
types) {
for (auto&& type : types) {
auto exec =
- internal::GenerateTypeAgnosticVarBinaryBase<ResolveIfElseExec,
ArrayKernelExec,
+ internal::GenerateTypeAgnosticVarBinaryBase<ResolveIfElseExec,
/*AllocateMem=*/std::true_type>(
*type);
// cond array needs to be boolean always
diff --git a/cpp/src/arrow/compute/kernels/scalar_string_ascii.cc
b/cpp/src/arrow/compute/kernels/scalar_string_ascii.cc
index d4dda5556b3..06ec8c999c6 100644
--- a/cpp/src/arrow/compute/kernels/scalar_string_ascii.cc
+++ b/cpp/src/arrow/compute/kernels/scalar_string_ascii.cc
@@ -16,6 +16,7 @@
// under the License.
#include <algorithm>
+#include <array>
#include <cctype>
#include <iterator>
#include <memory>
@@ -26,6 +27,7 @@
#include "arrow/compute/kernels/scalar_string_internal.h"
#include "arrow/compute/registry_internal.h"
#include "arrow/result.h"
+#include "arrow/status.h"
#include "arrow/util/config.h"
#include "arrow/util/logging_internal.h"
#include "arrow/util/macros.h"
@@ -71,12 +73,6 @@ RE2::Options MakeRE2Options(bool is_utf8, bool ignore_case =
false,
options.set_literal(literal);
return options;
}
-
-// Set RE2 encoding based on input type: Latin-1 for BinaryTypes and UTF-8 for
StringTypes
-template <typename T>
-RE2::Options MakeRE2Options(bool ignore_case = false, bool literal = false) {
- return MakeRE2Options(T::is_utf8, ignore_case, literal);
-}
#endif
// ----------------------------------------------------------------------
@@ -948,8 +944,8 @@ void AddAsciiStringReverse(FunctionRegistry* registry) {
auto func = std::make_shared<ScalarFunction>("binary_reverse",
Arity::Unary(),
binary_reverse_doc);
for (const auto& ty : BinaryTypes()) {
- DCHECK_OK(
- func->AddKernel({ty}, ty,
GenerateVarBinaryToVarBinary<BinaryReverse>(ty)));
+ DCHECK_OK(func->AddKernel({ty}, ty,
+
GenerateTypeAgnosticVarBinaryBase<BinaryReverse>(ty)));
}
DCHECK_OK(registry->AddFunction(std::move(func)));
}
@@ -1376,9 +1372,9 @@ template <typename Type>
struct MatchSubstring<Type, RegexSubstringMatcher> {
static Status Exec(KernelContext* ctx, const ExecSpan& batch, ExecResult*
out) {
// TODO Cache matcher across invocations (for regex compilation)
- ARROW_ASSIGN_OR_RAISE(auto matcher,
-
RegexSubstringMatcher::Make(MatchSubstringState::Get(ctx),
-
/*is_utf8=*/Type::is_utf8));
+ const bool is_utf8 = is_string_or_string_view(batch[0].type()->id());
+ ARROW_ASSIGN_OR_RAISE(auto matcher, RegexSubstringMatcher::Make(
+ MatchSubstringState::Get(ctx),
is_utf8));
return MatchSubstringImpl<Type, RegexSubstringMatcher>::Exec(ctx, batch,
out,
matcher.get());
}
@@ -1391,9 +1387,9 @@ struct MatchSubstring<Type, PlainSubstringMatcher> {
auto options = MatchSubstringState::Get(ctx);
if (options.ignore_case) {
#ifdef ARROW_WITH_RE2
- ARROW_ASSIGN_OR_RAISE(
- auto matcher, RegexSubstringMatcher::Make(options,
/*is_utf8=*/Type::is_utf8,
- /*literal=*/true));
+ const bool is_utf8 = is_string_or_string_view(batch[0].type()->id());
+ ARROW_ASSIGN_OR_RAISE(auto matcher, RegexSubstringMatcher::Make(options,
is_utf8,
+
/*literal=*/true));
return MatchSubstringImpl<Type, RegexSubstringMatcher>::Exec(ctx, batch,
out,
matcher.get());
#else
@@ -1414,9 +1410,9 @@ struct MatchSubstring<Type, PlainStartsWithMatcher> {
#ifdef ARROW_WITH_RE2
MatchSubstringOptions converted_options = options;
converted_options.pattern = "^" + RE2::QuoteMeta(options.pattern);
- ARROW_ASSIGN_OR_RAISE(
- auto matcher,
- RegexSubstringMatcher::Make(converted_options,
/*is_utf8=*/Type::is_utf8));
+ const bool is_utf8 = is_string_or_string_view(batch[0].type()->id());
+ ARROW_ASSIGN_OR_RAISE(auto matcher,
+ RegexSubstringMatcher::Make(converted_options,
is_utf8));
return MatchSubstringImpl<Type, RegexSubstringMatcher>::Exec(ctx, batch,
out,
matcher.get());
#else
@@ -1437,9 +1433,9 @@ struct MatchSubstring<Type, PlainEndsWithMatcher> {
#ifdef ARROW_WITH_RE2
MatchSubstringOptions converted_options = options;
converted_options.pattern = RE2::QuoteMeta(options.pattern) + "$";
- ARROW_ASSIGN_OR_RAISE(
- auto matcher,
- RegexSubstringMatcher::Make(converted_options,
/*is_utf8=*/Type::is_utf8));
+ const bool is_utf8 = is_string_or_string_view(batch[0].type()->id());
+ ARROW_ASSIGN_OR_RAISE(auto matcher,
+ RegexSubstringMatcher::Make(converted_options,
is_utf8));
return MatchSubstringImpl<Type, RegexSubstringMatcher>::Exec(ctx, batch,
out,
matcher.get());
#else
@@ -1500,27 +1496,50 @@ std::string MakeLikeRegex(const MatchSubstringOptions&
options) {
return like_pattern;
}
+struct MatchLikeConstants {
+ // NOTE: avoid making those constants global to avoid compiling regexes at
startup
+ RE2::Options regex_options;
+ // A LIKE pattern matching this regex can be translated into a substring
search.
+ RE2 like_pattern_is_substring_match{R"(%+([^%_]*[^\\%_])?%+)",
regex_options};
+ // A LIKE pattern matching this regex can be translated into a prefix search.
+ RE2 like_pattern_is_starts_with{R"(([^%_]*[^\\%_])?%+)", regex_options};
+ // A LIKE pattern matching this regex can be translated into a suffix search.
+ RE2 like_pattern_is_ends_with{R"(%+([^%_]*))", regex_options};
+
+ static Result<const MatchLikeConstants*> Instance(bool is_utf8) {
+ static const auto constants = MakeAll();
+ return constants[is_utf8].Map([](const auto& ptr) { return ptr.get(); });
+ }
+
+ private:
+ static Result<std::unique_ptr<MatchLikeConstants>> Make(bool is_utf8) {
+ auto constants = std::unique_ptr<MatchLikeConstants>(new
MatchLikeConstants(is_utf8));
+ RETURN_NOT_OK(RegexStatus(constants->like_pattern_is_substring_match) &
+ RegexStatus(constants->like_pattern_is_starts_with) &
+ RegexStatus(constants->like_pattern_is_ends_with));
+ return constants;
+ }
+
+ static std::array<Result<std::unique_ptr<MatchLikeConstants>>, 2> MakeAll() {
+ return {Make(false), Make(true)};
+ }
+
+ explicit MatchLikeConstants(bool is_utf8) :
regex_options(MakeRE2Options(is_utf8)) {}
+};
+
// Evaluate a SQL-like LIKE pattern by translating it to a regexp or
// substring search as appropriate. See what Apache Impala does:
//
https://github.com/apache/impala/blob/9c38568657d62b6f6d7b10aa1c721ba843374dd8/be/src/exprs/like-predicate.cc
-template <typename StringType>
+template <typename PhysicalType>
struct MatchLike {
+ static_assert(!is_string_or_string_view(PhysicalType::type_id),
+ "should only codegen on physical types");
+
static Status Exec(KernelContext* ctx, const ExecSpan& batch, ExecResult*
out) {
- // NOTE: avoid making those constants global to avoid compiling regexes at
startup
- static const RE2::Options kRE2Options = MakeRE2Options<StringType>();
- // A LIKE pattern matching this regex can be translated into a substring
search.
- static const RE2 kLikePatternIsSubstringMatch(R"(%+([^%_]*[^\\%_])?%+)",
kRE2Options);
- // A LIKE pattern matching this regex can be translated into a prefix
search.
- static const RE2 kLikePatternIsStartsWith(R"(([^%_]*[^\\%_])?%+)",
kRE2Options);
- // A LIKE pattern matching this regex can be translated into a suffix
search.
- static const RE2 kLikePatternIsEndsWith(R"(%+([^%_]*))", kRE2Options);
- static bool global_checked = false;
- if (ARROW_PREDICT_FALSE(!global_checked)) {
- RETURN_NOT_OK(RegexStatus(kLikePatternIsSubstringMatch));
- RETURN_NOT_OK(RegexStatus(kLikePatternIsStartsWith));
- RETURN_NOT_OK(RegexStatus(kLikePatternIsEndsWith));
- global_checked = true;
- }
+ ARROW_ASSIGN_OR_RAISE(
+ const auto like_constants,
+ MatchLikeConstants::Instance(
+ /*is_utf8=*/is_string_or_string_view(batch[0].type()->id())));
auto original_options = MatchSubstringState::Get(ctx);
auto original_state = ctx->state();
@@ -1530,24 +1549,29 @@ struct MatchLike {
bool matched = false;
if (!original_options.ignore_case) {
if ((matched = RE2::FullMatch(original_options.pattern,
- kLikePatternIsSubstringMatch, &pattern))) {
+
like_constants->like_pattern_is_substring_match,
+ &pattern))) {
MatchSubstringOptions converted_options{pattern,
original_options.ignore_case};
MatchSubstringState converted_state(converted_options);
ctx->SetState(&converted_state);
- status = MatchSubstring<StringType, PlainSubstringMatcher>::Exec(ctx,
batch, out);
+ status =
+ MatchSubstring<PhysicalType, PlainSubstringMatcher>::Exec(ctx,
batch, out);
} else if ((matched = RE2::FullMatch(original_options.pattern,
- kLikePatternIsStartsWith,
&pattern))) {
+
like_constants->like_pattern_is_starts_with,
+ &pattern))) {
MatchSubstringOptions converted_options{pattern,
original_options.ignore_case};
MatchSubstringState converted_state(converted_options);
ctx->SetState(&converted_state);
status =
- MatchSubstring<StringType, PlainStartsWithMatcher>::Exec(ctx,
batch, out);
+ MatchSubstring<PhysicalType, PlainStartsWithMatcher>::Exec(ctx,
batch, out);
} else if ((matched = RE2::FullMatch(original_options.pattern,
- kLikePatternIsEndsWith, &pattern)))
{
+
like_constants->like_pattern_is_ends_with,
+ &pattern))) {
MatchSubstringOptions converted_options{pattern,
original_options.ignore_case};
MatchSubstringState converted_state(converted_options);
ctx->SetState(&converted_state);
- status = MatchSubstring<StringType, PlainEndsWithMatcher>::Exec(ctx,
batch, out);
+ status =
+ MatchSubstring<PhysicalType, PlainEndsWithMatcher>::Exec(ctx,
batch, out);
}
}
@@ -1556,7 +1580,7 @@ struct MatchLike {
original_options.ignore_case};
MatchSubstringState converted_state(converted_options);
ctx->SetState(&converted_state);
- status = MatchSubstring<StringType, RegexSubstringMatcher>::Exec(ctx,
batch, out);
+ status = MatchSubstring<PhysicalType, RegexSubstringMatcher>::Exec(ctx,
batch, out);
}
ctx->SetState(original_state);
return status;
@@ -1616,7 +1640,8 @@ void AddAsciiStringMatchSubstring(FunctionRegistry*
registry) {
auto func = std::make_shared<ScalarFunction>("match_substring",
Arity::Unary(),
match_substring_doc);
for (const auto& ty : BaseBinaryTypes()) {
- auto exec = GenerateVarBinaryToVarBinary<MatchSubstring,
PlainSubstringMatcher>(ty);
+ auto exec =
+ GenerateTypeAgnosticVarBinaryBase<MatchSubstring,
PlainSubstringMatcher>(ty);
DCHECK_OK(
func->AddKernel({ty}, boolean(), std::move(exec),
MatchSubstringState::Init));
}
@@ -1627,7 +1652,7 @@ void AddAsciiStringMatchSubstring(FunctionRegistry*
registry) {
std::make_shared<ScalarFunction>("starts_with", Arity::Unary(),
starts_with_doc);
for (const auto& ty : BaseBinaryTypes()) {
auto exec =
- GenerateVarBinaryToVarBinary<MatchSubstring,
PlainStartsWithMatcher>(ty);
+ GenerateTypeAgnosticVarBinaryBase<MatchSubstring,
PlainStartsWithMatcher>(ty);
DCHECK_OK(
func->AddKernel({ty}, boolean(), std::move(exec),
MatchSubstringState::Init));
}
@@ -1637,7 +1662,8 @@ void AddAsciiStringMatchSubstring(FunctionRegistry*
registry) {
auto func =
std::make_shared<ScalarFunction>("ends_with", Arity::Unary(),
ends_with_doc);
for (const auto& ty : BaseBinaryTypes()) {
- auto exec = GenerateVarBinaryToVarBinary<MatchSubstring,
PlainEndsWithMatcher>(ty);
+ auto exec =
+ GenerateTypeAgnosticVarBinaryBase<MatchSubstring,
PlainEndsWithMatcher>(ty);
DCHECK_OK(
func->AddKernel({ty}, boolean(), std::move(exec),
MatchSubstringState::Init));
}
@@ -1648,7 +1674,8 @@ void AddAsciiStringMatchSubstring(FunctionRegistry*
registry) {
auto func = std::make_shared<ScalarFunction>("match_substring_regex",
Arity::Unary(),
match_substring_regex_doc);
for (const auto& ty : BaseBinaryTypes()) {
- auto exec = GenerateVarBinaryToVarBinary<MatchSubstring,
RegexSubstringMatcher>(ty);
+ auto exec =
+ GenerateTypeAgnosticVarBinaryBase<MatchSubstring,
RegexSubstringMatcher>(ty);
DCHECK_OK(
func->AddKernel({ty}, boolean(), std::move(exec),
MatchSubstringState::Init));
}
@@ -1658,7 +1685,7 @@ void AddAsciiStringMatchSubstring(FunctionRegistry*
registry) {
auto func =
std::make_shared<ScalarFunction>("match_like", Arity::Unary(),
match_like_doc);
for (const auto& ty : BaseBinaryTypes()) {
- auto exec = GenerateVarBinaryToVarBinary<MatchLike>(ty);
+ auto exec = GenerateTypeAgnosticVarBinaryBase<MatchLike>(ty);
DCHECK_OK(
func->AddKernel({ty}, boolean(), std::move(exec),
MatchSubstringState::Init));
}
@@ -1714,24 +1741,30 @@ struct FindSubstringRegex {
};
#endif
-template <typename InputType>
+template <typename InputPhysicalType>
struct FindSubstringExec {
- using OffsetType = typename TypeTraits<InputType>::OffsetType;
+ using OffsetType = typename TypeTraits<InputPhysicalType>::OffsetType;
+
+ static_assert(!is_string_or_string_view(InputPhysicalType::type_id),
+ "should only codegen on physical types");
+
static Status Exec(KernelContext* ctx, const ExecSpan& batch, ExecResult*
out) {
const MatchSubstringOptions& options = MatchSubstringState::Get(ctx);
+ const bool is_utf8 = is_string_or_string_view(batch[0].type()->id());
if (options.ignore_case) {
#ifdef ARROW_WITH_RE2
ARROW_ASSIGN_OR_RAISE(auto matcher,
- FindSubstringRegex::Make(options,
InputType::is_utf8, true));
- applicator::ScalarUnaryNotNullStateful<OffsetType, InputType,
FindSubstringRegex>
+ FindSubstringRegex::Make(options, is_utf8, true));
+ applicator::ScalarUnaryNotNullStateful<OffsetType, InputPhysicalType,
+ FindSubstringRegex>
kernel{std::move(matcher)};
return kernel.Exec(ctx, batch, out);
#else
return Status::NotImplemented("ignore_case requires RE2");
#endif
}
- applicator::ScalarUnaryNotNullStateful<OffsetType, InputType,
FindSubstring> kernel{
- FindSubstring(PlainSubstringMatcher(options))};
+ applicator::ScalarUnaryNotNullStateful<OffsetType, InputPhysicalType,
FindSubstring>
+ kernel{FindSubstring(PlainSubstringMatcher(options))};
return kernel.Exec(ctx, batch, out);
}
};
@@ -1744,13 +1777,19 @@ const FunctionDoc find_substring_doc(
{"strings"}, "MatchSubstringOptions", /*options_required=*/true);
#ifdef ARROW_WITH_RE2
-template <typename InputType>
+template <typename InputPhysicalType>
struct FindSubstringRegexExec {
- using OffsetType = typename TypeTraits<InputType>::OffsetType;
+ using OffsetType = typename TypeTraits<InputPhysicalType>::OffsetType;
+
+ static_assert(!is_string_or_string_view(InputPhysicalType::type_id),
+ "should only codegen on physical types");
+
static Status Exec(KernelContext* ctx, const ExecSpan& batch, ExecResult*
out) {
const MatchSubstringOptions& options = MatchSubstringState::Get(ctx);
- ARROW_ASSIGN_OR_RAISE(auto matcher, FindSubstringRegex::Make(options,
false));
- applicator::ScalarUnaryNotNullStateful<OffsetType, InputType,
FindSubstringRegex>
+ const bool is_utf8 = is_string_or_string_view(batch[0].type()->id());
+ ARROW_ASSIGN_OR_RAISE(auto matcher, FindSubstringRegex::Make(options,
is_utf8));
+ applicator::ScalarUnaryNotNullStateful<OffsetType, InputPhysicalType,
+ FindSubstringRegex>
kernel{std::move(matcher)};
return kernel.Exec(ctx, batch, out);
}
@@ -1771,7 +1810,7 @@ void AddAsciiStringFindSubstring(FunctionRegistry*
registry) {
for (const auto& ty : BaseBinaryTypes()) {
auto offset_type = offset_bit_width(ty->id()) == 64 ? int64() : int32();
DCHECK_OK(func->AddKernel({ty}, offset_type,
-
GenerateVarBinaryToVarBinary<FindSubstringExec>(ty),
+
GenerateTypeAgnosticVarBinaryBase<FindSubstringExec>(ty),
MatchSubstringState::Init));
}
DCHECK_OK(func->AddKernel({InputType(Type::FIXED_SIZE_BINARY)}, int32(),
@@ -1785,9 +1824,10 @@ void AddAsciiStringFindSubstring(FunctionRegistry*
registry) {
find_substring_regex_doc);
for (const auto& ty : BaseBinaryTypes()) {
auto offset_type = offset_bit_width(ty->id()) == 64 ? int64() : int32();
- DCHECK_OK(func->AddKernel({ty}, offset_type,
-
GenerateVarBinaryToVarBinary<FindSubstringRegexExec>(ty),
- MatchSubstringState::Init));
+ DCHECK_OK(
+ func->AddKernel({ty}, offset_type,
+
GenerateTypeAgnosticVarBinaryBase<FindSubstringRegexExec>(ty),
+ MatchSubstringState::Init));
}
DCHECK_OK(func->AddKernel({InputType(Type::FIXED_SIZE_BINARY)}, int32(),
FindSubstringRegexExec<FixedSizeBinaryType>::Exec,
@@ -1865,8 +1905,8 @@ struct CountSubstringRegexExec {
using OffsetType = typename TypeTraits<InputType>::OffsetType;
static Status Exec(KernelContext* ctx, const ExecSpan& batch, ExecResult*
out) {
const MatchSubstringOptions& options = MatchSubstringState::Get(ctx);
- ARROW_ASSIGN_OR_RAISE(
- auto counter, CountSubstringRegex::Make(options,
/*is_utf8=*/InputType::is_utf8));
+ const bool is_utf8 = is_string_or_string_view(batch[0].type()->id());
+ ARROW_ASSIGN_OR_RAISE(auto counter, CountSubstringRegex::Make(options,
is_utf8));
applicator::ScalarUnaryNotNullStateful<OffsetType, InputType,
CountSubstringRegex>
kernel{std::move(counter)};
return kernel.Exec(ctx, batch, out);
@@ -1881,9 +1921,9 @@ struct CountSubstringExec {
const MatchSubstringOptions& options = MatchSubstringState::Get(ctx);
if (options.ignore_case) {
#ifdef ARROW_WITH_RE2
- ARROW_ASSIGN_OR_RAISE(
- auto counter, CountSubstringRegex::Make(options,
/*is_utf8=*/InputType::is_utf8,
- /*literal=*/true));
+ const bool is_utf8 = is_string_or_string_view(batch[0].type()->id());
+ ARROW_ASSIGN_OR_RAISE(auto counter, CountSubstringRegex::Make(options,
is_utf8,
+
/*literal=*/true));
applicator::ScalarUnaryNotNullStateful<OffsetType, InputType,
CountSubstringRegex>
kernel{std::move(counter)};
return kernel.Exec(ctx, batch, out);
@@ -1920,7 +1960,7 @@ void AddAsciiStringCountSubstring(FunctionRegistry*
registry) {
for (const auto& ty : BaseBinaryTypes()) {
auto offset_type = offset_bit_width(ty->id()) == 64 ? int64() : int32();
DCHECK_OK(func->AddKernel({ty}, offset_type,
-
GenerateVarBinaryToVarBinary<CountSubstringExec>(ty),
+
GenerateTypeAgnosticVarBinaryBase<CountSubstringExec>(ty),
MatchSubstringState::Init));
}
DCHECK_OK(func->AddKernel({InputType(Type::FIXED_SIZE_BINARY)}, int32(),
@@ -1934,9 +1974,10 @@ void AddAsciiStringCountSubstring(FunctionRegistry*
registry) {
count_substring_regex_doc);
for (const auto& ty : BaseBinaryTypes()) {
auto offset_type = offset_bit_width(ty->id()) == 64 ? int64() : int32();
- DCHECK_OK(func->AddKernel({ty}, offset_type,
-
GenerateVarBinaryToVarBinary<CountSubstringRegexExec>(ty),
- MatchSubstringState::Init));
+ DCHECK_OK(
+ func->AddKernel({ty}, offset_type,
+
GenerateTypeAgnosticVarBinaryBase<CountSubstringRegexExec>(ty),
+ MatchSubstringState::Init));
}
DCHECK_OK(func->AddKernel({InputType(Type::FIXED_SIZE_BINARY)}, int32(),
CountSubstringRegexExec<FixedSizeBinaryType>::Exec,
@@ -1960,12 +2001,14 @@ struct ReplaceSubstring {
static Status Exec(KernelContext* ctx, const ExecSpan& batch, ExecResult*
out) {
// TODO Cache replacer across invocations (for regex compilation)
- ARROW_ASSIGN_OR_RAISE(auto replacer,
Replacer::Make(ReplaceState::Get(ctx)));
+ const bool is_utf8 = is_string_or_string_view(batch[0].type()->id());
+ ARROW_ASSIGN_OR_RAISE(auto replacer,
Replacer::Make(ReplaceState::Get(ctx), is_utf8));
return Replace(ctx, batch, *replacer, out);
}
+ template <typename ConcreteReplacer>
static Status Replace(KernelContext* ctx, const ExecSpan& batch,
- const Replacer& replacer, ExecResult* out) {
+ const ConcreteReplacer& replacer, ExecResult* out) {
ValueDataBuilder value_data_builder(ctx->memory_pool());
OffsetBuilder offset_builder(ctx->memory_pool());
@@ -1997,7 +2040,7 @@ struct PlainSubstringReplacer {
const ReplaceSubstringOptions& options_;
static Result<std::unique_ptr<PlainSubstringReplacer>> Make(
- const ReplaceSubstringOptions& options) {
+ const ReplaceSubstringOptions& options, bool is_utf8) {
return std::make_unique<PlainSubstringReplacer>(options);
}
@@ -2039,15 +2082,14 @@ struct PlainSubstringReplacer {
};
#ifdef ARROW_WITH_RE2
-template <typename Type>
struct RegexSubstringReplacer {
const ReplaceSubstringOptions& options_;
const RE2 regex_find_;
const RE2 regex_replacement_;
static Result<std::unique_ptr<RegexSubstringReplacer>> Make(
- const ReplaceSubstringOptions& options) {
- auto replacer = std::make_unique<RegexSubstringReplacer>(options);
+ const ReplaceSubstringOptions& options, bool is_utf8) {
+ auto replacer = std::make_unique<RegexSubstringReplacer>(options, is_utf8);
RETURN_NOT_OK(RegexStatus(replacer->regex_find_));
RETURN_NOT_OK(RegexStatus(replacer->regex_replacement_));
@@ -2064,10 +2106,10 @@ struct RegexSubstringReplacer {
// Using RE2::FindAndConsume we can only find the pattern if it is a group,
therefore
// we have 2 regexes, one with () around it, one without.
- explicit RegexSubstringReplacer(const ReplaceSubstringOptions& options)
+ explicit RegexSubstringReplacer(const ReplaceSubstringOptions& options, bool
is_utf8)
: options_(options),
- regex_find_("(" + options_.pattern + ")", MakeRE2Options<Type>()),
- regex_replacement_(options_.pattern, MakeRE2Options<Type>()) {}
+ regex_find_("(" + options_.pattern + ")", MakeRE2Options(is_utf8)),
+ regex_replacement_(options_.pattern, MakeRE2Options(is_utf8)) {}
Status ReplaceString(std::string_view s, TypedBufferBuilder<uint8_t>*
builder) const {
re2::StringPiece piece(s.data(), s.length());
@@ -2138,7 +2180,7 @@ const FunctionDoc replace_substring_doc(
#ifdef ARROW_WITH_RE2
template <typename Type>
-using ReplaceSubstringRegex = ReplaceSubstring<Type,
RegexSubstringReplacer<Type>>;
+using ReplaceSubstringRegex = ReplaceSubstring<Type, RegexSubstringReplacer>;
const FunctionDoc replace_substring_regex_doc(
"Replace matching non-overlapping substrings with replacement",
@@ -2155,7 +2197,7 @@ void AddAsciiStringReplaceSubstring(FunctionRegistry*
registry) {
auto func = std::make_shared<ScalarFunction>("replace_substring",
Arity::Unary(),
replace_substring_doc);
for (const auto& ty : BaseBinaryTypes()) {
- auto exec = GenerateVarBinaryToVarBinary<ReplaceSubstringPlain>(ty);
+ auto exec = GenerateTypeAgnosticVarBinaryBase<ReplaceSubstringPlain>(ty);
ScalarKernel kernel{{ty}, ty, std::move(exec), ReplaceState::Init};
kernel.mem_allocation = MemAllocation::NO_PREALLOCATE;
DCHECK_OK(func->AddKernel(std::move(kernel)));
@@ -2167,7 +2209,7 @@ void AddAsciiStringReplaceSubstring(FunctionRegistry*
registry) {
auto func = std::make_shared<ScalarFunction>(
"replace_substring_regex", Arity::Unary(),
replace_substring_regex_doc);
for (const auto& ty : BaseBinaryTypes()) {
- auto exec = GenerateVarBinaryToVarBinary<ReplaceSubstringRegex>(ty);
+ auto exec = GenerateTypeAgnosticVarBinaryBase<ReplaceSubstringRegex>(ty);
ScalarKernel kernel{{ty}, ty, std::move(exec), ReplaceState::Init};
kernel.mem_allocation = MemAllocation::NO_PREALLOCATE;
DCHECK_OK(func->AddKernel(std::move(kernel)));
@@ -2289,7 +2331,8 @@ struct ExtractRegex : public ExtractRegexBase {
static Status Exec(KernelContext* ctx, const ExecSpan& batch, ExecResult*
out) {
ExtractRegexOptions options = ExtractRegexState::Get(ctx);
- ARROW_ASSIGN_OR_RAISE(auto data, ExtractRegexData::Make(options,
Type::is_utf8));
+ const bool is_utf8 = is_string_or_string_view(batch[0].type()->id());
+ ARROW_ASSIGN_OR_RAISE(auto data, ExtractRegexData::Make(options, is_utf8));
return ExtractRegex(data).Extract(ctx, batch, out);
}
@@ -2346,7 +2389,7 @@ void AddAsciiStringExtractRegex(FunctionRegistry*
registry) {
for (const auto& ty : BaseBinaryTypes()) {
ScalarKernel kernel{{ty},
out_ty,
- GenerateVarBinaryToVarBinary<ExtractRegex>(ty),
+ GenerateTypeAgnosticVarBinaryBase<ExtractRegex>(ty),
ExtractRegexState::Init};
// Null values will be computed based on regex match or not
kernel.null_handling = NullHandling::COMPUTED_NO_PREALLOCATE;
@@ -2394,8 +2437,9 @@ struct ExtractRegexSpan : ExtractRegexBase {
static Status Exec(KernelContext* ctx, const ExecSpan& batch, ExecResult*
out) {
auto options = OptionsWrapper<ExtractRegexSpanOptions>::Get(ctx);
+ const bool is_utf8 = is_string_or_string_view(batch[0].type()->id());
ARROW_ASSIGN_OR_RAISE(auto data,
- ExtractRegexSpanData::Make(options.pattern,
Type::is_utf8));
+ ExtractRegexSpanData::Make(options.pattern,
is_utf8));
return ExtractRegexSpan{data}.Extract(ctx, batch, out);
}
@@ -2486,7 +2530,7 @@ void AddAsciiStringExtractRegexSpan(FunctionRegistry*
registry) {
OutputType output_type(ResolveExtractRegexSpanOutputType);
for (const auto& type : BaseBinaryTypes()) {
ScalarKernel kernel({type}, output_type,
- GenerateVarBinaryToVarBinary<ExtractRegexSpan>(type),
+
GenerateTypeAgnosticVarBinaryBase<ExtractRegexSpan>(type),
OptionsWrapper<ExtractRegexSpanOptions>::Init);
kernel.null_handling = NullHandling::COMPUTED_NO_PREALLOCATE;
kernel.mem_allocation = MemAllocation::NO_PREALLOCATE;
@@ -2789,7 +2833,7 @@ void AddAsciiStringSlice(FunctionRegistry* registry) {
auto func =
std::make_shared<ScalarFunction>("binary_slice", Arity::Unary(),
binary_slice_doc);
for (const auto& ty : BinaryTypes()) {
- auto exec = GenerateVarBinaryToVarBinary<SliceBytes>(ty);
+ auto exec = GenerateTypeAgnosticVarBinaryBase<SliceBytes>(ty);
DCHECK_OK(
func->AddKernel({ty}, ty, std::move(exec),
SliceBytesTransform::State::Init));
}
@@ -2810,7 +2854,7 @@ using SplitPatternState =
OptionsWrapper<SplitPatternOptions>;
struct SplitPatternFinder : public StringSplitFinderBase<SplitPatternOptions> {
using Options = SplitPatternOptions;
- Status PreExec(const SplitPatternOptions& options) override {
+ Status PreExec(const SplitPatternOptions& options, bool is_utf8) override {
if (options.pattern.length() == 0) {
return Status::Invalid("Empty separator");
}
@@ -2876,7 +2920,7 @@ void AddAsciiStringSplitPattern(FunctionRegistry*
registry) {
auto func = std::make_shared<ScalarFunction>("split_pattern", Arity::Unary(),
split_pattern_doc);
for (const auto& ty : BaseBinaryTypes()) {
- auto exec = GenerateVarBinaryToVarBinary<SplitPatternExec, ListType>(ty);
+ auto exec = GenerateTypeAgnosticVarBinaryBase<SplitPatternExec,
ListType>(ty);
DCHECK_OK(
func->AddKernel({ty}, {list(ty)}, std::move(exec),
SplitPatternState::Init));
}
@@ -2947,7 +2991,7 @@ void AddAsciiStringSplitWhitespace(FunctionRegistry*
registry) {
ascii_split_whitespace_doc,
&default_options);
for (const auto& ty : StringTypes()) {
- auto exec = GenerateVarBinaryToVarBinary<SplitWhitespaceAsciiExec,
ListType>(ty);
+ auto exec = GenerateTypeAgnosticVarBinaryBase<SplitWhitespaceAsciiExec,
ListType>(ty);
DCHECK_OK(func->AddKernel({ty}, {list(ty)}, std::move(exec),
StringSplitState::Init));
}
DCHECK_OK(registry->AddFunction(std::move(func)));
@@ -2957,13 +3001,12 @@ void AddAsciiStringSplitWhitespace(FunctionRegistry*
registry) {
// Split by regex
#ifdef ARROW_WITH_RE2
-template <typename Type>
struct SplitRegexFinder : public StringSplitFinderBase<SplitPatternOptions> {
using Options = SplitPatternOptions;
std::unique_ptr<RE2> regex_split;
- Status PreExec(const SplitPatternOptions& options) override {
+ Status PreExec(const SplitPatternOptions& options, bool is_utf8) override {
if (options.reverse) {
return Status::NotImplemented("Cannot split in reverse with regex");
}
@@ -2973,7 +3016,8 @@ struct SplitRegexFinder : public
StringSplitFinderBase<SplitPatternOptions> {
pattern.reserve(options.pattern.size() + 2);
pattern += options.pattern;
pattern += ')';
- regex_split = std::make_unique<RE2>(pattern, MakeRE2Options<Type>());
+ regex_split = std::make_unique<RE2>(
+ pattern, MakeRE2Options(is_utf8, /*ignore_case=*/false,
/*literal=*/false));
return RegexStatus(*regex_split);
}
@@ -3000,7 +3044,7 @@ struct SplitRegexFinder : public
StringSplitFinderBase<SplitPatternOptions> {
};
template <typename Type, typename ListType>
-using SplitRegexExec = StringSplitExec<Type, ListType, SplitRegexFinder<Type>>;
+using SplitRegexExec = StringSplitExec<Type, ListType, SplitRegexFinder>;
const FunctionDoc split_pattern_regex_doc(
"Split string according to regex pattern",
@@ -3016,7 +3060,7 @@ void AddAsciiStringSplitRegex(FunctionRegistry* registry)
{
auto func = std::make_shared<ScalarFunction>("split_pattern_regex",
Arity::Unary(),
split_pattern_regex_doc);
for (const auto& ty : BaseBinaryTypes()) {
- auto exec = GenerateVarBinaryToVarBinary<SplitRegexExec, ListType>(ty);
+ auto exec = GenerateTypeAgnosticVarBinaryBase<SplitRegexExec,
ListType>(ty);
DCHECK_OK(
func->AddKernel({ty}, {list(ty)}, std::move(exec),
SplitPatternState::Init));
}
@@ -3418,8 +3462,7 @@ const JoinOptions* GetDefaultJoinOptions() {
template <typename ListType>
void AddBinaryJoinForListType(ScalarFunction* func) {
for (const auto& ty : BaseBinaryTypes()) {
- auto exec =
- GenerateTypeAgnosticVarBinaryBase<BinaryJoin, ArrayKernelExec,
ListType>(*ty);
+ auto exec = GenerateTypeAgnosticVarBinaryBase<BinaryJoin, ListType>(*ty);
auto list_ty = std::make_shared<ListType>(ty);
DCHECK_OK(func->AddKernel({InputType(list_ty), InputType(ty)}, ty,
std::move(exec)));
}
@@ -3560,9 +3603,12 @@ struct BinaryRepeatTransform : public
StringBinaryTransformBase<Type1, Type2> {
}
};
-template <typename Type1, typename Type2>
+template <typename Type1, typename Type2,
+ typename PhysicalType1 = typename Type1::PhysicalType,
+ typename PhysicalType2 = typename Type2::PhysicalType>
using BinaryRepeat =
- StringBinaryTransformExec<Type1, Type2, BinaryRepeatTransform<Type1,
Type2>>;
+ StringBinaryTransformExec<PhysicalType1, PhysicalType2,
+ BinaryRepeatTransform<PhysicalType1,
PhysicalType2>>;
const FunctionDoc binary_repeat_doc(
"Repeat a binary string",
@@ -3573,7 +3619,7 @@ void AddAsciiStringRepeat(FunctionRegistry* registry) {
auto func = std::make_shared<ScalarCTypeToInt64Function>(
"binary_repeat", Arity::Binary(), binary_repeat_doc);
for (const auto& ty : BaseBinaryTypes()) {
- auto exec = GenerateVarBinaryToVarBinary<BinaryRepeat, Int64Type>(ty);
+ auto exec = GenerateTypeAgnosticVarBinaryBase<BinaryRepeat, Int64Type>(ty);
ScalarKernel kernel{{ty, int64()}, ty, exec};
DCHECK_OK(func->AddKernel(std::move(kernel)));
}
diff --git a/cpp/src/arrow/compute/kernels/scalar_string_internal.h
b/cpp/src/arrow/compute/kernels/scalar_string_internal.h
index 664f6b4c0b0..5c621e8d53f 100644
--- a/cpp/src/arrow/compute/kernels/scalar_string_internal.h
+++ b/cpp/src/arrow/compute/kernels/scalar_string_internal.h
@@ -21,6 +21,7 @@
#include "arrow/compute/api_scalar.h"
#include "arrow/compute/kernels/common_internal.h"
+#include "arrow/type_traits.h"
namespace arrow {
namespace compute {
@@ -68,10 +69,13 @@ static int64_t GetVarBinaryValuesLength(const ArraySpan&
span) {
///
/// and returns the number of codeunits of the `output` sequence or a negative
/// value if an invalid input sequence is detected.
-template <typename Type, typename StringTransform>
+template <typename PhysicalType, typename StringTransform>
struct StringTransformExecBase {
- using offset_type = typename Type::offset_type;
- using ArrayType = typename TypeTraits<Type>::ArrayType;
+ using offset_type = typename PhysicalType::offset_type;
+ using ArrayType = typename TypeTraits<PhysicalType>::ArrayType;
+
+ static_assert(!is_string_or_string_view(PhysicalType::type_id),
+ "should only codegen on physical types");
static Status Execute(KernelContext* ctx, StringTransform* transform,
const ExecSpan& batch, ExecResult* out) {
@@ -121,9 +125,11 @@ struct StringTransformExecBase {
}
};
-template <typename Type, typename StringTransform>
-struct StringTransformExec : public StringTransformExecBase<Type,
StringTransform> {
- using StringTransformExecBase<Type, StringTransform>::Execute;
+template <typename Type, typename StringTransform,
+ typename PhysicalType = typename Type::PhysicalType>
+struct StringTransformExec
+ : public StringTransformExecBase<PhysicalType, StringTransform> {
+ using StringTransformExecBase<PhysicalType, StringTransform>::Execute;
static Status Exec(KernelContext* ctx, const ExecSpan& batch, ExecResult*
out) {
StringTransform transform;
@@ -132,11 +138,12 @@ struct StringTransformExec : public
StringTransformExecBase<Type, StringTransfor
}
};
-template <typename Type, typename StringTransform>
+template <typename Type, typename StringTransform,
+ typename PhysicalType = typename Type::PhysicalType>
struct StringTransformExecWithState
- : public StringTransformExecBase<Type, StringTransform> {
+ : public StringTransformExecBase<PhysicalType, StringTransform> {
using State = typename StringTransform::State;
- using StringTransformExecBase<Type, StringTransform>::Execute;
+ using StringTransformExecBase<PhysicalType, StringTransform>::Execute;
static Status Exec(KernelContext* ctx, const ExecSpan& batch, ExecResult*
out) {
StringTransform transform(State::Get(ctx));
@@ -151,7 +158,7 @@ void MakeUnaryStringBatchKernel(
MemAllocation::type mem_allocation = MemAllocation::PREALLOCATE) {
auto func = std::make_shared<ScalarFunction>(name, Arity::Unary(),
std::move(doc));
for (const auto& ty : StringTypes()) {
- auto exec = GenerateVarBinaryToVarBinary<ExecFunctor>(ty);
+ auto exec = GenerateTypeAgnosticVarBinaryBase<ExecFunctor>(ty);
ScalarKernel kernel{{ty}, ty, std::move(exec)};
kernel.mem_allocation = mem_allocation;
ARROW_DCHECK_OK(func->AddKernel(std::move(kernel)));
@@ -216,6 +223,9 @@ static inline FunctionDoc StringClassifyDoc(std::string
class_summary,
template <typename Type, typename Predicate>
struct StringPredicateFunctor {
+ static_assert(!is_string_or_string_view(Type::type_id),
+ "should only codegen on physical types");
+
static Status Exec(KernelContext* ctx, const ExecSpan& batch, ExecResult*
out) {
Status st = Status::OK();
EnsureUtf8LookupTablesFilled();
@@ -237,7 +247,7 @@ void AddUnaryStringPredicate(std::string name,
FunctionRegistry* registry,
FunctionDoc doc) {
auto func = std::make_shared<ScalarFunction>(name, Arity::Unary(),
std::move(doc));
for (const auto& ty : StringTypes()) {
- auto exec = GenerateVarBinaryToVarBinary<StringPredicateFunctor,
Predicate>(ty);
+ auto exec = GenerateTypeAgnosticVarBinaryBase<StringPredicateFunctor,
Predicate>(ty);
ARROW_DCHECK_OK(func->AddKernel({ty}, boolean(), std::move(exec)));
}
ARROW_DCHECK_OK(registry->AddFunction(std::move(func)));
@@ -281,7 +291,7 @@ struct ReplaceStringSliceTransformBase : public
StringTransformBase {
template <typename Options>
struct StringSplitFinderBase {
virtual ~StringSplitFinderBase() = default;
- virtual Status PreExec(const Options& options) { return Status::OK(); }
+ virtual Status PreExec(const Options& options, bool is_utf8) { return
Status::OK(); }
// Derived classes should also define these methods:
// static bool Find(const uint8_t* begin, const uint8_t* end,
@@ -319,8 +329,9 @@ struct StringSplitExec {
}
Status Execute(KernelContext* ctx, const ExecSpan& batch, ExecResult* out) {
+ const bool is_utf8 = is_string_or_string_view(batch[0].type()->id());
SplitFinder finder;
- RETURN_NOT_OK(finder.PreExec(options));
+ RETURN_NOT_OK(finder.PreExec(options, is_utf8));
// TODO(wesm): refactor to not require creating ArrayData
const ArrayType input(batch[0].array.ToArrayData());
@@ -347,9 +358,11 @@ struct StringSplitExec {
*list_offsets++ = static_cast<list_offset_type>(builder.length());
}
// Assign string array to list child data
- std::shared_ptr<Array> string_array;
- RETURN_NOT_OK(builder.Finish(&string_array));
- output_list->child_data.push_back(string_array->data());
+ ARROW_ASSIGN_OR_RAISE(auto physical_array, builder.Finish());
+ // We got the physical type (e.g. binary instead of utf8), need to patch it
+ auto child_data = physical_array->data()->Copy();
+ child_data->type = batch[0].type()->GetSharedPtr();
+ output_list->child_data.push_back(std::move(child_data));
return Status::OK();
}
diff --git a/cpp/src/arrow/compute/kernels/scalar_string_test.cc
b/cpp/src/arrow/compute/kernels/scalar_string_test.cc
index b279f991f6e..bf5469ccd8b 100644
--- a/cpp/src/arrow/compute/kernels/scalar_string_test.cc
+++ b/cpp/src/arrow/compute/kernels/scalar_string_test.cc
@@ -30,7 +30,9 @@
#include "arrow/testing/gtest_util.h"
#include "arrow/type.h"
#include "arrow/type_fwd.h"
+#include "arrow/type_traits.h"
#include "arrow/util/config.h"
+#include "arrow/util/type_traits.h"
#include "arrow/util/value_parsing.h"
#ifdef ARROW_WITH_UTF8PROC
@@ -336,6 +338,44 @@ TYPED_TEST(TestBinaryKernels, NonUtf8Regex) {
{"letter": [1, 1], "digit": [2, 1]}])",
&options);
}
+
+ // On non-UTF8 input types, case insensitive matches should only account for
+ // ASCII letters.
+ // (this stresses that the "is_utf8" option is passed down correctly to RE2)
+ auto letters_array = this->MakeArray({"1ab2AB3", "1àb2ÀB3"});
+ {
+ MatchSubstringOptions options("(?i)(A|À)B");
+ // 5 is the byte index of "ÀB" in "1àb2ÀB3"
+ this->CheckUnary("find_substring_regex", letters_array,
this->offset_type(), "[1, 5]",
+ &options);
+ this->CheckUnary("count_substring_regex", letters_array,
this->offset_type(),
+ "[2, 1]", &options);
+ this->CheckUnary("match_substring_regex", letters_array, boolean(),
"[true, true]",
+ &options);
+ }
+ {
+ SplitPatternOptions options("(?i)(A|À)B");
+ this->CheckUnary("split_pattern_regex", letters_array, list(this->type()),
+ R"([["1", "2", "3"], ["1àb2", "3"]])", &options);
+ }
+ {
+ ReplaceSubstringOptions options("(?i)(A|À)B", "XY");
+ this->CheckUnary("replace_substring_regex", letters_array,
+ this->MakeArray({"1XY2XY3", "1àb2XY3"}), &options);
+ }
+ {
+ ExtractRegexOptions options("(?P<letters>(?i:(?:A|À)B))");
+ auto out_type = struct_({{"letters", this->type()}});
+ this->CheckUnary("extract_regex", letters_array, out_type,
+ R"([{"letters": "ab"}, {"letters": "ÀB"}])", &options);
+ }
+ {
+ ExtractRegexSpanOptions options("(?P<letters>(?i:(?:A|À)B))");
+ auto out_type = struct_({{"letters", fixed_size_list(this->offset_type(),
2)}});
+ // 5 is the byte index of "ÀB" in "1àb2ÀB3", 3 is its byte length
+ this->CheckUnary("extract_regex_span", letters_array, out_type,
+ R"([{"letters": [1, 2]}, {"letters": [5, 3]}])",
&options);
+ }
}
TYPED_TEST(TestBinaryKernels, NonUtf8WithNullRegex) {
@@ -509,6 +549,16 @@ TYPED_TEST(TestBaseBinaryKernels, FindSubstringIgnoreCase)
{
this->CheckUnary("find_substring",
R"-(["?aB)c", "acb", "c?Ab)", null, "?aBc", "AB)"])-",
this->offset_type(), "[0, -1, 1, null, -1, -1]", &options);
+ options.pattern = "?àb";
+ if (is_string_or_string_view(this->type()->id())) {
+ this->CheckUnary("find_substring",
+ R"-(["?àB)c", "àcb", "c?Àb)", null, "(?àBc", "ÀB)"])-",
+ this->offset_type(), "[0, -1, 1, null, 1, -1]", &options);
+ } else {
+ this->CheckUnary("find_substring",
+ R"-(["?àB)c", "àcb", "c?Àb)", null, "(?àBc", "ÀB)"])-",
+ this->offset_type(), "[0, -1, -1, null, 1, -1]",
&options);
+ }
}
TYPED_TEST(TestBaseBinaryKernels, FindSubstringRegex) {
@@ -597,6 +647,18 @@ TYPED_TEST(TestBaseBinaryKernels,
CountSubstringIgnoreCase) {
MatchSubstringOptions options_empty{"", /*ignore_case=*/true};
this->CheckUnary("count_substring", R"(["", null, "abc"])",
this->offset_type(),
"[1, null, 4]", &options_empty);
+
+ // case-folding is Unicode on string types, ASCII-only on binary types.
+ options.pattern = "àb";
+ if (is_string_or_string_view(this->type()->id())) {
+ this->CheckUnary("count_substring",
+ R"(["", null, "àb", "àBà", "bÀbÀ", "ÀbàbÀ", "bÀbÀcàbà"])",
+ this->offset_type(), "[0, null, 1, 1, 1, 2, 2]",
&options);
+ } else {
+ this->CheckUnary("count_substring",
+ R"(["", null, "àb", "àBà", "bÀbÀ", "ÀbàbÀ", "bÀbÀcàbà"])",
+ this->offset_type(), "[0, null, 1, 1, 0, 1, 1]",
&options);
+ }
}
TYPED_TEST(TestBaseBinaryKernels, CountSubstringRegexIgnoreCase) {
@@ -607,6 +669,17 @@ TYPED_TEST(TestBaseBinaryKernels,
CountSubstringRegexIgnoreCase) {
MatchSubstringOptions options_empty_match{"a*", /*ignore_case=*/true};
this->CheckUnary("count_substring_regex", R"(["", "bacAaAdaAaA", "c",
"AAA"])",
this->offset_type(), "[1, 7, 2, 2]", &options_empty_match);
+
+ // case-folding is Unicode on string types, ASCII-only on binary types.
+ options_as.pattern = "à+";
+ if (is_string_or_string_view(this->type()->id())) {
+ this->CheckUnary("count_substring_regex", R"(["", "bàcÀàÀdàÀàÀ", "àà",
"ÀÀÀ"])",
+ this->offset_type(), "[0, 3, 1, 1]", &options_as);
+ } else {
+ // In non-UTF8 mode, "à" is a two-character sequence, so "à+" does not
match "àà".
+ this->CheckUnary("count_substring_regex", R"(["", "bàcÀàÀdàÀàÀ", "àà",
"ÀÀÀ"])",
+ this->offset_type(), "[0, 4, 2, 0]", &options_as);
+ }
}
#else
TYPED_TEST(TestBaseBinaryKernels, CountSubstringIgnoreCase) {
@@ -1041,6 +1114,48 @@ TEST_F(TestFixedSizeBinaryKernels,
FindSubstringIgnoreCase) {
}
#endif
+#ifdef ARROW_WITH_RE2
+TYPED_TEST(TestStringKernels, Utf8Regex) {
+ // On UTF8 input types, case insensitive matches should account for all
letters.
+ // (this stresses that the "is_utf8" option is passed down correctly to RE2)
+ auto letters_array = this->MakeArray({"🙂ab2AB3", "🙂àb2ÀB3"});
+ {
+ MatchSubstringOptions options("(?i)(A|À)B");
+ // 4 is the byte index of "ab" in "🙂ab2AB3"
+ this->CheckUnary("find_substring_regex", letters_array,
this->offset_type(), "[4, 4]",
+ &options);
+ this->CheckUnary("count_substring_regex", letters_array,
this->offset_type(),
+ "[2, 2]", &options);
+ this->CheckUnary("match_substring_regex", letters_array, boolean(),
"[true, true]",
+ &options);
+ }
+ {
+ SplitPatternOptions options("(?i)(A|À)B");
+ this->CheckUnary("split_pattern_regex", letters_array, list(this->type()),
+ R"([["🙂", "2", "3"], ["🙂", "2", "3"]])", &options);
+ }
+ {
+ ReplaceSubstringOptions options("(?i)(A|À)B", "XY");
+ this->CheckUnary("replace_substring_regex", letters_array,
+ this->MakeArray({"🙂XY2XY3", "🙂XY2XY3"}), &options);
+ }
+ {
+ ExtractRegexOptions options("(?P<letters>(?i:(?:A|À)B))");
+ auto out_type = struct_({{"letters", this->type()}});
+ this->CheckUnary("extract_regex", letters_array, out_type,
+ R"([{"letters": "ab"}, {"letters": "àb"}])", &options);
+ }
+ {
+ ExtractRegexSpanOptions options("(?P<letters>(?i:(?:A|À)B))");
+ auto out_type = struct_({{"letters", fixed_size_list(this->offset_type(),
2)}});
+ // 4 is the byte index of "ab" in "🙂ab2AB3"
+ // 3 is the byte length of "àb"
+ this->CheckUnary("extract_regex_span", letters_array, out_type,
+ R"([{"letters": [4, 2]}, {"letters": [4, 3]}])",
&options);
+ }
+}
+#endif
+
TYPED_TEST(TestStringKernels, AsciiUpper) {
this->CheckUnary("ascii_upper", "[]", this->type(), "[]");
this->CheckUnary("ascii_upper", "[\"aAazZæÆ&\", null, \"\", \"bbb\"]",
this->type(),
@@ -1556,11 +1671,17 @@ TYPED_TEST(TestBaseBinaryKernels, MatchSubstring) {
}
#ifdef ARROW_WITH_RE2
-TYPED_TEST(TestStringKernels, MatchSubstringIgnoreCase) {
+TYPED_TEST(TestBaseBinaryKernels, MatchSubstringIgnoreCase) {
MatchSubstringOptions options_insensitive{"aé(", /*ignore_case=*/true};
- this->CheckUnary("match_substring", R"(["abc", "aEb", "baÉ(", "aé(", "ae(",
"Aé("])",
- boolean(), "[false, false, true, true, false, true]",
- &options_insensitive);
+ if (is_string_or_string_view(this->type()->id())) {
+ this->CheckUnary("match_substring", R"(["abc", "aEb", "baÉ(", "aé(",
"ae(", "Aé("])",
+ boolean(), "[false, false, true, true, false, true]",
+ &options_insensitive);
+ } else {
+ this->CheckUnary("match_substring", R"(["abc", "aEb", "baÉ(", "aé(",
"ae(", "Aé("])",
+ boolean(), "[false, false, false, true, false, true]",
+ &options_insensitive);
+ }
}
#else
TYPED_TEST(TestBaseBinaryKernels, MatchSubstringIgnoreCase) {
@@ -1598,6 +1719,15 @@ TYPED_TEST(TestBaseBinaryKernels,
MatchStartsWithIgnoreCase) {
boolean(), "[null, false, false, true, false, true]",
&options);
this->CheckUnary("starts_with", R"(["ABAB", "$ABAB", "ABAB$", "$AbAb",
"aBaB$"])",
boolean(), "[true, false, true, false, true]", &options);
+
+ options.pattern = "àBÀb";
+ if (is_string_or_string_view(this->type()->id())) {
+ this->CheckUnary("starts_with", R"(["ÀBÀB", "$ÀBÀB", "ÀBÀB$", "$ÀbÀb",
"àBÀB$"])",
+ boolean(), "[true, false, true, false, true]", &options);
+ } else {
+ this->CheckUnary("starts_with", R"(["ÀBÀB", "$ÀBÀB", "ÀBÀB$", "$ÀbÀb",
"àBÀB$"])",
+ boolean(), "[false, false, false, false, true]",
&options);
+ }
}
TYPED_TEST(TestBaseBinaryKernels, MatchEndsWithIgnoreCase) {
@@ -1607,6 +1737,15 @@ TYPED_TEST(TestBaseBinaryKernels,
MatchEndsWithIgnoreCase) {
boolean(), "[null, false, false, true, true, false]",
&options);
this->CheckUnary("ends_with", R"(["ABAB", "$ABAB", "ABAB$", "$AbAb",
"aBaB$"])",
boolean(), "[true, true, false, true, false]", &options);
+
+ options.pattern = "àBÀb";
+ if (is_string_or_string_view(this->type()->id())) {
+ this->CheckUnary("ends_with", R"(["ÀBÀB", "$àBÀB", "ÀBÀB$", "$ÀbÀb",
"àBÀB$"])",
+ boolean(), "[true, true, false, true, false]", &options);
+ } else {
+ this->CheckUnary("ends_with", R"(["ÀBÀB", "$àBÀB", "ÀBÀB$", "$ÀbÀb",
"àBÀB$"])",
+ boolean(), "[false, true, false, false, false]",
&options);
+ }
}
#else
TYPED_TEST(TestBaseBinaryKernels, MatchStartsWithIgnoreCase) {
@@ -1670,6 +1809,23 @@ TYPED_TEST(TestBaseBinaryKernels,
MatchSubstringRegexInvalid) {
CallFunction("match_substring_regex", {input}, &options));
}
+TYPED_TEST(TestBinaryKernels, MatchLikeIgnoreCase) {
+ // Case-folding is ASCII-only for binary types
+ MatchSubstringOptions insensitive_substring{"%e%", /*ignore_case=*/true};
+ this->CheckUnary("match_like", R"(["fooebar", "fooEbar", "é"])", boolean(),
+ "[true, true, false]", &insensitive_substring);
+ insensitive_substring.pattern = "%é%";
+ this->CheckUnary("match_like", R"(["fooébar", "fooÉbar", "e"])", boolean(),
+ "[true, false, false]", &insensitive_substring);
+
+ MatchSubstringOptions insensitive_regex{"_e%", /*ignore_case=*/true};
+ this->CheckUnary("match_like", R"(["aefoo", "aEfoo", "efoo"])", boolean(),
+ "[true, true, false]", &insensitive_regex);
+ insensitive_regex.pattern = "_é%";
+ this->CheckUnary("match_like", R"(["aéfoo", "aÉfoo", "éfoo"])", boolean(),
+ "[true, false, false]", &insensitive_regex);
+}
+
TYPED_TEST(TestStringKernels, MatchLike) {
auto inputs = R"(["foo", "bar", "foobar", "barfoo", "o", "\nfoo", "foo\n",
null])";
diff --git a/cpp/src/arrow/compute/kernels/scalar_string_utf8.cc
b/cpp/src/arrow/compute/kernels/scalar_string_utf8.cc
index fd340bba624..582d559dde2 100644
--- a/cpp/src/arrow/compute/kernels/scalar_string_utf8.cc
+++ b/cpp/src/arrow/compute/kernels/scalar_string_utf8.cc
@@ -45,7 +45,7 @@ void MakeUnaryStringUTF8TransformKernel(std::string name,
FunctionRegistry* regi
FunctionDoc doc) {
auto func = std::make_shared<ScalarFunction>(name, Arity::Unary(),
std::move(doc));
for (const auto& ty : StringTypes()) {
- auto exec = GenerateVarBinaryToVarBinary<Transformer>(ty);
+ auto exec = GenerateTypeAgnosticVarBinaryBase<Transformer>(ty);
DCHECK_OK(func->AddKernel({ty}, ty, std::move(exec)));
}
DCHECK_OK(registry->AddFunction(std::move(func)));
@@ -1144,7 +1144,7 @@ void AddUtf8StringReplaceSlice(FunctionRegistry*
registry) {
utf8_replace_slice_doc);
for (const auto& ty : StringTypes()) {
- auto exec = GenerateVarBinaryToVarBinary<Utf8ReplaceSlice>(ty);
+ auto exec = GenerateTypeAgnosticVarBinaryBase<Utf8ReplaceSlice>(ty);
DCHECK_OK(func->AddKernel({ty}, ty, std::move(exec),
ReplaceStringSliceTransformBase::State::Init));
}
@@ -1340,7 +1340,7 @@ void AddUtf8StringSlice(FunctionRegistry* registry) {
auto func = std::make_shared<ScalarFunction>("utf8_slice_codeunits",
Arity::Unary(),
utf8_slice_codeunits_doc);
for (const auto& ty : StringTypes()) {
- auto exec = GenerateVarBinaryToVarBinary<SliceCodeunits>(ty);
+ auto exec = GenerateTypeAgnosticVarBinaryBase<SliceCodeunits>(ty);
DCHECK_OK(
func->AddKernel({ty}, ty, std::move(exec),
SliceCodeunitsTransform::State::Init));
}
@@ -1355,7 +1355,8 @@ void AddUtf8StringSlice(FunctionRegistry* registry) {
struct SplitWhitespaceUtf8Finder : public StringSplitFinderBase<SplitOptions> {
using Options = SplitOptions;
- Status PreExec(const SplitOptions& options) override {
+ Status PreExec(const SplitOptions& options, bool is_utf8) override {
+ DCHECK(is_utf8);
EnsureUtf8LookupTablesFilled();
return Status::OK();
}
@@ -1426,7 +1427,7 @@ void AddUtf8StringSplitWhitespace(FunctionRegistry*
registry) {
std::make_shared<ScalarFunction>("utf8_split_whitespace", Arity::Unary(),
utf8_split_whitespace_doc,
&default_options);
for (const auto& ty : StringTypes()) {
- auto exec = GenerateVarBinaryToVarBinary<SplitWhitespaceUtf8Exec,
ListType>(ty);
+ auto exec = GenerateTypeAgnosticVarBinaryBase<SplitWhitespaceUtf8Exec,
ListType>(ty);
DCHECK_OK(func->AddKernel({ty}, {list(ty)}, std::move(exec),
StringSplitState::Init));
}
DCHECK_OK(registry->AddFunction(std::move(func)));
diff --git a/cpp/src/arrow/compute/kernels/vector_array_sort.cc
b/cpp/src/arrow/compute/kernels/vector_array_sort.cc
index ad9444037c0..1cfb11286e4 100644
--- a/cpp/src/arrow/compute/kernels/vector_array_sort.cc
+++ b/cpp/src/arrow/compute/kernels/vector_array_sort.cc
@@ -586,9 +586,8 @@ void AddArraySortingKernels(VectorKernel base,
VectorFunction* func) {
DCHECK_OK(func->AddKernel(base));
}
for (const auto& ty : BaseBinaryTypes()) {
- auto physical_type = GetPhysicalType(ty);
base.signature = KernelSignature::Make({ty}, uint64());
- base.exec = GenerateVarBinaryBase<ExecTemplate,
UInt64Type>(*physical_type);
+ base.exec = GenerateVarBinaryBase<ExecTemplate, UInt64Type>(ty);
DCHECK_OK(func->AddKernel(base));
}
base.signature = KernelSignature::Make({Type::FIXED_SIZE_BINARY}, uint64());
diff --git a/cpp/src/arrow/compute/kernels/vector_replace.cc
b/cpp/src/arrow/compute/kernels/vector_replace.cc
index ae39977aaea..6a9abfc0396 100644
--- a/cpp/src/arrow/compute/kernels/vector_replace.cc
+++ b/cpp/src/arrow/compute/kernels/vector_replace.cc
@@ -857,11 +857,10 @@ void RegisterVectorFunction(FunctionRegistry* registry,
Functor<FixedSizeBinaryType>::Exec,
ChunkedFunctor<FixedSizeBinaryType>::Exec,
registry, func.get());
for (const auto& ty : BaseBinaryTypes()) {
- AddKernel(
- ty->id(), Functor<FixedSizeBinaryType>::GetSignature(ty->id()),
- GenerateTypeAgnosticVarBinaryBase<Functor, ArrayKernelExec>(*ty),
- GenerateTypeAgnosticVarBinaryBase<ChunkedFunctor,
VectorKernel::ChunkedExec>(*ty),
- registry, func.get());
+ AddKernel(ty->id(), Functor<FixedSizeBinaryType>::GetSignature(ty->id()),
+ GenerateTypeAgnosticVarBinaryBase<Functor>(*ty),
+ GenerateTypeAgnosticVarBinaryBase<ChunkedFunctor>(*ty), registry,
+ func.get());
}
// TODO: list types
DCHECK_OK(registry->AddFunction(std::move(func)));
