This is an automated email from the ASF dual-hosted git repository.
pitrou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new f4265ecb0f5 GH-49889: [C++][Compute] Handle logical nulls in validity
kernels (#50269)
f4265ecb0f5 is described below
commit f4265ecb0f52d5f3c144c7d0acd02748baa40d7e
Author: Hadrian Reppas <[email protected]>
AuthorDate: Wed Sep 23 10:46:32 2026 -0400
GH-49889: [C++][Compute] Handle logical nulls in validity kernels (#50269)
### Rationale for this change
I was looking into #49889 and updated the handling of logical nulls in the
`is_valid`, `is_null`, and `true_unless_null` kernels. It turns out there is
some prior work here that I didn't see before I starting implementing my fix
(#35058, #35036, #37642). For some reason, that work has stalled.
### What changes are included in this PR?
This PR updates the `IsValidExec`, `IsNullExec`, and `TrueUnlessNullExec`
kernels so that they call out to a new function `SetLogicalNullBits`. The
`SetLogicalNullBits` function matches the logic in
`ArraySpan::ComputeLogicalNullCount()`. The logic is similar enough that they
may be some opportunities for code deduplication in `dict_util.cc` and
`ree_util.cc`.
### Are these changes tested?
Yes
### Are there any user-facing changes?
No
* GitHub Issue: #49889
Authored-by: Hadrian <[email protected]>
Signed-off-by: Antoine Pitrou <[email protected]>
---
cpp/src/arrow/compute/kernels/scalar_validity.cc | 152 +++++++++++----------
.../arrow/compute/kernels/scalar_validity_test.cc | 56 ++++++++
cpp/src/arrow/util/dict_util.cc | 119 +++++++++++++---
cpp/src/arrow/util/dict_util_internal.h | 18 +++
cpp/src/arrow/util/ree_util.cc | 30 ++++
cpp/src/arrow/util/ree_util.h | 10 ++
cpp/src/arrow/util/union_util.cc | 14 ++
cpp/src/arrow/util/union_util.h | 18 +++
8 files changed, 330 insertions(+), 87 deletions(-)
diff --git a/cpp/src/arrow/compute/kernels/scalar_validity.cc
b/cpp/src/arrow/compute/kernels/scalar_validity.cc
index 5913b756f1c..c995b8dff82 100644
--- a/cpp/src/arrow/compute/kernels/scalar_validity.cc
+++ b/cpp/src/arrow/compute/kernels/scalar_validity.cc
@@ -23,8 +23,11 @@
#include "arrow/util/bit_util.h"
#include "arrow/util/bitmap_ops.h"
+#include "arrow/util/dict_util_internal.h"
#include "arrow/util/float16.h"
#include "arrow/util/logging_internal.h"
+#include "arrow/util/ree_util.h"
+#include "arrow/util/union_util.h"
namespace arrow {
@@ -36,29 +39,7 @@ namespace compute {
namespace internal {
namespace {
-Status IsValidExec(KernelContext* ctx, const ExecSpan& batch, ExecResult* out)
{
- const ArraySpan& arr = batch[0].array;
- ArraySpan* out_span = out->array_span_mutable();
- if (arr.type->id() == Type::NA) {
- // Input is all nulls => output is entirely false.
- bit_util::SetBitsTo(out_span->buffers[1].data, out_span->offset,
out_span->length,
- false);
- return Status::OK();
- }
-
- DCHECK_EQ(out_span->offset, 0);
- DCHECK_LE(out_span->length, arr.length);
- if (arr.MayHaveNulls()) {
- // We could do a zero-copy optimization, but it isn't worth the added
complexity
- ::arrow::internal::CopyBitmap(arr.buffers[0].data, arr.offset, arr.length,
- out_span->buffers[1].data, out_span->offset);
- } else {
- // Input has no nulls => output is entirely true.
- bit_util::SetBitsTo(out_span->buffers[1].data, out_span->offset,
out_span->length,
- true);
- }
- return Status::OK();
-}
+using NanOptionsState = OptionsWrapper<NullOptions>;
struct IsFiniteOperator {
template <typename OutType, typename InType>
@@ -82,8 +63,6 @@ struct IsInfOperator {
}
};
-using NanOptionsState = OptionsWrapper<NullOptions>;
-
template <typename T>
static void SetNanBits(const ArraySpan& arr, uint8_t* out_bitmap, int64_t
out_offset) {
const T* data = arr.GetValues<T>(1);
@@ -101,45 +80,78 @@ static void SetNanBits(const ArraySpan& arr, uint8_t*
out_bitmap, int64_t out_of
}
}
-Status IsNullExec(KernelContext* ctx, const ExecSpan& batch, ExecResult* out) {
- const ArraySpan& arr = batch[0].array;
- ArraySpan* out_span = out->array_span_mutable();
- if (arr.type->id() == Type::NA) {
- bit_util::SetBitsTo(out_span->buffers[1].data, out_span->offset,
out_span->length,
- true);
- return Status::OK();
- }
-
- const auto& options = NanOptionsState::Get(ctx);
- uint8_t* out_bitmap = out_span->buffers[1].data;
- if (arr.GetNullCount() > 0) {
- // Input has nulls => output is the inverted null (validity) bitmap.
- InvertBitmap(arr.buffers[0].data, arr.offset, arr.length, out_bitmap,
- out_span->offset);
+// `nan_is_null` can only be true for the is_null kernel since the is_valid and
+// true_unless_null kernels currently do not take `NullOptions`
+Status SetLogicalNullBits(const ArraySpan& span, uint8_t* out_bitmap, int64_t
out_offset,
+ bool set_on_null, bool nan_is_null) {
+ const Type::type t = span.type->id();
+ if (t == Type::NA) {
+ // Input is all nulls, so all output bits are the same.
+ bit_util::SetBitsTo(out_bitmap, out_offset, span.length, set_on_null);
+ } else if (t == Type::SPARSE_UNION) {
+ // TODO: propagate `nan_is_null`
+ union_util::SetLogicalNullBitsSparse(span, out_bitmap, out_offset,
set_on_null);
+ } else if (t == Type::DENSE_UNION) {
+ // TODO: propagate `nan_is_null`
+ union_util::SetLogicalNullBitsDense(span, out_bitmap, out_offset,
set_on_null);
+ } else if (t == Type::RUN_END_ENCODED) {
+ // TODO: propagate `nan_is_null`
+ ree_util::SetLogicalNullBits(span, out_bitmap, out_offset, set_on_null);
+ } else if (t == Type::DICTIONARY) {
+ // TODO: propagate `nan_is_null`
+ dict_util::SetLogicalNullBits(span, out_bitmap, out_offset, set_on_null);
} else {
- // Input has no nulls => output is entirely false.
- bit_util::SetBitsTo(out_bitmap, out_span->offset, out_span->length, false);
- }
+ // Input is a type for which logical and physical nulls are the same, so
we can
+ // use GetNullCount() and the validity bitmap
+ if (span.GetNullCount() > 0) {
+ // Input has nulls. The output is either the validity bitmap or the
inverse of the
+ // validity bitmap.
+ if (set_on_null) {
+ InvertBitmap(span.buffers[0].data, span.offset, span.length,
out_bitmap,
+ out_offset);
+ } else {
+ CopyBitmap(span.buffers[0].data, span.offset, span.length, out_bitmap,
+ out_offset);
+ }
+ } else {
+ // Input has no nulls, so all output bits are the same.
+ bit_util::SetBitsTo(out_bitmap, out_offset, span.length, !set_on_null);
+ }
- if (is_floating(arr.type->id()) && options.nan_is_null) {
- switch (arr.type->id()) {
- case Type::FLOAT:
- SetNanBits<float>(arr, out_bitmap, out_span->offset);
- break;
- case Type::DOUBLE:
- SetNanBits<double>(arr, out_bitmap, out_span->offset);
- break;
- case Type::HALF_FLOAT:
- SetNanBits<uint16_t>(arr, out_bitmap, out_span->offset);
- break;
- default:
- return Status::NotImplemented("NaN detection not implemented for type
",
- arr.type->ToString());
+ // If nan_is_null, we must also check for nans.
+ if (is_floating(t) && nan_is_null) {
+ switch (t) {
+ case Type::FLOAT:
+ SetNanBits<float>(span, out_bitmap, out_offset);
+ break;
+ case Type::DOUBLE:
+ SetNanBits<double>(span, out_bitmap, out_offset);
+ break;
+ case Type::HALF_FLOAT:
+ SetNanBits<uint16_t>(span, out_bitmap, out_offset);
+ break;
+ default:
+ return Status::NotImplemented("NaN detection not implemented for
type ",
+ span.type->ToString());
+ }
}
}
return Status::OK();
}
+Status IsValidExec(KernelContext* ctx, const ExecSpan& batch, ExecResult* out)
{
+ ArraySpan* out_span = out->array_span_mutable();
+ return SetLogicalNullBits(batch[0].array, out_span->buffers[1].data,
out_span->offset,
+ /*set_on_null=*/false, /*nan_is_null=*/false);
+}
+
+Status IsNullExec(KernelContext* ctx, const ExecSpan& batch, ExecResult* out) {
+ ArraySpan* out_span = out->array_span_mutable();
+ return SetLogicalNullBits(batch[0].array, out_span->buffers[1].data,
out_span->offset,
+ /*set_on_null=*/true,
+
/*nan_is_null=*/NanOptionsState::Get(ctx).nan_is_null);
+}
+
struct IsNanOperator {
template <typename OutType, typename InType>
static constexpr OutType Call(KernelContext*, const InType& value, Status*) {
@@ -243,20 +255,18 @@ std::shared_ptr<ScalarFunction>
MakeIsNanFunction(std::string name, FunctionDoc
}
Status TrueUnlessNullExec(KernelContext* ctx, const ExecSpan& batch,
ExecResult* out) {
+ // Set all bits in the output's value bitmap to true
ArraySpan* out_span = out->array_span_mutable();
- if (out_span->buffers[0].data) {
- // If there is a validity bitmap computed above the kernel
- // invocation, we copy it to the output buffers
- ::arrow::internal::CopyBitmap(out_span->buffers[0].data, out_span->offset,
- out_span->length, out_span->buffers[1].data,
- out_span->offset);
- } else {
- // But for all-valid inputs, the engine will skip allocating a
- // validity bitmap, so we set everything to true
- bit_util::SetBitsTo(out_span->buffers[1].data, out_span->offset,
out_span->length,
- true);
- }
- return Status::OK();
+ bit_util::SetBitsTo(out_span->buffers[1].data, out_span->offset,
out_span->length,
+ true);
+
+ // Set the output's validity bitmap based on the nullity of the input array
+ // NOTE: alternatively, we could switch this kernel's null handling back to
+ // NullHandling::INTERSECTION and change the validity checks in exec.cc so
that
+ // they correctly handle logical nulls, but that would invove significant
changes
+ // in exec.cc which might have more side effects
+ return SetLogicalNullBits(batch[0].array, out_span->buffers[0].data,
out_span->offset,
+ /*set_on_null=*/false, /*nan_is_null=*/false);
}
const FunctionDoc is_valid_doc(
@@ -303,7 +313,7 @@ void RegisterScalarValidity(FunctionRegistry* registry) {
/*can_write_into_slices=*/true, &kNullOptions,
NanOptionsState::Init);
MakeFunction("true_unless_null", true_unless_null_doc, {InputType::Any()},
boolean(),
- TrueUnlessNullExec, registry, NullHandling::INTERSECTION,
+ TrueUnlessNullExec, registry,
NullHandling::COMPUTED_PREALLOCATE,
/*can_write_into_slices=*/false);
DCHECK_OK(registry->AddFunction(MakeIsFiniteFunction("is_finite",
is_finite_doc)));
diff --git a/cpp/src/arrow/compute/kernels/scalar_validity_test.cc
b/cpp/src/arrow/compute/kernels/scalar_validity_test.cc
index 4613176b48c..86895cdcec5 100644
--- a/cpp/src/arrow/compute/kernels/scalar_validity_test.cc
+++ b/cpp/src/arrow/compute/kernels/scalar_validity_test.cc
@@ -59,6 +59,62 @@ TEST_F(TestBooleanValidityKernels, TrueUnlessNull) {
type_singleton(), "[null, true, true, null]");
}
+void CheckValidityKernels(Datum input, Datum is_valid_expected) {
+ ASSERT_OK_AND_ASSIGN(auto is_null_expected,
compute::Invert(is_valid_expected));
+ BooleanScalar true_scalar(true);
+ NullScalar null_scalar;
+ ASSERT_OK_AND_ASSIGN(auto true_unlesss_null_expected,
+ compute::IfElse(is_valid_expected, true_scalar,
null_scalar));
+
+ CheckScalarUnary("is_valid", input, is_valid_expected);
+ CheckScalarUnary("is_null", input, is_null_expected);
+ CheckScalarUnary("true_unless_null", input, true_unlesss_null_expected);
+}
+
+TEST_F(TestBooleanValidityKernels, LogicalNulls) {
+ auto null_dict =
+ DictArrayFromJSON(dictionary(int8(), int8()), "[0, 2, 1]", "[0, 1,
null]");
+ CheckValidityKernels(null_dict, ArrayFromJSON(boolean(), "[true, false,
true]"));
+ auto null_index =
+ DictArrayFromJSON(dictionary(int8(), int32()), "[null, 1, 0]", "[8, 2]");
+ CheckValidityKernels(null_index, ArrayFromJSON(boolean(), "[false, true,
true]"));
+ auto null_dict_and_index = DictArrayFromJSON(dictionary(int8(), boolean()),
+ "[1, null, 2, 0]", "[true,
false, null]");
+ CheckValidityKernels(null_dict_and_index,
+ ArrayFromJSON(boolean(), "[true, false, false, true]"));
+ // No validity bitmap at all
+ auto no_nulls = DictArrayFromJSON(dictionary(int8(), utf8()), "[0, 1, 2, 2]",
+ R"(["itsy", "bitsy", "spider"])");
+ CheckValidityKernels(no_nulls, ArrayFromJSON(boolean(), "[true, true, true,
true]"));
+
+ CheckValidityKernels(ArrayFromJSON(float64(), "[1.0, NaN, null, 2.0]"),
+ ArrayFromJSON(boolean(), "[true, true, false, true]"));
+
+ ASSERT_OK_AND_ASSIGN(auto ree,
+ RunEndEncode(ArrayFromJSON(int64(), "[11, 11, null,
null, 12]")));
+ CheckValidityKernels(ree, ArrayFromJSON(boolean(), "[true, true, false,
false, true]"));
+
+ ArrayVector children{
+ ArrayFromJSON(int64(), "[1, 23, 45, null, null, -2, null]"),
+ ArrayFromJSON(float32(), "[null, 1.1, 2.2, null, -4.0, 1.5, 0.1]"),
+ ArrayFromJSON(utf8(), R"(["alpha", "", "beta", null, "gamma", "delta",
null])"),
+ };
+ auto type_ids = ArrayFromJSON(int8(), "[0, 1, 2, 2, 0, 2, 1]");
+ auto fields = {field("a", int64()), field("b", float32()), field("c",
utf8())};
+ SparseUnionArray sparse(sparse_union(fields), 7, children,
+ type_ids->data()->buffers[1]);
+ ASSERT_OK(sparse.ValidateFull());
+ CheckValidityKernels(
+ sparse, ArrayFromJSON(boolean(), "[true, true, true, false, false, true,
true]"));
+
+ auto offsets = ArrayFromJSON(int32(), "[0, 0, 0, 2, 3, 6, 3]");
+ DenseUnionArray dense(dense_union(fields), 7, children,
type_ids->data()->buffers[1],
+ offsets->data()->buffers[1]);
+ ASSERT_OK(dense.ValidateFull());
+ CheckValidityKernels(
+ dense, ArrayFromJSON(boolean(), "[true, false, true, true, false, false,
false]"));
+}
+
TEST_F(TestBooleanValidityKernels, IsValidIsNullNullType) {
CheckScalarUnary("is_null", std::make_shared<NullArray>(5),
ArrayFromJSON(boolean(), "[true, true, true, true, true]"));
diff --git a/cpp/src/arrow/util/dict_util.cc b/cpp/src/arrow/util/dict_util.cc
index c93517140ca..abcfd240e67 100644
--- a/cpp/src/arrow/util/dict_util.cc
+++ b/cpp/src/arrow/util/dict_util.cc
@@ -18,8 +18,11 @@
#include "arrow/util/dict_util_internal.h"
#include "arrow/array/array_dict.h"
+#include "arrow/util/bit_run_reader.h"
#include "arrow/util/bit_util.h"
+#include "arrow/util/bitmap_ops.h"
#include "arrow/util/checked_cast.h"
+#include "arrow/util/logging_internal.h"
namespace arrow {
namespace dict_util {
@@ -28,33 +31,68 @@ namespace {
template <typename IndexArrowType>
int64_t LogicalNullCount(const ArraySpan& span) {
- const auto* indices_null_bit_map = span.buffers[0].data;
+ const auto* indices_valid_bitmap = span.buffers[0].data;
const auto& dictionary_span = span.dictionary();
- const auto* dictionary_null_bit_map = dictionary_span.buffers[0].data;
+ const auto* dictionary_valid_bitmap = dictionary_span.buffers[0].data;
using CType = typename IndexArrowType::c_type;
const CType* indices_data = span.GetValues<CType>(1);
int64_t null_count = 0;
- for (int64_t i = 0; i < span.length; i++) {
- if (indices_null_bit_map != nullptr &&
- !bit_util::GetBit(indices_null_bit_map, i + span.offset)) {
- null_count++;
- continue;
- }
-
- CType current_index = indices_data[i];
- if (!bit_util::GetBit(dictionary_null_bit_map,
- current_index + dictionary_span.offset)) {
- null_count++;
- }
- }
+ DCHECK_OK(internal::VisitBitRuns(
+ indices_valid_bitmap, span.offset, span.length,
+ [=, &null_count](int64_t position, int64_t length, bool valid) {
+ if (valid) {
+ // This is a run of valid indicies, so we need to look each of them
up
+ // in the dictionary array to see if they are null
+ for (int64_t i = position; i < position + length; i++) {
+ CType index = indices_data[i];
+ if (!bit_util::GetBit(dictionary_valid_bitmap,
+ index + dictionary_span.offset)) {
+ null_count++;
+ }
+ }
+ } else {
+ // This is a run of null indices
+ null_count += length;
+ }
+ return Status::OK();
+ }));
return null_count;
}
+template <typename IndexArrowType>
+void SetLogicalNullBits(const ArraySpan& span, uint8_t* out_bitmap, int64_t
out_offset,
+ bool set_on_null) {
+ const auto* indices_valid_bitmap = span.buffers[0].data;
+ const auto& dictionary_span = span.dictionary();
+ const auto* dictionary_valid_bitmap = dictionary_span.buffers[0].data;
+
+ using CType = typename IndexArrowType::c_type;
+ const CType* indices_data = span.GetValues<CType>(1);
+ DCHECK_OK(internal::VisitBitRuns(
+ indices_valid_bitmap, span.offset, span.length,
+ [=](int64_t position, int64_t length, bool valid) {
+ if (valid) {
+ // This is a run of valid indicies, so we need to look each of them
up
+ // in the dictionary array to see if they are null
+ for (int64_t i = position; i < position + length; i++) {
+ CType index = indices_data[i];
+ bool is_null = !bit_util::GetBit(dictionary_valid_bitmap,
+ index + dictionary_span.offset);
+ bit_util::SetBitTo(out_bitmap, out_offset + i, is_null ==
set_on_null);
+ }
+ } else {
+ // This is a run of null indices
+ bit_util::SetBitsTo(out_bitmap, out_offset + position, length,
set_on_null);
+ }
+ return Status::OK();
+ }));
+}
+
} // namespace
int64_t LogicalNullCount(const ArraySpan& span) {
- if (span.dictionary().GetNullCount() == 0 || span.length == 0) {
+ if (span.dictionary().GetNullCount() == 0) {
return span.GetNullCount();
}
@@ -78,5 +116,54 @@ int64_t LogicalNullCount(const ArraySpan& span) {
return LogicalNullCount<Int64Type>(span);
}
}
+
+void SetLogicalNullBits(const ArraySpan& span, uint8_t* out_bitmap, int64_t
out_offset,
+ bool set_on_null) {
+ if (span.dictionary().GetNullCount() == 0) {
+ // No nulls in dictionary, so a value is a logical null if and only if
+ // its index is null
+ if (span.GetNullCount() == 0) {
+ // No null indices either (and possibly no validity bitmap at all), so
+ // there are no logical nulls
+ bit_util::SetBitsTo(out_bitmap, out_offset, span.length, !set_on_null);
+ } else if (set_on_null) {
+ internal::InvertBitmap(span.buffers[0].data, span.offset, span.length,
out_bitmap,
+ out_offset);
+ } else {
+ internal::CopyBitmap(span.buffers[0].data, span.offset, span.length,
out_bitmap,
+ out_offset);
+ }
+ return;
+ }
+
+ const auto& dict_array_type = internal::checked_cast<const
DictionaryType&>(*span.type);
+ switch (dict_array_type.index_type()->id()) {
+ case Type::UINT8:
+ SetLogicalNullBits<UInt8Type>(span, out_bitmap, out_offset, set_on_null);
+ break;
+ case Type::INT8:
+ SetLogicalNullBits<Int8Type>(span, out_bitmap, out_offset, set_on_null);
+ break;
+ case Type::UINT16:
+ SetLogicalNullBits<UInt16Type>(span, out_bitmap, out_offset,
set_on_null);
+ break;
+ case Type::INT16:
+ SetLogicalNullBits<Int16Type>(span, out_bitmap, out_offset, set_on_null);
+ break;
+ case Type::UINT32:
+ SetLogicalNullBits<UInt32Type>(span, out_bitmap, out_offset,
set_on_null);
+ break;
+ case Type::INT32:
+ SetLogicalNullBits<Int32Type>(span, out_bitmap, out_offset, set_on_null);
+ break;
+ case Type::UINT64:
+ SetLogicalNullBits<UInt64Type>(span, out_bitmap, out_offset,
set_on_null);
+ break;
+ default:
+ SetLogicalNullBits<Int64Type>(span, out_bitmap, out_offset, set_on_null);
+ break;
+ }
+}
+
} // namespace dict_util
} // namespace arrow
diff --git a/cpp/src/arrow/util/dict_util_internal.h
b/cpp/src/arrow/util/dict_util_internal.h
index a92733ae0f6..1cf70e7ca44 100644
--- a/cpp/src/arrow/util/dict_util_internal.h
+++ b/cpp/src/arrow/util/dict_util_internal.h
@@ -22,7 +22,25 @@
namespace arrow {
namespace dict_util {
+/// \brief Compute the logical null count of a dictionary-encoded array
+///
+/// This function considers a value to be a logical null if its index is null
or if
+/// it points to a null entry in the dictionary's validity bitmap. This means
that it
+/// only accounts for physical nulls in the dictionary. It does not recurse
into the
+/// dictionary values to find logical nulls, which would matter when those
values are
+/// themselves dictionary, run-end encoded, or union typed.
+ARROW_EXPORT
int64_t LogicalNullCount(const ArraySpan& span);
+/// \brief Populate a bitmap based on the logical nulls in a
dictionary-encoded array
+///
+/// `set_on_null` is true if we should set bits corresponding to nulls and
false if
+/// we should set bits corresponding to non-nulls
+///
+/// \see LogicalNullCount for how nulls in the dictionary are handled
+ARROW_EXPORT
+void SetLogicalNullBits(const ArraySpan& span, uint8_t* out_bitmap, int64_t
out_offset,
+ bool set_on_null);
+
} // namespace dict_util
} // namespace arrow
diff --git a/cpp/src/arrow/util/ree_util.cc b/cpp/src/arrow/util/ree_util.cc
index 461d6804b8c..8685f70e9ce 100644
--- a/cpp/src/arrow/util/ree_util.cc
+++ b/cpp/src/arrow/util/ree_util.cc
@@ -47,6 +47,23 @@ int64_t LogicalNullCount(const ArraySpan& span) {
return null_count;
}
+template <typename RunEndCType>
+void SetLogicalNullBits(const ArraySpan& span, uint8_t* out_bitmap, int64_t
out_offset,
+ bool set_on_null) {
+ const auto& values = ValuesArray(span);
+ const auto& values_bitmap = values.buffers[0].data;
+
+ RunEndEncodedArraySpan<RunEndCType> ree_span(span);
+ auto end = ree_span.end();
+ for (auto it = ree_span.begin(); it != end; ++it) {
+ const bool is_null =
+ values_bitmap &&
+ !bit_util::GetBit(values_bitmap, values.offset +
it.index_into_array());
+ bit_util::SetBitsTo(out_bitmap, out_offset, it.run_length(), set_on_null
== is_null);
+ out_offset += it.run_length();
+ }
+}
+
} // namespace
int64_t LogicalNullCount(const ArraySpan& span) {
@@ -61,6 +78,19 @@ int64_t LogicalNullCount(const ArraySpan& span) {
return LogicalNullCount<int64_t>(span);
}
+void SetLogicalNullBits(const ArraySpan& span, uint8_t* out_bitmap, int64_t
out_offset,
+ bool set_on_null) {
+ const auto type_id = RunEndsArray(span).type->id();
+ if (type_id == Type::INT16) {
+ SetLogicalNullBits<int16_t>(span, out_bitmap, out_offset, set_on_null);
+ } else if (type_id == Type::INT32) {
+ SetLogicalNullBits<int32_t>(span, out_bitmap, out_offset, set_on_null);
+ } else {
+ DCHECK_EQ(type_id, Type::INT64);
+ SetLogicalNullBits<int64_t>(span, out_bitmap, out_offset, set_on_null);
+ }
+}
+
namespace internal {
/// \pre 0 <= i < array_span.length()
diff --git a/cpp/src/arrow/util/ree_util.h b/cpp/src/arrow/util/ree_util.h
index 5c759f2e80d..8498c69225e 100644
--- a/cpp/src/arrow/util/ree_util.h
+++ b/cpp/src/arrow/util/ree_util.h
@@ -48,6 +48,7 @@ const RunEndCType* RunEnds(const ArraySpan& span) {
/// All the checks complete in O(1) time. Consequently, this function:
/// - DOES NOT check that run_ends is sorted and all-positive
/// - DOES NOT check the actual contents of the run_ends and values arrays
+ARROW_EXPORT
Status ValidateRunEndEncodedChildren(const RunEndEncodedType& type,
int64_t logical_length,
const std::shared_ptr<ArrayData>&
run_ends_data,
@@ -55,8 +56,17 @@ Status ValidateRunEndEncodedChildren(const
RunEndEncodedType& type,
int64_t null_count, int64_t
logical_offset);
/// \brief Compute the logical null count of an REE array
+ARROW_EXPORT
int64_t LogicalNullCount(const ArraySpan& span);
+/// \brief Populate a bitmap based on the logical nulls in an REE array
+///
+/// `set_on_null` is true if we should set bits corresponding to nulls and
false if
+/// we should set bits corresponding to non-nulls
+ARROW_EXPORT
+void SetLogicalNullBits(const ArraySpan& span, uint8_t* out_bitmap, int64_t
out_offset,
+ bool set_on_null);
+
namespace internal {
/// \brief Uses binary-search to find the physical offset given a logical
offset
diff --git a/cpp/src/arrow/util/union_util.cc b/cpp/src/arrow/util/union_util.cc
index 6b4d752d868..56438f537d7 100644
--- a/cpp/src/arrow/util/union_util.cc
+++ b/cpp/src/arrow/util/union_util.cc
@@ -55,4 +55,18 @@ int64_t LogicalDenseUnionNullCount(const ArraySpan& span) {
return null_count;
}
+void SetLogicalNullBitsSparse(const ArraySpan& span, uint8_t* out_bitmap,
+ int64_t out_offset, bool set_on_null) {
+ for (int64_t i = 0; i < span.length; i++) {
+ bit_util::SetBitTo(out_bitmap, out_offset + i, set_on_null ==
span.IsNull(i));
+ }
+}
+
+void SetLogicalNullBitsDense(const ArraySpan& span, uint8_t* out_bitmap,
+ int64_t out_offset, bool set_on_null) {
+ for (int64_t i = 0; i < span.length; i++) {
+ bit_util::SetBitTo(out_bitmap, out_offset + i, set_on_null ==
span.IsNull(i));
+ }
+}
+
} // namespace arrow::union_util
diff --git a/cpp/src/arrow/util/union_util.h b/cpp/src/arrow/util/union_util.h
index 0f30d5a3278..e267bb92e3f 100644
--- a/cpp/src/arrow/util/union_util.h
+++ b/cpp/src/arrow/util/union_util.h
@@ -22,10 +22,28 @@ namespace arrow {
namespace union_util {
/// \brief Compute the number of of logical nulls in a sparse union array
+ARROW_EXPORT
int64_t LogicalSparseUnionNullCount(const ArraySpan& span);
/// \brief Compute the number of of logical nulls in a dense union array
+ARROW_EXPORT
int64_t LogicalDenseUnionNullCount(const ArraySpan& span);
+/// \brief Populate a bitmap based on the logical nulls in a sparse union array
+///
+/// `set_on_null` is true if we should set bits corresponding to nulls and
false if
+/// we should set bits corresponding to non-nulls
+ARROW_EXPORT
+void SetLogicalNullBitsSparse(const ArraySpan& span, uint8_t* out_bitmap,
+ int64_t out_offset, bool set_on_null);
+
+/// \brief Populate a bitmap based on the logical nulls in a dense union array
+///
+/// `set_on_null` is true if we should set bits corresponding to nulls and
false if
+/// we should set bits corresponding to non-nulls
+ARROW_EXPORT
+void SetLogicalNullBitsDense(const ArraySpan& span, uint8_t* out_bitmap,
+ int64_t out_offset, bool set_on_null);
+
} // namespace union_util
} // namespace arrow
