This is an automated email from the ASF dual-hosted git repository.
pitrou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new 626407c899 GH-51224: [C++] Keep the correct nulls when winsorizing a
sliced array (#51251)
626407c899 is described below
commit 626407c899759168343d5cd86674136f05abf3b6
Author: Stefan Wang <[email protected]>
AuthorDate: Mon Sep 28 01:08:32 2026 -0700
GH-51224: [C++] Keep the correct nulls when winsorizing a sliced array
(#51251)
### Rationale for this change
Winsorizing a sliced array can put nulls in the wrong positions. Slices
containing only nulls or NaNs can also return values from outside the slice.
Closes https://github.com/apache/arrow/issues/51224.
### What changes are included in this PR?
Use the bitmap helper to match validity bits to the output values. Preserve
the input offset when returning the original buffers.
### Are these changes tested?
I ran the native C++ tests and PyArrow on macOS arm64 with Apple clang 21.
The new regression covers null-only, NaN-only and mixed null/NaN slices, with
chunked inputs as controls.
With PyArrow loading the native library being tested, save this as
repro-winsorize.py:
```python
import pyarrow as pa
import pyarrow.compute as pc
parent = pa.array([1.0, 2.0, None, 4.0, None, 6.0, 7.0, 8.0])
result = pc.winsorize(parent.slice(2, 5), lower_limit=0.0, upper_limit=1.0)
print("mixed slice:", result.to_pylist(), flush=True)
parent = pa.array([1.0, 2.0, None, None, None, 3.0])
nulls = pc.winsorize(parent.slice(2, 3), lower_limit=0.0, upper_limit=1.0)
print("all-null slice:", nulls.to_pylist(), flush=True)
assert result.to_pylist() == [None, 4.0, None, 6.0, 7.0]
assert nulls.to_pylist() == [None, None, None]
```
<details><summary>Raw logs</summary>
Before:
```console
$ python repro-winsorize.py
mixed slice: [0.0, 4.0, None, 6.0, None]
all-null slice: [1.0, 2.0, None]
AssertionError
```
After:
```console
$ python repro-winsorize.py
mixed slice: [None, 4.0, None, 6.0, 7.0]
all-null slice: [None, None, None]
exit_code=0
$ build/release/arrow-compute-vector-test
[ PASSED ] 1157 tests.
$ build/release/arrow-compute-aggregate-test
--gtest_filter='*Quantile*:*TDigest*'
[ PASSED ] 28 tests.
```
</details>
### Are there any user-facing changes?
Sliced inputs retain the correct values and null positions.
### Was AI used for this PR?
In accordance to the [AI generation
guidelines](https://arrow.apache.org/docs/dev/developers/overview.html#ai-generated-code),
please disclose below whether and how AI was used in this PR.
**PR code and description written by:**
- [ ] Human
- [x] AI
**Reviewed before submission by:**
- [x] Human
- [ ] AI
- [ ] Not reviewed
* GitHub Issue: #51224
Authored-by: 1fanwang <[email protected]>
Signed-off-by: Antoine Pitrou <[email protected]>
---
cpp/src/arrow/compute/kernels/vector_statistics.cc | 6 +++-
.../compute/kernels/vector_statistics_test.cc | 39 ++++++++++++++++++++++
2 files changed, 44 insertions(+), 1 deletion(-)
diff --git a/cpp/src/arrow/compute/kernels/vector_statistics.cc
b/cpp/src/arrow/compute/kernels/vector_statistics.cc
index 074f2ec0a7..cc2fbc4016 100644
--- a/cpp/src/arrow/compute/kernels/vector_statistics.cc
+++ b/cpp/src/arrow/compute/kernels/vector_statistics.cc
@@ -26,6 +26,7 @@
#include "arrow/compute/function.h"
#include "arrow/compute/kernel.h"
#include "arrow/compute/kernels/codegen_internal.h"
+#include "arrow/compute/kernels/util_internal.h"
#include "arrow/compute/registry.h"
#include "arrow/compute/registry_internal.h"
#include "arrow/result.h"
@@ -71,6 +72,7 @@ struct Winsorize {
// Only nulls and NaNs => return input as-is
out_data->null_count = data->null_count.load();
out_data->length = data->length;
+ out_data->offset = data->offset;
out_data->buffers = data->buffers;
return Status::OK();
}
@@ -127,7 +129,9 @@ struct Winsorize {
DCHECK_EQ(out->buffers.size(), data.buffers.size());
out->null_count = data.null_count.load();
out->length = data.length;
- out->buffers[0] = data.buffers[0];
+ out->offset = 0;
+ ARROW_ASSIGN_OR_RAISE(out->buffers[0],
+ GetOrCopyNullBitmapBuffer(data, ctx->memory_pool()));
ARROW_ASSIGN_OR_RAISE(out->buffers[1], ctx->Allocate(out->length *
sizeof(CType)));
// Avoid leaving uninitialized memory under null entries
std::memset(out->buffers[1]->mutable_data(), 0, out->length *
sizeof(CType));
diff --git a/cpp/src/arrow/compute/kernels/vector_statistics_test.cc
b/cpp/src/arrow/compute/kernels/vector_statistics_test.cc
index 97715cdaed..72a765be10 100644
--- a/cpp/src/arrow/compute/kernels/vector_statistics_test.cc
+++ b/cpp/src/arrow/compute/kernels/vector_statistics_test.cc
@@ -87,6 +87,45 @@ TEST_F(TestWinsorize, FloatingPoint) {
}
}
+TEST_F(TestWinsorize, SlicedInput) {
+ options_.lower_limit = 0.0;
+ options_.upper_limit = 1.0;
+ auto parent = ArrayFromJSON(float64(), "[1.1, 2.2, null, 4.4, null, 6.6,
7.7, 8.8]");
+ auto expected = ArrayFromJSON(float64(), "[null, 4.4, null, 6.6, 7.7]");
+ CheckWinsorize(parent->Slice(2, 5), expected);
+
+ options_.lower_limit = 0.25;
+ options_.upper_limit = 0.75;
+ auto dense = ArrayFromJSON(float64(), "[1.0, 2.0, 3.0, 44.0, 55.0, 66.0,
77.0]");
+ CheckWinsorize(dense->Slice(1, 5),
+ ArrayFromJSON(float64(), "[3.0, 3.0, 44.0, 55.0, 55.0]"));
+}
+
+TEST_F(TestWinsorize, SlicedChunkedInput) {
+ options_.lower_limit = 0.0;
+ options_.upper_limit = 1.0;
+ auto parent = ArrayFromJSON(float64(), "[1.1, 2.2, null, 4.4, null, 6.6,
7.7, 8.8]");
+ auto chunked = std::make_shared<ChunkedArray>(
+ ArrayVector{parent->Slice(2, 3), parent->Slice(5, 3)});
+ auto expected = std::make_shared<ChunkedArray>(ArrayVector{
+ ArrayFromJSON(float64(), "[null, 4.4, null]"),
+ ArrayFromJSON(float64(), "[6.6, 7.7, 8.8]"),
+ });
+ CheckWinsorize(chunked, expected);
+}
+
+TEST_F(TestWinsorize, SlicedInputWithoutQuantiles) {
+ for (const auto* json_input :
+ {"[1, 2, null, null, null, 3]", "[1, 2, NaN, null, NaN, 3]",
+ "[1, 2, NaN, NaN, NaN, 3]"}) {
+ auto parent = ArrayFromJSON(float64(), json_input);
+ auto sliced = parent->Slice(2, 3);
+ CheckWinsorize(sliced, sliced);
+ auto chunked = std::make_shared<ChunkedArray>(ArrayVector{sliced});
+ CheckWinsorize(chunked, chunked);
+ }
+}
+
TEST_F(TestWinsorize, Integral) {
for (auto type : IntTypes()) {
options_.lower_limit = 0.25;
