This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new 4170a315 test(parquet): reduce macOS race test memory pressure (#1275)
4170a315 is described below
commit 4170a3150de192460124d83b500c071d2c03c12e
Author: Matt Topol <[email protected]>
AuthorDate: Thu Sep 3 10:39:14 2026 -0400
test(parquet): reduce macOS race test memory pressure (#1275)
### Rationale for this change
The macOS test jobs run on 7 GB ARM64 runners and intermittently kill
`parquet/file` while running under the race detector. Two large-value
regression tests each write more than 1 GB and retained all encoded
output in memory. Package-level parallelism adds further pressure.
Example failure:
https://github.com/apache/arrow-go/actions/runs/33590782213/job/100362942846?pr=1241
### What changes are included in this PR?
- Replace the retained `bytes.Buffer` output in the two large-value
tests with a counting writer.
- Run Parquet packages serially (`-p=1`) on Darwin to bound
package-level memory pressure.
- Correct both macOS job names from AMD64 to ARM64.
### Are these changes tested?
- `go test -race -tags assert ./parquet/file -count=1`
- `go test -race -tags assert,noasm ./parquet/file -count=1`
- `bash -n ci/scripts/test.sh`
- `git diff --check`
Both formerly high-memory tests also passed independently under `-race
-tags assert,noasm`.
### Are there any user-facing changes?
No.
---
.github/workflows/test.yml | 4 ++--
ci/scripts/test.sh | 10 ++++++++--
parquet/file/large_value_test.go | 21 +++++++++++++++------
3 files changed, 25 insertions(+), 10 deletions(-)
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 6b7e713b..d3e13d50 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -161,7 +161,7 @@ jobs:
run: |
docker compose push debian-cgo-python
macos:
- name: AMD64 macOS 14 Go ${{ matrix.go }}
+ name: ARM64 macOS 14 Go ${{ matrix.go }}
runs-on: macos-14
timeout-minutes: 20
strategy:
@@ -188,7 +188,7 @@ jobs:
run: |
ci/scripts/test.sh $(pwd)
macos-cgo:
- name: AMD64 macOS 14 Go ${{ matrix.go }} - CGO
+ name: ARM64 macOS 14 Go ${{ matrix.go }} - CGO
runs-on: macos-14
timeout-minutes: 25
strategy:
diff --git a/ci/scripts/test.sh b/ci/scripts/test.sh
index 45064f0a..21677047 100755
--- a/ci/scripts/test.sh
+++ b/ci/scripts/test.sh
@@ -75,9 +75,15 @@ popd
pushd "${source_dir}/parquet"
-go test "${test_args[@]}" -tags assert ./...
+parquet_test_args=("${test_args[@]}")
+if [[ "$(go env GOOS)" = "darwin" ]]; then
+ # Keep package-level memory bounded on the 7 GB macOS ARM64 runners.
+ parquet_test_args+=("-p=1")
+fi
+
+go test "${parquet_test_args[@]}" -tags assert ./...
# run the tests again but with the noasm tag
-go test "${test_args[@]}" -tags assert,noasm ./...
+go test "${parquet_test_args[@]}" -tags assert,noasm ./...
popd
diff --git a/parquet/file/large_value_test.go b/parquet/file/large_value_test.go
index 7791169d..0c266979 100644
--- a/parquet/file/large_value_test.go
+++ b/parquet/file/large_value_test.go
@@ -34,12 +34,21 @@ import (
"github.com/stretchr/testify/require"
)
+type countingWriter struct {
+ bytesWritten int64
+}
+
+func (w *countingWriter) Write(p []byte) (int, error) {
+ w.bytesWritten += int64(len(p))
+ return len(p), nil
+}
+
// TestLargeByteArrayValuesDoNotOverflowInt32 tests that writing large byte
array
// values totalling over 1GB in a single WriteBatch call triggers adaptive
batch
// sizing and does not cause an int32 overflow panic in FlushCurrentPage.
//
-// Memory note: input values all share one 1.5MB buffer so input memory is low,
-// but the parquet output buffer grows to ~1GB (unavoidable for this boundary
test).
+// Memory note: input values all share one 1.5MB buffer so input memory is low.
+// Use a counting writer so the test does not retain more than 1GB of output.
func TestLargeByteArrayValuesDoNotOverflowInt32(t *testing.T) {
if runtime.GOARCH == "386" {
t.Skip("Skipping test on 32-bit architecture")
@@ -57,7 +66,7 @@ func TestLargeByteArrayValuesDoNotOverflowInt32(t *testing.T)
{
parquet.WithDataPageSize(1024*1024),
)
- out := &bytes.Buffer{}
+ out := &countingWriter{}
writer := file.NewParquetWriter(out, sc.Root(),
file.WithWriterProps(props))
defer writer.Close()
@@ -86,7 +95,7 @@ func TestLargeByteArrayValuesDoNotOverflowInt32(t *testing.T)
{
assert.NoError(t, colWriter.Close())
assert.NoError(t, rgw.Close())
assert.NoError(t, writer.Close())
- assert.Greater(t, out.Len(), 0)
+ assert.Greater(t, out.bytesWritten, int64(0))
}
// TestLargeStringArrayWithArrow tests the pqarrow integration path with large
values.
@@ -101,7 +110,7 @@ func TestLargeStringArrayWithArrow(t *testing.T) {
field := arrow.Field{Name: "large_strings", Type:
arrow.BinaryTypes.LargeString, Nullable: true}
arrowSchema := arrow.NewSchema([]arrow.Field{field}, nil)
- out := &bytes.Buffer{}
+ out := &countingWriter{}
props := parquet.NewWriterProperties(
parquet.WithStats(false),
parquet.WithVersion(parquet.V2_LATEST),
@@ -137,7 +146,7 @@ func TestLargeStringArrayWithArrow(t *testing.T) {
}
assert.NoError(t, pqw.Close())
- assert.Greater(t, out.Len(), 0)
+ assert.Greater(t, out.bytesWritten, int64(0))
}
// TestLargeByteArrayRoundTripCorrectness verifies that ByteArray values
