Script 'mail_helper' called by obssrc Hello community, here is the log from the commit of package apache-arrow for openSUSE:Factory checked in at 2026-10-20 12:42:40 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/apache-arrow (Old) and /work/SRC/openSUSE:Factory/.apache-arrow.new.3698736 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "apache-arrow" Tue Oct 20 12:42:40 2026 rev:25 rq:1386361 version:26.0.0 Changes: -------- --- /work/SRC/openSUSE:Factory/apache-arrow/apache-arrow.changes 2026-07-31 16:05:48.073284531 +0200 +++ /work/SRC/openSUSE:Factory/.apache-arrow.new.3698736/apache-arrow.changes 2026-10-20 12:43:12.445756042 +0200 @@ -1,0 +2,37 @@ +Sun Oct 18 15:05:00 UTC 2026 - Martin Pluskal <[email protected]> + +- Switch to system mimalloc and system simdjson: + * Link Factory mimalloc via use-system-mimalloc.patch + (adapted from Debian, gh#apache/arrow#42211 still open + upstream) and build with simdjson_SOURCE=SYSTEM (needs + simdjson >= 4.3.0, resolved from this project for now). + * Drop the bundled tarballs, the ARROW_*_URL exports and the + interim bundled() Provides; add BuildRequires on + pkgconfig(mimalloc) and pkgconfig(simdjson). + * Spec cleanup via spec-cleaner (drop Group tags, reorder). +- CVE-2024-41178 (boo#1228262): this package is not affected; + the flaw is in the Rust object_store crate and only the C++ + libraries are built here. +- Arrow C++ 26.0.0 already contains the 23.0.1 fix for + CVE-2026-25087 (boo#1258390), a use-after-free in IPC file + reading with pre-buffering. + +------------------------------------------------------------------- +Sat Oct 17 18:31:03 UTC 2026 - Martin Pluskal <[email protected]> + +- Update to 26.0.0 (libarrow soname 2500 -> 2600): + * Compute: new remainder/modulo and search-sorted kernels, scalar + AppendScalars, float16 hash kernels, duration temporal rounding, and + CSV reader fixes (embedded NUL bytes, missing or extra columns) + * Parquet: dense FIXED_LEN_BYTE_ARRAY decoding, settable MemoryPool on + readers, nesting-depth limit, synced thrift 2.14; Feather reader + and writer are deprecated + * Python: requires Python >= 3.11 and NumPy >= 2.0, adds 3.15 support, + drops 3.10 and pandas < 2.0.3; DLPack 1.3 producer support + * Hardening: heap-buffer-overflow fix in FillNullForward/Backward, + several Gandiva out-of-bounds read fixes, stricter binary and map + validation (no CVE IDs assigned upstream) + * Bundle simdjson 4.6.4 (now required for Parquet, not in Factory), + refresh mimalloc to 3.4.1 and the parquet-testing data snapshot + +------------------------------------------------------------------- --- /work/SRC/openSUSE:Factory/apache-arrow/python-pyarrow.changes 2026-09-28 10:42:37.525333202 +0200 +++ /work/SRC/openSUSE:Factory/.apache-arrow.new.3698736/python-pyarrow.changes 2026-10-20 12:43:12.521759221 +0200 @@ -1,0 +2,8 @@ +Sat Oct 17 18:31:03 UTC 2026 - Martin Pluskal <[email protected]> + +- Update to 26.0.0: + * Requires Python >= 3.11 and NumPy >= 2.0; adds Python 3.15 support, + drops 3.10 and pandas older than 2.0.3 + * DLPack 1.3 producer support and search-sorted compute kernel + +------------------------------------------------------------------- Old: ---- apache-arrow-25.0.0.tar.gz arrow-testing-25.0.0.tar.gz mimalloc-v3.3.1.tar.gz parquet-testing-25.0.0.tar.gz New: ---- apache-arrow-26.0.0.tar.gz arrow-testing-26.0.0.tar.gz parquet-testing-26.0.0.tar.gz use-system-mimalloc.patch ----------(New B)---------- New:/work/SRC/openSUSE:Factory/.apache-arrow.new.3698736/apache-arrow.changes-- Switch to system mimalloc and system simdjson: /work/SRC/openSUSE:Factory/.apache-arrow.new.3698736/apache-arrow.changes: * Link Factory mimalloc via use-system-mimalloc.patch /work/SRC/openSUSE:Factory/.apache-arrow.new.3698736/apache-arrow.changes- (adapted from Debian, gh#apache/arrow#42211 still open ----------(New E)---------- ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ apache-arrow.spec ++++++ --- /var/tmp/diff_new_pack.lxoq3u/_old 2026-10-20 12:43:13.787812185 +0200 +++ /var/tmp/diff_new_pack.lxoq3u/_new 2026-10-20 12:43:13.790812311 +0200 @@ -16,38 +16,32 @@ # -%bcond_without tests -%bcond_without flight - -# Remove static build due to devel-static packages being required by the generated CMake Targets -%bcond_with static -# Required for runtime dispatch, not yet packaged -%bcond_with xsimd - -%if %{suse_version} <= 1500 +%if 0%{?suse_version} <= 1500 # requires __has_builtin with keywords %define gccver 13 %endif - -%define sonum 2500 +%define sonum 2600 # See git submodule /testing pointing to the correct revision %define arrow_testing_commit 9ff285c88565f0f6abc855918c6a342e70e4909c # See git submodule /cpp/submodules/parquet-testing pointing to the correct revision -%define parquet_testing_commit e74785d85a4ecee829e1e405444d6a1b24b8bc9c -# See cpp/thirdparty/versions.txt, replace by BuildRequires: pkgconfig(mimalloc) as soon as gh#apache/arrow#42211 is resolved -%define arrow_mimalloc_build_version v3.3.1 +%define parquet_testing_commit 56653c437c8092f704a092d0d1d4e600124cd49f +%bcond_without tests +%bcond_without flight +# Remove static build due to devel-static packages being required by the generated CMake Targets +%bcond_with static +# Required for runtime dispatch, not yet packaged +%bcond_with xsimd Name: apache-arrow -Version: 25.0.0 +Version: 26.0.0 Release: 0 Summary: A development platform for in-memory data License: Apache-2.0 AND BSD-3-Clause AND BSD-2-Clause AND MIT -Group: Development/Tools/Other URL: https://arrow.apache.org/ # SourceRepository: https://github.com/apache/arrow Source0: https://github.com/apache/arrow/archive/apache-arrow-%{version}.tar.gz Source1: https://github.com/apache/arrow-testing/archive/%{arrow_testing_commit}.tar.gz#/arrow-testing-%{version}.tar.gz Source2: https://github.com/apache/parquet-testing/archive/%{parquet_testing_commit}.tar.gz#/parquet-testing-%{version}.tar.gz -Source3: https://github.com/microsoft/mimalloc/archive/%{arrow_mimalloc_build_version}.tar.gz#/mimalloc-%{arrow_mimalloc_build_version}.tar.gz +Patch0: use-system-mimalloc.patch BuildRequires: bison BuildRequires: cmake >= 3.25 BuildRequires: fdupes @@ -58,12 +52,6 @@ BuildRequires: libboost_filesystem-devel BuildRequires: libboost_headers-devel BuildRequires: libboost_process-devel -%if 0%{?suse_version} < 1699 -BuildRequires: libboost_system-devel -%endif -%if %{with static} -BuildRequires: libzstd-devel-static -%endif BuildRequires: pkgconfig BuildRequires: python-rpm-macros BuildRequires: python3-base @@ -85,11 +73,23 @@ BuildRequires: pkgconfig(liburiparser) >= 0.9.3 BuildRequires: pkgconfig(libutf8proc) >= 2.10.0 BuildRequires: pkgconfig(libzstd) >= 1.5.7 +# cpp/thirdparty/versions.txt wants mimalloc 3.4.1; Factory ships 3.3.2 +# with a compatible API (Debian links its system mimalloc the same way) +BuildRequires: pkgconfig(mimalloc) BuildRequires: pkgconfig(protobuf) >= 21.3 +# ARROW_SIMDJSON_REQUIRED_VERSION is 4.3.0 (ThirdpartyToolchain.cmake); +# resolved from the in-project simdjson package until it lands in Factory +BuildRequires: pkgconfig(simdjson) >= 4.3.0 BuildRequires: pkgconfig(sqlite3) BuildRequires: pkgconfig(thrift) >= 0.22.0 BuildRequires: pkgconfig(xsimd) >= 14.2.0 BuildRequires: pkgconfig(zlib) >= 1.3.1 +%if 0%{?suse_version} < 1699 +BuildRequires: libboost_system-devel +%endif +%if %{with static} +BuildRequires: libzstd-devel-static +%endif %if %{with tests} BuildRequires: timezone BuildRequires: pkgconfig(gmock) >= 1.10 @@ -106,7 +106,6 @@ %package -n libarrow%{sonum} Summary: Development platform for in-memory data - shared library -Group: System/Libraries %description -n libarrow%{sonum} Apache Arrow is a cross-language development platform for in-memory @@ -120,7 +119,6 @@ %package -n libarrow_acero%{sonum} Summary: Development platform for in-memory data - shared library -Group: System/Libraries %description -n libarrow_acero%{sonum} Apache Arrow is a cross-language development platform for in-memory @@ -134,7 +132,6 @@ %package -n libarrow_compute%{sonum} Summary: Development platform for in-memory data - shared library -Group: System/Libraries %description -n libarrow_compute%{sonum} Apache Arrow is a cross-language development platform for in-memory @@ -148,7 +145,6 @@ %package -n libarrow_flight%{sonum} Summary: Development platform for in-memory data - shared library -Group: System/Libraries %description -n libarrow_flight%{sonum} Apache Arrow is a cross-language development platform for in-memory @@ -162,7 +158,6 @@ %package -n libarrow_flight_sql%{sonum} Summary: Development platform for in-memory data - shared library -Group: System/Libraries %description -n libarrow_flight_sql%{sonum} Apache Arrow is a cross-language development platform for in-memory @@ -176,7 +171,6 @@ %package -n libarrow_dataset%{sonum} Summary: Development platform for in-memory data - shared library -Group: System/Libraries %description -n libarrow_dataset%{sonum} Apache Arrow is a cross-language development platform for in-memory @@ -190,7 +184,6 @@ %package -n libparquet%{sonum} Summary: Development platform for in-memory data - shared library -Group: System/Libraries %description -n libparquet%{sonum} Apache Arrow is a cross-language development platform for in-memory @@ -204,7 +197,6 @@ %package devel Summary: Development platform for in-memory data - development files -Group: Development/Libraries/C and C++ Requires: libarrow%{sonum} = %{version} Requires: libarrow_acero%{sonum} = %{version} Requires: libarrow_compute%{sonum} = %{version} @@ -214,10 +206,10 @@ Requires: libarrow_flight_sql%{sonum} = %{version} %endif %if %{with static} -Suggests: %{name}-devel-static = %{version} Suggests: %{name}-acero-devel-static = %{version} Suggests: %{name}-compute-devel-static = %{version} Suggests: %{name}-dataset-devel-static = %{version} +Suggests: %{name}-devel-static = %{version} %if %{with flight} Suggests: %{name}-flight-devel-static = %{version} Suggests: %{name}-flight-sql-devel-static = %{version} @@ -238,7 +230,6 @@ %if %{with static} %package devel-static Summary: Development platform for in-memory data - development files -Group: Development/Libraries/C and C++ Requires: %{name}-devel = %{version} %description devel-static @@ -253,7 +244,6 @@ %package acero-devel-static Summary: Development platform for in-memory data - development files -Group: Development/Libraries/C and C++ Requires: %{name}-devel = %{version} %description acero-devel-static @@ -268,7 +258,6 @@ %package compute-devel-static Summary: Development platform for in-memory data - development files -Group: Development/Libraries/C and C++ Requires: %{name}-devel = %{version} %description compute-devel-static @@ -283,7 +272,6 @@ %package flight-devel-static Summary: Development platform for in-memory data - development files -Group: Development/Libraries/C and C++ Requires: %{name}-devel = %{version} %description flight-devel-static @@ -298,7 +286,6 @@ %package flight-sql-devel-static Summary: Development platform for in-memory data - development files -Group: Development/Libraries/C and C++ Requires: %{name}-devel = %{version} %description flight-sql-devel-static @@ -313,7 +300,6 @@ %package dataset-devel-static Summary: Development platform for in-memory data - development files -Group: Development/Libraries/C and C++ Requires: %{name}-devel = %{version} %description dataset-devel-static @@ -329,7 +315,6 @@ %package -n apache-parquet-devel Summary: Development platform for in-memory data - development files -Group: Development/Libraries/C and C++ Requires: libparquet%{sonum} = %{version} %description -n apache-parquet-devel @@ -346,7 +331,6 @@ %if %{with static} %package -n apache-parquet-devel-static Summary: Development platform for in-memory data - development files -Group: Development/Libraries/C and C++ Requires: apache-parquet-devel = %{version} %description -n apache-parquet-devel-static @@ -362,7 +346,6 @@ %package -n apache-parquet-utils Summary: Development platform for in-memory data - development files -Group: Productivity/Scientific/Math %description -n apache-parquet-utils Apache Arrow is a cross-language development platform for in-memory @@ -376,13 +359,13 @@ %prep %setup -q -n arrow-apache-arrow-%{version} -a1 -a2 +%patch -P 0 -p1 %build %{?gccver:export CXX=g++-%{gccver}} %{?gccver:export CC=gcc-%{gccver}} export CFLAGS="%{optflags} -ffat-lto-objects" export CXXFLAGS="%{optflags} -ffat-lto-objects" -export ARROW_MIMALLOC_URL=%{SOURCE3} pushd cpp %cmake \ @@ -392,6 +375,7 @@ -DARROW_BUILD_TESTS:BOOL=%{?with_tests:ON}%{!?with_tests:OFF} \ -DARROW_BUILD_UTILITIES:BOOL=ON \ -DARROW_DEPENDENCY_SOURCE=SYSTEM \ + -Dsimdjson_SOURCE=SYSTEM \ -DARROW_BUILD_BENCHMARKS:BOOL=OFF \ %ifarch aarch64 -DARROW_SIMD_LEVEL:STRING=%{?with_xsimd:NEON}%{!?with_xsimd:NONE} \ @@ -464,7 +448,7 @@ export ARROW_TEST_DATA="${PWD}/arrow-testing-%{arrow_testing_commit}/data" pushd cpp export PYTHON=%{_bindir}/python3 -%ifarch %ix86 %arm32 +%ifarch %{ix86} %{arm32} GTEST_failing="TestDecimalFromReal*" GTEST_failing="${GTEST_failing}:*TestDecryptionConfiguration.TestDecryption*" %endif @@ -496,6 +480,7 @@ %post -n libarrow_flight_sql%{sonum} -p /sbin/ldconfig %postun -n libarrow_flight_sql%{sonum} -p /sbin/ldconfig %endif + %post -n libarrow_dataset%{sonum} -p /sbin/ldconfig %postun -n libarrow_dataset%{sonum} -p /sbin/ldconfig %post -n libparquet%{sonum} -p /sbin/ldconfig ++++++ python-pyarrow.spec ++++++ --- /var/tmp/diff_new_pack.lxoq3u/_old 2026-10-20 12:43:13.836814235 +0200 +++ /var/tmp/diff_new_pack.lxoq3u/_new 2026-10-20 12:43:13.839814360 +0200 @@ -16,19 +16,19 @@ # -%{?sle15_python_module_pythons} -%bcond_with xsimd %define plainpython python # See git submodule /testing pointing to the correct revision %define arrow_testing_commit 9ff285c88565f0f6abc855918c6a342e70e4909c # See git submodule /cpp/submodules/parquet-testing pointing to the correct revision -%define parquet_testing_commit e74785d85a4ecee829e1e405444d6a1b24b8bc9c -%if %{suse_version} <= 1500 +%define parquet_testing_commit 56653c437c8092f704a092d0d1d4e600124cd49f +%if 0%{?suse_version} <= 1500 # requires __has_builtin with keywords %define gccver 13 %endif +%bcond_with xsimd +%{?sle15_python_module_pythons} Name: python-pyarrow -Version: 25.0.0 +Version: 26.0.0 Release: 0 Summary: Python library for Apache Arrow License: Apache-2.0 AND BSD-3-Clause AND BSD-2-Clause AND MIT @@ -39,9 +39,9 @@ Source2: parquet-testing-%{version}.tar.gz Source99: python-pyarrow.rpmlintrc BuildRequires: %{python_module Cython >= 3.1} -BuildRequires: %{python_module devel >= 3.10} +BuildRequires: %{python_module devel >= 3.11} BuildRequires: %{python_module libcst >= 1.8.6} -BuildRequires: %{python_module numpy-devel >= 1.25} +BuildRequires: %{python_module numpy-devel >= 2.0} BuildRequires: %{python_module pip} BuildRequires: %{python_module scikit-build-core} BuildRequires: %{python_module setuptools_scm} @@ -59,7 +59,7 @@ BuildRequires: pkgconfig(gmock) >= 1.10 BuildRequires: pkgconfig(gtest) >= 1.10 BuildRequires: pkgconfig(parquet) = %{version} -Requires: python-numpy >= 1.25 +Requires: python-numpy >= 2.0 # SECTION test requirements BuildRequires: %{python_module hypothesis} BuildRequires: %{python_module pandas} @@ -87,7 +87,7 @@ Summary: Python library for Apache Arrow - header files Requires: python-Cython Requires: python-pyarrow = %{version} -Requires: %plainpython(abi) = %python_version +Requires: %plainpython(abi) = %{python_version} Supplements: (python-devel and python-pyarrow) %description devel @@ -97,7 +97,7 @@ platlib for consuming modules using cythonization. %prep -%setup -n arrow-apache-arrow-%{version} -a1 -a2 +%setup -q -n arrow-apache-arrow-%{version} -a1 -a2 %autopatch -p1 %build ++++++ apache-arrow-25.0.0.tar.gz -> apache-arrow-26.0.0.tar.gz ++++++ /work/SRC/openSUSE:Factory/apache-arrow/apache-arrow-25.0.0.tar.gz /work/SRC/openSUSE:Factory/.apache-arrow.new.3698736/apache-arrow-26.0.0.tar.gz differ: char 12, line 1 ++++++ arrow-testing-25.0.0.tar.gz -> arrow-testing-26.0.0.tar.gz ++++++ ++++ no output (probably identical) ++++++ parquet-testing-25.0.0.tar.gz -> parquet-testing-26.0.0.tar.gz ++++++ Binary files old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/bad_data/ARROW-GH-47662.parquet and new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/bad_data/ARROW-GH-47662.parquet differ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/bad_data/README.md new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/bad_data/README.md --- old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/bad_data/README.md 2026-03-04 07:49:07.000000000 +0100 +++ new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/bad_data/README.md 2026-09-15 09:24:18.000000000 +0200 @@ -33,3 +33,33 @@ * ARROW-GH-43605.parquet: dictionary index page uses rle encoding but 0 as rle bit-width. * ARROW-GH-45185.parquet: test case of https://github.com/apache/arrow/issues/45185 where repetition levels start with a 1 instead of 0. +* ARROW-GH-47662.parquet: test case identified in https://github.com/apache/arrow/issues/47662 + where a required column contains null values (an incorrect version of data/fixed_length_byte_array.parquet). + + +## Directory `variants` + +This subdirectory contains files with malformed variant structures. + +Robust implementations of variant decoders SHOULD reject these. + +| File | Malformed Structure | +|---------------------------------------------------------------|----------------------------------------------------------------------------| +| `variant/int_overflow_in_bounds_check.parquet` | Triggers an overflow if 32 bit multiplication is used to calculate ranges. | +| `variant/out_of_range_dictionary_size.parquet` | The dictionary is declared as larger than the data | +| `variant/malformed_child_inside_well_formed_parent.parquet` | Parent is well formed; child is malformed | +| `variant/out_of_range_child_offset.parquet` | The offset of an child element is out of range | +| `variant/out_of_range_element_count.parquet` | The number of declared array elements is larger than the data | +| `variant/bad_data/variants/over_deep_nested_children.parquet` | The hierarchy is excessively deep | + +The first of these is the most critical, as this can trigger a memory allocation of many GiB, which may affect the operations of other worker threads in a shared process; an oversized dictionary may also trigger excessive memory allocation. + +The out of range child and element files contain metadata referring to content past the end of the actual data field. +On languages with strict range check, this will fail on read; extra verification simply changes when the failure is detected. +For languages where range checks are not automatically, there is a risk of variant data referencing other data on the stack/in the heap. +As this data is read only, there's no _direct_ threat to the integrity of the process, but it is still highly dangerous. + +One notable file is `bad_data/variants/over_deep_nested_children.parquet`, which verifies that nested variant children over 500 levels deep is rejected. This number is subjective; it was chosen to be consistent with the JSON parser `org.apache.parquet.variant.VariantJsonParser`. + +Currently excluded from these tests is any with an explicit limit on the size of a variant. +Apache Spark places a limit on 128 MiB on each of the metadata and value fields here. \ No newline at end of file Binary files old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/bad_data/variants/duplicate_field_offsets.parquet and new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/bad_data/variants/duplicate_field_offsets.parquet differ Binary files old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/bad_data/variants/field_id_out_of_range.parquet and new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/bad_data/variants/field_id_out_of_range.parquet differ Binary files old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/bad_data/variants/int_overflow_in_bounds_check.parquet and new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/bad_data/variants/int_overflow_in_bounds_check.parquet differ Binary files old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/bad_data/variants/malformed_child_inside_well_formed_parent.parquet and new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/bad_data/variants/malformed_child_inside_well_formed_parent.parquet differ Binary files old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/bad_data/variants/negative_dictionary_size.parquet and new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/bad_data/variants/negative_dictionary_size.parquet differ Binary files old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/bad_data/variants/out_of_range_child_offset.parquet and new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/bad_data/variants/out_of_range_child_offset.parquet differ Binary files old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/bad_data/variants/out_of_range_dictionary_size.parquet and new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/bad_data/variants/out_of_range_dictionary_size.parquet differ Binary files old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/bad_data/variants/out_of_range_element_count.parquet and new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/bad_data/variants/out_of_range_element_count.parquet differ Binary files old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/bad_data/variants/over_deep_nested_children.parquet and new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/bad_data/variants/over_deep_nested_children.parquet differ Binary files old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/bad_data/variants/oversized_primitive_size.parquet and new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/bad_data/variants/oversized_primitive_size.parquet differ Binary files old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/bad_data/variants/short_string_length_exceeds_buffer.parquet and new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/bad_data/variants/short_string_length_exceeds_buffer.parquet differ Binary files old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/bad_data/variants/truncated_primitive_size.parquet and new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/bad_data/variants/truncated_primitive_size.parquet differ Binary files old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/bad_data/variants/unknown_primitive_type.parquet and new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/bad_data/variants/unknown_primitive_type.parquet differ Binary files old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/bad_data/variants/variant_version_2_header.parquet and new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/bad_data/variants/variant_version_2_header.parquet differ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/data/README.md new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/data/README.md --- old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/data/README.md 2026-03-04 07:49:07.000000000 +0100 +++ new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/data/README.md 2026-09-15 09:24:18.000000000 +0200 @@ -48,6 +48,7 @@ | large_string_map.brotli.parquet | MAP(STRING, INT32) with a string column chunk of more than 2GB. See [note](#large-string-map) below | | float16_nonzeros_and_nans.parquet | Float16 (logical type) column with NaNs and nonzero finite min/max values | | float16_zeros_and_nans.parquet | Float16 (logical type) column with NaNs and zeros as min/max values. . See [note](#float16-files) below | +| floating_orders_nan_count.parquet | FLOAT/DOUBLE/FLOAT16 columns in IEEE754 and TypeDefined orders across five row groups (no-NaN, mixed-NaN, all-NaN, zero-min, zero-max) to validate nan_count, signed NaNs, and IEEE-754 zero ordering in statistics and column index | | concatenated_gzip_members.parquet | 513 UINT64 numbers compressed using 2 concatenated gzip members in a single data page | | byte_stream_split.zstd.parquet | Standard normals with `BYTE_STREAM_SPLIT` encoding. See [note](#byte-stream-split) below | | incorrect_map_schema.parquet | Contains a Map schema without explicitly required keys, produced by Presto. See [note](#incorrect-map-schema) | @@ -57,10 +58,16 @@ | repeated_primitive_no_list.parquet | REPEATED INT32 and BYTE_ARRAY fields without LIST annotation. See [note](#REPEATED-primitive-fields-with-no-LIST-annotation) | | map_no_value.parquet | MAP with null values, MAP with INT32 keys and no values, and LIST<INT32> column with same values as the MAP keys. See [map_no_value.md](map_no_value.md) | | page_v2_empty_compressed.parquet | An INT32 column with DataPageV2, all values are null, the zero-sized data is compressed using ZSTD. This is a valid non-zero bytes ZSTD stream that uncompresses into 0 bytes. | +| int32_with_uuid_logical_type.parquet | A single required INT32 column `int32_uuid` (10 rows, values 0..9) annotated with the UUID logical type, which is only applicable to FIXED_LEN_BYTE_ARRAY(16). This is an unrecognized/incompatible logical-physical type combination; a reader should tolerate it by ignoring the annotation (reading the column as its physical INT32 type) and ignoring its statistics, rather than failing the whole file. | | datapage_v2_empty_datapage.snappy.parquet | A compressed FLOAT column with DataPageV2, a single row, value is null, the file uses Snappy compression, but there is no data for uncompression (see [related issue](https://github.com/apache/arrow-rs/issues/7388)). The zero bytes must not be attempted to be uncompressed, as this is an invalid Snappy stream. | | unknown-logical-type.parquet | A file containing a column annotated with a LogicalType whose identifier has been set to an abitrary high value to check the behaviour of an old reader reading a file written by a new writer containing an unsupported type (see [related issue](https://github.com/apache/arrow/issues/41764)). | | int96_from_spark.parquet | Single column of (deprecated) int96 values that originated as Apache Spark microsecond-resolution timestamps. Some values are outside the range typically representable by 64-bit nanosecond-resolution timestamps. See [int96_from_spark.md](int96_from_spark.md) for details. | +| int96_timestamp_order.parquet | Single `required int96` column written with the `INT96_TIMESTAMP_ORDER` column order ([parquet-format #584](https://github.com/apache/parquet-format/pull/584)). Values are chosen so a byte-wise comparison disagrees with the chronological order, so the min/max statistics (and column index) are only correct for a reader that honors the new order. See [int96_timestamp_order.md](int96_timestamp_order.md) for details. | | binary_truncated_min_max.parquet | A file containing six columns with exact, fully-truncated and partially-truncated max and min statistics and with the expected is_{min/max}_value_exact. (see [note](Binary-truncated-min-and-max-statistics)).| +| alp_extended.zstd.parquet | FLOAT and DOUBLE columns encoded using Adaptive Lossless floating-Point (ALP). See [note](#alp-encoding) below | +| json.parquet | A single optional BYTE_ARRAY column annotated with the JSON logical type (also carries the legacy converted type JSON). Four rows: `{"a":1}`, `{"a":1,"b":null}` (null field value inside a non-null document), `[1,null,3]` (null element inside an array), and a NULL row. See [note](#json-and-bson-logical-types) below. | +| bson.parquet | A single optional BYTE_ARRAY column annotated with the BSON logical type (also carries the legacy converted type BSON). Three rows: BSON `{"a":1}`, `{"a":1,"b":null}` (null field value inside a non-null document), and a NULL row. See [note](#json-and-bson-logical-types) below. | +| flba12_timestamp.parquet | Three `FIXED_LEN_BYTE_ARRAY(12)` columns annotated with `TIMESTAMP`. See [flba12_timestamp.md](flba12_timestamp.md) for details. | TODO: Document what each file is in the table above. @@ -142,6 +149,7 @@ public static final String BOOLEAN_FIELD_NAME = "boolean_field"; public static final String INT32_FIELD_NAME = "int32_field"; public static final String INT64_FIELD_NAME = "int64_field"; +public static final String INT64_ELEMENT_COLUMN_PATH = "int64_field.list.element"; public static final String INT96_FIELD_NAME = "int96_field"; public static final String FLOAT_FIELD_NAME = "float_field"; public static final String DOUBLE_FIELD_NAME = "double_field"; @@ -152,12 +160,21 @@ "schema", new PrimitiveType(REQUIRED, BOOLEAN, BOOLEAN_FIELD_NAME), Types.required(INT32).as(LogicalTypeAnnotation.timeType(true, MILLIS)).named(INT32_FIELD_NAME), - new PrimitiveType(REPEATED, INT64, INT64_FIELD_NAME), + Types.optionalGroup().repeatedGroup().required(INT64).named("element").named("list").as(LogicalTypeAnnotation.listType()).named(INT64_FIELD_NAME), Types.required(INT96).named(INT96_FIELD_NAME), new PrimitiveType(REQUIRED, FLOAT, FLOAT_FIELD_NAME), new PrimitiveType(REQUIRED, DOUBLE, DOUBLE_FIELD_NAME), new PrimitiveType(OPTIONAL, BINARY, BINARY_FIELD_NAME), Types.required(FIXED_LEN_BYTE_ARRAY).length(FIXED_LENGTH).named(FIXED_LENGTH_BINARY_FIELD_NAME)); + +// ColumnEncryptionProperties for the list field +Map<ColumnPath, ColumnEncryptionProperties> columnPropertiesMap = new HashMap<>(); +ColumnEncryptionProperties columnPropertiesInt64List = ColumnEncryptionProperties.builder( + ColumnPath.fromDotString(INT64_ELEMENT_COLUMN_PATH)) + .withKey(COLUMN_ENCRYPTION_KEYS[6]) + .withKeyID(COLUMN_ENCRYPTION_KEY_IDS[6]) + .build(); +columnPropertiesMap.put(columnPropertiesInt64List.getPath(), columnPropertiesInt64List); ``` ## Checksum Files @@ -577,3 +594,170 @@ {"utf8_full_truncation": "Julia Roberts", "binary_full_truncation": "Julia Roberts", "utf8_partial_truncation": "Julia Roberts", "binary_partial_truncation": "Julia Roberts", "utf8_no_truncation": "Julia Roberts", "binary_no_truncation": "Julia Roberts"} {"utf8_full_truncation": "Kevin Bacon", "binary_full_truncation": "Kevin Bacon", "utf8_partial_truncation": "πKevin Bacon", "binary_partial_truncation": "ΓΏΓΏ\u0001\u0002", "utf8_no_truncation": "Ke", "binary_no_truncation": "Ke"} ``` + +## ALP encoding + +`alp_extended.zstd.parquet` contains FLOAT and DOUBLE columns encoded with +[Adaptive Lossless floating-Point (ALP)](https://github.com/apache/parquet-format/blob/master/Encodings.md#adaptive-lossless-floating-point-alp--10) +(`ALP = 10`). +It was created with the code in this [PR](https://github.com/apache/arrow/pull/49154). + +All columns contain the same 9032 values, so decoders can bit-compare the +`ALP` columns against known results stored with `PLAIN` encoding. + + +| Column | Encoding | Rationale / coverage | +|-------------------------------------|---------------------------|-----------------------------------------------------------------------------------| +| `float_plain`, `double_plain` | `PLAIN` + zstd | In-file reference: readers can bit-compare the ALP columns against these | +| `float_alp_1024`, `double_alp_1024` | `ALP`, 1024-value vectors | The default vector size of 1024 values | +| `float_alp_4096`, `double_alp_4096` | `ALP`, 4096-value vectors | Readers must honor `log_vector_size` from the page header rather than assume 1024 | +| `float_alp_32`, `double_alp_32` | `ALP`, 32-value vectors | Many vectors per page, stresses the per-vector metadata loop | + + +### Data distribution (9032 rows) + +The "base distribution" means random values in `[-10.00, 10.00]` with exactly 2 +decimal digits (e.g. `9.43`), which are losslessly encodable by ALP (no exceptions). + +The contents of the 9032 rows are as follows: + +| Rows | Contents | Rationale / coverage | +|-----------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------| +| 0β1023 | base | Happy path: full vector, small frame-of-reference bit width, no exceptions | +| 1024β2047 | base, plus: NaN at 1024, 1500 and 2047 (three distinct bit patterns, see below), +Inf at 2000, βInf at 2001, β0.0 at 2002, subnormal (`5e-324` double / `1e-45` float) at 2003 | NaN / Inf and sign/precision edge values via the exception mechanism; exceptions at exact vector boundaries; NaN payload preservation | +| 2048β3071 | base, plus: `3.141592653589793` at 2500 | Exactly one exception: the full-mantissa value cannot round-trip as a decimal | +| 3072β4095 | base, plus: `44974934523.343` at 3100 and `-1243432432.3432` at 3711 | Large-magnitude values | +| 4096β5119 | every 2nd value full-mantissa random, rest base | Many (but not all) exceptions | +| 5120β6143 | all values full-mantissa random | All exceptions | +| 6144β7167 | base but with 4 decimal digits (e.g. `3.1416`) | Different exponent/factor than the other vectors | +| 7168β8191 | constant (all `7.77`) | `bit_width = 0` vectors | +| 8192β8999 | base, with nulls at every row ordinal divisible by 100 (row 8200, 8300, ... 8900) | Partial vector + null handling | +| 9000β9031 | random integer-valued values in `[β8e18, 8e18]`, with exactly -8e18 at 9000 and 8e18 at 9001 | Max FOR length (64-bit) `bit_width`; (FLOAT columns as exceptions) | + +The three NaNs use distinct bit patterns so readers are checked for preserving +non-canonical NaN payloads (ALP stores exception values bit-exactly): + +| Row | DOUBLE bits | FLOAT bits | Description | +|------|----------------------|--------------|---------------------------------| +| 1024 | `0x7FF8000000000000` | `0x7FC00000` | Canonical quiet NaN | +| 1500 | `0x7FF800DEADBEEF00` | `0x7FC0DEAD` | Quiet NaN with payload | +| 2047 | `0xFFF8000000000001` | `0xFFC00001` | Negative quiet NaN with payload | + +The file has five row groups: + +| Row group | Rows | Contents | +|-----------|-----------|---------------------------------------------------------------------| +| 0 | 0β6143 | Base + all exception cases (6144 rows, 1546 exceptions per column) | +| 1 | 6144β7167 | 4-decimal-digit values (different exponent/factor) | +| 2 | 7168β8191 | Constant `7.77` (`bit_width = 0`) | +| 3 | 8192β8999 | Partial trailing vector with 8 nulls | +| 4 | 9000β9031 | Large-magnitude values | + +To check conformance of an `ALP` decoder, read each `ALP`-encoded column and +compare the decoded values against the values from the corresponding +`PLAIN`-encoded column. The values should match exactly (bitwise). + +## JSON and BSON logical types + +`json.parquet` and `bson.parquet` each contain a single optional `BYTE_ARRAY` +column annotated with the `JSON` and `BSON` logical types respectively (both also +carry the equivalent legacy converted type). They give reader implementations a +small fixture for the JSON/BSON annotations, which no other file in this repo +exercises. The data is intentionally small and includes a NULL value, documents +with a null field value, and (for JSON) a null array element. + +`json.parquet` (4 rows): `{"a":1}`, `{"a":1,"b":null}` (null field value in a +non-null document), `[1,null,3]` (null element in an array), and one NULL row. + +`bson.parquet` (3 rows): BSON `{"a":1}` (`0c0000001061000100000000`), +`{"a":1,"b":null}` (`0f000000106100010000000a620000`), and one NULL row. + +Both files are generated by parquet-mr 1.18.0-SNAPSHOT using the following code: + +`json.parquet`: + +```java +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.parquet.example.data.Group; +import org.apache.parquet.example.data.simple.SimpleGroupFactory; +import org.apache.parquet.hadoop.ParquetWriter; +import org.apache.parquet.hadoop.example.ExampleParquetWriter; +import org.apache.parquet.hadoop.example.GroupWriteSupport; +import org.apache.parquet.hadoop.metadata.CompressionCodecName; +import org.apache.parquet.schema.LogicalTypeAnnotation; +import org.apache.parquet.schema.MessageType; +import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; +import org.apache.parquet.schema.Types; + +public class JsonFixture { + public static void main(String[] args) throws Exception { + MessageType schema = Types.buildMessage() + .optional(PrimitiveTypeName.BINARY) + .as(LogicalTypeAnnotation.jsonType()) + .named("json_field") + .named("json_fixture"); + Configuration conf = new Configuration(); + GroupWriteSupport.setSchema(schema, conf); + SimpleGroupFactory factory = new SimpleGroupFactory(schema); + String[] docs = {"{\"a\":1}", "{\"a\":1,\"b\":null}", "[1,null,3]"}; + try (ParquetWriter<Group> writer = ExampleParquetWriter.builder(new Path("json.parquet")) + .withConf(conf) + .withCompressionCodec(CompressionCodecName.UNCOMPRESSED) + .build()) { + for (String d : docs) { + writer.write(factory.newGroup().append("json_field", d)); // String -> UTF-8 BYTE_ARRAY + } + writer.write(factory.newGroup()); // optional field omitted -> NULL row + } + } +} +``` + +`bson.parquet`: + +```java +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.parquet.example.data.Group; +import org.apache.parquet.example.data.simple.SimpleGroupFactory; +import org.apache.parquet.hadoop.ParquetWriter; +import org.apache.parquet.hadoop.example.ExampleParquetWriter; +import org.apache.parquet.hadoop.example.GroupWriteSupport; +import org.apache.parquet.hadoop.metadata.CompressionCodecName; +import org.apache.parquet.io.api.Binary; +import org.apache.parquet.schema.LogicalTypeAnnotation; +import org.apache.parquet.schema.MessageType; +import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; +import org.apache.parquet.schema.Types; + +public class BsonFixture { + public static void main(String[] args) throws Exception { + MessageType schema = Types.buildMessage() + .optional(PrimitiveTypeName.BINARY) + .as(LogicalTypeAnnotation.bsonType()) + .named("bson_field") + .named("bson_fixture"); + Configuration conf = new Configuration(); + GroupWriteSupport.setSchema(schema, conf); + SimpleGroupFactory factory = new SimpleGroupFactory(schema); + byte[] doc1 = {0x0c,0x00,0x00,0x00,0x10,0x61,0x00,0x01,0x00,0x00,0x00,0x00}; // {"a":1} + byte[] doc2 = {0x0f,0x00,0x00,0x00,0x10,0x61,0x00,0x01,0x00,0x00,0x00,0x0a,0x62,0x00,0x00}; // {"a":1,"b":null} + try (ParquetWriter<Group> writer = ExampleParquetWriter.builder(new Path("bson.parquet")) + .withConf(conf) + .withCompressionCodec(CompressionCodecName.UNCOMPRESSED) + .build()) { + writer.write(factory.newGroup().append("bson_field", Binary.fromConstantByteArray(doc1))); + writer.write(factory.newGroup().append("bson_field", Binary.fromConstantByteArray(doc2))); + writer.write(factory.newGroup()); // optional field omitted -> NULL row + } + } +} +``` + +Column annotations read back with pyarrow (both files: `null_count` = 1): + +``` +json.parquet | physical: BYTE_ARRAY | logical: JSON | converted: JSON +bson.parquet | physical: BYTE_ARRAY | logical: BSON | converted: BSON +``` Binary files old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/data/aes256/encrypt_columns_and_footer.parquet.encrypted and new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/data/aes256/encrypt_columns_and_footer.parquet.encrypted differ Binary files old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/data/aes256/encrypt_columns_and_footer_ctr.parquet.encrypted and new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/data/aes256/encrypt_columns_and_footer_ctr.parquet.encrypted differ Binary files old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/data/aes256/encrypt_columns_and_footer_disable_aad_storage.parquet.encrypted and new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/data/aes256/encrypt_columns_and_footer_disable_aad_storage.parquet.encrypted differ Binary files old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/data/aes256/encrypt_columns_plaintext_footer.parquet.encrypted and new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/data/aes256/encrypt_columns_plaintext_footer.parquet.encrypted differ Binary files old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/data/aes256/uniform_encryption.parquet.encrypted and new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/data/aes256/uniform_encryption.parquet.encrypted differ Binary files old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/data/alp_extended.zstd.parquet and new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/data/alp_extended.zstd.parquet differ Binary files old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/data/bson.parquet and new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/data/bson.parquet differ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/data/fixed_length_byte_array.md new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/data/fixed_length_byte_array.md --- old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/data/fixed_length_byte_array.md 2026-03-04 07:49:07.000000000 +0100 +++ new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/data/fixed_length_byte_array.md 2026-09-15 09:24:18.000000000 +0200 @@ -31,14 +31,14 @@ writer.model.name: example Schema: message schema { - required fixed_len_byte_array(4) flba_field; + optional fixed_len_byte_array(4) flba_field; } -Row group 0: count: 1000 3.84 B records start: 4 total(compressed): 3.749 kB total(uncompressed):3.749 kB +Row group 0: count: 1000 3,94 B records start: 4 total(compressed): 3,848 kB total(uncompressed):3,848 kB -------------------------------------------------------------------------------- type encodings count avg size nulls min / max -flba_field FIXED[4] _ _ 1000 3.84 B 105 "0x00000001" / "0x000003E8" +flba_field FIXED[4] _ _ 1000 3,94 B 105 "0x00000001" / "0x000003E8" ``` # Column Index (from parquet-cli column-index command) @@ -59,15 +59,15 @@ page-9 6 0x00000001 0x00000064 offset index for column flba_field: - offset compressed size first row index -page-0 4 390 0 -page-1 394 390 100 -page-2 784 350 200 -page-3 1134 386 300 -page-4 1520 373 400 -page-5 1893 382 500 -page-6 2275 382 600 -page-7 2657 394 700 -page-8 3051 390 800 -page-9 3441 402 900 + offset compressed size first row index unencoded bytes +page-0 4 400 0 - +page-1 404 400 100 - +page-2 804 361 200 - +page-3 1165 396 300 - +page-4 1561 384 400 - +page-5 1945 392 500 - +page-6 2337 392 600 - +page-7 2729 404 700 - +page-8 3133 400 800 - +page-9 3533 411 900 - ``` Binary files old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/data/fixed_length_byte_array.parquet and new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/data/fixed_length_byte_array.parquet differ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/data/flba12_timestamp.md new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/data/flba12_timestamp.md --- old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/data/flba12_timestamp.md 1970-01-01 01:00:00.000000000 +0100 +++ new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/data/flba12_timestamp.md 2026-09-15 09:24:18.000000000 +0200 @@ -0,0 +1,49 @@ +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, + ~ software distributed under the License is distributed on an + ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + ~ KIND, either express or implied. See the License for the + ~ specific language governing permissions and limitations + ~ under the License. + --> + +# `flba12_timestamp.parquet` + +Three columns with physical type `FIXED_LEN_BYTE_ARRAY(12)` and logical type `TIMESTAMP`. + +## Schema + +``` +message flba12_timestamp { + optional fixed_len_byte_array(12) timestamp_millis (TIMESTAMP(MILLIS,true)); + optional fixed_len_byte_array(12) timestamp_micros (TIMESTAMP(MICROS,true)); + optional fixed_len_byte_array(12) timestamp_nanos (TIMESTAMP(NANOS,true)); +} +``` + +All columns use `TYPE_DEFINED_ORDER`. For the `timestamp_nanos` column, two out of the six rows +require more than 64 bits to represent. + +## Values + +The same six timestamp values appear in every column, only the unit scale differs. The values +below are interpreted as ISO-8601 timestamps in the Proleptic Gregorian calendar. + +| Row | Timestamp (UTC) | Epoch Seconds | Note | +|-----|-----------------------|------------------|---------------------------------------------| +| 0 | 1970-01-01T00:00:00Z | 0 | Epoch (all-zero bytes) | +| 1 | 1970-01-01T00:00:01Z | 1 | Small positive | +| 2 | 1969-12-31T23:59:59Z | -1 | Small negative (exercises two's-complement) | +| 3 | 2262-04-11T23:47:16Z | 9,223,372,036 | Near INT64-nanos max | +| 4 | 9999-12-31T23:59:59Z | 253,402,300,799 | Far future β NANOS exceeds INT64 | +| 5 | 0001-01-01T00:00:00Z | -62,135,596,800 | Far past β NANOS is below INT64 minimum | Binary files old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/data/flba12_timestamp.parquet and new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/data/flba12_timestamp.parquet differ Binary files old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/data/floating_orders_nan_count.parquet and new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/data/floating_orders_nan_count.parquet differ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/data/geospatial/README.md new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/data/geospatial/README.md --- old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/data/geospatial/README.md 2026-03-04 07:49:07.000000000 +0100 +++ new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/data/geospatial/README.md 2026-09-15 09:24:18.000000000 +0200 @@ -65,3 +65,27 @@ parameter set to an arbitrary string value. The Parquet format does not restrict the value of the crs parameter and implementations may choose to attempt interpreting the value or error. + +- `geography-points.parquet`: Contains a GEOGRAPHY column with 500 points + roughly equally spaced on a sphere using the golden ratio method. Points + are sorted by Hilbert curve and split into row groups of 10 rows each + to test spatial predicate pushdown. The points in the file contain + both the North and South Poles (`POINT (0 90)` and `POINT (0 -90)`). + At least one row group contains wraparound statistics (i.e., `xmin > xmax`). + +- `geography-lines.parquet`: Contains a GEOGRAPHY column with lines connecting + consecutive points from a spatially sorted point set. Lines are sorted by + Hilbert curve and split into row groups of 10 rows each to test spatial + predicate pushdown. There are two lines that intersect the North Pole, + two lines that intersect the South Pole, and several lines that contain + vertices on either side of the antimeridian. At least one row group + contains wraparound statistics (i.e., `xmin > xmax`). + +- `geography-polygons.parquet`: Contains a GEOGRAPHY column with polygons + created by buffering each of the 500 roughly equally spaced sphere points + with a 500km radius. Polygons are sorted by Hilbert curve and split into + row groups of 10 rows each to test spatial predicate pushdown. There is + a polygon that contains the North Pole and a polygon that contains the + South Pole and several polygons that contain vertices on either side of + the antimeridian. At least one row group contains wraparound statistics + (i.e., `xmin > xmax`). Binary files old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/data/geospatial/geography-lines.parquet and new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/data/geospatial/geography-lines.parquet differ Binary files old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/data/geospatial/geography-points.parquet and new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/data/geospatial/geography-points.parquet differ Binary files old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/data/geospatial/geography-polygons.parquet and new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/data/geospatial/geography-polygons.parquet differ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/data/geospatial/geospatial-gen-geography.py new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/data/geospatial/geospatial-gen-geography.py --- old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/data/geospatial/geospatial-gen-geography.py 1970-01-01 01:00:00.000000000 +0100 +++ new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/data/geospatial/geospatial-gen-geography.py 2026-09-15 09:24:18.000000000 +0200 @@ -0,0 +1,185 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import numpy as np +import pyarrow as pa +import pyarrow.parquet as parquet +import geoarrow.pyarrow as ga +import sedona.db + +from pathlib import Path + +HERE = Path(__file__).parent + + +def generate_points(n): + # Generate roughly equally spaced points on the sphere + golden_ratio = (1 + np.sqrt(5)) / 2 + indices = np.arange(n) + + phi = np.arcsin(2 * indices / (n - 1) - 1) if n > 1 else np.array([0.0]) + theta = 2 * np.pi * indices / golden_ratio + + lat = np.degrees(phi) + lon = np.degrees(theta) % 360 - 180 + + # Explicitly set poles at longitude 0 for numerical consistency + lon[0] = 0.0 # South pole (lat=-90) + lon[-1] = 0.0 # North pole (lat=90) + + array = ga.with_edge_type( + ga.with_crs(ga.make_point(lon, lat), ga.OGC_CRS84), ga.EdgeType.SPHERICAL + ) + return pa.table({"id": indices, "geometry": ga.as_wkb(array)}) + + +def write_points(): + sd = sedona.db.connect() + sd.create_data_frame(generate_points(500)).to_parquet( + HERE / "geography-points.parquet", + sort_by="geometry", + max_row_group_size=10, + geoparquet_version="none", + ) + + +def write_lines(): + sd = sedona.db.connect() + sd.create_data_frame(generate_points(500)).to_view("pts") + + # Sort so that points make sense sequentially + points_tab = sd.sql( + "SELECT * FROM pts ORDER BY sd_order(geometry)" + ).to_arrow_table() + + # Shift geometry column using PyArrow slicing + geom = points_tab.column("geometry") + ids = points_tab.column("id").slice(0, len(geom) - 1) + next_geom = geom.slice(1) # geometry[1:] + geom = geom.slice(0, len(geom) - 1) # geometry[:-1] + + paired = pa.table({"id": ids, "geometry": geom, "next_geom": next_geom}) + + sd.create_data_frame(paired).to_view("geoms") + sd.sql(""" + SELECT id, ST_MakeLine(geometry, next_geom) AS geometry + FROM geoms + """).to_parquet( + HERE / "geography-lines.parquet", + sort_by="geometry", + max_row_group_size=10, + geoparquet_version="none", + ) + + +def write_polygons(): + sd = sedona.db.connect() + sd.create_data_frame(generate_points(500)).to_view("pts") + sd.sql( + "SELECT id, ST_Buffer(geometry, 500000, 'quad_segs=1') as geometry from pts" + ).to_parquet( + HERE / "geography-polygons.parquet", + sort_by="geometry", + max_row_group_size=10, + geoparquet_version="none", + ) + + +def test_geography_parquet_files(): + """Test that geography parquet files have correct logical type and wraparound statistics.""" + files = [ + HERE / "geography-points.parquet", + HERE / "geography-lines.parquet", + HERE / "geography-polygons.parquet", + ] + + for path in files: + f = parquet.ParquetFile(path) + + # Check for Geography logical type + logical_type = f.metadata.schema.column(1).logical_type.to_json() + assert logical_type == '{"Type": "Geography"}', ( + f"{path.name}: Expected Geography logical type, got {logical_type}" + ) + print(f"{path.name}: Has Geography logical type") + + # Check that at least one row group has wraparound statistics (xmin > xmax) + has_wraparound = 0 + for i in range(f.metadata.num_row_groups): + stats = f.metadata.row_group(i).column(1).geo_statistics + if stats is not None and stats.xmin > stats.xmax: + has_wraparound += 1 + + assert has_wraparound > 0, ( + f"{path.name}: Expected at least one row group with antimeridian wraparound" + ) + print(f"{path.name}: Has {has_wraparound} row groups with wraparound stats") + + +def test_poles_intersection(): + """Test that each geography parquet file has at least one geometry intersecting each pole.""" + sd = sedona.db.connect() + files = [ + ("geography-points.parquet", HERE / "geography-points.parquet"), + ("geography-lines.parquet", HERE / "geography-lines.parquet"), + ("geography-polygons.parquet", HERE / "geography-polygons.parquet"), + ] + + for name, path in files: + sd.read_parquet(str(path)).to_view("geom_table", overwrite=True) + + # Test North Pole intersection + north_pole_count = ( + sd.sql(""" + SELECT COUNT(*) as cnt FROM geom_table + WHERE ST_Intersects(geometry, ST_GeogPoint(0, 90)) + """) + .to_arrow_table() + .column("cnt")[0] + .as_py() + ) + + assert north_pole_count >= 1, ( + f"{name}: Expected at least 1 geometry intersecting North Pole (0, 90), got {north_pole_count}" + ) + print(f"{name}: {north_pole_count} geometries intersect North Pole") + + # Test South Pole intersection + south_pole_count = ( + sd.sql(""" + SELECT COUNT(*) as cnt FROM geom_table + WHERE ST_Intersects(geometry, ST_GeogPoint(0, -90)) + """) + .to_arrow_table() + .column("cnt")[0] + .as_py() + ) + + assert south_pole_count >= 1, ( + f"{name}: Expected at least 1 geometry intersecting South Pole (0, -90), got {south_pole_count}" + ) + print(f"{name}: {south_pole_count} geometries intersect South Pole") + + print("\nPole intersection tests passed!") + + +if __name__ == "__main__": + write_points() + write_lines() + write_polygons() + test_geography_parquet_files() + test_poles_intersection() Binary files old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/data/int32_with_uuid_logical_type.parquet and new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/data/int32_with_uuid_logical_type.parquet differ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/data/int96_timestamp_order.md new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/data/int96_timestamp_order.md --- old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/data/int96_timestamp_order.md 1970-01-01 01:00:00.000000000 +0100 +++ new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/data/int96_timestamp_order.md 2026-09-15 09:24:18.000000000 +0200 @@ -0,0 +1,77 @@ +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, + ~ software distributed under the License is distributed on an + ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + ~ KIND, either express or implied. See the License for the + ~ specific language governing permissions and limitations + ~ under the License. + --> + +# `int96_timestamp_order.parquet` + +A single `required int96` column written with the `INT96_TIMESTAMP_ORDER` column order added in +[parquet-format #584](https://github.com/apache/parquet-format/pull/584). It exercises a reader's +ability to honor the new order: the column carries min/max statistics and a column index, and the +footer's `column_orders[0]` is set to `INT96_TIMESTAMP_ORDER` (union field 3) rather than +`TYPE_ORDER`. + +INT96 timestamps are 12 little-endian bytes: an 8-byte nanoseconds-within-the-day followed by a +4-byte Julian day. The defined order compares the Julian day (as a signed int32) first, then the +nanoseconds (as a signed int64) β i.e. chronological order. + +## Why this file is non-trivial + +The values are deliberately chosen so that a **byte-wise (lexicographic) comparison disagrees with +the chronological order**. Because the low-order nanosecond bytes come first in the little-endian +layout, a reader that compares the raw 12 bytes (or that ignores the new order) computes the wrong +min/max. A reader must implement the chronological comparison to pass. + +| Value | Julian day | nanos-of-day | Timestamp | first byte | +|----------------|------------|-------------------|------------------------------------|------------| +| EARLY | 2440000 | 123 | 1968-05-23 00:00:00.000000123 | `0x7B` | +| SAME_DAY_EARLY | 2440588 | 1000 | 1970-01-01 00:00:00.000001000 | `0xE8` | +| LATE_IN_DAY | 2440588 | 86399999999999 | 1970-01-01 23:59:59.999999999 | `0xFF` | +| NEXT_DAY | 2440589 | 0 | 1970-01-02 00:00:00.000000000 | `0x00` | + +Values are written to the file out of order: `LATE_IN_DAY, NEXT_DAY, EARLY, SAME_DAY_EARLY` (so that +the correct min/max are also neither the first nor the last value). + +- Correct (`INT96_TIMESTAMP_ORDER`) min/max: **EARLY / NEXT_DAY** +- Byte-wise (incorrect) min/max would be: **NEXT_DAY / LATE_IN_DAY** (ordered by the leading + nanosecond byte `0x00 < 0x7B < 0xE8 < 0xFF`) + +The min/max written to the statistics (and the column index) are therefore: + +``` +min = 0x 7B 00 00 00 00 00 00 00 40 3B 25 00 (EARLY: nanos 123, Julian day 2440000) +max = 0x 00 00 00 00 00 00 00 00 8D 3D 25 00 (NEXT_DAY: nanos 0, Julian day 2440589) +``` + +## How it was generated + +Written by parquet-java (parquet-mr 1.18.0-SNAPSHOT) via the +`TestInt96TimestampStatistics#writeInt96TimestampOrderInteropFile` test: + +``` +mvn -pl parquet-hadoop test \ + -Dtest='TestInt96TimestampStatistics#writeInt96TimestampOrderInteropFile' \ + -Dparquet.testing.data.dir=<parquet-testing>/data +``` + +## Schema + +``` +message int96_timestamp_order { + required int96 ts; +} +``` Binary files old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/data/int96_timestamp_order.parquet and new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/data/int96_timestamp_order.parquet differ Binary files old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/data/json.parquet and new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/data/json.parquet differ Binary files old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/shredded_variant/case-041-INVALID.parquet and new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/shredded_variant/case-041-INVALID.parquet differ Binary files old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/shredded_variant/case-041-INVALID_row-0.variant.bin and new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/shredded_variant/case-041-INVALID_row-0.variant.bin differ Binary files old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/shredded_variant/case-041.parquet and new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/shredded_variant/case-041.parquet differ Binary files old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/shredded_variant/case-041_row-0.variant.bin and new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/shredded_variant/case-041_row-0.variant.bin differ Binary files old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/shredded_variant/case-131-INVALID.parquet and new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/shredded_variant/case-131-INVALID.parquet differ Binary files old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/shredded_variant/case-131-INVALID_row-0.variant.bin and new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/shredded_variant/case-131-INVALID_row-0.variant.bin differ Binary files old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/shredded_variant/case-131.parquet and new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/shredded_variant/case-131.parquet differ Binary files old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/shredded_variant/case-131_row-0.variant.bin and new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/shredded_variant/case-131_row-0.variant.bin differ Binary files old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/shredded_variant/case-132-INVALID.parquet and new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/shredded_variant/case-132-INVALID.parquet differ Binary files old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/shredded_variant/case-132-INVALID_row-0.variant.bin and new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/shredded_variant/case-132-INVALID_row-0.variant.bin differ Binary files old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/shredded_variant/case-132.parquet and new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/shredded_variant/case-132.parquet differ Binary files old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/shredded_variant/case-132_row-0.variant.bin and new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/shredded_variant/case-132_row-0.variant.bin differ Binary files old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/shredded_variant/case-138-INVALID.parquet and new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/shredded_variant/case-138-INVALID.parquet differ Binary files old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/shredded_variant/case-138-INVALID_row-0.variant.bin and new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/shredded_variant/case-138-INVALID_row-0.variant.bin differ Binary files old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/shredded_variant/case-138.parquet and new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/shredded_variant/case-138.parquet differ Binary files old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/shredded_variant/case-138_row-0.variant.bin and new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/shredded_variant/case-138_row-0.variant.bin differ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/shredded_variant/cases.json new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/shredded_variant/cases.json --- old/parquet-testing-e74785d85a4ecee829e1e405444d6a1b24b8bc9c/shredded_variant/cases.json 2026-03-04 07:49:07.000000000 +0100 +++ new/parquet-testing-56653c437c8092f704a092d0d1d4e600124cd49f/shredded_variant/cases.json 2026-09-15 09:24:18.000000000 +0200 @@ -236,8 +236,9 @@ }, { "case_number" : 41, "test" : "testArrayMissingValueColumn", - "parquet_file" : "case-041.parquet", - "variant_file" : "case-041_row-0.variant.bin", + "notes" : "This parquet file is not valid according to the spec (the variant group omits the required 'value' column) and implementations can choose to error, or read the shredded value", + "parquet_file" : "case-041-INVALID.parquet", + "variant_file" : "case-041-INVALID_row-0.variant.bin", "variant" : "Variant(metadata=VariantMetadata(dict={}), value=VariantArray([Variant(type=STRING, value=comedy), Variant(type=STRING, value=drama)]))" }, { "case_number" : 42, @@ -775,14 +776,16 @@ }, { "case_number" : 131, "test" : "testMissingValueColumn", - "parquet_file" : "case-131.parquet", - "variant_file" : "case-131_row-0.variant.bin", + "notes" : "This parquet file is not valid according to the spec (the variant group omits the required 'value' column) and implementations can choose to error, or read the shredded value", + "parquet_file" : "case-131-INVALID.parquet", + "variant_file" : "case-131-INVALID_row-0.variant.bin", "variant" : "Variant(metadata=VariantMetadata(dict={}), value=Variant(type=INT32, value=34))" }, { "case_number" : 132, "test" : "testShreddedObjectMissingFieldValueColumn", - "parquet_file" : "case-132.parquet", - "variant_file" : "case-132_row-0.variant.bin", + "notes" : "This parquet file is not valid according to the spec (the shredded object field groups omit the required 'value' column) and implementations can choose to error, or read the shredded value", + "parquet_file" : "case-132-INVALID.parquet", + "variant_file" : "case-132-INVALID_row-0.variant.bin", "variant" : "Variant(metadata=VariantMetadata(dict={0 => a, 1 => b, 2 => c, 3 => d, 4 => e}), value=VariantObject(fields={b: Variant(type=STRING, value=iceberg)}))" }, { "case_number" : 133, @@ -816,7 +819,8 @@ }, { "case_number" : 138, "test" : "testShreddedObjectMissingValueColumn", - "parquet_file" : "case-138.parquet", - "variant_file" : "case-138_row-0.variant.bin", + "notes" : "This parquet file is not valid according to the spec (the variant group omits the required 'value' column) and implementations can choose to error, or read the shredded value", + "parquet_file" : "case-138-INVALID.parquet", + "variant_file" : "case-138-INVALID_row-0.variant.bin", "variant" : "Variant(metadata=VariantMetadata(dict={0 => a, 1 => b, 2 => c, 3 => d, 4 => e}), value=VariantObject(fields={a: Variant(type=INT16, value=1234), b: Variant(type=STRING, value=iceberg)}))" } ] \ No newline at end of file ++++++ use-system-mimalloc.patch ++++++ From: Dominique Belhachemi <[email protected]> Subject: Use system mimalloc instead of a vendored copy Bug: https://github.com/apache/arrow/issues/42211 Adapted from Debian's debian/patches/use-debian-mimalloc.patch (apache-arrow 25.0.0) to arrow 26.0.0: the vendored block gained an APPLE MI_TLS_MODEL_LOCAL special-case and a "Don't override default malloc" comment, both dropped along with the ExternalProject. The global ARROW_SHARED_LINK_LIBS list the Debian patch appends to no longer exists in arrow 26 (per-target SHARED_LINK_LIBS arg in BuildUtils.cmake instead); it is not needed here anyway since cpp/src/arrow/CMakeLists.txt already links mimalloc::mimalloc PRIVATE into the memory-pool objects that become part of libarrow. Linking the shared system mimalloc means libarrow carries a DT_NEEDED on libmimalloc built with MI_OVERRIDE=ON (unlike the vendored static build with MI_OVERRIDE=OFF/MI_LOCAL_DYNAMIC_TLS=ON), same trade-off as Debian. %check (memory_pool / IPC tests) exercises the pool. --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2603,75 +2603,11 @@ message(FATAL_ERROR "Can't use mimalloc with ARROW_ENABLE_THREADING=OFF") endif() - message(STATUS "Building (vendored) mimalloc from source") - # We only use a vendored mimalloc as we want to control its build options. - - set(MIMALLOC_LIB_BASE_NAME "mimalloc") - if(${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG") - set(MIMALLOC_LIB_BASE_NAME "${MIMALLOC_LIB_BASE_NAME}-${LOWERCASE_BUILD_TYPE}") - endif() - - set(MIMALLOC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/mimalloc_ep/src/mimalloc_ep") - set(MIMALLOC_INCLUDE_DIR "${MIMALLOC_PREFIX}/include") - set(MIMALLOC_STATIC_LIB - "${MIMALLOC_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${MIMALLOC_LIB_BASE_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) - - set(MIMALLOC_C_FLAGS ${EP_C_FLAGS}) - if(MINGW) - # Workaround https://github.com/microsoft/mimalloc/issues/910 on RTools40 - # This is still required as of mimalloc 3.3.1, tested as part of GH-49772 - set(MIMALLOC_C_FLAGS "${MIMALLOC_C_FLAGS} -DERROR_COMMITMENT_MINIMUM=635") - endif() - - set(MIMALLOC_CMAKE_ARGS - ${EP_COMMON_CMAKE_ARGS} - "-DCMAKE_C_FLAGS=${MIMALLOC_C_FLAGS}" - "-DCMAKE_INSTALL_PREFIX=${MIMALLOC_PREFIX}" - -DMI_INSTALL_TOPLEVEL=ON - # Don't override default malloc - -DMI_OVERRIDE=OFF - -DMI_OSX_INTERPOSE=OFF - -DMI_OSX_ZONE=OFF - # Allow usage through dlopen (i.e. when libarrow.so itself is dlopen'ed) - -DMI_LOCAL_DYNAMIC_TLS=ON - -DMI_BUILD_OBJECT=OFF - -DMI_BUILD_SHARED=OFF - -DMI_BUILD_TESTS=OFF - # GH-47229: Force mimalloc to generate armv8.0 binary - -DMI_NO_OPT_ARCH=ON) - - if(APPLE) - list(APPEND - MIMALLOC_CMAKE_ARGS - # GH-50428: Make sure several mimalloc instances can cohabit in the same process - # (also https://github.com/microsoft/mimalloc/issues/1327#issuecomment-4964140817) - -DMI_TLS_MODEL_LOCAL=ON) - endif() - - externalproject_add(mimalloc_ep - ${EP_COMMON_OPTIONS} - URL ${MIMALLOC_SOURCE_URL} - URL_HASH "SHA256=${ARROW_MIMALLOC_BUILD_SHA256_CHECKSUM}" - CMAKE_ARGS ${MIMALLOC_CMAKE_ARGS} - BUILD_BYPRODUCTS "${MIMALLOC_STATIC_LIB}") - - file(MAKE_DIRECTORY ${MIMALLOC_INCLUDE_DIR}) - - add_library(mimalloc::mimalloc STATIC IMPORTED) - set_target_properties(mimalloc::mimalloc PROPERTIES IMPORTED_LOCATION - "${MIMALLOC_STATIC_LIB}") - target_include_directories(mimalloc::mimalloc BEFORE - INTERFACE "${MIMALLOC_INCLUDE_DIR}") - target_link_libraries(mimalloc::mimalloc INTERFACE Threads::Threads) - if(WIN32) - target_link_libraries(mimalloc::mimalloc INTERFACE "bcrypt.lib" "psapi.lib") - endif() - add_dependencies(mimalloc::mimalloc mimalloc_ep) - - list(PREPEND ARROW_BUNDLED_STATIC_LIBS mimalloc::mimalloc) - - set(mimalloc_VENDORED TRUE) + # Use the system mimalloc (Factory ships 3.3.2 with a CMake config). + # Upstream mimalloc installs its export without a namespace, so keep + # the mimalloc::mimalloc name the rest of the build expects via an alias. + find_package(mimalloc REQUIRED) + add_library(mimalloc::mimalloc ALIAS mimalloc) endif() # ----------------------------------------------------------------------
