This is an automated email from the ASF dual-hosted git repository.
alamb pushed a change to branch
dependabot/npm_and_yarn/datafusion/wasmtest/datafusion-wasm-app/multi-96c788614a
in repository https://gitbox.apache.org/repos/asf/datafusion.git
from 6d88c63c53 chore(deps): bump on-headers and compression
add 3c95281974 feat: improve LiteralGuarantee for the case like `(a=1 AND
b=1) OR (a=2 AND b=3)` (#16762)
add 5dd706d631 [main] Update version to 49.0.0, add 49.0.0 changelog
(#16855)
add 518fb4b536 fix(build-wasm): put `arrow-ipc/zstd` dep under
`compression` feature flag (#16844)
add 3abd63870f chore(deps): bump serde_json from 1.0.140 to 1.0.141
(#16863)
add 14ac31d0e0 chore(deps): bump aws-config from 1.8.1 to 1.8.2 (#16864)
add 3a40625e2e test: Fix flaky join tests (#16860)
add 3d4fdf24ec chore(deps): bump rand from 0.9.1 to 0.9.2 (#16882)
add 51f04c872a Report error when `SessionState::sql_to_expr_with_alias`
does not consume all input (#16811)
add dbc03fa4f6 fix another flaky join test (#16880)
add d553ffdff8 Improve async_udf example and docs (#16846)
add 07516aa4a3 Chore: add unit tests for chr function (#16856)
add d9e963eab2 remove deprecated methods from FileScanConfig /
DataSourceExec (#16901)
add 3b4eda5c61 Support utf8view for spark hex (#16885)
add a6d4798c45 Fixes 3 bugs during serialization and deserialization of
physical plans (#16858)
add b7da86e9ad feat(spark): Implement Spark `string` function `luhn_check`
(#16848)
add bb1b55cfba chore(deps): bump aws-config from 1.8.2 to 1.8.3 (#16912)
add 070517a87f Derive UDF equality from PartialEq, Hash (#16842)
add 675b96c2b4 Ensure Substrait consumer can handle expressions in
VirtualTable (#16857)
add 8b9204cfeb Mutable Join Unwind (#16883)
add 2ff02b25ec feat(spark): implement Spark datetime function last_day
(#16828)
add 5e0b2d0554 fix(datafusion-proto): support serializing/deserilizing
ArrowFormat tables (#16875)
add 871d4b55dc ScalarValue Default + Min + Max (#16891)
add 4f533582d6 fix: `PlaceholderRowExec::partition_statistics` (#16851)
add 84c8881b64 minor: add is_superset() method for Interval's (#16895)
add 9deec2ad72 minor: implement with_new_expressions for
AggregateFunctionExpr (#16897)
add cbda394c0d Update enforce_distribution.rs (#16913)
add 4b9a468cc1 feat: Add
`ScalarValue::{new_one,new_zero,new_ten,distance}` support for `Decimal128` and
`Decimal256` (#16831)
add fd08e726dc Implement Helpers for ScopedTimerGuard and Time Structs
(#16911)
add 8bf7123687 Fix Partial Sort Get Slice Point Between Batches (#16881)
add 71dbdf6f19 fix: skip predicates on struct unnest in PushDownFilter
(#16790)
add e033d42251 optimize initcap function by avoiding memory allocation
(#16878)
add ff777eae44 Fix `schema_adapter` integration tests not running (#16835)
add 8d74be8ce4 Docs: Update Upgrading.md to reflect 49.0.0 is released
(#16853)
add e412859a04 Update release process (#16929)
add 939d3c544e Fix `next_up` and `next_down` behavior for zero float
values (#16745)
add 9f3e7dce69 Add Fetch Property to OutputRequirementExec (#16892)
add c6d5520716 feat: support distinct for window (#16925)
add e1a5cdf6db Support multiple ordered `array_agg` aggregations (#16625)
add ec8c34b711 chore(deps): bump tokio from 1.46.1 to 1.47.0 (#16952)
add d3def0ea8b chore(deps): bump serde_json from 1.0.140 to 1.0.141
(#16951)
add c3ec964445 chore: Remove attributes to allow dead_code that aren't
relevant anymore (#16953)
add 764d547edf speedup `date_trunc` (~7x faster) in some cases (#16859)
add b7c014a2c8 fix: regex bench (#16890)
add 5421825b62 feat: add multi level merge sort that will always fit in
memory (#15700)
add aab44fdaae feat: [datafusion-spark] Implement `next_day` function
(#16780)
add 56394e0a50 feat: Support distinct window for sum (#16943)
add 6f74fb047d chore(deps): bump rand from 0.9.1 to 0.9.2 (#16960)
add cca9d4c537 chore(deps): bump ctor from 0.4.2 to 0.4.3 (#16961)
add 949917a6e0 Remove references to DataFusion for Ray sub project now
that it is no longer maintained (#16966)
add f2086f3fa7 Add `temp_directory` and `max_temp_directory_size` runtime
config variables (#16934)
add 94e85488df disallow pushdown of volatile functions (#16861)
add 0183244a42 remove warning from every file open (#16968)
add 9b27952303 Pin github actions to commit sha (#16964)
add b10f453e7b Enable physical filter pushdown for hash joins (#16954)
add 677f7231be feat(spark): implement Spark math function rint (#16924)
add 5cdb7a2404 Include full DataType in TopKAggregateStream results
add 3dc3aaa86e Use tokio::task::coop::poll_proceed by default in
CooperativeStream (#16748)
add 853eee09b3 Add benchmark utility to profile peak memory usage (#16814)
add e6f4c7f8a5 chore(deps): bump indicatif from 0.17.11 to 0.18.0 (#16992)
add d376a32f1f Add `sql_parser.default_null_ordering` config option to
customize the default null ordering (#16963)
add 2a90ff606d Added Example for `Statistical Functions` in Docs (#16927)
add 31c3e58797 test(datafusion-cli): migrate tests to `insta` in
`print_format.rs` (#16993)
add 21362aa53d remove 'spill_record_batch_by_size' api (#16958)
add 6ea01d1336 chore(deps): bump serde_json from 1.0.141 to 1.0.142
(#17006)
add 494851a2bd Feature: Improve hash Expr performance (#16977)
add 9d6f923f85 Add tests for yielding in
`SpillManager::read_spill_as_stream` (#16616)
add 1a575a0835 feat(spark): implement Spark string function like/ilike
(#16962)
add a078cd257d Fix window_functions docs formatting (#17005)
add c0bdff912a Fix 'Analaysis' typo (#17015)
add f4c5a63c6b Fix random extra bullet for 'Analytical Functions' (#17014)
add c37dd5ea45 feat: Cache Parquet metadata in built in parquet reader
(#16971)
add b4e7147c85 fix: `ComposedPhysicalExtensionCodec` does not use the same
codec as encoding when decoding (#16986)
add 6d9b76e4a3 Perf: Port arrow-rs optimization for get_buffer_memory_size
and add fast path for no buffer for gc string view (#17008)
add 66d6995b8f Fix failing documentation check (#17026)
add 2968331e4c #16994 Ensure CooperativeExec#maintains_input_order returns
a Vec of the correct size (#16995)
add fa1f8c192d Upgrade arrow/parquet to 56.0.0 (#16690)
add 36f5f1456d test: Add logic tests for string_agg with order (#17033)
add eb2b8c07b2 fix error result in execute&pre_selection (#16930)
add cc78edc51e docs: Fix failing CI (#17041)
add 4c36226bc4 Docs: Add Examples to Config Options page (#17039)
add f10deb67a3 Add Tuning Guide for small data / short queries (#17040)
add 40015a8345 Docs: Update the crate configuration / build settings page
(#17038)
add 6d0073465a Implement `From<Option<String>>' for `ScalarValue` (#17043)
add e4f16dd60f feat: Add `Arc<ConfigOptions>` to `ScalarFunctionArgs`,
don't copy `ConfigOptions` on each query (#16970)
add 5f26e70f12 chore(deps): bump tokio-util from 0.7.15 to 0.7.16 (#17030)
add bf6f6316ae Add missing Substrait to DataFusion function name mappings
(#16950)
add 71b92bc9e7 refactor: use upstream inline_key_fast (#17044)
add b0c8dd61ac Make `AsyncScalarUDFImpl::invoke_async_with_args`
consistent with `ScalarUDFImpl::invoke_with_args` (#16902)
add 3472aa1349 Implement spark `array` function `array` (#16936)
add 6043be448f feat: spark crc32/sha1 (#17032)
add 79c4c057e6 Address memory over-accounting in array_agg (#16816)
add 183ff6643a Support `centroids` config for
`approx_percentile_cont_with_weight` (#17003)
add 2bbd6a14e0 chore(deps): bump aws-credential-types from 1.2.4 to 1.2.5
(#17053)
add 8147565bf1 Support Substrait functions and_not, xor, and between in
consumer built-in expression builder (#16984)
add 84bbce63af Derive UDWF equality from PartialEq, Hash (#17057)
add a9e6d4be4a feat: Limit the memory used in the file metadata cache
(#17031)
add d7a886cef0 fix return field for `is_null` and `is_not_null` expression
(#17056)
add 20bb7e6901 chore(deps): bump tokio from 1.47.0 to 1.47.1 (#17063)
add df153c2f81 Optimize char expression (#16076)
add 60ac1cc5f9 fix: Remove `datafusion.execution.parquet.cache_metadata`
config (#17062)
add a83237f3a9 Fix equality of parametrizable ArrayAgg function (#17065)
add 7bc9906513 Implement Spark `url` function `parse_url` (#16937)
add f9efba0e2c Add ExecutionPlan::reset_state (#17028)
add ac3a5735ff Derive UDAF equality from Eq, Hash (#17067)
add 541a67d12c Remove elements deprecated since v 45 (#17075)
add 060938bf68 Deprecate ScalarUDF::is_nullable (#17074)
add f0630fb4f7 Re-export `object_store` crate via DataFusion Core and
Common (#17070)
add 173989cc2f Docs: Add Tuning Guide for larger-than-memory queries
(#17069)
add 407a965d37 Link UdfEq and PtrEq to help understand relationship.
(#17082)
add cbc0614209 feat: Dynamic Parquet encryption and decryption properties
(#16779)
add 5f95734e1e Fix hash/equality issues for ScalarFunctionExpr (#17078)
add 9264bb88ab Fill missing methods in aliased UDF impls (#17080)
add d69359457e Improve Hash speed for ScalarFunctionExpr (#17099)
add ab794d29b3 chore(deps): bump clap from 4.5.42 to 4.5.43 (#17079)
add 6793af5829 minor: remove unused import in docstring of
datafusion_common::record_batch (#17106)
add 999b44ba6f Make macros in common::test_util hygenic (#17102)
add d987d2dc80 minor: remove unnecessary clippy:large_enum_variant allows
(#17108)
add 3d6541d119 minor: Improve equivalence handling of joins (#16893)
add 9463cf6792 Fix incorrect `NULL IN ()` optimization (#17092)
add 876b504cb0 Simplify comparisons and binary operations involving NULL
(#17088)
add c7d5904f01 Add `prettier` to the devcontainer (GitHub codespaces)
(#17019)
add c4f0375686 Set a lower threshold for clippy to flag large error
variants (#17109)
add ff9db020d7 chore(deps): bump rustyline from 16.0.0 to 17.0.0 (#17116)
add 14fb4a3c7d feat: Use Cached Metadata for ListingTable Statistics
(#17022)
add ff77b70200 Add dynamic filter (bounds) pushdown to HashJoinExec
(#16445)
add d375bfed45 Remove the "extended workflow" github action (#17119)
add f0e9334848 feat(spark): implement Spark math function mod/pmod (#16829)
add c5cb6f4ad7 feat(spark): implement Spark math function
bit_get/bit_count (#16942)
add ceab1e1653 feat: add `isodow` (ISO day-of-week) support to date_part
(Monday = 0) (#17112)
add 28e042dd47 feat(spark): implement spark datetime function
date_add/date_sub (#17024)
add 8494a3967a Derive `WindowUDFImpl` equality, hash from `Eq`, `Hash`
traits (#17081)
add 80d105840d chore(deps): bump sysinfo from 0.36.1 to 0.37.0 (#17124)
add a1431e2371 chore(deps): bump libc from 0.2.174 to 0.2.175 (#17121)
add 7d52145127 Preserve equivalence properties during projection pushdown
(#17129)
add 9d9cde499b chore: Enforce checks for RC branches (#17132)
add 0db14c8dd3 chore(deps): bump actions/checkout from 4.2.2 to 5.0.0
(#17149)
add 00c2191bc2 fix: Add missing member to visitor for
ConfigFileEncryptionProperties (#17103)
add 44daa9a7b8 minor: enhance comment in SortPreservingMergeStream.abort
(#17115)
add 31016f2233 Merge branch 'main' into
dependabot/npm_and_yarn/datafusion/wasmtest/datafusion-wasm-app/multi-96c788614a
No new revisions were added by this update.
Summary of changes:
.asf.yaml | 69 ++
.devcontainer/Dockerfile | 8 +-
.github/workflows/audit.yml | 2 +-
.github/workflows/dependencies.yml | 2 +-
.github/workflows/dev.yml | 8 +-
.github/workflows/docs.yaml | 6 +-
.github/workflows/docs_pr.yaml | 4 +-
.github/workflows/extended.yml | 52 +-
.github/workflows/labeler.yml | 4 +-
.github/workflows/large_files.yml | 2 +-
.github/workflows/pr_comment_commands.yml | 89 --
.github/workflows/rust.yml | 60 +-
.github/workflows/stale.yml | 2 +-
Cargo.lock | 990 ++++++++++++---------
Cargo.toml | 91 +-
README.md | 2 -
benchmarks/Cargo.toml | 5 +-
benchmarks/README.md | 61 ++
benchmarks/src/bin/mem_profile.rs | 360 ++++++++
benchmarks/src/clickbench.rs | 10 +-
benchmarks/src/h2o.rs | 19 +-
benchmarks/src/imdb/mod.rs | 3 +
benchmarks/src/imdb/run.rs | 15 +-
benchmarks/src/sort_tpch.rs | 16 +-
benchmarks/src/tpch/mod.rs | 3 +
benchmarks/src/tpch/run.rs | 15 +-
benchmarks/src/util/memory.rs | 57 ++
benchmarks/src/util/mod.rs | 2 +
clippy.toml | 12 +-
datafusion-cli/Cargo.toml | 8 +-
datafusion-cli/src/print_format.rs | 310 +++----
datafusion-examples/Cargo.toml | 4 +-
datafusion-examples/README.md | 2 +
datafusion-examples/examples/advanced_udwf.rs | 5 +-
datafusion-examples/examples/async_udf.rs | 306 +++----
.../examples/composed_extension_codec.rs | 76 +-
.../examples/parquet_encrypted_with_kms.rs | 301 +++++++
datafusion-examples/examples/sql_frontend.rs | 2 +-
datafusion/catalog/src/streaming.rs | 34 +-
datafusion/common-runtime/Cargo.toml | 2 +-
datafusion/common-runtime/src/common.rs | 21 +-
datafusion/common/Cargo.toml | 4 +-
datafusion/common/src/config.rs | 198 ++++-
datafusion/common/src/encryption.rs | 29 +-
.../common/src/file_options/parquet_writer.rs | 34 +-
datafusion/common/src/rounding.rs | 40 +-
datafusion/common/src/scalar/mod.rs | 982 +++++++++++++++++++-
datafusion/common/src/test_util.rs | 34 +-
datafusion/common/src/types/native.rs | 5 +-
datafusion/common/src/utils/mod.rs | 116 ---
datafusion/core/Cargo.toml | 3 +-
.../core/src/datasource/file_format/parquet.rs | 287 ++++--
datafusion/core/src/execution/context/mod.rs | 34 +-
datafusion/core/src/execution/session_state.rs | 33 +-
datafusion/core/src/lib.rs | 11 +-
datafusion/core/src/physical_planner.rs | 5 +
datafusion/core/src/test/object_store.rs | 24 +-
datafusion/core/tests/core_integration.rs | 3 +
datafusion/core/tests/execution/coop.rs | 83 +-
datafusion/core/tests/fuzz_cases/aggregate_fuzz.rs | 23 +-
.../core/tests/fuzz_cases/equivalence/ordering.rs | 2 +
.../tests/fuzz_cases/equivalence/projection.rs | 3 +
.../tests/fuzz_cases/equivalence/properties.rs | 2 +
datafusion/core/tests/fuzz_cases/mod.rs | 2 +
datafusion/core/tests/fuzz_cases/once_exec.rs | 115 +++
datafusion/core/tests/fuzz_cases/pruning.rs | 11 +-
.../spilling_fuzz_in_memory_constrained_env.rs | 654 ++++++++++++++
datafusion/core/tests/fuzz_cases/window_fuzz.rs | 3 +
.../schema_adapter_integration_tests.rs | 509 -----------
datafusion/core/tests/memory_limit/mod.rs | 10 +-
datafusion/core/tests/optimizer/mod.rs | 2 +-
datafusion/core/tests/parquet/custom_reader.rs | 7 +-
datafusion/core/tests/parquet/encryption.rs | 254 +++++-
datafusion/core/tests/parquet/file_statistics.rs | 3 +-
datafusion/core/tests/parquet/mod.rs | 5 +-
datafusion/core/tests/parquet/page_pruning.rs | 4 +-
datafusion/core/tests/parquet/row_group_pruning.rs | 14 +-
datafusion/core/tests/parquet/schema_adapter.rs | 3 +-
.../tests/physical_optimizer/enforce_sorting.rs | 138 ++-
.../physical_optimizer/filter_pushdown/mod.rs | 665 +++++++++++++-
.../physical_optimizer/filter_pushdown/util.rs | 5 +-
.../tests/physical_optimizer/join_selection.rs | 7 +-
.../physical_optimizer/partition_statistics.rs | 30 +
.../physical_optimizer/projection_pushdown.rs | 5 +
.../core/tests/physical_optimizer/test_utils.rs | 1 +
.../src/test => core/tests/schema_adapter}/mod.rs | 2 +-
.../schema_adapter_integration_tests.rs | 363 ++++++++
datafusion/core/tests/sql/runtime_config.rs | 81 ++
.../tests/user_defined/user_defined_aggregates.rs | 74 +-
.../user_defined/user_defined_scalar_functions.rs | 229 ++---
.../user_defined/user_defined_window_functions.rs | 90 +-
.../datasource-avro/src/avro_to_arrow/schema.rs | 2 +
datafusion/datasource-parquet/Cargo.toml | 1 +
datafusion/datasource-parquet/src/file_format.rs | 266 +++++-
datafusion/datasource-parquet/src/mod.rs | 2 +-
datafusion/datasource-parquet/src/opener.rs | 56 +-
datafusion/datasource-parquet/src/reader.rs | 151 +++-
datafusion/datasource-parquet/src/source.rs | 56 +-
datafusion/datasource/Cargo.toml | 2 +-
datafusion/datasource/src/file_scan_config.rs | 149 +---
datafusion/datasource/src/source.rs | 39 +-
datafusion/doc/src/lib.rs | 13 +-
datafusion/execution/Cargo.toml | 6 +
datafusion/execution/src/cache/cache_manager.rs | 105 ++-
datafusion/execution/src/cache/cache_unit.rs | 435 ++++++++-
datafusion/execution/src/cache/lru_queue.rs | 537 +++++++++++
datafusion/execution/src/cache/mod.rs | 1 +
datafusion/execution/src/config.rs | 72 +-
datafusion/execution/src/lib.rs | 2 +
datafusion/execution/src/parquet_encryption.rs | 81 ++
datafusion/execution/src/runtime_env.rs | 109 ++-
datafusion/expr-common/src/dyn_eq.rs | 64 ++
datafusion/expr-common/src/interval_arithmetic.rs | 225 +++++
datafusion/expr-common/src/lib.rs | 1 +
datafusion/expr-common/src/operator.rs | 63 +-
datafusion/expr/src/async_udf.rs | 49 +-
datafusion/expr/src/execution_props.rs | 21 +-
datafusion/expr/src/expr.rs | 37 +-
datafusion/expr/src/expr_fn.rs | 128 +--
datafusion/expr/src/lib.rs | 2 +
datafusion/expr/src/planner.rs | 2 +-
datafusion/expr/src/ptr_eq.rs | 135 +++
datafusion/expr/src/simplify.rs | 1 -
datafusion/expr/src/tree_node.rs | 12 +
datafusion/expr/src/udaf.rs | 130 ++-
datafusion/expr/src/udf.rs | 50 +-
datafusion/expr/src/udf_eq.rs | 194 ++++
datafusion/expr/src/udwf.rs | 99 +--
datafusion/expr/src/utils.rs | 176 +++-
datafusion/expr/src/window_state.rs | 10 +-
datafusion/ffi/src/session_config.rs | 20 +-
datafusion/ffi/src/udaf/accumulator_args.rs | 1 +
datafusion/ffi/src/udaf/mod.rs | 51 +-
datafusion/ffi/src/udf/mod.rs | 72 +-
datafusion/ffi/src/udwf/mod.rs | 48 +-
datafusion/functions-aggregate-common/src/order.rs | 15 +-
datafusion/functions-aggregate-common/src/utils.rs | 61 +-
.../src/approx_percentile_cont.rs | 22 +-
.../src/approx_percentile_cont_with_weight.rs | 139 +--
datafusion/functions-aggregate/src/array_agg.rs | 102 ++-
.../functions-aggregate/src/bit_and_or_xor.rs | 40 +-
datafusion/functions-aggregate/src/count.rs | 163 +++-
datafusion/functions-aggregate/src/first_last.rs | 56 +-
datafusion/functions-aggregate/src/regr.rs | 238 ++++-
datafusion/functions-aggregate/src/stddev.rs | 24 +-
datafusion/functions-aggregate/src/string_agg.rs | 30 +-
datafusion/functions-aggregate/src/sum.rs | 127 ++-
datafusion/functions-nested/benches/map.rs | 12 +-
datafusion/functions-window/src/cume_dist.rs | 17 +-
datafusion/functions-window/src/lead_lag.rs | 49 +-
datafusion/functions-window/src/macros.rs | 14 +-
datafusion/functions-window/src/nth_value.rs | 56 +-
datafusion/functions-window/src/ntile.rs | 20 +-
datafusion/functions-window/src/planner.rs | 15 +-
datafusion/functions-window/src/rank.rs | 77 +-
datafusion/functions-window/src/row_number.rs | 20 +-
datafusion/functions/benches/ascii.rs | 6 +
datafusion/functions/benches/character_length.rs | 6 +
datafusion/functions/benches/chr.rs | 3 +
datafusion/functions/benches/concat.rs | 3 +
datafusion/functions/benches/cot.rs | 4 +
datafusion/functions/benches/date_bin.rs | 9 +-
datafusion/functions/benches/date_trunc.rs | 9 +-
datafusion/functions/benches/encoding.rs | 7 +
datafusion/functions/benches/find_in_set.rs | 7 +
datafusion/functions/benches/gcd.rs | 5 +
datafusion/functions/benches/initcap.rs | 5 +
datafusion/functions/benches/isnan.rs | 4 +
datafusion/functions/benches/iszero.rs | 4 +
datafusion/functions/benches/lower.rs | 9 +
datafusion/functions/benches/ltrim.rs | 4 +
datafusion/functions/benches/make_date.rs | 16 +-
datafusion/functions/benches/nullif.rs | 3 +
datafusion/functions/benches/pad.rs | 3 +
datafusion/functions/benches/random.rs | 6 +-
datafusion/functions/benches/regx.rs | 2 +
datafusion/functions/benches/repeat.rs | 3 +
datafusion/functions/benches/reverse.rs | 7 +
datafusion/functions/benches/signum.rs | 4 +
datafusion/functions/benches/strpos.rs | 7 +
datafusion/functions/benches/substr.rs | 3 +
datafusion/functions/benches/substr_index.rs | 8 +-
datafusion/functions/benches/to_char.rs | 13 +-
datafusion/functions/benches/to_hex.rs | 5 +
datafusion/functions/benches/to_timestamp.rs | 10 +-
datafusion/functions/benches/trunc.rs | 5 +
datafusion/functions/benches/upper.rs | 4 +
datafusion/functions/benches/uuid.rs | 5 +
datafusion/functions/src/core/union_extract.rs | 6 +-
datafusion/functions/src/core/union_tag.rs | 3 +
datafusion/functions/src/core/version.rs | 3 +
datafusion/functions/src/datetime/date_bin.rs | 2 +
datafusion/functions/src/datetime/date_part.rs | 4 +-
datafusion/functions/src/datetime/date_trunc.rs | 91 +-
datafusion/functions/src/datetime/from_unixtime.rs | 3 +
datafusion/functions/src/datetime/make_date.rs | 2 +
datafusion/functions/src/datetime/to_char.rs | 7 +
datafusion/functions/src/datetime/to_date.rs | 5 +-
datafusion/functions/src/datetime/to_local_time.rs | 3 +
datafusion/functions/src/datetime/to_timestamp.rs | 3 +
datafusion/functions/src/math/log.rs | 11 +
datafusion/functions/src/math/power.rs | 6 +-
datafusion/functions/src/math/signum.rs | 3 +
datafusion/functions/src/regex/regexpcount.rs | 2 +
datafusion/functions/src/regex/regexpinstr.rs | 2 +
datafusion/functions/src/string/chr.rs | 122 +++
datafusion/functions/src/string/concat.rs | 2 +
datafusion/functions/src/string/concat_ws.rs | 3 +
datafusion/functions/src/string/contains.rs | 2 +
datafusion/functions/src/string/lower.rs | 2 +
datafusion/functions/src/string/upper.rs | 2 +
datafusion/functions/src/strings.rs | 39 +-
datafusion/functions/src/unicode/find_in_set.rs | 2 +
datafusion/functions/src/unicode/initcap.rs | 24 +-
datafusion/functions/src/utils.rs | 31 +-
datafusion/optimizer/src/analyzer/mod.rs | 8 -
datafusion/optimizer/src/analyzer/type_coercion.rs | 29 +-
datafusion/optimizer/src/optimizer.rs | 18 +-
datafusion/optimizer/src/push_down_filter.rs | 30 +-
.../src/simplify_expressions/expr_simplifier.rs | 329 +++----
.../src/simplify_expressions/simplify_exprs.rs | 1 +
.../src/simplify_expressions/unwrap_cast.rs | 14 +-
.../optimizer/src/simplify_expressions/utils.rs | 88 ++
.../optimizer/tests/optimizer_integration.rs | 2 +-
.../physical-expr-common/src/physical_expr.rs | 67 +-
datafusion/physical-expr/src/aggregate.rs | 43 +-
.../physical-expr/src/async_scalar_function.rs | 36 +-
.../physical-expr/src/equivalence/ordering.rs | 4 +
.../physical-expr/src/equivalence/projection.rs | 2 +
.../src/equivalence/properties/dependency.rs | 3 +
.../src/equivalence/properties/joins.rs | 8 +-
datafusion/physical-expr/src/expressions/binary.rs | 147 ++-
.../src/expressions/dynamic_filters.rs | 10 +-
.../physical-expr/src/expressions/is_not_null.rs | 5 -
.../physical-expr/src/expressions/is_null.rs | 5 -
.../physical-expr/src/expressions/literal.rs | 2 +-
datafusion/physical-expr/src/planner.rs | 6 +
datafusion/physical-expr/src/scalar_function.rs | 60 +-
datafusion/physical-expr/src/utils/guarantee.rs | 339 ++++++-
datafusion/physical-expr/src/window/aggregate.rs | 6 +-
datafusion/physical-expr/src/window/mod.rs | 6 -
.../physical-expr/src/window/sliding_aggregate.rs | 6 +-
datafusion/physical-expr/src/window/standard.rs | 4 +
datafusion/physical-expr/src/window/window_expr.rs | 6 +
.../physical-optimizer/src/enforce_distribution.rs | 22 +-
.../physical-optimizer/src/filter_pushdown.rs | 358 +++++++-
.../physical-optimizer/src/join_selection.rs | 52 +-
.../physical-optimizer/src/output_requirements.rs | 45 +-
datafusion/physical-plan/Cargo.toml | 1 +
.../group_values/single_group_by/bytes.rs | 13 +-
datafusion/physical-plan/src/aggregates/mod.rs | 148 +--
.../physical-plan/src/aggregates/row_hash.rs | 32 +-
.../src/aggregates/topk/hash_table.rs | 34 +-
datafusion/physical-plan/src/async_func.rs | 9 +-
datafusion/physical-plan/src/coalesce/mod.rs | 12 +-
datafusion/physical-plan/src/common.rs | 2 +-
datafusion/physical-plan/src/coop.rs | 22 +-
datafusion/physical-plan/src/execution_plan.rs | 87 +-
datafusion/physical-plan/src/filter_pushdown.rs | 35 +-
datafusion/physical-plan/src/joins/cross_join.rs | 12 +
datafusion/physical-plan/src/joins/hash_join.rs | 209 ++++-
datafusion/physical-plan/src/joins/test_utils.rs | 25 +
datafusion/physical-plan/src/joins/utils.rs | 4 +-
datafusion/physical-plan/src/lib.rs | 1 -
datafusion/physical-plan/src/metrics/value.rs | 118 +++
datafusion/physical-plan/src/placeholder_row.rs | 14 +-
datafusion/physical-plan/src/recursive_query.rs | 5 +-
datafusion/physical-plan/src/sorts/cursor.rs | 214 +----
datafusion/physical-plan/src/sorts/merge.rs | 15 +-
datafusion/physical-plan/src/sorts/mod.rs | 1 +
.../physical-plan/src/sorts/multi_level_merge.rs | 449 ++++++++++
datafusion/physical-plan/src/sorts/partial_sort.rs | 159 +++-
datafusion/physical-plan/src/sorts/sort.rs | 134 ++-
.../physical-plan/src/sorts/streaming_merge.rs | 92 +-
datafusion/physical-plan/src/spill/mod.rs | 19 +-
.../physical-plan/src/spill/spill_manager.rs | 70 +-
datafusion/physical-plan/src/test.rs | 2 +-
datafusion/physical-plan/src/union.rs | 7 +-
datafusion/physical-plan/src/values.rs | 330 -------
.../src/windows/bounded_window_agg_exec.rs | 1 +
datafusion/physical-plan/src/windows/mod.rs | 42 +-
datafusion/physical-plan/src/work_table.rs | 8 -
.../proto-common/proto/datafusion_common.proto | 42 +-
datafusion/proto-common/src/from_proto/mod.rs | 31 +-
datafusion/proto-common/src/generated/pbjson.rs | 799 ++++++++++++++---
datafusion/proto-common/src/generated/prost.rs | 68 +-
datafusion/proto-common/src/to_proto/mod.rs | 18 +-
datafusion/proto/Cargo.toml | 1 +
datafusion/proto/proto/datafusion.proto | 3 +
.../proto/src/generated/datafusion_proto_common.rs | 68 +-
datafusion/proto/src/generated/pbjson.rs | 49 +
datafusion/proto/src/generated/prost.rs | 11 +-
datafusion/proto/src/lib.rs | 5 +-
datafusion/proto/src/logical_plan/file_formats.rs | 14 -
datafusion/proto/src/logical_plan/mod.rs | 17 +-
datafusion/proto/src/logical_plan/to_proto.rs | 1 +
datafusion/proto/src/physical_plan/from_proto.rs | 106 +--
datafusion/proto/src/physical_plan/mod.rs | 508 ++++++-----
datafusion/proto/src/physical_plan/to_proto.rs | 6 +
datafusion/proto/tests/cases/mod.rs | 70 +-
.../proto/tests/cases/roundtrip_logical_plan.rs | 29 +-
.../proto/tests/cases/roundtrip_physical_plan.rs | 82 +-
.../tests/testdata/test.arrow} | Bin
datafusion/spark/Cargo.toml | 5 +
datafusion/spark/benches/char.rs | 3 +
datafusion/spark/src/function/array/mod.rs | 13 +-
.../src/function/array/spark_array.rs} | 195 ++--
datafusion/spark/src/function/bitwise/bit_count.rs | 319 +++++++
datafusion/spark/src/function/bitwise/bit_get.rs | 287 ++++++
datafusion/spark/src/function/bitwise/mod.rs | 20 +-
datafusion/spark/src/function/datetime/date_add.rs | 129 +++
datafusion/spark/src/function/datetime/date_sub.rs | 123 +++
datafusion/spark/src/function/datetime/last_day.rs | 125 +++
datafusion/spark/src/function/datetime/mod.rs | 40 +-
datafusion/spark/src/function/datetime/next_day.rs | 226 +++++
.../spark/src/function/functions_nested_utils.rs | 53 ++
datafusion/spark/src/function/hash/crc32.rs | 134 +++
datafusion/spark/src/function/hash/mod.rs | 12 +-
datafusion/spark/src/function/hash/sha1.rs | 145 +++
datafusion/spark/src/function/math/hex.rs | 13 +
datafusion/spark/src/function/math/mod.rs | 10 +-
datafusion/spark/src/function/math/modulus.rs | 617 +++++++++++++
datafusion/spark/src/function/math/rint.rs | 165 ++++
datafusion/spark/src/function/mod.rs | 1 +
datafusion/spark/src/function/string/char.rs | 58 +-
datafusion/spark/src/function/string/ilike.rs | 173 ++++
datafusion/spark/src/function/string/like.rs | 178 ++++
datafusion/spark/src/function/string/luhn_check.rs | 153 ++++
datafusion/spark/src/function/string/mod.rs | 25 +-
datafusion/spark/src/function/url/mod.rs | 13 +-
datafusion/spark/src/function/url/parse_url.rs | 301 +++++++
datafusion/spark/src/function/utils.rs | 17 +-
datafusion/sql/src/expr/function.rs | 12 +
datafusion/sql/src/expr/order_by.rs | 16 +-
datafusion/sql/src/parser.rs | 115 ++-
datafusion/sql/src/planner.rs | 62 ++
datafusion/sql/src/statement.rs | 4 +-
datafusion/sql/src/unparser/dialect.rs | 11 -
datafusion/sql/src/unparser/expr.rs | 19 +-
datafusion/sql/tests/sql_integration.rs | 43 +-
datafusion/sqllogictest/Cargo.toml | 4 +-
datafusion/sqllogictest/data/1.parquet | Bin 0 -> 1258 bytes
datafusion/sqllogictest/data/2.parquet | Bin 0 -> 1273 bytes
.../src/engines/datafusion_engine/error.rs | 2 +-
datafusion/sqllogictest/test_files/aggregate.slt | 138 +++
datafusion/sqllogictest/test_files/copy.slt | 1 -
datafusion/sqllogictest/test_files/cte.slt | 55 ++
datafusion/sqllogictest/test_files/errors.slt | 3 +-
datafusion/sqllogictest/test_files/expr.slt | 23 +
.../sqllogictest/test_files/expr/date_part.slt | 20 +
datafusion/sqllogictest/test_files/group_by.slt | 6 +-
.../sqllogictest/test_files/information_schema.slt | 8 +-
datafusion/sqllogictest/test_files/join.slt.part | 18 +-
datafusion/sqllogictest/test_files/joins.slt | 81 +-
.../test_files/listing_table_statistics.slt | 2 +-
datafusion/sqllogictest/test_files/order.slt | 92 ++
datafusion/sqllogictest/test_files/parquet.slt | 112 +++
.../test_files/parquet_filter_pushdown.slt | 47 +
.../sqllogictest/test_files/parquet_statistics.slt | 16 +-
datafusion/sqllogictest/test_files/predicates.slt | 67 ++
.../sqllogictest/test_files/push_down_filter.slt | 21 +-
.../sqllogictest/test_files/repartition_scan.slt | 8 +-
.../sqllogictest/test_files/spark/array/array.slt | 65 +-
.../test_files/spark/bitwise/bit_count.slt | 204 ++++-
.../test_files/spark/bitwise/bit_get.slt | 51 +-
.../test_files/spark/bitwise/getbit.slt | 52 +-
.../test_files/spark/datetime/date_add.slt | 45 +-
.../test_files/spark/datetime/date_sub.slt | 50 +-
.../test_files/spark/datetime/dateadd.slt | 36 +-
.../test_files/spark/datetime/last_day.slt | 99 ++-
.../test_files/spark/datetime/next_day.slt | 62 +-
.../sqllogictest/test_files/spark/hash/crc32.slt | 53 +-
.../sqllogictest/test_files/spark/hash/sha.slt | 48 +-
.../sqllogictest/test_files/spark/hash/sha1.slt | 48 +-
.../sqllogictest/test_files/spark/math/hex.slt | 10 +
.../sqllogictest/test_files/spark/math/mod.slt | 221 ++++-
.../sqllogictest/test_files/spark/math/pmod.slt | 320 ++++++-
.../sqllogictest/test_files/spark/math/rint.slt | 94 +-
.../test_files/spark/predicate/ilike.slt | 51 +-
.../test_files/spark/predicate/like.slt | 61 +-
.../test_files/spark/string/luhn_check.slt | 162 +++-
.../test_files/spark/url/parse_url.slt | 80 +-
datafusion/sqllogictest/test_files/window.slt | 144 +++
datafusion/substrait/src/extensions.rs | 2 +
.../logical_plan/consumer/expr/scalar_function.rs | 195 +++-
.../logical_plan/consumer/expr/window_function.rs | 1 +
.../src/logical_plan/consumer/rel/read_rel.rs | 30 +-
.../substrait/src/logical_plan/consumer/utils.rs | 2 +
.../logical_plan/producer/expr/scalar_function.rs | 21 +
.../logical_plan/producer/expr/window_function.rs | 1 +
.../tests/cases/builtin_expr_semantics_tests.rs | 124 +++
.../substrait/tests/cases/consumer_integration.rs | 83 ++
datafusion/substrait/tests/cases/mod.rs | 1 +
.../tests/cases/roundtrip_logical_plan.rs | 37 +-
.../test_plans/scalar_fn_logb_expr.substrait.json | 116 +++
.../scalar_fn_to_between_expr.substrait.json | 143 +++
..._to_built_in_binary_expr_and_not.substrait.json | 132 +++
...r_fn_to_built_in_binary_expr_xor.substrait.json | 132 +++
.../virtual_table_with_expressions.substrait.json | 75 ++
dev/changelog/49.0.0.md | 387 ++++++++
dev/release/README.md | 14 +-
dev/update_config_docs.sh | 103 ++-
docs/source/contributor-guide/testing.md | 18 +-
docs/source/index.rst | 2 -
.../library-user-guide/functions/adding-udfs.md | 74 +-
docs/source/library-user-guide/query-optimizer.md | 2 +-
docs/source/library-user-guide/upgrading.md | 173 +++-
docs/source/user-guide/configs.md | 115 ++-
docs/source/user-guide/crate-configuration.md | 78 +-
docs/source/user-guide/expressions.md | 42 +-
docs/source/user-guide/sql/aggregate_functions.md | 222 ++++-
docs/source/user-guide/sql/scalar_functions.md | 5 +-
docs/source/user-guide/sql/special_functions.md | 13 +-
docs/source/user-guide/sql/window_functions.md | 109 +--
414 files changed, 22764 insertions(+), 6046 deletions(-)
delete mode 100644 .github/workflows/pr_comment_commands.yml
create mode 100644 benchmarks/src/bin/mem_profile.rs
create mode 100644 benchmarks/src/util/memory.rs
create mode 100644 datafusion-examples/examples/parquet_encrypted_with_kms.rs
create mode 100644 datafusion/core/tests/fuzz_cases/once_exec.rs
create mode 100644
datafusion/core/tests/fuzz_cases/spilling_fuzz_in_memory_constrained_env.rs
delete mode 100644
datafusion/core/tests/integration_tests/schema_adapter_integration_tests.rs
copy datafusion/{expr/src/test => core/tests/schema_adapter}/mod.rs (95%)
create mode 100644
datafusion/core/tests/schema_adapter/schema_adapter_integration_tests.rs
create mode 100644 datafusion/execution/src/cache/lru_queue.rs
create mode 100644 datafusion/execution/src/parquet_encryption.rs
create mode 100644 datafusion/expr-common/src/dyn_eq.rs
create mode 100644 datafusion/expr/src/ptr_eq.rs
create mode 100644 datafusion/expr/src/udf_eq.rs
create mode 100644 datafusion/physical-plan/src/sorts/multi_level_merge.rs
delete mode 100644 datafusion/physical-plan/src/values.rs
copy datafusion/{core/tests/data/example.arrow =>
proto/tests/testdata/test.arrow} (100%)
copy datafusion/{functions-nested/src/make_array.rs =>
spark/src/function/array/spark_array.rs} (60%)
create mode 100644 datafusion/spark/src/function/bitwise/bit_count.rs
create mode 100644 datafusion/spark/src/function/bitwise/bit_get.rs
create mode 100644 datafusion/spark/src/function/datetime/date_add.rs
create mode 100644 datafusion/spark/src/function/datetime/date_sub.rs
create mode 100644 datafusion/spark/src/function/datetime/last_day.rs
create mode 100644 datafusion/spark/src/function/datetime/next_day.rs
create mode 100644 datafusion/spark/src/function/functions_nested_utils.rs
create mode 100644 datafusion/spark/src/function/hash/crc32.rs
create mode 100644 datafusion/spark/src/function/hash/sha1.rs
create mode 100644 datafusion/spark/src/function/math/modulus.rs
create mode 100644 datafusion/spark/src/function/math/rint.rs
create mode 100644 datafusion/spark/src/function/string/ilike.rs
create mode 100644 datafusion/spark/src/function/string/like.rs
create mode 100644 datafusion/spark/src/function/string/luhn_check.rs
create mode 100644 datafusion/spark/src/function/url/parse_url.rs
create mode 100644 datafusion/sqllogictest/data/1.parquet
create mode 100644 datafusion/sqllogictest/data/2.parquet
create mode 100644
datafusion/substrait/tests/cases/builtin_expr_semantics_tests.rs
create mode 100644
datafusion/substrait/tests/testdata/test_plans/scalar_fn_logb_expr.substrait.json
create mode 100644
datafusion/substrait/tests/testdata/test_plans/scalar_fn_to_between_expr.substrait.json
create mode 100644
datafusion/substrait/tests/testdata/test_plans/scalar_fn_to_built_in_binary_expr_and_not.substrait.json
create mode 100644
datafusion/substrait/tests/testdata/test_plans/scalar_fn_to_built_in_binary_expr_xor.substrait.json
create mode 100644
datafusion/substrait/tests/testdata/test_plans/virtual_table_with_expressions.substrait.json
create mode 100644 dev/changelog/49.0.0.md
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
