Copilot commented on code in PR #1763:
URL:
https://github.com/apache/datafusion-python/pull/1763#discussion_r4106596238
##########
python/tests/test_aggregation.py:
##########
@@ -317,11 +317,55 @@ def test_aggregate_100(df_aggregate_100, name, expr,
expected):
assert df.collect()[0].to_pydict() == expected_dict
+def test_any_value_skips_nulls_per_group():
+ ctx = SessionContext()
+ df = ctx.from_pydict(
+ {"g": ["x", "x", "y", "y", "z"], "v": [None, 7, 8, None, None]}
+ )
+ result = (
+ df.aggregate([column("g")], [f.any_value(column("v")).alias("v")])
+ .sort(column("g").sort())
+ .to_pydict()
+ )
+ assert result == {"g": ["x", "y", "z"], "v": [7, 8, None]}
+
+
[email protected](
+ ("expr", "expected"),
+ [
+ pytest.param(f.mean(column("v")), 2.0, id="mean"),
+ pytest.param(f.mean(column("v"), distinct=True), 3.0,
id="mean_distinct"),
+ pytest.param(
+ f.mean(column("v"), filter=column("v") > lit(1.0)), 5.0,
id="mean_filter"
+ ),
+ pytest.param(f.percentile_cont(column("v"), 0.5), 1.0,
id="percentile_cont"),
+ pytest.param(
+ f.percentile_cont(column("v"), 0.5, distinct=True),
+ 3.0,
+ id="percentile_cont_distinct",
+ ),
+ pytest.param(
+ f.quantile_cont(column("v"), 0.5, distinct=True),
+ 3.0,
+ id="quantile_cont_distinct",
+ ),
+ ],
+)
+def test_distinct_numeric_aggregates(expr, expected):
+ ctx = SessionContext()
+ df = ctx.from_pydict({"v": [1.0, 1.0, 1.0, 5.0]})
+ result = df.aggregate([], [expr.alias("r")]).collect_column("r")[0].as_py()
+ assert result == expected
+
+
data_test_bitwise_and_boolean_functions = [
+ ("any_value_filter", f.any_value(column("a"), filter=column("a") ==
lit(2)), [2]),
("bit_and", f.bit_and(column("a")), [0]),
("bit_and_filter", f.bit_and(column("a"), filter=column("a") != lit(2)),
[1]),
("bit_or", f.bit_or(column("b")), [6]),
("bit_or_filter", f.bit_or(column("b"), filter=column("a") != lit(3)),
[4]),
+ ("bit_and_distinct", f.bit_and(column("b"), distinct=True), [4]),
+ ("bit_or_distinct", f.bit_or(column("b"), distinct=True), [6]),
Review Comment:
These cases cannot verify that `distinct=True` is preserved: bitwise AND and
OR are idempotent, so removing duplicate inputs always yields the same result.
Add an assertion on the built expression/plan (for example, that its canonical
name contains `DISTINCT`) so the newly exposed argument fails the test if it is
silently dropped.
##########
python/datafusion/functions/__init__.py:
##########
@@ -6360,16 +6808,55 @@ def nth_value(
)
-def bit_and(expression: Expr, filter: Expr | None = None) -> Expr:
+def any_value(expression: Expr, filter: Expr | None = None) -> Expr:
+ """Returns an arbitrary non-null value from each group.
+
+ Returns NULL if every value in the group is NULL. Which value is returned
+ is not specified and may differ between runs.
+
+ If using the builder functions described in ref:`_aggregation` this
function ignores
Review Comment:
`ref:` is plain text rather than a Sphinx role, so the generated API
documentation will not link readers to the aggregation guide. Use the
established `:ref:` syntax.
##########
python/datafusion/dataframe.py:
##########
@@ -108,6 +108,32 @@ class ExplainFormat(Enum):
"""Graphviz DOT format for graph rendering."""
+class ExplainAnalyzeLevel(Enum):
Review Comment:
These new option enums are not re-exported from `datafusion.__init__`, even
though the sibling `ExplainFormat` used by the same method is available as
`datafusion.ExplainFormat`. This leaves the new `DataFrame.explain` API
inconsistent and forces users onto an internal module import; please export
both `ExplainAnalyzeLevel` and `ExplainMetricCategory` at the package root and
include them in `__all__`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
