Jefffrey commented on code in PR #21885:
URL: https://github.com/apache/datafusion/pull/21885#discussion_r3154578073
##########
datafusion/proto/src/physical_plan/mod.rs:
##########
@@ -716,18 +716,21 @@ impl protobuf::PhysicalPlanNode {
})?;
let filter_selectivity = filter.default_filter_selectivity.try_into();
- let projection = if !filter.projection.is_empty() {
- Some(
- filter
- .projection
- .iter()
- .map(|i| *i as usize)
- .collect::<Vec<_>>(),
- )
- } else {
+ // Determine if the projection is full to optimize used memory,
+ // storing `None` in this case.
Review Comment:
I feel we should focus more on the fact this logic is trying to preserve a
`None` across proto boundaries because of the limitations we face (with how its
encoded the same as `Some(vec![])`) instead of explaining it only as a memory
efficiency gain
##########
datafusion/proto/tests/cases/roundtrip_physical_plan.rs:
##########
@@ -3681,3 +3681,63 @@ async fn
roundtrip_issue_18602_complex_filter_decode_recursion() -> Result<()> {
roundtrip_test_sql_with_context(sql, &ctx).await
}
+
+#[tokio::test]
+async fn test_filter_exec_projection_serde_roundtrip() -> Result<()> {
+ let ctx = SessionContext::new();
+ let codec = DefaultPhysicalExtensionCodec {};
+
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("a", DataType::Int32, false),
+ Field::new("b", DataType::Int32, false),
+ Field::new("c", DataType::Int32, false),
+ ]));
+
+ let input: Arc<dyn ExecutionPlan> =
Arc::new(EmptyExec::new(Arc::clone(&schema)));
+
+ let predicate: Arc<dyn PhysicalExpr> = Arc::new(BinaryExpr::new(
+ Arc::new(Column::new("a", 0)),
+ Operator::Gt,
+ Arc::new(Literal::new(ScalarValue::Int32(Some(0)))),
+ ));
+
+ // Case 1: None -> should round-trip as None (return all columns)
+ let filter =
+ FilterExecBuilder::new(Arc::clone(&predicate),
Arc::clone(&input)).build()?;
+ let proto = PhysicalPlanNode::try_from_physical_plan(Arc::new(filter) as
_, &codec)?;
+ let roundtripped = proto.try_into_physical_plan(ctx.task_ctx().as_ref(),
&codec)?;
+ let rt = roundtripped.as_ref().downcast_ref::<FilterExec>().unwrap();
+ assert_eq!(
+ rt.projection().as_deref(),
+ None,
+ "None projection must stay None after roundtrip"
+ );
+
Review Comment:
What does this mean?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
