viirya commented on code in PR #26214:
URL: https://github.com/apache/datafusion/pull/26214#discussion_r4432889030
##########
datafusion/core/tests/sql/joins.rs:
##########
@@ -701,3 +701,118 @@ async fn
asof_join_unparser_preserves_right_preselection() -> Result<()> {
}
Ok(())
}
+
+/// Registers `o(a, b)` and `i(x, y)` with the fields that
+/// `RecordBatch::try_from_iter` infers, which count only the null bits.
+fn hidden_null_context(b: ArrayRef, y: ArrayRef) -> Result<SessionContext> {
+ let outer = RecordBatch::try_from_iter(vec![
+ (
+ "a",
+ Arc::new(Int32Array::from(vec![Some(1), None])) as ArrayRef,
+ ),
+ ("b", b),
+ ])?;
+ let inner = RecordBatch::try_from_iter(vec![
+ ("x", Arc::new(Int32Array::from(vec![1])) as ArrayRef),
+ ("y", y),
+ ])?;
+ assert!(!outer.schema().field_with_name("b")?.is_nullable());
+ assert!(!inner.schema().field_with_name("y")?.is_nullable());
+ let ctx = SessionContext::new();
+ ctx.register_batch("o", outer)?;
+ ctx.register_batch("i", inner)?;
+ Ok(ctx)
+}
+
+/// A dictionary with keys `keys` into `values`.
+fn dictionary(keys: Vec<i32>, values: ArrayRef) -> Result<ArrayRef> {
+ Ok(Arc::new(DictionaryArray::<Int32Type>::try_new(
+ Int32Array::from(keys),
+ values,
+ )?))
+}
+
+const HIDDEN_NULL_PROJECTED: &str = "SELECT a, (a, b) NOT IN (SELECT x, y FROM
i) AS r \
+ FROM o ORDER BY a NULLS LAST";
+const HIDDEN_NULL_FILTERED: &str =
+ "SELECT a FROM o WHERE (a, b) NOT IN (SELECT x, y FROM i)";
+
+/// A dictionary key can point to a NULL value without a null bit, so the
+/// field inferred for `b` is non-nullable although `b` holds a NULL. The
+/// multi-column `NOT IN` must still see that NULL: `(1, NULL) NOT IN ((1, 2))`
+/// is NULL, so a `WHERE` drops the row.
+#[tokio::test]
+async fn multi_column_not_in_sees_null_dictionary_values() -> Result<()> {
+ let ctx = hidden_null_context(
+ dictionary(vec![0, 1], Arc::new(Int32Array::from(vec![None,
Some(3)])))?,
+ dictionary(vec![0], Arc::new(Int32Array::from(vec![2])))?,
+ )?;
+ let projected = ctx.sql(HIDDEN_NULL_PROJECTED).await?.collect().await?;
+ assert_batches_eq!(
+ [
+ "+---+------+",
+ "| a | r |",
+ "+---+------+",
+ "| 1 | |",
+ "| | true |",
+ "+---+------+",
+ ],
+ &projected
+ );
+ let filtered = ctx.sql(HIDDEN_NULL_FILTERED).await?.collect().await?;
+ assert_batches_eq!(["+---+", "| a |", "+---+", "| |", "+---+"],
&filtered);
+ Ok(())
+}
+
+/// Every slot of a `Null` array is NULL without a null bit, so the inferred
+/// fields of `b` and `y` are non-nullable. Each row compares a NULL `b`, so
+/// `NOT IN` is NULL for both and a `WHERE` drops both.
+#[tokio::test]
+async fn multi_column_not_in_sees_null_type_values() -> Result<()> {
+ let ctx =
+ hidden_null_context(Arc::new(NullArray::new(2)),
Arc::new(NullArray::new(1)))?;
+ let projected = ctx.sql(HIDDEN_NULL_PROJECTED).await?.collect().await?;
+ assert_batches_eq!(
+ [
+ "+---+---+",
+ "| a | r |",
+ "+---+---+",
+ "| 1 | |",
+ "| | |",
+ "+---+---+",
+ ],
+ &projected
+ );
+ let filtered = ctx.sql(HIDDEN_NULL_FILTERED).await?.collect().await?;
+ assert_batches_eq!(["++", "++"], &filtered);
+ Ok(())
+}
+
+/// A cast to a common type makes the hidden NULL of a dictionary a real NULL
+/// under a type that hides none, and the cast keeps the non-nullable field.
+/// The element still stays a value key, on either side of the `IN`.
+#[tokio::test]
+async fn multi_column_not_in_keeps_cast_dictionary_as_value_key() ->
Result<()> {
Review Comment:
Moved to `null_aware_anti_join.slt` in 0eb4fda5b. `arrow_cast` does keep
`NOT NULL`, so the two new EXPLAINs build the dictionary column with it and put
an `Int64` column on the other side. One plan casts `multi_col_outer_dict.b` in
the outer projection and the other casts `multi_col_inner_dict.y` in the
subquery projection, and both show `null_aware(value_keys=2)`. Without the
column-type check in `may_hide_null`, both show `null_aware` instead.
`multi_column_not_in_keeps_cast_dictionary_as_value_key` is gone, and the
two hidden-NULL result tests stay in `joins.rs`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
