This is an automated email from the ASF dual-hosted git repository. wenchen pushed a commit to branch branch-3.4 in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.4 by this push: new 006babc71d1 [SPARK-41151][FOLLOW-UP][SQL] Improve the doc of `_metadata` generated columns nullability implementation 006babc71d1 is described below commit 006babc71d17ee8dfcda3f49d69f079383166e45 Author: yaohua <yaohua.z...@databricks.com> AuthorDate: Wed Feb 22 19:53:50 2023 +0800 [SPARK-41151][FOLLOW-UP][SQL] Improve the doc of `_metadata` generated columns nullability implementation ### What changes were proposed in this pull request? Add a doc of how `_metadata` nullability is implemented for generated metadata columns. ### Why are the changes needed? Improve readability ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? N/A Closes #40035 from Yaohua628/spark-41151-doc-follow-up. Lead-authored-by: yaohua <yaohua.z...@databricks.com> Co-authored-by: Yaohua Zhao <79476540+yaohua...@users.noreply.github.com> Signed-off-by: Wenchen Fan <wenc...@databricks.com> (cherry picked from commit 100056ad1b33e134d71239ec729e609e3a68f2c9) Signed-off-by: Wenchen Fan <wenc...@databricks.com> --- .../spark/sql/catalyst/expressions/namedExpressions.scala | 10 ++++++++++ .../spark/sql/execution/datasources/FileSourceStrategy.scala | 8 ++++++++ 2 files changed, 18 insertions(+) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala index d18cfea1629..52d96f92fdf 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala @@ -578,6 +578,16 @@ object FileSourceGeneratedMetadataAttribute { val FILE_SOURCE_GENERATED_METADATA_COL_ATTR_KEY = "__file_source_generated_metadata_col" + /** + * We keep generated metadata attributes nullability configurable here: + * 1. Before passing to readers, we create generated metadata attributes as nullable; + * Because, for row_index, the readers do not consider the column required. + * row_index can be generated with null in the process by readers. + * 2. When applying the projection, we change the nullability back to not-nullable; + * For row_index, it is generated with nulls which are then replaced, + * so it will not be null in the returned output. + * See `FileSourceStrategy` for more information + */ def apply(name: String, dataType: DataType, nullable: Boolean = false): AttributeReference = AttributeReference(name, dataType, nullable = nullable, new MetadataBuilder() diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategy.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategy.scala index f48e44d1aab..5838f9e5478 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategy.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategy.scala @@ -230,6 +230,14 @@ object FileSourceStrategy extends Strategy with PredicateHelper with Logging { case MetadataStructColumn(attr) => attr } + // We divide metadata columns into two categories: constant and generated. + // For constant metadata columns, we create these attributes as non-nullable + // when passing to readers, since the values are always available. + // For generated metadata columns, they are set as nullable when passed to readers, + // as the values will be null when trying to read the missing column from the file. + // They are then replaced by the actual values later in the process. + // All metadata columns will be non-null in the returned output. + // We then change the nullability to non-nullable in the metadata projection node below. val constantMetadataColumns: mutable.Buffer[Attribute] = mutable.Buffer.empty val generatedMetadataColumns: mutable.Buffer[Attribute] = mutable.Buffer.empty --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org