wombatu-kun commented on code in PR #20540:
URL: https://github.com/apache/hudi/pull/20540#discussion_r4511317445
##########
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/ExpressionIndexSupport.scala:
##########
@@ -230,7 +224,7 @@ class ExpressionIndexSupport(spark: SparkSession,
* @param queryColumns target columns to be included into the final table
* @return reshaped table according to the format outlined above
*/
- private def transpose(colStatsRecords:
HoodieData[HoodieMetadataColumnStats], queryColumns: Seq[String],
expressionIndexQuery: Expression): (HoodieData[Row], StructType) = {
+ private def transpose(colStatsRecords:
HoodieData[HoodieMetadataColumnStats], queryColumns: Seq[String], statsType:
DataType): (HoodieData[Row], StructType) = {
Review Comment:
`transpose` now gets `statsType` but still decodes V1 values with the source
column's type, so on a V1 index (created before table version 9) `year(ts)`
over a TIMESTAMP `ts` hits `asInstanceOf[Long]` on its INT stats, and `year(ts)
IN (2023, 2024)`, which used to skip the index, now fails the query from
`prunePartitions`. Could V1 values be decoded with `statsType` instead?
##########
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/ExpressionIndexSupport.scala:
##########
@@ -388,111 +379,44 @@ class ExpressionIndexSupport(spark: SparkSession,
metadataConfig.isEnabled && metaClient.getIndexMetadata.isPresent &&
!metaClient.getIndexMetadata.get().getIndexDefinitions.isEmpty
}
- private def filterQueriesWithFunctionalFilterKey(queryFilters:
Seq[Expression], sourceFieldOpt: Option[String],
- attributeFetcher:
Function1[Expression, Expression]): List[Tuple2[Expression, List[String]]] = {
- var expressionIndexQueries: List[Tuple2[Expression, List[String]]] =
List.empty
- for (query <- queryFilters) {
- filterQueryWithRecordKey(query, sourceFieldOpt,
attributeFetcher).foreach({
- case (exp: Expression, literals: List[String]) =>
- expressionIndexQueries = expressionIndexQueries :+ Tuple2.apply(exp,
literals)
- })
- }
-
- expressionIndexQueries
- }
-
/**
- * Searches for an index partition based on the specified index function and
target column name.
+ * Each expression index some query conjunct reads, with those conjuncts. A
conjunct reads an index when one of its
+ * terms is the index's `f(col)` ([[IndexedExpression.matches]]).
*
- * This method looks up the index definitions available in the metadata of a
`metaClient` instance
- * and attempts to find an index partition where the index function and the
source fields match
- * the provided arguments. If a matching index definition is found, the
partition identifier for
- * that index is returned.
- *
- * @param queryFilters A sequence of `Expression` objects to analyze. Each
expression should involve a single column
- * for the method to consider it (expressions involving
multiple columns are skipped).
- * @return An `Option` containing the index partition identifier if a
matching index definition is found.
- * Returns `None` if no matching index definition is found.
+ * Only `=` and `IN` conjuncts read an expression index for now, as before;
ranges follow with #20537.
*/
- private def getExpressionIndexPartitionAndLiterals(queryFilters:
Seq[Expression]): Option[Tuple3[String, Expression, List[String]]] = {
- val indexDefinitions =
metaClient.getIndexMetadata.get().getIndexDefinitions.asScala
- if (indexDefinitions.nonEmpty) {
- val functionDefinitions = indexDefinitions.values
- .filter(definition =>
MetadataPartitionType.fromPartitionPath(definition.getIndexName).equals(MetadataPartitionType.EXPRESSION_INDEX))
- .toList
- var indexPartitionAndLiteralsOpt: Option[Tuple3[String, Expression,
List[String]]] = Option.empty
- functionDefinitions.foreach(indexDefinition => {
- val queryInfoOpt = extractQueryAndLiterals(queryFilters,
indexDefinition)
- if (queryInfoOpt.isDefined) {
- indexPartitionAndLiteralsOpt =
Option.apply(Tuple3.apply(indexDefinition.getIndexName, queryInfoOpt.get._1,
queryInfoOpt.get._2))
- }
- })
- indexPartitionAndLiteralsOpt
- } else {
- Option.empty
- }
+ private def findIndexUses(queryFilters: Seq[Expression]): Seq[IndexUse] = {
+ val conjuncts =
queryFilters.flatMap(SparkBaseIndexSupport.splitConjuncts).filter(isEqualityOrIn)
+
metaClient.getIndexMetadata.get().getIndexDefinitions.values().asScala.toSeq
+ .filter(definition =>
MetadataPartitionType.fromPartitionPath(definition.getIndexName).equals(MetadataPartitionType.EXPRESSION_INDEX))
+ .flatMap { definition =>
+ val indexed = IndexedExpression(definition)
+ val uses = conjuncts.filter(_.find(indexed.matches).isDefined)
+ if (uses.isEmpty) None else Some(IndexUse(definition, indexed, uses))
+ }
}
+ private def isEqualityOrIn(conjunct: Expression): Boolean =
Leaf.parse(conjunct).exists(_.op match {
+ case Op.Eq | Op.In | Op.InSet(_) => true
Review Comment:
`Op.InSet` now routes an IN of more than 10 values here, and `Verdict` turns
each InSet value into a literal of its internal type, so `date_add(d, 1) IN
(...)` with eleven dates compares DATE stats with INT literals and fails the
query from `prunePartitions`, where master ignored InSet. Could `Verdict` build
those literals with the term's type, or InSet stay out of the expression index
until it does?
##########
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/IndexedExpression.scala:
##########
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hudi.dataskipping
+
+import org.apache.hudi.common.model.HoodieIndexDefinition
+import org.apache.hudi.common.util.StringUtils
+import
org.apache.hudi.core.index.expression.HoodieExpressionIndex.{DAYS_OPTION,
FORMAT_OPTION, IDENTITY_TRANSFORM, LENGTH_OPTION, PATTERN_OPTION,
POSITION_OPTION, REGEX_GROUP_INDEX_OPTION, REPLACEMENT_OPTION,
TRIM_STRING_OPTION}
+
+import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Cast,
DateAdd, DateFormatClass, DateSub, DayOfMonth, Expression, FromUnixTime, Hour,
Length, Literal, Lower, Month, ParseToDate, ParseToTimestamp, RegExpExtract,
RegExpReplace, StringSplit, StringTrim, StringTrimLeft, StringTrimRight,
Substring, UnixTimestamp, Upper, Year}
+import org.apache.spark.sql.catalyst.util.TimestampFormatter
+
+import scala.collection.JavaConverters._
+
+/**
+ * The expression an expression index holds stats for:
`function(sourceColumn)` with the index's options, as
+ * `ExpressionIndexSparkFunctions` computes it on write.
+ *
+ * A filter term reads the index only if [[matches]] says it is this
expression: the same function, the same
+ * options and the same column. Any other term, including the bare column of a
non-identity index, has no
+ * range in the index, so [[ExpressionIndexSource]] returns none and the file
is kept.
+ */
+case class IndexedExpression(function: String, sourceColumn: String, options:
Map[String, String]) {
+
+ def matches(term: Expression): Boolean = term.resolved && (term match {
+ case column: AttributeReference => function == IDENTITY_TRANSFORM &&
isSourceColumn(column)
+
+ case e: Year => function == "year" && isSource(e.child)
+ case e: Month => function == "month" && isSource(e.child)
+ case e: DayOfMonth => function == "day" && isSource(e.child)
+ case e: Hour => function == "hour" && isSource(e.child)
+ case e: DateFormatClass =>
+ function == "date_format" && isSource(e.left) && literalEquals(e.right,
options.get(FORMAT_OPTION))
+ case e: FromUnixTime =>
+ function == "from_unixtime" && isSource(e.sec) &&
literalEquals(e.format, Some(formatOrDefault))
+ case e: UnixTimestamp =>
+ function == "unix_timestamp" && isSource(e.timeExp) &&
literalEquals(e.format, Some(formatOrDefault))
+ case e: ParseToDate => function == "to_date" && isSource(e.left) &&
optionalLiteralEquals(e.format, FORMAT_OPTION)
Review Comment:
A format-less `to_date(col)` / `to_timestamp(col)` reaches this as
`Cast(col, DateType)` / `Cast(col, TimestampType)`, since the optimizer
replaces `ParseToDate` / `ParseToTimestamp` first (the `DateTimeFormatRule`
NOTE), so those indexes stop pruning where master still pruned them, and the
existing tests miss it because `resolveExpr` keeps the analyzer's shape. Is
that deferred to #19480, or could `matches` accept that `Cast` of the source
column for these two functions?
##########
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/ExpressionIndexSupport.scala:
##########
@@ -388,111 +379,44 @@ class ExpressionIndexSupport(spark: SparkSession,
metadataConfig.isEnabled && metaClient.getIndexMetadata.isPresent &&
!metaClient.getIndexMetadata.get().getIndexDefinitions.isEmpty
}
- private def filterQueriesWithFunctionalFilterKey(queryFilters:
Seq[Expression], sourceFieldOpt: Option[String],
- attributeFetcher:
Function1[Expression, Expression]): List[Tuple2[Expression, List[String]]] = {
- var expressionIndexQueries: List[Tuple2[Expression, List[String]]] =
List.empty
- for (query <- queryFilters) {
- filterQueryWithRecordKey(query, sourceFieldOpt,
attributeFetcher).foreach({
- case (exp: Expression, literals: List[String]) =>
- expressionIndexQueries = expressionIndexQueries :+ Tuple2.apply(exp,
literals)
- })
- }
-
- expressionIndexQueries
- }
-
/**
- * Searches for an index partition based on the specified index function and
target column name.
+ * Each expression index some query conjunct reads, with those conjuncts. A
conjunct reads an index when one of its
+ * terms is the index's `f(col)` ([[IndexedExpression.matches]]).
*
- * This method looks up the index definitions available in the metadata of a
`metaClient` instance
- * and attempts to find an index partition where the index function and the
source fields match
- * the provided arguments. If a matching index definition is found, the
partition identifier for
- * that index is returned.
- *
- * @param queryFilters A sequence of `Expression` objects to analyze. Each
expression should involve a single column
- * for the method to consider it (expressions involving
multiple columns are skipped).
- * @return An `Option` containing the index partition identifier if a
matching index definition is found.
- * Returns `None` if no matching index definition is found.
+ * Only `=` and `IN` conjuncts read an expression index for now, as before;
ranges follow with #20537.
*/
- private def getExpressionIndexPartitionAndLiterals(queryFilters:
Seq[Expression]): Option[Tuple3[String, Expression, List[String]]] = {
- val indexDefinitions =
metaClient.getIndexMetadata.get().getIndexDefinitions.asScala
- if (indexDefinitions.nonEmpty) {
- val functionDefinitions = indexDefinitions.values
- .filter(definition =>
MetadataPartitionType.fromPartitionPath(definition.getIndexName).equals(MetadataPartitionType.EXPRESSION_INDEX))
- .toList
- var indexPartitionAndLiteralsOpt: Option[Tuple3[String, Expression,
List[String]]] = Option.empty
- functionDefinitions.foreach(indexDefinition => {
- val queryInfoOpt = extractQueryAndLiterals(queryFilters,
indexDefinition)
- if (queryInfoOpt.isDefined) {
- indexPartitionAndLiteralsOpt =
Option.apply(Tuple3.apply(indexDefinition.getIndexName, queryInfoOpt.get._1,
queryInfoOpt.get._2))
- }
- })
- indexPartitionAndLiteralsOpt
- } else {
- Option.empty
- }
+ private def findIndexUses(queryFilters: Seq[Expression]): Seq[IndexUse] = {
+ val conjuncts =
queryFilters.flatMap(SparkBaseIndexSupport.splitConjuncts).filter(isEqualityOrIn)
+
metaClient.getIndexMetadata.get().getIndexDefinitions.values().asScala.toSeq
+ .filter(definition =>
MetadataPartitionType.fromPartitionPath(definition.getIndexName).equals(MetadataPartitionType.EXPRESSION_INDEX))
+ .flatMap { definition =>
+ val indexed = IndexedExpression(definition)
+ val uses = conjuncts.filter(_.find(indexed.matches).isDefined)
Review Comment:
A conjunct joins a use when any sub-expression matches, so a filter like
`date_format(ts, 'yyyy') = 2023` (Spark casts the indexed string to INT, which
no rule ranges) still reads the index and returns every file as `Some`, and
`lookupCandidateFilesInMetadataTable` then stops before the bloom filter and
column stats indexes. Could a conjunct join only when
`translateIntoExpressionIndexFilterExpr` gives something other than TRUE?
##########
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/ExpressionIndexSupport.scala:
##########
@@ -388,111 +379,44 @@ class ExpressionIndexSupport(spark: SparkSession,
metadataConfig.isEnabled && metaClient.getIndexMetadata.isPresent &&
!metaClient.getIndexMetadata.get().getIndexDefinitions.isEmpty
}
- private def filterQueriesWithFunctionalFilterKey(queryFilters:
Seq[Expression], sourceFieldOpt: Option[String],
- attributeFetcher:
Function1[Expression, Expression]): List[Tuple2[Expression, List[String]]] = {
- var expressionIndexQueries: List[Tuple2[Expression, List[String]]] =
List.empty
- for (query <- queryFilters) {
- filterQueryWithRecordKey(query, sourceFieldOpt,
attributeFetcher).foreach({
- case (exp: Expression, literals: List[String]) =>
- expressionIndexQueries = expressionIndexQueries :+ Tuple2.apply(exp,
literals)
- })
- }
-
- expressionIndexQueries
- }
-
/**
- * Searches for an index partition based on the specified index function and
target column name.
+ * Each expression index some query conjunct reads, with those conjuncts. A
conjunct reads an index when one of its
+ * terms is the index's `f(col)` ([[IndexedExpression.matches]]).
*
- * This method looks up the index definitions available in the metadata of a
`metaClient` instance
- * and attempts to find an index partition where the index function and the
source fields match
- * the provided arguments. If a matching index definition is found, the
partition identifier for
- * that index is returned.
- *
- * @param queryFilters A sequence of `Expression` objects to analyze. Each
expression should involve a single column
- * for the method to consider it (expressions involving
multiple columns are skipped).
- * @return An `Option` containing the index partition identifier if a
matching index definition is found.
- * Returns `None` if no matching index definition is found.
+ * Only `=` and `IN` conjuncts read an expression index for now, as before;
ranges follow with #20537.
*/
- private def getExpressionIndexPartitionAndLiterals(queryFilters:
Seq[Expression]): Option[Tuple3[String, Expression, List[String]]] = {
- val indexDefinitions =
metaClient.getIndexMetadata.get().getIndexDefinitions.asScala
- if (indexDefinitions.nonEmpty) {
- val functionDefinitions = indexDefinitions.values
- .filter(definition =>
MetadataPartitionType.fromPartitionPath(definition.getIndexName).equals(MetadataPartitionType.EXPRESSION_INDEX))
- .toList
- var indexPartitionAndLiteralsOpt: Option[Tuple3[String, Expression,
List[String]]] = Option.empty
- functionDefinitions.foreach(indexDefinition => {
- val queryInfoOpt = extractQueryAndLiterals(queryFilters,
indexDefinition)
- if (queryInfoOpt.isDefined) {
- indexPartitionAndLiteralsOpt =
Option.apply(Tuple3.apply(indexDefinition.getIndexName, queryInfoOpt.get._1,
queryInfoOpt.get._2))
- }
- })
- indexPartitionAndLiteralsOpt
- } else {
- Option.empty
- }
+ private def findIndexUses(queryFilters: Seq[Expression]): Seq[IndexUse] = {
+ val conjuncts =
queryFilters.flatMap(SparkBaseIndexSupport.splitConjuncts).filter(isEqualityOrIn)
+
metaClient.getIndexMetadata.get().getIndexDefinitions.values().asScala.toSeq
+ .filter(definition =>
MetadataPartitionType.fromPartitionPath(definition.getIndexName).equals(MetadataPartitionType.EXPRESSION_INDEX))
+ .flatMap { definition =>
+ val indexed = IndexedExpression(definition)
+ val uses = conjuncts.filter(_.find(indexed.matches).isDefined)
+ if (uses.isEmpty) None else Some(IndexUse(definition, indexed, uses))
+ }
}
+ private def isEqualityOrIn(conjunct: Expression): Boolean =
Leaf.parse(conjunct).exists(_.op match {
+ case Op.Eq | Op.In | Op.InSet(_) => true
+ case _ => false
+ })
+
/**
- * Extracts mappings from function names to column names from a sequence of
expressions.
- *
- * This method iterates over a given sequence of Spark SQL expressions and
identifies expressions
- * that contain function calls corresponding to keys in the
ExpressionIndexFunction. It supports only
- * expressions that are simple binary expressions involving a single column.
If an expression contains
- * one of the functions and operates on a single column, this method maps
the function name to the
- * column name.
+ * Bloom filter keys: the values of `f(col) = v` and `f(col) IN (...)`, as
strings like the writer added them. A file
+ * passes if it may hold any key, which is conservative across conjuncts. A
conjunct with a value that is not a
+ * non-null literal gives no keys, and is skipped.
*/
- private def extractQueryAndLiterals(queryFilters: Seq[Expression],
indexDefinition: HoodieIndexDefinition): Option[(Expression, List[String])] = {
- val attributeFetcher = (expr: Expression) => {
- expr match {
- case expression: UnaryExpression => expression.child
- case expression: DateFormatClass if
expression.right.asInstanceOf[Literal].value.toString.equals(indexDefinition.getExpressionIndexFormatOption())
=>
- expression.left
- case expression: FromUnixTime
- if
expression.format.asInstanceOf[Literal].value.toString.equals(indexDefinition.getExpressionIndexFormatOption(TimestampFormatter.defaultPattern()))
=>
- expression.sec
- case expression: UnixTimestamp if
expression.right.asInstanceOf[Literal].value.toString.equals(indexDefinition.getExpressionIndexFormatOption(TimestampFormatter.defaultPattern()))
=>
- expression.timeExp
- case expression: ParseToDate if (expression.format.isEmpty &&
StringUtils.isNullOrEmpty(indexDefinition.getExpressionIndexFormatOption()))
- || (expression.format.isDefined &&
expression.format.get.asInstanceOf[Literal].value.toString.equals(indexDefinition.getExpressionIndexFormatOption()))
=>
- expression.left
- case expression: ParseToTimestamp if (expression.format.isEmpty &&
StringUtils.isNullOrEmpty(indexDefinition.getExpressionIndexFormatOption()))
- || (expression.format.isDefined &&
expression.format.get.asInstanceOf[Literal].value.toString.equals(indexDefinition.getExpressionIndexFormatOption()))
=>
- expression.left
- case expression: DateAdd if expression.days.isInstanceOf[Literal] &&
expression.days.asInstanceOf[Literal].value.toString.equals(indexDefinition.getExpressionIndexDaysOption)
=>
- expression.startDate
- case expression: DateSub if expression.days.isInstanceOf[Literal] &&
expression.days.asInstanceOf[Literal].value.toString.equals(indexDefinition.getExpressionIndexDaysOption)
=>
- expression.startDate
- case expression: Substring if
expression.pos.asInstanceOf[Literal].value.toString.equals(indexDefinition.getExpressionIndexPositionOption)
- &&
expression.len.asInstanceOf[Literal].value.toString.equals(indexDefinition.getExpressionIndexLengthOption)=>
- expression.str
- case expression: StringTrim if (expression.trimStr.isEmpty &&
StringUtils.isNullOrEmpty(indexDefinition.getExpressionIndexTrimStringOption))
- || (expression.trimStr.isDefined &&
expression.trimStr.get.asInstanceOf[Literal].value.toString.equals(indexDefinition.getExpressionIndexTrimStringOption))
=> expression.srcStr
- case expression: StringTrimLeft if (expression.trimStr.isEmpty &&
StringUtils.isNullOrEmpty(indexDefinition.getExpressionIndexTrimStringOption))
- || (expression.trimStr.isDefined &&
expression.trimStr.get.asInstanceOf[Literal].value.toString.equals(indexDefinition.getExpressionIndexTrimStringOption))
=> expression.srcStr
- case expression: StringTrimRight if (expression.trimStr.isEmpty &&
StringUtils.isNullOrEmpty(indexDefinition.getExpressionIndexTrimStringOption))
- || (expression.trimStr.isDefined &&
expression.trimStr.get.asInstanceOf[Literal].value.toString.equals(indexDefinition.getExpressionIndexTrimStringOption))
=> expression.srcStr
- case expression: RegExpReplace if
expression.pos.asInstanceOf[Literal].value.toString.equals("1")
- &&
expression.regexp.asInstanceOf[Literal].value.toString.equals(indexDefinition.getExpressionIndexPatternOption)
- &&
expression.rep.asInstanceOf[Literal].value.toString.equals(indexDefinition.getExpressionIndexReplacementOption)
=>
- expression.subject
- case expression: RegExpExtract if
expression.regexp.asInstanceOf[Literal].value.toString.equals(indexDefinition.getExpressionIndexPatternOption)
- &&
expression.idx.asInstanceOf[Literal].value.toString.equals(indexDefinition.getExpressionIndexIndexOption)
=>
- expression.subject
- case expression: StringSplit if
expression.limit.asInstanceOf[Literal].value.toString.equals("-1")
- &&
expression.regex.asInstanceOf[Literal].value.toString.equals(indexDefinition.getExpressionIndexPatternOption)
=>
- expression.str
- case other => other
+ private def bloomFilterKeys(use: IndexUse): List[String] =
Review Comment:
The keys are Catalyst internal values, so a DATE or TIMESTAMP term probes
`19359` or epoch micros where the writer added `2023-01-02` or `2023-01-01
00:00:00.0` (its `Row.toString`), and the file holding the value is pruned.
Could keys be limited to string and integral terms, here or in a follow-up?
##########
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/DataSkippingUtils.scala:
##########
@@ -49,6 +49,13 @@ object DataSkippingUtils extends Logging {
}
}
+ /**
+ * Same as [[translateIntoColumnStatsIndexFilterExpr]], against one
expression index: only a term that is the
+ * index's `f(col)` reads its stats, every other leaf keeps the file.
+ */
+ def translateIntoExpressionIndexFilterExpr(dataTableFilterExpr: Expression,
source: ExpressionIndexSource): Expression =
+ StatsFilterCompiler.compile(dataTableFilterExpr, source, strictResolution
= true)
Review Comment:
With the expression index on this path, no production caller compiles with
`strictResolution = false` any more, so
`translateIntoColumnStatsIndexFilterExpr`'s `isExpressionIndex` and the golden
file's 256 `ei` lines only pin a mode nothing uses, and
`StatsFilterCompiler.compile` still says it is false for expression indexes.
Should they go in this PR?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]