wombatu-kun commented on code in PR #20540:
URL: https://github.com/apache/hudi/pull/20540#discussion_r4520862180


##########
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/ExpressionIndexSupport.scala:
##########
@@ -388,111 +379,44 @@ class ExpressionIndexSupport(spark: SparkSession,
     metadataConfig.isEnabled && metaClient.getIndexMetadata.isPresent && 
!metaClient.getIndexMetadata.get().getIndexDefinitions.isEmpty
   }
 
-  private def filterQueriesWithFunctionalFilterKey(queryFilters: 
Seq[Expression], sourceFieldOpt: Option[String],
-                                                   attributeFetcher: 
Function1[Expression, Expression]): List[Tuple2[Expression, List[String]]] = {
-    var expressionIndexQueries: List[Tuple2[Expression, List[String]]] = 
List.empty
-    for (query <- queryFilters) {
-      filterQueryWithRecordKey(query, sourceFieldOpt, 
attributeFetcher).foreach({
-        case (exp: Expression, literals: List[String]) =>
-          expressionIndexQueries = expressionIndexQueries :+ Tuple2.apply(exp, 
literals)
-      })
-    }
-
-    expressionIndexQueries
-  }
-
   /**
-   * Searches for an index partition based on the specified index function and 
target column name.
+   * Each expression index some query conjunct reads, with those conjuncts. A 
conjunct reads an index when one of its
+   * terms is the index's `f(col)` ([[IndexedExpression.matches]]).
    *
-   * This method looks up the index definitions available in the metadata of a 
`metaClient` instance
-   * and attempts to find an index partition where the index function and the 
source fields match
-   * the provided arguments. If a matching index definition is found, the 
partition identifier for
-   * that index is returned.
-   *
-   * @param queryFilters A sequence of `Expression` objects to analyze. Each 
expression should involve a single column
-   *                     for the method to consider it (expressions involving 
multiple columns are skipped).
-   * @return An `Option` containing the index partition identifier if a 
matching index definition is found.
-   *         Returns `None` if no matching index definition is found.
+   * Only `=` and `IN` conjuncts read an expression index for now, as before; 
ranges follow with #20537.
    */
-  private def getExpressionIndexPartitionAndLiterals(queryFilters: 
Seq[Expression]): Option[Tuple3[String, Expression, List[String]]] = {
-    val indexDefinitions = 
metaClient.getIndexMetadata.get().getIndexDefinitions.asScala
-    if (indexDefinitions.nonEmpty) {
-      val functionDefinitions = indexDefinitions.values
-        .filter(definition => 
MetadataPartitionType.fromPartitionPath(definition.getIndexName).equals(MetadataPartitionType.EXPRESSION_INDEX))
-        .toList
-      var indexPartitionAndLiteralsOpt: Option[Tuple3[String, Expression, 
List[String]]] = Option.empty
-      functionDefinitions.foreach(indexDefinition => {
-        val queryInfoOpt = extractQueryAndLiterals(queryFilters, 
indexDefinition)
-        if (queryInfoOpt.isDefined) {
-          indexPartitionAndLiteralsOpt = 
Option.apply(Tuple3.apply(indexDefinition.getIndexName, queryInfoOpt.get._1, 
queryInfoOpt.get._2))
-        }
-      })
-      indexPartitionAndLiteralsOpt
-    } else {
-      Option.empty
-    }
+  private def findIndexUses(queryFilters: Seq[Expression]): Seq[IndexUse] = {
+    val conjuncts = 
queryFilters.flatMap(SparkBaseIndexSupport.splitConjuncts).filter(isEqualityOrIn)
+    
metaClient.getIndexMetadata.get().getIndexDefinitions.values().asScala.toSeq
+      .filter(definition => 
MetadataPartitionType.fromPartitionPath(definition.getIndexName).equals(MetadataPartitionType.EXPRESSION_INDEX))
+      .flatMap { definition =>
+        val indexed = IndexedExpression(definition)
+        val uses = conjuncts.filter(_.find(indexed.matches).isDefined)
+        if (uses.isEmpty) None else Some(IndexUse(definition, indexed, uses))
+      }
   }
 
+  private def isEqualityOrIn(conjunct: Expression): Boolean = 
Leaf.parse(conjunct).exists(_.op match {
+    case Op.Eq | Op.In | Op.InSet(_) => true
+    case _ => false
+  })
+
   /**
-   * Extracts mappings from function names to column names from a sequence of 
expressions.
-   *
-   * This method iterates over a given sequence of Spark SQL expressions and 
identifies expressions
-   * that contain function calls corresponding to keys in the 
ExpressionIndexFunction. It supports only
-   * expressions that are simple binary expressions involving a single column. 
If an expression contains
-   * one of the functions and operates on a single column, this method maps 
the function name to the
-   * column name.
+   * Bloom filter keys: the values of `f(col) = v` and `f(col) IN (...)`, as 
strings like the writer added them. A file
+   * passes if it may hold any key, which is conservative across conjuncts. A 
conjunct with a value that is not a
+   * non-null literal gives no keys, and is skipped.
    */
-  private def extractQueryAndLiterals(queryFilters: Seq[Expression], 
indexDefinition: HoodieIndexDefinition): Option[(Expression, List[String])] = {
-    val attributeFetcher = (expr: Expression) => {
-      expr match {
-        case expression: UnaryExpression => expression.child
-        case expression: DateFormatClass if 
expression.right.asInstanceOf[Literal].value.toString.equals(indexDefinition.getExpressionIndexFormatOption())
 =>
-          expression.left
-        case expression: FromUnixTime
-          if 
expression.format.asInstanceOf[Literal].value.toString.equals(indexDefinition.getExpressionIndexFormatOption(TimestampFormatter.defaultPattern()))
 =>
-          expression.sec
-        case expression: UnixTimestamp if 
expression.right.asInstanceOf[Literal].value.toString.equals(indexDefinition.getExpressionIndexFormatOption(TimestampFormatter.defaultPattern()))
 =>
-          expression.timeExp
-        case expression: ParseToDate if (expression.format.isEmpty && 
StringUtils.isNullOrEmpty(indexDefinition.getExpressionIndexFormatOption()))
-          || (expression.format.isDefined && 
expression.format.get.asInstanceOf[Literal].value.toString.equals(indexDefinition.getExpressionIndexFormatOption()))
 =>
-          expression.left
-        case expression: ParseToTimestamp if (expression.format.isEmpty && 
StringUtils.isNullOrEmpty(indexDefinition.getExpressionIndexFormatOption()))
-          || (expression.format.isDefined && 
expression.format.get.asInstanceOf[Literal].value.toString.equals(indexDefinition.getExpressionIndexFormatOption()))
 =>
-          expression.left
-        case expression: DateAdd if expression.days.isInstanceOf[Literal] && 
expression.days.asInstanceOf[Literal].value.toString.equals(indexDefinition.getExpressionIndexDaysOption)
 =>
-          expression.startDate
-        case expression: DateSub if expression.days.isInstanceOf[Literal] && 
expression.days.asInstanceOf[Literal].value.toString.equals(indexDefinition.getExpressionIndexDaysOption)
 =>
-          expression.startDate
-        case expression: Substring if 
expression.pos.asInstanceOf[Literal].value.toString.equals(indexDefinition.getExpressionIndexPositionOption)
-          && 
expression.len.asInstanceOf[Literal].value.toString.equals(indexDefinition.getExpressionIndexLengthOption)=>
-          expression.str
-        case expression: StringTrim if (expression.trimStr.isEmpty && 
StringUtils.isNullOrEmpty(indexDefinition.getExpressionIndexTrimStringOption))
-          || (expression.trimStr.isDefined && 
expression.trimStr.get.asInstanceOf[Literal].value.toString.equals(indexDefinition.getExpressionIndexTrimStringOption))
  => expression.srcStr
-        case expression: StringTrimLeft if (expression.trimStr.isEmpty && 
StringUtils.isNullOrEmpty(indexDefinition.getExpressionIndexTrimStringOption))
-          || (expression.trimStr.isDefined && 
expression.trimStr.get.asInstanceOf[Literal].value.toString.equals(indexDefinition.getExpressionIndexTrimStringOption))
  => expression.srcStr
-        case expression: StringTrimRight if (expression.trimStr.isEmpty && 
StringUtils.isNullOrEmpty(indexDefinition.getExpressionIndexTrimStringOption))
-          || (expression.trimStr.isDefined && 
expression.trimStr.get.asInstanceOf[Literal].value.toString.equals(indexDefinition.getExpressionIndexTrimStringOption))
 => expression.srcStr
-        case expression: RegExpReplace if 
expression.pos.asInstanceOf[Literal].value.toString.equals("1")
-          && 
expression.regexp.asInstanceOf[Literal].value.toString.equals(indexDefinition.getExpressionIndexPatternOption)
-          && 
expression.rep.asInstanceOf[Literal].value.toString.equals(indexDefinition.getExpressionIndexReplacementOption)
 =>
-          expression.subject
-        case expression: RegExpExtract if 
expression.regexp.asInstanceOf[Literal].value.toString.equals(indexDefinition.getExpressionIndexPatternOption)
-          && 
expression.idx.asInstanceOf[Literal].value.toString.equals(indexDefinition.getExpressionIndexIndexOption)
 =>
-          expression.subject
-        case expression: StringSplit if 
expression.limit.asInstanceOf[Literal].value.toString.equals("-1")
-          && 
expression.regex.asInstanceOf[Literal].value.toString.equals(indexDefinition.getExpressionIndexPatternOption)
 =>
-          expression.str
-        case other => other
+  private def bloomFilterKeys(use: IndexUse): List[String] =

Review Comment:
   Not for `IN` over a cast: master never read the index for `date_add(ts, 1) 
IN (...)` on a STRING or TIMESTAMP `ts`, while a bloom index on it now drops 
the file holding the row. If the keys stay out of this PR, could they get an 
issue?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to