dengziming commented on code in PR #51686:
URL: https://github.com/apache/spark/pull/51686#discussion_r2239049853
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2ScanRelationPushDown.scala:
##########
@@ -140,39 +140,45 @@ object V2ScanRelationPushDown extends Rule[LogicalPlan]
with PredicateHelper {
val leftSideRequiredColumnNames =
getRequiredColumnNames(leftProjections, leftHolder)
val rightSideRequiredColumnNames =
getRequiredColumnNames(rightProjections, rightHolder)
- // Alias the duplicated columns from left side of the join. We are
creating the
- // Map[String, Int] to tell how many times each column name has occured
within one side.
- val leftSideNameCounts: Map[String, Int] =
-
leftSideRequiredColumnNames.groupBy(identity).view.mapValues(_.size).toMap
- val rightSideNameCounts: Map[String, Int] =
-
rightSideRequiredColumnNames.groupBy(identity).view.mapValues(_.size).toMap
- // It's more performant to call contains on Set than on Seq
- val rightSideColumnNamesSet = rightSideRequiredColumnNames.toSet
-
- val leftSideRequiredColumnsWithAliases = leftSideRequiredColumnNames.map
{ name =>
- val aliasName =
- if (leftSideNameCounts(name) > 1 ||
rightSideColumnNamesSet.contains(name)) {
- generateJoinOutputAlias(name)
+ def generateColumnAliasesForDuplicatedName(
+ leftColumns: Array[String],
+ rightColumns: Array[String]
+ ): (Array[SupportsPushDownJoin.ColumnWithAlias],
+ Array[SupportsPushDownJoin.ColumnWithAlias]) = {
+ // Count occurrences of each column name across both sides to
identify duplicates.
+ val allRequiredColumnNames = leftSideRequiredColumnNames ++
rightSideRequiredColumnNames
+ val allNameCounts: Map[String, Int] =
+ allRequiredColumnNames.groupBy(identity).view.mapValues(_.size).toMap
+ // Use Set for O(1) lookups when checking existing column names, claim
all names
+ // that appears only once to ensure they have highest priority.
+ val allClaimedAliases = mutable.HashSet.empty ++
allNameCounts.filter(_._2 == 1).keySet
+
+ def processColumn(name: String): SupportsPushDownJoin.ColumnWithAlias
= {
+ // Ensure a name that appears only once does not require an alias.
+ if (allNameCounts(name) == 1) {
+ new SupportsPushDownJoin.ColumnWithAlias(name, null)
} else {
- null
+ var attempt = 0
+ // Generate candidate alias: use original name for the first
attempt, then append
+ // suffix for more attempts.
+ var candidate = name
+ // Ensure candidate alias is unique by checking against existing
names.
+ while (allClaimedAliases.contains(candidate)) {
Review Comment:
@PetarVasiljevic-DB This is a very subtle case. I considered it first, but I
thought it was O(n) so I ignored it. Now I add an `aliasSuffixIndex` to avoid
O(n^2) case. PTAL again.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
