This is an automated email from the ASF dual-hosted git repository.

dkuzmenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/master by this push:
     new 3f6f940af3f HIVE-27357: Map-side SMB Join returns incorrect result 
when tables have different bucket size (Seonggon Namgung, reviewed by Attila 
Turoczy, Denys Kuzmenko)
3f6f940af3f is described below

commit 3f6f940af3f60cc28834268e5d7f5612e3b13c30
Author: seonggon <[email protected]>
AuthorDate: Tue Sep 3 17:43:10 2024 +0900

    HIVE-27357: Map-side SMB Join returns incorrect result when tables have 
different bucket size (Seonggon Namgung, reviewed by Attila Turoczy, Denys 
Kuzmenko)
    
    Closes #4336
---
 .../hive/ql/exec/tez/CustomPartitionVertex.java    | 100 +++++++--------
 .../smb_join_with_different_bucket_size.q          |  23 ++++
 .../llap/smb_join_with_different_bucket_size.q.out | 134 +++++++++++++++++++++
 3 files changed, 203 insertions(+), 54 deletions(-)

diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java
index bd3f004c3d9..1bf1ebfdf61 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.hive.ql.exec.tez;
 
 import java.io.IOException;
+import java.math.BigInteger;
 import java.nio.ByteBuffer;
 import java.util.*;
 import java.util.Map.Entry;
@@ -242,6 +243,8 @@ public class CustomPartitionVertex extends 
VertexManagerPlugin {
 
     Multimap<Integer, InputSplit> bucketToInitialSplitMap =
         getBucketSplitMapForPath(inputName, pathFileSplitsMap);
+    Preconditions.checkState(
+        bucketToInitialSplitMap.keySet().stream().allMatch(i -> 0 <= i && i < 
numBuckets));
 
     try {
       int totalResource = context.getTotalAvailableResource().getMemory();
@@ -356,7 +359,7 @@ public class CustomPartitionVertex extends 
VertexManagerPlugin {
           diEvent.setTargetIndex(task);
           taskEvents.add(diEvent);
         }
-        numSplitsForTask[task] = count;
+        numSplitsForTask[task] += count;
       }
     }
 
@@ -533,83 +536,72 @@ public class CustomPartitionVertex extends 
VertexManagerPlugin {
   private Multimap<Integer, InputSplit> getBucketSplitMapForPath(String 
inputName,
       Map<String, Set<FileSplit>> pathFileSplitsMap) {
 
+    boolean isSMBJoin = numInputsAffectingRootInputSpecUpdate != 1;
+    boolean isMainWork = mainWorkName.isEmpty() || 
inputName.compareTo(mainWorkName) == 0;
+    Preconditions.checkState(
+        isMainWork || isSMBJoin && inputToBucketMap != null && 
inputToBucketMap.containsKey(inputName),
+        "CustomPartitionVertex.inputToBucketMap is not defined for {}", 
inputName);
+    int inputBucketSize = isMainWork ? numBuckets : 
inputToBucketMap.get(inputName);
 
-    Multimap<Integer, InputSplit> bucketToInitialSplitMap =
-        ArrayListMultimap.create();
+    Multimap<Integer, InputSplit> bucketToSplitMap = 
ArrayListMultimap.create();
 
     boolean fallback = false;
-    Map<Integer, Integer> bucketIds = new HashMap<>();
     for (Map.Entry<String, Set<FileSplit>> entry : 
pathFileSplitsMap.entrySet()) {
       // Extract the buckedID from pathFilesMap, this is more accurate method,
       // however. it may not work in certain cases where buckets are named
       // after files used while loading data. In such case, fallback to old
       // potential inaccurate method.
       // The accepted file names are such as 000000_0, 000001_0_copy_1.
-      String bucketIdStr =
-              Utilities.getBucketFileNameFromPathSubString(entry.getKey());
+      String bucketIdStr = 
Utilities.getBucketFileNameFromPathSubString(entry.getKey());
       int bucketId = Utilities.getBucketIdFromFile(bucketIdStr);
-      if (bucketId == -1) {
+      if (bucketId < 0) {
         fallback = true;
-        LOG.info("Fallback to using older sort based logic to assign " +
-                "buckets to splits.");
-        bucketIds.clear();
+        LOG.info("Fallback to using older sort based logic to assign buckets 
to splits.");
+        bucketToSplitMap.clear();
         break;
       }
+
       // Make sure the bucketId is at max the numBuckets
-      bucketId = bucketId % numBuckets;
-      bucketIds.put(bucketId, bucketId);
-      for (FileSplit fsplit : entry.getValue()) {
-        bucketToInitialSplitMap.put(bucketId, fsplit);
-      }
+      bucketId %= inputBucketSize;
+
+      bucketToSplitMap.putAll(bucketId, entry.getValue());
     }
 
-    int bucketNum = 0;
     if (fallback) {
       // This is the old logic which assumes that the filenames are sorted in
       // alphanumeric order and mapped to appropriate bucket number.
+      int curSplitIndex = 0;
       for (Map.Entry<String, Set<FileSplit>> entry : 
pathFileSplitsMap.entrySet()) {
-        int bucketId = bucketNum % numBuckets;
-        for (FileSplit fsplit : entry.getValue()) {
-          bucketToInitialSplitMap.put(bucketId, fsplit);
-        }
-        bucketNum++;
+        int bucketId = curSplitIndex % inputBucketSize;
+        bucketToSplitMap.putAll(bucketId, entry.getValue());
+        curSplitIndex++;
       }
     }
 
-    // this is just for SMB join use-case. The numBuckets would be equal to 
that of the big table
-    // and the small table could have lesser number of buckets. In this case, 
we want to send the
-    // data from the right buckets to the big table side. For e.g. Big table 
has 8 buckets and small
-    // table has 4 buckets, bucket 0 of small table needs to be sent to bucket 
4 of the big table as
-    // well.
-    if (numInputsAffectingRootInputSpecUpdate != 1) {
-      // small table
-      if (fallback && bucketNum < numBuckets) {
-        // Old logic.
-        int loopedBucketId = 0;
-        for (; bucketNum < numBuckets; bucketNum++) {
-          for (InputSplit fsplit : 
bucketToInitialSplitMap.get(loopedBucketId)) {
-            bucketToInitialSplitMap.put(bucketNum, fsplit);
-          }
-          loopedBucketId++;
-        }
-      } else {
-        // new logic.
-        if (inputToBucketMap.containsKey(inputName)) {
-          int inputNumBuckets = inputToBucketMap.get(inputName);
-          if (inputNumBuckets < numBuckets) {
-            // Need to send the splits to multiple buckets
-            for (int i = 1; i < numBuckets / inputNumBuckets; i++) {
-              int bucketIdBase = i * inputNumBuckets;
-              for (Integer bucketId : bucketIds.keySet()) {
-                for (InputSplit fsplit : 
bucketToInitialSplitMap.get(bucketId)) {
-                  bucketToInitialSplitMap.put(bucketIdBase + bucketId, fsplit);
-                }
-              }
-            }
-          }
-        }
+    if (isSMBJoin && numBuckets != inputBucketSize) {
+      // This is just for SMB join use-case. The numBuckets would be equal to 
that of the big table
+      // and the small table could have different number of buckets. In this 
case, we want to send the
+      // data from the right buckets to the big table side. For e.g. Big table 
has 6 buckets and small
+      // table has 4 buckets, bucket 1 of small table needs to be sent to 
bucket 1, 3, 5 of the big table
+      // because (4*n + 1) % 6 can be 1, 3, or 5.
+
+      int gcd = 
BigInteger.valueOf(numBuckets).gcd(BigInteger.valueOf(inputBucketSize)).intValue();
+      Multimap<Integer, InputSplit> bucketIdRemainderSplitMap = 
ArrayListMultimap.create();
+      for (Entry<Integer, Collection<InputSplit>> entry: 
bucketToSplitMap.asMap().entrySet()) {
+        int smallTableBucketId = entry.getKey();
+        int remainder = smallTableBucketId % gcd;
+        bucketIdRemainderSplitMap.putAll(remainder, entry.getValue());
       }
+
+      Multimap<Integer, InputSplit> redistributedBucketSplitMap = 
ArrayListMultimap.create();
+      for (int bigTableBucketId = 0; bigTableBucketId < numBuckets; 
bigTableBucketId++) {
+        int remainder = bigTableBucketId % gcd;
+        redistributedBucketSplitMap.putAll(bigTableBucketId, 
bucketIdRemainderSplitMap.get(remainder));
+      }
+
+      bucketToSplitMap = redistributedBucketSplitMap;
     }
-    return bucketToInitialSplitMap;
+
+    return bucketToSplitMap;
   }
 }
diff --git 
a/ql/src/test/queries/clientpositive/smb_join_with_different_bucket_size.q 
b/ql/src/test/queries/clientpositive/smb_join_with_different_bucket_size.q
new file mode 100644
index 00000000000..14b8d2f66e2
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/smb_join_with_different_bucket_size.q
@@ -0,0 +1,23 @@
+-- SORT_QUERY_RESULTS
+
+SET hive.strict.checks.bucketing=true;
+SET hive.auto.convert.join=true;
+SET hive.auto.convert.sortmerge.join=true;
+SET hive.optimize.bucketmapjoin = true;
+SET hive.optimize.bucketmapjoin.sortedmerge = true;
+SET hive.auto.convert.join.noconditionaltask.size=1;
+SET hive.optimize.dynamic.partition.hashjoin=false;
+
+DROP TABLE IF EXISTS bucket2;
+CREATE TABLE bucket2(key string, value string) CLUSTERED BY (key) SORTED BY 
(key) INTO 2 BUCKETS;
+
+DROP TABLE IF EXISTS bucket3;
+CREATE TABLE bucket3(key string, value string) CLUSTERED BY (key) SORTED BY 
(key) INTO 3 BUCKETS;
+
+INSERT INTO TABLE bucket2 VALUES (1, 1), (2, 2), (7, 7), (6, 6), (14, 14), 
(11, 11);
+INSERT INTO TABLE bucket3 VALUES (1, 1), (2, 2), (7, 7), (6, 6), (14, 14), 
(11, 11);
+
+EXPLAIN
+SELECT * FROM bucket2 JOIN bucket3 on bucket2.key = bucket3.key;
+SELECT * FROM bucket2 JOIN bucket3 on bucket2.key = bucket3.key;
+
diff --git 
a/ql/src/test/results/clientpositive/llap/smb_join_with_different_bucket_size.q.out
 
b/ql/src/test/results/clientpositive/llap/smb_join_with_different_bucket_size.q.out
new file mode 100644
index 00000000000..3735b366e56
--- /dev/null
+++ 
b/ql/src/test/results/clientpositive/llap/smb_join_with_different_bucket_size.q.out
@@ -0,0 +1,134 @@
+PREHOOK: query: DROP TABLE IF EXISTS bucket2
+PREHOOK: type: DROPTABLE
+PREHOOK: Output: database:default
+POSTHOOK: query: DROP TABLE IF EXISTS bucket2
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Output: database:default
+PREHOOK: query: CREATE TABLE bucket2(key string, value string) CLUSTERED BY 
(key) SORTED BY (key) INTO 2 BUCKETS
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@bucket2
+POSTHOOK: query: CREATE TABLE bucket2(key string, value string) CLUSTERED BY 
(key) SORTED BY (key) INTO 2 BUCKETS
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@bucket2
+PREHOOK: query: DROP TABLE IF EXISTS bucket3
+PREHOOK: type: DROPTABLE
+PREHOOK: Output: database:default
+POSTHOOK: query: DROP TABLE IF EXISTS bucket3
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Output: database:default
+PREHOOK: query: CREATE TABLE bucket3(key string, value string) CLUSTERED BY 
(key) SORTED BY (key) INTO 3 BUCKETS
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@bucket3
+POSTHOOK: query: CREATE TABLE bucket3(key string, value string) CLUSTERED BY 
(key) SORTED BY (key) INTO 3 BUCKETS
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@bucket3
+PREHOOK: query: INSERT INTO TABLE bucket2 VALUES (1, 1), (2, 2), (7, 7), (6, 
6), (14, 14), (11, 11)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@bucket2
+POSTHOOK: query: INSERT INTO TABLE bucket2 VALUES (1, 1), (2, 2), (7, 7), (6, 
6), (14, 14), (11, 11)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@bucket2
+POSTHOOK: Lineage: bucket2.key SCRIPT []
+POSTHOOK: Lineage: bucket2.value SCRIPT []
+PREHOOK: query: INSERT INTO TABLE bucket3 VALUES (1, 1), (2, 2), (7, 7), (6, 
6), (14, 14), (11, 11)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@bucket3
+POSTHOOK: query: INSERT INTO TABLE bucket3 VALUES (1, 1), (2, 2), (7, 7), (6, 
6), (14, 14), (11, 11)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@bucket3
+POSTHOOK: Lineage: bucket3.key SCRIPT []
+POSTHOOK: Lineage: bucket3.value SCRIPT []
+PREHOOK: query: EXPLAIN
+SELECT * FROM bucket2 JOIN bucket3 on bucket2.key = bucket3.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@bucket2
+PREHOOK: Input: default@bucket3
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN
+SELECT * FROM bucket2 JOIN bucket3 on bucket2.key = bucket3.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@bucket2
+POSTHOOK: Input: default@bucket3
+#### A masked pattern was here ####
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-1
+    Tez
+#### A masked pattern was here ####
+      Vertices:
+        Map 1 
+            Map Operator Tree:
+                TableScan
+                  alias: bucket3
+                  filterExpr: key is not null (type: boolean)
+                  Statistics: Num rows: 6 Data size: 1020 Basic stats: 
COMPLETE Column stats: COMPLETE
+                  Filter Operator
+                    predicate: key is not null (type: boolean)
+                    Statistics: Num rows: 6 Data size: 1020 Basic stats: 
COMPLETE Column stats: COMPLETE
+                    Select Operator
+                      expressions: key (type: string), value (type: string)
+                      outputColumnNames: _col0, _col1
+                      Statistics: Num rows: 6 Data size: 1020 Basic stats: 
COMPLETE Column stats: COMPLETE
+                      Dummy Store
+            Map Operator Tree:
+                TableScan
+                  alias: bucket2
+                  filterExpr: key is not null (type: boolean)
+                  Statistics: Num rows: 6 Data size: 1020 Basic stats: 
COMPLETE Column stats: COMPLETE
+                  Filter Operator
+                    predicate: key is not null (type: boolean)
+                    Statistics: Num rows: 6 Data size: 1020 Basic stats: 
COMPLETE Column stats: COMPLETE
+                    Select Operator
+                      expressions: key (type: string), value (type: string)
+                      outputColumnNames: _col0, _col1
+                      Statistics: Num rows: 6 Data size: 1020 Basic stats: 
COMPLETE Column stats: COMPLETE
+                      Merge Join Operator
+                        condition map:
+                             Inner Join 0 to 1
+                        keys:
+                          0 _col0 (type: string)
+                          1 _col0 (type: string)
+                        outputColumnNames: _col0, _col1, _col2, _col3
+                        Statistics: Num rows: 6 Data size: 2040 Basic stats: 
COMPLETE Column stats: COMPLETE
+                        File Output Operator
+                          compressed: false
+                          Statistics: Num rows: 6 Data size: 2040 Basic stats: 
COMPLETE Column stats: COMPLETE
+                          table:
+                              input format: 
org.apache.hadoop.mapred.SequenceFileInputFormat
+                              output format: 
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                              serde: 
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+            Execution mode: llap
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        ListSink
+
+PREHOOK: query: SELECT * FROM bucket2 JOIN bucket3 on bucket2.key = bucket3.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@bucket2
+PREHOOK: Input: default@bucket3
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM bucket2 JOIN bucket3 on bucket2.key = 
bucket3.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@bucket2
+POSTHOOK: Input: default@bucket3
+#### A masked pattern was here ####
+1      1       1       1
+11     11      11      11
+14     14      14      14
+2      2       2       2
+6      6       6       6
+7      7       7       7

Reply via email to