This is an automated email from the ASF dual-hosted git repository.
dkuzmenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 3f6f940af3f HIVE-27357: Map-side SMB Join returns incorrect result
when tables have different bucket size (Seonggon Namgung, reviewed by Attila
Turoczy, Denys Kuzmenko)
3f6f940af3f is described below
commit 3f6f940af3f60cc28834268e5d7f5612e3b13c30
Author: seonggon <[email protected]>
AuthorDate: Tue Sep 3 17:43:10 2024 +0900
HIVE-27357: Map-side SMB Join returns incorrect result when tables have
different bucket size (Seonggon Namgung, reviewed by Attila Turoczy, Denys
Kuzmenko)
Closes #4336
---
.../hive/ql/exec/tez/CustomPartitionVertex.java | 100 +++++++--------
.../smb_join_with_different_bucket_size.q | 23 ++++
.../llap/smb_join_with_different_bucket_size.q.out | 134 +++++++++++++++++++++
3 files changed, 203 insertions(+), 54 deletions(-)
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java
index bd3f004c3d9..1bf1ebfdf61 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.hive.ql.exec.tez;
import java.io.IOException;
+import java.math.BigInteger;
import java.nio.ByteBuffer;
import java.util.*;
import java.util.Map.Entry;
@@ -242,6 +243,8 @@ public class CustomPartitionVertex extends
VertexManagerPlugin {
Multimap<Integer, InputSplit> bucketToInitialSplitMap =
getBucketSplitMapForPath(inputName, pathFileSplitsMap);
+ Preconditions.checkState(
+ bucketToInitialSplitMap.keySet().stream().allMatch(i -> 0 <= i && i <
numBuckets));
try {
int totalResource = context.getTotalAvailableResource().getMemory();
@@ -356,7 +359,7 @@ public class CustomPartitionVertex extends
VertexManagerPlugin {
diEvent.setTargetIndex(task);
taskEvents.add(diEvent);
}
- numSplitsForTask[task] = count;
+ numSplitsForTask[task] += count;
}
}
@@ -533,83 +536,72 @@ public class CustomPartitionVertex extends
VertexManagerPlugin {
private Multimap<Integer, InputSplit> getBucketSplitMapForPath(String
inputName,
Map<String, Set<FileSplit>> pathFileSplitsMap) {
+ boolean isSMBJoin = numInputsAffectingRootInputSpecUpdate != 1;
+ boolean isMainWork = mainWorkName.isEmpty() ||
inputName.compareTo(mainWorkName) == 0;
+ Preconditions.checkState(
+ isMainWork || isSMBJoin && inputToBucketMap != null &&
inputToBucketMap.containsKey(inputName),
+ "CustomPartitionVertex.inputToBucketMap is not defined for {}",
inputName);
+ int inputBucketSize = isMainWork ? numBuckets :
inputToBucketMap.get(inputName);
- Multimap<Integer, InputSplit> bucketToInitialSplitMap =
- ArrayListMultimap.create();
+ Multimap<Integer, InputSplit> bucketToSplitMap =
ArrayListMultimap.create();
boolean fallback = false;
- Map<Integer, Integer> bucketIds = new HashMap<>();
for (Map.Entry<String, Set<FileSplit>> entry :
pathFileSplitsMap.entrySet()) {
// Extract the buckedID from pathFilesMap, this is more accurate method,
// however. it may not work in certain cases where buckets are named
// after files used while loading data. In such case, fallback to old
// potential inaccurate method.
// The accepted file names are such as 000000_0, 000001_0_copy_1.
- String bucketIdStr =
- Utilities.getBucketFileNameFromPathSubString(entry.getKey());
+ String bucketIdStr =
Utilities.getBucketFileNameFromPathSubString(entry.getKey());
int bucketId = Utilities.getBucketIdFromFile(bucketIdStr);
- if (bucketId == -1) {
+ if (bucketId < 0) {
fallback = true;
- LOG.info("Fallback to using older sort based logic to assign " +
- "buckets to splits.");
- bucketIds.clear();
+ LOG.info("Fallback to using older sort based logic to assign buckets
to splits.");
+ bucketToSplitMap.clear();
break;
}
+
// Make sure the bucketId is at max the numBuckets
- bucketId = bucketId % numBuckets;
- bucketIds.put(bucketId, bucketId);
- for (FileSplit fsplit : entry.getValue()) {
- bucketToInitialSplitMap.put(bucketId, fsplit);
- }
+ bucketId %= inputBucketSize;
+
+ bucketToSplitMap.putAll(bucketId, entry.getValue());
}
- int bucketNum = 0;
if (fallback) {
// This is the old logic which assumes that the filenames are sorted in
// alphanumeric order and mapped to appropriate bucket number.
+ int curSplitIndex = 0;
for (Map.Entry<String, Set<FileSplit>> entry :
pathFileSplitsMap.entrySet()) {
- int bucketId = bucketNum % numBuckets;
- for (FileSplit fsplit : entry.getValue()) {
- bucketToInitialSplitMap.put(bucketId, fsplit);
- }
- bucketNum++;
+ int bucketId = curSplitIndex % inputBucketSize;
+ bucketToSplitMap.putAll(bucketId, entry.getValue());
+ curSplitIndex++;
}
}
- // this is just for SMB join use-case. The numBuckets would be equal to
that of the big table
- // and the small table could have lesser number of buckets. In this case,
we want to send the
- // data from the right buckets to the big table side. For e.g. Big table
has 8 buckets and small
- // table has 4 buckets, bucket 0 of small table needs to be sent to bucket
4 of the big table as
- // well.
- if (numInputsAffectingRootInputSpecUpdate != 1) {
- // small table
- if (fallback && bucketNum < numBuckets) {
- // Old logic.
- int loopedBucketId = 0;
- for (; bucketNum < numBuckets; bucketNum++) {
- for (InputSplit fsplit :
bucketToInitialSplitMap.get(loopedBucketId)) {
- bucketToInitialSplitMap.put(bucketNum, fsplit);
- }
- loopedBucketId++;
- }
- } else {
- // new logic.
- if (inputToBucketMap.containsKey(inputName)) {
- int inputNumBuckets = inputToBucketMap.get(inputName);
- if (inputNumBuckets < numBuckets) {
- // Need to send the splits to multiple buckets
- for (int i = 1; i < numBuckets / inputNumBuckets; i++) {
- int bucketIdBase = i * inputNumBuckets;
- for (Integer bucketId : bucketIds.keySet()) {
- for (InputSplit fsplit :
bucketToInitialSplitMap.get(bucketId)) {
- bucketToInitialSplitMap.put(bucketIdBase + bucketId, fsplit);
- }
- }
- }
- }
- }
+ if (isSMBJoin && numBuckets != inputBucketSize) {
+ // This is just for SMB join use-case. The numBuckets would be equal to
that of the big table
+ // and the small table could have different number of buckets. In this
case, we want to send the
+ // data from the right buckets to the big table side. For e.g. Big table
has 6 buckets and small
+ // table has 4 buckets, bucket 1 of small table needs to be sent to
bucket 1, 3, 5 of the big table
+ // because (4*n + 1) % 6 can be 1, 3, or 5.
+
+ int gcd =
BigInteger.valueOf(numBuckets).gcd(BigInteger.valueOf(inputBucketSize)).intValue();
+ Multimap<Integer, InputSplit> bucketIdRemainderSplitMap =
ArrayListMultimap.create();
+ for (Entry<Integer, Collection<InputSplit>> entry:
bucketToSplitMap.asMap().entrySet()) {
+ int smallTableBucketId = entry.getKey();
+ int remainder = smallTableBucketId % gcd;
+ bucketIdRemainderSplitMap.putAll(remainder, entry.getValue());
}
+
+ Multimap<Integer, InputSplit> redistributedBucketSplitMap =
ArrayListMultimap.create();
+ for (int bigTableBucketId = 0; bigTableBucketId < numBuckets;
bigTableBucketId++) {
+ int remainder = bigTableBucketId % gcd;
+ redistributedBucketSplitMap.putAll(bigTableBucketId,
bucketIdRemainderSplitMap.get(remainder));
+ }
+
+ bucketToSplitMap = redistributedBucketSplitMap;
}
- return bucketToInitialSplitMap;
+
+ return bucketToSplitMap;
}
}
diff --git
a/ql/src/test/queries/clientpositive/smb_join_with_different_bucket_size.q
b/ql/src/test/queries/clientpositive/smb_join_with_different_bucket_size.q
new file mode 100644
index 00000000000..14b8d2f66e2
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/smb_join_with_different_bucket_size.q
@@ -0,0 +1,23 @@
+-- SORT_QUERY_RESULTS
+
+SET hive.strict.checks.bucketing=true;
+SET hive.auto.convert.join=true;
+SET hive.auto.convert.sortmerge.join=true;
+SET hive.optimize.bucketmapjoin = true;
+SET hive.optimize.bucketmapjoin.sortedmerge = true;
+SET hive.auto.convert.join.noconditionaltask.size=1;
+SET hive.optimize.dynamic.partition.hashjoin=false;
+
+DROP TABLE IF EXISTS bucket2;
+CREATE TABLE bucket2(key string, value string) CLUSTERED BY (key) SORTED BY
(key) INTO 2 BUCKETS;
+
+DROP TABLE IF EXISTS bucket3;
+CREATE TABLE bucket3(key string, value string) CLUSTERED BY (key) SORTED BY
(key) INTO 3 BUCKETS;
+
+INSERT INTO TABLE bucket2 VALUES (1, 1), (2, 2), (7, 7), (6, 6), (14, 14),
(11, 11);
+INSERT INTO TABLE bucket3 VALUES (1, 1), (2, 2), (7, 7), (6, 6), (14, 14),
(11, 11);
+
+EXPLAIN
+SELECT * FROM bucket2 JOIN bucket3 on bucket2.key = bucket3.key;
+SELECT * FROM bucket2 JOIN bucket3 on bucket2.key = bucket3.key;
+
diff --git
a/ql/src/test/results/clientpositive/llap/smb_join_with_different_bucket_size.q.out
b/ql/src/test/results/clientpositive/llap/smb_join_with_different_bucket_size.q.out
new file mode 100644
index 00000000000..3735b366e56
--- /dev/null
+++
b/ql/src/test/results/clientpositive/llap/smb_join_with_different_bucket_size.q.out
@@ -0,0 +1,134 @@
+PREHOOK: query: DROP TABLE IF EXISTS bucket2
+PREHOOK: type: DROPTABLE
+PREHOOK: Output: database:default
+POSTHOOK: query: DROP TABLE IF EXISTS bucket2
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Output: database:default
+PREHOOK: query: CREATE TABLE bucket2(key string, value string) CLUSTERED BY
(key) SORTED BY (key) INTO 2 BUCKETS
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@bucket2
+POSTHOOK: query: CREATE TABLE bucket2(key string, value string) CLUSTERED BY
(key) SORTED BY (key) INTO 2 BUCKETS
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@bucket2
+PREHOOK: query: DROP TABLE IF EXISTS bucket3
+PREHOOK: type: DROPTABLE
+PREHOOK: Output: database:default
+POSTHOOK: query: DROP TABLE IF EXISTS bucket3
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Output: database:default
+PREHOOK: query: CREATE TABLE bucket3(key string, value string) CLUSTERED BY
(key) SORTED BY (key) INTO 3 BUCKETS
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@bucket3
+POSTHOOK: query: CREATE TABLE bucket3(key string, value string) CLUSTERED BY
(key) SORTED BY (key) INTO 3 BUCKETS
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@bucket3
+PREHOOK: query: INSERT INTO TABLE bucket2 VALUES (1, 1), (2, 2), (7, 7), (6,
6), (14, 14), (11, 11)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@bucket2
+POSTHOOK: query: INSERT INTO TABLE bucket2 VALUES (1, 1), (2, 2), (7, 7), (6,
6), (14, 14), (11, 11)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@bucket2
+POSTHOOK: Lineage: bucket2.key SCRIPT []
+POSTHOOK: Lineage: bucket2.value SCRIPT []
+PREHOOK: query: INSERT INTO TABLE bucket3 VALUES (1, 1), (2, 2), (7, 7), (6,
6), (14, 14), (11, 11)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@bucket3
+POSTHOOK: query: INSERT INTO TABLE bucket3 VALUES (1, 1), (2, 2), (7, 7), (6,
6), (14, 14), (11, 11)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@bucket3
+POSTHOOK: Lineage: bucket3.key SCRIPT []
+POSTHOOK: Lineage: bucket3.value SCRIPT []
+PREHOOK: query: EXPLAIN
+SELECT * FROM bucket2 JOIN bucket3 on bucket2.key = bucket3.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@bucket2
+PREHOOK: Input: default@bucket3
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN
+SELECT * FROM bucket2 JOIN bucket3 on bucket2.key = bucket3.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@bucket2
+POSTHOOK: Input: default@bucket3
+#### A masked pattern was here ####
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: bucket3
+ filterExpr: key is not null (type: boolean)
+ Statistics: Num rows: 6 Data size: 1020 Basic stats:
COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 6 Data size: 1020 Basic stats:
COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 6 Data size: 1020 Basic stats:
COMPLETE Column stats: COMPLETE
+ Dummy Store
+ Map Operator Tree:
+ TableScan
+ alias: bucket2
+ filterExpr: key is not null (type: boolean)
+ Statistics: Num rows: 6 Data size: 1020 Basic stats:
COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 6 Data size: 1020 Basic stats:
COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 6 Data size: 1020 Basic stats:
COMPLETE Column stats: COMPLETE
+ Merge Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 6 Data size: 2040 Basic stats:
COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 6 Data size: 2040 Basic stats:
COMPLETE Column stats: COMPLETE
+ table:
+ input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: llap
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT * FROM bucket2 JOIN bucket3 on bucket2.key = bucket3.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@bucket2
+PREHOOK: Input: default@bucket3
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM bucket2 JOIN bucket3 on bucket2.key =
bucket3.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@bucket2
+POSTHOOK: Input: default@bucket3
+#### A masked pattern was here ####
+1 1 1 1
+11 11 11 11
+14 14 14 14
+2 2 2 2
+6 6 6 6
+7 7 7 7