This is an automated email from the ASF dual-hosted git repository.

deniskuzZ pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/master by this push:
     new 74cbaeb5230 HIVE-30079: Size a map join hash table from the smaller 
key signal (#6805)
74cbaeb5230 is described below

commit 74cbaeb5230116b34c84a1e34ddfb0984ad83aac
Author: Denys Kuzmenko <[email protected]>
AuthorDate: Thu Oct 29 09:44:48 2026 +0100

    HIVE-30079: Size a map join hash table from the smaller key signal (#6805)
---
 .../hadoop/hive/ql/exec/HashTableLoader.java       |  11 ++
 .../hadoop/hive/ql/exec/tez/HashTableLoader.java   |   9 +-
 .../fast/VectorMapJoinFastHashTableLoader.java     |  28 ++-
 .../exec/TestHashTableLoaderKeyCountForSizing.java |  66 +++++++
 .../fast/TestVectorMapJoinFastHashTableLoader.java | 212 +++++++++++++++++++++
 5 files changed, 313 insertions(+), 13 deletions(-)

diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableLoader.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableLoader.java
index e24b3eacf8c..27b2891508f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableLoader.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableLoader.java
@@ -18,6 +18,8 @@
  */
 package org.apache.hadoop.hive.ql.exec;
 
+import java.util.stream.LongStream;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.ql.exec.mr.ExecMapperContext;
 import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainer;
@@ -39,4 +41,13 @@ void init(ExecMapperContext context, MapredContext 
mrContext, Configuration hcon
 
   void load(MapJoinTableContainer[] mapJoinTables, 
MapJoinTableContainerSerDe[] mapJoinTableSerdes)
       throws HiveException;
+
+  /**
+   * Returns the key count to size a map join hash table for: the smaller 
positive of the
+   * optimizer's estimate and the APPROXIMATE_INPUT_RECORDS counter, or -1 
when neither is positive.
+   * An overestimate is allocated before the first row is read; an 
underestimate only costs a rehash.
+   */
+  static long keyCountForSizing(long estKeyCount, long inputRecords) {
+    return LongStream.of(estKeyCount, inputRecords).filter(count -> count > 
0).min().orElse(-1);
+  }
 }
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java
index 3feda516da3..ed97ba803ea 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java
@@ -18,6 +18,8 @@
  */
 package org.apache.hadoop.hive.ql.exec.tez;
 
+import static org.apache.hadoop.hive.ql.exec.HashTableLoader.keyCountForSizing;
+
 import java.io.IOException;
 import java.lang.management.ManagementFactory;
 import java.util.Collections;
@@ -227,7 +229,7 @@ public void load(MapJoinTableContainer[] mapJoinTables,
         } catch (Exception e) {
           LOG.debug("Failed to get value for counter 
APPROXIMATE_INPUT_RECORDS", e);
         }
-        long keyCount = Math.max(estKeyCount, inputRecords);
+        long keyCount = keyCountForSizing(estKeyCount, inputRecords);
 
         long memory = 0;
         if (useHybridGraceHashJoin) {
@@ -251,8 +253,9 @@ public void load(MapJoinTableContainer[] mapJoinTables,
         }
 
         LOG.info("Loading hash table for input: {} cacheKey: {} 
tableContainer: {} smallTablePos: {} " +
-                        "estKeyCount : {} keyCount : {}", inputName, cacheKey,
-                tableContainer.getClass().getSimpleName(), pos, estKeyCount, 
keyCount);
+                        "estKeyCount : {} inputRecords : {} keyCount : {}",
+                inputName, cacheKey, 
tableContainer.getClass().getSimpleName(), pos, estKeyCount,
+                inputRecords, keyCount);
 
         tableContainer.setSerde(keyCtx, valCtx);
         long startTime = System.currentTimeMillis();
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTableLoader.java
 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTableLoader.java
index 349637f4925..ca34dc84e6b 100644
--- 
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTableLoader.java
+++ 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTableLoader.java
@@ -18,10 +18,13 @@
  */
 package org.apache.hadoop.hive.ql.exec.vector.mapjoin.fast;
 
+import static org.apache.hadoop.hive.ql.exec.HashTableLoader.keyCountForSizing;
+
 import java.io.IOException;
 import java.util.Collections;
 import java.util.Map;
 import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.LinkedBlockingQueue;
@@ -150,19 +153,21 @@ public void resetBeforeOffer(HashTableElementBatch 
elementBatch) {
     }
   }
 
-  private void submitQueueDrainThreads(VectorMapJoinFastTableContainer 
vectorMapJoinFastTableContainer)
-      throws InterruptedException, IOException, SerDeException {
+  private CompletableFuture<Void> submitQueueDrainThreads(
+      VectorMapJoinFastTableContainer vectorMapJoinFastTableContainer) {
+    CompletableFuture<?>[] loadFutures = new 
CompletableFuture<?>[numLoadThreads];
     for (int partitionId = 0; partitionId < numLoadThreads; partitionId++) {
       int finalPartitionId = partitionId;
-      this.loadExecService.submit(() -> {
+      loadFutures[partitionId] = CompletableFuture.runAsync(() -> {
         try {
           LOG.info("Partition id {} with Queue size {}", finalPartitionId, 
loadBatchQueues[finalPartitionId].size());
           drainAndLoadForPartition(finalPartitionId, 
vectorMapJoinFastTableContainer);
         } catch (IOException | InterruptedException | SerDeException | 
HiveException e) {
-          throw new RuntimeException("Failed to start HT Load threads", e);
+          throw new RuntimeException("Hash table drain thread failed for 
partition " + finalPartitionId, e);
         }
-      });
+      }, this.loadExecService);
     }
+    return CompletableFuture.allOf(loadFutures);
   }
 
   private void drainAndLoadForPartition(int partitionId, 
VectorMapJoinFastTableContainer tableContainer)
@@ -258,19 +263,20 @@ public void load(MapJoinTableContainer[] mapJoinTables,
         } catch (Exception e) {
           LOG.debug("Failed to get value for counter 
APPROXIMATE_INPUT_RECORDS", e);
         }
-        long keyCount = Math.max(estKeyCount, inputRecords);
-        initHTLoadingService(keyCount);
+        long keyCount = keyCountForSizing(estKeyCount, inputRecords);
+        initHTLoadingService(Math.max(estKeyCount, inputRecords));
 
         VectorMapJoinFastTableContainer tableContainer =
             new VectorMapJoinFastTableContainer(desc, hconf, keyCount, 
numLoadThreads);
 
         LOG.info("Loading hash table for input: {} cacheKey: {} 
tableContainer: {} smallTablePos: {} " +
-                "estKeyCount : {} keyCount : {}", inputName, cacheKey,
-                tableContainer.getClass().getSimpleName(), pos, estKeyCount, 
keyCount);
+                "estKeyCount : {} inputRecords : {} keyCount : {}",
+                inputName, cacheKey, 
tableContainer.getClass().getSimpleName(), pos, estKeyCount,
+                inputRecords, keyCount);
 
         tableContainer.setSerde(null, null); // No SerDes here.
         // Submit parallel loading Threads
-        submitQueueDrainThreads(tableContainer);
+        CompletableFuture<Void> drainThreadsFuture = 
submitQueueDrainThreads(tableContainer);
 
         long receivedEntries = 0;
         long startTime = System.currentTimeMillis();
@@ -310,6 +316,8 @@ public void load(MapJoinTableContainer[] mapJoinTables,
         if (!loadExecService.awaitTermination(2, TimeUnit.MINUTES)) {
           throw new HiveException("Failed to complete the hash table loader. 
Loading timed out.");
         }
+        // the executor terminates even when a drain thread throws; join() 
rethrows that failure
+        drainThreadsFuture.join();
         batchPool.clear();
         LOG.info("Total received entries: {} Threads {} HT entries: {}", 
receivedEntries, numLoadThreads, totalEntries.get());
 
diff --git 
a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestHashTableLoaderKeyCountForSizing.java
 
b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestHashTableLoaderKeyCountForSizing.java
new file mode 100644
index 00000000000..089f8383501
--- /dev/null
+++ 
b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestHashTableLoaderKeyCountForSizing.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hive.ql.exec;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * The rule both hash table loaders size from. Every case is one call, so the 
two loaders cannot
+ * drift apart on it.
+ */
+public class TestHashTableLoaderKeyCountForSizing {
+
+  /**
+   * The counter counts rows, the estimate distinct keys. A table sized from 
the smaller can at
+   * worst rehash; one sized from a larger count can ask for more than the 
memory threshold and
+   * fail the query.
+   */
+  @Test
+  public void aRecordCountDoesNotSizeAKeyIndexedTable() {
+    assertEquals(65_000_000L, HashTableLoader.keyCountForSizing(65_000_000L, 
320_243_730L));
+  }
+
+  /**
+   * An estimate can be far too high as well: 21,666,666 keys where the 
counter reads 18,166.
+   * Sizing from the estimate allocates 512 MiB of slot arrays per task, the 
counter 0.5 MiB.
+   */
+  @Test
+  public void aRecordCountBoundsAnEstimateFromAbove() {
+    assertEquals(18_166L, HashTableLoader.keyCountForSizing(21_666_666L, 
18_166L));
+  }
+
+  /** Without an estimate the counter is all there is, which is what 
HIVE-23953 added it for. */
+  @Test
+  public void theCounterIsUsedAloneWhenTheEstimateIsAbsent() {
+    assertEquals(320_243_730L, HashTableLoader.keyCountForSizing(-1L, 
320_243_730L));
+  }
+
+  /**
+   * 0 is how an absent signal actually arrives: TezCounters.findCounter 
materialises a counter at
+   * zero when the input never announced, so an empty small table must not pin 
the size to zero.
+   */
+  @Test
+  public void zeroIsTreatedAsAbsentOnEitherSide() {
+    assertEquals(500L, HashTableLoader.keyCountForSizing(0L, 500L));
+    assertEquals(500L, HashTableLoader.keyCountForSizing(500L, 0L));
+    assertEquals(-1L, HashTableLoader.keyCountForSizing(0L, 0L));
+  }
+}
diff --git 
a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastHashTableLoader.java
 
b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastHashTableLoader.java
new file mode 100644
index 00000000000..c8dcd3db5cc
--- /dev/null
+++ 
b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastHashTableLoader.java
@@ -0,0 +1,212 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hive.ql.exec.vector.mapjoin.fast;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import com.google.common.base.Throwables;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.exec.MapJoinOperator;
+import org.apache.hadoop.hive.ql.exec.mapjoin.MapJoinMemoryExhaustionError;
+import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainer;
+import org.apache.hadoop.hive.ql.exec.tez.TezContext;
+import org.apache.hadoop.hive.ql.plan.MapJoinDesc;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.tez.common.counters.TaskCounter;
+import org.apache.tez.common.counters.TezCounter;
+import org.apache.tez.common.counters.TezCounters;
+import org.apache.tez.runtime.api.AbstractLogicalInput;
+import org.apache.tez.runtime.api.InputContext;
+import org.apache.tez.runtime.api.ProcessorContext;
+import org.apache.tez.runtime.library.api.KeyValueReader;
+import org.junit.Test;
+import org.mockito.MockedConstruction;
+import org.mockito.Mockito;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertThrows;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * Drives {@link VectorMapJoinFastHashTableLoader}: what it sizes the 
container for, and what it
+ * does when a drain thread dies.
+ */
+public class TestVectorMapJoinFastHashTableLoader {
+
+  private static final int NUM_THREADS = 4;
+  private static final int ROWS = 4096;
+  // initHTLoadingService only uses more than one thread above FIRST_SIZE_UP
+  private static final long EST_KEYS = 4L * 
VectorMapJoinFastHashTable.FIRST_SIZE_UP;
+
+  private static final KeyValueReader EMPTY_ROWS = new KeyValueReader() {
+    @Override
+    public boolean next() {
+      return false;
+    }
+
+    @Override
+    public Object getCurrentKey() {
+      throw new IllegalStateException();
+    }
+
+    @Override
+    public Object getCurrentValue() {
+      throw new IllegalStateException();
+    }
+  };
+
+  /** Feeds ROWS distinct 4-byte keys; key i hashes to partition i % 
NUM_THREADS. */
+  private static final class Rows extends KeyValueReader {
+    private int idx = -1;
+    private final BytesWritable key = new BytesWritable(new byte[4]);
+    private final BytesWritable value = new BytesWritable(new byte[] {1, 2, 3, 
4});
+
+    @Override
+    public boolean next() {
+      return ++idx < ROWS;
+    }
+
+    @Override
+    public Object getCurrentKey() {
+      byte[] b = key.getBytes();
+      b[0] = (byte) idx;
+      b[1] = (byte) (idx >>> 8);
+      b[2] = (byte) (idx >>> 16);
+      b[3] = (byte) (idx >>> 24);
+      return key;
+    }
+
+    @Override
+    public Object getCurrentValue() {
+      return value;
+    }
+  }
+
+  private static long hashOf(BytesWritable k) {
+    byte[] b = k.getBytes();
+    return (b[0] & 0xffL) | ((b[1] & 0xffL) << 8) | ((b[2] & 0xffL) << 16) | 
((b[3] & 0xffL) << 24);
+  }
+
+  /** A loader on NUM_THREADS reading one small-table input that announces 
estKeys and inputRecords. */
+  private static VectorMapJoinFastHashTableLoader loaderFor(long estKeys, long 
inputRecords,
+      KeyValueReader reader) throws Exception {
+    HiveConf hconf = new HiveConf();
+    hconf.setIntVar(HiveConf.ConfVars.HIVE_MAPJOIN_PARALEL_HASHTABLE_THREADS, 
NUM_THREADS);
+
+    Map<Integer, String> parentToInput = new HashMap<>();
+    parentToInput.put(1, "smallTableInput");
+    Map<Integer, Long> parentKeyCounts = new HashMap<>();
+    parentKeyCounts.put(1, estKeys);
+
+    MapJoinDesc desc = mock(MapJoinDesc.class);
+    when(desc.getParentToInput()).thenReturn(parentToInput);
+    when(desc.getParentKeyCounts()).thenReturn(parentKeyCounts);
+    when(desc.getMemoryMonitorInfo()).thenReturn(null);
+    when(desc.getPosBigTable()).thenReturn(0);
+
+    MapJoinOperator joinOp = mock(MapJoinOperator.class);
+    when(joinOp.getConf()).thenReturn(desc);
+    when(joinOp.getCacheKey()).thenReturn("cacheKey");
+
+    TezCounter counter = mock(TezCounter.class);
+    when(counter.getValue()).thenReturn(inputRecords);
+    TezCounters counters = mock(TezCounters.class);
+    when(counters.findCounter(anyString(), anyString())).thenReturn(counter);
+    
when(counters.findCounter(TaskCounter.APPROXIMATE_INPUT_RECORDS)).thenReturn(counter);
+    ProcessorContext processorContext = mock(ProcessorContext.class);
+    when(processorContext.getCounters()).thenReturn(counters);
+    InputContext inputContext = mock(InputContext.class);
+    when(inputContext.getCounters()).thenReturn(counters);
+    AbstractLogicalInput input = mock(AbstractLogicalInput.class);
+    when(input.getContext()).thenReturn(inputContext);
+    when(input.getReader()).thenReturn(reader);
+    TezContext tezContext = mock(TezContext.class);
+    when(tezContext.getTezProcessorContext()).thenReturn(processorContext);
+    when(tezContext.getInput(anyString())).thenReturn(input);
+    return new VectorMapJoinFastHashTableLoader(tezContext, hconf, joinOp);
+  }
+
+  /**
+   * Estimate and counter straddle FIRST_SIZE_UP: the table is sized from the 
smaller, the thread
+   * count still follows the larger.
+   */
+  @Test
+  public void sizesFromTheSmallerSignalAndThreadsFromTheLarger() throws 
Exception {
+    long estKeys = VectorMapJoinFastHashTable.FIRST_SIZE_UP / 2;
+
+    List<Object> ctorArgs = new ArrayList<>();
+    try (MockedConstruction<VectorMapJoinFastTableContainer> construction = 
Mockito.mockConstruction(
+        VectorMapJoinFastTableContainer.class, (container, ctx) -> 
ctorArgs.addAll(ctx.arguments()))) {
+      loaderFor(estKeys, EST_KEYS, EMPTY_ROWS).load(new 
MapJoinTableContainer[2], null);
+    }
+
+    assertEquals("the table is sized from the smaller signal", estKeys, 
ctorArgs.get(2));
+    assertEquals("the threads follow the larger signal", NUM_THREADS, 
ctorArgs.get(3));
+  }
+
+  @Test
+  public void drainThreadErrorMustNotProduceAShortHashTable() throws Exception 
{
+    AtomicInteger loadedRows = new AtomicInteger();
+    MapJoinMemoryExhaustionError injected =
+        new MapJoinMemoryExhaustionError("simulated expand failure while 
draining partition 0");
+
+    MapJoinTableContainer[] tables = new MapJoinTableContainer[2];
+    Throwable thrown;
+    VectorMapJoinFastTableContainer tableContainer;
+
+    try (MockedConstruction<VectorMapJoinFastTableContainer> construction =
+        Mockito.mockConstruction(VectorMapJoinFastTableContainer.class, 
(container, ctx) -> {
+          when(container.getHashCode(any(BytesWritable.class)))
+              .thenAnswer(inv -> hashOf(inv.getArgument(0)));
+          doAnswer(inv -> {
+            long hashCode = inv.getArgument(0);
+            // the loader routes a row to partition (numLoadThreads - 1) & 
hashCode
+            if (((NUM_THREADS - 1) & hashCode) == 0) {
+              throw injected;
+            }
+            loadedRows.incrementAndGet();
+            return null;
+          }).when(container).putRow(anyLong(), any(BytesWritable.class), 
any(BytesWritable.class));
+        })) {
+
+      VectorMapJoinFastHashTableLoader loader = loaderFor(EST_KEYS, EST_KEYS, 
new Rows());
+      thrown = assertThrows(Throwable.class, () -> loader.load(tables, null));
+      tableContainer = construction.constructed().get(0);
+    }
+
+    assertEquals("one of " + NUM_THREADS + " partitions must have been 
dropped",
+        ROWS - ROWS / NUM_THREADS, loadedRows.get());
+    assertNull("a hash table missing a partition must never be published", 
tables[1]);
+    verify(tableContainer, never()).seal();
+    assertSame("the drain thread's Error must reach the caller", injected, 
Throwables.getRootCause(thrown));
+  }
+}

Reply via email to