This is an automated email from the ASF dual-hosted git repository.
deniskuzZ pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 74cbaeb5230 HIVE-30079: Size a map join hash table from the smaller
key signal (#6805)
74cbaeb5230 is described below
commit 74cbaeb5230116b34c84a1e34ddfb0984ad83aac
Author: Denys Kuzmenko <[email protected]>
AuthorDate: Thu Oct 29 09:44:48 2026 +0100
HIVE-30079: Size a map join hash table from the smaller key signal (#6805)
---
.../hadoop/hive/ql/exec/HashTableLoader.java | 11 ++
.../hadoop/hive/ql/exec/tez/HashTableLoader.java | 9 +-
.../fast/VectorMapJoinFastHashTableLoader.java | 28 ++-
.../exec/TestHashTableLoaderKeyCountForSizing.java | 66 +++++++
.../fast/TestVectorMapJoinFastHashTableLoader.java | 212 +++++++++++++++++++++
5 files changed, 313 insertions(+), 13 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableLoader.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableLoader.java
index e24b3eacf8c..27b2891508f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableLoader.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableLoader.java
@@ -18,6 +18,8 @@
*/
package org.apache.hadoop.hive.ql.exec;
+import java.util.stream.LongStream;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.ql.exec.mr.ExecMapperContext;
import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainer;
@@ -39,4 +41,13 @@ void init(ExecMapperContext context, MapredContext
mrContext, Configuration hcon
void load(MapJoinTableContainer[] mapJoinTables,
MapJoinTableContainerSerDe[] mapJoinTableSerdes)
throws HiveException;
+
+ /**
+ * Returns the key count to size a map join hash table for: the smaller
positive of the
+ * optimizer's estimate and the APPROXIMATE_INPUT_RECORDS counter, or -1
when neither is positive.
+ * An overestimate is allocated before the first row is read; an
underestimate only costs a rehash.
+ */
+ static long keyCountForSizing(long estKeyCount, long inputRecords) {
+ return LongStream.of(estKeyCount, inputRecords).filter(count -> count >
0).min().orElse(-1);
+ }
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java
index 3feda516da3..ed97ba803ea 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java
@@ -18,6 +18,8 @@
*/
package org.apache.hadoop.hive.ql.exec.tez;
+import static org.apache.hadoop.hive.ql.exec.HashTableLoader.keyCountForSizing;
+
import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.util.Collections;
@@ -227,7 +229,7 @@ public void load(MapJoinTableContainer[] mapJoinTables,
} catch (Exception e) {
LOG.debug("Failed to get value for counter
APPROXIMATE_INPUT_RECORDS", e);
}
- long keyCount = Math.max(estKeyCount, inputRecords);
+ long keyCount = keyCountForSizing(estKeyCount, inputRecords);
long memory = 0;
if (useHybridGraceHashJoin) {
@@ -251,8 +253,9 @@ public void load(MapJoinTableContainer[] mapJoinTables,
}
LOG.info("Loading hash table for input: {} cacheKey: {}
tableContainer: {} smallTablePos: {} " +
- "estKeyCount : {} keyCount : {}", inputName, cacheKey,
- tableContainer.getClass().getSimpleName(), pos, estKeyCount,
keyCount);
+ "estKeyCount : {} inputRecords : {} keyCount : {}",
+ inputName, cacheKey,
tableContainer.getClass().getSimpleName(), pos, estKeyCount,
+ inputRecords, keyCount);
tableContainer.setSerde(keyCtx, valCtx);
long startTime = System.currentTimeMillis();
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTableLoader.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTableLoader.java
index 349637f4925..ca34dc84e6b 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTableLoader.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTableLoader.java
@@ -18,10 +18,13 @@
*/
package org.apache.hadoop.hive.ql.exec.vector.mapjoin.fast;
+import static org.apache.hadoop.hive.ql.exec.HashTableLoader.keyCountForSizing;
+
import java.io.IOException;
import java.util.Collections;
import java.util.Map;
import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingQueue;
@@ -150,19 +153,21 @@ public void resetBeforeOffer(HashTableElementBatch
elementBatch) {
}
}
- private void submitQueueDrainThreads(VectorMapJoinFastTableContainer
vectorMapJoinFastTableContainer)
- throws InterruptedException, IOException, SerDeException {
+ private CompletableFuture<Void> submitQueueDrainThreads(
+ VectorMapJoinFastTableContainer vectorMapJoinFastTableContainer) {
+ CompletableFuture<?>[] loadFutures = new
CompletableFuture<?>[numLoadThreads];
for (int partitionId = 0; partitionId < numLoadThreads; partitionId++) {
int finalPartitionId = partitionId;
- this.loadExecService.submit(() -> {
+ loadFutures[partitionId] = CompletableFuture.runAsync(() -> {
try {
LOG.info("Partition id {} with Queue size {}", finalPartitionId,
loadBatchQueues[finalPartitionId].size());
drainAndLoadForPartition(finalPartitionId,
vectorMapJoinFastTableContainer);
} catch (IOException | InterruptedException | SerDeException |
HiveException e) {
- throw new RuntimeException("Failed to start HT Load threads", e);
+ throw new RuntimeException("Hash table drain thread failed for
partition " + finalPartitionId, e);
}
- });
+ }, this.loadExecService);
}
+ return CompletableFuture.allOf(loadFutures);
}
private void drainAndLoadForPartition(int partitionId,
VectorMapJoinFastTableContainer tableContainer)
@@ -258,19 +263,20 @@ public void load(MapJoinTableContainer[] mapJoinTables,
} catch (Exception e) {
LOG.debug("Failed to get value for counter
APPROXIMATE_INPUT_RECORDS", e);
}
- long keyCount = Math.max(estKeyCount, inputRecords);
- initHTLoadingService(keyCount);
+ long keyCount = keyCountForSizing(estKeyCount, inputRecords);
+ initHTLoadingService(Math.max(estKeyCount, inputRecords));
VectorMapJoinFastTableContainer tableContainer =
new VectorMapJoinFastTableContainer(desc, hconf, keyCount,
numLoadThreads);
LOG.info("Loading hash table for input: {} cacheKey: {}
tableContainer: {} smallTablePos: {} " +
- "estKeyCount : {} keyCount : {}", inputName, cacheKey,
- tableContainer.getClass().getSimpleName(), pos, estKeyCount,
keyCount);
+ "estKeyCount : {} inputRecords : {} keyCount : {}",
+ inputName, cacheKey,
tableContainer.getClass().getSimpleName(), pos, estKeyCount,
+ inputRecords, keyCount);
tableContainer.setSerde(null, null); // No SerDes here.
// Submit parallel loading Threads
- submitQueueDrainThreads(tableContainer);
+ CompletableFuture<Void> drainThreadsFuture =
submitQueueDrainThreads(tableContainer);
long receivedEntries = 0;
long startTime = System.currentTimeMillis();
@@ -310,6 +316,8 @@ public void load(MapJoinTableContainer[] mapJoinTables,
if (!loadExecService.awaitTermination(2, TimeUnit.MINUTES)) {
throw new HiveException("Failed to complete the hash table loader.
Loading timed out.");
}
+ // the executor terminates even when a drain thread throws; join()
rethrows that failure
+ drainThreadsFuture.join();
batchPool.clear();
LOG.info("Total received entries: {} Threads {} HT entries: {}",
receivedEntries, numLoadThreads, totalEntries.get());
diff --git
a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestHashTableLoaderKeyCountForSizing.java
b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestHashTableLoaderKeyCountForSizing.java
new file mode 100644
index 00000000000..089f8383501
--- /dev/null
+++
b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestHashTableLoaderKeyCountForSizing.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hive.ql.exec;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * The rule both hash table loaders size from. Every case is one call, so the
two loaders cannot
+ * drift apart on it.
+ */
+public class TestHashTableLoaderKeyCountForSizing {
+
+ /**
+ * The counter counts rows, the estimate distinct keys. A table sized from
the smaller can at
+ * worst rehash; one sized from a larger count can ask for more than the
memory threshold and
+ * fail the query.
+ */
+ @Test
+ public void aRecordCountDoesNotSizeAKeyIndexedTable() {
+ assertEquals(65_000_000L, HashTableLoader.keyCountForSizing(65_000_000L,
320_243_730L));
+ }
+
+ /**
+ * An estimate can be far too high as well: 21,666,666 keys where the
counter reads 18,166.
+ * Sizing from the estimate allocates 512 MiB of slot arrays per task, the
counter 0.5 MiB.
+ */
+ @Test
+ public void aRecordCountBoundsAnEstimateFromAbove() {
+ assertEquals(18_166L, HashTableLoader.keyCountForSizing(21_666_666L,
18_166L));
+ }
+
+ /** Without an estimate the counter is all there is, which is what
HIVE-23953 added it for. */
+ @Test
+ public void theCounterIsUsedAloneWhenTheEstimateIsAbsent() {
+ assertEquals(320_243_730L, HashTableLoader.keyCountForSizing(-1L,
320_243_730L));
+ }
+
+ /**
+ * 0 is how an absent signal actually arrives: TezCounters.findCounter
materialises a counter at
+ * zero when the input never announced, so an empty small table must not pin
the size to zero.
+ */
+ @Test
+ public void zeroIsTreatedAsAbsentOnEitherSide() {
+ assertEquals(500L, HashTableLoader.keyCountForSizing(0L, 500L));
+ assertEquals(500L, HashTableLoader.keyCountForSizing(500L, 0L));
+ assertEquals(-1L, HashTableLoader.keyCountForSizing(0L, 0L));
+ }
+}
diff --git
a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastHashTableLoader.java
b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastHashTableLoader.java
new file mode 100644
index 00000000000..c8dcd3db5cc
--- /dev/null
+++
b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/TestVectorMapJoinFastHashTableLoader.java
@@ -0,0 +1,212 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hive.ql.exec.vector.mapjoin.fast;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import com.google.common.base.Throwables;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.exec.MapJoinOperator;
+import org.apache.hadoop.hive.ql.exec.mapjoin.MapJoinMemoryExhaustionError;
+import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainer;
+import org.apache.hadoop.hive.ql.exec.tez.TezContext;
+import org.apache.hadoop.hive.ql.plan.MapJoinDesc;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.tez.common.counters.TaskCounter;
+import org.apache.tez.common.counters.TezCounter;
+import org.apache.tez.common.counters.TezCounters;
+import org.apache.tez.runtime.api.AbstractLogicalInput;
+import org.apache.tez.runtime.api.InputContext;
+import org.apache.tez.runtime.api.ProcessorContext;
+import org.apache.tez.runtime.library.api.KeyValueReader;
+import org.junit.Test;
+import org.mockito.MockedConstruction;
+import org.mockito.Mockito;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertThrows;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * Drives {@link VectorMapJoinFastHashTableLoader}: what it sizes the
container for, and what it
+ * does when a drain thread dies.
+ */
+public class TestVectorMapJoinFastHashTableLoader {
+
+ private static final int NUM_THREADS = 4;
+ private static final int ROWS = 4096;
+ // initHTLoadingService only uses more than one thread above FIRST_SIZE_UP
+ private static final long EST_KEYS = 4L *
VectorMapJoinFastHashTable.FIRST_SIZE_UP;
+
+ private static final KeyValueReader EMPTY_ROWS = new KeyValueReader() {
+ @Override
+ public boolean next() {
+ return false;
+ }
+
+ @Override
+ public Object getCurrentKey() {
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public Object getCurrentValue() {
+ throw new IllegalStateException();
+ }
+ };
+
+ /** Feeds ROWS distinct 4-byte keys; key i hashes to partition i %
NUM_THREADS. */
+ private static final class Rows extends KeyValueReader {
+ private int idx = -1;
+ private final BytesWritable key = new BytesWritable(new byte[4]);
+ private final BytesWritable value = new BytesWritable(new byte[] {1, 2, 3,
4});
+
+ @Override
+ public boolean next() {
+ return ++idx < ROWS;
+ }
+
+ @Override
+ public Object getCurrentKey() {
+ byte[] b = key.getBytes();
+ b[0] = (byte) idx;
+ b[1] = (byte) (idx >>> 8);
+ b[2] = (byte) (idx >>> 16);
+ b[3] = (byte) (idx >>> 24);
+ return key;
+ }
+
+ @Override
+ public Object getCurrentValue() {
+ return value;
+ }
+ }
+
+ private static long hashOf(BytesWritable k) {
+ byte[] b = k.getBytes();
+ return (b[0] & 0xffL) | ((b[1] & 0xffL) << 8) | ((b[2] & 0xffL) << 16) |
((b[3] & 0xffL) << 24);
+ }
+
+ /** A loader on NUM_THREADS reading one small-table input that announces
estKeys and inputRecords. */
+ private static VectorMapJoinFastHashTableLoader loaderFor(long estKeys, long
inputRecords,
+ KeyValueReader reader) throws Exception {
+ HiveConf hconf = new HiveConf();
+ hconf.setIntVar(HiveConf.ConfVars.HIVE_MAPJOIN_PARALEL_HASHTABLE_THREADS,
NUM_THREADS);
+
+ Map<Integer, String> parentToInput = new HashMap<>();
+ parentToInput.put(1, "smallTableInput");
+ Map<Integer, Long> parentKeyCounts = new HashMap<>();
+ parentKeyCounts.put(1, estKeys);
+
+ MapJoinDesc desc = mock(MapJoinDesc.class);
+ when(desc.getParentToInput()).thenReturn(parentToInput);
+ when(desc.getParentKeyCounts()).thenReturn(parentKeyCounts);
+ when(desc.getMemoryMonitorInfo()).thenReturn(null);
+ when(desc.getPosBigTable()).thenReturn(0);
+
+ MapJoinOperator joinOp = mock(MapJoinOperator.class);
+ when(joinOp.getConf()).thenReturn(desc);
+ when(joinOp.getCacheKey()).thenReturn("cacheKey");
+
+ TezCounter counter = mock(TezCounter.class);
+ when(counter.getValue()).thenReturn(inputRecords);
+ TezCounters counters = mock(TezCounters.class);
+ when(counters.findCounter(anyString(), anyString())).thenReturn(counter);
+
when(counters.findCounter(TaskCounter.APPROXIMATE_INPUT_RECORDS)).thenReturn(counter);
+ ProcessorContext processorContext = mock(ProcessorContext.class);
+ when(processorContext.getCounters()).thenReturn(counters);
+ InputContext inputContext = mock(InputContext.class);
+ when(inputContext.getCounters()).thenReturn(counters);
+ AbstractLogicalInput input = mock(AbstractLogicalInput.class);
+ when(input.getContext()).thenReturn(inputContext);
+ when(input.getReader()).thenReturn(reader);
+ TezContext tezContext = mock(TezContext.class);
+ when(tezContext.getTezProcessorContext()).thenReturn(processorContext);
+ when(tezContext.getInput(anyString())).thenReturn(input);
+ return new VectorMapJoinFastHashTableLoader(tezContext, hconf, joinOp);
+ }
+
+ /**
+ * Estimate and counter straddle FIRST_SIZE_UP: the table is sized from the
smaller, the thread
+ * count still follows the larger.
+ */
+ @Test
+ public void sizesFromTheSmallerSignalAndThreadsFromTheLarger() throws
Exception {
+ long estKeys = VectorMapJoinFastHashTable.FIRST_SIZE_UP / 2;
+
+ List<Object> ctorArgs = new ArrayList<>();
+ try (MockedConstruction<VectorMapJoinFastTableContainer> construction =
Mockito.mockConstruction(
+ VectorMapJoinFastTableContainer.class, (container, ctx) ->
ctorArgs.addAll(ctx.arguments()))) {
+ loaderFor(estKeys, EST_KEYS, EMPTY_ROWS).load(new
MapJoinTableContainer[2], null);
+ }
+
+ assertEquals("the table is sized from the smaller signal", estKeys,
ctorArgs.get(2));
+ assertEquals("the threads follow the larger signal", NUM_THREADS,
ctorArgs.get(3));
+ }
+
+ @Test
+ public void drainThreadErrorMustNotProduceAShortHashTable() throws Exception
{
+ AtomicInteger loadedRows = new AtomicInteger();
+ MapJoinMemoryExhaustionError injected =
+ new MapJoinMemoryExhaustionError("simulated expand failure while
draining partition 0");
+
+ MapJoinTableContainer[] tables = new MapJoinTableContainer[2];
+ Throwable thrown;
+ VectorMapJoinFastTableContainer tableContainer;
+
+ try (MockedConstruction<VectorMapJoinFastTableContainer> construction =
+ Mockito.mockConstruction(VectorMapJoinFastTableContainer.class,
(container, ctx) -> {
+ when(container.getHashCode(any(BytesWritable.class)))
+ .thenAnswer(inv -> hashOf(inv.getArgument(0)));
+ doAnswer(inv -> {
+ long hashCode = inv.getArgument(0);
+ // the loader routes a row to partition (numLoadThreads - 1) &
hashCode
+ if (((NUM_THREADS - 1) & hashCode) == 0) {
+ throw injected;
+ }
+ loadedRows.incrementAndGet();
+ return null;
+ }).when(container).putRow(anyLong(), any(BytesWritable.class),
any(BytesWritable.class));
+ })) {
+
+ VectorMapJoinFastHashTableLoader loader = loaderFor(EST_KEYS, EST_KEYS,
new Rows());
+ thrown = assertThrows(Throwable.class, () -> loader.load(tables, null));
+ tableContainer = construction.constructed().get(0);
+ }
+
+ assertEquals("one of " + NUM_THREADS + " partitions must have been
dropped",
+ ROWS - ROWS / NUM_THREADS, loadedRows.get());
+ assertNull("a hash table missing a partition must never be published",
tables[1]);
+ verify(tableContainer, never()).seal();
+ assertSame("the drain thread's Error must reach the caller", injected,
Throwables.getRootCause(thrown));
+ }
+}