This is an automated email from the ASF dual-hosted git repository.
deniskuzZ pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new abbab5499c4 HIVE-30044: LLAP delete delta tail is read from the
filesystem even on a cache hit (#6778)
abbab5499c4 is described below
commit abbab5499c45236297d0822a99e066983a34763e
Author: Denys Kuzmenko <[email protected]>
AuthorDate: Fri Sep 18 21:47:38 2026 +0200
HIVE-30044: LLAP delete delta tail is read from the filesystem even on a
cache hit (#6778)
---
.../ql/io/orc/VectorizedOrcAcidRowBatchReader.java | 8 +-
...stVectorizedOrcAcidRowBatchReaderTailCache.java | 98 ++++++++++++++++++++++
2 files changed, 104 insertions(+), 2 deletions(-)
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java
b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java
index c4a92f624e2..8490575c961 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java
@@ -706,7 +706,8 @@ private OrcRawRecordMerger.KeyInterval
findOriginalMinMaxKeys(OrcSplit orcSplit,
return keyIntervalTmp;
}
- private static class ReaderData implements Closeable {
+ @VisibleForTesting
+ static class ReaderData implements Closeable {
OrcTail orcTail;
Reader reader;
@@ -728,12 +729,15 @@ public void close() throws IOException {
* @return ReaderData object where the orcTail is not null. Reader can be
null, but if we had to create
* one we return that as well for further reuse.
*/
- private static ReaderData getOrcReaderData(Path path, Configuration conf,
CacheTag cacheTag, Object fileKey) throws IOException {
+ @VisibleForTesting
+ static ReaderData getOrcReaderData(Path path, Configuration conf, CacheTag
cacheTag, Object fileKey)
+ throws IOException {
ReaderData readerData = new ReaderData();
if (shouldReadDeleteDeltasWithLlap(conf, true)) {
try {
readerData.orcTail = LlapProxy.getIo().getOrcTailFromCache(path, conf,
cacheTag, fileKey);
readerData.reader = OrcFile.createReader(path,
OrcFile.readerOptions(conf).orcTail(readerData.orcTail));
+ return readerData;
} catch (IllegalCacheConfigurationException icce) {
throw new IOException("LLAP cache is not configured properly while
delete delta caching is turned on", icce);
}
diff --git
a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorizedOrcAcidRowBatchReaderTailCache.java
b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorizedOrcAcidRowBatchReaderTailCache.java
new file mode 100644
index 00000000000..2ed5ea0a8a0
--- /dev/null
+++
b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorizedOrcAcidRowBatchReaderTailCache.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.io.orc;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertSame;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.mockStatic;
+import static org.mockito.Mockito.when;
+
+import java.io.File;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.llap.io.api.LlapIo;
+import org.apache.hadoop.hive.llap.io.api.LlapProxy;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import
org.apache.hadoop.hive.ql.io.orc.VectorizedOrcAcidRowBatchReader.ReaderData;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.orc.OrcFile;
+import org.apache.orc.Reader;
+import org.apache.orc.TypeDescription;
+import org.apache.orc.Writer;
+import org.apache.orc.impl.OrcTail;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.mockito.MockedStatic;
+
+/**
+ * The delete delta tail served from the LLAP metadata cache has to be the one
the reader uses:
+ * re-reading it from the filesystem makes
hive.llap.io.cache.deletedeltas=metadata do nothing and
+ * costs a footer read per split at the 'all' level.
+ */
+public class TestVectorizedOrcAcidRowBatchReaderTailCache {
+
+ @Rule
+ public TemporaryFolder temp = new TemporaryFolder();
+
+ private Path path;
+ private JobConf conf;
+
+ @Before
+ public void writeOrcFile() throws Exception {
+ File file = new File(temp.getRoot(), "delete_delta_file.orc");
+ path = new Path(file.toURI());
+ conf = new JobConf();
+ TypeDescription schema = TypeDescription.fromString("struct<id:int>");
+ try (Writer writer = OrcFile.createWriter(path,
OrcFile.writerOptions(conf).setSchema(schema))) {
+ VectorizedRowBatch batch = schema.createRowBatch();
+ ((LongColumnVector) batch.cols[0]).vector[0] = 1;
+ batch.size = 1;
+ writer.addRowBatch(batch);
+ }
+ HiveConf.setVar(conf, ConfVars.HIVE_EXECUTION_MODE, "llap");
+ HiveConf.setVar(conf, ConfVars.LLAP_IO_CACHE_DELETEDELTAS, "metadata");
+ }
+
+ @Test
+ public void testCachedTailIsTheOneReturned() throws Exception {
+ OrcTail cachedTail;
+ try (Reader reader = OrcFile.createReader(path,
OrcFile.readerOptions(conf))) {
+ cachedTail = new OrcTail(reader.getFileTail(),
reader.getSerializedFileFooter());
+ }
+ LlapIo<?> llapIo = mock(LlapIo.class);
+ when(llapIo.getOrcTailFromCache(any(), any(), any(),
any())).thenReturn(cachedTail);
+
+ try (MockedStatic<LlapProxy> proxy = mockStatic(LlapProxy.class)) {
+ proxy.when(LlapProxy::isDaemon).thenReturn(true);
+ proxy.when(LlapProxy::getIo).thenReturn(llapIo);
+
+ try (ReaderData readerData =
+ VectorizedOrcAcidRowBatchReader.getOrcReaderData(path, conf, null,
null)) {
+ assertSame("the cached tail must not be replaced by a filesystem
read", cachedTail, readerData.orcTail);
+ assertNotNull(readerData.reader);
+ }
+ }
+ }
+}
