This is an automated email from the ASF dual-hosted git repository.
dkuzmenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 90391c68288 HIVE-28452: Iceberg: Cache delete files on executors
(Denys Kuzmenko, reviewed by Butao Zhang)
90391c68288 is described below
commit 90391c682881494c6753602822aa83d33faf1c8b
Author: Denys Kuzmenko <[email protected]>
AuthorDate: Fri Aug 23 14:13:25 2024 +0200
HIVE-28452: Iceberg: Cache delete files on executors (Denys Kuzmenko,
reviewed by Butao Zhang)
Closes #5397
---
iceberg/checkstyle/checkstyle.xml | 4 +-
.../apache/iceberg/data/CachingDeleteLoader.java | 55 +++++++++++++++++++
.../iceberg/mr/hive/vector/HiveDeleteFilter.java | 13 ++++-
.../mr/hive/vector/HiveVectorizedReader.java | 3 +-
.../mr/mapreduce/AbstractIcebergRecordReader.java | 18 ++-----
.../mr/mapreduce/IcebergMergeRecordReader.java | 10 ++--
.../iceberg/mr/mapreduce/IcebergRecordReader.java | 61 ++++++++++++----------
.../mr/hive/vector/TestHiveVectorizedReader.java | 5 +-
8 files changed, 115 insertions(+), 54 deletions(-)
diff --git a/iceberg/checkstyle/checkstyle.xml
b/iceberg/checkstyle/checkstyle.xml
index 7a595e6cbe1..8f2cb2f9a1b 100644
--- a/iceberg/checkstyle/checkstyle.xml
+++ b/iceberg/checkstyle/checkstyle.xml
@@ -423,7 +423,9 @@
<property name="processJavadoc" value="true"/>
</module>
<module name="UpperEll"/> <!-- Java Style Guide: Numeric Literals -->
- <module name="VisibilityModifier"/> <!-- Java Coding Guidelines:
Minimize mutability -->
+ <module name="VisibilityModifier"> <!-- Java Coding Guidelines:
Minimize mutability -->
+ <property name="protectedAllowed" value="true"/>
+ </module>
<module name="WhitespaceAfter"/> <!-- Java Style Guide: Horizontal
whitespace -->
<module name="WhitespaceAround"> <!-- Java Style Guide: Horizontal
whitespace -->
<property name="allowEmptyConstructors" value="true"/>
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/data/CachingDeleteLoader.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/data/CachingDeleteLoader.java
new file mode 100644
index 00000000000..aeef6b38602
--- /dev/null
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/data/CachingDeleteLoader.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.data;
+
+import java.util.function.Function;
+import java.util.function.Supplier;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.exec.ObjectCache;
+import org.apache.hadoop.hive.ql.exec.ObjectCacheFactory;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.iceberg.DeleteFile;
+import org.apache.iceberg.io.InputFile;
+
+public class CachingDeleteLoader extends BaseDeleteLoader {
+ private final ObjectCache cache;
+
+ public CachingDeleteLoader(Function<DeleteFile, InputFile> loadInputFile,
Configuration conf) {
+ super(loadInputFile);
+
+ String queryId = HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_QUERY_ID);
+ this.cache = ObjectCacheFactory.getCache(conf, queryId, false);
+ }
+
+ @Override
+ protected boolean canCache(long size) {
+ return cache != null;
+ }
+
+ @Override
+ protected <V> V getOrLoad(String key, Supplier<V> valueSupplier, long
valueSize) {
+ try {
+ return cache.retrieve(key, valueSupplier::get);
+ } catch (HiveException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveDeleteFilter.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveDeleteFilter.java
index a87470ae4d9..8ed185e7996 100644
---
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveDeleteFilter.java
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveDeleteFilter.java
@@ -22,12 +22,15 @@ package org.apache.iceberg.mr.hive.vector;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.NoSuchElementException;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.iceberg.FileScanTask;
import org.apache.iceberg.MetadataColumns;
import org.apache.iceberg.Schema;
import org.apache.iceberg.StructLike;
+import org.apache.iceberg.data.CachingDeleteLoader;
import org.apache.iceberg.data.DeleteFilter;
+import org.apache.iceberg.data.DeleteLoader;
import org.apache.iceberg.io.CloseableIterable;
import org.apache.iceberg.io.CloseableIterator;
import org.apache.iceberg.io.FileIO;
@@ -40,11 +43,19 @@ public class HiveDeleteFilter extends DeleteFilter<HiveRow>
{
private final FileIO io;
private final HiveStructLike asStructLike;
+ private final Configuration conf;
- public HiveDeleteFilter(FileIO io, FileScanTask task, Schema tableSchema,
Schema requestedSchema) {
+ public HiveDeleteFilter(FileIO io, FileScanTask task, Schema tableSchema,
Schema requestedSchema,
+ Configuration conf) {
super((task.file()).path().toString(), task.deletes(), tableSchema,
requestedSchema);
this.io = io;
this.asStructLike = new HiveStructLike(this.requiredSchema().asStruct());
+ this.conf = conf;
+ }
+
+ @Override
+ protected DeleteLoader newDeleteLoader() {
+ return new CachingDeleteLoader(this::loadInputFile, conf);
}
@Override
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveVectorizedReader.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveVectorizedReader.java
index c92b4617cbd..dc13857d196 100644
---
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveVectorizedReader.java
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveVectorizedReader.java
@@ -88,7 +88,8 @@ public class HiveVectorizedReader {
Schema requiredSchema = readSchema;
if (!task.deletes().isEmpty()) {
- deleteFilter = new HiveDeleteFilter(table.io(), task, table.schema(),
prepareSchemaForDeleteFilter(readSchema));
+ deleteFilter = new HiveDeleteFilter(table.io(), task, table.schema(),
prepareSchemaForDeleteFilter(readSchema),
+ context.getConfiguration());
requiredSchema = deleteFilter.requiredSchema();
// TODO: take requiredSchema and adjust readColumnIds below accordingly
for equality delete cases
// and remove below limitation
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/AbstractIcebergRecordReader.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/AbstractIcebergRecordReader.java
index 53ee0d870f8..e9a86f40e62 100644
---
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/AbstractIcebergRecordReader.java
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/AbstractIcebergRecordReader.java
@@ -39,9 +39,9 @@ import org.apache.iceberg.mr.hive.IcebergAcidUtil;
public abstract class AbstractIcebergRecordReader<T> extends
RecordReader<Void, T> {
private TaskAttemptContext context;
- private Configuration conf;
- private Table table;
- private Schema expectedSchema;
+ protected Configuration conf;
+ protected Table table;
+ protected Schema expectedSchema;
private String nameMapping;
private boolean reuseContainers;
private boolean caseSensitive;
@@ -106,26 +106,14 @@ public abstract class AbstractIcebergRecordReader<T>
extends RecordReader<Void,
return context;
}
- public Configuration getConf() {
- return conf;
- }
-
public boolean isReuseContainers() {
return reuseContainers;
}
- public Schema getExpectedSchema() {
- return expectedSchema;
- }
-
public String getNameMapping() {
return nameMapping;
}
- public Table getTable() {
- return table;
- }
-
public InputFormatConfig.InMemoryDataModel getInMemoryDataModel() {
return inMemoryDataModel;
}
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergMergeRecordReader.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergMergeRecordReader.java
index 7871e8325d3..17808c77f7c 100644
---
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergMergeRecordReader.java
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergMergeRecordReader.java
@@ -55,11 +55,11 @@ public final class IcebergMergeRecordReader<T> extends
AbstractIcebergRecordRead
}
private CloseableIterator<T> nextTask() {
- CloseableIterator<T> closeableIterator =
openGeneric(mergeSplit.getContentFile(), getTable().schema()).iterator();
+ CloseableIterator<T> closeableIterator =
openGeneric(mergeSplit.getContentFile(), table.schema()).iterator();
if (mergeSplit.getContentFile() instanceof DeleteFile) {
- Schema deleteSchema =
IcebergAcidUtil.createSerdeSchemaForDelete(getTable().schema().columns());
+ Schema deleteSchema =
IcebergAcidUtil.createSerdeSchemaForDelete(table.schema().columns());
return new
IcebergAcidUtil.MergeTaskVirtualColumnAwareIterator<>(closeableIterator,
- deleteSchema, getConf(), mergeSplit.getContentFile(), getTable());
+ deleteSchema, conf, mergeSplit.getContentFile(), table);
} else {
return closeableIterator;
}
@@ -95,8 +95,8 @@ public final class IcebergMergeRecordReader<T> extends
AbstractIcebergRecordRead
} else if (contentFile instanceof DeleteFile) {
schema = new Schema(MetadataColumns.DELETE_FILE_PATH,
MetadataColumns.DELETE_FILE_POS);
}
- InputFile inputFile =
getTable().encryption().decrypt(EncryptedFiles.encryptedInput(
- getTable().io().newInputFile(contentFile.path().toString()),
+ InputFile inputFile =
table.encryption().decrypt(EncryptedFiles.encryptedInput(
+ table.io().newInputFile(contentFile.path().toString()),
contentFile.keyMetadata()));
CloseableIterable<T> iterable;
switch (contentFile.format()) {
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java
index 43efb4fc552..916082fe469 100644
---
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java
@@ -21,7 +21,6 @@ package org.apache.iceberg.mr.mapreduce;
import java.io.IOException;
import java.util.Collections;
-import java.util.Iterator;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
@@ -44,7 +43,9 @@ import org.apache.iceberg.Schema;
import org.apache.iceberg.Table;
import org.apache.iceberg.avro.Avro;
import org.apache.iceberg.common.DynMethods;
+import org.apache.iceberg.data.CachingDeleteLoader;
import org.apache.iceberg.data.DeleteFilter;
+import org.apache.iceberg.data.DeleteLoader;
import org.apache.iceberg.data.GenericDeleteFilter;
import org.apache.iceberg.data.IdentityPartitionConverters;
import org.apache.iceberg.data.avro.DataReader;
@@ -62,6 +63,7 @@ import org.apache.iceberg.mr.hive.IcebergAcidUtil;
import org.apache.iceberg.orc.ORC;
import org.apache.iceberg.parquet.Parquet;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Iterables;
import org.apache.iceberg.types.Type;
import org.apache.iceberg.types.TypeUtil;
import org.apache.iceberg.types.Types;
@@ -89,7 +91,7 @@ public final class IcebergRecordReader<T> extends
AbstractIcebergRecordReader<T>
}
}
- private Iterator<FileScanTask> tasks;
+ private Iterable<FileScanTask> tasks;
private CloseableIterator<T> currentIterator;
private T current;
@@ -98,32 +100,28 @@ public final class IcebergRecordReader<T> extends
AbstractIcebergRecordReader<T>
// For now IcebergInputFormat does its own split planning and does not
accept FileSplit instances
super.initialize(split, newContext);
CombinedScanTask task = ((IcebergSplit) split).task();
- this.tasks = task.files().iterator();
+ this.tasks = task.files();
this.currentIterator = nextTask();
}
private CloseableIterator<T> nextTask() {
- CloseableIterator<T> closeableIterator = open(tasks.next(),
getExpectedSchema()).iterator();
- if (!isFetchVirtualColumns() || Utilities.getIsVectorized(getConf())) {
+ CloseableIterator<T> closeableIterator = CloseableIterable.concat(
+ Iterables.transform(tasks, task -> open(task,
expectedSchema))).iterator();
+ if (!isFetchVirtualColumns() || Utilities.getIsVectorized(conf)) {
return closeableIterator;
}
- return new
IcebergAcidUtil.VirtualColumnAwareIterator<T>(closeableIterator,
getExpectedSchema(),
- getConf(), getTable());
+ return new IcebergAcidUtil.VirtualColumnAwareIterator<>(closeableIterator,
+ expectedSchema, conf, table);
}
@Override
public boolean nextKeyValue() throws IOException {
- while (true) {
- if (currentIterator.hasNext()) {
- current = currentIterator.next();
- return true;
- } else if (tasks.hasNext()) {
- currentIterator.close();
- this.currentIterator = nextTask();
- } else {
- currentIterator.close();
- return false;
- }
+ if (currentIterator.hasNext()) {
+ current = currentIterator.next();
+ return true;
+ } else {
+ currentIterator.close();
+ return false;
}
}
@@ -149,24 +147,24 @@ public final class IcebergRecordReader<T> extends
AbstractIcebergRecordReader<T>
Expression residual = HiveIcebergInputFormat.residualForTask(task,
getContext().getConfiguration());
// TODO: We have to take care of the EncryptionManager when LLAP and
vectorization is used
- CloseableIterable<T> iterator =
HIVE_VECTORIZED_READER_BUILDER.invoke(getTable(), path, task,
+ CloseableIterable<T> iterator =
HIVE_VECTORIZED_READER_BUILDER.invoke(table, path, task,
idToConstant, getContext(), residual, readSchema);
return applyResidualFiltering(iterator, residual, readSchema);
}
- private CloseableIterable<T> openGeneric(FileScanTask task, Schema
readSchema) {
+ private CloseableIterable openGeneric(FileScanTask task, Schema readSchema) {
if (task.isDataTask()) {
// When querying metadata tables, the currentTask is a DataTask and the
data has to
// be fetched from the task instead of reading it from files.
IcebergInternalRecordWrapper wrapper =
- new IcebergInternalRecordWrapper(getTable().schema().asStruct(),
readSchema.asStruct());
- return (CloseableIterable) CloseableIterable.transform(((DataTask)
task).rows(), row -> wrapper.wrap(row));
+ new IcebergInternalRecordWrapper(table.schema().asStruct(),
readSchema.asStruct());
+ return CloseableIterable.transform(((DataTask) task).rows(),
wrapper::wrap);
}
DataFile file = task.file();
- InputFile inputFile =
getTable().encryption().decrypt(EncryptedFiles.encryptedInput(
- getTable().io().newInputFile(file.path().toString()),
+ InputFile inputFile =
table.encryption().decrypt(EncryptedFiles.encryptedInput(
+ table.io().newInputFile(file.path().toString()),
file.keyMetadata()));
CloseableIterable<T> iterable;
@@ -197,7 +195,12 @@ public final class IcebergRecordReader<T> extends
AbstractIcebergRecordReader<T>
case HIVE:
return openVectorized(currentTask, readSchema);
case GENERIC:
- DeleteFilter deletes = new GenericDeleteFilter(getTable().io(),
currentTask, getTable().schema(), readSchema);
+ DeleteFilter deletes = new GenericDeleteFilter(table.io(),
currentTask, table.schema(), readSchema) {
+ @Override
+ protected DeleteLoader newDeleteLoader() {
+ return new CachingDeleteLoader(this::loadInputFile, conf);
+ }
+ };
Schema requiredSchema = deletes.requiredSchema();
return deletes.filter(openGeneric(currentTask, requiredSchema));
default:
@@ -277,13 +280,13 @@ public final class IcebergRecordReader<T> extends
AbstractIcebergRecordReader<T>
private Map<Integer, ?> constantsMap(FileScanTask task, BiFunction<Type,
Object, Object> converter) {
PartitionSpec spec = task.spec();
Set<Integer> idColumns = spec.identitySourceIds();
- Schema partitionSchema = TypeUtil.select(getExpectedSchema(), idColumns);
+ Schema partitionSchema = TypeUtil.select(expectedSchema, idColumns);
boolean projectsIdentityPartitionColumns =
!partitionSchema.columns().isEmpty();
- if (getExpectedSchema().findField(MetadataColumns.PARTITION_COLUMN_ID) !=
null) {
- Types.StructType partitionType = Partitioning.partitionType(getTable());
+ if (expectedSchema.findField(MetadataColumns.PARTITION_COLUMN_ID) != null)
{
+ Types.StructType partitionType = Partitioning.partitionType(table);
return PartitionUtil.constantsMap(task, partitionType, converter);
} else if (projectsIdentityPartitionColumns) {
- Types.StructType partitionType = Partitioning.partitionType(getTable());
+ Types.StructType partitionType = Partitioning.partitionType(table);
return PartitionUtil.constantsMap(task, partitionType, converter);
} else {
return Collections.emptyMap();
diff --git
a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/vector/TestHiveVectorizedReader.java
b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/vector/TestHiveVectorizedReader.java
index fae2e207ba2..0209c8237a4 100644
---
a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/vector/TestHiveVectorizedReader.java
+++
b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/vector/TestHiveVectorizedReader.java
@@ -98,8 +98,9 @@ public class TestHiveVectorizedReader {
try (MockedStatic<ParquetFileReader> mockedParquetFileReader =
Mockito.mockStatic(ParquetFileReader.class,
Mockito.CALLS_REAL_METHODS)) {
for (InputSplit split : splits) {
- try (RecordReader<Void, ?> reader =
inputFormat.createRecordReader(split, context)) {
- reader.initialize(split, context);
+ try (RecordReader<Void, ?> r = inputFormat.createRecordReader(split,
context)) {
+ r.initialize(split, context);
+ r.nextKeyValue();
}
}
mockedParquetFileReader.verify(times(1), () ->