This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 2de099c97269 feat: Support read and write of HFile footer metadata
(#19129)
2de099c97269 is described below
commit 2de099c972695ad4d15d08f7478504eaa5edd4ff
Author: Shuo Cheng <[email protected]>
AuthorDate: Wed Jul 1 13:53:54 2026 +0800
feat: Support read and write of HFile footer metadata (#19129)
* feat: Support read and write of HFile footer metadata
---
.../org/apache/hudi/common/util/HFileUtils.java | 26 +++++++++++++++++++++-
.../io/storage/hadoop/HoodieAvroHFileWriter.java | 6 +++++
.../io/hadoop/TestHoodieHFileReaderWriter.java | 24 ++++++++++++++++++++
3 files changed, 55 insertions(+), 1 deletion(-)
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/util/HFileUtils.java
b/hudi-common/src/main/java/org/apache/hudi/common/util/HFileUtils.java
index 864bd76e2b14..be74096edd63 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/util/HFileUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/util/HFileUtils.java
@@ -34,10 +34,14 @@ import
org.apache.hudi.common.util.collection.ClosableIterator;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIOException;
+import org.apache.hudi.exception.MetadataNotFoundException;
import org.apache.hudi.io.compress.CompressionCodec;
import org.apache.hudi.io.hfile.HFileContext;
+import org.apache.hudi.io.hfile.HFileReader;
import org.apache.hudi.io.hfile.HFileWriter;
import org.apache.hudi.io.hfile.HFileWriterImpl;
+import org.apache.hudi.io.hfile.UTF8StringKey;
+import org.apache.hudi.io.storage.HFileReaderFactory;
import org.apache.hudi.io.storage.HoodieAvroHFileReaderImplBase;
import org.apache.hudi.io.storage.HoodieFileReader;
import org.apache.hudi.io.storage.HoodieIOFactory;
@@ -54,6 +58,7 @@ import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.OutputStream;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -66,6 +71,7 @@ import static
org.apache.hudi.common.config.HoodieStorageConfig.BLOOM_FILTER_NUM
import static
org.apache.hudi.common.config.HoodieStorageConfig.BLOOM_FILTER_TYPE;
import static
org.apache.hudi.common.config.HoodieStorageConfig.HFILE_COMPRESSION_ALGORITHM_NAME;
import static
org.apache.hudi.common.config.HoodieStorageConfig.HFILE_WITH_BLOOM_FILTER_ENABLED;
+import static org.apache.hudi.common.util.StringUtils.fromUTF8Bytes;
import static org.apache.hudi.common.util.StringUtils.getUTF8Bytes;
/**
@@ -101,7 +107,25 @@ public class HFileUtils extends FileFormatUtils {
@Override
public Map<String, String> readFooter(HoodieStorage storage, boolean
required, StoragePath filePath, String... footerNames) {
- throw new UnsupportedOperationException("HFileUtils does not support
readFooter");
+ Map<String, String> footerVals = new HashMap<>();
+ try (HFileReader reader = HFileReaderFactory.builder()
+ .withStorage(storage)
+ .withPath(filePath)
+ .build()
+ .createHFileReader()) {
+ for (String footerName : footerNames) {
+ Option<byte[]> footerValue = reader.getMetaInfo(new
UTF8StringKey(footerName));
+ if (footerValue.isPresent()) {
+ footerVals.put(footerName, fromUTF8Bytes(footerValue.get()));
+ } else if (required) {
+ throw new MetadataNotFoundException(
+ "Could not find metadata key in HFile footer. Key " + footerName
+ ", file: " + filePath);
+ }
+ }
+ return footerVals;
+ } catch (IOException io) {
+ throw new HoodieIOException("Unable to read footer for HFile: " +
filePath, io);
+ }
}
@Override
diff --git
a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroHFileWriter.java
b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroHFileWriter.java
index 15f088cbd8aa..a74aa5aa571c 100644
---
a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroHFileWriter.java
+++
b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroHFileWriter.java
@@ -49,6 +49,7 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.OutputStream;
+import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
import static org.apache.hudi.common.util.StringUtils.EMPTY_STRING;
@@ -128,6 +129,11 @@ public class HoodieAvroHFileWriter
return !isWrapperFileSystem || wrapperFs.get().getBytesWritten(file) <
maxFileSize;
}
+ @Override
+ public void addFooterMetadata(Map<String, String> footerMetadata) {
+ footerMetadata.forEach((key, value) -> writer.appendFileInfo(key,
getUTF8Bytes(value)));
+ }
+
@Override
public void writeAvro(String recordKey, IndexedRecord record) throws
IOException {
if (!this.hfileConfig.isAllowDuplicatesOnHfileWrites()) {
diff --git
a/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHFileReaderWriter.java
b/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHFileReaderWriter.java
index f2225ff3ca66..9029085df4db 100644
---
a/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHFileReaderWriter.java
+++
b/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHFileReaderWriter.java
@@ -31,9 +31,12 @@ import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.schema.HoodieSchema;
import org.apache.hudi.common.schema.HoodieSchemaType;
import org.apache.hudi.common.table.HoodieTableConfig;
+import org.apache.hudi.common.table.log.NativeLogFooterMetadata;
import org.apache.hudi.common.testutils.HoodieTestUtils;
+import org.apache.hudi.common.util.HFileUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.ClosableIterator;
+import org.apache.hudi.exception.MetadataNotFoundException;
import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.io.hfile.HFileReader;
import org.apache.hudi.io.hfile.UTF8StringKey;
@@ -201,6 +204,27 @@ public class TestHoodieHFileReaderWriter extends
TestHoodieReaderWriterBase {
}
}
+ @Test
+ public void testReadFooterMetadata() throws Exception {
+ HoodieSchema schema =
getSchemaFromResource(TestHoodieOrcReaderWriter.class,
"/exampleSchemaWithMetaFields.avsc");
+ HoodieAvroHFileWriter writer = createWriter(schema, false);
+ Map<String, String> footerMetadata = new TreeMap<>();
+ footerMetadata.put(NativeLogFooterMetadata.FOOTER_METADATA_KEY,
"{\"SCHEMA\":\"schema\"}");
+ footerMetadata.put("custom", "value");
+ writer.addFooterMetadata(footerMetadata);
+ writer.close();
+
+ HoodieStorage storage = HoodieTestUtils.getStorage(getFilePath());
+ Map<String, String> footer = new HFileUtils().readFooter(
+ storage, false, getFilePath(),
NativeLogFooterMetadata.FOOTER_METADATA_KEY, "custom", "missing");
+
+ assertEquals(2, footer.size());
+ assertEquals("{\"SCHEMA\":\"schema\"}",
footer.get(NativeLogFooterMetadata.FOOTER_METADATA_KEY));
+ assertEquals("value", footer.get("custom"));
+ assertThrows(MetadataNotFoundException.class,
+ () -> new HFileUtils().readFooter(storage, true, getFilePath(),
"missing"));
+ }
+
@ParameterizedTest
@MethodSource("populateMetaFieldsAndTestAvroWithMeta")
public void testWriteReadHFileWithMetaFields(boolean populateMetaFields,
boolean testAvroWithMeta) throws Exception {
