This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 38e2e12556fa feat: switch default log format version to v2 (#19091)
38e2e12556fa is described below
commit 38e2e12556faf39f9eeff32d6ac090672aa209f0
Author: Danny Chan <[email protected]>
AuthorDate: Tue Jun 30 10:01:13 2026 +0800
feat: switch default log format version to v2 (#19091)
---
.../hudi/io/HoodieNativeLogFormatWriter.java | 3 +--
.../hudi/common/table/log/HoodieLogFormat.java | 7 ++++++-
.../table/log/TestHoodieLogFormatVersion.java | 22 +++++++++++++++++++---
.../common/table/log/HoodieLogFormatWriter.java | 7 +++----
.../common/functional/TestHoodieLogFormat.java | 20 ++++++++++----------
5 files changed, 39 insertions(+), 20 deletions(-)
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieNativeLogFormatWriter.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieNativeLogFormatWriter.java
index 1b35f4fbe51d..8c7867ec7ea8 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieNativeLogFormatWriter.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieNativeLogFormatWriter.java
@@ -61,7 +61,6 @@ import static
org.apache.hudi.common.model.LogExtensions.DELETE_LOG_EXTENSION;
*/
public class HoodieNativeLogFormatWriter extends HoodieLogFormat.Writer {
- private static final int NATIVE_LOG_FORMAT_VERSION = 2;
private static final String LOG_FORMAT_METADATA_FOOTER_KEY =
"hudi.log.format.metadata";
private final HoodieWriteConfig writeConfig;
@@ -222,7 +221,7 @@ public class HoodieNativeLogFormatWriter extends
HoodieLogFormat.Writer {
private Map<String, String> getFooterMetadata(Map<HeaderMetadataType,
String> header) throws IOException {
Map<String, String> logFormatMetadata = new LinkedHashMap<>();
- logFormatMetadata.put(HeaderMetadataType.VERSION.name(),
String.valueOf(NATIVE_LOG_FORMAT_VERSION));
+ logFormatMetadata.put(HeaderMetadataType.VERSION.name(),
String.valueOf(HoodieLogFormat.CURRENT_VERSION));
header.forEach((key, value) -> {
if (value != null) {
logFormatMetadata.put(key.name(), value);
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormat.java
b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormat.java
index 3877a682240d..45ff2624dc81 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormat.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormat.java
@@ -56,7 +56,12 @@ public interface HoodieLogFormat {
* The current version of the log format. Anytime the log format changes
this version needs to be bumped and
* corresponding changes need to be made to {@link HoodieLogFormatVersion}
*/
- int CURRENT_VERSION = 1;
+ int CURRENT_VERSION = 2;
+
+ /**
+ * The current version of the inline log format.
+ */
+ int INLINE_LOG_FORMAT_VERSION = 1;
String UNKNOWN_WRITE_TOKEN = "1-0-1";
diff --git
a/hudi-common/src/test/java/org/apache/hudi/common/table/log/TestHoodieLogFormatVersion.java
b/hudi-common/src/test/java/org/apache/hudi/common/table/log/TestHoodieLogFormatVersion.java
index 7665e694e6b7..b3b5379295da 100755
---
a/hudi-common/src/test/java/org/apache/hudi/common/table/log/TestHoodieLogFormatVersion.java
+++
b/hudi-common/src/test/java/org/apache/hudi/common/table/log/TestHoodieLogFormatVersion.java
@@ -20,6 +20,7 @@ package org.apache.hudi.common.table.log;
import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -29,43 +30,57 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
public class TestHoodieLogFormatVersion {
private static HoodieLogFormatVersion verDefault =
new HoodieLogFormatVersion(HoodieLogFormatVersion.DEFAULT_VERSION);
- private static HoodieLogFormatVersion verCurrent =
+ private static HoodieLogFormatVersion verInline =
+ new HoodieLogFormatVersion(HoodieLogFormat.INLINE_LOG_FORMAT_VERSION);
+ private static HoodieLogFormatVersion verCurrent =
new HoodieLogFormatVersion(HoodieLogFormat.CURRENT_VERSION);
+ @Test
+ public void testCurrentVersionValues() {
+ assertEquals(1, HoodieLogFormat.INLINE_LOG_FORMAT_VERSION);
+ assertEquals(2, HoodieLogFormat.CURRENT_VERSION);
+ }
+
@Test
public void testHasMagicHeader() {
assertTrue(verDefault.hasMagicHeader());
+ assertTrue(verInline.hasMagicHeader());
assertTrue(verCurrent.hasMagicHeader());
}
@Test
public void testHasContent() {
assertTrue(verDefault.hasContent());
+ assertTrue(verInline.hasContent());
assertTrue(verCurrent.hasContent());
}
@Test
public void testHasContentLength() {
assertTrue(verDefault.hasContentLength());
+ assertTrue(verInline.hasContentLength());
assertTrue(verCurrent.hasContentLength());
}
@Test
public void testHasOrdinal() {
assertTrue(verDefault.hasOrdinal());
+ assertTrue(verInline.hasOrdinal());
assertTrue(verCurrent.hasOrdinal());
}
@Test
public void testHasHeader() {
assertFalse(verDefault.hasHeader());
+ assertTrue(verInline.hasHeader());
assertTrue(verCurrent.hasHeader());
}
@Test
public void testHasFooter() {
assertFalse(verDefault.hasFooter());
- assertTrue(verCurrent.hasFooter());
+ assertTrue(verInline.hasFooter());
+ assertFalse(verCurrent.hasFooter());
HoodieLogFormatVersion verNew =
new HoodieLogFormatVersion(HoodieLogFormat.CURRENT_VERSION + 1);
@@ -75,7 +90,8 @@ public class TestHoodieLogFormatVersion {
@Test
public void testHasLogBlockLength() {
assertFalse(verDefault.hasLogBlockLength());
- assertTrue(verCurrent.hasLogBlockLength());
+ assertTrue(verInline.hasLogBlockLength());
+ assertFalse(verCurrent.hasLogBlockLength());
HoodieLogFormatVersion verNew =
new HoodieLogFormatVersion(HoodieLogFormat.CURRENT_VERSION + 1);
diff --git
a/hudi-hadoop-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatWriter.java
b/hudi-hadoop-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatWriter.java
index fe24bd600f1e..be30dfc015e6 100644
---
a/hudi-hadoop-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatWriter.java
+++
b/hudi-hadoop-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatWriter.java
@@ -127,9 +127,8 @@ public class HoodieLogFormatWriter extends
HoodieLogFormat.Writer {
@Override
public AppendResult appendBlocks(List<HoodieLogBlock> blocks) throws
IOException {
try {
- // Find current version
- HoodieLogFormat.LogFormatVersion currentLogFormatVersion =
- new HoodieLogFormatVersion(HoodieLogFormat.CURRENT_VERSION);
+ HoodieLogFormat.LogFormatVersion inlineLogFormatVersion =
+ new
HoodieLogFormatVersion(HoodieLogFormat.INLINE_LOG_FORMAT_VERSION);
FSDataOutputStream originalOutputStream = getOutputStream();
long startPos = originalOutputStream.getPos();
@@ -153,7 +152,7 @@ public class HoodieLogFormatWriter extends
HoodieLogFormat.Writer {
outputStream.writeLong(getLogBlockLength(content.size(),
headerBytes.length, footerBytes.length));
// 3. Write the version of this log block
- outputStream.writeInt(currentLogFormatVersion.getVersion());
+ outputStream.writeInt(inlineLogFormatVersion.getVersion());
// 4. Write the block type
outputStream.writeInt(block.getBlockType().ordinal());
diff --git
a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormat.java
b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormat.java
index e63a27149d3c..65f1da8a48bc 100755
---
a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormat.java
+++
b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormat.java
@@ -985,7 +985,7 @@ public class TestHoodieLogFormat extends
HoodieCommonTestHarness {
// Write out a length that does not confirm with the content
outputStream.writeLong(474);
outputStream.writeInt(HoodieLogBlockType.AVRO_DATA_BLOCK.ordinal());
- outputStream.writeInt(HoodieLogFormat.CURRENT_VERSION);
+ outputStream.writeInt(HoodieLogFormat.INLINE_LOG_FORMAT_VERSION);
// Write out a length that does not confirm with the content
outputStream.writeLong(400);
// Write out incomplete content
@@ -1016,7 +1016,7 @@ public class TestHoodieLogFormat extends
HoodieCommonTestHarness {
// Write out a length that does not confirm with the content
outputStream.writeLong(1000);
outputStream.writeInt(HoodieLogBlockType.AVRO_DATA_BLOCK.ordinal());
- outputStream.writeInt(HoodieLogFormat.CURRENT_VERSION);
+ outputStream.writeInt(HoodieLogFormat.INLINE_LOG_FORMAT_VERSION);
// Write out a length that does not confirm with the content
outputStream.writeLong(500);
// Write out some bytes
@@ -1162,7 +1162,7 @@ public class TestHoodieLogFormat extends
HoodieCommonTestHarness {
// Write out a length that does not confirm with the content
outputStream.writeLong(474);
outputStream.writeInt(HoodieLogBlockType.AVRO_DATA_BLOCK.ordinal());
- outputStream.writeInt(HoodieLogFormat.CURRENT_VERSION);
+ outputStream.writeInt(HoodieLogFormat.INLINE_LOG_FORMAT_VERSION);
// Write out a length that does not confirm with the content
outputStream.writeLong(400);
// Write out incomplete content
@@ -1341,7 +1341,7 @@ public class TestHoodieLogFormat extends
HoodieCommonTestHarness {
// Write out a length that does not confirm with the content
outputStream.writeLong(1000);
- outputStream.writeInt(HoodieLogFormat.CURRENT_VERSION);
+ outputStream.writeInt(HoodieLogFormat.INLINE_LOG_FORMAT_VERSION);
outputStream.writeInt(HoodieLogBlockType.AVRO_DATA_BLOCK.ordinal());
// Write out some header
@@ -2220,7 +2220,7 @@ public class TestHoodieLogFormat extends
HoodieCommonTestHarness {
outputStream.write(HoodieLogFormat.MAGIC);
outputStream.writeLong(1000);
outputStream.writeInt(HoodieLogBlockType.AVRO_DATA_BLOCK.ordinal());
- outputStream.writeInt(HoodieLogFormat.CURRENT_VERSION);
+ outputStream.writeInt(HoodieLogFormat.INLINE_LOG_FORMAT_VERSION);
// Write out a length that does not confirm with the content
outputStream.writeLong(100);
outputStream.flush();
@@ -2232,7 +2232,7 @@ public class TestHoodieLogFormat extends
HoodieCommonTestHarness {
outputStream.write(HoodieLogFormat.MAGIC);
outputStream.writeLong(1000);
outputStream.writeInt(HoodieLogBlockType.AVRO_DATA_BLOCK.ordinal());
- outputStream.writeInt(HoodieLogFormat.CURRENT_VERSION);
+ outputStream.writeInt(HoodieLogFormat.INLINE_LOG_FORMAT_VERSION);
// Write out a length that does not confirm with the content
outputStream.writeLong(100);
outputStream.flush();
@@ -2256,7 +2256,7 @@ public class TestHoodieLogFormat extends
HoodieCommonTestHarness {
outputStream.write(HoodieLogFormat.MAGIC);
outputStream.writeLong(1000);
outputStream.writeInt(HoodieLogBlockType.AVRO_DATA_BLOCK.ordinal());
- outputStream.writeInt(HoodieLogFormat.CURRENT_VERSION);
+ outputStream.writeInt(HoodieLogFormat.INLINE_LOG_FORMAT_VERSION);
// Write out a length that does not confirm with the content
outputStream.writeLong(100);
outputStream.flush();
@@ -2353,7 +2353,7 @@ public class TestHoodieLogFormat extends
HoodieCommonTestHarness {
outputStream.write(HoodieLogFormat.MAGIC);
outputStream.writeLong(1000);
outputStream.writeInt(HoodieLogBlockType.AVRO_DATA_BLOCK.ordinal());
- outputStream.writeInt(HoodieLogFormat.CURRENT_VERSION);
+ outputStream.writeInt(HoodieLogFormat.INLINE_LOG_FORMAT_VERSION);
// Write out a length that does not confirm with the content
outputStream.writeLong(100);
outputStream.flush();
@@ -2365,7 +2365,7 @@ public class TestHoodieLogFormat extends
HoodieCommonTestHarness {
outputStream.write(HoodieLogFormat.MAGIC);
outputStream.writeLong(1000);
outputStream.writeInt(HoodieLogBlockType.AVRO_DATA_BLOCK.ordinal());
- outputStream.writeInt(HoodieLogFormat.CURRENT_VERSION);
+ outputStream.writeInt(HoodieLogFormat.INLINE_LOG_FORMAT_VERSION);
// Write out a length that does not confirm with the content
outputStream.writeLong(100);
outputStream.flush();
@@ -3007,7 +3007,7 @@ public class TestHoodieLogFormat extends
HoodieCommonTestHarness {
outputStream.write(HoodieLogFormat.MAGIC);
// Write out a length that does not confirm with the content
outputStream.writeLong(473);
- outputStream.writeInt(HoodieLogFormat.CURRENT_VERSION);
+ outputStream.writeInt(HoodieLogFormat.INLINE_LOG_FORMAT_VERSION);
outputStream.writeInt(10000); // an invalid block type
// Write out a length that does not confirm with the content
