This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new a33b2a5e03f [HUDI-7834] Create placeholder table versions and
introduce new hoodie table property to track initial table version (#11406)
a33b2a5e03f is described below
commit a33b2a5e03f434e3ce270a128be626ae9e9e78c9
Author: Balaji Varadarajan <vbal...@apache.org>
AuthorDate: Fri Jun 7 18:43:33 2024 -0700
[HUDI-7834] Create placeholder table versions and introduce new hoodie
table property to track initial table version (#11406)
Co-authored-by: Balaji Varadarajan <bal...@applied.co>
Co-authored-by: Y Ethan Guo <ethan.guoyi...@gmail.com>
---
.../apache/hudi/cli/commands/RepairsCommand.java | 4 +++
.../upgrade/EightToSevenDowngradeHandler.java | 37 +++++++++++++++++++
.../table/upgrade/SevenToEightUpgradeHandler.java | 38 ++++++++++++++++++++
.../table/upgrade/SevenToSixDowngradeHandler.java | 40 +++++++++++++++++++++
.../table/upgrade/SixToSevenUpgradeHandler.java | 42 ++++++++++++++++++++++
.../hudi/table/upgrade/UpgradeDowngrade.java | 8 +++++
.../hudi/common/table/HoodieTableConfig.java | 16 +++++++++
.../hudi/common/table/HoodieTableMetaClient.java | 3 ++
.../hudi/common/table/HoodieTableVersion.java | 10 ++++--
.../common/table/TestHoodieTableMetaClient.java | 1 +
.../RepairOverwriteHoodiePropsProcedure.scala | 5 ++-
.../sql/hudi/procedure/TestRepairsProcedure.scala | 1 +
.../TestUpgradeOrDowngradeProcedure.scala | 4 +--
13 files changed, 203 insertions(+), 6 deletions(-)
diff --git
a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java
b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java
index 57ec8ccf57b..569136e0b50 100644
--- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java
@@ -161,6 +161,10 @@ public class RepairsCommand {
newProps.load(fileInputStream);
}
Map<String, String> oldProps = client.getTableConfig().propsMap();
+ // Copy Initial Version from old-props to new-props
+ if (oldProps.containsKey(HoodieTableConfig.INITIAL_VERSION.key())) {
+ newProps.put(HoodieTableConfig.INITIAL_VERSION.key(),
oldProps.get(HoodieTableConfig.INITIAL_VERSION.key()));
+ }
HoodieTableConfig.create(client.getStorage(), client.getMetaPath(),
newProps);
// reload new props as checksum would have been added
newProps =
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/EightToSevenDowngradeHandler.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/EightToSevenDowngradeHandler.java
new file mode 100644
index 00000000000..3bb22481681
--- /dev/null
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/EightToSevenDowngradeHandler.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.upgrade;
+
+import org.apache.hudi.common.config.ConfigProperty;
+import org.apache.hudi.common.engine.HoodieEngineContext;
+import org.apache.hudi.config.HoodieWriteConfig;
+
+import java.util.Collections;
+import java.util.Map;
+
+/**
+ * Version 7 is going to be placeholder version for bridge release 0.16.0.
+ * Version 8 is the placeholder version to track 1.x.
+ */
+public class EightToSevenDowngradeHandler implements DowngradeHandler {
+ @Override
+ public Map<ConfigProperty, String> downgrade(HoodieWriteConfig config,
HoodieEngineContext context, String instantTime, SupportsUpgradeDowngrade
upgradeDowngradeHelper) {
+ return Collections.emptyMap();
+ }
+}
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/SevenToEightUpgradeHandler.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/SevenToEightUpgradeHandler.java
new file mode 100644
index 00000000000..9ed4f192786
--- /dev/null
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/SevenToEightUpgradeHandler.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.upgrade;
+
+import org.apache.hudi.common.config.ConfigProperty;
+import org.apache.hudi.common.engine.HoodieEngineContext;
+import org.apache.hudi.config.HoodieWriteConfig;
+
+import java.util.Collections;
+import java.util.Map;
+
+/**
+ * Version 7 is going to be placeholder version for bridge release 0.16.0.
+ * Version 8 is the placeholder version to track 1.x.
+ */
+public class SevenToEightUpgradeHandler implements UpgradeHandler {
+ @Override
+ public Map<ConfigProperty, String> upgrade(HoodieWriteConfig config,
HoodieEngineContext context,
+ String instantTime,
SupportsUpgradeDowngrade upgradeDowngradeHelper) {
+ return Collections.emptyMap();
+ }
+}
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/SevenToSixDowngradeHandler.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/SevenToSixDowngradeHandler.java
new file mode 100644
index 00000000000..5cf9b42bab2
--- /dev/null
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/SevenToSixDowngradeHandler.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.upgrade;
+
+import org.apache.hudi.common.config.ConfigProperty;
+import org.apache.hudi.common.engine.HoodieEngineContext;
+import org.apache.hudi.config.HoodieWriteConfig;
+
+import java.util.Collections;
+import java.util.Map;
+
+/**
+ * There is no format change between table version 6 and 7.
+ * Table version 7 is meant for required bridge release upgrade before
upgrading to 1.0.
+ * Version 7 is going to be placeholder version for bridge release 0.16.0.
+ * Version 8 is the placeholder version to track 1.x.
+ */
+public class SevenToSixDowngradeHandler implements DowngradeHandler {
+
+ @Override
+ public Map<ConfigProperty, String> downgrade(HoodieWriteConfig config,
HoodieEngineContext context, String instantTime, SupportsUpgradeDowngrade
upgradeDowngradeHelper) {
+ return Collections.emptyMap();
+ }
+}
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/SixToSevenUpgradeHandler.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/SixToSevenUpgradeHandler.java
new file mode 100644
index 00000000000..eb3c97bfcef
--- /dev/null
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/SixToSevenUpgradeHandler.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.upgrade;
+
+import org.apache.hudi.common.config.ConfigProperty;
+import org.apache.hudi.common.engine.HoodieEngineContext;
+import org.apache.hudi.config.HoodieWriteConfig;
+
+import java.util.Collections;
+import java.util.Map;
+
+/**
+ * There is no format change between table version 6 and 7.
+ * Table version 7 is meant for required bridge release upgrade before
upgrading to 1.0.
+ * Version 7 is going to be placeholder version for bridge release 0.16.0.
+ * Version 8 is the placeholder version to track 1.x.
+ */
+public class SixToSevenUpgradeHandler implements UpgradeHandler {
+
+ @Override
+ public Map<ConfigProperty, String> upgrade(HoodieWriteConfig config,
HoodieEngineContext context,
+ String instantTime,
+ SupportsUpgradeDowngrade
upgradeDowngradeHelper) {
+ return Collections.emptyMap();
+ }
+}
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/UpgradeDowngrade.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/UpgradeDowngrade.java
index b5177a5746b..c089f671481 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/UpgradeDowngrade.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/UpgradeDowngrade.java
@@ -174,6 +174,10 @@ public class UpgradeDowngrade {
return new FourToFiveUpgradeHandler().upgrade(config, context,
instantTime, upgradeDowngradeHelper);
} else if (fromVersion == HoodieTableVersion.FIVE && toVersion ==
HoodieTableVersion.SIX) {
return new FiveToSixUpgradeHandler().upgrade(config, context,
instantTime, upgradeDowngradeHelper);
+ } else if (fromVersion == HoodieTableVersion.SIX && toVersion ==
HoodieTableVersion.SEVEN) {
+ return new SixToSevenUpgradeHandler().upgrade(config, context,
instantTime, upgradeDowngradeHelper);
+ } else if (fromVersion == HoodieTableVersion.SEVEN && toVersion ==
HoodieTableVersion.EIGHT) {
+ return new SevenToEightUpgradeHandler().upgrade(config, context,
instantTime, upgradeDowngradeHelper);
} else {
throw new HoodieUpgradeDowngradeException(fromVersion.versionCode(),
toVersion.versionCode(), true);
}
@@ -192,6 +196,10 @@ public class UpgradeDowngrade {
return new FiveToFourDowngradeHandler().downgrade(config, context,
instantTime, upgradeDowngradeHelper);
} else if (fromVersion == HoodieTableVersion.SIX && toVersion ==
HoodieTableVersion.FIVE) {
return new SixToFiveDowngradeHandler().downgrade(config, context,
instantTime, upgradeDowngradeHelper);
+ } else if (fromVersion == HoodieTableVersion.SEVEN && toVersion ==
HoodieTableVersion.SIX) {
+ return new SevenToSixDowngradeHandler().downgrade(config, context,
instantTime, upgradeDowngradeHelper);
+ } else if (fromVersion == HoodieTableVersion.EIGHT && toVersion ==
HoodieTableVersion.SEVEN) {
+ return new EightToSevenDowngradeHandler().downgrade(config, context,
instantTime, upgradeDowngradeHelper);
} else {
throw new HoodieUpgradeDowngradeException(fromVersion.versionCode(),
toVersion.versionCode(), false);
}
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java
b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java
index 9210d1521ed..87263a13f9d 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java
@@ -116,6 +116,13 @@ public class HoodieTableConfig extends HoodieConfig {
.withDocumentation("Version of table, used for running upgrade/downgrade
steps between releases with potentially "
+ "breaking/backwards compatible changes.");
+ public static final ConfigProperty<HoodieTableVersion> INITIAL_VERSION =
ConfigProperty
+ .key("hoodie.table.initial.version")
+ .defaultValue(HoodieTableVersion.ZERO)
+ .withDocumentation("Initial Version of table when the table was
created. Used for upgrade/downgrade"
+ + " to identify what upgrade/downgrade paths happened on the
table. This is only configured "
+ + "when the table is initially setup.");
+
public static final ConfigProperty<String> PRECOMBINE_FIELD = ConfigProperty
.key("hoodie.table.precombine.field")
.noDefaultValue()
@@ -512,6 +519,15 @@ public class HoodieTableConfig extends HoodieConfig {
: VERSION.defaultValue();
}
+ /**
+ * @return the hoodie.table.initial.version from hoodie.properties file.
+ */
+ public HoodieTableVersion getTableInitialVersion() {
+ return contains(INITIAL_VERSION)
+ ? HoodieTableVersion.versionFromCode(getInt(INITIAL_VERSION))
+ : INITIAL_VERSION.defaultValue();
+ }
+
public void setTableVersion(HoodieTableVersion tableVersion) {
setValue(VERSION, Integer.toString(tableVersion.versionCode()));
}
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java
b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java
index b41e447de08..e8e99ff9a0c 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java
@@ -76,6 +76,7 @@ import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
+import static org.apache.hudi.common.table.HoodieTableConfig.INITIAL_VERSION;
import static org.apache.hudi.common.util.ConfigUtils.containsConfigProperty;
import static org.apache.hudi.common.util.ConfigUtils.getStringWithAltKeys;
import static org.apache.hudi.common.util.StringUtils.getUTF8Bytes;
@@ -623,6 +624,8 @@ public class HoodieTableMetaClient implements Serializable {
}
initializeBootstrapDirsIfNotExists(basePath, storage);
+ // When the table is initialized, set the initial version to be the
current version.
+ props.put(INITIAL_VERSION.key(),
String.valueOf(HoodieTableVersion.current().versionCode()));
HoodieTableConfig.create(storage, metaPathDir, props);
}
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableVersion.java
b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableVersion.java
index a0ec92473ca..f3ed871e125 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableVersion.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableVersion.java
@@ -40,8 +40,12 @@ public enum HoodieTableVersion {
// 0.12.0 onwards
FIVE(5),
// 0.14.0 onwards
- SIX(6);
-
+ SIX(6),
+ // 0.16.0
+ SEVEN(7),
+ // 1.0 beta
+ EIGHT(8);
+
private final int versionCode;
HoodieTableVersion(int versionCode) {
@@ -53,7 +57,7 @@ public enum HoodieTableVersion {
}
public static HoodieTableVersion current() {
- return SIX;
+ return EIGHT;
}
public static HoodieTableVersion versionFromCode(int versionCode) {
diff --git
a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableMetaClient.java
b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableMetaClient.java
index 0ab72016893..b4e2fca80d3 100644
---
a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableMetaClient.java
+++
b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableMetaClient.java
@@ -63,6 +63,7 @@ public class TestHoodieTableMetaClient extends
HoodieCommonTestHarness {
"Metapath should be ${basepath}/.hoodie");
assertTrue(metaClient.getTableConfig().getProps().containsKey(HoodieTableConfig.TABLE_CHECKSUM.key()));
assertTrue(HoodieTableConfig.validateChecksum(metaClient.getTableConfig().getProps()));
+ assertEquals(HoodieTableVersion.current(),
metaClient.getTableConfig().getTableInitialVersion());
}
@Test
diff --git
a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairOverwriteHoodiePropsProcedure.scala
b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairOverwriteHoodiePropsProcedure.scala
index 3273c737747..fcc883c4aaf 100644
---
a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairOverwriteHoodiePropsProcedure.scala
+++
b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairOverwriteHoodiePropsProcedure.scala
@@ -20,7 +20,6 @@ package org.apache.spark.sql.hudi.command.procedures
import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient}
import org.apache.hudi.hadoop.fs.HadoopFSUtils
-import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
import org.apache.spark.internal.Logging
import org.apache.spark.sql.Row
@@ -68,6 +67,10 @@ class RepairOverwriteHoodiePropsProcedure extends
BaseProcedure with ProcedureBu
var newProps = new Properties
loadNewProps(overwriteFilePath, newProps)
val oldProps = metaClient.getTableConfig.propsMap
+ // Copy Initial Version from old props to new props
+ if (oldProps.containsKey(HoodieTableConfig.INITIAL_VERSION.key)) {
+ newProps.put(HoodieTableConfig.INITIAL_VERSION.key,
oldProps.get(HoodieTableConfig.INITIAL_VERSION.key))
+ }
HoodieTableConfig.create(metaClient.getStorage, metaClient.getMetaPath,
newProps)
// reload new props as checksum would have been added
newProps = HoodieTableMetaClient.reload(metaClient).getTableConfig.getProps
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestRepairsProcedure.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestRepairsProcedure.scala
index 154d9ecfb5a..c8c7e0cda8c 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestRepairsProcedure.scala
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestRepairsProcedure.scala
@@ -141,6 +141,7 @@ class TestRepairsProcedure extends
HoodieSparkProcedureTestBase {
|[hoodie.datasource.write.partitionpath.urlencode,false,null]
|[hoodie.table.checksum,,]
|[hoodie.table.create.schema,,]
+ |[hoodie.table.initial.version,8,8]
|[hoodie.table.keygenerator.type,NON_PARTITION,null]
|[hoodie.table.name,,]
|[hoodie.table.precombine.field,ts,null]
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestUpgradeOrDowngradeProcedure.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestUpgradeOrDowngradeProcedure.scala
index bc6f266eb94..3db05e6572d 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestUpgradeOrDowngradeProcedure.scala
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestUpgradeOrDowngradeProcedure.scala
@@ -54,10 +54,10 @@ class TestUpgradeOrDowngradeProcedure extends
HoodieSparkProcedureTestBase {
var metaClient = createMetaClient(spark, tablePath)
// verify hoodie.table.version of the original table
- assertResult(HoodieTableVersion.SIX.versionCode) {
+ assertResult(HoodieTableVersion.EIGHT.versionCode) {
metaClient.getTableConfig.getTableVersion.versionCode()
}
- assertTableVersionFromPropertyFile(metaClient,
HoodieTableVersion.SIX.versionCode)
+ assertTableVersionFromPropertyFile(metaClient,
HoodieTableVersion.EIGHT.versionCode)
// downgrade table to ZERO
checkAnswer(s"""call downgrade_table(table => '$tableName', to_version
=> 'ZERO')""")(Seq(true))
