This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new a91bdece2c1 [HUDI-7079] Disable new file format for metadata table
(#10058)
a91bdece2c1 is described below
commit a91bdece2c1782410009915881ff6488fdfbf973
Author: Jon Vexler <jbvex...@gmail.com>
AuthorDate: Fri Nov 10 13:04:42 2023 -0500
[HUDI-7079] Disable new file format for metadata table (#10058)
Disable the new file format for the metadata table because we haven't
implemented hfile base files yet. Also made new method for duplicated code
Co-authored-by: Jonathan Vexler <=>
---
.../java/org/apache/hudi/client/BaseHoodieTableServiceClient.java | 2 +-
.../src/main/java/org/apache/hudi/table/HoodieTable.java | 2 +-
.../java/org/apache/hudi/table/action/compact/CompactHelpers.java | 2 +-
.../org/apache/hudi/common/bootstrap/index/HFileBootstrapIndex.java | 4 +---
.../java/org/apache/hudi/common/table/HoodieTableMetaClient.java | 5 +++++
.../src/main/scala/org/apache/hudi/DefaultSource.scala | 4 ++--
6 files changed, 11 insertions(+), 8 deletions(-)
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieTableServiceClient.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieTableServiceClient.java
index 8b39a46cb07..112b8111004 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieTableServiceClient.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieTableServiceClient.java
@@ -952,7 +952,7 @@ public abstract class BaseHoodieTableServiceClient<I, T, O>
extends BaseHoodieCl
// from the async indexer (`HoodieIndexer`).
// TODO(HUDI-5733): This should be cleaned up once the proper fix of
rollbacks in the
// metadata table is landed.
- if (isMetadataTable(metaClient.getBasePathV2().toString())) {
+ if (metaClient.isMetadataTable()) {
return
inflightInstantsStream.map(HoodieInstant::getTimestamp).filter(entry -> {
if (curInstantTime.isPresent()) {
return !entry.equals(curInstantTime.get());
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java
index c44d3b0f4cb..aa521fcc3e5 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java
@@ -1010,7 +1010,7 @@ public abstract class HoodieTable<T, I, K, O> implements
Serializable {
// Only execute metadata table deletion when all the following conditions
are met
// (1) This is data table
// (2) Metadata table is disabled in HoodieWriteConfig for the writer
- return !HoodieTableMetadata.isMetadataTable(metaClient.getBasePath())
+ return !metaClient.isMetadataTable()
&& !config.isMetadataTableEnabled();
}
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/CompactHelpers.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/CompactHelpers.java
index 6a1e3f14062..78ea56fd9ad 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/CompactHelpers.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/CompactHelpers.java
@@ -106,7 +106,7 @@ public class CompactHelpers<T, I, K, O> {
}
public Option<InstantRange> getInstantRange(HoodieTableMetaClient
metaClient) {
- return
HoodieTableMetadata.isMetadataTable(metaClient.getBasePathV2().toString())
+ return metaClient.isMetadataTable()
? Option.of(getMetadataLogReaderInstantRange(metaClient)) :
Option.empty();
}
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/HFileBootstrapIndex.java
b/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/HFileBootstrapIndex.java
index 27314f150dc..744f3dc3a2d 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/HFileBootstrapIndex.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/HFileBootstrapIndex.java
@@ -34,7 +34,6 @@ import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.io.storage.HoodieHFileUtils;
-import org.apache.hudi.metadata.HoodieTableMetadata;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -106,8 +105,7 @@ public class HFileBootstrapIndex extends BootstrapIndex {
FileSystem fs = metaClient.getFs();
// The metadata table is never bootstrapped, so the bootstrap index is
always absent
// for the metadata table. The fs.exists calls are avoided for metadata
table.
- isPresent =
!HoodieTableMetadata.isMetadataTable(metaClient.getBasePathV2().toString())
- && fs.exists(indexByPartitionPath) && fs.exists(indexByFilePath);
+ isPresent = !metaClient.isMetadataTable() &&
fs.exists(indexByPartitionPath) && fs.exists(indexByFilePath);
} catch (IOException ioe) {
throw new HoodieIOException(ioe.getMessage(), ioe);
}
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java
b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java
index e7aba0b502b..9512e61f089 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java
@@ -57,6 +57,7 @@ import org.apache.hudi.exception.TableNotFoundException;
import org.apache.hudi.hadoop.CachingPath;
import org.apache.hudi.hadoop.SerializablePath;
import org.apache.hudi.keygen.constant.KeyGeneratorType;
+import org.apache.hudi.metadata.HoodieTableMetadata;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
@@ -382,6 +383,10 @@ public class HoodieTableMetaClient implements Serializable
{
return timelineLayoutVersion;
}
+ public Boolean isMetadataTable() {
+ return HoodieTableMetadata.isMetadataTable(getBasePathV2().toString());
+ }
+
/**
* Get the FS implementation for this table.
*/
diff --git
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/DefaultSource.scala
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/DefaultSource.scala
index bdaf1257e57..c619f61cb2c 100644
---
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/DefaultSource.scala
+++
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/DefaultSource.scala
@@ -235,8 +235,8 @@ object DefaultSource {
Option(schema)
}
- val useNewPaquetFileFormat = parameters.getOrElse(
- USE_NEW_HUDI_PARQUET_FILE_FORMAT.key,
USE_NEW_HUDI_PARQUET_FILE_FORMAT.defaultValue).toBoolean
+ val useNewPaquetFileFormat =
parameters.getOrElse(USE_NEW_HUDI_PARQUET_FILE_FORMAT.key,
+ USE_NEW_HUDI_PARQUET_FILE_FORMAT.defaultValue).toBoolean &&
!metaClient.isMetadataTable
if (metaClient.getCommitsTimeline.filterCompletedInstants.countInstants()
== 0) {
new EmptyRelation(sqlContext, resolveSchema(metaClient, parameters,
Some(schema)))
} else if (isCdcQuery) {
