This is an automated email from the ASF dual-hosted git repository. danny0405 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push: new 0804a79f20c [HUDI-7761] Make the ManifestWriter Extendable (#11253) 0804a79f20c is described below commit 0804a79f20c680b1d369c5e84d7c9ae4412ad472 Author: csivaguru <csivag...@gmail.com> AuthorDate: Sat May 18 17:35:18 2024 -0700 [HUDI-7761] Make the ManifestWriter Extendable (#11253) --- .../java/org/apache/hudi/gcp/bigquery/BigQuerySchemaResolver.java | 4 ++-- .../java/org/apache/hudi/sync/common/util/ManifestFileWriter.java | 7 +++---- .../org/apache/hudi/sync/common/util/TestManifestFileWriter.java | 3 +-- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/hudi-gcp/src/main/java/org/apache/hudi/gcp/bigquery/BigQuerySchemaResolver.java b/hudi-gcp/src/main/java/org/apache/hudi/gcp/bigquery/BigQuerySchemaResolver.java index 361f869a9fe..27b5143944f 100644 --- a/hudi-gcp/src/main/java/org/apache/hudi/gcp/bigquery/BigQuerySchemaResolver.java +++ b/hudi-gcp/src/main/java/org/apache/hudi/gcp/bigquery/BigQuerySchemaResolver.java @@ -38,7 +38,7 @@ import java.util.stream.Collectors; /** * Extracts the BigQuery schema from a Hudi table. */ -class BigQuerySchemaResolver { +public class BigQuerySchemaResolver { private static final BigQuerySchemaResolver INSTANCE = new BigQuerySchemaResolver(TableSchemaResolver::new); private final Function<HoodieTableMetaClient, TableSchemaResolver> tableSchemaResolverSupplier; @@ -48,7 +48,7 @@ class BigQuerySchemaResolver { this.tableSchemaResolverSupplier = tableSchemaResolverSupplier; } - static BigQuerySchemaResolver getInstance() { + public static BigQuerySchemaResolver getInstance() { return INSTANCE; } diff --git a/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/util/ManifestFileWriter.java b/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/util/ManifestFileWriter.java index 3eaf80dddfe..eb52fe4586d 100644 --- a/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/util/ManifestFileWriter.java +++ b/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/util/ManifestFileWriter.java @@ -54,7 +54,7 @@ public class ManifestFileWriter { private final HoodieTableMetaClient metaClient; private final boolean useFileListingFromMetadata; - private ManifestFileWriter(HoodieTableMetaClient metaClient, boolean useFileListingFromMetadata) { + protected ManifestFileWriter(HoodieTableMetaClient metaClient, boolean useFileListingFromMetadata) { this.metaClient = metaClient; this.useFileListingFromMetadata = useFileListingFromMetadata; } @@ -64,7 +64,7 @@ public class ManifestFileWriter { */ public synchronized void writeManifestFile(boolean useAbsolutePath) { try { - List<String> baseFiles = fetchLatestBaseFilesForAllPartitions(metaClient, useFileListingFromMetadata, useAbsolutePath) + List<String> baseFiles = fetchLatestBaseFilesForAllPartitions(useAbsolutePath) .collect(Collectors.toList()); if (baseFiles.isEmpty()) { LOG.warn("No base file to generate manifest file."); @@ -86,8 +86,7 @@ public class ManifestFileWriter { } @VisibleForTesting - public static Stream<String> fetchLatestBaseFilesForAllPartitions(HoodieTableMetaClient metaClient, - boolean useFileListingFromMetadata, boolean useAbsolutePath) { + public Stream<String> fetchLatestBaseFilesForAllPartitions(boolean useAbsolutePath) { try { StorageConfiguration storageConf = metaClient.getStorageConf(); HoodieLocalEngineContext engContext = new HoodieLocalEngineContext(storageConf); diff --git a/hudi-sync/hudi-sync-common/src/test/java/org/apache/hudi/sync/common/util/TestManifestFileWriter.java b/hudi-sync/hudi-sync-common/src/test/java/org/apache/hudi/sync/common/util/TestManifestFileWriter.java index 2db602b65c2..13caefd1d0f 100644 --- a/hudi-sync/hudi-sync-common/src/test/java/org/apache/hudi/sync/common/util/TestManifestFileWriter.java +++ b/hudi-sync/hudi-sync-common/src/test/java/org/apache/hudi/sync/common/util/TestManifestFileWriter.java @@ -34,7 +34,6 @@ import java.util.List; import java.util.stream.IntStream; import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS; -import static org.apache.hudi.sync.common.util.ManifestFileWriter.fetchLatestBaseFilesForAllPartitions; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -51,7 +50,7 @@ public class TestManifestFileWriter extends HoodieCommonTestHarness { // Generate 10 files under each partition createTestDataForPartitionedTable(metaClient, 10); ManifestFileWriter manifestFileWriter = ManifestFileWriter.builder().setMetaClient(metaClient).build(); - assertEquals(30, fetchLatestBaseFilesForAllPartitions(metaClient, false, false).count()); + assertEquals(30, manifestFileWriter.fetchLatestBaseFilesForAllPartitions(false).count()); } @Test