This is an automated email from the ASF dual-hosted git repository.
codope pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new a5eb6dbbb0e [MINOR] Test table cleanup (#9110)
a5eb6dbbb0e is described below
commit a5eb6dbbb0e531821250c1f0217d29f85410ac49
Author: Sagar Sumit <sagarsumi...@gmail.com>
AuthorDate: Tue Jul 4 15:25:49 2023 +0530
[MINOR] Test table cleanup (#9110)
---
.../client/functional/TestHoodieMetadataBase.java | 6 ++-
.../apache/hudi/io/TestHoodieTimelineArchiver.java | 4 +-
.../java/org/apache/hudi/table/TestCleaner.java | 6 ++-
.../hudi/common/testutils/HoodieTestTable.java | 49 +++++++---------------
4 files changed, 25 insertions(+), 40 deletions(-)
diff --git
a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieMetadataBase.java
b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieMetadataBase.java
index 27fd2822986..7a3006277e9 100644
---
a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieMetadataBase.java
+++
b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieMetadataBase.java
@@ -18,7 +18,6 @@
package org.apache.hudi.client.functional;
-import org.apache.hadoop.fs.Path;
import org.apache.hudi.client.HoodieTimelineArchiver;
import org.apache.hudi.common.config.HoodieMetadataConfig;
import org.apache.hudi.common.config.HoodieStorageConfig;
@@ -45,6 +44,8 @@ import
org.apache.hudi.metadata.SparkHoodieBackedTableMetadataWriter;
import org.apache.hudi.table.HoodieSparkTable;
import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.testutils.HoodieClientTestHarness;
+
+import org.apache.hadoop.fs.Path;
import org.junit.jupiter.api.AfterEach;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -116,8 +117,9 @@ public class TestHoodieMetadataBase extends
HoodieClientTestHarness {
this.writeConfig = writeConfig;
if (enableMetadataTable) {
metadataWriter = SparkHoodieBackedTableMetadataWriter.create(hadoopConf,
writeConfig, context);
+ // reload because table configs could have been updated
+ metaClient = HoodieTableMetaClient.reload(metaClient);
testTable = HoodieMetadataTestTable.of(metaClient, metadataWriter);
- testTable.updateFilesPartitionInTableConfig();
} else {
testTable = HoodieTestTable.of(metaClient);
}
diff --git
a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/TestHoodieTimelineArchiver.java
b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/TestHoodieTimelineArchiver.java
index 7251988e4fd..2306376e1a4 100644
---
a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/TestHoodieTimelineArchiver.java
+++
b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/TestHoodieTimelineArchiver.java
@@ -138,8 +138,9 @@ public class TestHoodieTimelineArchiver extends
HoodieClientTestHarness {
private void initWriteConfigAndMetatableWriter(HoodieWriteConfig
writeConfig, boolean enableMetadataTable) throws IOException {
if (enableMetadataTable) {
metadataWriter = SparkHoodieBackedTableMetadataWriter.create(hadoopConf,
writeConfig, context);
+ // reload because table configs could have been updated
+ metaClient = HoodieTableMetaClient.reload(metaClient);
testTable = HoodieMetadataTestTable.of(metaClient, metadataWriter);
- testTable.updateFilesPartitionInTableConfig();
} else {
testTable = HoodieTestTable.of(metaClient);
}
@@ -1484,6 +1485,7 @@ public class TestHoodieTimelineArchiver extends
HoodieClientTestHarness {
/**
* Test archival functionality when there are inflights files.
* Archive should hold on to the greatest completed commit that is less than
the oldes inflight commit.
+ *
* @throws Exception
*/
@Test
diff --git
a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestCleaner.java
b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestCleaner.java
index efe8e5a688c..d1e77613691 100644
---
a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestCleaner.java
+++
b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestCleaner.java
@@ -925,8 +925,9 @@ public class TestCleaner extends HoodieCleanerTestBase {
.withPath(basePath)
.build();
HoodieTableMetadataWriter metadataWriter =
SparkHoodieBackedTableMetadataWriter.create(hadoopConf, config, context);
+ // reload because table configs could have been updated
+ metaClient = HoodieTableMetaClient.reload(metaClient);
HoodieTestTable testTable = HoodieMetadataTestTable.of(metaClient,
metadataWriter);
- testTable.updateFilesPartitionInTableConfig();
String p1 = "part_1";
String p2 = "part_2";
@@ -1028,6 +1029,8 @@ public class TestCleaner extends HoodieCleanerTestBase {
// multiple versions with pending compaction. File Slices (6 - 7) have
multiple file-slices but not under
// compactions
// FileIds 2-5 will be under compaction
+ // reload because table configs could have been updated
+ metaClient = HoodieTableMetaClient.reload(metaClient);
HoodieTestTable testTable = HoodieTestTable.of(metaClient);
testTable.withPartitionMetaFiles(partition);
@@ -1094,7 +1097,6 @@ public class TestCleaner extends HoodieCleanerTestBase {
part1ToFileId = new HashMap<>();
part1ToFileId.put(partition, Arrays.asList(file7P1));
commitWithMdt("013", part1ToFileId, testTable, metadataWriter, true, true);
- testTable.updateFilesPartitionInTableConfig();
// Clean now
metaClient = HoodieTableMetaClient.reload(metaClient);
diff --git
a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java
b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java
index a6775889970..43df68ce986 100644
---
a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java
+++
b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java
@@ -60,8 +60,6 @@ import org.apache.hudi.common.util.StringUtils;
import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieIOException;
-import org.apache.hudi.metadata.HoodieMetadataPayload;
-import org.apache.hudi.metadata.MetadataPartitionType;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -88,12 +86,10 @@ import java.util.stream.Stream;
import static java.time.temporal.ChronoUnit.SECONDS;
import static java.util.Collections.singletonMap;
-import static
org.apache.hudi.common.config.HoodieMetadataConfig.DEFAULT_METADATA_POPULATE_META_FIELDS;
import static org.apache.hudi.common.model.HoodieTableType.MERGE_ON_READ;
import static org.apache.hudi.common.model.WriteOperationType.CLUSTER;
import static org.apache.hudi.common.model.WriteOperationType.COMPACT;
import static org.apache.hudi.common.model.WriteOperationType.UPSERT;
-import static org.apache.hudi.common.table.HoodieTableConfig.ARCHIVELOG_FOLDER;
import static
org.apache.hudi.common.table.timeline.HoodieTimeline.CLEAN_ACTION;
import static
org.apache.hudi.common.table.timeline.HoodieTimeline.REPLACE_COMMIT_ACTION;
import static org.apache.hudi.common.testutils.FileCreateUtils.baseFileName;
@@ -125,7 +121,6 @@ import static
org.apache.hudi.common.util.CleanerUtils.convertCleanMetadata;
import static org.apache.hudi.common.util.CommitUtils.buildMetadata;
import static org.apache.hudi.common.util.CommitUtils.getCommitActionType;
import static org.apache.hudi.common.util.StringUtils.EMPTY_STRING;
-import static
org.apache.hudi.metadata.HoodieTableMetadata.METADATA_TABLE_NAME_SUFFIX;
/**
* Test Hoodie Table for testing only.
@@ -144,7 +139,7 @@ public class HoodieTestTable {
protected final FileSystem fs;
protected HoodieTableMetaClient metaClient;
protected String currentInstantTime;
- private boolean isNonPartitioned = false;
+ private boolean isNonPartitioned = false;
protected HoodieTestTable(String basePath, FileSystem fs,
HoodieTableMetaClient metaClient) {
ValidationUtils.checkArgument(Objects.equals(basePath,
metaClient.getBasePath()));
@@ -160,22 +155,6 @@ public class HoodieTestTable {
return new HoodieTestTable(metaClient.getBasePath(),
metaClient.getRawFs(), metaClient);
}
- public void updateFilesPartitionInTableConfig() throws IOException {
- metaClient.getTableConfig().setMetadataPartitionState(metaClient,
MetadataPartitionType.FILES, true);
- this.metaClient = HoodieTableMetaClient.reload(metaClient);
-
- HoodieTableMetaClient.withPropertyBuilder()
- .setTableType(HoodieTableType.MERGE_ON_READ)
- .setTableName("test_table" + METADATA_TABLE_NAME_SUFFIX)
- .setArchiveLogFolder(ARCHIVELOG_FOLDER.defaultValue())
- .setPayloadClassName(HoodieMetadataPayload.class.getName())
- .setBaseFileFormat(HoodieFileFormat.HFILE.toString())
- .setRecordKeyFields(HoodieMetadataPayload.KEY_FIELD_NAME)
- .setPopulateMetaFields(DEFAULT_METADATA_POPULATE_META_FIELDS)
-
.setKeyGeneratorClassProp("org.apache.hudi.metadata.HoodieTableMetadataKeyGenerator")
- .initTable(metaClient.getHadoopConf(), metaClient.getMetaPath() +
"/metadata");
- }
-
public void setNonPartitioned() {
this.isNonPartitioned = true;
}
@@ -482,7 +461,7 @@ public class HoodieTestTable {
public HoodieTestTable addInflightCompaction(String instantTime,
HoodieCommitMetadata commitMetadata) throws Exception {
List<FileSlice> fileSlices = new ArrayList<>();
for (Map.Entry<String, List<HoodieWriteStat>> entry :
commitMetadata.getPartitionToWriteStats().entrySet()) {
- for (HoodieWriteStat stat: entry.getValue()) {
+ for (HoodieWriteStat stat : entry.getValue()) {
fileSlices.add(new FileSlice(entry.getKey(), instantTime,
stat.getPath()));
}
}
@@ -503,27 +482,27 @@ public class HoodieTestTable {
forReplaceCommit(instantTime);
WriteOperationType operationType = WriteOperationType.DELETE_PARTITION;
Pair<HoodieRequestedReplaceMetadata, HoodieReplaceCommitMetadata> metas =
- generateReplaceCommitMetadata(instantTime, partition, fileIds,
Option.empty(), operationType);
+ generateReplaceCommitMetadata(instantTime, partition, fileIds,
Option.empty(), operationType);
return addReplaceCommit(instantTime, Option.of(metas.getLeft()),
Option.empty(), metas.getRight());
}
private Pair<HoodieRequestedReplaceMetadata, HoodieReplaceCommitMetadata>
generateReplaceCommitMetadata(
- String instantTime, String partition, List<String> replacedFileIds,
Option<String> newFileId, WriteOperationType operationType) {
+ String instantTime, String partition, List<String> replacedFileIds,
Option<String> newFileId, WriteOperationType operationType) {
HoodieRequestedReplaceMetadata requestedReplaceMetadata = new
HoodieRequestedReplaceMetadata();
requestedReplaceMetadata.setOperationType(operationType.toString());
requestedReplaceMetadata.setVersion(1);
List<HoodieSliceInfo> sliceInfos = replacedFileIds.stream()
- .map(replacedFileId ->
HoodieSliceInfo.newBuilder().setFileId(replacedFileId).build())
- .collect(Collectors.toList());
+ .map(replacedFileId ->
HoodieSliceInfo.newBuilder().setFileId(replacedFileId).build())
+ .collect(Collectors.toList());
List<HoodieClusteringGroup> clusteringGroups = new ArrayList<>();
clusteringGroups.add(HoodieClusteringGroup.newBuilder()
-
.setVersion(1).setNumOutputFileGroups(1).setMetrics(Collections.emptyMap())
- .setSlices(sliceInfos).build());
+
.setVersion(1).setNumOutputFileGroups(1).setMetrics(Collections.emptyMap())
+ .setSlices(sliceInfos).build());
requestedReplaceMetadata.setExtraMetadata(Collections.emptyMap());
requestedReplaceMetadata.setClusteringPlan(HoodieClusteringPlan.newBuilder()
- .setVersion(1).setExtraMetadata(Collections.emptyMap())
-
.setStrategy(HoodieClusteringStrategy.newBuilder().setStrategyClassName("").setVersion(1).build())
- .setInputGroups(clusteringGroups).build());
+ .setVersion(1).setExtraMetadata(Collections.emptyMap())
+
.setStrategy(HoodieClusteringStrategy.newBuilder().setStrategyClassName("").setVersion(1).build())
+ .setInputGroups(clusteringGroups).build());
HoodieReplaceCommitMetadata replaceMetadata = new
HoodieReplaceCommitMetadata();
replacedFileIds.forEach(replacedFileId ->
replaceMetadata.addReplaceFileId(partition, replacedFileId));
@@ -850,7 +829,7 @@ public class HoodieTestTable {
for (Map.Entry<String, List<String>> entry : partitionFiles.entrySet()) {
deleteFilesInPartition(entry.getKey(), entry.getValue());
}
- for (Map.Entry<String, List<String>> entry: extraFiles.entrySet()) {
+ for (Map.Entry<String, List<String>> entry : extraFiles.entrySet()) {
if (partitionFiles.containsKey(entry.getKey())) {
partitionFiles.get(entry.getKey()).addAll(entry.getValue());
}
@@ -864,7 +843,7 @@ public class HoodieTestTable {
List<HoodieInstant> commitsToRollback =
metaClient.getActiveTimeline().getCommitsTimeline()
.filterCompletedInstants().findInstantsAfter(commitToRestoreTo).getReverseOrderedInstants().collect(Collectors.toList());
Map<String, List<HoodieRollbackMetadata>> rollbackMetadataMap = new
HashMap<>();
- for (HoodieInstant commitInstantToRollback: commitsToRollback) {
+ for (HoodieInstant commitInstantToRollback : commitsToRollback) {
Option<HoodieCommitMetadata> commitMetadata =
getCommitMeta(commitInstantToRollback);
if (!commitMetadata.isPresent()) {
throw new IllegalArgumentException("Instant to rollback not present in
timeline: " + commitInstantToRollback.getTimestamp());
@@ -877,7 +856,7 @@ public class HoodieTestTable {
}
}
- HoodieRestoreMetadata restoreMetadata =
TimelineMetadataUtils.convertRestoreMetadata(restoreTime,1000L,
+ HoodieRestoreMetadata restoreMetadata =
TimelineMetadataUtils.convertRestoreMetadata(restoreTime, 1000L,
commitsToRollback, rollbackMetadataMap);
return addRestore(restoreTime, restoreMetadata);
}
