This is an automated email from the ASF dual-hosted git repository. sivabalan pushed a commit to branch release-0.5.3 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit ae48ecbe232eb55267d1a138baeec13baa1fb249 Author: Balaji Varadarajan <varad...@uber.com> AuthorDate: Wed Jun 3 00:35:14 2020 -0700 [HUDI-990] Timeline API : filterCompletedAndCompactionInstants needs to handle requested state correctly. Also ensure timeline gets reloaded after we revert committed transactions --- .../client/embedded/EmbeddedTimelineService.java | 4 +++- .../apache/hudi/table/HoodieCopyOnWriteTable.java | 2 ++ .../apache/hudi/table/HoodieMergeOnReadTable.java | 2 ++ .../org/apache/hudi/table/TestMergeOnReadTable.java | 3 +++ .../table/timeline/HoodieDefaultTimeline.java | 2 +- .../table/view/FileSystemViewStorageConfig.java | 21 +++++++++++++++++++++ 6 files changed, 32 insertions(+), 2 deletions(-) diff --git a/hudi-client/src/main/java/org/apache/hudi/client/embedded/EmbeddedTimelineService.java b/hudi-client/src/main/java/org/apache/hudi/client/embedded/EmbeddedTimelineService.java index 5afee3f..c7c4f7b 100644 --- a/hudi-client/src/main/java/org/apache/hudi/client/embedded/EmbeddedTimelineService.java +++ b/hudi-client/src/main/java/org/apache/hudi/client/embedded/EmbeddedTimelineService.java @@ -89,7 +89,9 @@ public class EmbeddedTimelineService { * Retrieves proper view storage configs for remote clients to access this service. */ public FileSystemViewStorageConfig getRemoteFileSystemViewConfig() { - return FileSystemViewStorageConfig.newBuilder().withStorageType(FileSystemViewStorageType.REMOTE_FIRST) + FileSystemViewStorageType viewStorageType = config.shouldEnableBackupForRemoteFileSystemView() + ? FileSystemViewStorageType.REMOTE_FIRST : FileSystemViewStorageType.REMOTE_ONLY; + return FileSystemViewStorageConfig.newBuilder().withStorageType(viewStorageType) .withRemoteServerHost(hostAddr).withRemoteServerPort(serverPort).build(); } diff --git a/hudi-client/src/main/java/org/apache/hudi/table/HoodieCopyOnWriteTable.java b/hudi-client/src/main/java/org/apache/hudi/table/HoodieCopyOnWriteTable.java index 4c91c77..c74af2d 100644 --- a/hudi-client/src/main/java/org/apache/hudi/table/HoodieCopyOnWriteTable.java +++ b/hudi-client/src/main/java/org/apache/hudi/table/HoodieCopyOnWriteTable.java @@ -359,6 +359,8 @@ public class HoodieCopyOnWriteTable<T extends HoodieRecordPayload> extends Hoodi if (instant.isCompleted()) { LOG.info("Unpublishing instant " + instant); instant = activeTimeline.revertToInflight(instant); + // reload meta-client to reflect latest timeline status + metaClient.reloadActiveTimeline(); } // For Requested State (like failure during index lookup), there is nothing to do rollback other than diff --git a/hudi-client/src/main/java/org/apache/hudi/table/HoodieMergeOnReadTable.java b/hudi-client/src/main/java/org/apache/hudi/table/HoodieMergeOnReadTable.java index 938a5fd..5f56369 100644 --- a/hudi-client/src/main/java/org/apache/hudi/table/HoodieMergeOnReadTable.java +++ b/hudi-client/src/main/java/org/apache/hudi/table/HoodieMergeOnReadTable.java @@ -179,6 +179,8 @@ public class HoodieMergeOnReadTable<T extends HoodieRecordPayload> extends Hoodi if (instant.isCompleted()) { LOG.error("Un-publishing instant " + instant + ", deleteInstants=" + deleteInstants); instant = this.getActiveTimeline().revertToInflight(instant); + // reload meta-client to reflect latest timeline status + metaClient.reloadActiveTimeline(); } List<HoodieRollbackStat> allRollbackStats = new ArrayList<>(); diff --git a/hudi-client/src/test/java/org/apache/hudi/table/TestMergeOnReadTable.java b/hudi-client/src/test/java/org/apache/hudi/table/TestMergeOnReadTable.java index fdc968d..9f3eaea 100644 --- a/hudi-client/src/test/java/org/apache/hudi/table/TestMergeOnReadTable.java +++ b/hudi-client/src/test/java/org/apache/hudi/table/TestMergeOnReadTable.java @@ -44,6 +44,7 @@ import org.apache.hudi.common.table.TableFileSystemView.SliceView; import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieInstant.State; +import org.apache.hudi.common.table.view.FileSystemViewStorageConfig; import org.apache.hudi.common.table.view.HoodieTableFileSystemView; import org.apache.hudi.common.util.Option; import org.apache.hudi.config.HoodieCompactionConfig; @@ -1219,6 +1220,8 @@ public class TestMergeOnReadTable extends HoodieClientTestHarness { .withInlineCompaction(false).withMaxNumDeltaCommitsBeforeCompaction(1).build()) .withStorageConfig(HoodieStorageConfig.newBuilder().limitFileSize(1024 * 1024 * 1024).build()) .withEmbeddedTimelineServerEnabled(true).forTable("test-trip-table") + .withFileSystemViewConfig(new FileSystemViewStorageConfig.Builder() + .withEnableBackupForRemoteFileSystemView(false).build()) .withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(indexType).build()); } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieDefaultTimeline.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieDefaultTimeline.java index 86431c9..6451749 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieDefaultTimeline.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieDefaultTimeline.java @@ -107,7 +107,7 @@ public class HoodieDefaultTimeline implements HoodieTimeline { @Override public HoodieTimeline filterCompletedAndCompactionInstants() { - return new HoodieDefaultTimeline(instants.stream().filter(s -> !s.isInflight() + return new HoodieDefaultTimeline(instants.stream().filter(s -> s.isCompleted() || s.getAction().equals(HoodieTimeline.COMPACTION_ACTION)), details); } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewStorageConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewStorageConfig.java index 93c5507..d805dfb 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewStorageConfig.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewStorageConfig.java @@ -55,6 +55,15 @@ public class FileSystemViewStorageConfig extends DefaultHoodieConfig { private static final Double DEFAULT_MEM_FRACTION_FOR_PENDING_COMPACTION = 0.01; private static final Long DEFAULT_MAX_MEMORY_FOR_VIEW = 100 * 1024 * 1024L; // 100 MB + /** + * Configs to control whether backup needs to be configured if clients were not able to reach + * timeline service. + */ + public static final String REMOTE_BACKUP_VIEW_HANDLER_ENABLE = + "hoodie.filesystem.remote.backup.view.enable"; + // Need to be disabled only for tests. + public static final String DEFAULT_REMOTE_BACKUP_VIEW_HANDLER_ENABLE = "true"; + public static FileSystemViewStorageConfig.Builder newBuilder() { return new Builder(); } @@ -98,6 +107,10 @@ public class FileSystemViewStorageConfig extends DefaultHoodieConfig { return FileSystemViewStorageType.valueOf(props.getProperty(FILESYSTEM_SECONDARY_VIEW_STORAGE_TYPE)); } + public boolean shouldEnableBackupForRemoteFileSystemView() { + return Boolean.parseBoolean(props.getProperty(REMOTE_BACKUP_VIEW_HANDLER_ENABLE)); + } + public String getRocksdbBasePath() { return props.getProperty(ROCKSDB_BASE_PATH_PROP); } @@ -166,6 +179,11 @@ public class FileSystemViewStorageConfig extends DefaultHoodieConfig { return this; } + public Builder withEnableBackupForRemoteFileSystemView(boolean enable) { + props.setProperty(REMOTE_BACKUP_VIEW_HANDLER_ENABLE, Boolean.toString(enable)); + return this; + } + public FileSystemViewStorageConfig build() { setDefaultOnCondition(props, !props.containsKey(FILESYSTEM_VIEW_STORAGE_TYPE), FILESYSTEM_VIEW_STORAGE_TYPE, DEFAULT_VIEW_STORAGE_TYPE.name()); @@ -188,6 +206,9 @@ public class FileSystemViewStorageConfig extends DefaultHoodieConfig { setDefaultOnCondition(props, !props.containsKey(ROCKSDB_BASE_PATH_PROP), ROCKSDB_BASE_PATH_PROP, DEFAULT_ROCKSDB_BASE_PATH); + setDefaultOnCondition(props, !props.containsKey(REMOTE_BACKUP_VIEW_HANDLER_ENABLE), + REMOTE_BACKUP_VIEW_HANDLER_ENABLE, DEFAULT_REMOTE_BACKUP_VIEW_HANDLER_ENABLE); + // Validations FileSystemViewStorageType.valueOf(props.getProperty(FILESYSTEM_VIEW_STORAGE_TYPE)); FileSystemViewStorageType.valueOf(props.getProperty(FILESYSTEM_SECONDARY_VIEW_STORAGE_TYPE));