wgtmac commented on code in PR #542:
URL: https://github.com/apache/iceberg-cpp/pull/542#discussion_r2796630441
##########
src/iceberg/transaction.cc:
##########
@@ -428,4 +425,9 @@ Transaction::NewUpdateSnapshotReference() {
return update_ref;
}
+Result<std::shared_ptr<SnapshotManager>> Transaction::NewSnapshotManager() {
+ // SnapshotManager has its own commit logic, so it is not added to the
pending updates.
Review Comment:
We can still call `AddUpdate` just like other updates and do not set
`last_update_committed_` to false for `SnapshotManager`. However, it seems that
`SnapshotManager` cannot be retried easily so I'm fine to keep the current
approach as is.
##########
src/iceberg/table.cc:
##########
@@ -222,6 +223,13 @@ Result<std::shared_ptr<UpdatePartitionStatistics>>
Table::NewUpdatePartitionStat
return transaction->NewUpdatePartitionStatistics();
}
+Result<std::shared_ptr<SnapshotManager>> Table::NewSnapshotManager() {
+ ICEBERG_ASSIGN_OR_RAISE(
+ auto transaction, Transaction::Make(shared_from_this(),
Transaction::Kind::kUpdate,
+ /*auto_commit=*/false));
Review Comment:
Here `auto_commit` means that users do not need to call
`Transaction::Commit()` after calling `SnapshotManager::Commit()`. Should we
set this to true and handle the commit logic internally?
##########
src/iceberg/test/update_test_base.h:
##########
@@ -43,6 +43,7 @@ class UpdateTestBase : public ::testing::Test {
void SetUp() override {
InitializeFileIO();
RegisterTableFromResource("TableMetadataV2Valid.json");
+ RegisterMinimalTableFromResource("TableMetadataV2ValidMinimal.json");
Review Comment:
It looks weird to register two resources in the per-case setup function but
each case uses only one of them. Does it make sense to use class inheritance
for each resource?
##########
src/iceberg/update/snapshot_manager.cc:
##########
@@ -0,0 +1,201 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/update/snapshot_manager.h"
+
+#include <memory>
+#include <string>
+
+#include "iceberg/result.h"
+#include "iceberg/snapshot.h"
+#include "iceberg/table.h"
+#include "iceberg/table_metadata.h"
+#include "iceberg/transaction.h"
+#include "iceberg/update/fast_append.h"
+#include "iceberg/update/set_snapshot.h"
+#include "iceberg/update/update_snapshot_reference.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+Result<std::shared_ptr<SnapshotManager>> SnapshotManager::Make(
+ std::shared_ptr<Transaction> transaction) {
+ ICEBERG_PRECHECK(transaction != nullptr, "Invalid input transaction: null");
+ return std::shared_ptr<SnapshotManager>(new
SnapshotManager(std::move(transaction)));
+}
+
+SnapshotManager::SnapshotManager(std::shared_ptr<Transaction> transaction)
+ : PendingUpdate(std::move(transaction)) {}
+
+SnapshotManager::~SnapshotManager() = default;
+
+SnapshotManager& SnapshotManager::Cherrypick(int64_t snapshot_id) {
+ ICEBERG_BUILDER_RETURN_IF_ERROR(CommitIfRefUpdatesExist());
+ // TODO(anyone): Implement cherrypick operation
+ ICEBERG_BUILDER_CHECK(false, "Cherrypick operation not yet implemented");
+ return *this;
+}
+
+SnapshotManager& SnapshotManager::SetCurrentSnapshot(int64_t snapshot_id) {
+ ICEBERG_BUILDER_RETURN_IF_ERROR(CommitIfRefUpdatesExist());
+ ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto set_snapshot,
transaction_->NewSetSnapshot());
+ set_snapshot->SetCurrentSnapshot(snapshot_id);
+ ICEBERG_BUILDER_RETURN_IF_ERROR(set_snapshot->Commit());
+ return *this;
+}
+
+SnapshotManager& SnapshotManager::RollbackToTime(int64_t timestamp_ms) {
+ ICEBERG_BUILDER_RETURN_IF_ERROR(CommitIfRefUpdatesExist());
+ ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto set_snapshot,
transaction_->NewSetSnapshot());
+ set_snapshot->RollbackToTime(timestamp_ms);
+ ICEBERG_BUILDER_RETURN_IF_ERROR(set_snapshot->Commit());
+ return *this;
+}
+
+SnapshotManager& SnapshotManager::RollbackTo(int64_t snapshot_id) {
+ ICEBERG_BUILDER_RETURN_IF_ERROR(CommitIfRefUpdatesExist());
+ ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto set_snapshot,
transaction_->NewSetSnapshot());
+ set_snapshot->RollbackTo(snapshot_id);
+ ICEBERG_BUILDER_RETURN_IF_ERROR(set_snapshot->Commit());
+ return *this;
+}
+
+SnapshotManager& SnapshotManager::CreateBranch(const std::string& name) {
+ if (base().current_snapshot_id != kInvalidSnapshotId) {
+ ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto current_snapshot, base().Snapshot());
+ ICEBERG_DCHECK(current_snapshot != nullptr, "Current snapshot should not
be null");
+ return CreateBranch(name, current_snapshot->snapshot_id);
+ }
+ const auto& current_refs = base().refs;
+ ICEBERG_BUILDER_CHECK(!base().refs.contains(name), "Ref {} already exists",
name);
+ ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto fast_append,
transaction_->NewFastAppend());
+ ICEBERG_BUILDER_RETURN_IF_ERROR(fast_append->SetTargetBranch(name).Commit());
+ return *this;
+}
+
+SnapshotManager& SnapshotManager::CreateBranch(const std::string& name,
+ int64_t snapshot_id) {
+ ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto update_ref,
UpdateSnapshotReferencesOperation());
+ update_ref->CreateBranch(name, snapshot_id);
+ return *this;
+}
+
+SnapshotManager& SnapshotManager::CreateTag(const std::string& name,
+ int64_t snapshot_id) {
+ ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto update_ref,
UpdateSnapshotReferencesOperation());
+ update_ref->CreateTag(name, snapshot_id);
+ return *this;
+}
+
+SnapshotManager& SnapshotManager::RemoveBranch(const std::string& name) {
+ ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto update_ref,
UpdateSnapshotReferencesOperation());
+ update_ref->RemoveBranch(name);
+ return *this;
+}
+
+SnapshotManager& SnapshotManager::RemoveTag(const std::string& name) {
+ ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto update_ref,
UpdateSnapshotReferencesOperation());
+ update_ref->RemoveTag(name);
+ return *this;
+}
+
+SnapshotManager& SnapshotManager::ReplaceTag(const std::string& name,
+ int64_t snapshot_id) {
+ ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto update_ref,
UpdateSnapshotReferencesOperation());
+ update_ref->ReplaceTag(name, snapshot_id);
+ return *this;
+}
+
+SnapshotManager& SnapshotManager::ReplaceBranch(const std::string& name,
+ int64_t snapshot_id) {
+ ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto update_ref,
UpdateSnapshotReferencesOperation());
+ update_ref->ReplaceBranch(name, snapshot_id);
+ return *this;
+}
+
+SnapshotManager& SnapshotManager::ReplaceBranch(const std::string& from,
+ const std::string& to) {
+ ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto update_ref,
UpdateSnapshotReferencesOperation());
+ update_ref->ReplaceBranch(from, to);
+ return *this;
+}
+
+SnapshotManager& SnapshotManager::FastForwardBranch(const std::string& from,
+ const std::string& to) {
+ ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto update_ref,
UpdateSnapshotReferencesOperation());
+ update_ref->FastForward(from, to);
+ return *this;
+}
+
+SnapshotManager& SnapshotManager::RenameBranch(const std::string& name,
+ const std::string& new_name) {
+ ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto update_ref,
UpdateSnapshotReferencesOperation());
+ update_ref->RenameBranch(name, new_name);
+ return *this;
+}
+
+SnapshotManager& SnapshotManager::SetMinSnapshotsToKeep(const std::string&
branch_name,
+ int32_t
min_snapshots_to_keep) {
+ ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto update_ref,
UpdateSnapshotReferencesOperation());
+ update_ref->SetMinSnapshotsToKeep(branch_name, min_snapshots_to_keep);
+ return *this;
+}
+
+SnapshotManager& SnapshotManager::SetMaxSnapshotAgeMs(const std::string&
branch_name,
+ int64_t
max_snapshot_age_ms) {
+ ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto update_ref,
UpdateSnapshotReferencesOperation());
+ update_ref->SetMaxSnapshotAgeMs(branch_name, max_snapshot_age_ms);
+ return *this;
+}
+
+SnapshotManager& SnapshotManager::SetMaxRefAgeMs(const std::string& name,
+ int64_t max_ref_age_ms) {
+ ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto update_ref,
UpdateSnapshotReferencesOperation());
+ update_ref->SetMaxRefAgeMs(name, max_ref_age_ms);
+ return *this;
+}
+
+Status SnapshotManager::Commit() {
+ transaction_->EnableAutoCommit();
Review Comment:
I think we need to read and backup `Transaction::auto_commit_` in the
SnapshotManager ctor and then restore it before exiting this function,
otherwise any `CommitIfRefUpdatesExist()` before `SnapshotManager::Commit()`
will set `Transaction::committed_` to true.
##########
src/iceberg/update/snapshot_manager.cc:
##########
@@ -0,0 +1,201 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/update/snapshot_manager.h"
+
+#include <memory>
+#include <string>
+
+#include "iceberg/result.h"
+#include "iceberg/snapshot.h"
+#include "iceberg/table.h"
+#include "iceberg/table_metadata.h"
+#include "iceberg/transaction.h"
+#include "iceberg/update/fast_append.h"
+#include "iceberg/update/set_snapshot.h"
+#include "iceberg/update/update_snapshot_reference.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+Result<std::shared_ptr<SnapshotManager>> SnapshotManager::Make(
+ std::shared_ptr<Transaction> transaction) {
+ ICEBERG_PRECHECK(transaction != nullptr, "Invalid input transaction: null");
+ return std::shared_ptr<SnapshotManager>(new
SnapshotManager(std::move(transaction)));
+}
+
+SnapshotManager::SnapshotManager(std::shared_ptr<Transaction> transaction)
+ : PendingUpdate(std::move(transaction)) {}
+
+SnapshotManager::~SnapshotManager() = default;
+
+SnapshotManager& SnapshotManager::Cherrypick(int64_t snapshot_id) {
+ ICEBERG_BUILDER_RETURN_IF_ERROR(CommitIfRefUpdatesExist());
+ // TODO(anyone): Implement cherrypick operation
+ ICEBERG_BUILDER_CHECK(false, "Cherrypick operation not yet implemented");
+ return *this;
+}
+
+SnapshotManager& SnapshotManager::SetCurrentSnapshot(int64_t snapshot_id) {
+ ICEBERG_BUILDER_RETURN_IF_ERROR(CommitIfRefUpdatesExist());
+ ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto set_snapshot,
transaction_->NewSetSnapshot());
+ set_snapshot->SetCurrentSnapshot(snapshot_id);
+ ICEBERG_BUILDER_RETURN_IF_ERROR(set_snapshot->Commit());
+ return *this;
+}
+
+SnapshotManager& SnapshotManager::RollbackToTime(int64_t timestamp_ms) {
+ ICEBERG_BUILDER_RETURN_IF_ERROR(CommitIfRefUpdatesExist());
+ ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto set_snapshot,
transaction_->NewSetSnapshot());
+ set_snapshot->RollbackToTime(timestamp_ms);
+ ICEBERG_BUILDER_RETURN_IF_ERROR(set_snapshot->Commit());
+ return *this;
+}
+
+SnapshotManager& SnapshotManager::RollbackTo(int64_t snapshot_id) {
+ ICEBERG_BUILDER_RETURN_IF_ERROR(CommitIfRefUpdatesExist());
+ ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto set_snapshot,
transaction_->NewSetSnapshot());
+ set_snapshot->RollbackTo(snapshot_id);
+ ICEBERG_BUILDER_RETURN_IF_ERROR(set_snapshot->Commit());
+ return *this;
+}
+
+SnapshotManager& SnapshotManager::CreateBranch(const std::string& name) {
+ if (base().current_snapshot_id != kInvalidSnapshotId) {
+ ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto current_snapshot, base().Snapshot());
+ ICEBERG_DCHECK(current_snapshot != nullptr, "Current snapshot should not
be null");
+ return CreateBranch(name, current_snapshot->snapshot_id);
+ }
+ const auto& current_refs = base().refs;
+ ICEBERG_BUILDER_CHECK(!base().refs.contains(name), "Ref {} already exists",
name);
+ ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto fast_append,
transaction_->NewFastAppend());
+ ICEBERG_BUILDER_RETURN_IF_ERROR(fast_append->SetTargetBranch(name).Commit());
+ return *this;
+}
+
+SnapshotManager& SnapshotManager::CreateBranch(const std::string& name,
+ int64_t snapshot_id) {
+ ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto update_ref,
UpdateSnapshotReferencesOperation());
+ update_ref->CreateBranch(name, snapshot_id);
+ return *this;
+}
+
+SnapshotManager& SnapshotManager::CreateTag(const std::string& name,
+ int64_t snapshot_id) {
+ ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto update_ref,
UpdateSnapshotReferencesOperation());
+ update_ref->CreateTag(name, snapshot_id);
+ return *this;
+}
+
+SnapshotManager& SnapshotManager::RemoveBranch(const std::string& name) {
+ ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto update_ref,
UpdateSnapshotReferencesOperation());
+ update_ref->RemoveBranch(name);
+ return *this;
+}
+
+SnapshotManager& SnapshotManager::RemoveTag(const std::string& name) {
+ ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto update_ref,
UpdateSnapshotReferencesOperation());
+ update_ref->RemoveTag(name);
+ return *this;
+}
+
+SnapshotManager& SnapshotManager::ReplaceTag(const std::string& name,
+ int64_t snapshot_id) {
+ ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto update_ref,
UpdateSnapshotReferencesOperation());
+ update_ref->ReplaceTag(name, snapshot_id);
+ return *this;
+}
+
+SnapshotManager& SnapshotManager::ReplaceBranch(const std::string& name,
+ int64_t snapshot_id) {
+ ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto update_ref,
UpdateSnapshotReferencesOperation());
+ update_ref->ReplaceBranch(name, snapshot_id);
+ return *this;
+}
+
+SnapshotManager& SnapshotManager::ReplaceBranch(const std::string& from,
+ const std::string& to) {
+ ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto update_ref,
UpdateSnapshotReferencesOperation());
+ update_ref->ReplaceBranch(from, to);
+ return *this;
+}
+
+SnapshotManager& SnapshotManager::FastForwardBranch(const std::string& from,
+ const std::string& to) {
+ ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto update_ref,
UpdateSnapshotReferencesOperation());
+ update_ref->FastForward(from, to);
+ return *this;
+}
+
+SnapshotManager& SnapshotManager::RenameBranch(const std::string& name,
+ const std::string& new_name) {
+ ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto update_ref,
UpdateSnapshotReferencesOperation());
+ update_ref->RenameBranch(name, new_name);
+ return *this;
+}
+
+SnapshotManager& SnapshotManager::SetMinSnapshotsToKeep(const std::string&
branch_name,
+ int32_t
min_snapshots_to_keep) {
+ ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto update_ref,
UpdateSnapshotReferencesOperation());
+ update_ref->SetMinSnapshotsToKeep(branch_name, min_snapshots_to_keep);
+ return *this;
+}
+
+SnapshotManager& SnapshotManager::SetMaxSnapshotAgeMs(const std::string&
branch_name,
+ int64_t
max_snapshot_age_ms) {
+ ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto update_ref,
UpdateSnapshotReferencesOperation());
+ update_ref->SetMaxSnapshotAgeMs(branch_name, max_snapshot_age_ms);
+ return *this;
+}
+
+SnapshotManager& SnapshotManager::SetMaxRefAgeMs(const std::string& name,
+ int64_t max_ref_age_ms) {
+ ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto update_ref,
UpdateSnapshotReferencesOperation());
+ update_ref->SetMaxRefAgeMs(name, max_ref_age_ms);
+ return *this;
+}
+
+Status SnapshotManager::Commit() {
+ transaction_->EnableAutoCommit();
Review Comment:
To verify this issue, we can create a `SnapshotManager` from `Table &
Transaction` and then call `SetCurrentSnapshot` and `RollbackXXX` for multiple
times to see it they can be committed successfully.
##########
src/iceberg/transaction.h:
##########
@@ -129,14 +141,15 @@ class ICEBERG_EXPORT Transaction : public
std::enable_shared_from_this<Transacti
private:
friend class PendingUpdate;
+ friend class SnapshotManager;
Review Comment:
Is this required?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
