Fokko commented on code in PR #349:
URL: https://github.com/apache/iceberg-rust/pull/349#discussion_r1579472707
##########
crates/iceberg/src/transaction.rs:
##########
@@ -121,6 +166,270 @@ impl<'a> Transaction<'a> {
}
}
+/// FastAppendAction is a transaction action for fast append data files to the
table.
+pub struct FastAppendAction<'a> {
+ tx: Transaction<'a>,
+
+ parent_snapshot_id: Option<i64>,
+ snapshot_id: i64,
+ schema: Schema,
+ schema_id: i32,
+ format_version: FormatVersion,
+ partition_spec: PartitionSpec,
+ key_metadata: Vec<u8>,
+
+ commit_uuid: String,
+ manifest_id: i64,
+
+ appended_data_files: Vec<DataFile>,
+}
+
+impl<'a> FastAppendAction<'a> {
+ #[allow(clippy::too_many_arguments)]
+ pub(crate) fn new(
+ tx: Transaction<'a>,
+ parent_snapshot_id: Option<i64>,
+ snapshot_id: i64,
+ schema: Schema,
+ schema_id: i32,
+ format_version: FormatVersion,
+ partition_spec: PartitionSpec,
+ key_metadata: Vec<u8>,
+ commit_uuid: String,
+ ) -> Result<Self> {
+ Ok(Self {
+ tx,
+ parent_snapshot_id,
+ snapshot_id,
+ schema,
+ schema_id,
+ format_version,
+ partition_spec,
+ key_metadata,
+ commit_uuid,
+ manifest_id: 0,
+ appended_data_files: vec![],
+ })
+ }
+
+ /// Add data files to the snapshot.
+ pub fn add_data_files(
+ &mut self,
+ data_file: impl IntoIterator<Item = DataFile>,
+ ) -> Result<&mut Self> {
+ self.appended_data_files.extend(data_file);
+ Ok(self)
+ }
+
+ fn generate_manifest_file_path(&mut self) -> String {
+ let manifest_id = self.manifest_id;
+ self.manifest_id += 1;
+ format!(
+ "{}/{}/{}-m{}.{}",
+ self.tx.table.metadata().location(),
+ META_ROOT_PATH,
+ &self.commit_uuid,
+ manifest_id,
+ DataFileFormat::Avro
+ )
+ }
+
+ async fn manifest_from_parent_snapshot(&self) -> Result<Vec<ManifestFile>>
{
+ if let Some(snapshot) = self.tx.table.metadata().current_snapshot() {
+ let manifest_list = snapshot
+ .load_manifest_list(self.tx.table.file_io(),
&self.tx.table.metadata_ref())
+ .await?;
+ let mut manifest_files =
Vec::with_capacity(manifest_list.entries().len());
+ for entry in manifest_list.entries() {
+ // From:
https://github.com/apache/iceberg-python/blob/659a951d6397ab280cae80206fe6e8e4be2d3738/pyiceberg/table/__init__.py#L2921
+ // Why we need this?
+ if entry.added_snapshot_id == self.snapshot_id {
+ continue;
+ }
+ let manifest =
entry.load_manifest(self.tx.table.file_io()).await?;
+ // Skip manifest with all delete entries.
+ if manifest.entries().iter().all(|entry| !entry.is_alive()) {
+ continue;
+ }
+ manifest_files.push(entry.clone());
+ }
+ Ok(manifest_files)
+ } else {
+ Ok(vec![])
+ }
+ }
+
+ // Write manifest file for added data files and return the ManifestFile
for ManifestList.
+ async fn manifest_for_data_file(&mut self) -> Result<ManifestFile> {
+ let appended_data_files = std::mem::take(&mut
self.appended_data_files);
+ let manifest_entries = appended_data_files
+ .into_iter()
+ .map(|data_file| {
+ ManifestEntry::builder()
+ .status(crate::spec::ManifestStatus::Added)
+ .snapshot_id(self.snapshot_id)
Review Comment:
This is different for V1 and V2. For V1 we do want to write this, for V2
typically not. This is because when the commit fails due to a conflict, we have
to rewrite the manifest-list but can-reuse the already written manifests.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org
For additional commands, e-mail: issues-h...@iceberg.apache.org
