Fokko commented on code in PR #5201: URL: https://github.com/apache/iceberg/pull/5201#discussion_r918601035
########## python/pyiceberg/table/snapshots.py: ########## @@ -0,0 +1,94 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from enum import Enum +from typing import ( + Dict, + List, + Optional, + Union, +) + +from pydantic import Field, root_validator + +from pyiceberg.utils.iceberg_base_model import IcebergBaseModel + +OPERATION = "operation" + + +class Operation(Enum): + """Describes the operation + + Possible operation values are: + - append: Only data files were added and no files were removed. + - replace: Data and delete files were added and removed without changing table data; i.e., compaction, changing the data file format, or relocating data files. + - overwrite: Data and delete files were added and removed in a logical overwrite operation. + - delete: Data files were removed and their contents logically deleted and/or delete files were added to delete rows. + """ + + APPEND = "append" + REPLACE = "replace" + OVERWRITE = "overwrite" + DELETE = "delete" + + +class Summary(IcebergBaseModel): + """ + The snapshot summary’s operation field is used by some operations, + like snapshot expiration, to skip processing certain snapshots. + """ + + __root__: Dict[str, Union[str, Operation]] + + @root_validator + def check_operation(cls, values: Dict[str, Dict[str, Union[str, Operation]]]) -> Dict[str, Dict[str, Union[str, Operation]]]: + if operation := values["__root__"].get(OPERATION): + if isinstance(operation, str): + values["__root__"][OPERATION] = Operation(operation.lower()) + else: + raise ValueError("Operation not set") + return values + + def __init__(self, operation: Optional[Operation] = None, __root__: Dict[str, Union[str, Operation]] = None, **data): + super().__init__(__root__={"operation": operation, **data} if not __root__ else __root__) + + @property + def operation(self) -> Operation: + operation = self.__root__[OPERATION] + if isinstance(operation, Operation): + return operation + else: + # Should not happen + raise ValueError(f"Unknown type of operation: {operation}") + + @property + def additional_properties(self) -> Dict[str, str]: + return { + k: v for k, v in self.__root__.items() if k != OPERATION # type: ignore # We know that they are all string, and we don't want to check + } + + +class Snapshot(IcebergBaseModel): + snapshot_id: int = Field(alias="snapshot-id") + parent_snapshot_id: Optional[int] = Field(alias="parent-snapshot-id") + sequence_number: Optional[int] = Field(alias="sequence-number", default=None) + timestamp_ms: int = Field(alias="timestamp-ms") + manifest_list: Optional[str] = Field( + alias="manifest-list", description="Location of the snapshot's manifest list file", default=None + ) + manifests: List[str] = Field(default_factory=list, repr=False, exclude=True) Review Comment: Thanks for the historical context. Sounds like an excellent idea. I'll drop this one, and make the manifest-list mandatory. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
