timsaucer commented on code in PR #1264:
URL:
https://github.com/apache/datafusion-python/pull/1264#discussion_r2424148424
##########
python/datafusion/dataframe.py:
##########
@@ -1206,3 +1265,48 @@ def fill_null(self, value: Any, subset: list[str] | None
= None) -> DataFrame:
- For columns not in subset, the original column is kept unchanged
"""
return DataFrame(self.df.fill_null(value, subset))
+
+
+class InsertOp(Enum):
+ """Insert operation mode.
+
+ These modes are used by the table writing feature to define how record
+ batches should be written to a table.
+ """
+
+ APPEND = InsertOpInternal.APPEND
+ """Appends new rows to the existing table without modifying any existing
rows."""
+
+ REPLACE = InsertOpInternal.REPLACE
+ """Replace existing rows that collide with the inserted rows.
+
+ Replacement is typically based on a unique key or primary key.
+ """
+
+ OVERWRITE = InsertOpInternal.OVERWRITE
+ """Overwrites all existing rows in the table with the new rows."""
+
+
+class DataFrameWriteOptions:
+ """Writer options for DataFrame.
+
+ There is no guarantee the table provider supports all writer options.
+ See the individual implementation and documentation for details.
+ """
+
+ def __init__(
+ self,
+ insert_operation: InsertOp | None = None,
+ single_file_output: bool = False,
+ partition_by: str | Sequence[str] | None = None,
+ sort_by: Expr | SortExpr | Sequence[Expr] | Sequence[SortExpr] | None
= None,
+ ) -> None:
+ """Instantiate writer options for DataFrame."""
+ if isinstance(partition_by, str):
+ partition_by = [partition_by]
+
+ sort_by_raw = sort_list_to_raw_sort_list(sort_by)
+
+ self._raw_write_options = DataFrameWriteOptionsInternal(
+ insert_operation, single_file_output, partition_by, sort_by_raw
+ )
Review Comment:
Very good catch. I have corrected it and added unit test coverage. It's
difficult to add more coverage for multi file output and partition by since the
built in writers don't support them.
##########
python/datafusion/dataframe.py:
##########
@@ -1206,3 +1265,48 @@ def fill_null(self, value: Any, subset: list[str] | None
= None) -> DataFrame:
- For columns not in subset, the original column is kept unchanged
"""
return DataFrame(self.df.fill_null(value, subset))
+
+
+class InsertOp(Enum):
+ """Insert operation mode.
+
+ These modes are used by the table writing feature to define how record
+ batches should be written to a table.
+ """
+
+ APPEND = InsertOpInternal.APPEND
+ """Appends new rows to the existing table without modifying any existing
rows."""
+
+ REPLACE = InsertOpInternal.REPLACE
+ """Replace existing rows that collide with the inserted rows.
+
+ Replacement is typically based on a unique key or primary key.
+ """
+
+ OVERWRITE = InsertOpInternal.OVERWRITE
+ """Overwrites all existing rows in the table with the new rows."""
+
+
+class DataFrameWriteOptions:
+ """Writer options for DataFrame.
+
+ There is no guarantee the table provider supports all writer options.
+ See the individual implementation and documentation for details.
+ """
+
+ def __init__(
+ self,
+ insert_operation: InsertOp | None = None,
+ single_file_output: bool = False,
+ partition_by: str | Sequence[str] | None = None,
+ sort_by: Expr | SortExpr | Sequence[Expr] | Sequence[SortExpr] | None
= None,
+ ) -> None:
+ """Instantiate writer options for DataFrame."""
+ if isinstance(partition_by, str):
+ partition_by = [partition_by]
+
+ sort_by_raw = sort_list_to_raw_sort_list(sort_by)
+
+ self._raw_write_options = DataFrameWriteOptionsInternal(
+ insert_operation, single_file_output, partition_by, sort_by_raw
+ )
Review Comment:
Very good catch. I have corrected it and added unit test coverage. It's
difficult to add more coverage for multi file output and partition by since I
think the built in writers don't support them.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
