uranusjr commented on code in PR #36085:
URL: https://github.com/apache/airflow/pull/36085#discussion_r1418634989
##########
airflow/providers/weaviate/operators/weaviate.py:
##########
@@ -51,21 +57,38 @@ def __init__(
self,
conn_id: str,
class_name: str,
- input_json: list[dict[str, Any]],
+ input_json: list[dict[str, Any]] | pd.DataFrame | None = None,
+ input_data: list[dict[str, Any]] | pd.DataFrame | None = None,
+ vector_col: str = "Vector",
**kwargs: Any,
) -> None:
self.batch_params = kwargs.pop("batch_params", {})
self.hook_params = kwargs.pop("hook_params", {})
super().__init__(**kwargs)
self.class_name = class_name
self.conn_id = conn_id
- self.input_json = input_json
+ self.vector_col = vector_col
+ self.input_data = input_data
+ if input_json:
+ warnings.warn(
+ "Passing 'input_json' to WeaviateIngestOperator is deprecated
and"
+ " you should use 'input_data' instead",
+ AirflowProviderDeprecationWarning,
+ )
+ self.input_data = input_json
+ if self.input_data is None:
+ raise ValueError("Either input_json or input_data is required")
@cached_property
def hook(self) -> WeaviateHook:
"""Return an instance of the WeaviateHook."""
return WeaviateHook(conn_id=self.conn_id, **self.hook_params)
def execute(self, context: Context) -> None:
- self.log.debug("Input json: %s", self.input_json)
- self.hook.batch_data(self.class_name, self.input_json,
**self.batch_params)
+ self.log.debug("Input data: %s", self.input_data)
+ self.hook.batch_data(
+ self.class_name,
+ self.input_data, # type: ignore
Review Comment:
I tweaked a check to make Mypy understand the attribute can never be None.
The trick is to move the check before assignment. Since Mypy infers attribute
types on assignment time, if you check after assignment, Mypy will still assume
the None case is needed because technically you can do e.g.
```python
op = WeaviateIngestOperator(input_data=[...])
op.input_data = None # Takes advantage of the None case.
op.execute({})
```
so we need to eliminate the None case before the assignment so Mypy does not
consider the None case to be valid on the attribute at all.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@airflow.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
