kaxil commented on code in PR #26134:
URL: https://github.com/apache/airflow/pull/26134#discussion_r962163277
##########
airflow/lineage/__init__.py:
##########
@@ -64,33 +54,38 @@ def get_backend() -> Optional[LineageBackend]:
return None
-def _get_instance(meta: Metadata):
- """Instantiate an object from Metadata"""
- cls = import_string(meta.type_name)
- return structure(meta.data, cls)
+def _render_object(obj: Any, context: "Context") -> dict:
+ return context['ti'].task.render_template(obj, context)
-def _render_object(obj: Any, context) -> Any:
- """Renders a attr annotated object. Will set non serializable attributes
to none"""
- return structure(
- json.loads(
- ENV.from_string(json.dumps(unstructure(obj), default=lambda o:
None))
- .render(lazy_mapping_from_context(context))
- .encode('utf-8')
- ),
- type(obj),
- )
+def _deserialize(serialized: dict):
+ from airflow.serialization.serialized_objects import BaseSerialization
+ # This is only use in the worker side, so it is okay to "blindly" import
the specified class here.
+ cls = import_string(serialized['__type'])
+ return cls(**BaseSerialization.deserialize(serialized['__var']))
-def _to_dataset(obj: Any, source: str) -> Optional[Metadata]:
- """Create Metadata from attr annotated object"""
- if not attr.has(obj):
- return None
- type_name = obj.__module__ + '.' + obj.__class__.__name__
- data = unstructure(obj)
+def _serialize(objs: List[Any], source: str):
+ """Serialize an attrs-decorated class to JSON"""
+ from airflow.serialization.serialized_objects import BaseSerialization
- return Metadata(type_name, source, data)
+ for obj in objs:
+ if not attr.has(obj):
+ continue
Review Comment:
Should we remove the following line?:
https://github.com/apache/airflow/blob/6a152773d7a6ab340d87fed81a571670b36bf356/airflow/lineage/__init__.py#L170-L171
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
