mik-laj commented on a change in pull request #8807:
URL: https://github.com/apache/airflow/pull/8807#discussion_r425584091
##########
File path: backport_packages/setup_backport_packages.py
##########
@@ -422,40 +449,680 @@ def usage():
print(text)
print()
print("You can see all packages configured by specifying
list-backport-packages as first argument")
+ print()
+ print("You can generate release notes by specifying:
update-package-release-notes YYYY.MM.DD [PACKAGES]")
+ print()
+ print("You can specify additional suffix when generating packages by using
--version-suffix <SUFFIX>\n"
+ "before package name when you run sdist or bdist\n")
+
+
+def is_imported_from_same_module(the_class: str, imported_name: str) -> bool:
+ """
+ Is the class imported from another module?
+
+ :param the_class: the class object itself
+ :param imported_name: name of the imported class
+ :return: true if the class was imported from another module
+ """
+ return ".".join(imported_name.split(".")[:-1]) == the_class.__module__
+
+
+def is_example_dag(imported_name: str) -> bool:
+ """
+ Is the class an example_dag class?
+
+ :param imported_name: name where the class is imported from
+ :return: true if it is an example_dags class
+ """
+ return ".example_dags." in imported_name
+
+
+def is_from_the_expected_package(the_class: Type, expected_package: str) ->
bool:
+ """
+ Returns true if the class is from the package expected.
+ :param the_class: the class object
+ :param expected_package: package expected for the class
+ :return:
+ """
+ return the_class.__module__.startswith(expected_package)
+
+
+def inherits_from(the_class: Type, expected_ancestor: Type) -> bool:
+ """
+ Returns true if the class inherits (directly or indirectly) from the class
specified.
+ :param the_class: The class to check
+ :param expected_ancestor: expected class to inherit from
+ :return: true is the class inherits from the class expected
+ """
+ if expected_ancestor is None:
+ return False
+ import inspect
+ mro = inspect.getmro(the_class)
+ return the_class is not expected_ancestor and expected_ancestor in mro
+
+
+def is_class(the_class: Type) -> bool:
+ """
+ Returns true if the object passed is a class
+ :param the_class: the class to pass
+ :return: true if it is a class
+ """
+ import inspect
+ return inspect.isclass(the_class)
+
+
+def is_bigquery_class(imported_name: str) -> bool:
+ """
+ Returns true if the object passed is a class
+ :param imported_name: name of the class imported
+ :return: true if it is a class
+ """
+ return is_bigquery_non_dts_module(module_name=imported_name.split(".")[-2])
+
+
+def has_expected_string_in_name(the_class: Type, expected_string:
Optional[str]) -> bool:
+ """
+ In case expected_string is different than None then it checks for presence
of the string in the
+ imported_name.
+ :param the_class: name of the imported object
+ :param expected_string: string to expect
+ :return: true if the expected_string is None or the expected string is
found in the imported name
+ """
+ return expected_string is None or expected_string in the_class.__module__
+
+
+def find_all_subclasses(expected_package: str,
+ expected_ancestor: Type,
+ expected_string: Optional[str] = None,
+ exclude_class_type=None) -> Set[str]:
+ """
+ Returns set of classes containing all subclasses in package specified.
+
+ :param expected_package: full package name where to look for the classes
+ :param expected_ancestor: type of the object the method looks for
+ :param expected_string: this string is expected to appear in the package
name
+ :param exclude_class_type: exclude class of this type (Sensor are also
Operators so they should be
+ excluded from the Operator list)
+ """
+ subclasses = set()
+ for imported_name, the_class in globals().items():
+ if is_class(the_class=the_class) \
+ and not is_example_dag(imported_name=imported_name) \
+ and is_from_the_expected_package(the_class=the_class,
expected_package=expected_package) \
+ and is_imported_from_same_module(the_class=the_class,
imported_name=imported_name) \
+ and has_expected_string_in_name(the_class=the_class,
expected_string=expected_string) \
+ and inherits_from(the_class=the_class,
expected_ancestor=expected_ancestor) \
+ and not inherits_from(the_class=the_class,
expected_ancestor=exclude_class_type) \
+ and not is_bigquery_class(imported_name=imported_name):
+ subclasses.add(imported_name)
+ return subclasses
+
+
+def get_new_and_moved_classes(classes: Set[str],
+ dict_of_moved_classes: Dict[str, str]) ->
Tuple[List[str], Dict[str, str]]:
+ """
+ Splits the set of classes into new and moved, depending on their presence
in the dict of objects
+ retrieved from the test_contrib_to_core.
+
+ :param classes: set of classes found
+ :param dict_of_moved_classes: dictionary of classes that were moved from
contrib to core
+ :return:
+ """
+ new_objects = []
+ moved_objects = {}
+ for obj in classes:
+ if obj in dict_of_moved_classes:
+ moved_objects[obj] = dict_of_moved_classes[obj]
+ del dict_of_moved_classes[obj]
+ else:
+ new_objects.append(obj)
+ new_objects.sort()
+ return new_objects, moved_objects
+
+
+def strip_package_from_class(base_package: str, class_name: str) -> str:
+ """
+ Strips base package name from the class (if it starts with the package
name).
+ """
+ if class_name.startswith(base_package):
+ return class_name[len(base_package) + 1:]
+ else:
+ return class_name
+
+
+def convert_class_name_to_url(base_url: str, class_name) -> str:
+ """
+ Converts the class name to URL that the class can be reached
+
+ :param base_url: base URL to use
+ :param class_name: name of the class
+ :return: URL to the class
+ """
+ return base_url + "/".join(class_name.split(".")[:-1]) + ".py"
+
+
+def get_class_code_link(base_package: str, class_name: str, git_tag: str) ->
str:
+ """
+ Provides markdown link for the class passed as parameter.
+
+ :param base_package: base package to strip from most names
+ :param class_name: name of the class
+ :param git_tag: tag to use for the URL link
+ :return: URL to the class
+ """
+ url_prefix = f'https://github.com/apache/airflow/blob/{git_tag}/'
+ return f'[{strip_package_from_class(base_package, class_name)}]' \
+ f'({convert_class_name_to_url(url_prefix, class_name)})'
+
+
+def convert_new_classes_to_table(class_list: List[str], full_package_name:
str, class_type: str) -> str:
+ """
+ Converts new classes tp a markdown table.
+
+ :param class_list: list of classes to convert to markup
+ :param full_package_name: name of the provider package
+ :param class_type: type of classes -> operators, sensors etc.
+ :return:
+ """
+ from tabulate import tabulate
+ headers = [f"New Airflow 2.0 {class_type}: `{full_package_name}` package"]
+ table = [(get_class_code_link(full_package_name, obj, "master"),) for obj
in class_list]
+ return tabulate(table, headers=headers, tablefmt="pipe")
+
+
+def convert_moved_objects_to_table(class_dict: Dict[str, str],
+ full_package_name: str, class_type: str) ->
str:
+ """
+ Converts moved classes to a markdown table
+ :param class_dict: dictionary of classes (to -> from)
+ :param full_package_name: name of the provider package
+ :param class_type: type of classes -> operators, sensors etc.
+ :return:
+ """
+ from tabulate import tabulate
+ headers = [f"Airflow 2.0 {class_type}: `{full_package_name}` package",
+ "Airflow 1.10.* previous location (usually `airflow.contrib`)"]
+ table = [
+ (get_class_code_link(full_package_name, obj, "master"),
+ get_class_code_link("airflow.contrib", class_dict[obj],
"v1-10-stable"))
+ for obj in sorted(class_dict.keys())
+ ]
+ return tabulate(table, headers=headers, tablefmt="pipe")
+
+
+def get_package_class_summary(full_package_name: str) -> Dict[str, Any]:
+ """
+ Gets summary of the package in the form of dictionary containing all types
of classes
+ :param full_package_name: full package name
+ :return: dictionary of objects usable as context for Jinja2 templates
+ """
+ from airflow.secrets import BaseSecretsBackend
+ from airflow.sensors.base_sensor_operator import BaseSensorOperator
+ from airflow.hooks.base_hook import BaseHook
+ from airflow.models.baseoperator import BaseOperator
+ from typing_extensions import Protocol
+ operators = find_all_subclasses(expected_package=full_package_name,
+ expected_ancestor=BaseOperator,
+ expected_string=".operators.",
+ exclude_class_type=BaseSensorOperator)
+ sensors = find_all_subclasses(expected_package=full_package_name,
+ expected_ancestor=BaseSensorOperator,
+ expected_string='.sensors.')
+ hooks = find_all_subclasses(expected_package=full_package_name,
+ expected_ancestor=BaseHook,
+ expected_string='.hooks.')
+ protocols = find_all_subclasses(expected_package=full_package_name,
+ expected_ancestor=Protocol)
+ secrets = find_all_subclasses(expected_package=full_package_name,
+ expected_ancestor=BaseSecretsBackend)
+ new_operators, moved_operators = get_new_and_moved_classes(operators,
MOVED_OPERATORS_DICT)
+ new_sensors, moved_sensors = get_new_and_moved_classes(sensors,
MOVED_SENSORS_DICT)
+ new_hooks, moved_hooks = get_new_and_moved_classes(hooks, MOVED_HOOKS_DICT)
+ new_protocols, moved_protocols = get_new_and_moved_classes(protocols,
MOVED_PROTOCOLS_DICT)
+ new_secrets, moved_secrets = get_new_and_moved_classes(secrets,
MOVED_SECRETS_DICT)
+ class_summary = {
+ "NEW_OPERATORS": new_operators,
+ "MOVED_OPERATORS": moved_operators,
+ "NEW_SENSORS": new_sensors,
+ "MOVED_SENSORS": moved_sensors,
+ "NEW_HOOKS": new_hooks,
+ "MOVED_HOOKS": moved_hooks,
+ "NEW_PROTOCOLS": new_protocols,
+ "MOVED_PROTOCOLS": moved_protocols,
+ "NEW_SECRETS": new_secrets,
+ "MOVED_SECRETS": moved_secrets,
+ }
+ for from_name, to_name, object_type in [
+ ("NEW_OPERATORS", "NEW_OPERATORS_TABLE", "operators"),
+ ("NEW_SENSORS", "NEW_SENSORS_TABLE", "sensors"),
+ ("NEW_HOOKS", "NEW_HOOKS_TABLE", "hooks"),
+ ("NEW_PROTOCOLS", "NEW_PROTOCOLS_TABLE", "protocols"),
+ ("NEW_SECRETS", "NEW_SECRETS_TABLE", "secrets"),
+ ]:
+ class_summary[to_name] =
convert_new_classes_to_table(class_summary[from_name],
+
full_package_name,
+ object_type)
+ for from_name, to_name, object_type in [
+ ("MOVED_OPERATORS", "MOVED_OPERATORS_TABLE", "operators"),
+ ("MOVED_SENSORS", "MOVED_SENSORS_TABLE", "sensors"),
+ ("MOVED_HOOKS", "MOVED_HOOKS_TABLE", "hooks"),
+ ("MOVED_PROTOCOLS", "MOVED_PROTOCOLS_TABLE", "protocols"),
+ ("MOVED_SECRETS", "MOVED_SECRETS_TABLE", "protocols"),
+ ]:
+ class_summary[to_name] =
convert_moved_objects_to_table(class_summary[from_name],
+
full_package_name,
+ object_type)
+ return class_summary
+
+
+def render_template(template_name: str, context: Dict[str, Any]) -> str:
+ """
+ Renders template based on it's name. Reads the template from
<name>_TEMPLATE.md.jinja2 in current dir.
+ :param template_name: name of the template to use
+ :param context: Jinja2 context
+ :return: rendered template
+ """
+ from jinja2 import Template
+ template_file_path = os.path.join(MY_DIR_PATH,
f"{template_name}_TEMPLATE.md.jinja2")
+ with open(template_file_path, "rt") as template_file:
+ # remove comment
+ template = Template(template_file.read(), autoescape=True)
Review comment:
By default, Jinja uses a dangerous undefined value. When an undefined
variable is used, the empty value is displayed.
Here is an example of how to render templates more securely.
```python
def render_template(template_name: str, *args, **kwargs) -> str:
"""Render Jinja template"""
def render_template(template_name: str, *args, **kwargs) -> str:
"""Render Jinja template"""
template_loader = jinja2.FileSystemLoader(searchpath=TPL_PATH)
template_env = jinja2.Environment(
loader=template_loader, undefined=jinja2.StrictUndefined,
trim_blocks=True, lstrip_blocks=True
)
template = template_env.get_template(template_name)
content: str = template.render(*args, **kwargs)
return content
```
It's may be related to:
https://github.com/apache/airflow/pull/8807#discussion_r425580282 but I'm not
sure.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
