[GitHub] [airflow] Bowrna commented on a change in pull request #21145: enter the shell breeze2 environment
Bowrna commented on a change in pull request #21145:
URL: https://github.com/apache/airflow/pull/21145#discussion_r816105059
##
File path: dev/breeze/src/airflow_breeze/shell/enter_shell.py
##
@@ -0,0 +1,237 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import sys
+from pathlib import Path
+from typing import Dict
+
+import click
+from inputimeout import TimeoutOccurred, inputimeout
+
+from airflow_breeze import global_constants
+from airflow_breeze.cache import (
+check_cache_and_write_if_not_cached,
+read_from_cache_file,
+write_to_cache_file,
+)
+from airflow_breeze.ci.build_image import build_image
+from airflow_breeze.console import console
+from airflow_breeze.global_constants import (
+FLOWER_HOST_PORT,
+MSSQL_HOST_PORT,
+MSSQL_VERSION,
+MYSQL_HOST_PORT,
+MYSQL_VERSION,
+POSTGRES_HOST_PORT,
+POSTGRES_VERSION,
+REDIS_HOST_PORT,
+SSH_PORT,
+WEBSERVER_HOST_PORT,
+)
+from airflow_breeze.shell.shell_builder import ShellBuilder
+from airflow_breeze.utils.docker_command_utils import (
+check_docker_compose_version,
+check_docker_resources,
+check_docker_version,
+)
+from airflow_breeze.utils.path_utils import BUILD_CACHE_DIR
+from airflow_breeze.utils.run_utils import (
+filter_out_none,
+fix_group_permissions,
+get_latest_sha,
+instruct_build_image,
+instruct_for_setup,
+is_repo_rebased,
+md5sum_check_if_build_is_needed,
+run_command,
+)
+from airflow_breeze.visuals import ASCIIART, ASCIIART_STYLE, CHEATSHEET,
CHEATSHEET_STYLE
+
+PARAMS_TO_ENTER_SHELL = {
+"HOST_USER_ID": "host_user_id",
+"HOST_GROUP_ID": "host_group_id",
+"COMPOSE_FILE": "compose_files",
+"PYTHON_MAJOR_MINOR_VERSION": "python_version",
+"BACKEND": "backend",
+"AIRFLOW_VERSION": "airflow_version",
+"INSTALL_AIRFLOW_VERSION": "install_airflow_version",
+"AIRFLOW_SOURCES": "airflow_sources",
+"AIRFLOW_CI_IMAGE": "airflow_ci_image_name",
+"AIRFLOW_CI_IMAGE_WITH_TAG": "airflow_ci_image_name_with_tag",
+"AIRFLOW_PROD_IMAGE": "airflow_prod_image_name",
+"AIRFLOW_IMAGE_KUBERNETES": "airflow_image_kubernetes",
+"SQLITE_URL": "sqlite_url",
+"USE_AIRFLOW_VERSION": "use_airflow_version",
+"SKIP_TWINE_CHECK": "skip_twine_check",
+"USE_PACKAGES_FROM_DIST": "use_packages_from_dist",
+"EXECUTOR": "executor",
+"START_AIRFLOW": "start_airflow",
+"ENABLED_INTEGRATIONS": "enabled_integrations",
+"GITHUB_ACTIONS": "github_actions",
+"ISSUE_ID": "issue_id",
+"NUM_RUNS": "num_runs",
+"VERSION_SUFFIX_FOR_SVN": "version_suffix_for_svn",
+"VERSION_SUFFIX_FOR_PYPI": "version_suffix_for_pypi",
+}
+
+PARAMS_FOR_SHELL_CONSTANTS = {
+"SSH_PORT": SSH_PORT,
+"WEBSERVER_HOST_PORT": WEBSERVER_HOST_PORT,
+"FLOWER_HOST_PORT": FLOWER_HOST_PORT,
+"REDIS_HOST_PORT": REDIS_HOST_PORT,
+"MYSQL_HOST_PORT": MYSQL_HOST_PORT,
+"MYSQL_VERSION": MYSQL_VERSION,
+"MSSQL_HOST_PORT": MSSQL_HOST_PORT,
+"MSSQL_VERSION": MSSQL_VERSION,
+"POSTGRES_HOST_PORT": POSTGRES_HOST_PORT,
+"POSTGRES_VERSION": POSTGRES_VERSION,
+}
+
+PARAMS_IN_CACHE = {
+'python_version': 'PYTHON_MAJOR_MINOR_VERSION',
+'backend': 'BACKEND',
+'executor': 'EXECUTOR',
+'postgres_version': 'POSTGRES_VERSION',
+'mysql_version': 'MYSQL_VERSION',
+'mssql_version': 'MSSQL_VERSION',
+}
+
+DEFAULT_VALUES_FOR_PARAM = {
+'python_version': 'DEFAULT_PYTHON_MAJOR_MINOR_VERSION',
+'backend': 'DEFAULT_BACKEND',
+'executor': 'DEFAULT_EXECUTOR',
+'postgres_version': 'POSTGRES_VERSION',
+'mysql_version': 'MYSQL_VERSION',
+'mssql_version': 'MSSQL_VERSION',
+}
+
+
+def construct_env_variables_docker_compose_command(shell_params: ShellBuilder)
-> Dict[str, str]:
+env_variables: Dict[str, str] = {}
+for param_name in PARAMS_TO_ENTER_SHELL:
+param_value = PARAMS_TO_ENTER_SHELL[param_name]
+env_variables[param_name] = str(getattr(shell_params, param_value))
+for constant_param_name in PARAMS_FOR_SHELL_CONSTANTS:
+constant_param_value = PARAMS_FOR_SHELL_CONSTANTS[constant_param_name]
+env_variables[constant_param_name] = str(constant_param_value)
+return env_
[GitHub] [airflow] Bowrna commented on a change in pull request #21145: enter the shell breeze2 environment
Bowrna commented on a change in pull request #21145:
URL: https://github.com/apache/airflow/pull/21145#discussion_r816095542
##
File path: dev/breeze/src/airflow_breeze/shell/enter_shell.py
##
@@ -0,0 +1,237 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import sys
+from pathlib import Path
+from typing import Dict
+
+import click
+from inputimeout import TimeoutOccurred, inputimeout
+
+from airflow_breeze import global_constants
+from airflow_breeze.cache import (
+check_cache_and_write_if_not_cached,
+read_from_cache_file,
+write_to_cache_file,
+)
+from airflow_breeze.ci.build_image import build_image
+from airflow_breeze.console import console
+from airflow_breeze.global_constants import (
+FLOWER_HOST_PORT,
+MSSQL_HOST_PORT,
+MSSQL_VERSION,
+MYSQL_HOST_PORT,
+MYSQL_VERSION,
+POSTGRES_HOST_PORT,
+POSTGRES_VERSION,
+REDIS_HOST_PORT,
+SSH_PORT,
+WEBSERVER_HOST_PORT,
+)
+from airflow_breeze.shell.shell_builder import ShellBuilder
+from airflow_breeze.utils.docker_command_utils import (
+check_docker_compose_version,
+check_docker_resources,
+check_docker_version,
+)
+from airflow_breeze.utils.path_utils import BUILD_CACHE_DIR
+from airflow_breeze.utils.run_utils import (
+filter_out_none,
+fix_group_permissions,
+get_latest_sha,
+instruct_build_image,
+instruct_for_setup,
+is_repo_rebased,
+md5sum_check_if_build_is_needed,
+run_command,
+)
+from airflow_breeze.visuals import ASCIIART, ASCIIART_STYLE, CHEATSHEET,
CHEATSHEET_STYLE
+
+PARAMS_TO_ENTER_SHELL = {
+"HOST_USER_ID": "host_user_id",
+"HOST_GROUP_ID": "host_group_id",
+"COMPOSE_FILE": "compose_files",
+"PYTHON_MAJOR_MINOR_VERSION": "python_version",
+"BACKEND": "backend",
+"AIRFLOW_VERSION": "airflow_version",
+"INSTALL_AIRFLOW_VERSION": "install_airflow_version",
+"AIRFLOW_SOURCES": "airflow_sources",
+"AIRFLOW_CI_IMAGE": "airflow_ci_image_name",
+"AIRFLOW_CI_IMAGE_WITH_TAG": "airflow_ci_image_name_with_tag",
+"AIRFLOW_PROD_IMAGE": "airflow_prod_image_name",
+"AIRFLOW_IMAGE_KUBERNETES": "airflow_image_kubernetes",
+"SQLITE_URL": "sqlite_url",
+"USE_AIRFLOW_VERSION": "use_airflow_version",
+"SKIP_TWINE_CHECK": "skip_twine_check",
+"USE_PACKAGES_FROM_DIST": "use_packages_from_dist",
+"EXECUTOR": "executor",
+"START_AIRFLOW": "start_airflow",
+"ENABLED_INTEGRATIONS": "enabled_integrations",
+"GITHUB_ACTIONS": "github_actions",
+"ISSUE_ID": "issue_id",
+"NUM_RUNS": "num_runs",
+"VERSION_SUFFIX_FOR_SVN": "version_suffix_for_svn",
+"VERSION_SUFFIX_FOR_PYPI": "version_suffix_for_pypi",
+}
+
+PARAMS_FOR_SHELL_CONSTANTS = {
+"SSH_PORT": SSH_PORT,
+"WEBSERVER_HOST_PORT": WEBSERVER_HOST_PORT,
+"FLOWER_HOST_PORT": FLOWER_HOST_PORT,
+"REDIS_HOST_PORT": REDIS_HOST_PORT,
+"MYSQL_HOST_PORT": MYSQL_HOST_PORT,
+"MYSQL_VERSION": MYSQL_VERSION,
+"MSSQL_HOST_PORT": MSSQL_HOST_PORT,
+"MSSQL_VERSION": MSSQL_VERSION,
+"POSTGRES_HOST_PORT": POSTGRES_HOST_PORT,
+"POSTGRES_VERSION": POSTGRES_VERSION,
+}
+
+PARAMS_IN_CACHE = {
+'python_version': 'PYTHON_MAJOR_MINOR_VERSION',
+'backend': 'BACKEND',
+'executor': 'EXECUTOR',
+'postgres_version': 'POSTGRES_VERSION',
+'mysql_version': 'MYSQL_VERSION',
+'mssql_version': 'MSSQL_VERSION',
+}
+
+DEFAULT_VALUES_FOR_PARAM = {
+'python_version': 'DEFAULT_PYTHON_MAJOR_MINOR_VERSION',
+'backend': 'DEFAULT_BACKEND',
+'executor': 'DEFAULT_EXECUTOR',
+'postgres_version': 'POSTGRES_VERSION',
+'mysql_version': 'MYSQL_VERSION',
+'mssql_version': 'MSSQL_VERSION',
+}
+
+
+def construct_env_variables_docker_compose_command(shell_params: ShellBuilder)
-> Dict[str, str]:
+env_variables: Dict[str, str] = {}
+for param_name in PARAMS_TO_ENTER_SHELL:
+param_value = PARAMS_TO_ENTER_SHELL[param_name]
+env_variables[param_name] = str(getattr(shell_params, param_value))
+for constant_param_name in PARAMS_FOR_SHELL_CONSTANTS:
+constant_param_value = PARAMS_FOR_SHELL_CONSTANTS[constant_param_name]
+env_variables[constant_param_name] = str(constant_param_value)
+return env_
[GitHub] [airflow] Bowrna commented on a change in pull request #21145: enter the shell breeze2 environment
Bowrna commented on a change in pull request #21145:
URL: https://github.com/apache/airflow/pull/21145#discussion_r815436090
##
File path: dev/breeze/src/airflow_breeze/breeze.py
##
@@ -75,19 +111,78 @@ def version():
@option_verbose
-@main.command()
-def shell(verbose: bool):
+@main.command(
+context_settings=dict(
+ignore_unknown_options=True,
+allow_extra_args=True,
+),
+)
+@option_python_version
+@option_backend
+@click.option('--integration', type=click.Choice(ALLOWED_INTEGRATIONS),
multiple=True)
+@click.option('-L', '--build-cache-local', is_flag=True)
+@click.option('-U', '--build-cache-pulled', is_flag=True)
+@click.option('-X', '--build-cache-disabled', is_flag=True)
+@click.option('--production-image', is_flag=True)
Review comment:
Let me try first. If i didn't get success, i will get your help.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@airflow.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [airflow] Bowrna commented on a change in pull request #21145: enter the shell breeze2 environment
Bowrna commented on a change in pull request #21145:
URL: https://github.com/apache/airflow/pull/21145#discussion_r815429816
##
File path: dev/breeze/src/airflow_breeze/breeze.py
##
@@ -75,19 +111,78 @@ def version():
@option_verbose
-@main.command()
-def shell(verbose: bool):
+@main.command(
+context_settings=dict(
+ignore_unknown_options=True,
+allow_extra_args=True,
+),
+)
+@option_python_version
+@option_backend
+@click.option('--integration', type=click.Choice(ALLOWED_INTEGRATIONS),
multiple=True)
+@click.option('-L', '--build-cache-local', is_flag=True)
+@click.option('-U', '--build-cache-pulled', is_flag=True)
+@click.option('-X', '--build-cache-disabled', is_flag=True)
+@click.option('--production-image', is_flag=True)
Review comment:
@potiuk wrt [this](https://github.com/apache/airflow/pull/21848) issue,
we don't need the `--production-image` flag and any process related to that
flag, am I right?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@airflow.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [airflow] Bowrna commented on a change in pull request #21145: enter the shell breeze2 environment
Bowrna commented on a change in pull request #21145:
URL: https://github.com/apache/airflow/pull/21145#discussion_r815295778
##
File path: dev/breeze/src/airflow_breeze/shell/enter_shell.py
##
@@ -0,0 +1,199 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import sys
+from pathlib import Path
+from typing import Dict
+
+import click
+
+from airflow_breeze import global_constants
+from airflow_breeze.cache import (
+check_cache_and_write_if_not_cached,
+read_from_cache_file,
+write_to_cache_file,
+)
+from airflow_breeze.ci.build_image import build_image
+from airflow_breeze.console import console
+from airflow_breeze.global_constants import (
+FLOWER_HOST_PORT,
+MSSQL_HOST_PORT,
+MSSQL_VERSION,
+MYSQL_HOST_PORT,
+MYSQL_VERSION,
+POSTGRES_HOST_PORT,
+POSTGRES_VERSION,
+REDIS_HOST_PORT,
+SSH_PORT,
+WEBSERVER_HOST_PORT,
+)
+from airflow_breeze.shell.shell_builder import ShellBuilder
+from airflow_breeze.utils.docker_command_utils import (
+check_docker_compose_version,
+check_docker_resources,
+check_docker_version,
+)
+from airflow_breeze.utils.path_utils import BUILD_CACHE_DIR
+from airflow_breeze.utils.run_utils import (
+filter_out_none,
+fix_group_permissions,
+instruct_build_image,
+instruct_for_setup,
+md5sum_check_if_build_is_needed,
+run_command,
+)
+from airflow_breeze.visuals import ASCIIART, ASCIIART_STYLE, CHEATSHEET,
CHEATSHEET_STYLE
+
+PARAMS_TO_ENTER_SHELL = {
+"HOST_USER_ID": "host_user_id",
+"HOST_GROUP_ID": "host_group_id",
+"COMPOSE_FILE": "compose_files",
+"PYTHON_MAJOR_MINOR_VERSION": "python_version",
+"BACKEND": "backend",
+"AIRFLOW_VERSION": "airflow_version",
+"INSTALL_AIRFLOW_VERSION": "install_airflow_version",
+"AIRFLOW_SOURCES": "airflow_sources",
+"AIRFLOW_CI_IMAGE": "airflow_ci_image_name",
+"AIRFLOW_CI_IMAGE_WITH_TAG": "airflow_ci_image_name_with_tag",
+"AIRFLOW_PROD_IMAGE": "airflow_prod_image_name",
+"AIRFLOW_IMAGE_KUBERNETES": "airflow_image_kubernetes",
+"SQLITE_URL": "sqlite_url",
+"USE_AIRFLOW_VERSION": "use_airflow_version",
+"SKIP_TWINE_CHECK": "skip_twine_check",
+"USE_PACKAGES_FROM_DIST": "use_packages_from_dist",
+"EXECUTOR": "executor",
+"START_AIRFLOW": "start_airflow",
+"ENABLED_INTEGRATIONS": "enabled_integrations",
+"GITHUB_ACTIONS": "github_actions",
+"ISSUE_ID": "issue_id",
+"NUM_RUNS": "num_runs",
+"VERSION_SUFFIX_FOR_SVN": "version_suffix_for_svn",
+"VERSION_SUFFIX_FOR_PYPI": "version_suffix_for_pypi",
+}
+
+PARAMS_FOR_SHELL_CONSTANTS = {
+"SSH_PORT": SSH_PORT,
+"WEBSERVER_HOST_PORT": WEBSERVER_HOST_PORT,
+"FLOWER_HOST_PORT": FLOWER_HOST_PORT,
+"REDIS_HOST_PORT": REDIS_HOST_PORT,
+"MYSQL_HOST_PORT": MYSQL_HOST_PORT,
+"MYSQL_VERSION": MYSQL_VERSION,
+"MSSQL_HOST_PORT": MSSQL_HOST_PORT,
+"MSSQL_VERSION": MSSQL_VERSION,
+"POSTGRES_HOST_PORT": POSTGRES_HOST_PORT,
+"POSTGRES_VERSION": POSTGRES_VERSION,
+}
+
+PARAMS_IN_CACHE = {
+'python_version': 'PYTHON_MAJOR_MINOR_VERSION',
+'backend': 'BACKEND',
+'executor': 'EXECUTOR',
+'postgres_version': 'POSTGRES_VERSION',
+'mysql_version': 'MYSQL_VERSION',
+'mssql_version': 'MSSQL_VERSION',
+}
+
+DEFAULT_VALUES_FOR_PARAM = {
+'python_version': 'DEFAULT_PYTHON_MAJOR_MINOR_VERSION',
+'backend': 'DEFAULT_BACKEND',
+'executor': 'DEFAULT_EXECUTOR',
+'postgres_version': 'POSTGRES_VERSION',
+'mysql_version': 'MYSQL_VERSION',
+'mssql_version': 'MSSQL_VERSION',
+}
+
+
+def construct_env_variables_docker_compose_command(shell_params: ShellBuilder)
-> Dict[str, str]:
+env_variables: Dict[str, str] = {}
+for param_name in PARAMS_TO_ENTER_SHELL:
+param_value = PARAMS_TO_ENTER_SHELL[param_name]
+env_variables[param_name] = str(getattr(shell_params, param_value))
+for constant_param_name in PARAMS_FOR_SHELL_CONSTANTS:
+constant_param_value = PARAMS_FOR_SHELL_CONSTANTS[constant_param_name]
+env_variables[constant_param_name] = str(constant_param_value)
+return env_variables
+
+
+def build_image_checks(verbose: bool, shell_params: ShellBuilder):
+fix_group_
[GitHub] [airflow] Bowrna commented on a change in pull request #21145: enter the shell breeze2 environment
Bowrna commented on a change in pull request #21145: URL: https://github.com/apache/airflow/pull/21145#discussion_r813834570 ## File path: dev/breeze/tests/test_run_utils.py ## @@ -0,0 +1,24 @@ +from pathlib import Path +from unittest import mock +import stat +import os +import pytest +from airflow_breeze.utils.run_utils import change_directory_permission, change_file_permission + Review comment: @potiuk Please share your reviews about this test code that i tried. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@airflow.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [airflow] Bowrna commented on a change in pull request #21145: enter the shell breeze2 environment
Bowrna commented on a change in pull request #21145: URL: https://github.com/apache/airflow/pull/21145#discussion_r812797300 ## File path: dev/breeze/tests/test_cache.py ## @@ -62,6 +67,17 @@ def test_read_from_cache_file(param): if param_value is None: assert None is param_value else: -allowed, param_list = check_if_values_allowed(param + 's', param_value) +allowed, param_list = check_if_values_allowed(param, param_value) if allowed: assert param_value in param_list + + Review comment: @potiuk Could you share your views on the above question too? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@airflow.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [airflow] Bowrna commented on a change in pull request #21145: enter the shell breeze2 environment
Bowrna commented on a change in pull request #21145:
URL: https://github.com/apache/airflow/pull/21145#discussion_r812722254
##
File path: dev/breeze/src/airflow_breeze/utils/run_utils.py
##
@@ -73,3 +97,110 @@ def check_package_installed(package_name: str) -> bool:
Please install using https://pre-commit.com/#install to
continue[/]\n"
)
return is_installed
+
+
+def get_filesystem_type(filepath):
+root_type = "unknown"
+for part in psutil.disk_partitions():
+if part.mountpoint == '/':
+root_type = part.fstype
+continue
+if filepath.startswith(part.mountpoint):
+return part.fstype
+
+return root_type
+
+
+def calculate_md5_checksum_for_files(md5sum_cache_dir: Path):
+not_modified_files = []
+modified_files = []
+for calculate_md5_file in FILES_FOR_REBUILD_CHECK:
+file_to_get_md5 = Path(AIRFLOW_SOURCE, calculate_md5_file)
+md5_checksum = generate_md5(file_to_get_md5)
+sub_dir_name = file_to_get_md5.parts[-2]
+actual_file_name = file_to_get_md5.parts[-1]
+cache_file_name = Path(md5sum_cache_dir, sub_dir_name + '-' +
actual_file_name + '.md5sum')
+file_content = md5_checksum + ' ' + str(file_to_get_md5) + '\n'
+is_modified = update_md5checksum_in_cache(file_content,
cache_file_name)
+if is_modified:
+modified_files.append(calculate_md5_file)
+else:
+not_modified_files.append(calculate_md5_file)
+return modified_files, not_modified_files
+
+
+def md5sum_check_if_build_is_needed(md5sum_cache_dir: Path, the_image_type:
str) -> bool:
+build_needed = False
+modified_files, not_modified_files =
calculate_md5_checksum_for_files(md5sum_cache_dir)
+if len(modified_files) > 0:
+console.print('The following files are modified: ', modified_files)
+console.print(f'Likely {the_image_type} image needs rebuild')
+build_needed = True
+else:
+console.print(
+f'Docker image build is not needed for {the_image_type} build as
no important files are changed!'
+)
+return build_needed
+
+
+def instruct_build_image(the_image_type: str, python_version: str):
+console.print(f'The {the_image_type} image for python version
{python_version} may be outdated')
+console.print('Please run this command at earliest convenience:')
+if the_image_type == 'CI':
+console.print(f'./Breeze2 build-ci-image --python {python_version}')
+else:
+console.print(f'./Breeze2 build-prod-image --python {python_version}')
+console.print("If you run it via pre-commit as individual hook, you can
run 'pre-commit run build'.")
+
+
+def instruct_for_setup():
+CMDNAME = 'Breeze2'
+console.print(f"You can setup autocomplete by running {CMDNAME}
setup-autocomplete'")
+console.print(" You can toggle ascii/cheatsheet by running:")
+console.print(f" * {CMDNAME} toggle-suppress-cheatsheet")
+console.print(f" * {CMDNAME} toggle-suppress-asciiart")
+
+
+@contextlib.contextmanager
+def working_directory(source_path: Path):
+# Equivalent of pushd and popd in bash script.
+# https://stackoverflow.com/a/42441759/3101838
+prev_cwd = Path.cwd()
+os.chdir(source_path)
+try:
+yield
+finally:
+os.chdir(prev_cwd)
+
+
+def change_file_permission(file_to_fix: Path):
Review comment:
@potiuk Could you tell me if i can use mock to test the
`change_file_permission` part too?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@airflow.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [airflow] Bowrna commented on a change in pull request #21145: enter the shell breeze2 environment
Bowrna commented on a change in pull request #21145: URL: https://github.com/apache/airflow/pull/21145#discussion_r812673596 ## File path: dev/breeze/tests/test_cache.py ## @@ -62,6 +67,17 @@ def test_read_from_cache_file(param): if param_value is None: assert None is param_value else: -allowed, param_list = check_if_values_allowed(param + 's', param_value) +allowed, param_list = check_if_values_allowed(param, param_value) if allowed: assert param_value in param_list + + Review comment: @potiuk Could you share your views on this test that I tried? In the below code, If i tried `mock_path.assert_called_with(AIRFLOW_SOURCES / ".build"/ param)` it didn't work. Can you explain why it doesn't work? I tried to find myself. But that part is not very clear to me -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@airflow.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [airflow] Bowrna commented on a change in pull request #21145: enter the shell breeze2 environment
Bowrna commented on a change in pull request #21145:
URL: https://github.com/apache/airflow/pull/21145#discussion_r811960847
##
File path: dev/breeze/tests/test_docker_command_utils.py
##
@@ -0,0 +1,173 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from unittest import mock
+from unittest.mock import call
+
+from airflow_breeze.utils.docker_command_utils import
check_docker_compose_version, check_docker_version
+
+
+@mock.patch('airflow_breeze.utils.docker_command_utils.run_command')
+@mock.patch('airflow_breeze.utils.docker_command_utils.console')
+def test_check_docker_version_unknown(mock_console, mock_run_command):
+check_docker_version(verbose=True)
+expected_run_command_calls = [
+call(['docker', 'info'], verbose=True, suppress_console_print=True,
capture_output=True, text=True),
+call(
+['docker', 'version', '--format', '{{.Client.Version}}'],
+verbose=True,
+suppress_console_print=True,
+capture_output=True,
+text=True,
+),
+]
+
+mock_run_command.assert_has_calls(expected_run_command_calls)
+mock_console.print.assert_called_with(
+"Your version of docker is unknown. If the scripts fail, please make
sure to"
+" install docker at least: 20.10.0 version."
+)
+
+
+@mock.patch('airflow_breeze.utils.docker_command_utils.check_docker_permission')
+@mock.patch('airflow_breeze.utils.docker_command_utils.run_command')
+@mock.patch('airflow_breeze.utils.docker_command_utils.console')
+def test_check_docker_version_too_low(mock_console, mock_run_command,
mock_check_docker_permission):
+mock_check_docker_permission.return_value = False
+mock_run_command.return_value.returncode = 0
+mock_run_command.return_value.stdout = "0.9"
+check_docker_version(verbose=True)
+mock_check_docker_permission.assert_called_with(True)
+mock_run_command.assert_called_with(
+['docker', 'version', '--format', '{{.Client.Version}}'],
+verbose=True,
+suppress_console_print=True,
+capture_output=True,
+text=True,
+)
+mock_console.print.assert_called_with(
+"Your version of docker is too old:0.9. Please upgrade to
at least 20.10.0"
+)
+
+
+@mock.patch('airflow_breeze.utils.docker_command_utils.check_docker_permission')
+@mock.patch('airflow_breeze.utils.docker_command_utils.run_command')
+@mock.patch('airflow_breeze.utils.docker_command_utils.console')
+def test_check_docker_version_ok(mock_console, mock_run_command,
mock_check_docker_permission):
+mock_check_docker_permission.return_value = False
+mock_run_command.return_value.returncode = 0
+mock_run_command.return_value.stdout = "20.10.0"
+check_docker_version(verbose=True)
+mock_check_docker_permission.assert_called_with(True)
+mock_run_command.assert_called_with(
+['docker', 'version', '--format', '{{.Client.Version}}'],
+verbose=True,
+suppress_console_print=True,
+capture_output=True,
+text=True,
+)
+mock_console.print.assert_called_with("Good version of Docker: 20.10.0.")
+
+
+@mock.patch('airflow_breeze.utils.docker_command_utils.check_docker_permission')
+@mock.patch('airflow_breeze.utils.docker_command_utils.run_command')
+@mock.patch('airflow_breeze.utils.docker_command_utils.console')
+def test_check_docker_version_higher(mock_console, mock_run_command,
mock_check_docker_permission):
+mock_check_docker_permission.return_value = False
+mock_run_command.return_value.returncode = 0
+mock_run_command.return_value.stdout = "21.10.0"
+check_docker_version(verbose=True)
+mock_check_docker_permission.assert_called_with(True)
+mock_run_command.assert_called_with(
+['docker', 'version', '--format', '{{.Client.Version}}'],
+verbose=True,
+suppress_console_print=True,
+capture_output=True,
+text=True,
+)
+mock_console.print.assert_called_with("Good version of Docker: 21.10.0.")
+
+
+@mock.patch('airflow_breeze.utils.docker_command_utils.run_command')
+@mock.patch('airflow_breeze.utils.docker_command_utils.console')
+def test_che
[GitHub] [airflow] Bowrna commented on a change in pull request #21145: enter the shell breeze2 environment
Bowrna commented on a change in pull request #21145:
URL: https://github.com/apache/airflow/pull/21145#discussion_r811960847
##
File path: dev/breeze/tests/test_docker_command_utils.py
##
@@ -0,0 +1,173 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from unittest import mock
+from unittest.mock import call
+
+from airflow_breeze.utils.docker_command_utils import
check_docker_compose_version, check_docker_version
+
+
+@mock.patch('airflow_breeze.utils.docker_command_utils.run_command')
+@mock.patch('airflow_breeze.utils.docker_command_utils.console')
+def test_check_docker_version_unknown(mock_console, mock_run_command):
+check_docker_version(verbose=True)
+expected_run_command_calls = [
+call(['docker', 'info'], verbose=True, suppress_console_print=True,
capture_output=True, text=True),
+call(
+['docker', 'version', '--format', '{{.Client.Version}}'],
+verbose=True,
+suppress_console_print=True,
+capture_output=True,
+text=True,
+),
+]
+
+mock_run_command.assert_has_calls(expected_run_command_calls)
+mock_console.print.assert_called_with(
+"Your version of docker is unknown. If the scripts fail, please make
sure to"
+" install docker at least: 20.10.0 version."
+)
+
+
+@mock.patch('airflow_breeze.utils.docker_command_utils.check_docker_permission')
+@mock.patch('airflow_breeze.utils.docker_command_utils.run_command')
+@mock.patch('airflow_breeze.utils.docker_command_utils.console')
+def test_check_docker_version_too_low(mock_console, mock_run_command,
mock_check_docker_permission):
+mock_check_docker_permission.return_value = False
+mock_run_command.return_value.returncode = 0
+mock_run_command.return_value.stdout = "0.9"
+check_docker_version(verbose=True)
+mock_check_docker_permission.assert_called_with(True)
+mock_run_command.assert_called_with(
+['docker', 'version', '--format', '{{.Client.Version}}'],
+verbose=True,
+suppress_console_print=True,
+capture_output=True,
+text=True,
+)
+mock_console.print.assert_called_with(
+"Your version of docker is too old:0.9. Please upgrade to
at least 20.10.0"
+)
+
+
+@mock.patch('airflow_breeze.utils.docker_command_utils.check_docker_permission')
+@mock.patch('airflow_breeze.utils.docker_command_utils.run_command')
+@mock.patch('airflow_breeze.utils.docker_command_utils.console')
+def test_check_docker_version_ok(mock_console, mock_run_command,
mock_check_docker_permission):
+mock_check_docker_permission.return_value = False
+mock_run_command.return_value.returncode = 0
+mock_run_command.return_value.stdout = "20.10.0"
+check_docker_version(verbose=True)
+mock_check_docker_permission.assert_called_with(True)
+mock_run_command.assert_called_with(
+['docker', 'version', '--format', '{{.Client.Version}}'],
+verbose=True,
+suppress_console_print=True,
+capture_output=True,
+text=True,
+)
+mock_console.print.assert_called_with("Good version of Docker: 20.10.0.")
+
+
+@mock.patch('airflow_breeze.utils.docker_command_utils.check_docker_permission')
+@mock.patch('airflow_breeze.utils.docker_command_utils.run_command')
+@mock.patch('airflow_breeze.utils.docker_command_utils.console')
+def test_check_docker_version_higher(mock_console, mock_run_command,
mock_check_docker_permission):
+mock_check_docker_permission.return_value = False
+mock_run_command.return_value.returncode = 0
+mock_run_command.return_value.stdout = "21.10.0"
+check_docker_version(verbose=True)
+mock_check_docker_permission.assert_called_with(True)
+mock_run_command.assert_called_with(
+['docker', 'version', '--format', '{{.Client.Version}}'],
+verbose=True,
+suppress_console_print=True,
+capture_output=True,
+text=True,
+)
+mock_console.print.assert_called_with("Good version of Docker: 21.10.0.")
+
+
+@mock.patch('airflow_breeze.utils.docker_command_utils.run_command')
+@mock.patch('airflow_breeze.utils.docker_command_utils.console')
+def test_che
[GitHub] [airflow] Bowrna commented on a change in pull request #21145: enter the shell breeze2 environment
Bowrna commented on a change in pull request #21145:
URL: https://github.com/apache/airflow/pull/21145#discussion_r811626590
##
File path: dev/breeze/tests/test_docker_command_utils.py
##
@@ -0,0 +1,173 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from unittest import mock
+from unittest.mock import call
+
+from airflow_breeze.utils.docker_command_utils import
check_docker_compose_version, check_docker_version
+
+
+@mock.patch('airflow_breeze.utils.docker_command_utils.run_command')
+@mock.patch('airflow_breeze.utils.docker_command_utils.console')
+def test_check_docker_version_unknown(mock_console, mock_run_command):
+check_docker_version(verbose=True)
+expected_run_command_calls = [
+call(['docker', 'info'], verbose=True, suppress_console_print=True,
capture_output=True, text=True),
+call(
+['docker', 'version', '--format', '{{.Client.Version}}'],
+verbose=True,
+suppress_console_print=True,
+capture_output=True,
+text=True,
+),
+]
+
+mock_run_command.assert_has_calls(expected_run_command_calls)
+mock_console.print.assert_called_with(
+"Your version of docker is unknown. If the scripts fail, please make
sure to"
+" install docker at least: 20.10.0 version."
+)
+
+
+@mock.patch('airflow_breeze.utils.docker_command_utils.check_docker_permission')
+@mock.patch('airflow_breeze.utils.docker_command_utils.run_command')
+@mock.patch('airflow_breeze.utils.docker_command_utils.console')
+def test_check_docker_version_too_low(mock_console, mock_run_command,
mock_check_docker_permission):
+mock_check_docker_permission.return_value = False
+mock_run_command.return_value.returncode = 0
+mock_run_command.return_value.stdout = "0.9"
+check_docker_version(verbose=True)
+mock_check_docker_permission.assert_called_with(True)
+mock_run_command.assert_called_with(
+['docker', 'version', '--format', '{{.Client.Version}}'],
+verbose=True,
+suppress_console_print=True,
+capture_output=True,
+text=True,
+)
+mock_console.print.assert_called_with(
+"Your version of docker is too old:0.9. Please upgrade to
at least 20.10.0"
+)
+
+
+@mock.patch('airflow_breeze.utils.docker_command_utils.check_docker_permission')
+@mock.patch('airflow_breeze.utils.docker_command_utils.run_command')
+@mock.patch('airflow_breeze.utils.docker_command_utils.console')
+def test_check_docker_version_ok(mock_console, mock_run_command,
mock_check_docker_permission):
+mock_check_docker_permission.return_value = False
+mock_run_command.return_value.returncode = 0
+mock_run_command.return_value.stdout = "20.10.0"
+check_docker_version(verbose=True)
+mock_check_docker_permission.assert_called_with(True)
+mock_run_command.assert_called_with(
+['docker', 'version', '--format', '{{.Client.Version}}'],
+verbose=True,
+suppress_console_print=True,
+capture_output=True,
+text=True,
+)
+mock_console.print.assert_called_with("Good version of Docker: 20.10.0.")
+
+
+@mock.patch('airflow_breeze.utils.docker_command_utils.check_docker_permission')
+@mock.patch('airflow_breeze.utils.docker_command_utils.run_command')
+@mock.patch('airflow_breeze.utils.docker_command_utils.console')
+def test_check_docker_version_higher(mock_console, mock_run_command,
mock_check_docker_permission):
+mock_check_docker_permission.return_value = False
+mock_run_command.return_value.returncode = 0
+mock_run_command.return_value.stdout = "21.10.0"
+check_docker_version(verbose=True)
+mock_check_docker_permission.assert_called_with(True)
+mock_run_command.assert_called_with(
+['docker', 'version', '--format', '{{.Client.Version}}'],
+verbose=True,
+suppress_console_print=True,
+capture_output=True,
+text=True,
+)
+mock_console.print.assert_called_with("Good version of Docker: 21.10.0.")
+
+
+@mock.patch('airflow_breeze.utils.docker_command_utils.run_command')
+@mock.patch('airflow_breeze.utils.docker_command_utils.console')
+def test_che
[GitHub] [airflow] Bowrna commented on a change in pull request #21145: enter the shell breeze2 environment
Bowrna commented on a change in pull request #21145:
URL: https://github.com/apache/airflow/pull/21145#discussion_r811287573
##
File path: dev/breeze/tests/test_docker_command_utils.py
##
@@ -0,0 +1,173 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from unittest import mock
+from unittest.mock import call
+
+from airflow_breeze.utils.docker_command_utils import
check_docker_compose_version, check_docker_version
+
+
+@mock.patch('airflow_breeze.utils.docker_command_utils.run_command')
+@mock.patch('airflow_breeze.utils.docker_command_utils.console')
+def test_check_docker_version_unknown(mock_console, mock_run_command):
+check_docker_version(verbose=True)
+expected_run_command_calls = [
+call(['docker', 'info'], verbose=True, suppress_console_print=True,
capture_output=True, text=True),
+call(
+['docker', 'version', '--format', '{{.Client.Version}}'],
+verbose=True,
+suppress_console_print=True,
+capture_output=True,
+text=True,
+),
+]
+
+mock_run_command.assert_has_calls(expected_run_command_calls)
+mock_console.print.assert_called_with(
+"Your version of docker is unknown. If the scripts fail, please make
sure to"
+" install docker at least: 20.10.0 version."
+)
+
+
+@mock.patch('airflow_breeze.utils.docker_command_utils.check_docker_permission')
+@mock.patch('airflow_breeze.utils.docker_command_utils.run_command')
+@mock.patch('airflow_breeze.utils.docker_command_utils.console')
+def test_check_docker_version_too_low(mock_console, mock_run_command,
mock_check_docker_permission):
+mock_check_docker_permission.return_value = False
+mock_run_command.return_value.returncode = 0
+mock_run_command.return_value.stdout = "0.9"
+check_docker_version(verbose=True)
+mock_check_docker_permission.assert_called_with(True)
+mock_run_command.assert_called_with(
+['docker', 'version', '--format', '{{.Client.Version}}'],
+verbose=True,
+suppress_console_print=True,
+capture_output=True,
+text=True,
+)
+mock_console.print.assert_called_with(
+"Your version of docker is too old:0.9. Please upgrade to
at least 20.10.0"
+)
+
+
+@mock.patch('airflow_breeze.utils.docker_command_utils.check_docker_permission')
+@mock.patch('airflow_breeze.utils.docker_command_utils.run_command')
+@mock.patch('airflow_breeze.utils.docker_command_utils.console')
+def test_check_docker_version_ok(mock_console, mock_run_command,
mock_check_docker_permission):
+mock_check_docker_permission.return_value = False
+mock_run_command.return_value.returncode = 0
+mock_run_command.return_value.stdout = "20.10.0"
+check_docker_version(verbose=True)
+mock_check_docker_permission.assert_called_with(True)
+mock_run_command.assert_called_with(
+['docker', 'version', '--format', '{{.Client.Version}}'],
+verbose=True,
+suppress_console_print=True,
+capture_output=True,
+text=True,
+)
+mock_console.print.assert_called_with("Good version of Docker: 20.10.0.")
+
+
+@mock.patch('airflow_breeze.utils.docker_command_utils.check_docker_permission')
+@mock.patch('airflow_breeze.utils.docker_command_utils.run_command')
+@mock.patch('airflow_breeze.utils.docker_command_utils.console')
+def test_check_docker_version_higher(mock_console, mock_run_command,
mock_check_docker_permission):
+mock_check_docker_permission.return_value = False
+mock_run_command.return_value.returncode = 0
+mock_run_command.return_value.stdout = "21.10.0"
+check_docker_version(verbose=True)
+mock_check_docker_permission.assert_called_with(True)
+mock_run_command.assert_called_with(
+['docker', 'version', '--format', '{{.Client.Version}}'],
+verbose=True,
+suppress_console_print=True,
+capture_output=True,
+text=True,
+)
+mock_console.print.assert_called_with("Good version of Docker: 21.10.0.")
+
+
+@mock.patch('airflow_breeze.utils.docker_command_utils.run_command')
+@mock.patch('airflow_breeze.utils.docker_command_utils.console')
+def test_che
[GitHub] [airflow] Bowrna commented on a change in pull request #21145: enter the shell breeze2 environment
Bowrna commented on a change in pull request #21145:
URL: https://github.com/apache/airflow/pull/21145#discussion_r810231042
##
File path: dev/breeze/src/airflow_breeze/ci/build_params.py
##
@@ -106,8 +116,8 @@ def commit_sha(self):
def docker_cache_ci_directive(self) -> List:
docker_cache_ci_directive = []
if self.docker_cache == "pulled":
-docker_cache_ci_directive.append("--cache-from")
-docker_cache_ci_directive.append(self.airflow_ci_image_name)
+
docker_cache_ci_directive.append(f"--cache-from={self.airflow_ci_image_name_with_cache}")
Review comment:
@potiuk Changes in cache-from flag in docker command construction
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@airflow.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [airflow] Bowrna commented on a change in pull request #21145: enter the shell breeze2 environment
Bowrna commented on a change in pull request #21145:
URL: https://github.com/apache/airflow/pull/21145#discussion_r810230692
##
File path: dev/breeze/src/airflow_breeze/ci/build_image.py
##
@@ -83,11 +83,17 @@ def construct_docker_command(ci_image: BuildParams) ->
List[str]:
def build_image(verbose, **kwargs):
ci_image_params = BuildParams(**filter_out_none(**kwargs))
+ci_image_cache_dir = Path(BUILD_CACHE_DIR, ci_image_params.airflow_branch)
+ci_image_cache_dir.mkdir(parents=True, exist_ok=True)
+touch_cache_file(
+f"built_{ci_image_params.python_version}",
+root_dir=ci_image_cache_dir,
+)
is_cached, value = check_cache_and_write_if_not_cached(
"PYTHON_MAJOR_MINOR_VERSION", ci_image_params.python_version
)
if is_cached:
ci_image_params.python_version = value
cmd = construct_docker_command(ci_image_params)
-output = run_command(cmd, verbose=verbose, text=True)
+output = run_command(cmd, verbose=verbose, cwd=AIRFLOW_SOURCE, text=True)
Review comment:
@potiuk Changes done for CWD in subprocess
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@airflow.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [airflow] Bowrna commented on a change in pull request #21145: enter the shell breeze2 environment
Bowrna commented on a change in pull request #21145:
URL: https://github.com/apache/airflow/pull/21145#discussion_r805174798
##
File path: dev/breeze/src/airflow_breeze/shell/enter_shell.py
##
@@ -0,0 +1,176 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from pathlib import Path
+from typing import Dict, List
+
+from airflow_breeze import global_constants
+from airflow_breeze.cache import (
+check_cache_and_write_if_not_cached,
+read_from_cache_file,
+write_to_cache_file,
+)
+from airflow_breeze.console import console
+from airflow_breeze.global_constants import (
+FLOWER_HOST_PORT,
+MSSQL_HOST_PORT,
+MSSQL_VERSION,
+MYSQL_HOST_PORT,
+MYSQL_VERSION,
+POSTGRES_HOST_PORT,
+POSTGRES_VERSION,
+REDIS_HOST_PORT,
+SSH_PORT,
+WEBSERVER_HOST_PORT,
+)
+from airflow_breeze.shell.shell_builder import ShellBuilder
+from airflow_breeze.utils.docker_command_utils import (
+check_docker_compose_version,
+check_docker_resources,
+check_docker_version,
+)
+from airflow_breeze.utils.path_utils import BUILD_CACHE_DIR
+from airflow_breeze.utils.run_utils import (
+filter_out_none,
+instruct_for_setup,
+md5sum_check_if_build_is_needed,
+run_command,
+)
+from airflow_breeze.visuals import ASCIIART, ASCIIART_STYLE, CHEATSHEET,
CHEATSHEET_STYLE
+
+PARAMS_TO_ENTER_SHELL = {
+"HOST_USER_ID": "host_user_id",
+"HOST_GROUP_ID": "host_group_id",
+"COMPOSE_FILE": "compose_files",
+"PYTHON_MAJOR_MINOR_VERSION": "python_version",
+"BACKEND": "backend",
+"AIRFLOW_VERSION": "airflow_version",
+"INSTALL_AIRFLOW_VERSION": "install_airflow_version",
+"AIRFLOW_SOURCES": "airflow_sources",
+"AIRFLOW_CI_IMAGE": "airflow_ci_image_name",
+"AIRFLOW_CI_IMAGE_WITH_TAG": "airflow_ci_image_name_with_tag",
+"AIRFLOW_PROD_IMAGE": "airflow_prod_image_name",
+"AIRFLOW_IMAGE_KUBERNETES": "airflow_image_kubernetes",
+"SQLITE_URL": "sqlite_url",
+"USE_AIRFLOW_VERSION": "use_airflow_version",
+"SKIP_TWINE_CHECK": "skip_twine_check",
+"USE_PACKAGES_FROM_DIST": "use_packages_from_dist",
+"EXECUTOR": "executor",
+"START_AIRFLOW": "start_airflow",
+"ENABLED_INTEGRATIONS": "enabled_integrations",
+"GITHUB_ACTIONS": "github_actions",
+"ISSUE_ID": "issue_id",
+"NUM_RUNS": "num_runs",
+"VERSION_SUFFIX_FOR_SVN": "version_suffix_for_svn",
+"VERSION_SUFFIX_FOR_PYPI": "version_suffix_for_pypi",
+}
+
+PARAMS_FOR_SHELL_CONSTANTS = {
+"SSH_PORT": SSH_PORT,
+"WEBSERVER_HOST_PORT": WEBSERVER_HOST_PORT,
+"FLOWER_HOST_PORT": FLOWER_HOST_PORT,
+"REDIS_HOST_PORT": REDIS_HOST_PORT,
+"MYSQL_HOST_PORT": MYSQL_HOST_PORT,
+"MYSQL_VERSION": MYSQL_VERSION,
+"MSSQL_HOST_PORT": MSSQL_HOST_PORT,
+"MSSQL_VERSION": MSSQL_VERSION,
+"POSTGRES_HOST_PORT": POSTGRES_HOST_PORT,
+"POSTGRES_VERSION": POSTGRES_VERSION,
+}
+
+PARAMS_IN_CACHE = {
+'python_version': 'PYTHON_MAJOR_MINOR_VERSION',
+'backend': 'BACKEND',
+'executor': 'EXECUTOR',
+'postgres_version': 'POSTGRES_VERSION',
+'mysql_version': 'MYSQL_VERSION',
+'mssql_version': 'MSSQL_VERSION',
+}
+
+DEFAULT_VALUES_FOR_PARAM = {
+'python_version': 'DEFAULT_PYTHON_MAJOR_MINOR_VERSION',
+'backend': 'DEFAULT_BACKEND',
+'executor': 'DEFAULT_EXECUTOR',
+'postgres_version': 'POSTGRES_VERSION',
+'mysql_version': 'MYSQL_VERSION',
+'mssql_version': 'MSSQL_VERSION',
+}
+
+
+def construct_arguments_docker_compose_command(shell_params: ShellBuilder) ->
List[str]:
+args_command = []
+for param_name in PARAMS_TO_ENTER_SHELL:
+param_value = PARAMS_TO_ENTER_SHELL[param_name]
+args_command.append("-e")
+args_command.append(param_name + '=' + str(getattr(shell_params,
param_value)))
+for constant_param_name in PARAMS_FOR_SHELL_CONSTANTS:
+constant_param_value = PARAMS_FOR_SHELL_CONSTANTS[constant_param_name]
+args_command.append("-e")
+args_command.append(constant_param_name + '=' +
str(constant_param_value))
+return args_command
+
+
Review comment:
@potiuk this is how I tried to implement running docker-compose command
with `-e` flags.
##
File path: dev/breez
[GitHub] [airflow] Bowrna commented on a change in pull request #21145: enter the shell breeze2 environment
Bowrna commented on a change in pull request #21145:
URL: https://github.com/apache/airflow/pull/21145#discussion_r805338267
##
File path: dev/breeze/src/airflow_breeze/shell/enter_shell.py
##
@@ -0,0 +1,176 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from pathlib import Path
+from typing import Dict, List
+
+from airflow_breeze import global_constants
+from airflow_breeze.cache import (
+check_cache_and_write_if_not_cached,
+read_from_cache_file,
+write_to_cache_file,
+)
+from airflow_breeze.console import console
+from airflow_breeze.global_constants import (
+FLOWER_HOST_PORT,
+MSSQL_HOST_PORT,
+MSSQL_VERSION,
+MYSQL_HOST_PORT,
+MYSQL_VERSION,
+POSTGRES_HOST_PORT,
+POSTGRES_VERSION,
+REDIS_HOST_PORT,
+SSH_PORT,
+WEBSERVER_HOST_PORT,
+)
+from airflow_breeze.shell.shell_builder import ShellBuilder
+from airflow_breeze.utils.docker_command_utils import (
+check_docker_compose_version,
+check_docker_resources,
+check_docker_version,
+)
+from airflow_breeze.utils.path_utils import BUILD_CACHE_DIR
+from airflow_breeze.utils.run_utils import (
+filter_out_none,
+instruct_for_setup,
+md5sum_check_if_build_is_needed,
+run_command,
+)
+from airflow_breeze.visuals import ASCIIART, ASCIIART_STYLE, CHEATSHEET,
CHEATSHEET_STYLE
+
+PARAMS_TO_ENTER_SHELL = {
+"HOST_USER_ID": "host_user_id",
+"HOST_GROUP_ID": "host_group_id",
+"COMPOSE_FILE": "compose_files",
+"PYTHON_MAJOR_MINOR_VERSION": "python_version",
+"BACKEND": "backend",
+"AIRFLOW_VERSION": "airflow_version",
+"INSTALL_AIRFLOW_VERSION": "install_airflow_version",
+"AIRFLOW_SOURCES": "airflow_sources",
+"AIRFLOW_CI_IMAGE": "airflow_ci_image_name",
+"AIRFLOW_CI_IMAGE_WITH_TAG": "airflow_ci_image_name_with_tag",
+"AIRFLOW_PROD_IMAGE": "airflow_prod_image_name",
+"AIRFLOW_IMAGE_KUBERNETES": "airflow_image_kubernetes",
+"SQLITE_URL": "sqlite_url",
+"USE_AIRFLOW_VERSION": "use_airflow_version",
+"SKIP_TWINE_CHECK": "skip_twine_check",
+"USE_PACKAGES_FROM_DIST": "use_packages_from_dist",
+"EXECUTOR": "executor",
+"START_AIRFLOW": "start_airflow",
+"ENABLED_INTEGRATIONS": "enabled_integrations",
+"GITHUB_ACTIONS": "github_actions",
+"ISSUE_ID": "issue_id",
+"NUM_RUNS": "num_runs",
+"VERSION_SUFFIX_FOR_SVN": "version_suffix_for_svn",
+"VERSION_SUFFIX_FOR_PYPI": "version_suffix_for_pypi",
+}
+
+PARAMS_FOR_SHELL_CONSTANTS = {
+"SSH_PORT": SSH_PORT,
+"WEBSERVER_HOST_PORT": WEBSERVER_HOST_PORT,
+"FLOWER_HOST_PORT": FLOWER_HOST_PORT,
+"REDIS_HOST_PORT": REDIS_HOST_PORT,
+"MYSQL_HOST_PORT": MYSQL_HOST_PORT,
+"MYSQL_VERSION": MYSQL_VERSION,
+"MSSQL_HOST_PORT": MSSQL_HOST_PORT,
+"MSSQL_VERSION": MSSQL_VERSION,
+"POSTGRES_HOST_PORT": POSTGRES_HOST_PORT,
+"POSTGRES_VERSION": POSTGRES_VERSION,
+}
+
+PARAMS_IN_CACHE = {
+'python_version': 'PYTHON_MAJOR_MINOR_VERSION',
+'backend': 'BACKEND',
+'executor': 'EXECUTOR',
+'postgres_version': 'POSTGRES_VERSION',
+'mysql_version': 'MYSQL_VERSION',
+'mssql_version': 'MSSQL_VERSION',
+}
+
+DEFAULT_VALUES_FOR_PARAM = {
+'python_version': 'DEFAULT_PYTHON_MAJOR_MINOR_VERSION',
+'backend': 'DEFAULT_BACKEND',
+'executor': 'DEFAULT_EXECUTOR',
+'postgres_version': 'POSTGRES_VERSION',
+'mysql_version': 'MYSQL_VERSION',
+'mssql_version': 'MSSQL_VERSION',
+}
+
+
+def construct_arguments_docker_compose_command(shell_params: ShellBuilder) ->
List[str]:
+args_command = []
+for param_name in PARAMS_TO_ENTER_SHELL:
+param_value = PARAMS_TO_ENTER_SHELL[param_name]
+args_command.append("-e")
+args_command.append(param_name + '=' + str(getattr(shell_params,
param_value)))
+for constant_param_name in PARAMS_FOR_SHELL_CONSTANTS:
+constant_param_value = PARAMS_FOR_SHELL_CONSTANTS[constant_param_name]
+args_command.append("-e")
+args_command.append(constant_param_name + '=' +
str(constant_param_value))
+return args_command
+
+
Review comment:
ok @potiuk i have added this part.
--
This is an automated message from the Apache Git Service.
To respond to the mes
[GitHub] [airflow] Bowrna commented on a change in pull request #21145: enter the shell breeze2 environment
Bowrna commented on a change in pull request #21145:
URL: https://github.com/apache/airflow/pull/21145#discussion_r805295828
##
File path: dev/breeze/src/airflow_breeze/shell/enter_shell.py
##
@@ -0,0 +1,176 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from pathlib import Path
+from typing import Dict, List
+
+from airflow_breeze import global_constants
+from airflow_breeze.cache import (
+check_cache_and_write_if_not_cached,
+read_from_cache_file,
+write_to_cache_file,
+)
+from airflow_breeze.console import console
+from airflow_breeze.global_constants import (
+FLOWER_HOST_PORT,
+MSSQL_HOST_PORT,
+MSSQL_VERSION,
+MYSQL_HOST_PORT,
+MYSQL_VERSION,
+POSTGRES_HOST_PORT,
+POSTGRES_VERSION,
+REDIS_HOST_PORT,
+SSH_PORT,
+WEBSERVER_HOST_PORT,
+)
+from airflow_breeze.shell.shell_builder import ShellBuilder
+from airflow_breeze.utils.docker_command_utils import (
+check_docker_compose_version,
+check_docker_resources,
+check_docker_version,
+)
+from airflow_breeze.utils.path_utils import BUILD_CACHE_DIR
+from airflow_breeze.utils.run_utils import (
+filter_out_none,
+instruct_for_setup,
+md5sum_check_if_build_is_needed,
+run_command,
+)
+from airflow_breeze.visuals import ASCIIART, ASCIIART_STYLE, CHEATSHEET,
CHEATSHEET_STYLE
+
+PARAMS_TO_ENTER_SHELL = {
+"HOST_USER_ID": "host_user_id",
+"HOST_GROUP_ID": "host_group_id",
+"COMPOSE_FILE": "compose_files",
+"PYTHON_MAJOR_MINOR_VERSION": "python_version",
+"BACKEND": "backend",
+"AIRFLOW_VERSION": "airflow_version",
+"INSTALL_AIRFLOW_VERSION": "install_airflow_version",
+"AIRFLOW_SOURCES": "airflow_sources",
+"AIRFLOW_CI_IMAGE": "airflow_ci_image_name",
+"AIRFLOW_CI_IMAGE_WITH_TAG": "airflow_ci_image_name_with_tag",
+"AIRFLOW_PROD_IMAGE": "airflow_prod_image_name",
+"AIRFLOW_IMAGE_KUBERNETES": "airflow_image_kubernetes",
+"SQLITE_URL": "sqlite_url",
+"USE_AIRFLOW_VERSION": "use_airflow_version",
+"SKIP_TWINE_CHECK": "skip_twine_check",
+"USE_PACKAGES_FROM_DIST": "use_packages_from_dist",
+"EXECUTOR": "executor",
+"START_AIRFLOW": "start_airflow",
+"ENABLED_INTEGRATIONS": "enabled_integrations",
+"GITHUB_ACTIONS": "github_actions",
+"ISSUE_ID": "issue_id",
+"NUM_RUNS": "num_runs",
+"VERSION_SUFFIX_FOR_SVN": "version_suffix_for_svn",
+"VERSION_SUFFIX_FOR_PYPI": "version_suffix_for_pypi",
+}
+
+PARAMS_FOR_SHELL_CONSTANTS = {
+"SSH_PORT": SSH_PORT,
+"WEBSERVER_HOST_PORT": WEBSERVER_HOST_PORT,
+"FLOWER_HOST_PORT": FLOWER_HOST_PORT,
+"REDIS_HOST_PORT": REDIS_HOST_PORT,
+"MYSQL_HOST_PORT": MYSQL_HOST_PORT,
+"MYSQL_VERSION": MYSQL_VERSION,
+"MSSQL_HOST_PORT": MSSQL_HOST_PORT,
+"MSSQL_VERSION": MSSQL_VERSION,
+"POSTGRES_HOST_PORT": POSTGRES_HOST_PORT,
+"POSTGRES_VERSION": POSTGRES_VERSION,
+}
+
+PARAMS_IN_CACHE = {
+'python_version': 'PYTHON_MAJOR_MINOR_VERSION',
+'backend': 'BACKEND',
+'executor': 'EXECUTOR',
+'postgres_version': 'POSTGRES_VERSION',
+'mysql_version': 'MYSQL_VERSION',
+'mssql_version': 'MSSQL_VERSION',
+}
+
+DEFAULT_VALUES_FOR_PARAM = {
+'python_version': 'DEFAULT_PYTHON_MAJOR_MINOR_VERSION',
+'backend': 'DEFAULT_BACKEND',
+'executor': 'DEFAULT_EXECUTOR',
+'postgres_version': 'POSTGRES_VERSION',
+'mysql_version': 'MYSQL_VERSION',
+'mssql_version': 'MSSQL_VERSION',
+}
+
+
+def construct_arguments_docker_compose_command(shell_params: ShellBuilder) ->
List[str]:
+args_command = []
+for param_name in PARAMS_TO_ENTER_SHELL:
+param_value = PARAMS_TO_ENTER_SHELL[param_name]
+args_command.append("-e")
+args_command.append(param_name + '=' + str(getattr(shell_params,
param_value)))
+for constant_param_name in PARAMS_FOR_SHELL_CONSTANTS:
+constant_param_value = PARAMS_FOR_SHELL_CONSTANTS[constant_param_name]
+args_command.append("-e")
+args_command.append(constant_param_name + '=' +
str(constant_param_value))
+return args_command
+
+
Review comment:
@potiuk Do you think it's better to log all the env variable used during
this docker-compose command?
--
This is an
[GitHub] [airflow] Bowrna commented on a change in pull request #21145: enter the shell breeze2 environment
Bowrna commented on a change in pull request #21145:
URL: https://github.com/apache/airflow/pull/21145#discussion_r805291826
##
File path: dev/breeze/src/airflow_breeze/shell/enter_shell.py
##
@@ -0,0 +1,176 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from pathlib import Path
+from typing import Dict, List
+
+from airflow_breeze import global_constants
+from airflow_breeze.cache import (
+check_cache_and_write_if_not_cached,
+read_from_cache_file,
+write_to_cache_file,
+)
+from airflow_breeze.console import console
+from airflow_breeze.global_constants import (
+FLOWER_HOST_PORT,
+MSSQL_HOST_PORT,
+MSSQL_VERSION,
+MYSQL_HOST_PORT,
+MYSQL_VERSION,
+POSTGRES_HOST_PORT,
+POSTGRES_VERSION,
+REDIS_HOST_PORT,
+SSH_PORT,
+WEBSERVER_HOST_PORT,
+)
+from airflow_breeze.shell.shell_builder import ShellBuilder
+from airflow_breeze.utils.docker_command_utils import (
+check_docker_compose_version,
+check_docker_resources,
+check_docker_version,
+)
+from airflow_breeze.utils.path_utils import BUILD_CACHE_DIR
+from airflow_breeze.utils.run_utils import (
+filter_out_none,
+instruct_for_setup,
+md5sum_check_if_build_is_needed,
+run_command,
+)
+from airflow_breeze.visuals import ASCIIART, ASCIIART_STYLE, CHEATSHEET,
CHEATSHEET_STYLE
+
+PARAMS_TO_ENTER_SHELL = {
+"HOST_USER_ID": "host_user_id",
+"HOST_GROUP_ID": "host_group_id",
+"COMPOSE_FILE": "compose_files",
+"PYTHON_MAJOR_MINOR_VERSION": "python_version",
+"BACKEND": "backend",
+"AIRFLOW_VERSION": "airflow_version",
+"INSTALL_AIRFLOW_VERSION": "install_airflow_version",
+"AIRFLOW_SOURCES": "airflow_sources",
+"AIRFLOW_CI_IMAGE": "airflow_ci_image_name",
+"AIRFLOW_CI_IMAGE_WITH_TAG": "airflow_ci_image_name_with_tag",
+"AIRFLOW_PROD_IMAGE": "airflow_prod_image_name",
+"AIRFLOW_IMAGE_KUBERNETES": "airflow_image_kubernetes",
+"SQLITE_URL": "sqlite_url",
+"USE_AIRFLOW_VERSION": "use_airflow_version",
+"SKIP_TWINE_CHECK": "skip_twine_check",
+"USE_PACKAGES_FROM_DIST": "use_packages_from_dist",
+"EXECUTOR": "executor",
+"START_AIRFLOW": "start_airflow",
+"ENABLED_INTEGRATIONS": "enabled_integrations",
+"GITHUB_ACTIONS": "github_actions",
+"ISSUE_ID": "issue_id",
+"NUM_RUNS": "num_runs",
+"VERSION_SUFFIX_FOR_SVN": "version_suffix_for_svn",
+"VERSION_SUFFIX_FOR_PYPI": "version_suffix_for_pypi",
+}
+
+PARAMS_FOR_SHELL_CONSTANTS = {
+"SSH_PORT": SSH_PORT,
+"WEBSERVER_HOST_PORT": WEBSERVER_HOST_PORT,
+"FLOWER_HOST_PORT": FLOWER_HOST_PORT,
+"REDIS_HOST_PORT": REDIS_HOST_PORT,
+"MYSQL_HOST_PORT": MYSQL_HOST_PORT,
+"MYSQL_VERSION": MYSQL_VERSION,
+"MSSQL_HOST_PORT": MSSQL_HOST_PORT,
+"MSSQL_VERSION": MSSQL_VERSION,
+"POSTGRES_HOST_PORT": POSTGRES_HOST_PORT,
+"POSTGRES_VERSION": POSTGRES_VERSION,
+}
+
+PARAMS_IN_CACHE = {
+'python_version': 'PYTHON_MAJOR_MINOR_VERSION',
+'backend': 'BACKEND',
+'executor': 'EXECUTOR',
+'postgres_version': 'POSTGRES_VERSION',
+'mysql_version': 'MYSQL_VERSION',
+'mssql_version': 'MSSQL_VERSION',
+}
+
+DEFAULT_VALUES_FOR_PARAM = {
+'python_version': 'DEFAULT_PYTHON_MAJOR_MINOR_VERSION',
+'backend': 'DEFAULT_BACKEND',
+'executor': 'DEFAULT_EXECUTOR',
+'postgres_version': 'POSTGRES_VERSION',
+'mysql_version': 'MYSQL_VERSION',
+'mssql_version': 'MSSQL_VERSION',
+}
+
+
+def construct_arguments_docker_compose_command(shell_params: ShellBuilder) ->
List[str]:
+args_command = []
+for param_name in PARAMS_TO_ENTER_SHELL:
+param_value = PARAMS_TO_ENTER_SHELL[param_name]
+args_command.append("-e")
+args_command.append(param_name + '=' + str(getattr(shell_params,
param_value)))
+for constant_param_name in PARAMS_FOR_SHELL_CONSTANTS:
+constant_param_value = PARAMS_FOR_SHELL_CONSTANTS[constant_param_name]
+args_command.append("-e")
+args_command.append(constant_param_name + '=' +
str(constant_param_value))
+return args_command
+
+
Review comment:
sure @potiuk I will first solve the issue by fixing it in os.environ and
then handle the verbose case too. thanks
--
[GitHub] [airflow] Bowrna commented on a change in pull request #21145: enter the shell breeze2 environment
Bowrna commented on a change in pull request #21145:
URL: https://github.com/apache/airflow/pull/21145#discussion_r805174798
##
File path: dev/breeze/src/airflow_breeze/shell/enter_shell.py
##
@@ -0,0 +1,176 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from pathlib import Path
+from typing import Dict, List
+
+from airflow_breeze import global_constants
+from airflow_breeze.cache import (
+check_cache_and_write_if_not_cached,
+read_from_cache_file,
+write_to_cache_file,
+)
+from airflow_breeze.console import console
+from airflow_breeze.global_constants import (
+FLOWER_HOST_PORT,
+MSSQL_HOST_PORT,
+MSSQL_VERSION,
+MYSQL_HOST_PORT,
+MYSQL_VERSION,
+POSTGRES_HOST_PORT,
+POSTGRES_VERSION,
+REDIS_HOST_PORT,
+SSH_PORT,
+WEBSERVER_HOST_PORT,
+)
+from airflow_breeze.shell.shell_builder import ShellBuilder
+from airflow_breeze.utils.docker_command_utils import (
+check_docker_compose_version,
+check_docker_resources,
+check_docker_version,
+)
+from airflow_breeze.utils.path_utils import BUILD_CACHE_DIR
+from airflow_breeze.utils.run_utils import (
+filter_out_none,
+instruct_for_setup,
+md5sum_check_if_build_is_needed,
+run_command,
+)
+from airflow_breeze.visuals import ASCIIART, ASCIIART_STYLE, CHEATSHEET,
CHEATSHEET_STYLE
+
+PARAMS_TO_ENTER_SHELL = {
+"HOST_USER_ID": "host_user_id",
+"HOST_GROUP_ID": "host_group_id",
+"COMPOSE_FILE": "compose_files",
+"PYTHON_MAJOR_MINOR_VERSION": "python_version",
+"BACKEND": "backend",
+"AIRFLOW_VERSION": "airflow_version",
+"INSTALL_AIRFLOW_VERSION": "install_airflow_version",
+"AIRFLOW_SOURCES": "airflow_sources",
+"AIRFLOW_CI_IMAGE": "airflow_ci_image_name",
+"AIRFLOW_CI_IMAGE_WITH_TAG": "airflow_ci_image_name_with_tag",
+"AIRFLOW_PROD_IMAGE": "airflow_prod_image_name",
+"AIRFLOW_IMAGE_KUBERNETES": "airflow_image_kubernetes",
+"SQLITE_URL": "sqlite_url",
+"USE_AIRFLOW_VERSION": "use_airflow_version",
+"SKIP_TWINE_CHECK": "skip_twine_check",
+"USE_PACKAGES_FROM_DIST": "use_packages_from_dist",
+"EXECUTOR": "executor",
+"START_AIRFLOW": "start_airflow",
+"ENABLED_INTEGRATIONS": "enabled_integrations",
+"GITHUB_ACTIONS": "github_actions",
+"ISSUE_ID": "issue_id",
+"NUM_RUNS": "num_runs",
+"VERSION_SUFFIX_FOR_SVN": "version_suffix_for_svn",
+"VERSION_SUFFIX_FOR_PYPI": "version_suffix_for_pypi",
+}
+
+PARAMS_FOR_SHELL_CONSTANTS = {
+"SSH_PORT": SSH_PORT,
+"WEBSERVER_HOST_PORT": WEBSERVER_HOST_PORT,
+"FLOWER_HOST_PORT": FLOWER_HOST_PORT,
+"REDIS_HOST_PORT": REDIS_HOST_PORT,
+"MYSQL_HOST_PORT": MYSQL_HOST_PORT,
+"MYSQL_VERSION": MYSQL_VERSION,
+"MSSQL_HOST_PORT": MSSQL_HOST_PORT,
+"MSSQL_VERSION": MSSQL_VERSION,
+"POSTGRES_HOST_PORT": POSTGRES_HOST_PORT,
+"POSTGRES_VERSION": POSTGRES_VERSION,
+}
+
+PARAMS_IN_CACHE = {
+'python_version': 'PYTHON_MAJOR_MINOR_VERSION',
+'backend': 'BACKEND',
+'executor': 'EXECUTOR',
+'postgres_version': 'POSTGRES_VERSION',
+'mysql_version': 'MYSQL_VERSION',
+'mssql_version': 'MSSQL_VERSION',
+}
+
+DEFAULT_VALUES_FOR_PARAM = {
+'python_version': 'DEFAULT_PYTHON_MAJOR_MINOR_VERSION',
+'backend': 'DEFAULT_BACKEND',
+'executor': 'DEFAULT_EXECUTOR',
+'postgres_version': 'POSTGRES_VERSION',
+'mysql_version': 'MYSQL_VERSION',
+'mssql_version': 'MSSQL_VERSION',
+}
+
+
+def construct_arguments_docker_compose_command(shell_params: ShellBuilder) ->
List[str]:
+args_command = []
+for param_name in PARAMS_TO_ENTER_SHELL:
+param_value = PARAMS_TO_ENTER_SHELL[param_name]
+args_command.append("-e")
+args_command.append(param_name + '=' + str(getattr(shell_params,
param_value)))
+for constant_param_name in PARAMS_FOR_SHELL_CONSTANTS:
+constant_param_value = PARAMS_FOR_SHELL_CONSTANTS[constant_param_name]
+args_command.append("-e")
+args_command.append(constant_param_name + '=' +
str(constant_param_value))
+return args_command
+
+
Review comment:
@potiuk this is how I tried to implement running docker-compose command
with `-e` flags.
--
This is an automated mes
[GitHub] [airflow] Bowrna commented on a change in pull request #21145: enter the shell breeze2 environment
Bowrna commented on a change in pull request #21145:
URL: https://github.com/apache/airflow/pull/21145#discussion_r799265614
##
File path: dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
##
@@ -53,28 +67,175 @@
]
-def get_extra_docker_flags(all: bool, airflow_sources: str) -> List:
+def get_extra_docker_flags(all: bool, selected: bool, airflow_sources: str) ->
List:
# get_extra_docker_flags(False, str(airflow_source))
+# add verbosity
EXTRA_DOCKER_FLAGS = []
if all:
EXTRA_DOCKER_FLAGS.extend(["-v",
f"{airflow_sources}:/opt/airflow/:cached"])
-else:
+elif selected:
for flag in NECESSARY_HOST_VOLUMES:
EXTRA_DOCKER_FLAGS.extend(["-v", airflow_sources + flag])
+else:
+console.print('Skip mounting host volumes to Docker')
EXTRA_DOCKER_FLAGS.extend(["-v", f"{airflow_sources}/files:/files"])
EXTRA_DOCKER_FLAGS.extend(["-v", f"{airflow_sources}/dist:/dist"])
EXTRA_DOCKER_FLAGS.extend(["--rm"])
EXTRA_DOCKER_FLAGS.extend(["--env-file",
f"{airflow_sources}/scripts/ci/docker-compose/_docker.env"])
return EXTRA_DOCKER_FLAGS
-def check_docker_resources(
-verbose: bool, mount_all_flag: bool, airflow_sources: str,
airflow_ci_image_name: str
-):
-extra_docker_flags = get_extra_docker_flags(mount_all_flag,
airflow_sources)
+def check_docker_resources(verbose: bool, airflow_sources: str,
airflow_ci_image_name: str):
+extra_docker_flags = get_extra_docker_flags(
+MOUNT_ALL_LOCAL_SOURCES, MOUNT_SELECTED_LOCAL_SOURCES, airflow_sources
+)
cmd = []
cmd.extend(["docker", "run", "-t"])
cmd.extend(extra_docker_flags)
cmd.extend(["--entrypoint", "/bin/bash", airflow_ci_image_name])
cmd.extend(["-c", "python
/opt/airflow/scripts/in_container/run_resource_check.py"])
run_command(cmd, verbose=verbose, text=True)
+
+
+def check_docker_permission(verbose) -> bool:
+permission_denied = False
+docker_permission_command = ["docker", "info"]
+try:
+_ = run_command(
+docker_permission_command,
+verbose=verbose,
+suppress_console_print=True,
+capture_output=True,
+text=True,
+)
+except subprocess.CalledProcessError as ex:
+permission_denied = True
+if ex.stdout and 'Got permission denied while trying to connect' in
ex.stdout:
+console.print('ERROR: You have `permission denied` error when
trying to communicate with docker.')
+console.print(
+'Most likely you need to add your user to `docker` group: \
+https://docs.docker.com/ engine/install/linux-postinstall/ .'
+)
+return permission_denied
+
+
+def compare_version(current_version: str, min_version: str) -> bool:
+return version.parse(current_version) > version.parse(min_version)
+
+
+def check_docker_version(verbose: bool):
+permission_denied = check_docker_permission(verbose)
+if not permission_denied:
+docker_version_command = ['docker', 'version', '--format',
'{{.Client.Version}}']
+docker_version = ''
+docker_version_output = run_command(
+docker_version_command,
+verbose=verbose,
+suppress_console_print=True,
+capture_output=True,
+text=True,
+)
+if docker_version_output.returncode == 0:
+docker_version = docker_version_output.stdout.strip()
+if docker_version == '':
+console.print(
+f'Your version of docker is unknown. If the scripts fail,
please make sure to \
+install docker at least: {MIN_DOCKER_VERSION} version.'
+)
+else:
+good_version = compare_version(docker_version, MIN_DOCKER_VERSION)
+if good_version:
+console.print(f'Good version of Docker: {docker_version}.')
+else:
+console.print(
+f'Your version of docker is too old:{docker_version}.
Please upgrade to \
+at least ${MIN_DOCKER_VERSION}'
+)
+
+
+def check_docker_compose_version(verbose: bool):
+version_pattern = re.compile(r'(\d+)\.(\d+)\.(\d+)')
+docker_compose_version_command = ["docker-compose", "--version"]
+docker_compose_version_output = run_command(
+docker_compose_version_command,
+verbose=verbose,
+suppress_console_print=True,
+capture_output=True,
+text=True,
+)
+if docker_compose_version_output.returncode == 0:
+docker_compose_version = docker_compose_version_output.stdout
+version_extracted = version_pattern.search(docker_compose_version)
+if version_extracted is not None:
+version = '.'.join(version_extracted.groups())
+good_version = compare_version(version, MIN_DOCKER_COMPOSE_VERSION)
+if good_version
