This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch v1-10-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 5661dc95a7e157b612fe817057048486275d735a Author: Jarek Potiuk <jarek.pot...@polidea.com> AuthorDate: Tue Jun 9 09:33:16 2020 +0200 Improved cloud tool available in the trimmed down CI container (#9167) * Improved cloud tool available in the trimmed down CI container The tools now have shebangs which make them available for python tools. Also /opt/airflow is now mounted from the host Airflow sources which makes it possible for the tools to copy files directly to/from the sources of Airflow. It also contains one small change for Linux users - the files created by docker gcloud are created with root user so in order to fix that the directories mounted from the host are fixed when you exit the tool - their ownership is changed to be owned by the host user (cherry picked from commit de9d3401f9668c703a8f0665769fd17268f607ed) --- .pre-commit-config.yaml | 2 +- BREEZE.rst | 3 + Dockerfile.ci | 21 +---- breeze | 1 + requirements/requirements-python2.7.txt | 79 +++++++++--------- requirements/requirements-python3.5.txt | 95 +++++++++++----------- requirements/requirements-python3.6.txt | 99 ++++++++++++----------- requirements/requirements-python3.7.txt | 97 +++++++++++----------- requirements/setup-2.7.md5 | 2 +- requirements/setup-3.5.md5 | 2 +- requirements/setup-3.6.md5 | 2 +- requirements/setup-3.7.md5 | 2 +- scripts/ci/ci_docs.sh | 3 + scripts/ci/ci_fix_ownership.sh | 3 + scripts/ci/ci_flake8.sh | 6 ++ scripts/ci/ci_mypy.sh | 3 + scripts/ci/docker-compose/forward-credentials.yml | 2 + scripts/ci/docker-compose/local-prod.yml | 2 + scripts/ci/docker-compose/local.yml | 2 + scripts/ci/in_container/_in_container_utils.sh | 16 ++-- scripts/ci/libraries/_initialization.sh | 7 ++ scripts/ci/libraries/_runs.sh | 6 ++ scripts/ci/prepare_tool_scripts.sh | 64 +++++++++++++++ 23 files changed, 308 insertions(+), 211 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4979f24..7e49b77 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -60,7 +60,7 @@ repos: - id: insert-license name: Add license for all JS/CSS files files: \.(js|css)$ - exclude: ^\.github/.*$ + exclude: ^\.github/.*$|^airflow/www/static/.*|^airflow/www_rbac/static/.*$ args: - --comment-style - "/**| *| */" diff --git a/BREEZE.rst b/BREEZE.rst index aec975c..3a1a6a0 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -243,6 +243,9 @@ it is downloaded, it will stay until you remove the downloaded images from your For each of those CLI credentials are taken (automatically) from the credentials you have defined in your ${HOME} directory on host. +Those tools also have host Airflow source directory mounted in /opt/airflow path +so you can directly transfer files to/from your airflow host sources. + Those are currently installed CLIs (they are available as aliases to the docker commands): +-----------------------+----------+-------------------------------------------------+-------------------+ diff --git a/Dockerfile.ci b/Dockerfile.ci index 8035fa2..8051431 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -307,26 +307,7 @@ RUN if [[ -n "${ADDITIONAL_PYTHON_DEPS}" ]]; then \ pip install ${ADDITIONAL_PYTHON_DEPS}; \ fi -RUN \ - AWSCLI_IMAGE="amazon/aws-cli:latest" && \ - AZURECLI_IMAGE="mcr.microsoft.com/azure-cli:latest" && \ - GCLOUD_IMAGE="gcr.io/google.com/cloudsdktool/cloud-sdk:latest" && \ - echo "docker run --rm -it -v \${HOST_HOME}/.aws:/root/.aws ${AWSCLI_IMAGE} \"\$@\"" \ - > /usr/bin/aws && \ - echo "docker pull ${AWSCLI_IMAGE}" > /usr/bin/aws-update && \ - echo "docker run --rm -it -v \${HOST_HOME}/.azure:/root/.azure ${AZURECLI_IMAGE} \"\$@\"" \ - > /usr/bin/az && \ - echo "docker pull ${AZURECLI_IMAGE}" > /usr/bin/az-update && \ - echo "docker run --rm -it -v \${HOST_HOME}/.config:/root/.config ${GCLOUD_IMAGE} bq \"\$@\"" \ - > /usr/bin/bq && \ - echo "docker pull ${GCLOUD_IMAGE}" > /usr/bin/bq-update && \ - echo "docker run --rm -it -v \${HOST_HOME}/.config:/root/.config ${GCLOUD_IMAGE} gcloud \"\$@\"" \ - > /usr/bin/gcloud && \ - echo "docker pull ${GCLOUD_IMAGE}" > /usr/bin/gcloud-update && \ - echo "docker run --rm -it -v \${HOST_HOME}/.config:/root/.config ${GCLOUD_IMAGE} gsutil \"\$@\"" \ - > /usr/bin/gsutil && \ - echo "docker pull ${GCLOUD_IMAGE}" > /usr/bin/gsutil-update && \ - chmod a+x /usr/bin/aws /usr/bin/az /usr/bin/bq /usr/bin/gcloud /usr/bin/gsutil +RUN scripts/ci/prepare_tool_scripts.sh WORKDIR ${AIRFLOW_SOURCES} diff --git a/breeze b/breeze index d9cfd92..005f917 100755 --- a/breeze +++ b/breeze @@ -443,6 +443,7 @@ export POSTGRES_HOST_PORT="${POSTGRES_HOST_PORT}" export POSTGRES_VERSION="${POSTGRES_VERSION}" export MYSQL_HOST_PORT="${MYSQL_HOST_PORT}" export MYSQL_VERSION="${MYSQL_VERSION}" +export AIRFLOW_SOURCES="${AIRFLOW_SOURCES}" export MYSQL_ENCODING="${MYSQL_ENCODING}" export AIRFLOW_CI_IMAGE="${AIRFLOW_CI_IMAGE}" export AIRFOW_PROD_IMAGE="${AIRFLOW_PROD_IMAGE}" diff --git a/requirements/requirements-python2.7.txt b/requirements/requirements-python2.7.txt index 323e15a..909d880 100644 --- a/requirements/requirements-python2.7.txt +++ b/requirements/requirements-python2.7.txt @@ -8,18 +8,18 @@ Flask-Caching==1.3.3 Flask-JWT-Extended==3.24.1 Flask-Login==0.4.1 Flask-OpenID==1.2.5 -Flask-SQLAlchemy==2.4.1 +Flask-SQLAlchemy==2.4.3 Flask-WTF==0.14.3 Flask==1.1.2 JPype1==0.7.1 -JayDeBeApi==1.1.1 +JayDeBeApi==1.2.2 Jinja2==2.10.3 -Mako==1.1.2 +Mako==1.1.3 Markdown==2.6.11 MarkupSafe==1.1.1 PyHive==0.6.2 PyJWT==1.7.1 -PyNaCl==1.3.0 +PyNaCl==1.4.0 PySmbClient==0.1.5 PyYAML==5.3.1 Pygments==2.5.2 @@ -29,10 +29,10 @@ Sphinx==1.8.5 Unidecode==1.1.1 WTForms==2.3.1 Werkzeug==0.16.1 -adal==1.2.3 +adal==1.2.4 alabaster==0.7.12 alembic==1.4.2 -amqp==2.5.2 +amqp==2.6.0 analytics-python==1.2.9 ansiwrap==0.8.4 apipkg==1.5 @@ -45,14 +45,14 @@ astroid==1.6.6 atlasclient==1.0.0 atomicwrites==1.4.0 attrs==19.3.0 -aws-sam-translator==1.23.0 -aws-xray-sdk==2.5.0 +aws-sam-translator==1.24.0 +aws-xray-sdk==2.6.0 azure-common==1.1.25 azure-cosmos==3.1.2 azure-datalake-store==0.0.48 azure-mgmt-containerinstance==1.5.0 azure-mgmt-nspkg==3.0.2 -azure-mgmt-resource==9.0.0 +azure-mgmt-resource==10.0.0 azure-nspkg==3.0.2 azure-storage-blob==2.1.0 azure-storage-common==2.1.0 @@ -69,18 +69,18 @@ beautifulsoup4==4.7.1 billiard==3.6.3.0 bleach==3.1.5 blinker==1.4 -boto3==1.13.15 +boto3==1.13.25 boto==2.49.0 -botocore==1.16.15 +botocore==1.16.25 cached-property==1.5.1 cachetools==3.1.1 cassandra-driver==3.20.2 cattrs==0.9.2 -celery==4.4.2 -certifi==2020.4.5.1 +celery==4.4.5 +certifi==2020.4.5.2 cffi==1.14.0 cfgv==2.0.1 -cfn-lint==0.32.1 +cfn-lint==0.33.0 cgroupspy==0.1.6 chardet==3.0.4 click==6.7 @@ -92,7 +92,7 @@ contextdecorator==0.10.0 contextlib2==0.6.0.post1 cookies==2.2.1 coverage==5.1 -croniter==0.3.31 +croniter==0.3.32 cryptography==2.9.2 cx-Oracle==7.3.0 datadog==0.36.0 @@ -115,7 +115,7 @@ execnet==1.7.1 fastavro==0.23.4 filelock==3.0.12 flake8-colors==0.1.6 -flake8==3.8.1 +flake8==3.8.3 flaky==3.6.1 flask-swagger==0.2.13 flower==0.9.4 @@ -126,27 +126,27 @@ future-fstrings==1.2.0 future==0.18.2 futures==3.3.0 gcsfs==0.2.3 -google-api-core==1.17.0 -google-api-python-client==1.8.3 +google-api-core==1.19.1 +google-api-python-client==1.9.1 google-auth-httplib2==0.0.3 google-auth-oauthlib==0.4.1 -google-auth==1.15.0 -google-cloud-bigquery==1.24.0 +google-auth==1.16.1 +google-cloud-bigquery==1.25.0 google-cloud-bigtable==1.2.1 google-cloud-container==0.5.0 google-cloud-core==1.3.0 google-cloud-dlp==0.15.0 google-cloud-language==1.3.0 google-cloud-secret-manager==1.0.0 -google-cloud-spanner==1.16.0 +google-cloud-spanner==1.17.0 google-cloud-speech==1.3.2 google-cloud-storage==1.28.1 google-cloud-texttospeech==1.0.1 google-cloud-translate==2.0.1 google-cloud-videointelligence==1.14.0 google-cloud-vision==1.0.0 -google-resumable-media==0.5.0 -googleapis-common-protos==1.51.0 +google-resumable-media==0.5.1 +googleapis-common-protos==1.52.0 graphviz==0.14 grpc-google-iam-v1==0.12.3 grpcio-gcp==0.2.2 @@ -156,13 +156,13 @@ hdfs==2.5.8 hmsclient==0.1.1 httplib2==0.18.1 humanize==0.5.1 -hvac==0.10.2 -identify==1.4.16 +hvac==0.10.3 +identify==1.4.19 idna==2.9 ijson==2.6.1 imagesize==1.2.0 -importlib-metadata==1.6.0 -importlib-resources==1.5.0 +importlib-metadata==1.6.1 +importlib-resources==2.0.0 inflection==0.3.1 ipaddress==1.0.23 ipdb==0.13.2 @@ -184,7 +184,7 @@ jupyter-client==5.3.4 jupyter-core==4.6.3 kombu==4.6.3 kubernetes==11.0.0 -lazy-object-proxy==1.4.3 +lazy-object-proxy==1.5.0 ldap3==2.7 linecache2==1.0.0 lockfile==0.12.2 @@ -198,14 +198,15 @@ mongomock==3.19.0 monotonic==1.5 more-itertools==5.0.0 moto==1.3.14 -msrest==0.6.14 +msrest==0.6.15 msrestazure==0.6.3 multi-key-dict==2.0.3 mysqlclient==1.3.14 +natsort==6.2.1 nbconvert==5.6.1 nbformat==4.4.0 networkx==2.2 -nodeenv==1.3.5 +nodeenv==1.4.0 nteract-scrapbook==0.3.1 ntlm-auth==1.4.0 numpy==1.16.6 @@ -230,7 +231,7 @@ pre-commit==1.21.0 presto-python-client==0.7.0 prison==0.1.0 prompt-toolkit==1.0.18 -protobuf==3.12.1 +protobuf==3.12.2 psutil==5.7.0 psycopg2-binary==2.8.5 ptyprocess==0.6.0 @@ -250,26 +251,28 @@ pymssql==2.1.4 pyparsing==2.4.7 pyrsistent==0.16.0 pysftp==0.2.9 -pytest-cov==2.8.1 +pytest-cov==2.9.0 pytest-forked==1.1.3 pytest-instafail==0.4.1.post0 pytest-rerunfailures==9.0 pytest-timeout==1.3.4 pytest-xdist==1.32.0 -pytest==4.6.10 +pytest==4.6.11 python-daemon==2.1.2 python-dateutil==2.8.1 python-editor==1.0.4 python-http-client==3.2.7 python-jenkins==1.7.0 python-jose==3.1.0 +python-nvd3==0.15.0 python-openid==2.2.5 +python-slugify==4.0.0 pytz==2020.1 pytzdata==2019.3 pywinrm==0.4.1 pyzmq==19.0.1 -qds-sdk==1.15.2 -redis==3.5.2 +qds-sdk==1.16.0 +redis==3.5.3 requests-futures==0.9.4 requests-kerberos==0.12.0 requests-mock==1.8.0 @@ -309,14 +312,14 @@ tabulate==0.8.7 tenacity==4.12.0 termcolor==1.1.0 testpath==0.4.4 -text-unidecode==1.2 +text-unidecode==1.3 textwrap3==0.9.2 thrift-sasl==0.4.2 thrift==0.13.0 tokenize-rt==3.2.0 toml==0.10.1 tornado==5.1.1 -tqdm==4.46.0 +tqdm==4.46.1 traceback2==1.4.0 traitlets==4.3.3 typing-extensions==3.7.4.2 @@ -329,7 +332,7 @@ urllib3==1.25.9 vertica-python==0.10.4 vine==1.3.0 virtualenv==20.0.21 -wcwidth==0.1.9 +wcwidth==0.2.4 webencodings==0.5.1 websocket-client==0.57.0 wrapt==1.12.1 diff --git a/requirements/requirements-python3.5.txt b/requirements/requirements-python3.5.txt index bee756e..a167c60 100644 --- a/requirements/requirements-python3.5.txt +++ b/requirements/requirements-python3.5.txt @@ -8,70 +8,70 @@ Flask-Caching==1.3.3 Flask-JWT-Extended==3.24.1 Flask-Login==0.4.1 Flask-OpenID==1.2.5 -Flask-SQLAlchemy==2.4.1 +Flask-SQLAlchemy==2.4.3 Flask-WTF==0.14.3 Flask==1.1.2 JPype1==0.7.1 -JayDeBeApi==1.1.1 +JayDeBeApi==1.2.2 Jinja2==2.10.3 -Mako==1.1.2 +Mako==1.1.3 Markdown==2.6.11 MarkupSafe==1.1.1 PyHive==0.6.2 PyJWT==1.7.1 -PyNaCl==1.3.0 +PyNaCl==1.4.0 PySmbClient==0.1.5 PyYAML==5.3.1 Pygments==2.6.1 SQLAlchemy-JSONField==0.9.0 SQLAlchemy==1.3.17 -Sphinx==3.0.3 +Sphinx==3.1.0 Unidecode==1.1.1 WTForms==2.3.1 Werkzeug==0.16.1 -adal==1.2.3 +adal==1.2.4 alabaster==0.7.12 alembic==1.4.2 -amqp==2.5.2 +amqp==2.6.0 analytics-python==1.2.9 ansiwrap==0.8.4 apipkg==1.5 -apispec==3.3.0 +apispec==3.3.1 appdirs==1.4.4 argcomplete==1.11.1 asn1crypto==1.3.0 aspy.yaml==1.3.0 -astroid==2.4.1 +astroid==2.4.2 atlasclient==1.0.0 attrs==19.3.0 -aws-sam-translator==1.23.0 -aws-xray-sdk==2.5.0 +aws-sam-translator==1.24.0 +aws-xray-sdk==2.6.0 azure-common==1.1.25 azure-cosmos==3.1.2 azure-datalake-store==0.0.48 azure-mgmt-containerinstance==1.5.0 -azure-mgmt-resource==9.0.0 +azure-mgmt-resource==10.0.0 azure-nspkg==3.0.2 azure-storage-blob==2.1.0 azure-storage-common==2.1.0 azure-storage==0.36.0 -backcall==0.1.0 +backcall==0.2.0 bcrypt==3.1.7 beautifulsoup4==4.7.1 billiard==3.6.3.0 blinker==1.4 -boto3==1.13.15 +boto3==1.13.25 boto==2.49.0 -botocore==1.16.15 +botocore==1.16.25 cached-property==1.5.1 cachetools==4.1.0 cassandra-driver==3.20.2 cattrs==0.9.2 -celery==4.4.2 -certifi==2020.4.5.1 +celery==4.4.5 +certifi==2020.4.5.2 cffi==1.14.0 cfgv==2.0.1 -cfn-lint==0.32.1 +cfn-lint==0.33.0 cgroupspy==0.1.6 chardet==3.0.4 click==6.7 @@ -80,7 +80,7 @@ colorama==0.4.3 colorlog==4.0.2 configparser==3.5.3 coverage==5.1 -croniter==0.3.31 +croniter==0.3.32 cryptography==2.9.2 cx-Oracle==7.3.0 datadog==0.36.0 @@ -102,7 +102,7 @@ execnet==1.7.1 fastavro==0.23.4 filelock==3.0.12 flake8-colors==0.1.6 -flake8==3.8.1 +flake8==3.8.3 flaky==3.6.1 flask-swagger==0.2.13 flower==0.9.4 @@ -112,27 +112,27 @@ funcsigs==1.0.2 future-fstrings==1.2.0 future==0.18.2 gcsfs==0.6.2 -google-api-core==1.17.0 -google-api-python-client==1.8.3 +google-api-core==1.19.1 +google-api-python-client==1.9.1 google-auth-httplib2==0.0.3 google-auth-oauthlib==0.4.1 -google-auth==1.15.0 -google-cloud-bigquery==1.24.0 +google-auth==1.16.1 +google-cloud-bigquery==1.25.0 google-cloud-bigtable==1.2.1 google-cloud-container==0.5.0 google-cloud-core==1.3.0 google-cloud-dlp==0.15.0 google-cloud-language==1.3.0 google-cloud-secret-manager==1.0.0 -google-cloud-spanner==1.16.0 +google-cloud-spanner==1.17.0 google-cloud-speech==1.3.2 google-cloud-storage==1.28.1 google-cloud-texttospeech==1.0.1 google-cloud-translate==2.0.1 google-cloud-videointelligence==1.14.0 google-cloud-vision==1.0.0 -google-resumable-media==0.5.0 -googleapis-common-protos==1.51.0 +google-resumable-media==0.5.1 +googleapis-common-protos==1.52.0 graphviz==0.14 grpc-google-iam-v1==0.12.3 grpcio-gcp==0.2.2 @@ -142,14 +142,14 @@ hdfs==2.5.8 hmsclient==0.1.1 httplib2==0.18.1 humanize==0.5.1 -hvac==0.10.2 -identify==1.4.16 +hvac==0.10.3 +identify==1.4.19 idna==2.9 ijson==2.6.1 imagesize==1.2.0 -importlib-metadata==1.6.0 -importlib-resources==1.5.0 -inflection==0.4.0 +importlib-metadata==1.6.1 +importlib-resources==2.0.0 +inflection==0.5.0 ipdb==0.13.2 ipython-genutils==0.2.0 ipython==7.9.0 @@ -168,9 +168,9 @@ jsonschema==3.2.0 junit-xml==1.9 jupyter-client==6.1.3 jupyter-core==4.6.3 -kombu==4.6.8 +kombu==4.6.10 kubernetes==11.0.0 -lazy-object-proxy==1.4.3 +lazy-object-proxy==1.5.0 ldap3==2.7 lockfile==0.12.2 marshmallow-enum==1.5.1 @@ -181,19 +181,20 @@ mock==3.0.5 mongomock==3.19.0 more-itertools==8.3.0 moto==1.3.14 -msrest==0.6.14 +msrest==0.6.15 msrestazure==0.6.3 multi-key-dict==2.0.3 mypy-extensions==0.4.3 mypy==0.720 mysqlclient==1.3.14 +natsort==7.0.1 nbclient==0.1.0 nbformat==5.0.6 networkx==2.4 -nodeenv==1.3.5 +nodeenv==1.4.0 nteract-scrapbook==0.4.1 ntlm-auth==1.4.0 -numpy==1.18.4 +numpy==1.18.5 oauthlib==3.1.0 oscrypto==1.2.0 packaging==20.4 @@ -215,7 +216,7 @@ pre-commit==1.21.0 presto-python-client==0.7.0 prison==0.1.0 prompt-toolkit==2.0.10 -protobuf==3.12.1 +protobuf==3.12.2 psutil==5.7.0 psycopg2-binary==2.8.5 ptyprocess==0.6.0 @@ -236,26 +237,28 @@ pymssql==2.1.4 pyparsing==2.4.7 pyrsistent==0.16.0 pysftp==0.2.9 -pytest-cov==2.8.1 +pytest-cov==2.9.0 pytest-forked==1.1.3 pytest-instafail==0.4.1.post0 pytest-rerunfailures==9.0 pytest-timeout==1.3.4 pytest-xdist==1.32.0 -pytest==5.4.2 +pytest==5.4.3 python-daemon==2.1.2 python-dateutil==2.8.1 python-editor==1.0.4 python-http-client==3.2.7 python-jenkins==1.7.0 python-jose==3.1.0 +python-nvd3==0.15.0 +python-slugify==4.0.0 python3-openid==3.1.0 pytz==2020.1 pytzdata==2019.3 pywinrm==0.4.1 pyzmq==19.0.1 -qds-sdk==1.15.2 -redis==3.5.2 +qds-sdk==1.16.0 +redis==3.5.3 requests-futures==0.9.4 requests-kerberos==0.12.0 requests-mock==1.8.0 @@ -274,7 +277,7 @@ setproctitle==1.1.10 six==1.15.0 slackclient==1.3.2 snowballstemmer==2.0.0 -snowflake-connector-python==2.2.6 +snowflake-connector-python==2.2.7 snowflake-sqlalchemy==1.2.3 soupsieve==2.0.1 sphinx-argparse==0.2.5 @@ -295,14 +298,14 @@ sshtunnel==0.1.5 tabulate==0.8.7 tenacity==4.12.0 termcolor==1.1.0 -text-unidecode==1.2 +text-unidecode==1.3 textwrap3==0.9.2 thrift-sasl==0.4.2 thrift==0.13.0 tokenize-rt==3.2.0 toml==0.10.1 tornado==5.1.1 -tqdm==4.46.0 +tqdm==4.46.1 traitlets==4.3.3 typed-ast==1.4.1 typing-extensions==3.7.4.2 @@ -314,7 +317,7 @@ urllib3==1.25.9 vertica-python==0.10.4 vine==1.3.0 virtualenv==20.0.21 -wcwidth==0.1.9 +wcwidth==0.2.4 websocket-client==0.57.0 wrapt==1.12.1 xmltodict==0.12.0 diff --git a/requirements/requirements-python3.6.txt b/requirements/requirements-python3.6.txt index e020794..b189c42 100644 --- a/requirements/requirements-python3.6.txt +++ b/requirements/requirements-python3.6.txt @@ -8,32 +8,32 @@ Flask-Caching==1.3.3 Flask-JWT-Extended==3.24.1 Flask-Login==0.4.1 Flask-OpenID==1.2.5 -Flask-SQLAlchemy==2.4.1 +Flask-SQLAlchemy==2.4.3 Flask-WTF==0.14.3 Flask==1.1.2 JPype1==0.7.1 -JayDeBeApi==1.1.1 +JayDeBeApi==1.2.2 Jinja2==2.10.3 -Mako==1.1.2 +Mako==1.1.3 Markdown==2.6.11 MarkupSafe==1.1.1 PyHive==0.6.2 PyJWT==1.7.1 -PyNaCl==1.3.0 +PyNaCl==1.4.0 PySmbClient==0.1.5 PyYAML==5.3.1 Pygments==2.6.1 SQLAlchemy-JSONField==0.9.0 -SQLAlchemy-Utils==0.36.5 +SQLAlchemy-Utils==0.36.6 SQLAlchemy==1.3.17 -Sphinx==3.0.3 +Sphinx==3.1.0 Unidecode==1.1.1 WTForms==2.3.1 Werkzeug==0.16.1 -adal==1.2.3 +adal==1.2.4 alabaster==0.7.12 alembic==1.4.2 -amqp==2.5.2 +amqp==2.6.0 analytics-python==1.2.9 ansiwrap==0.8.4 apipkg==1.5 @@ -41,39 +41,39 @@ apispec==1.3.3 appdirs==1.4.4 argcomplete==1.11.1 asn1crypto==1.3.0 -astroid==2.4.1 +astroid==2.4.2 async-generator==1.10 atlasclient==1.0.0 attrs==19.3.0 -aws-sam-translator==1.23.0 -aws-xray-sdk==2.5.0 +aws-sam-translator==1.24.0 +aws-xray-sdk==2.6.0 azure-common==1.1.25 azure-cosmos==3.1.2 azure-datalake-store==0.0.48 azure-mgmt-containerinstance==1.5.0 -azure-mgmt-resource==9.0.0 +azure-mgmt-resource==10.0.0 azure-nspkg==3.0.2 azure-storage-blob==2.1.0 azure-storage-common==2.1.0 azure-storage==0.36.0 -backcall==0.1.0 +backcall==0.2.0 bcrypt==3.1.7 beautifulsoup4==4.7.1 billiard==3.6.3.0 black==19.10b0 blinker==1.4 -boto3==1.13.15 +boto3==1.13.25 boto==2.49.0 -botocore==1.16.15 +botocore==1.16.25 cached-property==1.5.1 cachetools==4.1.0 cassandra-driver==3.20.2 cattrs==0.9.2 -celery==4.4.2 -certifi==2020.4.5.1 +celery==4.4.5 +certifi==2020.4.5.2 cffi==1.14.0 cfgv==3.1.0 -cfn-lint==0.32.1 +cfn-lint==0.33.0 cgroupspy==0.1.6 chardet==3.0.4 click==6.7 @@ -82,7 +82,7 @@ colorama==0.4.3 colorlog==4.0.2 configparser==3.5.3 coverage==5.1 -croniter==0.3.31 +croniter==0.3.32 cryptography==2.9.2 cx-Oracle==7.3.0 datadog==0.36.0 @@ -104,7 +104,7 @@ execnet==1.7.1 fastavro==0.23.4 filelock==3.0.12 flake8-colors==0.1.6 -flake8==3.8.1 +flake8==3.8.3 flaky==3.6.1 flask-swagger==0.2.13 flower==0.9.4 @@ -114,27 +114,27 @@ funcsigs==1.0.2 future-fstrings==1.2.0 future==0.18.2 gcsfs==0.6.2 -google-api-core==1.17.0 -google-api-python-client==1.8.3 +google-api-core==1.19.1 +google-api-python-client==1.9.1 google-auth-httplib2==0.0.3 google-auth-oauthlib==0.4.1 -google-auth==1.15.0 -google-cloud-bigquery==1.24.0 +google-auth==1.16.1 +google-cloud-bigquery==1.25.0 google-cloud-bigtable==1.2.1 google-cloud-container==0.5.0 google-cloud-core==1.3.0 google-cloud-dlp==0.15.0 google-cloud-language==1.3.0 google-cloud-secret-manager==1.0.0 -google-cloud-spanner==1.16.0 +google-cloud-spanner==1.17.0 google-cloud-speech==1.3.2 google-cloud-storage==1.28.1 google-cloud-texttospeech==1.0.1 google-cloud-translate==2.0.1 google-cloud-videointelligence==1.14.0 google-cloud-vision==1.0.0 -google-resumable-media==0.5.0 -googleapis-common-protos==1.51.0 +google-resumable-media==0.5.1 +googleapis-common-protos==1.52.0 graphviz==0.14 grpc-google-iam-v1==0.12.3 grpcio-gcp==0.2.2 @@ -144,17 +144,17 @@ hdfs==2.5.8 hmsclient==0.1.1 httplib2==0.18.1 humanize==0.5.1 -hvac==0.10.2 -identify==1.4.16 +hvac==0.10.3 +identify==1.4.19 idna==2.9 ijson==2.6.1 imagesize==1.2.0 -importlib-metadata==1.6.0 -importlib-resources==1.5.0 -inflection==0.4.0 +importlib-metadata==1.6.1 +importlib-resources==2.0.0 +inflection==0.5.0 ipdb==0.13.2 ipython-genutils==0.2.0 -ipython==7.14.0 +ipython==7.15.0 iso8601==0.1.12 isodate==0.6.0 itsdangerous==1.1.0 @@ -170,33 +170,34 @@ jsonschema==3.2.0 junit-xml==1.9 jupyter-client==6.1.3 jupyter-core==4.6.3 -kombu==4.6.8 +kombu==4.6.10 kubernetes==11.0.0 -lazy-object-proxy==1.4.3 +lazy-object-proxy==1.5.0 ldap3==2.7 lockfile==0.12.2 marshmallow-enum==1.5.1 -marshmallow-sqlalchemy==0.23.0 +marshmallow-sqlalchemy==0.23.1 marshmallow==2.21.0 mccabe==0.6.1 mock==4.0.2 mongomock==3.19.0 more-itertools==8.3.0 moto==1.3.14 -msrest==0.6.14 +msrest==0.6.15 msrestazure==0.6.3 multi-key-dict==2.0.3 mypy-extensions==0.4.3 mypy==0.720 mysqlclient==1.3.14 +natsort==7.0.1 nbclient==0.3.1 nbformat==5.0.6 nest-asyncio==1.3.3 networkx==2.4 -nodeenv==1.3.5 +nodeenv==1.4.0 nteract-scrapbook==0.4.1 ntlm-auth==1.4.0 -numpy==1.18.4 +numpy==1.18.5 oauthlib==3.1.0 oscrypto==1.2.0 packaging==20.4 @@ -213,11 +214,11 @@ pexpect==4.8.0 pickleshare==0.7.5 pinotdb==0.1.1 pluggy==0.13.1 -pre-commit==2.4.0 +pre-commit==2.5.0 presto-python-client==0.7.0 prison==0.1.3 prompt-toolkit==3.0.5 -protobuf==3.12.1 +protobuf==3.12.2 psutil==5.7.0 psycopg2-binary==2.8.5 ptyprocess==0.6.0 @@ -238,13 +239,13 @@ pymssql==2.1.4 pyparsing==2.4.7 pyrsistent==0.16.0 pysftp==0.2.9 -pytest-cov==2.8.1 +pytest-cov==2.9.0 pytest-forked==1.1.3 pytest-instafail==0.4.1.post0 pytest-rerunfailures==9.0 pytest-timeout==1.3.4 pytest-xdist==1.32.0 -pytest==5.4.2 +pytest==5.4.3 python-daemon==2.1.2 python-dateutil==2.8.1 python-editor==1.0.4 @@ -258,9 +259,9 @@ pytz==2020.1 pytzdata==2019.3 pywinrm==0.4.1 pyzmq==19.0.1 -qds-sdk==1.15.2 -redis==3.5.2 -regex==2020.5.14 +qds-sdk==1.16.0 +redis==3.5.3 +regex==2020.6.8 requests-futures==0.9.4 requests-kerberos==0.12.0 requests-mock==1.8.0 @@ -279,7 +280,7 @@ setproctitle==1.1.10 six==1.15.0 slackclient==1.3.2 snowballstemmer==2.0.0 -snowflake-connector-python==2.2.6 +snowflake-connector-python==2.2.7 snowflake-sqlalchemy==1.2.3 soupsieve==2.0.1 sphinx-argparse==0.2.5 @@ -306,7 +307,7 @@ thrift-sasl==0.4.2 thrift==0.13.0 toml==0.10.1 tornado==5.1.1 -tqdm==4.46.0 +tqdm==4.46.1 traitlets==4.3.3 typed-ast==1.4.1 typing-extensions==3.7.4.2 @@ -318,7 +319,7 @@ urllib3==1.25.9 vertica-python==0.10.4 vine==1.3.0 virtualenv==20.0.21 -wcwidth==0.1.9 +wcwidth==0.2.4 websocket-client==0.57.0 wrapt==1.12.1 xmltodict==0.12.0 diff --git a/requirements/requirements-python3.7.txt b/requirements/requirements-python3.7.txt index d0dee9e..a890025 100644 --- a/requirements/requirements-python3.7.txt +++ b/requirements/requirements-python3.7.txt @@ -8,32 +8,32 @@ Flask-Caching==1.3.3 Flask-JWT-Extended==3.24.1 Flask-Login==0.4.1 Flask-OpenID==1.2.5 -Flask-SQLAlchemy==2.4.1 +Flask-SQLAlchemy==2.4.3 Flask-WTF==0.14.3 Flask==1.1.2 JPype1==0.7.1 -JayDeBeApi==1.1.1 +JayDeBeApi==1.2.2 Jinja2==2.10.3 -Mako==1.1.2 +Mako==1.1.3 Markdown==2.6.11 MarkupSafe==1.1.1 PyHive==0.6.2 PyJWT==1.7.1 -PyNaCl==1.3.0 +PyNaCl==1.4.0 PySmbClient==0.1.5 PyYAML==5.3.1 Pygments==2.6.1 SQLAlchemy-JSONField==0.9.0 -SQLAlchemy-Utils==0.36.5 +SQLAlchemy-Utils==0.36.6 SQLAlchemy==1.3.17 -Sphinx==3.0.3 +Sphinx==3.1.0 Unidecode==1.1.1 WTForms==2.3.1 Werkzeug==0.16.1 -adal==1.2.3 +adal==1.2.4 alabaster==0.7.12 alembic==1.4.2 -amqp==2.5.2 +amqp==2.6.0 analytics-python==1.2.9 ansiwrap==0.8.4 apipkg==1.5 @@ -41,39 +41,39 @@ apispec==1.3.3 appdirs==1.4.4 argcomplete==1.11.1 asn1crypto==1.3.0 -astroid==2.4.1 +astroid==2.4.2 async-generator==1.10 atlasclient==1.0.0 attrs==19.3.0 -aws-sam-translator==1.23.0 -aws-xray-sdk==2.5.0 +aws-sam-translator==1.24.0 +aws-xray-sdk==2.6.0 azure-common==1.1.25 azure-cosmos==3.1.2 azure-datalake-store==0.0.48 azure-mgmt-containerinstance==1.5.0 -azure-mgmt-resource==9.0.0 +azure-mgmt-resource==10.0.0 azure-nspkg==3.0.2 azure-storage-blob==2.1.0 azure-storage-common==2.1.0 azure-storage==0.36.0 -backcall==0.1.0 +backcall==0.2.0 bcrypt==3.1.7 beautifulsoup4==4.7.1 billiard==3.6.3.0 black==19.10b0 blinker==1.4 -boto3==1.13.15 +boto3==1.13.25 boto==2.49.0 -botocore==1.16.15 +botocore==1.16.25 cached-property==1.5.1 cachetools==4.1.0 cassandra-driver==3.20.2 cattrs==0.9.2 -celery==4.4.2 -certifi==2020.4.5.1 +celery==4.4.5 +certifi==2020.4.5.2 cffi==1.14.0 cfgv==3.1.0 -cfn-lint==0.32.1 +cfn-lint==0.33.0 cgroupspy==0.1.6 chardet==3.0.4 click==6.7 @@ -82,7 +82,7 @@ colorama==0.4.3 colorlog==4.0.2 configparser==3.5.3 coverage==5.1 -croniter==0.3.31 +croniter==0.3.32 cryptography==2.9.2 cx-Oracle==7.3.0 datadog==0.36.0 @@ -104,7 +104,7 @@ execnet==1.7.1 fastavro==0.23.4 filelock==3.0.12 flake8-colors==0.1.6 -flake8==3.8.1 +flake8==3.8.3 flaky==3.6.1 flask-swagger==0.2.13 flower==0.9.4 @@ -114,27 +114,27 @@ funcsigs==1.0.2 future-fstrings==1.2.0 future==0.18.2 gcsfs==0.6.2 -google-api-core==1.17.0 -google-api-python-client==1.8.3 +google-api-core==1.19.1 +google-api-python-client==1.9.1 google-auth-httplib2==0.0.3 google-auth-oauthlib==0.4.1 -google-auth==1.15.0 -google-cloud-bigquery==1.24.0 +google-auth==1.16.1 +google-cloud-bigquery==1.25.0 google-cloud-bigtable==1.2.1 google-cloud-container==0.5.0 google-cloud-core==1.3.0 google-cloud-dlp==0.15.0 google-cloud-language==1.3.0 google-cloud-secret-manager==1.0.0 -google-cloud-spanner==1.16.0 +google-cloud-spanner==1.17.0 google-cloud-speech==1.3.2 google-cloud-storage==1.28.1 google-cloud-texttospeech==1.0.1 google-cloud-translate==2.0.1 google-cloud-videointelligence==1.14.0 google-cloud-vision==1.0.0 -google-resumable-media==0.5.0 -googleapis-common-protos==1.51.0 +google-resumable-media==0.5.1 +googleapis-common-protos==1.52.0 graphviz==0.14 grpc-google-iam-v1==0.12.3 grpcio-gcp==0.2.2 @@ -144,16 +144,16 @@ hdfs==2.5.8 hmsclient==0.1.1 httplib2==0.18.1 humanize==0.5.1 -hvac==0.10.2 -identify==1.4.16 +hvac==0.10.3 +identify==1.4.19 idna==2.9 ijson==2.6.1 imagesize==1.2.0 -importlib-metadata==1.6.0 -inflection==0.4.0 +importlib-metadata==1.6.1 +inflection==0.5.0 ipdb==0.13.2 ipython-genutils==0.2.0 -ipython==7.14.0 +ipython==7.15.0 iso8601==0.1.12 isodate==0.6.0 itsdangerous==1.1.0 @@ -169,33 +169,34 @@ jsonschema==3.2.0 junit-xml==1.9 jupyter-client==6.1.3 jupyter-core==4.6.3 -kombu==4.6.8 +kombu==4.6.10 kubernetes==11.0.0 -lazy-object-proxy==1.4.3 +lazy-object-proxy==1.5.0 ldap3==2.7 lockfile==0.12.2 marshmallow-enum==1.5.1 -marshmallow-sqlalchemy==0.23.0 +marshmallow-sqlalchemy==0.23.1 marshmallow==2.21.0 mccabe==0.6.1 mock==4.0.2 mongomock==3.19.0 more-itertools==8.3.0 moto==1.3.14 -msrest==0.6.14 +msrest==0.6.15 msrestazure==0.6.3 multi-key-dict==2.0.3 mypy-extensions==0.4.3 mypy==0.720 mysqlclient==1.3.14 +natsort==7.0.1 nbclient==0.3.1 nbformat==5.0.6 nest-asyncio==1.3.3 networkx==2.4 -nodeenv==1.3.5 +nodeenv==1.4.0 nteract-scrapbook==0.4.1 ntlm-auth==1.4.0 -numpy==1.18.4 +numpy==1.18.5 oauthlib==3.1.0 oscrypto==1.2.0 packaging==20.4 @@ -212,11 +213,11 @@ pexpect==4.8.0 pickleshare==0.7.5 pinotdb==0.1.1 pluggy==0.13.1 -pre-commit==2.4.0 +pre-commit==2.5.0 presto-python-client==0.7.0 prison==0.1.3 prompt-toolkit==3.0.5 -protobuf==3.12.1 +protobuf==3.12.2 psutil==5.7.0 psycopg2-binary==2.8.5 ptyprocess==0.6.0 @@ -237,13 +238,13 @@ pymssql==2.1.4 pyparsing==2.4.7 pyrsistent==0.16.0 pysftp==0.2.9 -pytest-cov==2.8.1 +pytest-cov==2.9.0 pytest-forked==1.1.3 pytest-instafail==0.4.1.post0 pytest-rerunfailures==9.0 pytest-timeout==1.3.4 pytest-xdist==1.32.0 -pytest==5.4.2 +pytest==5.4.3 python-daemon==2.1.2 python-dateutil==2.8.1 python-editor==1.0.4 @@ -257,9 +258,9 @@ pytz==2020.1 pytzdata==2019.3 pywinrm==0.4.1 pyzmq==19.0.1 -qds-sdk==1.15.2 -redis==3.5.2 -regex==2020.5.14 +qds-sdk==1.16.0 +redis==3.5.3 +regex==2020.6.8 requests-futures==0.9.4 requests-kerberos==0.12.0 requests-mock==1.8.0 @@ -278,7 +279,7 @@ setproctitle==1.1.10 six==1.15.0 slackclient==1.3.2 snowballstemmer==2.0.0 -snowflake-connector-python==2.2.6 +snowflake-connector-python==2.2.7 snowflake-sqlalchemy==1.2.3 soupsieve==2.0.1 sphinx-argparse==0.2.5 @@ -305,7 +306,7 @@ thrift-sasl==0.4.2 thrift==0.13.0 toml==0.10.1 tornado==5.1.1 -tqdm==4.46.0 +tqdm==4.46.1 traitlets==4.3.3 typed-ast==1.4.1 typing-extensions==3.7.4.2 @@ -316,7 +317,7 @@ urllib3==1.25.9 vertica-python==0.10.4 vine==1.3.0 virtualenv==20.0.21 -wcwidth==0.1.9 +wcwidth==0.2.4 websocket-client==0.57.0 wrapt==1.12.1 xmltodict==0.12.0 diff --git a/requirements/setup-2.7.md5 b/requirements/setup-2.7.md5 index 93454c3..fa9e77d 100644 --- a/requirements/setup-2.7.md5 +++ b/requirements/setup-2.7.md5 @@ -1 +1 @@ -4b8bf8ce351fd07604aa11e00ff2cba4 /opt/airflow/setup.py +a42983dcaa7e536b9c343d1d59ae5b32 /opt/airflow/setup.py diff --git a/requirements/setup-3.5.md5 b/requirements/setup-3.5.md5 index 93454c3..fa9e77d 100644 --- a/requirements/setup-3.5.md5 +++ b/requirements/setup-3.5.md5 @@ -1 +1 @@ -4b8bf8ce351fd07604aa11e00ff2cba4 /opt/airflow/setup.py +a42983dcaa7e536b9c343d1d59ae5b32 /opt/airflow/setup.py diff --git a/requirements/setup-3.6.md5 b/requirements/setup-3.6.md5 index 93454c3..fa9e77d 100644 --- a/requirements/setup-3.6.md5 +++ b/requirements/setup-3.6.md5 @@ -1 +1 @@ -4b8bf8ce351fd07604aa11e00ff2cba4 /opt/airflow/setup.py +a42983dcaa7e536b9c343d1d59ae5b32 /opt/airflow/setup.py diff --git a/requirements/setup-3.7.md5 b/requirements/setup-3.7.md5 index 93454c3..fa9e77d 100644 --- a/requirements/setup-3.7.md5 +++ b/requirements/setup-3.7.md5 @@ -1 +1 @@ -4b8bf8ce351fd07604aa11e00ff2cba4 /opt/airflow/setup.py +a42983dcaa7e536b9c343d1d59ae5b32 /opt/airflow/setup.py diff --git a/scripts/ci/ci_docs.sh b/scripts/ci/ci_docs.sh index e84de9b..417f6a6 100755 --- a/scripts/ci/ci_docs.sh +++ b/scripts/ci/ci_docs.sh @@ -28,6 +28,9 @@ function run_docs() { --env VERBOSE_COMMANDS \ --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ + --env HOST_OS="$(uname -s)" \ + --env HOST_HOME="${HOME}" \ + --env HOST_AIRFLOW_SOURCES="${AIRFLOW_SOURCES}" \ --rm \ "${AIRFLOW_CI_IMAGE}" \ "--" "/opt/airflow/docs/build" \ diff --git a/scripts/ci/ci_fix_ownership.sh b/scripts/ci/ci_fix_ownership.sh index 775cc58..81f5ac1 100755 --- a/scripts/ci/ci_fix_ownership.sh +++ b/scripts/ci/ci_fix_ownership.sh @@ -33,11 +33,14 @@ export AIRFLOW_IMAGE=${AIRFLOW_CI_IMAGE} export WEBSERVER_HOST_PORT=28080 HOST_USER_ID="$(id -ur)" HOST_GROUP_ID="$(id -gr)" +HOST_OS="$(uname -s)" export HOST_USER_ID export HOST_GROUP_ID +export HOST_OS docker-compose \ -f "${MY_DIR}/docker-compose/base.yml" \ -f "${MY_DIR}/docker-compose/local.yml" \ + -f "${MY_DIR}/docker-compose/forward-credentials.yml" \ run airflow /opt/airflow/scripts/ci/in_container/run_fix_ownership.sh diff --git a/scripts/ci/ci_flake8.sh b/scripts/ci/ci_flake8.sh index 5a2da56..fac9be4 100755 --- a/scripts/ci/ci_flake8.sh +++ b/scripts/ci/ci_flake8.sh @@ -31,6 +31,9 @@ function run_flake8() { --env VERBOSE_COMMANDS \ --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ + --env HOST_OS="$(uname -s)" \ + --env HOST_HOME="${HOME}" \ + --env HOST_AIRFLOW_SOURCES="${AIRFLOW_SOURCES}" \ --rm \ "${AIRFLOW_CI_IMAGE}" \ "--" "/opt/airflow/scripts/ci/in_container/run_flake8.sh" \ @@ -43,6 +46,9 @@ function run_flake8() { --env VERBOSE_COMMANDS \ --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ + --env HOST_OS="$(uname -s)" \ + --env HOST_HOME="${HOME}" \ + --env HOST_AIRFLOW_SOURCES="${AIRFLOW_SOURCES}" \ --rm \ "${AIRFLOW_CI_IMAGE}" \ "--" "/opt/airflow/scripts/ci/in_container/run_flake8.sh" "${FILES[@]}" \ diff --git a/scripts/ci/ci_mypy.sh b/scripts/ci/ci_mypy.sh index 8898d30..962d2c0 100755 --- a/scripts/ci/ci_mypy.sh +++ b/scripts/ci/ci_mypy.sh @@ -33,6 +33,9 @@ function run_mypy() { --env VERBOSE_COMMANDS \ --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ + --env HOST_OS="$(uname -s)" \ + --env HOST_HOME="${HOME}" \ + --env HOST_AIRFLOW_SOURCES="${AIRFLOW_SOURCES}" \ "-v" "${AIRFLOW_SOURCES}/.mypy_cache:/opt/airflow/.mypy_cache" \ --rm \ "${AIRFLOW_CI_IMAGE}" \ diff --git a/scripts/ci/docker-compose/forward-credentials.yml b/scripts/ci/docker-compose/forward-credentials.yml index 875b1ce..0eb0dbb 100644 --- a/scripts/ci/docker-compose/forward-credentials.yml +++ b/scripts/ci/docker-compose/forward-credentials.yml @@ -22,6 +22,8 @@ services: # Useful for gcloud/aws/kubernetes etc. authorisation to be passed # To inside docker. Use with care - your credentials will be available to # Everything you install in Docker + # If you add it here - also add it to "in_container_fix_ownership" method in + # the _in_container_utils.sh file to make it friendly for Linux users volumes: - ${HOME}/.aws:/root/.aws:cached - ${HOME}/.azure:/root/.azure:cached diff --git a/scripts/ci/docker-compose/local-prod.yml b/scripts/ci/docker-compose/local-prod.yml index ae8317d..a82b4f8 100644 --- a/scripts/ci/docker-compose/local-prod.yml +++ b/scripts/ci/docker-compose/local-prod.yml @@ -40,4 +40,6 @@ services: - HOST_USER_ID - HOST_GROUP_ID - HOST_HOME=${HOME} + - HOST_AIRFLOW_SOURCES=${AIRFLOW_SOURCES} + - HOST_OS - PYTHONDONTWRITEBYTECODE diff --git a/scripts/ci/docker-compose/local.yml b/scripts/ci/docker-compose/local.yml index 9603417..3c9e40b 100644 --- a/scripts/ci/docker-compose/local.yml +++ b/scripts/ci/docker-compose/local.yml @@ -60,6 +60,8 @@ services: - HOST_USER_ID - HOST_GROUP_ID - HOST_HOME=${HOME} + - HOST_AIRFLOW_SOURCES=${AIRFLOW_SOURCES} + - HOST_OS - PYTHONDONTWRITEBYTECODE ports: - "${WEBSERVER_HOST_PORT}:8080" diff --git a/scripts/ci/in_container/_in_container_utils.sh b/scripts/ci/in_container/_in_container_utils.sh index 9197a04..5d9fbb6 100644 --- a/scripts/ci/in_container/_in_container_utils.sh +++ b/scripts/ci/in_container/_in_container_utils.sh @@ -89,13 +89,19 @@ function in_container_cleanup_pycache() { # # Fixes ownership of files generated in container - if they are owned by root, they will be owned by -# The host user. +# The host user. Only needed if the host is Linux - on Mac, ownership of files is automatically +# changed to the Host user via osxfs filesystem # function in_container_fix_ownership() { - set +o pipefail - sudo find "${AIRFLOW_SOURCES}" -print0 -user root \ - | sudo xargs --null chown -v "${HOST_USER_ID}.${HOST_GROUP_ID}" --no-dereference >/dev/null 2>&1 - set -o pipefail + if [[ ${HOST_OS:=} == "Linux" ]]; then + set +o pipefail + echo "Fixing ownership of mounted files" + sudo find "${AIRFLOW_SOURCES}" -print0 -user root \ + | sudo xargs --null chown "${HOST_USER_ID}.${HOST_GROUP_ID}" --no-dereference >/dev/null 2>&1 + sudo find "/root/.aws" "/root/.azure" "/root/.config" "/root/.docker" -print0 -user root \ + | sudo xargs --null chown "${HOST_USER_ID}.${HOST_GROUP_ID}" --no-dereference || true >/dev/null 2>&1 + set -o pipefail + fi } function in_container_go_to_airflow_sources() { diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh index 9f86ba4..c8cd9f7 100644 --- a/scripts/ci/libraries/_initialization.sh +++ b/scripts/ci/libraries/_initialization.sh @@ -114,6 +114,13 @@ function initialize_common_environment { HOST_GROUP_ID="$(id -gr)" export HOST_GROUP_ID + # Set host OS. This is used to set the ownership properly when exiting + # The container on Linux - all files created inside docker are created with root user + # but they should be restored back to the host user + HOST_OS="$(uname -s)" + export HOST_OS + + # Add the right volume mount for sources, depending which mount strategy is used if [[ ${MOUNT_SOURCE_DIR_FOR_STATIC_CHECKS} == "true" ]]; then print_info diff --git a/scripts/ci/libraries/_runs.sh b/scripts/ci/libraries/_runs.sh index cd5a44a..76b674d 100644 --- a/scripts/ci/libraries/_runs.sh +++ b/scripts/ci/libraries/_runs.sh @@ -25,6 +25,9 @@ function run_docs() { --env VERBOSE_COMMANDS \ --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ + --env HOST_OS="$(uname -s)" \ + --env HOST_HOME="${HOME}" \ + --env HOST_AIRFLOW_SOURCES="${AIRFLOW_SOURCES}" \ --rm \ "${AIRFLOW_CI_IMAGE}" \ "--" "/opt/airflow/docs/build" \ @@ -40,6 +43,9 @@ function run_generate_requirements() { --env VERBOSE_COMMANDS \ --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ + --env HOST_OS="$(uname -s)" \ + --env HOST_HOME="${HOME}" \ + --env HOST_AIRFLOW_SOURCES="${AIRFLOW_SOURCES}" \ --env UPGRADE_WHILE_GENERATING_REQUIREMENTS \ --env PYTHON_MAJOR_MINOR_VERSION \ --env CHECK_REQUIREMENTS_ONLY \ diff --git a/scripts/ci/prepare_tool_scripts.sh b/scripts/ci/prepare_tool_scripts.sh new file mode 100755 index 0000000..7a98c50 --- /dev/null +++ b/scripts/ci/prepare_tool_scripts.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +set -euo pipefail + +function prepare_tool_script() { + IMAGE="${1}" + VOLUME="${2}" + TOOL="${3}" + COMMAND="${4:-}" + + TARGET_TOOL_PATH="/usr/bin/${TOOL}" + TARGET_TOOL_UPDATE_PATH="/usr/bin/${TOOL}-update" + + cat >"${TARGET_TOOL_PATH}" <<EOF +#!/usr/bin/env bash +docker run --rm -it \ + -v "\${HOST_AIRFLOW_SOURCES}/tmp:/tmp" \ + -v "\${HOST_AIRFLOW_SOURCES}/files:/files" \ + -v "\${HOST_AIRFLOW_SOURCES}:/opt/airflow" \ + -v "\${HOST_HOME}/${VOLUME}:/root/${VOLUME}" \ + "${IMAGE}" ${COMMAND} "\$@" +RES=\$? +if [[ \${HOST_OS} == "Linux" ]]; then + docker run --rm \ + -v "\${HOST_AIRFLOW_SOURCES}/tmp:/tmp" \ + -v "\${HOST_AIRFLOW_SOURCES}/files:/files" \ + -v "\${HOST_HOME}/${VOLUME}:/root/${VOLUME}" \ + "\${AIRFLOW_CI_IMAGE}" bash -c \ + "find '/tmp/' '/files/' '/root/${VOLUME}' -user root -print0 | xargs --null chown '\${HOST_USER_ID}.\${HOST_GROUP_ID}' --no-dereference" >/dev/null 2>&1 +fi +exit \${RES} +EOF + + cat >"${TARGET_TOOL_UPDATE_PATH}" <<EOF +#!/usr/bin/env bash +docker pull "${IMAGE}" +EOF + + chmod a+x "${TARGET_TOOL_PATH}" "${TARGET_TOOL_UPDATE_PATH}" +} + +GCLOUD_IMAGE="gcr.io/google.com/cloudsdktool/cloud-sdk:latest" + +prepare_tool_script "amazon/aws-cli:latest" ".aws" aws +prepare_tool_script "mcr.microsoft.com/azure-cli:latest" ".azure" az az +prepare_tool_script "${GCLOUD_IMAGE}" ".config/gcloud" bq bq +prepare_tool_script "${GCLOUD_IMAGE}" ".config/gcloud" gcloud gcloud +prepare_tool_script "${GCLOUD_IMAGE}" ".config/gcloud" gsutil gsutil