This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a commit to branch v2-7-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 5847d00812a2e794a9cd13aa96ff3636df89e5e0 Author: Andrey Anshin <andrey.ans...@taragol.is> AuthorDate: Sat Sep 16 21:09:33 2023 +0400 Use latest LTS clients of Oracle MySQL and MariaDB (#33722) (cherry picked from commit 18716c8b4ff50fcc8c0def4aa20f7c5628d2189d) --- Dockerfile | 87 +++++++++++++++++++++++++++------------- Dockerfile.ci | 87 +++++++++++++++++++++++++++------------- scripts/docker/install_mysql.sh | 89 ++++++++++++++++++++++++++++------------- 3 files changed, 179 insertions(+), 84 deletions(-) diff --git a/Dockerfile b/Dockerfile index dc32cbc45b..17f5457064 100644 --- a/Dockerfile +++ b/Dockerfile @@ -178,54 +178,68 @@ COPY <<"EOF" /install_mysql.sh set -euo pipefail declare -a packages -MYSQL_VERSION="8.0" -readonly MYSQL_VERSION -MARIADB_VERSION="10.5" -readonly MARIADB_VERSION +MYSQL_LTS_VERSION="8.0" +MARIADB_LTS_VERSION="10.11" +readonly MYSQL_LTS_VERSION +readonly MARIADB_LTS_VERSION COLOR_BLUE=$'\e[34m' readonly COLOR_BLUE COLOR_YELLOW=$'\e[1;33m' readonly COLOR_YELLOW +COLOR_RED=$'\e[1;31m' +readonly COLOR_RED COLOR_RESET=$'\e[0m' readonly COLOR_RESET : "${INSTALL_MYSQL_CLIENT:?Should be true or false}" : "${INSTALL_MYSQL_CLIENT_TYPE:-mysql}" +export_key() { + local key="${1}" + local name="${2:-mysql}" + + echo "${COLOR_BLUE}Verify and export GPG public key ${key}${COLOR_RESET}" + GNUPGHOME="$(mktemp -d)" + export GNUPGHOME + set +e + for keyserver in $(shuf -e ha.pool.sks-keyservers.net hkp://p80.pool.sks-keyservers.net:80 \ + keyserver.ubuntu.com hkp://keyserver.ubuntu.com:80) + do + gpg --keyserver "${keyserver}" --recv-keys "${key}" 2>&1 && break + done + set -e + gpg --export "${key}" > "/etc/apt/trusted.gpg.d/${name}.gpg" + gpgconf --kill all + rm -rf "${GNUPGHOME}" + unset GNUPGHOME +} + install_mysql_client() { if [[ "${1}" == "dev" ]]; then packages=("libmysqlclient-dev" "mysql-client") elif [[ "${1}" == "prod" ]]; then + # `libmysqlclientXX` where XX is number, and it should be increased every new GA MySQL release, for example + # 18 - MySQL 5.6.48 + # 20 - MySQL 5.7.42 + # 21 - MySQL 8.0.34 + # 22 - MySQL 8.1 packages=("libmysqlclient21" "mysql-client") else echo - echo "Specify either prod or dev" + echo "${COLOR_RED}Specify either prod or dev${COLOR_RESET}" echo exit 1 fi + export_key "467B942D3A79BD29" "mysql" + echo - echo "${COLOR_BLUE}Installing mysql client version ${MYSQL_VERSION}: ${1}${COLOR_RESET}" + echo "${COLOR_BLUE}Installing Oracle MySQL client version ${MYSQL_LTS_VERSION}: ${1}${COLOR_RESET}" echo - local key="467B942D3A79BD29" - readonly key - - GNUPGHOME="$(mktemp -d)" - export GNUPGHOME - set +e - for keyserver in $(shuf -e ha.pool.sks-keyservers.net hkp://p80.pool.sks-keyservers.net:80 \ - keyserver.ubuntu.com hkp://keyserver.ubuntu.com:80) - do - gpg --keyserver "${keyserver}" --recv-keys "${key}" 2>&1 && break - done - set -e - gpg --export "${key}" > /etc/apt/trusted.gpg.d/mysql.gpg - gpgconf --kill all - rm -rf "${GNUPGHOME}" - unset GNUPGHOME - echo "deb http://repo.mysql.com/apt/debian/ $(lsb_release -cs) mysql-${MYSQL_VERSION}" > /etc/apt/sources.list.d/mysql.list + echo "deb http://repo.mysql.com/apt/debian/ $(lsb_release -cs) mysql-${MYSQL_LTS_VERSION}" > \ + /etc/apt/sources.list.d/mysql.list apt-get update apt-get install --no-install-recommends -y "${packages[@]}" apt-get autoremove -yqq --purge @@ -233,21 +247,38 @@ install_mysql_client() { } install_mariadb_client() { + # List of compatible package Oracle MySQL -> MariaDB: + # `mysql-client` -> `mariadb-client` or `mariadb-client-compat` (11+) + # `libmysqlclientXX` (where XX is a number) -> `libmariadb3-compat` + # `libmysqlclient-dev` -> `libmariadb-dev-compat` + # + # Different naming against Debian repo which we used before + # that some of packages might contains `-compat` suffix, Debian repo -> MariaDB repo: + # `libmariadb-dev` -> `libmariadb-dev-compat` + # `mariadb-client-core` -> `mariadb-client` or `mariadb-client-compat` (11+) if [[ "${1}" == "dev" ]]; then - packages=("libmariadb-dev" "mariadb-client-core-${MARIADB_VERSION}") + packages=("libmariadb-dev-compat" "mariadb-client") elif [[ "${1}" == "prod" ]]; then - packages=("mariadb-client-core-${MARIADB_VERSION}") + packages=("libmariadb3-compat" "mariadb-client") else echo - echo "Specify either prod or dev" + echo "${COLOR_RED}Specify either prod or dev${COLOR_RESET}" echo exit 1 fi + export_key "0xF1656F24C74CD1D8" "mariadb" + echo - echo "${COLOR_BLUE}Installing MariaDB client version ${MARIADB_VERSION}: ${1}${COLOR_RESET}" - echo "${COLOR_YELLOW}MariaDB client binary compatible with MySQL client.${COLOR_RESET}" + echo "${COLOR_BLUE}Installing MariaDB client version ${MARIADB_LTS_VERSION}: ${1}${COLOR_RESET}" + echo "${COLOR_YELLOW}MariaDB client protocol-compatible with MySQL client.${COLOR_RESET}" echo + + curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - + echo "deb [arch=amd64,arm64] https://archive.mariadb.org/mariadb-${MARIADB_LTS_VERSION}/repo/debian/ $(lsb_release -cs) main" > \ + /etc/apt/sources.list.d/mariadb.list + # Make sure that dependencies from MariaDB repo are preferred over Debian dependencies + printf "Package: *\nPin: release o=MariaDB\nPin-Priority: 999\n" > /etc/apt/preferences.d/mariadb apt-get update apt-get install --no-install-recommends -y "${packages[@]}" apt-get autoremove -yqq --purge diff --git a/Dockerfile.ci b/Dockerfile.ci index b6e1054efe..b2da5f134b 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -138,54 +138,68 @@ COPY <<"EOF" /install_mysql.sh set -euo pipefail declare -a packages -MYSQL_VERSION="8.0" -readonly MYSQL_VERSION -MARIADB_VERSION="10.5" -readonly MARIADB_VERSION +MYSQL_LTS_VERSION="8.0" +MARIADB_LTS_VERSION="10.11" +readonly MYSQL_LTS_VERSION +readonly MARIADB_LTS_VERSION COLOR_BLUE=$'\e[34m' readonly COLOR_BLUE COLOR_YELLOW=$'\e[1;33m' readonly COLOR_YELLOW +COLOR_RED=$'\e[1;31m' +readonly COLOR_RED COLOR_RESET=$'\e[0m' readonly COLOR_RESET : "${INSTALL_MYSQL_CLIENT:?Should be true or false}" : "${INSTALL_MYSQL_CLIENT_TYPE:-mysql}" +export_key() { + local key="${1}" + local name="${2:-mysql}" + + echo "${COLOR_BLUE}Verify and export GPG public key ${key}${COLOR_RESET}" + GNUPGHOME="$(mktemp -d)" + export GNUPGHOME + set +e + for keyserver in $(shuf -e ha.pool.sks-keyservers.net hkp://p80.pool.sks-keyservers.net:80 \ + keyserver.ubuntu.com hkp://keyserver.ubuntu.com:80) + do + gpg --keyserver "${keyserver}" --recv-keys "${key}" 2>&1 && break + done + set -e + gpg --export "${key}" > "/etc/apt/trusted.gpg.d/${name}.gpg" + gpgconf --kill all + rm -rf "${GNUPGHOME}" + unset GNUPGHOME +} + install_mysql_client() { if [[ "${1}" == "dev" ]]; then packages=("libmysqlclient-dev" "mysql-client") elif [[ "${1}" == "prod" ]]; then + # `libmysqlclientXX` where XX is number, and it should be increased every new GA MySQL release, for example + # 18 - MySQL 5.6.48 + # 20 - MySQL 5.7.42 + # 21 - MySQL 8.0.34 + # 22 - MySQL 8.1 packages=("libmysqlclient21" "mysql-client") else echo - echo "Specify either prod or dev" + echo "${COLOR_RED}Specify either prod or dev${COLOR_RESET}" echo exit 1 fi + export_key "467B942D3A79BD29" "mysql" + echo - echo "${COLOR_BLUE}Installing mysql client version ${MYSQL_VERSION}: ${1}${COLOR_RESET}" + echo "${COLOR_BLUE}Installing Oracle MySQL client version ${MYSQL_LTS_VERSION}: ${1}${COLOR_RESET}" echo - local key="467B942D3A79BD29" - readonly key - - GNUPGHOME="$(mktemp -d)" - export GNUPGHOME - set +e - for keyserver in $(shuf -e ha.pool.sks-keyservers.net hkp://p80.pool.sks-keyservers.net:80 \ - keyserver.ubuntu.com hkp://keyserver.ubuntu.com:80) - do - gpg --keyserver "${keyserver}" --recv-keys "${key}" 2>&1 && break - done - set -e - gpg --export "${key}" > /etc/apt/trusted.gpg.d/mysql.gpg - gpgconf --kill all - rm -rf "${GNUPGHOME}" - unset GNUPGHOME - echo "deb http://repo.mysql.com/apt/debian/ $(lsb_release -cs) mysql-${MYSQL_VERSION}" > /etc/apt/sources.list.d/mysql.list + echo "deb http://repo.mysql.com/apt/debian/ $(lsb_release -cs) mysql-${MYSQL_LTS_VERSION}" > \ + /etc/apt/sources.list.d/mysql.list apt-get update apt-get install --no-install-recommends -y "${packages[@]}" apt-get autoremove -yqq --purge @@ -193,21 +207,38 @@ install_mysql_client() { } install_mariadb_client() { + # List of compatible package Oracle MySQL -> MariaDB: + # `mysql-client` -> `mariadb-client` or `mariadb-client-compat` (11+) + # `libmysqlclientXX` (where XX is a number) -> `libmariadb3-compat` + # `libmysqlclient-dev` -> `libmariadb-dev-compat` + # + # Different naming against Debian repo which we used before + # that some of packages might contains `-compat` suffix, Debian repo -> MariaDB repo: + # `libmariadb-dev` -> `libmariadb-dev-compat` + # `mariadb-client-core` -> `mariadb-client` or `mariadb-client-compat` (11+) if [[ "${1}" == "dev" ]]; then - packages=("libmariadb-dev" "mariadb-client-core-${MARIADB_VERSION}") + packages=("libmariadb-dev-compat" "mariadb-client") elif [[ "${1}" == "prod" ]]; then - packages=("mariadb-client-core-${MARIADB_VERSION}") + packages=("libmariadb3-compat" "mariadb-client") else echo - echo "Specify either prod or dev" + echo "${COLOR_RED}Specify either prod or dev${COLOR_RESET}" echo exit 1 fi + export_key "0xF1656F24C74CD1D8" "mariadb" + echo - echo "${COLOR_BLUE}Installing MariaDB client version ${MARIADB_VERSION}: ${1}${COLOR_RESET}" - echo "${COLOR_YELLOW}MariaDB client binary compatible with MySQL client.${COLOR_RESET}" + echo "${COLOR_BLUE}Installing MariaDB client version ${MARIADB_LTS_VERSION}: ${1}${COLOR_RESET}" + echo "${COLOR_YELLOW}MariaDB client protocol-compatible with MySQL client.${COLOR_RESET}" echo + + curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - + echo "deb [arch=amd64,arm64] https://archive.mariadb.org/mariadb-${MARIADB_LTS_VERSION}/repo/debian/ $(lsb_release -cs) main" > \ + /etc/apt/sources.list.d/mariadb.list + # Make sure that dependencies from MariaDB repo are preferred over Debian dependencies + printf "Package: *\nPin: release o=MariaDB\nPin-Priority: 999\n" > /etc/apt/preferences.d/mariadb apt-get update apt-get install --no-install-recommends -y "${packages[@]}" apt-get autoremove -yqq --purge diff --git a/scripts/docker/install_mysql.sh b/scripts/docker/install_mysql.sh index 6b6ae7abb9..9b4d51b2f5 100644 --- a/scripts/docker/install_mysql.sh +++ b/scripts/docker/install_mysql.sh @@ -18,54 +18,70 @@ set -euo pipefail declare -a packages -MYSQL_VERSION="8.0" -readonly MYSQL_VERSION -MARIADB_VERSION="10.5" -readonly MARIADB_VERSION +# https://dev.mysql.com/blog-archive/introducing-mysql-innovation-and-long-term-support-lts-versions/ +MYSQL_LTS_VERSION="8.0" +# https://mariadb.org/about/#maintenance-policy +MARIADB_LTS_VERSION="10.11" +readonly MYSQL_LTS_VERSION +readonly MARIADB_LTS_VERSION COLOR_BLUE=$'\e[34m' readonly COLOR_BLUE COLOR_YELLOW=$'\e[1;33m' readonly COLOR_YELLOW +COLOR_RED=$'\e[1;31m' +readonly COLOR_RED COLOR_RESET=$'\e[0m' readonly COLOR_RESET : "${INSTALL_MYSQL_CLIENT:?Should be true or false}" : "${INSTALL_MYSQL_CLIENT_TYPE:-mysql}" +export_key() { + local key="${1}" + local name="${2:-mysql}" + + echo "${COLOR_BLUE}Verify and export GPG public key ${key}${COLOR_RESET}" + GNUPGHOME="$(mktemp -d)" + export GNUPGHOME + set +e + for keyserver in $(shuf -e ha.pool.sks-keyservers.net hkp://p80.pool.sks-keyservers.net:80 \ + keyserver.ubuntu.com hkp://keyserver.ubuntu.com:80) + do + gpg --keyserver "${keyserver}" --recv-keys "${key}" 2>&1 && break + done + set -e + gpg --export "${key}" > "/etc/apt/trusted.gpg.d/${name}.gpg" + gpgconf --kill all + rm -rf "${GNUPGHOME}" + unset GNUPGHOME +} + install_mysql_client() { if [[ "${1}" == "dev" ]]; then packages=("libmysqlclient-dev" "mysql-client") elif [[ "${1}" == "prod" ]]; then + # `libmysqlclientXX` where XX is number, and it should be increased every new GA MySQL release, for example + # 18 - MySQL 5.6.48 + # 20 - MySQL 5.7.42 + # 21 - MySQL 8.0.34 + # 22 - MySQL 8.1 packages=("libmysqlclient21" "mysql-client") else echo - echo "Specify either prod or dev" + echo "${COLOR_RED}Specify either prod or dev${COLOR_RESET}" echo exit 1 fi + export_key "467B942D3A79BD29" "mysql" + echo - echo "${COLOR_BLUE}Installing mysql client version ${MYSQL_VERSION}: ${1}${COLOR_RESET}" + echo "${COLOR_BLUE}Installing Oracle MySQL client version ${MYSQL_LTS_VERSION}: ${1}${COLOR_RESET}" echo - local key="467B942D3A79BD29" - readonly key - - GNUPGHOME="$(mktemp -d)" - export GNUPGHOME - set +e - for keyserver in $(shuf -e ha.pool.sks-keyservers.net hkp://p80.pool.sks-keyservers.net:80 \ - keyserver.ubuntu.com hkp://keyserver.ubuntu.com:80) - do - gpg --keyserver "${keyserver}" --recv-keys "${key}" 2>&1 && break - done - set -e - gpg --export "${key}" > /etc/apt/trusted.gpg.d/mysql.gpg - gpgconf --kill all - rm -rf "${GNUPGHOME}" - unset GNUPGHOME - echo "deb http://repo.mysql.com/apt/debian/ $(lsb_release -cs) mysql-${MYSQL_VERSION}" > /etc/apt/sources.list.d/mysql.list + echo "deb http://repo.mysql.com/apt/debian/ $(lsb_release -cs) mysql-${MYSQL_LTS_VERSION}" > \ + /etc/apt/sources.list.d/mysql.list apt-get update apt-get install --no-install-recommends -y "${packages[@]}" apt-get autoremove -yqq --purge @@ -73,21 +89,38 @@ install_mysql_client() { } install_mariadb_client() { + # List of compatible package Oracle MySQL -> MariaDB: + # `mysql-client` -> `mariadb-client` or `mariadb-client-compat` (11+) + # `libmysqlclientXX` (where XX is a number) -> `libmariadb3-compat` + # `libmysqlclient-dev` -> `libmariadb-dev-compat` + # + # Different naming against Debian repo which we used before + # that some of packages might contains `-compat` suffix, Debian repo -> MariaDB repo: + # `libmariadb-dev` -> `libmariadb-dev-compat` + # `mariadb-client-core` -> `mariadb-client` or `mariadb-client-compat` (11+) if [[ "${1}" == "dev" ]]; then - packages=("libmariadb-dev" "mariadb-client-core-${MARIADB_VERSION}") + packages=("libmariadb-dev-compat" "mariadb-client") elif [[ "${1}" == "prod" ]]; then - packages=("mariadb-client-core-${MARIADB_VERSION}") + packages=("libmariadb3-compat" "mariadb-client") else echo - echo "Specify either prod or dev" + echo "${COLOR_RED}Specify either prod or dev${COLOR_RESET}" echo exit 1 fi + export_key "0xF1656F24C74CD1D8" "mariadb" + echo - echo "${COLOR_BLUE}Installing MariaDB client version ${MARIADB_VERSION}: ${1}${COLOR_RESET}" - echo "${COLOR_YELLOW}MariaDB client binary compatible with MySQL client.${COLOR_RESET}" + echo "${COLOR_BLUE}Installing MariaDB client version ${MARIADB_LTS_VERSION}: ${1}${COLOR_RESET}" + echo "${COLOR_YELLOW}MariaDB client protocol-compatible with MySQL client.${COLOR_RESET}" echo + + curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - + echo "deb [arch=amd64,arm64] https://archive.mariadb.org/mariadb-${MARIADB_LTS_VERSION}/repo/debian/ $(lsb_release -cs) main" > \ + /etc/apt/sources.list.d/mariadb.list + # Make sure that dependencies from MariaDB repo are preferred over Debian dependencies + printf "Package: *\nPin: release o=MariaDB\nPin-Priority: 999\n" > /etc/apt/preferences.d/mariadb apt-get update apt-get install --no-install-recommends -y "${packages[@]}" apt-get autoremove -yqq --purge