This is an automated email from the ASF dual-hosted git repository. maximebeauchemin pushed a commit to branch faster_docker in repository https://gitbox.apache.org/repos/asf/superset.git
commit 89b7dd1c1a3dacf72e1159f7708c68a5b6574d5c Author: Maxime Beauchemin <[email protected]> AuthorDate: Fri Feb 9 10:01:43 2024 -0800 feat: faster docker builds --- Dockerfile | 30 ++++++++++++++++++++------- scripts/get_package_version.py | 11 ++++++++++ setup.py | 46 ++++++++++++++++++++++++++++-------------- version.txt | 1 + 4 files changed, 66 insertions(+), 22 deletions(-) diff --git a/Dockerfile b/Dockerfile index fc3e667037..0f24bdbba9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -47,11 +47,24 @@ COPY ./superset-frontend ./ # This seems to be the most expensive step RUN npm run ${BUILD_CMD} +###################################################################### +# Python pre-lean image +###################################################################### +FROM python:${PY_VER} AS pre_lean +WORKDIR /app +RUN mkdir scripts ./superset-frontend/ +COPY --chown=superset:superset ./scripts/get_package_version.py ./scripts +COPY --chown=superset:superset ./superset-frontend/package.json ./superset-frontend/ +# Generate ./version.txt from package.json (the source of thruth) +RUN python ./scripts/get_package_version.py > ./version.txt + + ###################################################################### # Final lean image... ###################################################################### FROM python:${PY_VER} AS lean + WORKDIR /app ENV LANG=C.UTF-8 \ LC_ALL=C.UTF-8 \ @@ -61,9 +74,7 @@ ENV LANG=C.UTF-8 \ SUPERSET_HOME="/app/superset_home" \ SUPERSET_PORT=8088 -RUN mkdir -p ${PYTHONPATH} superset/static superset-frontend apache_superset.egg-info requirements \ - && useradd --user-group -d ${SUPERSET_HOME} -m --no-log-init --shell /bin/bash superset \ - && apt-get update -qq && apt-get install -yqq --no-install-recommends \ +RUN apt-get update -qq && apt-get install -yqq --no-install-recommends \ build-essential \ curl \ default-libmysqlclient-dev \ @@ -72,13 +83,18 @@ RUN mkdir -p ${PYTHONPATH} superset/static superset-frontend apache_superset.egg libpq-dev \ libecpg-dev \ libldap2-dev \ - && touch superset/static/version_info.json \ - && chown -R superset:superset ./* \ && rm -rf /var/lib/apt/lists/* +RUN mkdir -p ${PYTHONPATH} superset/static superset-frontend apache_superset.egg-info requirements \ + && useradd --user-group -d ${SUPERSET_HOME} -m --no-log-init --shell /bin/bash superset \ + && touch superset/static/version_info.json \ + && chown -R superset:superset ./* + COPY --chown=superset:superset setup.py MANIFEST.in README.md ./ -# setup.py uses the version information in package.json -COPY --chown=superset:superset superset-frontend/package.json superset-frontend/ + +# version_info.json is used by setup.py +COPY --chown=superset:superset --from=pre_lean /app/version.txt ./ + RUN --mount=type=bind,target=./requirements/local.txt,src=./requirements/local.txt \ --mount=type=bind,target=./requirements/development.txt,src=./requirements/development.txt \ --mount=type=bind,target=./requirements/base.txt,src=./requirements/base.txt \ diff --git a/scripts/get_package_version.py b/scripts/get_package_version.py new file mode 100644 index 0000000000..60b5723569 --- /dev/null +++ b/scripts/get_package_version.py @@ -0,0 +1,11 @@ +import json +import os + +BASE_DIR = os.path.abspath(os.path.dirname(__file__)) +PACKAGE_JSON = os.path.join(BASE_DIR, "../", "superset-frontend", "package.json") + + +if os.path.exists(PACKAGE_JSON): + # package.json is the source of truth for version info + with open(PACKAGE_JSON) as f: + print(json.load(f)["version"]) diff --git a/setup.py b/setup.py index 97df81a695..7428ac2710 100644 --- a/setup.py +++ b/setup.py @@ -22,12 +22,21 @@ from setuptools import find_packages, setup BASE_DIR = os.path.abspath(os.path.dirname(__file__)) PACKAGE_JSON = os.path.join(BASE_DIR, "superset-frontend", "package.json") +VERSION_INFO_FILE = os.path.join(BASE_DIR, "superset", "static", "version_info.json") +DOCKER_VERSION_FILE = os.path.join(BASE_DIR, "version.txt") -with open(PACKAGE_JSON) as package_file: - version_string = json.load(package_file)["version"] -with open("README.md", encoding="utf-8") as f: - long_description = f.read() +def get_version() -> str: + version = "" + if os.path.exists(PACKAGE_JSON): + # package.json is the source of truth + with open(PACKAGE_JSON) as f: + version = json.load(f)["version"] + elif os.path.exists(DOCKER_VERSION_FILE): + # to improve docker caching, we prepare a small version.txt + with open(DOCKER_VERSION_FILE) as f: + version = f.read().strip() + return version def get_git_sha() -> str: @@ -38,24 +47,31 @@ def get_git_sha() -> str: return "" -GIT_SHA = get_git_sha() -version_info = {"GIT_SHA": GIT_SHA, "version": version_string} -print("-==-" * 15) -print("VERSION: " + version_string) -print("GIT SHA: " + GIT_SHA) -print("-==-" * 15) +def stamp_version() -> None: + """Leaving a trace in stdout and static assets""" + GIT_SHA = get_git_sha() + version_info = {"GIT_SHA": GIT_SHA, "version": VERSION} + print("-==-" * 15) + print(f"VERSION: {VERSION}") + print(f"GIT SHA: {GIT_SHA}") + print("-==-" * 15) -VERSION_INFO_FILE = os.path.join(BASE_DIR, "superset", "static", "version_info.json") + with open(VERSION_INFO_FILE, "w") as version_file: + json.dump(version_info, version_file) -with open(VERSION_INFO_FILE, "w") as version_file: - json.dump(version_info, version_file) + +VERSION = get_version() +stamp_version() + +with open("README.md", encoding="utf-8") as f: + long_description = f.read() setup( name="apache-superset", description="A modern, enterprise-ready business intelligence web application", long_description=long_description, long_description_content_type="text/markdown", - version=version_string, + version=VERSION, packages=find_packages(), include_package_data=True, zip_safe=False, @@ -211,7 +227,7 @@ setup( author="Apache Software Foundation", author_email="[email protected]", url="https://superset.apache.org/";, - download_url="https://www.apache.org/dist/superset/"; + version_string, + download_url="https://www.apache.org/dist/superset/"; + VERSION, classifiers=[ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", diff --git a/version.txt b/version.txt new file mode 100644 index 0000000000..cb676de104 --- /dev/null +++ b/version.txt @@ -0,0 +1 @@ +0.0.0-dev
