This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new cf510a3 Make Production Dockerfile OpenShift-compatible (#9545)
cf510a3 is described below
commit cf510a30fb4bd2aae0860712f02e205b1d6c8f53
Author: Jarek Potiuk <jarek.pot...@polidea.com>
AuthorDate: Sat Jun 27 14:29:55 2020 +0200
Make Production Dockerfile OpenShift-compatible (#9545)
OpenShift (and other Kubernetes platforms) often use the approach
that they start containers with random user and root group. This is
described in the
https://docs.openshift.com/container-platform/3.7/creating_images/guidelines.html
All the files created by the "airflow" user are now belonging to
'root' group and the root group has the same access to those
files as the Airflow user.
Additionally, the random user gets automatically added
/etc/passwd entry which is name 'default'. The name of the user
can be set by setting the USER_NAME variable when starting the
container.
Closes #9248
Closes #8706
---
Dockerfile | 34 ++++++++++++++++++++++++----------
IMAGES.rst | 12 +++++++++++-
scripts/ci/libraries/_build_images.sh | 1 -
scripts/prod/entrypoint_prod.sh | 9 +++++++++
4 files changed, 44 insertions(+), 12 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index 4af6976..62930be 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -214,8 +214,7 @@ RUN pip install --user
"${AIRFLOW_INSTALL_SOURCES}[${AIRFLOW_EXTRAS}]${AIRFLOW_I
find /root/.local/ -name '*.pyc' -print0 | xargs -0 rm -r && \
find /root/.local/ -type d -name '__pycache__' -print0 | xargs -0 rm -r
-RUN \
-
AIRFLOW_SITE_PACKAGE="/root/.local/lib/python${PYTHON_MAJOR_MINOR_VERSION}/site-packages/airflow";
\
+RUN
AIRFLOW_SITE_PACKAGE="/root/.local/lib/python${PYTHON_MAJOR_MINOR_VERSION}/site-packages/airflow";
\
if [[ -f "${AIRFLOW_SITE_PACKAGE}/www_rbac/package.json" ]]; then \
WWW_DIR="${AIRFLOW_SITE_PACKAGE}/www_rbac"; \
elif [[ -f "${AIRFLOW_SITE_PACKAGE}/www/package.json" ]]; then \
@@ -227,6 +226,10 @@ RUN \
rm -rf "${WWW_DIR}/node_modules"; \
fi
+# make sure that all directories and files in .local are also group accessible
+RUN find /root/.local -executable -print0 | xargs --null chmod g+x && \
+ find /root/.local -print0 | xargs --null chmod g+rw
+
##############################################################################################
# This is the actual Airflow image - much smaller than the build one. We copy
# installed Airflow and all it's dependencies from the build image to make it
smaller.
@@ -325,36 +328,47 @@ RUN pip install --upgrade pip==${PIP_VERSION}
ENV AIRFLOW_UID=${AIRFLOW_UID}
ENV AIRFLOW_GID=${AIRFLOW_GID}
+ENV AIRFLOW__CORE__LOAD_EXAMPLES="false"
+
+ARG AIRFLOW_USER_HOME_DIR=/home/airflow
+ENV AIRFLOW_USER_HOME_DIR=${AIRFLOW_USER_HOME_DIR}
+
RUN addgroup --gid "${AIRFLOW_GID}" "airflow" && \
adduser --quiet "airflow" --uid "${AIRFLOW_UID}" \
- --ingroup "airflow" \
- --home /home/airflow
+ --gid "${AIRFLOW_GID}" \
+ --home "${AIRFLOW_USER_HOME_DIR}"
ARG AIRFLOW_HOME
ENV AIRFLOW_HOME=${AIRFLOW_HOME}
+# Make Airflow files belong to the root group and are accessible. This is to
accomodate the guidelines from
+# OpenShift
https://docs.openshift.com/enterprise/3.0/creating_images/guidelines.html
RUN mkdir -pv "${AIRFLOW_HOME}"; \
mkdir -pv "${AIRFLOW_HOME}/dags"; \
mkdir -pv "${AIRFLOW_HOME}/logs"; \
- chown -R "airflow" "${AIRFLOW_HOME}"
+ chown -R "airflow:root" "${AIRFLOW_USER_HOME_DIR}" "${AIRFLOW_HOME}"; \
+ find "${AIRFLOW_HOME}" -executable -print0 | xargs --null chmod g+x && \
+ find "${AIRFLOW_HOME}" -print0 | xargs --null chmod g+rw
-COPY --chown=airflow:airflow --from=airflow-build-image /root/.local
"/home/airflow/.local"
+COPY --chown=airflow:root --from=airflow-build-image /root/.local
"${AIRFLOW_USER_HOME_DIR}/.local"
COPY scripts/prod/entrypoint_prod.sh /entrypoint
COPY scripts/prod/clean-logs.sh /clean-logs
RUN chmod a+x /entrypoint /clean-logs
-USER airflow
+# Make /etc/passwd root-group-writeable so that user can be dynamically added
by OpenShift
+# See https://github.com/apache/airflow/issues/9248
+RUN chmod g=u /etc/passwd
-ENV PATH="/home/airflow/.local/bin:${PATH}"
+ENV PATH="${AIRFLOW_USER_HOME_DIR}/.local/bin:${PATH}"
ENV GUNICORN_CMD_ARGS="--worker-tmp-dir /dev/shm"
WORKDIR ${AIRFLOW_HOME}
-ENV AIRFLOW__CORE__LOAD_EXAMPLES="false"
-
EXPOSE 8080
+USER ${AIRFLOW_UID}
+
ENTRYPOINT ["/usr/bin/dumb-init", "--", "/entrypoint"]
CMD ["--help"]
diff --git a/IMAGES.rst b/IMAGES.rst
index 8b4b8df..99eaf77 100644
--- a/IMAGES.rst
+++ b/IMAGES.rst
@@ -399,7 +399,12 @@ The following build arguments (``--build-arg`` in docker
build command) can be u
+------------------------------------------+------------------------------------------+------------------------------------------+
| ``AIRFLOW_UID`` | ``50000``
| Airflow user UID |
+------------------------------------------+------------------------------------------+------------------------------------------+
-| ``AIRFLOW_GID`` | ``50000``
| Airflow group GID |
+| ``AIRFLOW_GID`` | ``50000``
| Airflow group GID. Note that most files |
+| |
| created on behalf of airflow user belong |
+| |
| to the ``root`` group (0) to keep |
+| |
| OpenShift Guidelines compatibility |
++------------------------------------------+------------------------------------------+------------------------------------------+
+| ``AIRFLOW_USER_HOME_DIR`` | ``/home/airflow``
| Home directory of the Airflow user |
+------------------------------------------+------------------------------------------+------------------------------------------+
| ``PIP_VERSION`` | ``19.0.2``
| version of PIP to use |
+------------------------------------------+------------------------------------------+------------------------------------------+
@@ -621,6 +626,11 @@ Using the PROD image
The PROD image entrypoint works as follows:
+* In case the user is not "airflow" (with undefined user id) and the group id
of the user is set to 0 (root),
+ then the user is dynamically added to /etc/passwd at entry using USER_NAME
variable to define the user name.
+ This is in order to accommodate the
+ `OpenShift
Guidelines<https://docs.openshift.com/enterprise/3.0/creating_images/guidelines.html>`_
+
* If ``AIRFLOW__CORE__SQL_ALCHEMY_CONN`` variable is passed to the container
and it is either mysql or postgres
SQL alchemy connection, then the connection is checked and the script waits
until the database is reachable.
diff --git a/scripts/ci/libraries/_build_images.sh
b/scripts/ci/libraries/_build_images.sh
index fcb2cc8..0f1d9aa 100644
--- a/scripts/ci/libraries/_build_images.sh
+++ b/scripts/ci/libraries/_build_images.sh
@@ -678,7 +678,6 @@ function build_prod_image() {
--build-arg ADDITIONAL_AIRFLOW_EXTRAS="${ADDITIONAL_AIRFLOW_EXTRAS}" \
--build-arg ADDITIONAL_PYTHON_DEPS="${ADDITIONAL_PYTHON_DEPS}" \
--build-arg ADDITIONAL_DEV_DEPS="${ADDITIONAL_DEV_DEPS}" \
- --build-arg ADDITIONAL_RUNTIME_DEPS="${ADDITIONAL_RUNTIME_DEPS}" \
"${DOCKER_CACHE_PROD_BUILD_DIRECTIVE[@]}" \
-t "${AIRFLOW_PROD_BUILD_IMAGE}" \
--target "airflow-build-image" \
diff --git a/scripts/prod/entrypoint_prod.sh b/scripts/prod/entrypoint_prod.sh
index 309170c..5e3b241 100755
--- a/scripts/prod/entrypoint_prod.sh
+++ b/scripts/prod/entrypoint_prod.sh
@@ -90,6 +90,15 @@ function verify_db_connection {
fi
}
+if ! whoami &> /dev/null; then
+ if [[ -w /etc/passwd ]]; then
+ echo "${USER_NAME:-default}:x:$(id -u):0:${USER_NAME:-default}
user:${AIRFLOW_USER_HOME_DIR}:/sbin/nologin" \
+ >> /etc/passwd
+ fi
+ export HOME="${AIRFLOW_USER_HOME_DIR}"
+fi
+
+
# if no DB configured - use sqlite db by default
AIRFLOW__CORE__SQL_ALCHEMY_CONN="${AIRFLOW__CORE__SQL_ALCHEMY_CONN:="sqlite:///${AIRFLOW_HOME}/airflow.db"}"
