This is an automated email from the ASF dual-hosted git repository.

chesnay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink-docker.git

commit 2991299ee8d42ddc741417864e75fc1832984fe5
Author: Chesnay Schepler <ches...@apache.org>
AuthorDate: Fri Apr 24 10:42:05 2020 +0200

    [FLINK-17367] Add script for building with configurable download URL
---
 .gitignore                        |  3 ++-
 1.10/scala_2.11-debian/Dockerfile | 29 ++++++++++++-----------
 1.10/scala_2.12-debian/Dockerfile | 29 ++++++++++++-----------
 1.9/scala_2.11-debian/Dockerfile  | 29 ++++++++++++-----------
 1.9/scala_2.12-debian/Dockerfile  | 29 ++++++++++++-----------
 Dockerfile-debian.template        | 29 ++++++++++++-----------
 add-custom.sh                     | 49 +++++++++++++++++++++++++++++++++++++++
 add-version.sh                    | 27 ++++-----------------
 common.sh                         | 26 +++++++++++++++++++++
 9 files changed, 161 insertions(+), 89 deletions(-)

diff --git a/.gitignore b/.gitignore
index c34e36b..feb151f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,3 @@
 .*.swp
-.idea
\ No newline at end of file
+.idea
+dev
\ No newline at end of file
diff --git a/1.10/scala_2.11-debian/Dockerfile 
b/1.10/scala_2.11-debian/Dockerfile
index 5c84be8..f7b5f6d 100644
--- a/1.10/scala_2.11-debian/Dockerfile
+++ b/1.10/scala_2.11-debian/Dockerfile
@@ -46,7 +46,8 @@ RUN set -ex; \
 # Configure Flink version
 ENV 
FLINK_TGZ_URL=https://www.apache.org/dyn/closer.cgi?action=download&filename=flink/flink-1.10.0/flink-1.10.0-bin-scala_2.11.tgz
 \
     
FLINK_ASC_URL=https://www.apache.org/dist/flink/flink-1.10.0/flink-1.10.0-bin-scala_2.11.tgz.asc
 \
-    GPG_KEY=BB137807CEFBE7DD2616556710B12A1F89C115E8
+    GPG_KEY=BB137807CEFBE7DD2616556710B12A1F89C115E8 \
+    SKIP_GPG=false
 
 # Prepare environment
 ENV FLINK_HOME=/opt/flink
@@ -58,19 +59,21 @@ WORKDIR $FLINK_HOME
 # Install Flink
 RUN set -ex; \
   wget -nv -O flink.tgz "$FLINK_TGZ_URL"; \
-  wget -nv -O flink.tgz.asc "$FLINK_ASC_URL"; \
   \
-  export GNUPGHOME="$(mktemp -d)"; \
-  for server in ha.pool.sks-keyservers.net $(shuf -e \
-                          hkp://p80.pool.sks-keyservers.net:80 \
-                          keyserver.ubuntu.com \
-                          hkp://keyserver.ubuntu.com:80 \
-                          pgp.mit.edu) ; do \
-      gpg --batch --keyserver "$server" --recv-keys "$GPG_KEY" && break || : ; 
\
-  done && \
-  gpg --batch --verify flink.tgz.asc flink.tgz; \
-  gpgconf --kill all; \
-  rm -rf "$GNUPGHOME" flink.tgz.asc; \
+  if [ "$SKIP_GPG" = "false" ]; then \
+    wget -nv -O flink.tgz.asc "$FLINK_ASC_URL"; \
+    export GNUPGHOME="$(mktemp -d)"; \
+    for server in ha.pool.sks-keyservers.net $(shuf -e \
+                            hkp://p80.pool.sks-keyservers.net:80 \
+                            keyserver.ubuntu.com \
+                            hkp://keyserver.ubuntu.com:80 \
+                            pgp.mit.edu) ; do \
+        gpg --batch --keyserver "$server" --recv-keys "$GPG_KEY" && break || : 
; \
+    done && \
+    gpg --batch --verify flink.tgz.asc flink.tgz; \
+    gpgconf --kill all; \
+    rm -rf "$GNUPGHOME" flink.tgz.asc; \
+  fi; \
   \
   tar -xf flink.tgz --strip-components=1; \
   rm flink.tgz; \
diff --git a/1.10/scala_2.12-debian/Dockerfile 
b/1.10/scala_2.12-debian/Dockerfile
index c8e1c09..cf22764 100644
--- a/1.10/scala_2.12-debian/Dockerfile
+++ b/1.10/scala_2.12-debian/Dockerfile
@@ -46,7 +46,8 @@ RUN set -ex; \
 # Configure Flink version
 ENV 
FLINK_TGZ_URL=https://www.apache.org/dyn/closer.cgi?action=download&filename=flink/flink-1.10.0/flink-1.10.0-bin-scala_2.12.tgz
 \
     
FLINK_ASC_URL=https://www.apache.org/dist/flink/flink-1.10.0/flink-1.10.0-bin-scala_2.12.tgz.asc
 \
-    GPG_KEY=BB137807CEFBE7DD2616556710B12A1F89C115E8
+    GPG_KEY=BB137807CEFBE7DD2616556710B12A1F89C115E8 \
+    SKIP_GPG=false
 
 # Prepare environment
 ENV FLINK_HOME=/opt/flink
@@ -58,19 +59,21 @@ WORKDIR $FLINK_HOME
 # Install Flink
 RUN set -ex; \
   wget -nv -O flink.tgz "$FLINK_TGZ_URL"; \
-  wget -nv -O flink.tgz.asc "$FLINK_ASC_URL"; \
   \
-  export GNUPGHOME="$(mktemp -d)"; \
-  for server in ha.pool.sks-keyservers.net $(shuf -e \
-                          hkp://p80.pool.sks-keyservers.net:80 \
-                          keyserver.ubuntu.com \
-                          hkp://keyserver.ubuntu.com:80 \
-                          pgp.mit.edu) ; do \
-      gpg --batch --keyserver "$server" --recv-keys "$GPG_KEY" && break || : ; 
\
-  done && \
-  gpg --batch --verify flink.tgz.asc flink.tgz; \
-  gpgconf --kill all; \
-  rm -rf "$GNUPGHOME" flink.tgz.asc; \
+  if [ "$SKIP_GPG" = "false" ]; then \
+    wget -nv -O flink.tgz.asc "$FLINK_ASC_URL"; \
+    export GNUPGHOME="$(mktemp -d)"; \
+    for server in ha.pool.sks-keyservers.net $(shuf -e \
+                            hkp://p80.pool.sks-keyservers.net:80 \
+                            keyserver.ubuntu.com \
+                            hkp://keyserver.ubuntu.com:80 \
+                            pgp.mit.edu) ; do \
+        gpg --batch --keyserver "$server" --recv-keys "$GPG_KEY" && break || : 
; \
+    done && \
+    gpg --batch --verify flink.tgz.asc flink.tgz; \
+    gpgconf --kill all; \
+    rm -rf "$GNUPGHOME" flink.tgz.asc; \
+  fi; \
   \
   tar -xf flink.tgz --strip-components=1; \
   rm flink.tgz; \
diff --git a/1.9/scala_2.11-debian/Dockerfile b/1.9/scala_2.11-debian/Dockerfile
index 8a82570..8f71da8 100644
--- a/1.9/scala_2.11-debian/Dockerfile
+++ b/1.9/scala_2.11-debian/Dockerfile
@@ -46,7 +46,8 @@ RUN set -ex; \
 # Configure Flink version
 ENV 
FLINK_TGZ_URL=https://www.apache.org/dyn/closer.cgi?action=download&filename=flink/flink-1.9.3/flink-1.9.3-bin-scala_2.11.tgz
 \
     
FLINK_ASC_URL=https://www.apache.org/dist/flink/flink-1.9.3/flink-1.9.3-bin-scala_2.11.tgz.asc
 \
-    GPG_KEY=6B6291A8502BA8F0913AE04DDEB95B05BF075300
+    GPG_KEY=6B6291A8502BA8F0913AE04DDEB95B05BF075300 \
+    SKIP_GPG=false
 
 # Prepare environment
 ENV FLINK_HOME=/opt/flink
@@ -58,19 +59,21 @@ WORKDIR $FLINK_HOME
 # Install Flink
 RUN set -ex; \
   wget -nv -O flink.tgz "$FLINK_TGZ_URL"; \
-  wget -nv -O flink.tgz.asc "$FLINK_ASC_URL"; \
   \
-  export GNUPGHOME="$(mktemp -d)"; \
-  for server in ha.pool.sks-keyservers.net $(shuf -e \
-                          hkp://p80.pool.sks-keyservers.net:80 \
-                          keyserver.ubuntu.com \
-                          hkp://keyserver.ubuntu.com:80 \
-                          pgp.mit.edu) ; do \
-      gpg --batch --keyserver "$server" --recv-keys "$GPG_KEY" && break || : ; 
\
-  done && \
-  gpg --batch --verify flink.tgz.asc flink.tgz; \
-  gpgconf --kill all; \
-  rm -rf "$GNUPGHOME" flink.tgz.asc; \
+  if [ "$SKIP_GPG" = "false" ]; then \
+    wget -nv -O flink.tgz.asc "$FLINK_ASC_URL"; \
+    export GNUPGHOME="$(mktemp -d)"; \
+    for server in ha.pool.sks-keyservers.net $(shuf -e \
+                            hkp://p80.pool.sks-keyservers.net:80 \
+                            keyserver.ubuntu.com \
+                            hkp://keyserver.ubuntu.com:80 \
+                            pgp.mit.edu) ; do \
+        gpg --batch --keyserver "$server" --recv-keys "$GPG_KEY" && break || : 
; \
+    done && \
+    gpg --batch --verify flink.tgz.asc flink.tgz; \
+    gpgconf --kill all; \
+    rm -rf "$GNUPGHOME" flink.tgz.asc; \
+  fi; \
   \
   tar -xf flink.tgz --strip-components=1; \
   rm flink.tgz; \
diff --git a/1.9/scala_2.12-debian/Dockerfile b/1.9/scala_2.12-debian/Dockerfile
index dc30005..095e9d2 100644
--- a/1.9/scala_2.12-debian/Dockerfile
+++ b/1.9/scala_2.12-debian/Dockerfile
@@ -46,7 +46,8 @@ RUN set -ex; \
 # Configure Flink version
 ENV 
FLINK_TGZ_URL=https://www.apache.org/dyn/closer.cgi?action=download&filename=flink/flink-1.9.3/flink-1.9.3-bin-scala_2.12.tgz
 \
     
FLINK_ASC_URL=https://www.apache.org/dist/flink/flink-1.9.3/flink-1.9.3-bin-scala_2.12.tgz.asc
 \
-    GPG_KEY=6B6291A8502BA8F0913AE04DDEB95B05BF075300
+    GPG_KEY=6B6291A8502BA8F0913AE04DDEB95B05BF075300 \
+    SKIP_GPG=false
 
 # Prepare environment
 ENV FLINK_HOME=/opt/flink
@@ -58,19 +59,21 @@ WORKDIR $FLINK_HOME
 # Install Flink
 RUN set -ex; \
   wget -nv -O flink.tgz "$FLINK_TGZ_URL"; \
-  wget -nv -O flink.tgz.asc "$FLINK_ASC_URL"; \
   \
-  export GNUPGHOME="$(mktemp -d)"; \
-  for server in ha.pool.sks-keyservers.net $(shuf -e \
-                          hkp://p80.pool.sks-keyservers.net:80 \
-                          keyserver.ubuntu.com \
-                          hkp://keyserver.ubuntu.com:80 \
-                          pgp.mit.edu) ; do \
-      gpg --batch --keyserver "$server" --recv-keys "$GPG_KEY" && break || : ; 
\
-  done && \
-  gpg --batch --verify flink.tgz.asc flink.tgz; \
-  gpgconf --kill all; \
-  rm -rf "$GNUPGHOME" flink.tgz.asc; \
+  if [ "$SKIP_GPG" = "false" ]; then \
+    wget -nv -O flink.tgz.asc "$FLINK_ASC_URL"; \
+    export GNUPGHOME="$(mktemp -d)"; \
+    for server in ha.pool.sks-keyservers.net $(shuf -e \
+                            hkp://p80.pool.sks-keyservers.net:80 \
+                            keyserver.ubuntu.com \
+                            hkp://keyserver.ubuntu.com:80 \
+                            pgp.mit.edu) ; do \
+        gpg --batch --keyserver "$server" --recv-keys "$GPG_KEY" && break || : 
; \
+    done && \
+    gpg --batch --verify flink.tgz.asc flink.tgz; \
+    gpgconf --kill all; \
+    rm -rf "$GNUPGHOME" flink.tgz.asc; \
+  fi; \
   \
   tar -xf flink.tgz --strip-components=1; \
   rm flink.tgz; \
diff --git a/Dockerfile-debian.template b/Dockerfile-debian.template
index 0a206e6..7222314 100644
--- a/Dockerfile-debian.template
+++ b/Dockerfile-debian.template
@@ -46,7 +46,8 @@ RUN set -ex; \
 # Configure Flink version
 ENV FLINK_TGZ_URL=%%BINARY_DOWNLOAD_URL%% \
     FLINK_ASC_URL=%%ASC_DOWNLOAD_URL%% \
-    GPG_KEY=%%GPG_KEY%%
+    GPG_KEY=%%GPG_KEY%% \
+    CHECK_GPG=%%CHECK_GPG%%
 
 # Prepare environment
 ENV FLINK_HOME=/opt/flink
@@ -58,19 +59,21 @@ WORKDIR $FLINK_HOME
 # Install Flink
 RUN set -ex; \
   wget -nv -O flink.tgz "$FLINK_TGZ_URL"; \
-  wget -nv -O flink.tgz.asc "$FLINK_ASC_URL"; \
   \
-  export GNUPGHOME="$(mktemp -d)"; \
-  for server in ha.pool.sks-keyservers.net $(shuf -e \
-                          hkp://p80.pool.sks-keyservers.net:80 \
-                          keyserver.ubuntu.com \
-                          hkp://keyserver.ubuntu.com:80 \
-                          pgp.mit.edu) ; do \
-      gpg --batch --keyserver "$server" --recv-keys "$GPG_KEY" && break || : ; 
\
-  done && \
-  gpg --batch --verify flink.tgz.asc flink.tgz; \
-  gpgconf --kill all; \
-  rm -rf "$GNUPGHOME" flink.tgz.asc; \
+  if [ "$CHECK_GPG" = "true" ]; then \
+    wget -nv -O flink.tgz.asc "$FLINK_ASC_URL"; \
+    export GNUPGHOME="$(mktemp -d)"; \
+    for server in ha.pool.sks-keyservers.net $(shuf -e \
+                            hkp://p80.pool.sks-keyservers.net:80 \
+                            keyserver.ubuntu.com \
+                            hkp://keyserver.ubuntu.com:80 \
+                            pgp.mit.edu) ; do \
+        gpg --batch --keyserver "$server" --recv-keys "$GPG_KEY" && break || : 
; \
+    done && \
+    gpg --batch --verify flink.tgz.asc flink.tgz; \
+    gpgconf --kill all; \
+    rm -rf "$GNUPGHOME" flink.tgz.asc; \
+  fi; \
   \
   tar -xf flink.tgz --strip-components=1; \
   rm flink.tgz; \
diff --git a/add-custom.sh b/add-custom.sh
new file mode 100755
index 0000000..c863f3f
--- /dev/null
+++ b/add-custom.sh
@@ -0,0 +1,49 @@
+#!/bin/bash -e
+
+# Use this script to build the Dockerfiles against an arbitrary
+# Flink distribution.
+# This is exlusively for development purposes.
+
+source "$(dirname "$0")"/common.sh
+
+function usage() {
+    echo >&2 "usage: $0 -u binary-download-url [-n name]"
+}
+
+binary_download_url=
+name=custom
+
+while getopts u:n:h arg; do
+  case "$arg" in
+    u)
+      binary_download_url=$OPTARG
+      ;;
+    n)
+      name=$OPTARG
+      ;;
+    h)
+      usage
+      exit 0
+      ;;
+    \?)
+      usage
+      exit 1
+      ;;
+  esac
+done
+
+if [ -z "${binary_download_url}" ]; then
+    usage
+    exit 1
+fi
+
+mkdir -p "dev"
+
+echo -n >&2 "Generating Dockerfiles..."
+for source_variant in "${SOURCE_VARIANTS[@]}"; do
+  dir="dev/${name}-${source_variant}"
+  rm -rf "${dir}"
+
+  generate "${dir}" "${binary_download_url}" "" "" false ${source_variant}
+done
+echo >&2 " done."
diff --git a/add-version.sh b/add-version.sh
index 779113b..329ed70 100755
--- a/add-version.sh
+++ b/add-version.sh
@@ -10,6 +10,8 @@
 #
 # See other repos (e.g. httpd, cassandra) for update.sh examples.
 
+source "$(dirname "$0")"/common.sh
+
 function usage() {
     echo >&2 "usage: $0 -r flink-release -f flink-version"
 }
@@ -57,7 +59,6 @@ if [[ ! "$flink_version" =~ ^$flink_release\.+ ]]; then
 fi
 
 # Defaults, can vary between versions
-source_variants=( debian )
 scala_versions=( 2.11 2.12 )
 gpg_key=
 
@@ -95,28 +96,8 @@ fi
 
 mkdir "$flink_release"
 
-function generate() {
-    dir=$1
-    binary_download_url=$2
-    asc_download_url=$3
-    gpg_key=$4
-    source_variant=$5
-
-    mkdir "$dir"
-    cp docker-entrypoint.sh "$dir/docker-entrypoint.sh"
-
-    # '&' has special semantics in sed replacement patterns
-    escaped_binary_download_url=$(echo "$binary_download_url" | sed 
's/&/\\\&/')
-
-    sed \
-        -e "s,%%BINARY_DOWNLOAD_URL%%,${escaped_binary_download_url}," \
-        -e "s,%%ASC_DOWNLOAD_URL%%,$asc_download_url," \
-        -e "s/%%GPG_KEY%%/$gpg_key/" \
-        "Dockerfile-$source_variant.template" > "$dir/Dockerfile"
-}
-
 echo -n >&2 "Generating Dockerfiles..."
-for source_variant in "${source_variants[@]}"; do
+for source_variant in "${SOURCE_VARIANTS[@]}"; do
     for scala_version in "${scala_versions[@]}"; do
         dir="$flink_release/scala_${scala_version}-${source_variant}"
 
@@ -126,7 +107,7 @@ for source_variant in "${source_variants[@]}"; do
         # Not all mirrors have the .asc files
         flink_asc_url=https://www.apache.org/dist/${flink_url_file_path}.asc
 
-        generate "${dir}" "${flink_tgz_url}" "${flink_asc_url}" ${gpg_key} 
${source_variant}
+        generate "${dir}" "${flink_tgz_url}" "${flink_asc_url}" ${gpg_key} 
true ${source_variant}
     done
 done
 echo >&2 " done."
diff --git a/common.sh b/common.sh
new file mode 100644
index 0000000..0232930
--- /dev/null
+++ b/common.sh
@@ -0,0 +1,26 @@
+#!/bin/bash -e
+
+# Defaults, can vary between versions
+export SOURCE_VARIANTS=( debian )
+
+function generate() {
+    dir=$1
+    binary_download_url=$2
+    asc_download_url=$3
+    gpg_key=$4
+    check_gpg=$5
+    source_variant=$6
+
+    mkdir "$dir"
+    cp docker-entrypoint.sh "$dir/docker-entrypoint.sh"
+
+    # '&' has special semantics in sed replacement patterns
+    escaped_binary_download_url=$(echo "$binary_download_url" | sed 
's/&/\\\&/')
+
+    sed \
+        -e "s,%%BINARY_DOWNLOAD_URL%%,${escaped_binary_download_url}," \
+        -e "s,%%ASC_DOWNLOAD_URL%%,$asc_download_url," \
+        -e "s/%%GPG_KEY%%/$gpg_key/" \
+        -e "s/%%CHECK_GPG%%/${check_gpg}/" \
+        "Dockerfile-$source_variant.template" > "$dir/Dockerfile"
+}

Reply via email to