This is an automated email from the ASF dual-hosted git repository.
yikun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-docker.git
The following commit(s) were added to refs/heads/master by this push:
new 896a36e [SPARK-40845] Add template support for SPARK_GPG_KEY and fix
GPG verify
896a36e is described below
commit 896a36e36c094bf1480f4819005e2982ea8af417
Author: Yikun Jiang <yikunk...@gmail.com>
AuthorDate: Thu Oct 20 15:38:03 2022 +0800
[SPARK-40845] Add template support for SPARK_GPG_KEY and fix GPG verify
### What changes were proposed in this pull request?
This patch:
- Add template support for `SPARK_GPG_KEY`.
- Fix a bug on GPG verified. (Change `||` to `;`)
- Use opengpg.org instead of gpg.com becasue it would be uploaded in [spark
release process](https://spark.apache.org/release-process.html).
### Why are the changes needed?
Each version have specific GPG key to verified, so we need to set GPG
version separately.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
CI passed
Run `./add-dockerfiles.sh 3.3.0` and see GPG set correctly
Closes #16 from Yikun/GPG.
Authored-by: Yikun Jiang <yikunk...@gmail.com>
Signed-off-by: Yikun Jiang <yikunk...@gmail.com>
---
3.3.0/scala2.12-java11-python3-r-ubuntu/Dockerfile | 6 +++---
3.3.0/scala2.12-java11-python3-ubuntu/Dockerfile | 6 +++---
3.3.0/scala2.12-java11-r-ubuntu/Dockerfile | 6 +++---
3.3.0/scala2.12-java11-ubuntu/Dockerfile | 6 +++---
Dockerfile.template | 6 +++---
tools/template.py | 6 ++++++
6 files changed, 21 insertions(+), 15 deletions(-)
diff --git a/3.3.0/scala2.12-java11-python3-r-ubuntu/Dockerfile
b/3.3.0/scala2.12-java11-python3-r-ubuntu/Dockerfile
index be9cbb0..ac16bdd 100644
--- a/3.3.0/scala2.12-java11-python3-r-ubuntu/Dockerfile
+++ b/3.3.0/scala2.12-java11-python3-r-ubuntu/Dockerfile
@@ -45,7 +45,7 @@ RUN set -ex && \
# https://downloads.apache.org/spark/KEYS
ENV
SPARK_TGZ_URL=https://dlcdn.apache.org/spark/spark-3.3.0/spark-3.3.0-bin-hadoop3.tgz
\
SPARK_TGZ_ASC_URL=https://downloads.apache.org/spark/spark-3.3.0/spark-3.3.0-bin-hadoop3.tgz.asc
\
- GPG_KEY=E298A3A825C0D65DFD57CBB651716619E084DAB9
+ GPG_KEY=80FB8EBE8EBA68504989703491B5DC815DBF10D3
RUN set -ex; \
export SPARK_TMP="$(mktemp -d)"; \
@@ -53,8 +53,8 @@ RUN set -ex; \
wget -nv -O spark.tgz "$SPARK_TGZ_URL"; \
wget -nv -O spark.tgz.asc "$SPARK_TGZ_ASC_URL"; \
export GNUPGHOME="$(mktemp -d)"; \
- gpg --keyserver hkps://keyserver.pgp.com --recv-key "$GPG_KEY" || \
- gpg --keyserver hkps://keyserver.ubuntu.com --recv-keys "$GPG_KEY" || \
+ gpg --keyserver hkps://keys.openpgp.org --recv-key "$GPG_KEY" || \
+ gpg --keyserver hkps://keyserver.ubuntu.com --recv-keys "$GPG_KEY"; \
gpg --batch --verify spark.tgz.asc spark.tgz; \
gpgconf --kill all; \
rm -rf "$GNUPGHOME" spark.tgz.asc; \
diff --git a/3.3.0/scala2.12-java11-python3-ubuntu/Dockerfile
b/3.3.0/scala2.12-java11-python3-ubuntu/Dockerfile
index 096c7eb..c6e433d 100644
--- a/3.3.0/scala2.12-java11-python3-ubuntu/Dockerfile
+++ b/3.3.0/scala2.12-java11-python3-ubuntu/Dockerfile
@@ -44,7 +44,7 @@ RUN set -ex && \
# https://downloads.apache.org/spark/KEYS
ENV
SPARK_TGZ_URL=https://dlcdn.apache.org/spark/spark-3.3.0/spark-3.3.0-bin-hadoop3.tgz
\
SPARK_TGZ_ASC_URL=https://downloads.apache.org/spark/spark-3.3.0/spark-3.3.0-bin-hadoop3.tgz.asc
\
- GPG_KEY=E298A3A825C0D65DFD57CBB651716619E084DAB9
+ GPG_KEY=80FB8EBE8EBA68504989703491B5DC815DBF10D3
RUN set -ex; \
export SPARK_TMP="$(mktemp -d)"; \
@@ -52,8 +52,8 @@ RUN set -ex; \
wget -nv -O spark.tgz "$SPARK_TGZ_URL"; \
wget -nv -O spark.tgz.asc "$SPARK_TGZ_ASC_URL"; \
export GNUPGHOME="$(mktemp -d)"; \
- gpg --keyserver hkps://keyserver.pgp.com --recv-key "$GPG_KEY" || \
- gpg --keyserver hkps://keyserver.ubuntu.com --recv-keys "$GPG_KEY" || \
+ gpg --keyserver hkps://keys.openpgp.org --recv-key "$GPG_KEY" || \
+ gpg --keyserver hkps://keyserver.ubuntu.com --recv-keys "$GPG_KEY"; \
gpg --batch --verify spark.tgz.asc spark.tgz; \
gpgconf --kill all; \
rm -rf "$GNUPGHOME" spark.tgz.asc; \
diff --git a/3.3.0/scala2.12-java11-r-ubuntu/Dockerfile
b/3.3.0/scala2.12-java11-r-ubuntu/Dockerfile
index 2e085a2..975e444 100644
--- a/3.3.0/scala2.12-java11-r-ubuntu/Dockerfile
+++ b/3.3.0/scala2.12-java11-r-ubuntu/Dockerfile
@@ -42,7 +42,7 @@ RUN set -ex && \
# https://downloads.apache.org/spark/KEYS
ENV
SPARK_TGZ_URL=https://dlcdn.apache.org/spark/spark-3.3.0/spark-3.3.0-bin-hadoop3.tgz
\
SPARK_TGZ_ASC_URL=https://downloads.apache.org/spark/spark-3.3.0/spark-3.3.0-bin-hadoop3.tgz.asc
\
- GPG_KEY=E298A3A825C0D65DFD57CBB651716619E084DAB9
+ GPG_KEY=80FB8EBE8EBA68504989703491B5DC815DBF10D3
RUN set -ex; \
export SPARK_TMP="$(mktemp -d)"; \
@@ -50,8 +50,8 @@ RUN set -ex; \
wget -nv -O spark.tgz "$SPARK_TGZ_URL"; \
wget -nv -O spark.tgz.asc "$SPARK_TGZ_ASC_URL"; \
export GNUPGHOME="$(mktemp -d)"; \
- gpg --keyserver hkps://keyserver.pgp.com --recv-key "$GPG_KEY" || \
- gpg --keyserver hkps://keyserver.ubuntu.com --recv-keys "$GPG_KEY" || \
+ gpg --keyserver hkps://keys.openpgp.org --recv-key "$GPG_KEY" || \
+ gpg --keyserver hkps://keyserver.ubuntu.com --recv-keys "$GPG_KEY"; \
gpg --batch --verify spark.tgz.asc spark.tgz; \
gpgconf --kill all; \
rm -rf "$GNUPGHOME" spark.tgz.asc; \
diff --git a/3.3.0/scala2.12-java11-ubuntu/Dockerfile
b/3.3.0/scala2.12-java11-ubuntu/Dockerfile
index 5858e2d..c38e5dc 100644
--- a/3.3.0/scala2.12-java11-ubuntu/Dockerfile
+++ b/3.3.0/scala2.12-java11-ubuntu/Dockerfile
@@ -41,7 +41,7 @@ RUN set -ex && \
# https://downloads.apache.org/spark/KEYS
ENV
SPARK_TGZ_URL=https://dlcdn.apache.org/spark/spark-3.3.0/spark-3.3.0-bin-hadoop3.tgz
\
SPARK_TGZ_ASC_URL=https://downloads.apache.org/spark/spark-3.3.0/spark-3.3.0-bin-hadoop3.tgz.asc
\
- GPG_KEY=E298A3A825C0D65DFD57CBB651716619E084DAB9
+ GPG_KEY=80FB8EBE8EBA68504989703491B5DC815DBF10D3
RUN set -ex; \
export SPARK_TMP="$(mktemp -d)"; \
@@ -49,8 +49,8 @@ RUN set -ex; \
wget -nv -O spark.tgz "$SPARK_TGZ_URL"; \
wget -nv -O spark.tgz.asc "$SPARK_TGZ_ASC_URL"; \
export GNUPGHOME="$(mktemp -d)"; \
- gpg --keyserver hkps://keyserver.pgp.com --recv-key "$GPG_KEY" || \
- gpg --keyserver hkps://keyserver.ubuntu.com --recv-keys "$GPG_KEY" || \
+ gpg --keyserver hkps://keys.openpgp.org --recv-key "$GPG_KEY" || \
+ gpg --keyserver hkps://keyserver.ubuntu.com --recv-keys "$GPG_KEY"; \
gpg --batch --verify spark.tgz.asc spark.tgz; \
gpgconf --kill all; \
rm -rf "$GNUPGHOME" spark.tgz.asc; \
diff --git a/Dockerfile.template b/Dockerfile.template
index a031b16..2b90fe5 100644
--- a/Dockerfile.template
+++ b/Dockerfile.template
@@ -51,7 +51,7 @@ RUN set -ex && \
# https://downloads.apache.org/spark/KEYS
ENV SPARK_TGZ_URL=https://dlcdn.apache.org/spark/spark-{{ SPARK_VERSION
}}/spark-{{ SPARK_VERSION }}-bin-hadoop3.tgz \
SPARK_TGZ_ASC_URL=https://downloads.apache.org/spark/spark-{{
SPARK_VERSION }}/spark-{{ SPARK_VERSION }}-bin-hadoop3.tgz.asc \
- GPG_KEY=E298A3A825C0D65DFD57CBB651716619E084DAB9
+ GPG_KEY={{ SPARK_GPG_KEY }}
RUN set -ex; \
export SPARK_TMP="$(mktemp -d)"; \
@@ -59,8 +59,8 @@ RUN set -ex; \
wget -nv -O spark.tgz "$SPARK_TGZ_URL"; \
wget -nv -O spark.tgz.asc "$SPARK_TGZ_ASC_URL"; \
export GNUPGHOME="$(mktemp -d)"; \
- gpg --keyserver hkps://keyserver.pgp.com --recv-key "$GPG_KEY" || \
- gpg --keyserver hkps://keyserver.ubuntu.com --recv-keys "$GPG_KEY" || \
+ gpg --keyserver hkps://keys.openpgp.org --recv-key "$GPG_KEY" || \
+ gpg --keyserver hkps://keyserver.ubuntu.com --recv-keys "$GPG_KEY"; \
gpg --batch --verify spark.tgz.asc spark.tgz; \
gpgconf --kill all; \
rm -rf "$GNUPGHOME" spark.tgz.asc; \
diff --git a/tools/template.py b/tools/template.py
index e5e9a65..721f968 100755
--- a/tools/template.py
+++ b/tools/template.py
@@ -21,6 +21,11 @@ from argparse import ArgumentParser
from jinja2 import Environment, FileSystemLoader
+GPG_KEY_DICT = {
+ # issuer "maxg...@apache.org"
+ "3.3.0": "80FB8EBE8EBA68504989703491B5DC815DBF10D3",
+}
+
def parse_opts():
parser = ArgumentParser(prog="template")
@@ -76,6 +81,7 @@ def main():
HAVE_PY=opts.pyspark,
HAVE_R=opts.sparkr,
SPARK_VERSION=opts.spark_version,
+ SPARK_GPG_KEY=GPG_KEY_DICT.get(opts.spark_version),
)
)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
