This is an automated email from the ASF dual-hosted git repository. djwang pushed a commit to branch merge-with-upstream in repository https://gitbox.apache.org/repos/asf/cloudberry-pxf.git
commit 26c0920e3513c12a2caeb1255c5ae9a54a7d9fbe Author: Alexander Denissov <[email protected]> AuthorDate: Thu Sep 15 17:00:27 2022 -0700 added specs for gpdb7-rhel8-test-pxf docker image (#867) --- concourse/docker/pxf-dev-base/README.md | 19 ++- concourse/docker/pxf-dev-base/cloudbuild.yaml | 33 ++++- .../docker/pxf-dev-base/gpdb7/rhel8/Dockerfile | 25 ++-- concourse/pipelines/cloudbuild_pipeline.yml | 148 +++++++++++++++++++-- 4 files changed, 197 insertions(+), 28 deletions(-) diff --git a/concourse/docker/pxf-dev-base/README.md b/concourse/docker/pxf-dev-base/README.md index 69418887..b905fb1c 100644 --- a/concourse/docker/pxf-dev-base/README.md +++ b/concourse/docker/pxf-dev-base/README.md @@ -132,7 +132,9 @@ following command to build the image: ### Docker gpdb7-rocky8-test-pxf-image image -Build this image for Greenplum 7 running on Rocky 8. Run the following +TODO: ===> remove this if Greenplum 7 will not be supported on Centos7 <=== + +Build this image for Greenplum 7 running on CentOS 7. Run the following command to build the image: pushd ~/workspace/pxf/concourse/docker/pxf-dev-base/ @@ -145,6 +147,21 @@ command to build the image: . popd +### Docker gpdb7-rhel8-test-pxf-image image + +Build this image for Greenplum 7 running on Rhel 8. Run the following +command to build the image: + + pushd ~/workspace/pxf/concourse/docker/pxf-dev-base/ + docker build \ + --build-arg=BASE_IMAGE=gcr.io/data-gpdb-private-images/gpdb7-rhel8-test:latest \ + --build-arg=GINKGO_VERSION=${GINKGO_VERSION} \ + --tag=gpdb7-rhel8-test-pxf \ + -f ~/workspace/pxf/concourse/docker/pxf-dev-base/gpdb7/rhel8/Dockerfile \ + . + popd + +### Docker gpdb7-ubuntu18.04-test-pxf-image image ### Docker gpdb7-rocky9-test-pxf-image image diff --git a/concourse/docker/pxf-dev-base/cloudbuild.yaml b/concourse/docker/pxf-dev-base/cloudbuild.yaml index 1b999d0c..d7fd8d10 100644 --- a/concourse/docker/pxf-dev-base/cloudbuild.yaml +++ b/concourse/docker/pxf-dev-base/cloudbuild.yaml @@ -224,7 +224,33 @@ steps: - apache-maven-binaries - gpdb7-rocky8-test-pxf-image-cache - # Greenplum 7 Rocky 9 Image +# Greenplum 7 Rhel 8 Image +- name: 'gcr.io/cloud-builders/docker' + id: gpdb7-rhel8-test-pxf-image-cache + entrypoint: 'bash' + args: + - '-c' + - | + mkdir -p /workspace/build + docker pull gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb7-rhel8-test-pxf:latest || exit 0 + waitFor: ['-'] + +- name: 'gcr.io/cloud-builders/docker' + id: gpdb7-rhel8-test-pxf-image + args: + - 'build' + - '--build-arg=BASE_IMAGE=${_PRIVATE_BASE_IMAGE_REPOSITORY}/gpdb7-rhel8-test:latest' + - '--build-arg=GINKGO_VERSION=${_GINKGO_VERSION}' + - '--tag=gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb7-rhel8-test-pxf:$COMMIT_SHA' + - '--cache-from' + - 'gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb7-rhel8-test-pxf:latest' + - '-f' + - 'concourse/docker/pxf-dev-base/gpdb7/rhel8/Dockerfile' + - '/workspace/build/' + waitFor: + - gpdb7-rhel8-test-pxf-image-cache + +# Greenplum 7 Ubuntu 18.04 Image - name: 'gcr.io/cloud-builders/docker' id: gpdb7-rocky9-test-pxf-image-cache entrypoint: 'bash' @@ -265,5 +291,6 @@ images: - 'gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb6-rocky9-test-pxf:$COMMIT_SHA' - 'gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb6-ubuntu18.04-test-pxf:$COMMIT_SHA' - 'gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb6-oel7-test-pxf:$COMMIT_SHA' -- 'gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb7-rocky8-test-pxf:$COMMIT_SHA' -- 'gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb7-rocky9-test-pxf:$COMMIT_SHA' +- 'gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb7-centos7-test-pxf:$COMMIT_SHA' +- 'gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb7-rhel8-test-pxf:$COMMIT_SHA' +- 'gcr.io/$PROJECT_ID/gpdb-pxf-dev/gpdb7-ubuntu18.04-test-pxf:$COMMIT_SHA' diff --git a/concourse/docker/pxf-dev-base/gpdb7/rhel8/Dockerfile b/concourse/docker/pxf-dev-base/gpdb7/rhel8/Dockerfile index 39ccd433..6217f14c 100644 --- a/concourse/docker/pxf-dev-base/gpdb7/rhel8/Dockerfile +++ b/concourse/docker/pxf-dev-base/gpdb7/rhel8/Dockerfile @@ -2,16 +2,10 @@ ARG BASE_IMAGE=gcr.io/data-gpdb-private-images/gpdb7-rhel8-test:latest FROM ${BASE_IMAGE} -ARG GO_VERSION -ARG GO_SHA256SUM - -ADD apache-maven.tar.gz /usr/share +ARG GINKGO_VERSION # install Go utilities -RUN mkdir -p /tmp/pxf_src/ && cd /tmp \ - && wget -O go.tgz -q https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz \ - && echo "${GO_SHA256SUM} /tmp/go.tgz" | sha256sum --check \ - && rm -rf /usr/local/go && tar -C /usr/local -xzf go.tgz && rm go.tgz +RUN GOPATH=/opt/go /usr/local/go/bin/go install github.com/onsi/ginkgo/ginkgo@v${GINKGO_VERSION} # add minio software RUN useradd -s /sbin/nologin -d /opt/minio minio \ @@ -22,8 +16,17 @@ RUN useradd -s /sbin/nologin -d /opt/minio minio \ && chmod +x /opt/minio/bin/minio \ && chown -R minio:minio /opt/minio +ARG MAVEN_VERSION=3.6.3 +ARG USER_HOME_DIR="/root" +ARG BASE_URL=https://apache.osuosl.org/maven/maven-3/${MAVEN_VERSION}/binaries + # install dependencies that are missing on the base images -RUN ln -s /usr/share/apache-maven-*/bin/mvn /usr/bin/mvn +RUN curl -fsSL -o /tmp/apache-maven.tar.gz ${BASE_URL}/apache-maven-${MAVEN_VERSION}-bin.tar.gz \ + && mkdir -p /usr/share/maven \ + && tar -xzf /tmp/apache-maven.tar.gz -C /usr/share/maven --strip-components=1 \ + && rm -f /tmp/apache-maven.tar.gz \ + && ln -s /usr/share/maven/bin/mvn /usr/bin/mvn \ + && /usr/bin/pip2 install paramiko --no-cache-dir # create rsa key for the root user RUN ssh-keygen -t rsa -N "" -f /root/.ssh/id_rsa \ @@ -58,11 +61,11 @@ RUN ssh-keygen -t rsa -N "" -f /root/.ssh/id_rsa \ && echo >> ~gpadmin/.pxfrc 'export PGPORT=5432' \ && echo >> ~gpadmin/.pxfrc 'export GOPATH=/opt/go' \ && echo >> ~gpadmin/.pxfrc 'export GPHOME=$(find /usr/local/ -name greenplum-db* -type d | head -n1)' \ + && echo >> ~gpadmin/.pxfrc 'export LD_LIBRARY_PATH=${GPHOME}/lib:${LD_LIBRARY_PATH-}' \ && echo >> ~gpadmin/.pxfrc 'export GPHD_ROOT=/singlecluster' \ && echo >> ~gpadmin/.pxfrc 'export PXF_HOME=/usr/local/pxf-gp7' \ && echo >> ~gpadmin/.pxfrc 'export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk' \ - && echo >> ~gpadmin/.pxfrc 'export PG_CONFIG=${GPHOME}/bin/pg_config' \ - && echo >> ~gpadmin/.pxfrc 'export PATH=${PXF_HOME}/bin:${GOPATH}/bin:/usr/local/go/bin:$PATH' \ + && echo >> ~gpadmin/.pxfrc 'export PATH=${GPHOME}/bin:${PXF_HOME}/bin:${GOPATH}/bin:/usr/local/go/bin:$JAVA_HOME/bin:$PATH' \ && ln -s ~gpadmin/.pxfrc ~root \ && echo >> ~gpadmin/.bashrc 'source ~/.pxfrc' \ && chown -R gpadmin:gpadmin ~gpadmin diff --git a/concourse/pipelines/cloudbuild_pipeline.yml b/concourse/pipelines/cloudbuild_pipeline.yml index d82a92bb..b8709082 100644 --- a/concourse/pipelines/cloudbuild_pipeline.yml +++ b/concourse/pipelines/cloudbuild_pipeline.yml @@ -92,16 +92,138 @@ jobs: - name: pxf_src run: path: pxf_src/concourse/scripts/check_docker_images_and_tag.bash - - task: trigger_builds - image: gcloud - params: - GOOGLE_CREDENTIALS: ((ud/pxf/secrets/pxf-cloudbuild-service-account-key)) - GOOGLE_PROJECT_ID: ((ud/pxf/common/google-project-id)) - GOOGLE_ZONE: ((ud/pxf/common/google-zone)) - TRIGGER_LIST: "pxf-dev-server" - config: - platform: linux - inputs: - - name: pxf_src - run: - path: pxf_src/concourse/scripts/trigger_cloudbuild.bash + +## ====================================================================== +## RESOURCES +## ====================================================================== +resources: + - name: gcloud + type: registry-image + icon: docker + source: + repository: google/cloud-sdk + tag: slim + + - name: pxf-dev-base-cloudbuild-trigger + type: git + icon: git + source: + branch: ((pxf-git-branch)) + uri: ((ud/pxf/common/git-remote)) + paths: + - concourse/docker/pxf-dev-base + + - name: dockerfile-gpdb-pxf-mapr + type: git + icon: git + source: + branch: ((pxf-git-branch)) + uri: ((ud/pxf/common/git-remote)) + paths: + - concourse/docker/mapr + + - name: base-images-trigger + type: git + icon: git + source: + uri: ((ud/pxf/common/gp-image-baking-repo-uri)) + branch: ((ud/pxf/common/gp-image-baking-repo-branch)) + private_key: ((ud/pxf/secrets/gp-image-baking-repo-key)) + paths: + - images/docker/gpdb5/gpdb5-centos7-build-test + - images/docker/gpdb6/gpdb6-centos7-test + - images/docker/gpdb6/gpdb6-ubuntu18.04-test + - images/docker/gpdb6/gpdb6-oel7-test + - images/docker/gpdb6/gpdb6-rhel8-test + - images/docker/gpdb7/gpdb7-centos7-test + - images/docker/gpdb7/gpdb7-rhel8-test + +## ====================================================================== +## JOBS +## ====================================================================== +jobs: + - name: Check upstream base images + max_in_flight: 1 + plan: + - in_parallel: + - get: gcloud + - get: pxf_src + resource: pxf-dev-base-cloudbuild-trigger + - get: base-images-trigger + trigger: true + - task: trigger_builds + image: gcloud + params: + GOOGLE_CREDENTIALS: ((ud/pxf/secrets/pxf-cloudbuild-service-account-key)) + GOOGLE_PROJECT_ID: ((ud/pxf/common/google-project-id)) + GOOGLE_ZONE: ((ud/pxf/common/google-zone)) + TRIGGER_LIST: "pxf-dev-base rpmrebuild" + config: + platform: linux + inputs: + - name: pxf_src + run: + path: pxf_src/concourse/scripts/trigger_cloudbuild.bash + + - name: Check pxf-dev-base images + max_in_flight: 1 + plan: + - in_parallel: + - get: gcloud + - get: pxf_src + resource: pxf-dev-base-cloudbuild-trigger + trigger: true + - get: base-images-trigger + passed: [Check upstream base images] + trigger: true + - task: check_docker_images_and_tag + timeout: 30m + image: gcloud + params: + GOOGLE_CREDENTIALS: ((ud/pxf/secrets/pxf-cloudbuild-service-account-key)) + GOOGLE_PROJECT_ID: ((ud/pxf/common/google-project-id)) + GOOGLE_ZONE: ((ud/pxf/common/google-zone)) + IMAGE_LIST: "gpdb5-centos7-test-pxf gpdb6-centos7-test-pxf gpdb6-rhel8-test-pxf gpdb6-ubuntu18.04-test-pxf gpdb6-oel7-test-pxf gpdb7-centos7-test-pxf gpdb7-rhel8-test-pxf gpdb7-ubuntu18.04-test-pxf" + config: + platform: linux + inputs: + - name: pxf_src + run: + path: pxf_src/concourse/scripts/check_docker_images_and_tag.bash + - task: trigger_builds + image: gcloud + params: + GOOGLE_CREDENTIALS: ((ud/pxf/secrets/pxf-cloudbuild-service-account-key)) + GOOGLE_PROJECT_ID: ((ud/pxf/common/google-project-id)) + GOOGLE_ZONE: ((ud/pxf/common/google-zone)) + TRIGGER_LIST: "pxf-dev-server pxf-dev-mapr" + config: + platform: linux + inputs: + - name: pxf_src + run: + path: pxf_src/concourse/scripts/trigger_cloudbuild.bash + + - name: Check pxf-dev-mapr images + max_in_flight: 1 + plan: + - in_parallel: + - get: gcloud + - get: pxf_src + resource: dockerfile-gpdb-pxf-mapr + trigger: true + - do: *pxf_dev_mapr_image + + - name: Check pxf-dev-mapr images 2 + max_in_flight: 1 + plan: + - in_parallel: + - get: gcloud + - get: pxf_src + resource: pxf-dev-base-cloudbuild-trigger + passed: [Check pxf-dev-base images] + trigger: true + - get: base-images-trigger + passed: [Check pxf-dev-base images] + trigger: true + - do: *pxf_dev_mapr_image --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
