This is an automated email from the ASF dual-hosted git repository. djwang pushed a commit to branch merge-with-upstream in repository https://gitbox.apache.org/repos/asf/cloudberry-pxf.git
commit 18056b1f2d933df518535a7e5afc325be6e5a609 Author: Himanshu Pandey <[email protected]> AuthorDate: Mon Sep 12 00:37:51 2022 -0700 Temporary add GPDB header file to Install directory Manually --- .../pipelines/templates/dev_build_pipeline-tpl.yml | 173 +++++++++++++++++++-- concourse/scripts/build.bash | 16 +- concourse/scripts/initialize_gpdb.bash | 1 + concourse/tasks/build.yml | 2 + 4 files changed, 179 insertions(+), 13 deletions(-) diff --git a/concourse/pipelines/templates/dev_build_pipeline-tpl.yml b/concourse/pipelines/templates/dev_build_pipeline-tpl.yml index 659ac660..b2ace48a 100644 --- a/concourse/pipelines/templates/dev_build_pipeline-tpl.yml +++ b/concourse/pipelines/templates/dev_build_pipeline-tpl.yml @@ -142,11 +142,70 @@ resources: {% include 'resources/pxf-artifact-tpl.yml' %} {% endcall %} +- name: gpdb7_rhel8_rpm_latest-0 + type: gcs + icon: google-drive + source: + bucket: ((ud/pxf/common/gpdb-concourse-resources-prod-bucket-name)) + json_key: ((concourse-gcs-resources-service-account-key)) + regexp: server/published/master/greenplum-db-(7.*)-rhel8-x86_64.rpm + ## ---------- PXF 5 Artifact --------------- {% if multinode_el7 %} [[macros.resource_pxf5_gp6_el7_artifact()]] {% endif %} +## ---------- PXF Build Artifacts ---------- +{% set gp_ver = None %} + +{% for gp_ver in range(5, 8) %} +- name: pxf_gp[[gp_ver]]_tarball_rhel7 + type: gcs + icon: google-drive + source: + bucket: ((ud/pxf/common/build-bucket-name)) + json_key: ((ud/pxf/secrets/pxf-storage-service-account-key)) + versioned_file: dev/[[user]]-[[branch]]/snapshots/pxf6/pxf-gp[[gp_ver]].el7.tar.gz +{% endfor %} + +{% set gp_ver = 6 %} +- name: pxf_gp[[gp_ver]]_tarball_ubuntu18 + type: gcs + icon: google-drive + source: + bucket: ((ud/pxf/common/build-bucket-name)) + json_key: ((ud/pxf/secrets/pxf-storage-service-account-key)) + versioned_file: dev/[[user]]-[[branch]]/snapshots/pxf6/pxf-gp[[gp_ver]]-ubuntu18.04.tar.gz +{% set gp_ver = None %} + +{% for gp_ver in range(6, 7) %} +- name: pxf_gp[[gp_ver]]_tarball_rhel8 + type: gcs + icon: google-drive + source: + bucket: ((ud/pxf/common/build-bucket-name)) + json_key: ((ud/pxf/secrets/pxf-storage-service-account-key)) + versioned_file: dev/[[user]]-[[branch]]/snapshots/pxf6/pxf-gp[[gp_ver]].el8.tar.gz +{% endfor %} + + +- name: pxf_gp7_headerfile + type: gcs + icon: google-drive + source: + bucket: ((ud/pxf/common/build-resources-bucket-name)) + json_key: ((ud/pxf/secrets/pxf-storage-service-account-key)) + versioned_file: build-dependencies/extaccess.h + + +- name: pxf-build-dependencies + type: gcs + icon: google-drive + source: + bucket: ((ud/pxf/common/build-resources-bucket-name)) + json_key: ((ud/pxf/secrets/pxf-storage-service-account-key)) + versioned_file: build-dependencies/pxf-build-dependencies.tar.gz + ## ---------- Auxiliary Resources ---------- {% if use_ccp %} [[macros.resource_terraform('gpdb')]] @@ -161,13 +220,29 @@ resources: ## ====================================================================== jobs: -## ---------- Build and Basic single node Test Swimlanes ---------- -{# include a build job for all OS other than OEL (which does not have a separate build job) #} -{% call(x) macros.for_each_config(build_test_pxf_combinations) %} - {% if x.test_os != 'oel' %} - {% include 'jobs/build-tpl.yml' %} - {% endif %} -{% endcall %} +## ---------- Centos 7 Swimlane ---------- +{% for gp_ver in range(5, 8) %} +- name: Build PXF-GP[[gp_ver]] on RHEL7 + plan: + - in_parallel: + - get: pxf_src + trigger: true + - get: gpdb_package + resource: gpdb[[gp_ver]]_rhel7_rpm_latest-0 + - get: gpdb[[gp_ver]]-pxf-dev-centos7-image + - get: pxf-build-dependencies +{% if gp_ver == 7 %} + - get: pxf_gp7_headerfile +{% endif %} + - task: Build PXF-GP[[gp_ver]] on RHEL7 + image: gpdb[[gp_ver]]-pxf-dev-centos7-image + file: pxf_src/concourse/tasks/build.yml + params: + LICENSE: ((ud/pxf/common/rpm-license)) + VENDOR: ((ud/pxf/common/rpm-vendor)) + - put: pxf_gp[[gp_ver]]_tarball_rhel7 + params: + file: dist/pxf-gp[[gp_ver]]-*.el7.tar.gz {# include a non-FDW test job for all OS other than OEL (unless requested in the Makefile) #} {% call(x) macros.for_each_config(build_test_pxf_combinations) %} @@ -212,9 +287,87 @@ jobs: {% set use_impers = 'false' if (test_feature in clouds or test_feature == default_distro) else 'true' %} {% do x.update({'use_impers': use_impers}) %} - {# define a protocol if the feature is a cloud test #} - {% set protocol = test_feature if test_feature in clouds else '' %} - {% do x.update({'protocol': protocol}) %} +## -- PXF GP7 REHL 8 -- + + {% set gp_ver = 7 %} +- name: Build PXF-GP[[gp_ver]] on RHEL8 + plan: + - in_parallel: + - get: pxf_src + trigger: true + - get: gpdb_package + resource: gpdb[[gp_ver]]_rhel8_rpm_latest-0 + - get: gpdb[[gp_ver]]-pxf-dev-rhel8-image + - get: pxf-build-dependencies + - task: Build PXF-GP[[gp_ver]] on RHEL8 + image: gpdb[[gp_ver]]-pxf-dev-rhel8-image + file: pxf_src/concourse/tasks/build.yml + params: + LICENSE: ((ud/pxf/common/rpm-license)) + VENDOR: ((ud/pxf/common/rpm-vendor)) + GOOGLE_CREDENTIALS: ((ud/pxf/secrets/ccp-ci-service-account-key)) + GOOGLE_PROJECT_ID: ((ud/pxf/common/google-project-id)) + GOOGLE_ZONE: ((ud/pxf/common/google-zone)) + - put: pxf_gp[[gp_ver]]_tarball_rhel8 + params: + file: dist/pxf-gp[[gp_ver]]-*.el8.tar.gz + +- name: Test PXF-GP[[gp_ver]]-HDP2 on RHEL8 + plan: + - in_parallel: + - get: pxf_src + passed: [Build PXF-GP[[gp_ver]] on RHEL8] + trigger: true + - get: pxf_tarball + resource: pxf_gp[[gp_ver]]_tarball_rhel8 + passed: [Build PXF-GP[[gp_ver]] on RHEL8] + - get: gpdb_package + resource: gpdb[[gp_ver]]_rhel8_rpm_latest-0 + passed: [Build PXF-GP[[gp_ver]] on RHEL8] + - get: gpdb[[gp_ver]]-pxf-dev-rhel8-image + passed: [Build PXF-GP[[gp_ver]] on RHEL8] + - get: pxf-automation-dependencies + - get: singlecluster + resource: singlecluster-hdp2 + - task: Test PXF-GP[[gp_ver]]-HDP2 on RHEL8 + file: pxf_src/concourse/tasks/test.yml + image: gpdb[[gp_ver]]-pxf-dev-rhel8-image + params: + ACCESS_KEY_ID: ((tf-machine-access-key-id)) + GP_VER: [[gp_ver]] + GROUP: gpdb,proxy,profile +SECRET_ACCESS_KEY: ((tf-machine-secret-access-key)) + {% set gp_ver = None %} + +## ---------- Extended Tests ---------- +{% set gp_ver = 6 %} +{% if hdp2 %} +- name: Test PXF-GP[[gp_ver]]-HDP2-NO-IMPERS on RHEL7 + plan: + - in_parallel: + - get: pxf_src + passed: [Build PXF-GP[[gp_ver]] on RHEL7] + trigger: true + - get: pxf_tarball + resource: pxf_gp[[gp_ver]]_tarball_rhel7 + passed: [Build PXF-GP[[gp_ver]] on RHEL7] + - get: gpdb_package + resource: gpdb[[gp_ver]]_rhel7_rpm_latest-0 + passed: [Build PXF-GP[[gp_ver]] on RHEL7] + - get: gpdb[[gp_ver]]-pxf-dev-centos7-image + - get: pxf-automation-dependencies + - get: singlecluster + resource: singlecluster-hdp2 + - task: Test PXF-GP[[gp_ver]]-HDP2-NO-IMPERS on RHEL7 + file: pxf_src/concourse/tasks/test.yml + image: gpdb[[gp_ver]]-pxf-dev-centos7-image + params: + GP_VER: [[gp_ver]] + IMPERSONATION: false +{% if slack_notification %} + <<: *slack_alert +{% endif %} +{% endif %} {# define a feature name to go into the test job name, unless it is a test against a distro/cloud only #} {% set feature = test_feature if test_feature not in distros and test_feature not in clouds else '' %} diff --git a/concourse/scripts/build.bash b/concourse/scripts/build.bash index 6cfb731b..abef1583 100755 --- a/concourse/scripts/build.bash +++ b/concourse/scripts/build.bash @@ -97,9 +97,19 @@ function package_pxf_fdw() { install_gpdb # installation of GPDB from RPM/DEB doesn't ensure that the installation location will match the version # given in the gpdb_package, so set the GPHOME after installation -# In case we are testing a dev version (i.e: 6.25.3+dev.6.54a3437), GPDB_VERSION%%+* will remove any extra string from + onwards -# TODO: revert this change when there are publicly available distributions of rocky9 GPDB rpms -GPHOME=$(find /usr/local/ -name "greenplum-db-${GPDB_VERSION%%+*}*") +GPHOME=$(find /usr/local/ -name "greenplum-db-${GPDB_VERSION}*") + +HEADER_FILE_GP7=pxf_gp7_headerfile +if [[ ${GPDB_VERSION:0:1} -ge 7 ]]; then + #PROJECT=${GOOGLE_PROJECT_ID:-} + + #gcloud config set project "$PROJECT" + #gcloud auth activate-service-account --key-file=<(echo "$GOOGLE_CREDENTIALS") + + mkdir ${GPHOME}/include/postgresql/server/extension/gp_exttable_fdw + cp ${HEADER_FILE_GP7}/extaccess.h ${GPHOME}/include/postgresql/server/extension/gp_exttable_fdw +fi + inflate_dependencies compile_pxf package_pxf diff --git a/concourse/scripts/initialize_gpdb.bash b/concourse/scripts/initialize_gpdb.bash index 02743dea..7630e9a7 100755 --- a/concourse/scripts/initialize_gpdb.bash +++ b/concourse/scripts/initialize_gpdb.bash @@ -21,6 +21,7 @@ sed -e "s/MASTER_HOSTNAME=mdw/MASTER_HOSTNAME=\$(hostname -f)/g" \ "${GPHOME}/docs/cli_help/gpconfigs/gpinitsystem_config" >~gpadmin/gpconfigs/gpinitsystem_config chmod +w ~gpadmin/gpconfigs/gpinitsystem_config + #Script to start segments and create directories. hostname -f >/tmp/hosts.txt diff --git a/concourse/tasks/build.yml b/concourse/tasks/build.yml index 5e9f7924..2e6a61ee 100644 --- a/concourse/tasks/build.yml +++ b/concourse/tasks/build.yml @@ -9,6 +9,8 @@ inputs: optional: true - name: pxf_src - name: gpdb_package +- name: pxf_gp7_headerfile + optional: true outputs: - name: dist --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
