Module: Mesa Branch: main Commit: 95a7b65c1422b857eefa21358ca0229ab1a44587 URL: http://cgit.freedesktop.org/mesa/mesa/commit/?id=95a7b65c1422b857eefa21358ca0229ab1a44587
Author: David Heidelberg <[email protected]> Date: Thu Jul 28 01:19:21 2022 +0200 ci: replace gzip usage with zstd where posible v2: added missing zstd to arm_build.sh Reviewed-by: Emma Anholt <[email protected]> Signed-off-by: David Heidelberg <[email protected]> Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/17776> --- .gitlab-ci/common/init-stage2.sh | 4 ++-- .gitlab-ci/container/create-rootfs.sh | 3 ++- .gitlab-ci/container/debian/arm_build.sh | 3 ++- .gitlab-ci/container/debian/x86_build-base.sh | 3 ++- .gitlab-ci/container/debian/x86_build.sh | 3 ++- .gitlab-ci/container/debian/x86_test-base.sh | 3 ++- .gitlab-ci/image-tags.yml | 12 ++++++------ .gitlab-ci/lava/lava-gitlab-ci.yml | 6 +++--- .gitlab-ci/lava/lava_job_submitter.py | 2 +- .gitlab-ci/prepare-artifacts.sh | 4 ++-- .gitlab-ci/test/gitlab-ci.yml | 2 +- 11 files changed, 25 insertions(+), 20 deletions(-) diff --git a/.gitlab-ci/common/init-stage2.sh b/.gitlab-ci/common/init-stage2.sh index ff73ec8a06f..ad6528d1948 100755 --- a/.gitlab-ci/common/init-stage2.sh +++ b/.gitlab-ci/common/init-stage2.sh @@ -149,9 +149,9 @@ cleanup # upload artifacts if [ -n "$MINIO_RESULTS_UPLOAD" ]; then - tar -czf results.tar.gz results/; + tar --zstd -cf results.tar.zst results/; ci-fairy minio login --token-file "${CI_JOB_JWT_FILE}"; - ci-fairy minio cp results.tar.gz minio://"$MINIO_RESULTS_UPLOAD"/results.tar.gz; + ci-fairy minio cp results.tar.zst minio://"$MINIO_RESULTS_UPLOAD"/results.tar.zst; fi # We still need to echo the hwci: mesa message, as some scripts rely on it, such diff --git a/.gitlab-ci/container/create-rootfs.sh b/.gitlab-ci/container/create-rootfs.sh index 7b529ec7016..53f7b7c085e 100644 --- a/.gitlab-ci/container/create-rootfs.sh +++ b/.gitlab-ci/container/create-rootfs.sh @@ -92,7 +92,8 @@ apt-get -y install --no-install-recommends \ waffle-utils \ wget \ xinit \ - xserver-xorg-core + xserver-xorg-core \ + zstd # Needed for ci-fairy, this revision is able to upload files to # MinIO and doesn't depend on git diff --git a/.gitlab-ci/container/debian/arm_build.sh b/.gitlab-ci/container/debian/arm_build.sh index aa03ade8243..e462c62ec45 100644 --- a/.gitlab-ci/container/debian/arm_build.sh +++ b/.gitlab-ci/container/debian/arm_build.sh @@ -54,7 +54,8 @@ apt-get -y install \ u-boot-tools \ wget \ xz-utils \ - zlib1g-dev + zlib1g-dev \ + zstd # Not available anymore in bullseye apt-get install -y --no-remove -t buster \ diff --git a/.gitlab-ci/container/debian/x86_build-base.sh b/.gitlab-ci/container/debian/x86_build-base.sh index 042d6c9ae2c..92c4ad60227 100644 --- a/.gitlab-ci/container/debian/x86_build-base.sh +++ b/.gitlab-ci/container/debian/x86_build-base.sh @@ -73,7 +73,8 @@ apt-get install -y --no-remove \ x11proto-gl-dev \ x11proto-randr-dev \ xz-utils \ - zlib1g-dev + zlib1g-dev \ + zstd # Needed for ci-fairy, this revision is able to upload files to MinIO pip3 install git+http://gitlab.freedesktop.org/freedesktop/ci-templates@34f4ade99434043f88e164933f570301fd18b125 diff --git a/.gitlab-ci/container/debian/x86_build.sh b/.gitlab-ci/container/debian/x86_build.sh index 80476e68eb0..67b939e7872 100644 --- a/.gitlab-ci/container/debian/x86_build.sh +++ b/.gitlab-ci/container/debian/x86_build.sh @@ -48,7 +48,8 @@ apt-get install -y --no-remove \ procps \ spirv-tools \ strace \ - time + time \ + zstd . .gitlab-ci/container/container_pre_build.sh diff --git a/.gitlab-ci/container/debian/x86_test-base.sh b/.gitlab-ci/container/debian/x86_test-base.sh index 9d522ee5a84..41e3b3c66f2 100644 --- a/.gitlab-ci/container/debian/x86_test-base.sh +++ b/.gitlab-ci/container/debian/x86_test-base.sh @@ -57,7 +57,8 @@ apt-get install -y --no-remove \ waffle-utils \ xauth \ xvfb \ - zlib1g + zlib1g \ + zstd apt-get install -y --no-install-recommends \ $STABLE_EPHEMERAL diff --git a/.gitlab-ci/image-tags.yml b/.gitlab-ci/image-tags.yml index ec01a13fcbb..a84b4dc31ec 100644 --- a/.gitlab-ci/image-tags.yml +++ b/.gitlab-ci/image-tags.yml @@ -1,21 +1,21 @@ variables: DEBIAN_X86_BUILD_BASE_IMAGE: "debian/x86_build-base" - DEBIAN_BASE_TAG: "2022-07-01-bb-llvm13" + DEBIAN_BASE_TAG: "2022-08-04-zstd" DEBIAN_X86_BUILD_IMAGE_PATH: "debian/x86_build" - DEBIAN_BUILD_TAG: "2022-07-14-directx-headers" + DEBIAN_BUILD_TAG: "2022-08-04-zstd" DEBIAN_X86_BUILD_MINGW_IMAGE_PATH: "debian/x86_build-mingw" - DEBIAN_BUILD_MINGW_TAG: "2022-07-14-directx-headers" + DEBIAN_BUILD_MINGW_TAG: "2022-08-04-zstd" DEBIAN_X86_TEST_BASE_IMAGE: "debian/x86_test-base" DEBIAN_X86_TEST_IMAGE_PATH: "debian/x86_test-gl" - DEBIAN_X86_TEST_GL_TAG: "2022-07-06-virgl-update" - DEBIAN_X86_TEST_VK_TAG: "2022-07-18-apitrace-11-1" + DEBIAN_X86_TEST_GL_TAG: "2022-08-04-zstd" + DEBIAN_X86_TEST_VK_TAG: "2022-08-04-zstd" FEDORA_X86_BUILD_TAG: "2022-04-24-spirv-tools-5" - KERNEL_ROOTFS_TAG: "2022-07-27-rootfs-zstd" + KERNEL_ROOTFS_TAG: "2022-08-04-zstd" WINDOWS_X64_VS_PATH: "windows/x64_vs" WINDOWS_X64_VS_TAG: "2022-06-15-vs-winsdk" diff --git a/.gitlab-ci/lava/lava-gitlab-ci.yml b/.gitlab-ci/lava/lava-gitlab-ci.yml index ef73ab18e9c..dce6e32d439 100755 --- a/.gitlab-ci/lava/lava-gitlab-ci.yml +++ b/.gitlab-ci/lava/lava-gitlab-ci.yml @@ -12,9 +12,9 @@ BASE_SYSTEM_MAINLINE_HOST_PATH: "${BASE_SYSTEM_HOST_PREFIX}/${FDO_UPSTREAM_REPO}/${DISTRIBUTION_TAG}/${ARCH}" BASE_SYSTEM_FORK_HOST_PATH: "${BASE_SYSTEM_HOST_PREFIX}/${CI_PROJECT_PATH}/${DISTRIBUTION_TAG}/${ARCH}" # per-job build artifacts - BUILD_PATH: "${PIPELINE_ARTIFACTS_BASE}/${CI_PROJECT_NAME}-${ARCH}.tar.gz" + BUILD_PATH: "${PIPELINE_ARTIFACTS_BASE}/${CI_PROJECT_NAME}-${ARCH}.tar.zst" JOB_ROOTFS_OVERLAY_PATH: "${JOB_ARTIFACTS_BASE}/job-rootfs-overlay.tar.gz" - JOB_RESULTS_PATH: "${JOB_ARTIFACTS_BASE}/results.tar.gz" + JOB_RESULTS_PATH: "${JOB_ARTIFACTS_BASE}/results.tar.zst" MINIO_RESULTS_UPLOAD: "${JOB_ARTIFACTS_BASE}" PIGLIT_NO_WINDOW: 1 VISIBILITY_GROUP: "Collabora+fdo" @@ -30,7 +30,7 @@ tags: - $RUNNER_TAG after_script: - - wget -q "https://${JOB_RESULTS_PATH}"; -O- | tar -xz + - wget -q "https://${JOB_RESULTS_PATH}"; -O- | tar --zstd -x .lava-test:armhf: variables: diff --git a/.gitlab-ci/lava/lava_job_submitter.py b/.gitlab-ci/lava/lava_job_submitter.py index 820aa048e28..9c4af85cfde 100755 --- a/.gitlab-ci/lava/lava_job_submitter.py +++ b/.gitlab-ci/lava/lava_job_submitter.py @@ -166,7 +166,7 @@ def generate_lava_yaml(args): run_steps += [ 'mkdir -p {}'.format(args.ci_project_dir), - 'wget -S --progress=dot:giga -O- {} | tar -xz -C {}'.format(args.build_url, args.ci_project_dir), + 'wget -S --progress=dot:giga -O- {} | tar --zstd -x -C {}'.format(args.build_url, args.ci_project_dir), 'wget -S --progress=dot:giga -O- {} | tar -xz -C /'.format(args.job_rootfs_overlay_url), # Sleep a bit to give time for bash to dump shell xtrace messages into diff --git a/.gitlab-ci/prepare-artifacts.sh b/.gitlab-ci/prepare-artifacts.sh index 46d4ab0e3a6..3933a2c984f 100755 --- a/.gitlab-ci/prepare-artifacts.sh +++ b/.gitlab-ci/prepare-artifacts.sh @@ -52,8 +52,8 @@ cp -Rp .gitlab-ci/b2c artifacts/ if [ -n "$MINIO_ARTIFACT_NAME" ]; then # Pass needed files to the test stage - MINIO_ARTIFACT_NAME="$MINIO_ARTIFACT_NAME.tar.gz" - gzip -c artifacts/install.tar > ${MINIO_ARTIFACT_NAME} + MINIO_ARTIFACT_NAME="$MINIO_ARTIFACT_NAME.tar.zst" + zstd artifacts/install.tar -o ${MINIO_ARTIFACT_NAME} ci-fairy minio login --token-file "${CI_JOB_JWT_FILE}" ci-fairy minio cp ${MINIO_ARTIFACT_NAME} minio://${PIPELINE_ARTIFACTS_BASE}/${MINIO_ARTIFACT_NAME} fi diff --git a/.gitlab-ci/test/gitlab-ci.yml b/.gitlab-ci/test/gitlab-ci.yml index c05c0b49d1b..d7611b15b33 100644 --- a/.gitlab-ci/test/gitlab-ci.yml +++ b/.gitlab-ci/test/gitlab-ci.yml @@ -124,7 +124,7 @@ # improve it even more (see https://docs.mesa3d.org/ci/bare-metal.html for # setup). - echo -e "\e[0Ksection_start:$(date +%s):artifacts_download[collapsed=true]\r\e[0KDownloading artifacts from minio" - - wget ${FDO_HTTP_CACHE_URI:-}https://${PIPELINE_ARTIFACTS_BASE}/${MINIO_ARTIFACT_NAME}.tar.gz -S --progress=dot:giga -O- | tar -xz + - wget ${FDO_HTTP_CACHE_URI:-}https://${PIPELINE_ARTIFACTS_BASE}/${MINIO_ARTIFACT_NAME}.tar.zst -S --progress=dot:giga -O- | tar --zstd -x - echo -e "\e[0Ksection_end:$(date +%s):artifacts_download\r\e[0K" artifacts: when: always
