[clang] [OffloadBundler] Compress bundles over 4GB (PR #122307)

2025-01-20 Thread LLVM Continuous Integration via cfe-commits 

llvm-ci wrote:

LLVM Buildbot has detected a new failure on builder `clang-aarch64-global-isel` 
running on `linaro-clang-aarch64-global-isel` while building `clang` at step 7 
"ninja check 1".

Full details are available at: 
https://lab.llvm.org/buildbot/#/builders/125/builds/5078


Here is the relevant piece of the build log for the reference

```
Step 7 (ninja check 1) failure: stage 1 checked (failure)
 TEST 'Clang :: Driver/clang-offload-bundler-zlib.c' FAILED 

Exit Code: 1

Command Output (stderr):
--
RUN: at line 7: 
/home/tcwg-buildbot/worker/clang-aarch64-global-isel/stage1/bin/clang -O0 
-target aarch64-unknown-linux-gnu 
/home/tcwg-buildbot/worker/clang-aarch64-global-isel/llvm/clang/test/Driver/clang-offload-bundler-zlib.c
 -c -emit-llvm -o 
/home/tcwg-buildbot/worker/clang-aarch64-global-isel/stage1/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.bc
+ /home/tcwg-buildbot/worker/clang-aarch64-global-isel/stage1/bin/clang -O0 
-target aarch64-unknown-linux-gnu 
/home/tcwg-buildbot/worker/clang-aarch64-global-isel/llvm/clang/test/Driver/clang-offload-bundler-zlib.c
 -c -emit-llvm -o 
/home/tcwg-buildbot/worker/clang-aarch64-global-isel/stage1/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.bc
RUN: at line 12: touch 
/home/tcwg-buildbot/worker/clang-aarch64-global-isel/stage1/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.empty
+ touch 
/home/tcwg-buildbot/worker/clang-aarch64-global-isel/stage1/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.empty
RUN: at line 17: echo 'Content of device file 1' > 
/home/tcwg-buildbot/worker/clang-aarch64-global-isel/stage1/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt1
+ echo 'Content of device file 1'
RUN: at line 18: echo 'Content of device file 2' > 
/home/tcwg-buildbot/worker/clang-aarch64-global-isel/stage1/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt2
+ echo 'Content of device file 2'
RUN: at line 23: env OFFLOAD_BUNDLER_COMPRESS=1 OFFLOAD_BUNDLER_VERBOSE=1
clang-offload-bundler -type=bc 
-targets=hip-amdgcn-amd-amdhsa--gfx900,hip-amdgcn-amd-amdhsa--gfx906
-input=/home/tcwg-buildbot/worker/clang-aarch64-global-isel/stage1/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt1
 
-input=/home/tcwg-buildbot/worker/clang-aarch64-global-isel/stage1/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt2
 
-output=/home/tcwg-buildbot/worker/clang-aarch64-global-isel/stage1/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.hip.bundle.bc
 2>&1 |
/home/tcwg-buildbot/worker/clang-aarch64-global-isel/stage1/bin/FileCheck 
-check-prefix=COMPRESS 
/home/tcwg-buildbot/worker/clang-aarch64-global-isel/llvm/clang/test/Driver/clang-offload-bundler-zlib.c
+ env OFFLOAD_BUNDLER_COMPRESS=1 OFFLOAD_BUNDLER_VERBOSE=1 
clang-offload-bundler -type=bc 
-targets=hip-amdgcn-amd-amdhsa--gfx900,hip-amdgcn-amd-amdhsa--gfx906 
-input=/home/tcwg-buildbot/worker/clang-aarch64-global-isel/stage1/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt1
 
-input=/home/tcwg-buildbot/worker/clang-aarch64-global-isel/stage1/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt2
 
-output=/home/tcwg-buildbot/worker/clang-aarch64-global-isel/stage1/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.hip.bundle.bc
+ /home/tcwg-buildbot/worker/clang-aarch64-global-isel/stage1/bin/FileCheck 
-check-prefix=COMPRESS 
/home/tcwg-buildbot/worker/clang-aarch64-global-isel/llvm/clang/test/Driver/clang-offload-bundler-zlib.c
/home/tcwg-buildbot/worker/clang-aarch64-global-isel/llvm/clang/test/Driver/clang-offload-bundler-zlib.c:59:14:
 error: COMPRESS: expected string not found in input
// COMPRESS: Compressed bundle format version: 3
 ^
:4:21: note: scanning from here
Compression level: 6
^
:7:1: note: possible intended match here
Compression rate: 1.86
^

Input file: 
Check file: 
/home/tcwg-buildbot/worker/clang-aarch64-global-isel/llvm/clang/test/Driver/clang-offload-bundler-zlib.c

-dump-input=help explains the following input dump.

Input was:
<<
1: Compressed bundle format version: 2 
2: Total file size (including headers): 125 bytes 
3: Compression method used: zlib 
4: Compression level: 6 
check:59'0 X error: no match found
5: Binary size before compression: 188 bytes 
check:59'0 ~~
6: Binary size after compression: 101 bytes 
check:59'0 ~
7: Compression rate: 1.86 
check:59'0 ~~~
check:59'1 ?   possible intended match
8: Compression ratio: 53.72% 
check:59'0 ~~
9: Compression speed: 0.14 MB/s 
check:59'0 ~~

[clang] [OffloadBundler] Compress bundles over 4GB (PR #122307)

2025-01-20 Thread LLVM Continuous Integration via cfe-commits 

llvm-ci wrote:

LLVM Buildbot has detected a new failure on builder 
`clang-aarch64-sve2-vla-2stage` running on `linaro-g4-01` while building 
`clang` at step 12 "ninja check 2".

Full details are available at: 
https://lab.llvm.org/buildbot/#/builders/199/builds/1090


Here is the relevant piece of the build log for the reference

```
Step 12 (ninja check 2) failure: stage 2 checked (failure)
 TEST 'Clang :: Driver/clang-offload-bundler-zlib.c' FAILED 

Exit Code: 1

Command Output (stderr):
--
RUN: at line 7: 
/home/tcwg-buildbot/worker/clang-aarch64-sve2-vla-2stage/stage2/bin/clang -O0 
-target aarch64-unknown-linux-gnu 
/home/tcwg-buildbot/worker/clang-aarch64-sve2-vla-2stage/llvm/clang/test/Driver/clang-offload-bundler-zlib.c
 -c -emit-llvm -o 
/home/tcwg-buildbot/worker/clang-aarch64-sve2-vla-2stage/stage2/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.bc
+ /home/tcwg-buildbot/worker/clang-aarch64-sve2-vla-2stage/stage2/bin/clang -O0 
-target aarch64-unknown-linux-gnu 
/home/tcwg-buildbot/worker/clang-aarch64-sve2-vla-2stage/llvm/clang/test/Driver/clang-offload-bundler-zlib.c
 -c -emit-llvm -o 
/home/tcwg-buildbot/worker/clang-aarch64-sve2-vla-2stage/stage2/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.bc
RUN: at line 12: touch 
/home/tcwg-buildbot/worker/clang-aarch64-sve2-vla-2stage/stage2/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.empty
+ touch 
/home/tcwg-buildbot/worker/clang-aarch64-sve2-vla-2stage/stage2/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.empty
RUN: at line 17: echo 'Content of device file 1' > 
/home/tcwg-buildbot/worker/clang-aarch64-sve2-vla-2stage/stage2/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt1
+ echo 'Content of device file 1'
RUN: at line 18: echo 'Content of device file 2' > 
/home/tcwg-buildbot/worker/clang-aarch64-sve2-vla-2stage/stage2/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt2
+ echo 'Content of device file 2'
RUN: at line 23: env OFFLOAD_BUNDLER_COMPRESS=1 OFFLOAD_BUNDLER_VERBOSE=1
clang-offload-bundler -type=bc 
-targets=hip-amdgcn-amd-amdhsa--gfx900,hip-amdgcn-amd-amdhsa--gfx906
-input=/home/tcwg-buildbot/worker/clang-aarch64-sve2-vla-2stage/stage2/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt1
 
-input=/home/tcwg-buildbot/worker/clang-aarch64-sve2-vla-2stage/stage2/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt2
 
-output=/home/tcwg-buildbot/worker/clang-aarch64-sve2-vla-2stage/stage2/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.hip.bundle.bc
 2>&1 |
/home/tcwg-buildbot/worker/clang-aarch64-sve2-vla-2stage/stage2/bin/FileCheck 
-check-prefix=COMPRESS 
/home/tcwg-buildbot/worker/clang-aarch64-sve2-vla-2stage/llvm/clang/test/Driver/clang-offload-bundler-zlib.c
+ env OFFLOAD_BUNDLER_COMPRESS=1 OFFLOAD_BUNDLER_VERBOSE=1 
clang-offload-bundler -type=bc 
-targets=hip-amdgcn-amd-amdhsa--gfx900,hip-amdgcn-amd-amdhsa--gfx906 
-input=/home/tcwg-buildbot/worker/clang-aarch64-sve2-vla-2stage/stage2/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt1
 
-input=/home/tcwg-buildbot/worker/clang-aarch64-sve2-vla-2stage/stage2/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt2
 
-output=/home/tcwg-buildbot/worker/clang-aarch64-sve2-vla-2stage/stage2/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.hip.bundle.bc
+ /home/tcwg-buildbot/worker/clang-aarch64-sve2-vla-2stage/stage2/bin/FileCheck 
-check-prefix=COMPRESS 
/home/tcwg-buildbot/worker/clang-aarch64-sve2-vla-2stage/llvm/clang/test/Driver/clang-offload-bundler-zlib.c
/home/tcwg-buildbot/worker/clang-aarch64-sve2-vla-2stage/llvm/clang/test/Driver/clang-offload-bundler-zlib.c:59:14:
 error: COMPRESS: expected string not found in input
// COMPRESS: Compressed bundle format version: 3
 ^
:4:21: note: scanning from here
Compression level: 6
^
:7:1: note: possible intended match here
Compression rate: 1.86
^

Input file: 
Check file: 
/home/tcwg-buildbot/worker/clang-aarch64-sve2-vla-2stage/llvm/clang/test/Driver/clang-offload-bundler-zlib.c

-dump-input=help explains the following input dump.

Input was:
<<
1: Compressed bundle format version: 2 
2: Total file size (including headers): 125 bytes 
3: Compression method used: zlib 
4: Compression level: 6 
check:59'0 X error: no match found
5: Binary size before compression: 188 bytes 
check:59'0 ~~
6: Binary size after compression: 101 bytes 
check:59'0 ~
7: Compression rate: 1.86 
check:59'0 ~~~
check:59'1 ?   possible intended match
8: Compression ratio: 53.72% 
check:59'0 ~~

[clang] [OffloadBundler] Compress bundles over 4GB (PR #122307)

2025-01-20 Thread LLVM Continuous Integration via cfe-commits 

llvm-ci wrote:

LLVM Buildbot has detected a new failure on builder `llvm-clang-win-x-aarch64` 
running on `as-builder-2` while building `clang` at step 10 "test-check-clang".

Full details are available at: 
https://lab.llvm.org/buildbot/#/builders/193/builds/5054


Here is the relevant piece of the build log for the reference

```
Step 10 (test-check-clang) failure: Test just built components: check-clang 
completed (failure)
 TEST 'Clang :: Driver/clang-offload-bundler-zlib.c' FAILED 

Exit Code: 1

Command Output (stdout):
--
# RUN: at line 7
c:\buildbot\as-builder-2\x-aarch64\build\bin\clang.exe -O0 -target 
aarch64-unknown-linux-gnu 
C:\buildbot\as-builder-2\x-aarch64\llvm-project\clang\test\Driver\clang-offload-bundler-zlib.c
 -c -emit-llvm -o 
C:\buildbot\as-builder-2\x-aarch64\build\tools\clang\test\Driver\Output\clang-offload-bundler-zlib.c.tmp.bc
# executed command: 'c:\buildbot\as-builder-2\x-aarch64\build\bin\clang.exe' 
-O0 -target aarch64-unknown-linux-gnu 
'C:\buildbot\as-builder-2\x-aarch64\llvm-project\clang\test\Driver\clang-offload-bundler-zlib.c'
 -c -emit-llvm -o 
'C:\buildbot\as-builder-2\x-aarch64\build\tools\clang\test\Driver\Output\clang-offload-bundler-zlib.c.tmp.bc'
# RUN: at line 12
touch 
C:\buildbot\as-builder-2\x-aarch64\build\tools\clang\test\Driver\Output\clang-offload-bundler-zlib.c.tmp.empty
# executed command: touch 
'C:\buildbot\as-builder-2\x-aarch64\build\tools\clang\test\Driver\Output\clang-offload-bundler-zlib.c.tmp.empty'
# RUN: at line 17
echo 'Content of device file 1' > 
C:\buildbot\as-builder-2\x-aarch64\build\tools\clang\test\Driver\Output\clang-offload-bundler-zlib.c.tmp.tgt1
# executed command: echo 'Content of device file 1'
# RUN: at line 18
echo 'Content of device file 2' > 
C:\buildbot\as-builder-2\x-aarch64\build\tools\clang\test\Driver\Output\clang-offload-bundler-zlib.c.tmp.tgt2
# executed command: echo 'Content of device file 2'
# RUN: at line 23
env OFFLOAD_BUNDLER_COMPRESS=1 OFFLOAD_BUNDLER_VERBOSE=1
clang-offload-bundler -type=bc 
-targets=hip-amdgcn-amd-amdhsa--gfx900,hip-amdgcn-amd-amdhsa--gfx906
-input=C:\buildbot\as-builder-2\x-aarch64\build\tools\clang\test\Driver\Output\clang-offload-bundler-zlib.c.tmp.tgt1
 
-input=C:\buildbot\as-builder-2\x-aarch64\build\tools\clang\test\Driver\Output\clang-offload-bundler-zlib.c.tmp.tgt2
 
-output=C:\buildbot\as-builder-2\x-aarch64\build\tools\clang\test\Driver\Output\clang-offload-bundler-zlib.c.tmp.hip.bundle.bc
 2>&1 |c:\buildbot\as-builder-2\x-aarch64\build\bin\filecheck.exe 
-check-prefix=COMPRESS 
C:\buildbot\as-builder-2\x-aarch64\llvm-project\clang\test\Driver\clang-offload-bundler-zlib.c
# executed command: env OFFLOAD_BUNDLER_COMPRESS=1 OFFLOAD_BUNDLER_VERBOSE=1 
clang-offload-bundler -type=bc 
-targets=hip-amdgcn-amd-amdhsa--gfx900,hip-amdgcn-amd-amdhsa--gfx906 
'-input=C:\buildbot\as-builder-2\x-aarch64\build\tools\clang\test\Driver\Output\clang-offload-bundler-zlib.c.tmp.tgt1'
 
'-input=C:\buildbot\as-builder-2\x-aarch64\build\tools\clang\test\Driver\Output\clang-offload-bundler-zlib.c.tmp.tgt2'
 
'-output=C:\buildbot\as-builder-2\x-aarch64\build\tools\clang\test\Driver\Output\clang-offload-bundler-zlib.c.tmp.hip.bundle.bc'
# executed command: 
'c:\buildbot\as-builder-2\x-aarch64\build\bin\filecheck.exe' 
-check-prefix=COMPRESS 
'C:\buildbot\as-builder-2\x-aarch64\llvm-project\clang\test\Driver\clang-offload-bundler-zlib.c'
# .---command stderr
# | 
C:\buildbot\as-builder-2\x-aarch64\llvm-project\clang\test\Driver\clang-offload-bundler-zlib.c:59:14:
 error: COMPRESS: expected string not found in input
# | // COMPRESS: Compressed bundle format version: 3
# |  ^
# | :4:21: note: scanning from here
# | Compression level: 6
# | ^
# | :7:1: note: possible intended match here
# | Compression rate: 1.86
# | ^
# | 
# | Input file: 
# | Check file: 
C:\buildbot\as-builder-2\x-aarch64\llvm-project\clang\test\Driver\clang-offload-bundler-zlib.c
# | 
# | -dump-input=help explains the following input dump.
# | 
# | Input was:
# | <<
# | 1: Compressed bundle format version: 2 
# | 2: Total file size (including headers): 125 bytes 
# | 3: Compression method used: zlib 
# | 4: Compression level: 6 
# | check:59'0 X error: no match found
# | 5: Binary size before compression: 188 bytes 
# | check:59'0 ~~
# | 6: Binary size after compression: 101 bytes 
# | check:59'0 ~
# | 7: Compression rate: 1.86 
# | check:59'0 ~~~
...

```



https://github.com/llvm/llvm-project/pull/122307
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] [OffloadBundler] Compress bundles over 4GB (PR #122307)

2025-01-20 Thread LLVM Continuous Integration via cfe-commits 

llvm-ci wrote:

LLVM Buildbot has detected a new failure on builder 
`llvm-clang-x86_64-gcc-ubuntu` running on `sie-linux-worker3` while building 
`clang` at step 6 "test-build-unified-tree-check-all".

Full details are available at: 
https://lab.llvm.org/buildbot/#/builders/174/builds/11719


Here is the relevant piece of the build log for the reference

```
Step 6 (test-build-unified-tree-check-all) failure: test (failure)
 TEST 'Clang :: Driver/clang-offload-bundler-zlib.c' FAILED 

Exit Code: 1

Command Output (stderr):
--
RUN: at line 7: 
/home/buildbot/buildbot-root/llvm-clang-x86_64-gcc-ubuntu/build/bin/clang -O0 
-target x86_64-unknown-linux-gnu 
/home/buildbot/buildbot-root/llvm-clang-x86_64-gcc-ubuntu/llvm-project/clang/test/Driver/clang-offload-bundler-zlib.c
 -c -emit-llvm -o 
/home/buildbot/buildbot-root/llvm-clang-x86_64-gcc-ubuntu/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.bc
+ /home/buildbot/buildbot-root/llvm-clang-x86_64-gcc-ubuntu/build/bin/clang -O0 
-target x86_64-unknown-linux-gnu 
/home/buildbot/buildbot-root/llvm-clang-x86_64-gcc-ubuntu/llvm-project/clang/test/Driver/clang-offload-bundler-zlib.c
 -c -emit-llvm -o 
/home/buildbot/buildbot-root/llvm-clang-x86_64-gcc-ubuntu/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.bc
RUN: at line 12: touch 
/home/buildbot/buildbot-root/llvm-clang-x86_64-gcc-ubuntu/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.empty
+ touch 
/home/buildbot/buildbot-root/llvm-clang-x86_64-gcc-ubuntu/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.empty
RUN: at line 17: echo 'Content of device file 1' > 
/home/buildbot/buildbot-root/llvm-clang-x86_64-gcc-ubuntu/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt1
+ echo 'Content of device file 1'
RUN: at line 18: echo 'Content of device file 2' > 
/home/buildbot/buildbot-root/llvm-clang-x86_64-gcc-ubuntu/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt2
+ echo 'Content of device file 2'
RUN: at line 23: env OFFLOAD_BUNDLER_COMPRESS=1 OFFLOAD_BUNDLER_VERBOSE=1
clang-offload-bundler -type=bc 
-targets=hip-amdgcn-amd-amdhsa--gfx900,hip-amdgcn-amd-amdhsa--gfx906
-input=/home/buildbot/buildbot-root/llvm-clang-x86_64-gcc-ubuntu/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt1
 
-input=/home/buildbot/buildbot-root/llvm-clang-x86_64-gcc-ubuntu/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt2
 
-output=/home/buildbot/buildbot-root/llvm-clang-x86_64-gcc-ubuntu/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.hip.bundle.bc
 2>&1 |
/home/buildbot/buildbot-root/llvm-clang-x86_64-gcc-ubuntu/build/bin/FileCheck 
-check-prefix=COMPRESS 
/home/buildbot/buildbot-root/llvm-clang-x86_64-gcc-ubuntu/llvm-project/clang/test/Driver/clang-offload-bundler-zlib.c
+ /home/buildbot/buildbot-root/llvm-clang-x86_64-gcc-ubuntu/build/bin/FileCheck 
-check-prefix=COMPRESS 
/home/buildbot/buildbot-root/llvm-clang-x86_64-gcc-ubuntu/llvm-project/clang/test/Driver/clang-offload-bundler-zlib.c
+ env OFFLOAD_BUNDLER_COMPRESS=1 OFFLOAD_BUNDLER_VERBOSE=1 
clang-offload-bundler -type=bc 
-targets=hip-amdgcn-amd-amdhsa--gfx900,hip-amdgcn-amd-amdhsa--gfx906 
-input=/home/buildbot/buildbot-root/llvm-clang-x86_64-gcc-ubuntu/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt1
 
-input=/home/buildbot/buildbot-root/llvm-clang-x86_64-gcc-ubuntu/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt2
 
-output=/home/buildbot/buildbot-root/llvm-clang-x86_64-gcc-ubuntu/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.hip.bundle.bc
/home/buildbot/buildbot-root/llvm-clang-x86_64-gcc-ubuntu/llvm-project/clang/test/Driver/clang-offload-bundler-zlib.c:59:14:
 error: COMPRESS: expected string not found in input
// COMPRESS: Compressed bundle format version: 3
 ^
:4:21: note: scanning from here
Compression level: 6
^
:7:1: note: possible intended match here
Compression rate: 1.86
^

Input file: 
Check file: 
/home/buildbot/buildbot-root/llvm-clang-x86_64-gcc-ubuntu/llvm-project/clang/test/Driver/clang-offload-bundler-zlib.c

-dump-input=help explains the following input dump.

Input was:
<<
1: Compressed bundle format 
version: 2 
2: Total file size (including headers): 
125 bytes 
3: Compression method used: 
zlib 
check:36   ^
4: Compression level: 6 

check:37   ^~~~
check:59'0 X e

[clang] [OffloadBundler] Compress bundles over 4GB (PR #122307)

2025-01-20 Thread LLVM Continuous Integration via cfe-commits 

llvm-ci wrote:

LLVM Buildbot has detected a new failure on builder `premerge-monolithic-linux` 
running on `premerge-linux-1` while building `clang` at step 7 
"test-build-unified-tree-check-all".

Full details are available at: 
https://lab.llvm.org/buildbot/#/builders/153/builds/20312


Here is the relevant piece of the build log for the reference

```
Step 7 (test-build-unified-tree-check-all) failure: test (failure)
 TEST 'Clang :: Driver/clang-offload-bundler-zlib.c' FAILED 

Exit Code: 1

Command Output (stderr):
--
RUN: at line 7: /build/buildbot/premerge-monolithic-linux/build/bin/clang -O0 
-target x86_64-unknown-linux-gnu 
/build/buildbot/premerge-monolithic-linux/llvm-project/clang/test/Driver/clang-offload-bundler-zlib.c
 -c -emit-llvm -o 
/build/buildbot/premerge-monolithic-linux/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.bc
+ /build/buildbot/premerge-monolithic-linux/build/bin/clang -O0 -target 
x86_64-unknown-linux-gnu 
/build/buildbot/premerge-monolithic-linux/llvm-project/clang/test/Driver/clang-offload-bundler-zlib.c
 -c -emit-llvm -o 
/build/buildbot/premerge-monolithic-linux/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.bc
RUN: at line 12: touch 
/build/buildbot/premerge-monolithic-linux/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.empty
+ touch 
/build/buildbot/premerge-monolithic-linux/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.empty
RUN: at line 17: echo 'Content of device file 1' > 
/build/buildbot/premerge-monolithic-linux/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt1
+ echo 'Content of device file 1'
RUN: at line 18: echo 'Content of device file 2' > 
/build/buildbot/premerge-monolithic-linux/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt2
+ echo 'Content of device file 2'
RUN: at line 23: env OFFLOAD_BUNDLER_COMPRESS=1 OFFLOAD_BUNDLER_VERBOSE=1
clang-offload-bundler -type=bc 
-targets=hip-amdgcn-amd-amdhsa--gfx900,hip-amdgcn-amd-amdhsa--gfx906
-input=/build/buildbot/premerge-monolithic-linux/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt1
 
-input=/build/buildbot/premerge-monolithic-linux/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt2
 
-output=/build/buildbot/premerge-monolithic-linux/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.hip.bundle.bc
 2>&1 |/build/buildbot/premerge-monolithic-linux/build/bin/FileCheck 
-check-prefix=COMPRESS 
/build/buildbot/premerge-monolithic-linux/llvm-project/clang/test/Driver/clang-offload-bundler-zlib.c
+ env OFFLOAD_BUNDLER_COMPRESS=1 OFFLOAD_BUNDLER_VERBOSE=1 
clang-offload-bundler -type=bc 
-targets=hip-amdgcn-amd-amdhsa--gfx900,hip-amdgcn-amd-amdhsa--gfx906 
-input=/build/buildbot/premerge-monolithic-linux/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt1
 
-input=/build/buildbot/premerge-monolithic-linux/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt2
 
-output=/build/buildbot/premerge-monolithic-linux/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.hip.bundle.bc
+ /build/buildbot/premerge-monolithic-linux/build/bin/FileCheck 
-check-prefix=COMPRESS 
/build/buildbot/premerge-monolithic-linux/llvm-project/clang/test/Driver/clang-offload-bundler-zlib.c
/build/buildbot/premerge-monolithic-linux/llvm-project/clang/test/Driver/clang-offload-bundler-zlib.c:59:14:
 error: COMPRESS: expected string not found in input
// COMPRESS: Compressed bundle format version: 3
 ^
:4:21: note: scanning from here
Compression level: 6
^
:7:1: note: possible intended match here
Compression rate: 1.86
^

Input file: 
Check file: 
/build/buildbot/premerge-monolithic-linux/llvm-project/clang/test/Driver/clang-offload-bundler-zlib.c

-dump-input=help explains the following input dump.

Input was:
<<
1: Compressed bundle format version: 2 
2: Total file size (including headers): 125 bytes 
3: Compression method used: zlib 
4: Compression level: 6 
check:59'0 X error: no match found
5: Binary size before compression: 188 bytes 
check:59'0 ~~
6: Binary size after compression: 101 bytes 
check:59'0 ~
7: Compression rate: 1.86 
check:59'0 ~~~
check:59'1 ?   possible intended match
8: Compression ratio: 53.72% 
check:59'0 ~~
9: Compression speed: 1.02 MB/s 
check:59'0 ~
   10: Truncated MD5 hash: 0x2bcafb899a29dc94 
...

```



https://github.com/llvm/llvm-project/pull/122307
___
cfe-commits mailing list
cfe

[clang] [OffloadBundler] Compress bundles over 4GB (PR #122307)

2025-01-20 Thread LLVM Continuous Integration via cfe-commits 

llvm-ci wrote:

LLVM Buildbot has detected a new failure on builder `clang-debian-cpp20` 
running on `clang-debian-cpp20` while building `clang` at step 6 
"test-build-unified-tree-check-all".

Full details are available at: 
https://lab.llvm.org/buildbot/#/builders/108/builds/8358


Here is the relevant piece of the build log for the reference

```
Step 6 (test-build-unified-tree-check-all) failure: test (failure)
 TEST 'Clang :: Driver/clang-offload-bundler-zlib.c' FAILED 

Exit Code: 1

Command Output (stderr):
--
RUN: at line 7: 
/vol/worker/clang-debian-cpp20/clang-debian-cpp20/build/bin/clang -O0 -target 
x86_64-unknown-linux-gnu 
/vol/worker/clang-debian-cpp20/clang-debian-cpp20/llvm-project/clang/test/Driver/clang-offload-bundler-zlib.c
 -c -emit-llvm -o 
/vol/worker/clang-debian-cpp20/clang-debian-cpp20/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.bc
+ /vol/worker/clang-debian-cpp20/clang-debian-cpp20/build/bin/clang -O0 -target 
x86_64-unknown-linux-gnu 
/vol/worker/clang-debian-cpp20/clang-debian-cpp20/llvm-project/clang/test/Driver/clang-offload-bundler-zlib.c
 -c -emit-llvm -o 
/vol/worker/clang-debian-cpp20/clang-debian-cpp20/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.bc
RUN: at line 12: touch 
/vol/worker/clang-debian-cpp20/clang-debian-cpp20/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.empty
+ touch 
/vol/worker/clang-debian-cpp20/clang-debian-cpp20/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.empty
RUN: at line 17: echo 'Content of device file 1' > 
/vol/worker/clang-debian-cpp20/clang-debian-cpp20/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt1
+ echo 'Content of device file 1'
RUN: at line 18: echo 'Content of device file 2' > 
/vol/worker/clang-debian-cpp20/clang-debian-cpp20/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt2
+ echo 'Content of device file 2'
RUN: at line 23: env OFFLOAD_BUNDLER_COMPRESS=1 OFFLOAD_BUNDLER_VERBOSE=1
clang-offload-bundler -type=bc 
-targets=hip-amdgcn-amd-amdhsa--gfx900,hip-amdgcn-amd-amdhsa--gfx906
-input=/vol/worker/clang-debian-cpp20/clang-debian-cpp20/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt1
 
-input=/vol/worker/clang-debian-cpp20/clang-debian-cpp20/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt2
 
-output=/vol/worker/clang-debian-cpp20/clang-debian-cpp20/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.hip.bundle.bc
 2>&1 |
/vol/worker/clang-debian-cpp20/clang-debian-cpp20/build/bin/FileCheck 
-check-prefix=COMPRESS 
/vol/worker/clang-debian-cpp20/clang-debian-cpp20/llvm-project/clang/test/Driver/clang-offload-bundler-zlib.c
+ /vol/worker/clang-debian-cpp20/clang-debian-cpp20/build/bin/FileCheck 
-check-prefix=COMPRESS 
/vol/worker/clang-debian-cpp20/clang-debian-cpp20/llvm-project/clang/test/Driver/clang-offload-bundler-zlib.c
+ env OFFLOAD_BUNDLER_COMPRESS=1 OFFLOAD_BUNDLER_VERBOSE=1 
clang-offload-bundler -type=bc 
-targets=hip-amdgcn-amd-amdhsa--gfx900,hip-amdgcn-amd-amdhsa--gfx906 
-input=/vol/worker/clang-debian-cpp20/clang-debian-cpp20/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt1
 
-input=/vol/worker/clang-debian-cpp20/clang-debian-cpp20/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt2
 
-output=/vol/worker/clang-debian-cpp20/clang-debian-cpp20/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.hip.bundle.bc
/vol/worker/clang-debian-cpp20/clang-debian-cpp20/llvm-project/clang/test/Driver/clang-offload-bundler-zlib.c:59:14:
 error: COMPRESS: expected string not found in input
// COMPRESS: Compressed bundle format version: 3
 ^
:4:21: note: scanning from here
Compression level: 6
^
:7:1: note: possible intended match here
Compression rate: 1.86
^

Input file: 
Check file: 
/vol/worker/clang-debian-cpp20/clang-debian-cpp20/llvm-project/clang/test/Driver/clang-offload-bundler-zlib.c

-dump-input=help explains the following input dump.

Input was:
<<
1: Compressed bundle format version: 2 
2: Total file size (including headers): 125 bytes 
3: Compression method used: zlib 
4: Compression level: 6 
check:59'0 X error: no match found
5: Binary size before compression: 188 bytes 
check:59'0 ~~
6: Binary size after compression: 101 bytes 
check:59'0 ~
7: Compression rate: 1.86 
check:59'0 ~~~
check:59'1 ?   possible intended match
8: Compression ratio: 53.72% 
check:59'0 ~~
9: Compression speed: 1.14 MB/s 
check:59'0 ~
   10: Trunc

[clang] [OffloadBundler] Compress bundles over 4GB (PR #122307)

2025-01-20 Thread LLVM Continuous Integration via cfe-commits 

llvm-ci wrote:

LLVM Buildbot has detected a new failure on builder `clang-armv8-quick` running 
on `linaro-clang-armv8-quick` while building `clang` at step 5 "ninja check 1".

Full details are available at: 
https://lab.llvm.org/buildbot/#/builders/154/builds/10570


Here is the relevant piece of the build log for the reference

```
Step 5 (ninja check 1) failure: stage 1 checked (failure)
 TEST 'Clang :: Driver/clang-offload-bundler-zlib.c' FAILED 

Exit Code: 1

Command Output (stderr):
--
RUN: at line 7: /home/tcwg-buildbot/worker/clang-armv8-quick/stage1/bin/clang 
-O0 -target armv8l-unknown-linux-gnueabihf 
/home/tcwg-buildbot/worker/clang-armv8-quick/llvm/clang/test/Driver/clang-offload-bundler-zlib.c
 -c -emit-llvm -o 
/home/tcwg-buildbot/worker/clang-armv8-quick/stage1/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.bc
+ /home/tcwg-buildbot/worker/clang-armv8-quick/stage1/bin/clang -O0 -target 
armv8l-unknown-linux-gnueabihf 
/home/tcwg-buildbot/worker/clang-armv8-quick/llvm/clang/test/Driver/clang-offload-bundler-zlib.c
 -c -emit-llvm -o 
/home/tcwg-buildbot/worker/clang-armv8-quick/stage1/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.bc
RUN: at line 12: touch 
/home/tcwg-buildbot/worker/clang-armv8-quick/stage1/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.empty
+ touch 
/home/tcwg-buildbot/worker/clang-armv8-quick/stage1/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.empty
RUN: at line 17: echo 'Content of device file 1' > 
/home/tcwg-buildbot/worker/clang-armv8-quick/stage1/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt1
+ echo 'Content of device file 1'
RUN: at line 18: echo 'Content of device file 2' > 
/home/tcwg-buildbot/worker/clang-armv8-quick/stage1/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt2
+ echo 'Content of device file 2'
RUN: at line 23: env OFFLOAD_BUNDLER_COMPRESS=1 OFFLOAD_BUNDLER_VERBOSE=1
clang-offload-bundler -type=bc 
-targets=hip-amdgcn-amd-amdhsa--gfx900,hip-amdgcn-amd-amdhsa--gfx906
-input=/home/tcwg-buildbot/worker/clang-armv8-quick/stage1/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt1
 
-input=/home/tcwg-buildbot/worker/clang-armv8-quick/stage1/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt2
 
-output=/home/tcwg-buildbot/worker/clang-armv8-quick/stage1/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.hip.bundle.bc
 2>&1 |/home/tcwg-buildbot/worker/clang-armv8-quick/stage1/bin/FileCheck 
-check-prefix=COMPRESS 
/home/tcwg-buildbot/worker/clang-armv8-quick/llvm/clang/test/Driver/clang-offload-bundler-zlib.c
+ env OFFLOAD_BUNDLER_COMPRESS=1 OFFLOAD_BUNDLER_VERBOSE=1 
clang-offload-bundler -type=bc 
-targets=hip-amdgcn-amd-amdhsa--gfx900,hip-amdgcn-amd-amdhsa--gfx906 
-input=/home/tcwg-buildbot/worker/clang-armv8-quick/stage1/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt1
 
-input=/home/tcwg-buildbot/worker/clang-armv8-quick/stage1/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt2
 
-output=/home/tcwg-buildbot/worker/clang-armv8-quick/stage1/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.hip.bundle.bc
+ /home/tcwg-buildbot/worker/clang-armv8-quick/stage1/bin/FileCheck 
-check-prefix=COMPRESS 
/home/tcwg-buildbot/worker/clang-armv8-quick/llvm/clang/test/Driver/clang-offload-bundler-zlib.c
/home/tcwg-buildbot/worker/clang-armv8-quick/llvm/clang/test/Driver/clang-offload-bundler-zlib.c:59:14:
 error: COMPRESS: expected string not found in input
// COMPRESS: Compressed bundle format version: 3
 ^
:4:21: note: scanning from here
Compression level: 6
^
:7:1: note: possible intended match here
Compression rate: 1.86
^

Input file: 
Check file: 
/home/tcwg-buildbot/worker/clang-armv8-quick/llvm/clang/test/Driver/clang-offload-bundler-zlib.c

-dump-input=help explains the following input dump.

Input was:
<<
1: Compressed bundle format version: 2 
2: Total file size (including headers): 125 bytes 
3: Compression method used: zlib 
4: Compression level: 6 
check:59'0 X error: no match found
5: Binary size before compression: 188 bytes 
check:59'0 ~~
6: Binary size after compression: 101 bytes 
check:59'0 ~
7: Compression rate: 1.86 
check:59'0 ~~~
check:59'1 ?   possible intended match
8: Compression ratio: 53.72% 
check:59'0 ~~
9: Compression speed: 0.02 MB/s 
check:59'0 ~
   10: Truncated MD5 hash: 0x2bcafb899a29dc94 
...

```



https://github.com/llvm/llvm-project/pull/122307
___
cfe-commits

[clang] [OffloadBundler] Compress bundles over 4GB (PR #122307)

2025-01-20 Thread LLVM Continuous Integration via cfe-commits 

llvm-ci wrote:

LLVM Buildbot has detected a new failure on builder `clang-aarch64-quick` 
running on `linaro-clang-aarch64-quick` while building `clang` at step 5 "ninja 
check 1".

Full details are available at: 
https://lab.llvm.org/buildbot/#/builders/65/builds/10933


Here is the relevant piece of the build log for the reference

```
Step 5 (ninja check 1) failure: stage 1 checked (failure)
 TEST 'Clang :: Driver/clang-offload-bundler-zlib.c' FAILED 

Exit Code: 1

Command Output (stderr):
--
RUN: at line 7: /home/tcwg-buildbot/worker/clang-aarch64-quick/stage1/bin/clang 
-O0 -target aarch64-unknown-linux-gnu 
/home/tcwg-buildbot/worker/clang-aarch64-quick/llvm/clang/test/Driver/clang-offload-bundler-zlib.c
 -c -emit-llvm -o 
/home/tcwg-buildbot/worker/clang-aarch64-quick/stage1/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.bc
+ /home/tcwg-buildbot/worker/clang-aarch64-quick/stage1/bin/clang -O0 -target 
aarch64-unknown-linux-gnu 
/home/tcwg-buildbot/worker/clang-aarch64-quick/llvm/clang/test/Driver/clang-offload-bundler-zlib.c
 -c -emit-llvm -o 
/home/tcwg-buildbot/worker/clang-aarch64-quick/stage1/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.bc
RUN: at line 12: touch 
/home/tcwg-buildbot/worker/clang-aarch64-quick/stage1/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.empty
+ touch 
/home/tcwg-buildbot/worker/clang-aarch64-quick/stage1/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.empty
RUN: at line 17: echo 'Content of device file 1' > 
/home/tcwg-buildbot/worker/clang-aarch64-quick/stage1/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt1
+ echo 'Content of device file 1'
RUN: at line 18: echo 'Content of device file 2' > 
/home/tcwg-buildbot/worker/clang-aarch64-quick/stage1/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt2
+ echo 'Content of device file 2'
RUN: at line 23: env OFFLOAD_BUNDLER_COMPRESS=1 OFFLOAD_BUNDLER_VERBOSE=1
clang-offload-bundler -type=bc 
-targets=hip-amdgcn-amd-amdhsa--gfx900,hip-amdgcn-amd-amdhsa--gfx906
-input=/home/tcwg-buildbot/worker/clang-aarch64-quick/stage1/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt1
 
-input=/home/tcwg-buildbot/worker/clang-aarch64-quick/stage1/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt2
 
-output=/home/tcwg-buildbot/worker/clang-aarch64-quick/stage1/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.hip.bundle.bc
 2>&1 |/home/tcwg-buildbot/worker/clang-aarch64-quick/stage1/bin/FileCheck 
-check-prefix=COMPRESS 
/home/tcwg-buildbot/worker/clang-aarch64-quick/llvm/clang/test/Driver/clang-offload-bundler-zlib.c
+ /home/tcwg-buildbot/worker/clang-aarch64-quick/stage1/bin/FileCheck 
-check-prefix=COMPRESS 
/home/tcwg-buildbot/worker/clang-aarch64-quick/llvm/clang/test/Driver/clang-offload-bundler-zlib.c
+ env OFFLOAD_BUNDLER_COMPRESS=1 OFFLOAD_BUNDLER_VERBOSE=1 
clang-offload-bundler -type=bc 
-targets=hip-amdgcn-amd-amdhsa--gfx900,hip-amdgcn-amd-amdhsa--gfx906 
-input=/home/tcwg-buildbot/worker/clang-aarch64-quick/stage1/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt1
 
-input=/home/tcwg-buildbot/worker/clang-aarch64-quick/stage1/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt2
 
-output=/home/tcwg-buildbot/worker/clang-aarch64-quick/stage1/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.hip.bundle.bc
/home/tcwg-buildbot/worker/clang-aarch64-quick/llvm/clang/test/Driver/clang-offload-bundler-zlib.c:59:14:
 error: COMPRESS: expected string not found in input
// COMPRESS: Compressed bundle format version: 3
 ^
:4:21: note: scanning from here
Compression level: 6
^
:7:1: note: possible intended match here
Compression rate: 1.86
^

Input file: 
Check file: 
/home/tcwg-buildbot/worker/clang-aarch64-quick/llvm/clang/test/Driver/clang-offload-bundler-zlib.c

-dump-input=help explains the following input dump.

Input was:
<<
1: Compressed bundle format version: 2 
2: Total file size (including headers): 125 bytes 
3: Compression method used: zlib 
4: Compression level: 6 
check:59'0 X error: no match found
5: Binary size before compression: 188 bytes 
check:59'0 ~~
6: Binary size after compression: 101 bytes 
check:59'0 ~
7: Compression rate: 1.86 
check:59'0 ~~~
check:59'1 ?   possible intended match
8: Compression ratio: 53.72% 
check:59'0 ~~
9: Compression speed: 0.54 MB/s 
check:59'0 ~
   10: Truncated MD5 hash: 0x2bcafb899a29dc94 
...

```



https://github.com/llvm/llvm-project/pull/122307
__

[clang] [OffloadBundler] Compress bundles over 4GB (PR #122307)

2025-01-20 Thread LLVM Continuous Integration via cfe-commits 

llvm-ci wrote:

LLVM Buildbot has detected a new failure on builder `arc-builder` running on 
`arc-worker` while building `clang` at step 6 
"test-build-unified-tree-check-all".

Full details are available at: 
https://lab.llvm.org/buildbot/#/builders/3/builds/10570


Here is the relevant piece of the build log for the reference

```
Step 6 (test-build-unified-tree-check-all) failure: test (failure)
 TEST 'Clang :: Driver/clang-offload-bundler-zlib.c' FAILED 

Exit Code: 1

Command Output (stderr):
--
RUN: at line 7: /buildbot/worker/arc-folder/build/bin/clang -O0 -target 
x86_64-unknown-linux-gnu 
/buildbot/worker/arc-folder/llvm-project/clang/test/Driver/clang-offload-bundler-zlib.c
 -c -emit-llvm -o 
/buildbot/worker/arc-folder/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.bc
+ /buildbot/worker/arc-folder/build/bin/clang -O0 -target 
x86_64-unknown-linux-gnu 
/buildbot/worker/arc-folder/llvm-project/clang/test/Driver/clang-offload-bundler-zlib.c
 -c -emit-llvm -o 
/buildbot/worker/arc-folder/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.bc
RUN: at line 12: touch 
/buildbot/worker/arc-folder/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.empty
+ touch 
/buildbot/worker/arc-folder/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.empty
RUN: at line 17: echo 'Content of device file 1' > 
/buildbot/worker/arc-folder/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt1
+ echo 'Content of device file 1'
RUN: at line 18: echo 'Content of device file 2' > 
/buildbot/worker/arc-folder/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt2
+ echo 'Content of device file 2'
RUN: at line 23: env OFFLOAD_BUNDLER_COMPRESS=1 OFFLOAD_BUNDLER_VERBOSE=1
clang-offload-bundler -type=bc 
-targets=hip-amdgcn-amd-amdhsa--gfx900,hip-amdgcn-amd-amdhsa--gfx906
-input=/buildbot/worker/arc-folder/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt1
 
-input=/buildbot/worker/arc-folder/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt2
 
-output=/buildbot/worker/arc-folder/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.hip.bundle.bc
 2>&1 |/buildbot/worker/arc-folder/build/bin/FileCheck 
-check-prefix=COMPRESS 
/buildbot/worker/arc-folder/llvm-project/clang/test/Driver/clang-offload-bundler-zlib.c
+ env OFFLOAD_BUNDLER_COMPRESS=1 OFFLOAD_BUNDLER_VERBOSE=1 
clang-offload-bundler -type=bc 
-targets=hip-amdgcn-amd-amdhsa--gfx900,hip-amdgcn-amd-amdhsa--gfx906 
-input=/buildbot/worker/arc-folder/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt1
 
-input=/buildbot/worker/arc-folder/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt2
 
-output=/buildbot/worker/arc-folder/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.hip.bundle.bc
+ /buildbot/worker/arc-folder/build/bin/FileCheck -check-prefix=COMPRESS 
/buildbot/worker/arc-folder/llvm-project/clang/test/Driver/clang-offload-bundler-zlib.c
/buildbot/worker/arc-folder/llvm-project/clang/test/Driver/clang-offload-bundler-zlib.c:59:14:
 error: COMPRESS: expected string not found in input
// COMPRESS: Compressed bundle format version: 3
 ^
:4:21: note: scanning from here
Compression level: 6
^
:7:1: note: possible intended match here
Compression rate: 1.86
^

Input file: 
Check file: 
/buildbot/worker/arc-folder/llvm-project/clang/test/Driver/clang-offload-bundler-zlib.c

-dump-input=help explains the following input dump.

Input was:
<<
1: Compressed bundle format version: 2 
2: Total file size (including headers): 125 bytes 
3: Compression method used: zlib 
4: Compression level: 6 
check:59'0 X error: no match found
5: Binary size before compression: 188 bytes 
check:59'0 ~~
6: Binary size after compression: 101 bytes 
check:59'0 ~
7: Compression rate: 1.86 
check:59'0 ~~~
check:59'1 ?   possible intended match
8: Compression ratio: 53.72% 
check:59'0 ~~
9: Compression speed: 1.46 MB/s 
check:59'0 ~
   10: Truncated MD5 hash: 0x2bcafb899a29dc94 
...

```



https://github.com/llvm/llvm-project/pull/122307
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] [OffloadBundler] Compress bundles over 4GB (PR #122307)

2025-01-20 Thread LLVM Continuous Integration via cfe-commits 

llvm-ci wrote:

LLVM Buildbot has detected a new failure on builder 
`llvm-clang-x86_64-sie-ubuntu-fast` running on `sie-linux-worker` while 
building `clang` at step 6 "test-build-unified-tree-check-all".

Full details are available at: 
https://lab.llvm.org/buildbot/#/builders/144/builds/16114


Here is the relevant piece of the build log for the reference

```
Step 6 (test-build-unified-tree-check-all) failure: test (failure)
 TEST 'Clang :: Driver/clang-offload-bundler-zlib.c' FAILED 

Exit Code: 1

Command Output (stderr):
--
RUN: at line 7: 
/home/buildbot/buildbot-root/llvm-clang-x86_64-sie-ubuntu-fast/build/bin/clang 
-O0 -target x86_64-scei-ps4 
/home/buildbot/buildbot-root/llvm-clang-x86_64-sie-ubuntu-fast/llvm-project/clang/test/Driver/clang-offload-bundler-zlib.c
 -c -emit-llvm -o 
/home/buildbot/buildbot-root/llvm-clang-x86_64-sie-ubuntu-fast/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.bc
+ 
/home/buildbot/buildbot-root/llvm-clang-x86_64-sie-ubuntu-fast/build/bin/clang 
-O0 -target x86_64-scei-ps4 
/home/buildbot/buildbot-root/llvm-clang-x86_64-sie-ubuntu-fast/llvm-project/clang/test/Driver/clang-offload-bundler-zlib.c
 -c -emit-llvm -o 
/home/buildbot/buildbot-root/llvm-clang-x86_64-sie-ubuntu-fast/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.bc
RUN: at line 12: touch 
/home/buildbot/buildbot-root/llvm-clang-x86_64-sie-ubuntu-fast/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.empty
+ touch 
/home/buildbot/buildbot-root/llvm-clang-x86_64-sie-ubuntu-fast/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.empty
RUN: at line 17: echo 'Content of device file 1' > 
/home/buildbot/buildbot-root/llvm-clang-x86_64-sie-ubuntu-fast/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt1
+ echo 'Content of device file 1'
RUN: at line 18: echo 'Content of device file 2' > 
/home/buildbot/buildbot-root/llvm-clang-x86_64-sie-ubuntu-fast/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt2
+ echo 'Content of device file 2'
RUN: at line 23: env OFFLOAD_BUNDLER_COMPRESS=1 OFFLOAD_BUNDLER_VERBOSE=1
clang-offload-bundler -type=bc 
-targets=hip-amdgcn-amd-amdhsa--gfx900,hip-amdgcn-amd-amdhsa--gfx906
-input=/home/buildbot/buildbot-root/llvm-clang-x86_64-sie-ubuntu-fast/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt1
 
-input=/home/buildbot/buildbot-root/llvm-clang-x86_64-sie-ubuntu-fast/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt2
 
-output=/home/buildbot/buildbot-root/llvm-clang-x86_64-sie-ubuntu-fast/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.hip.bundle.bc
 2>&1 |
/home/buildbot/buildbot-root/llvm-clang-x86_64-sie-ubuntu-fast/build/bin/FileCheck
 -check-prefix=COMPRESS 
/home/buildbot/buildbot-root/llvm-clang-x86_64-sie-ubuntu-fast/llvm-project/clang/test/Driver/clang-offload-bundler-zlib.c
+ env OFFLOAD_BUNDLER_COMPRESS=1 OFFLOAD_BUNDLER_VERBOSE=1 
clang-offload-bundler -type=bc 
-targets=hip-amdgcn-amd-amdhsa--gfx900,hip-amdgcn-amd-amdhsa--gfx906 
-input=/home/buildbot/buildbot-root/llvm-clang-x86_64-sie-ubuntu-fast/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt1
 
-input=/home/buildbot/buildbot-root/llvm-clang-x86_64-sie-ubuntu-fast/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.tgt2
 
-output=/home/buildbot/buildbot-root/llvm-clang-x86_64-sie-ubuntu-fast/build/tools/clang/test/Driver/Output/clang-offload-bundler-zlib.c.tmp.hip.bundle.bc
+ 
/home/buildbot/buildbot-root/llvm-clang-x86_64-sie-ubuntu-fast/build/bin/FileCheck
 -check-prefix=COMPRESS 
/home/buildbot/buildbot-root/llvm-clang-x86_64-sie-ubuntu-fast/llvm-project/clang/test/Driver/clang-offload-bundler-zlib.c
/home/buildbot/buildbot-root/llvm-clang-x86_64-sie-ubuntu-fast/llvm-project/clang/test/Driver/clang-offload-bundler-zlib.c:59:14:
 error: COMPRESS: expected string not found in input
// COMPRESS: Compressed bundle format version: 3
 ^
:4:21: note: scanning from here
Compression level: 6
^
:7:1: note: possible intended match here
Compression rate: 1.86
^

Input file: 
Check file: 
/home/buildbot/buildbot-root/llvm-clang-x86_64-sie-ubuntu-fast/llvm-project/clang/test/Driver/clang-offload-bundler-zlib.c

-dump-input=help explains the following input dump.

Input was:
<<
1: Compressed bundle format 
version: 2 
2: Total file size (including headers): 
125 bytes 
3: Compression method used: 
zlib 
check:36   ^
4: Compression level: 6 
[0

[clang] [OffloadBundler] Compress bundles over 4GB (PR #122307)

2025-01-20 Thread Yaxun Liu via cfe-commits 

https://github.com/yxsamliu closed 
https://github.com/llvm/llvm-project/pull/122307
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] [OffloadBundler] Compress bundles over 4GB (PR #122307)

2025-01-20 Thread Joseph Huber via cfe-commits 

jhuber6 wrote:

> currently HIP runtime loads fat binary by itself for non-compressed fat 
> binary. It needs to switch to use comgr for loading fat binary. Some math 
> libs compile assembly code to code objects then bundle them to fat binary. 
> Such math libs need to switch to use tools for packing offload binary. Also, 
> zstd compression of code objects for different GPU arch by taking advantage 
> of redundancy in ISA is critical for math libraries. The offload binary needs 
> to support zstd compression.

We could probably just keep backwards compatibility by checking file magic now 
that the bundles have them. Does it do anything special during optimization? 
The offload binary is just a big blob, basically looking like this. I figured 
any optimization algorithm would be able to figure out the repetition itself.
```




...


```

https://github.com/llvm/llvm-project/pull/122307
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] [OffloadBundler] Compress bundles over 4GB (PR #122307)

2025-01-20 Thread Yaxun Liu via cfe-commits 

yxsamliu wrote:

> > > Seems fine. What's required to move HIP over to using the binary format 
> > > natively by the way? Guessing we'd need to update a bunch of tools in the 
> > > fork.
> > 
> > 
> > need to teach comgr to load the new offload binary
> 
> Should be easy enough, since there's a library function for that in LLVM. Is 
> comgr the only location that would need to change? Nothing in ROCr or HIP 
> runtime?

currently HIP runtime loads fat binary by itself for non-compressed fat binary. 
It needs to switch to use comgr for loading fat binary. Some math libs compile 
assembly code to code objects then bundle them to fat binary. Such math libs 
need to switch to use tools for packing offload binary. Also, zstd compression 
of code objects for different GPU arch by taking advantage of redundancy in ISA 
is critical for math libraries. The offload binary needs to support zstd 
compression.

https://github.com/llvm/llvm-project/pull/122307
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] [OffloadBundler] Compress bundles over 4GB (PR #122307)

2025-01-20 Thread Joseph Huber via cfe-commits 

jhuber6 wrote:

> > Seems fine. What's required to move HIP over to using the binary format 
> > natively by the way? Guessing we'd need to update a bunch of tools in the 
> > fork.
> 
> need to teach comgr to load the new offload binary

Should be easy enough, since there's a library function for that in LLVM. Is 
comgr the only location that would need to change? Nothing in ROCr or HIP 
runtime?

https://github.com/llvm/llvm-project/pull/122307
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] [OffloadBundler] Compress bundles over 4GB (PR #122307)

2025-01-20 Thread Yaxun Liu via cfe-commits 

yxsamliu wrote:

> Seems fine. What's required to move HIP over to using the binary format 
> natively by the way? Guessing we'd need to update a bunch of tools in the 
> fork.

need to teach comgr to load the new offload binary

https://github.com/llvm/llvm-project/pull/122307
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] [OffloadBundler] Compress bundles over 4GB (PR #122307)

2025-01-20 Thread Joseph Huber via cfe-commits 

https://github.com/jhuber6 approved this pull request.

Seems fine. What's required to move HIP over to using the binary format 
natively by the way? Guessing we'd need to update a bunch of tools in the fork.

https://github.com/llvm/llvm-project/pull/122307
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] [OffloadBundler] Compress bundles over 4GB (PR #122307)

2025-01-20 Thread Yaxun Liu via cfe-commits 

yxsamliu wrote:

ping

https://github.com/llvm/llvm-project/pull/122307
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] [OffloadBundler] Compress bundles over 4GB (PR #122307)

2025-01-20 Thread Yaxun Liu via cfe-commits 

https://github.com/yxsamliu updated 
https://github.com/llvm/llvm-project/pull/122307

>From eb980d1e6f3919ab3ee86ff5ac8ea0805dd8afe1 Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" 
Date: Thu, 9 Jan 2025 09:19:35 -0500
Subject: [PATCH] [OffloadBundler] Compress bundles over 4GB

Added initial support for version 3 of the compressed offload bundle format,
which uses 64-bit fields for Total File Size and Uncompressed Binary Size.
This enables support for files larger than 4GB. The support is currently
experimental and can be enabled by setting the environment variable
`COMPRESSED_BUNDLE_FORMAT_VERSION=3`.
---
 clang/docs/ClangOffloadBundler.rst|   2 +
 clang/docs/ReleaseNotes.rst   |   1 +
 clang/include/clang/Driver/OffloadBundler.h   |  50 +--
 clang/lib/Driver/OffloadBundler.cpp   | 138 ++
 .../test/Driver/clang-offload-bundler-zlib.c  |  24 +++
 5 files changed, 172 insertions(+), 43 deletions(-)

diff --git a/clang/docs/ClangOffloadBundler.rst 
b/clang/docs/ClangOffloadBundler.rst
index 3c241027d405ca..bceb4060992fc8 100644
--- a/clang/docs/ClangOffloadBundler.rst
+++ b/clang/docs/ClangOffloadBundler.rst
@@ -542,3 +542,5 @@ The compressed offload bundle begins with a header followed 
by the compressed bi
 
 - **Compressed Data**:
 The actual compressed binary data follows the header. Its size can be 
inferred from the total size of the file minus the header size.
+
+> **Note**: Version 3 of the format is under development. It uses 64-bit 
fields for Total File Size and Uncompressed Binary Size to support files larger 
than 4GB. To experiment with version 3, set the environment variable 
`COMPRESSED_BUNDLE_FORMAT_VERSION=3`. This support is experimental and not 
recommended for production use.
\ No newline at end of file
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index f6d5c346021d60..9061095594036c 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -1131,6 +1131,7 @@ RISC-V Support
 CUDA/HIP Language Changes
 ^
 - Fixed a bug about overriding a constexpr pure-virtual member function with a 
non-constexpr virtual member function which causes compilation failure when 
including standard C++ header `format`.
+- Added initial support for version 3 of the compressed offload bundle format, 
which uses 64-bit fields for Total File Size and Uncompressed Binary Size. This 
enables support for files larger than 4GB. The support is currently 
experimental and can be enabled by setting the environment variable 
`COMPRESSED_BUNDLE_FORMAT_VERSION=3`.
 
 CUDA Support
 
diff --git a/clang/include/clang/Driver/OffloadBundler.h 
b/clang/include/clang/Driver/OffloadBundler.h
index 57ecbdcb7d040e..31c11e25ecd9f2 100644
--- a/clang/include/clang/Driver/OffloadBundler.h
+++ b/clang/include/clang/Driver/OffloadBundler.h
@@ -39,6 +39,7 @@ class OffloadBundlerConfig {
   bool Verbose = false;
   llvm::compression::Format CompressionFormat;
   int CompressionLevel;
+  uint16_t CompressedBundleVersion;
 
   unsigned BundleAlignment = 1;
   unsigned HostInputIndex = ~0u;
@@ -100,36 +101,63 @@ struct OffloadTargetInfo {
 // - Version (2 bytes)
 // - Compression Method (2 bytes) - Uses the values from
 // llvm::compression::Format.
-// - Total file size (4 bytes). Available in version 2 and above.
-// - Uncompressed Size (4 bytes).
+// - Total file size (4 bytes in V2, 8 bytes in V3).
+// - Uncompressed Size (4 bytes in V1/V2, 8 bytes in V3).
 // - Truncated MD5 Hash (8 bytes).
 // - Compressed Data (variable length).
-
 class CompressedOffloadBundle {
 private:
   static inline const size_t MagicSize = 4;
   static inline const size_t VersionFieldSize = sizeof(uint16_t);
   static inline const size_t MethodFieldSize = sizeof(uint16_t);
-  static inline const size_t FileSizeFieldSize = sizeof(uint32_t);
-  static inline const size_t UncompressedSizeFieldSize = sizeof(uint32_t);
+  // Legacy size fields for V1/V2
+  static inline const size_t FileSizeFieldSizeV2 = sizeof(uint32_t);
+  static inline const size_t UncompressedSizeFieldSizeV2 = sizeof(uint32_t);
+  // New size fields for V3
+  static inline const size_t FileSizeFieldSizeV3 = sizeof(uint64_t);
+  static inline const size_t UncompressedSizeFieldSizeV3 = sizeof(uint64_t);
   static inline const size_t HashFieldSize = sizeof(uint64_t);
+
+  // Keep V1 header size for backward compatibility
   static inline const size_t V1HeaderSize =
   MagicSize + VersionFieldSize + MethodFieldSize +
-  UncompressedSizeFieldSize + HashFieldSize;
+  UncompressedSizeFieldSizeV2 + HashFieldSize;
+
+  // Keep V2 header size for backward compatibility
   static inline const size_t V2HeaderSize =
-  MagicSize + VersionFieldSize + FileSizeFieldSize + MethodFieldSize +
-  UncompressedSizeFieldSize + HashFieldSize;
+  MagicSize + VersionFieldSize + FileSizeFieldSizeV2 + MethodFieldSize +
+  UncompressedSizeFieldSizeV

[clang] [OffloadBundler] Compress bundles over 4GB (PR #122307)

2025-01-09 Thread Yaxun Liu via cfe-commits 

https://github.com/yxsamliu updated 
https://github.com/llvm/llvm-project/pull/122307

>From a067566b7422f84323b6f44df41918fe503fcb84 Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" 
Date: Thu, 9 Jan 2025 09:19:35 -0500
Subject: [PATCH] [OffloadBundler] Compress bundles over 4GB

Added initial support for version 3 of the compressed offload bundle format,
which uses 64-bit fields for Total File Size and Uncompressed Binary Size.
This enables support for files larger than 4GB. The support is currently
experimental and can be enabled by setting the environment variable
`COMPRESSED_BUNDLE_FORMAT_VERSION=3`.
---
 clang/docs/ClangOffloadBundler.rst|   2 +
 clang/docs/ReleaseNotes.rst   |   1 +
 clang/include/clang/Driver/OffloadBundler.h   |  50 +--
 clang/lib/Driver/OffloadBundler.cpp   | 138 ++
 .../test/Driver/clang-offload-bundler-zlib.c  |  24 +++
 5 files changed, 172 insertions(+), 43 deletions(-)

diff --git a/clang/docs/ClangOffloadBundler.rst 
b/clang/docs/ClangOffloadBundler.rst
index 3c241027d405ca..bceb4060992fc8 100644
--- a/clang/docs/ClangOffloadBundler.rst
+++ b/clang/docs/ClangOffloadBundler.rst
@@ -542,3 +542,5 @@ The compressed offload bundle begins with a header followed 
by the compressed bi
 
 - **Compressed Data**:
 The actual compressed binary data follows the header. Its size can be 
inferred from the total size of the file minus the header size.
+
+> **Note**: Version 3 of the format is under development. It uses 64-bit 
fields for Total File Size and Uncompressed Binary Size to support files larger 
than 4GB. To experiment with version 3, set the environment variable 
`COMPRESSED_BUNDLE_FORMAT_VERSION=3`. This support is experimental and not 
recommended for production use.
\ No newline at end of file
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 07a1a4195427d8..1cd2e5db5c1f79 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -1059,6 +1059,7 @@ RISC-V Support
 CUDA/HIP Language Changes
 ^
 - Fixed a bug about overriding a constexpr pure-virtual member function with a 
non-constexpr virtual member function which causes compilation failure when 
including standard C++ header `format`.
+- Added initial support for version 3 of the compressed offload bundle format, 
which uses 64-bit fields for Total File Size and Uncompressed Binary Size. This 
enables support for files larger than 4GB. The support is currently 
experimental and can be enabled by setting the environment variable 
`COMPRESSED_BUNDLE_FORMAT_VERSION=3`.
 
 CUDA Support
 
diff --git a/clang/include/clang/Driver/OffloadBundler.h 
b/clang/include/clang/Driver/OffloadBundler.h
index 57ecbdcb7d040e..31c11e25ecd9f2 100644
--- a/clang/include/clang/Driver/OffloadBundler.h
+++ b/clang/include/clang/Driver/OffloadBundler.h
@@ -39,6 +39,7 @@ class OffloadBundlerConfig {
   bool Verbose = false;
   llvm::compression::Format CompressionFormat;
   int CompressionLevel;
+  uint16_t CompressedBundleVersion;
 
   unsigned BundleAlignment = 1;
   unsigned HostInputIndex = ~0u;
@@ -100,36 +101,63 @@ struct OffloadTargetInfo {
 // - Version (2 bytes)
 // - Compression Method (2 bytes) - Uses the values from
 // llvm::compression::Format.
-// - Total file size (4 bytes). Available in version 2 and above.
-// - Uncompressed Size (4 bytes).
+// - Total file size (4 bytes in V2, 8 bytes in V3).
+// - Uncompressed Size (4 bytes in V1/V2, 8 bytes in V3).
 // - Truncated MD5 Hash (8 bytes).
 // - Compressed Data (variable length).
-
 class CompressedOffloadBundle {
 private:
   static inline const size_t MagicSize = 4;
   static inline const size_t VersionFieldSize = sizeof(uint16_t);
   static inline const size_t MethodFieldSize = sizeof(uint16_t);
-  static inline const size_t FileSizeFieldSize = sizeof(uint32_t);
-  static inline const size_t UncompressedSizeFieldSize = sizeof(uint32_t);
+  // Legacy size fields for V1/V2
+  static inline const size_t FileSizeFieldSizeV2 = sizeof(uint32_t);
+  static inline const size_t UncompressedSizeFieldSizeV2 = sizeof(uint32_t);
+  // New size fields for V3
+  static inline const size_t FileSizeFieldSizeV3 = sizeof(uint64_t);
+  static inline const size_t UncompressedSizeFieldSizeV3 = sizeof(uint64_t);
   static inline const size_t HashFieldSize = sizeof(uint64_t);
+
+  // Keep V1 header size for backward compatibility
   static inline const size_t V1HeaderSize =
   MagicSize + VersionFieldSize + MethodFieldSize +
-  UncompressedSizeFieldSize + HashFieldSize;
+  UncompressedSizeFieldSizeV2 + HashFieldSize;
+
+  // Keep V2 header size for backward compatibility
   static inline const size_t V2HeaderSize =
-  MagicSize + VersionFieldSize + FileSizeFieldSize + MethodFieldSize +
-  UncompressedSizeFieldSize + HashFieldSize;
+  MagicSize + VersionFieldSize + FileSizeFieldSizeV2 + MethodFieldSize +
+  UncompressedSizeFieldSizeV

[clang] [OffloadBundler] Compress bundles over 4GB (PR #122307)

2025-01-09 Thread Yaxun Liu via cfe-commits 

yxsamliu wrote:

> I've never been a fan of this type of implicit behavior, since it will change 
> spuriously depending on what the user wants.
> 
> I also wonder if we couldn't do something similar with ELF compression so we 
> don't need to manually do all this stuff.

Linux only supports compressed debug info sections.

The env var is for debugging purpose only. After runtime support is mature, we 
will remove the env var and also eventually remove support of generation of old 
versions of compressed bundle format.

https://github.com/llvm/llvm-project/pull/122307
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] [OffloadBundler] Compress bundles over 4GB (PR #122307)

2025-01-09 Thread Joseph Huber via cfe-commits 

https://github.com/jhuber6 commented:

I've never been a fan of this type of implicit behavior, since it will change 
spuriously depending on what the user wants.

I also wonder if we couldn't do something similar with ELF compression so we 
don't need to manually do all this stuff.

https://github.com/llvm/llvm-project/pull/122307
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang] [OffloadBundler] Compress bundles over 4GB (PR #122307)

2025-01-09 Thread via cfe-commits 

llvmbot wrote:



@llvm/pr-subscribers-clang-driver

@llvm/pr-subscribers-clang

Author: Yaxun (Sam) Liu (yxsamliu)


Changes

Added initial support for version 3 of the compressed offload bundle format, 
which uses 64-bit fields for Total File Size and Uncompressed Binary Size. This 
enables support for files larger than 4GB. The support is currently 
experimental and can be enabled by setting the environment variable 
`COMPRESSED_BUNDLE_FORMAT_VERSION=3`.

---
Full diff: https://github.com/llvm/llvm-project/pull/122307.diff


5 Files Affected:

- (modified) clang/docs/ClangOffloadBundler.rst (+2) 
- (modified) clang/docs/ReleaseNotes.rst (+1) 
- (modified) clang/include/clang/Driver/OffloadBundler.h (+39-11) 
- (modified) clang/lib/Driver/OffloadBundler.cpp (+106-32) 
- (modified) clang/test/Driver/clang-offload-bundler-zlib.c (+24) 


``diff
diff --git a/clang/docs/ClangOffloadBundler.rst 
b/clang/docs/ClangOffloadBundler.rst
index 3c241027d405ca..bceb4060992fc8 100644
--- a/clang/docs/ClangOffloadBundler.rst
+++ b/clang/docs/ClangOffloadBundler.rst
@@ -542,3 +542,5 @@ The compressed offload bundle begins with a header followed 
by the compressed bi
 
 - **Compressed Data**:
 The actual compressed binary data follows the header. Its size can be 
inferred from the total size of the file minus the header size.
+
+> **Note**: Version 3 of the format is under development. It uses 64-bit 
fields for Total File Size and Uncompressed Binary Size to support files larger 
than 4GB. To experiment with version 3, set the environment variable 
`COMPRESSED_BUNDLE_FORMAT_VERSION=3`. This support is experimental and not 
recommended for production use.
\ No newline at end of file
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 93915e5db7d131..ea3c27bde40fb8 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -1053,6 +1053,7 @@ RISC-V Support
 
 CUDA/HIP Language Changes
 ^
+* Added initial support for version 3 of the compressed offload bundle format, 
which uses 64-bit fields for Total File Size and Uncompressed Binary Size. This 
enables support for files larger than 4GB. The support is currently 
experimental and can be enabled by setting the environment variable 
`COMPRESSED_BUNDLE_FORMAT_VERSION=3`.
 
 CUDA Support
 
diff --git a/clang/include/clang/Driver/OffloadBundler.h 
b/clang/include/clang/Driver/OffloadBundler.h
index 57ecbdcb7d040e..31c11e25ecd9f2 100644
--- a/clang/include/clang/Driver/OffloadBundler.h
+++ b/clang/include/clang/Driver/OffloadBundler.h
@@ -39,6 +39,7 @@ class OffloadBundlerConfig {
   bool Verbose = false;
   llvm::compression::Format CompressionFormat;
   int CompressionLevel;
+  uint16_t CompressedBundleVersion;
 
   unsigned BundleAlignment = 1;
   unsigned HostInputIndex = ~0u;
@@ -100,36 +101,63 @@ struct OffloadTargetInfo {
 // - Version (2 bytes)
 // - Compression Method (2 bytes) - Uses the values from
 // llvm::compression::Format.
-// - Total file size (4 bytes). Available in version 2 and above.
-// - Uncompressed Size (4 bytes).
+// - Total file size (4 bytes in V2, 8 bytes in V3).
+// - Uncompressed Size (4 bytes in V1/V2, 8 bytes in V3).
 // - Truncated MD5 Hash (8 bytes).
 // - Compressed Data (variable length).
-
 class CompressedOffloadBundle {
 private:
   static inline const size_t MagicSize = 4;
   static inline const size_t VersionFieldSize = sizeof(uint16_t);
   static inline const size_t MethodFieldSize = sizeof(uint16_t);
-  static inline const size_t FileSizeFieldSize = sizeof(uint32_t);
-  static inline const size_t UncompressedSizeFieldSize = sizeof(uint32_t);
+  // Legacy size fields for V1/V2
+  static inline const size_t FileSizeFieldSizeV2 = sizeof(uint32_t);
+  static inline const size_t UncompressedSizeFieldSizeV2 = sizeof(uint32_t);
+  // New size fields for V3
+  static inline const size_t FileSizeFieldSizeV3 = sizeof(uint64_t);
+  static inline const size_t UncompressedSizeFieldSizeV3 = sizeof(uint64_t);
   static inline const size_t HashFieldSize = sizeof(uint64_t);
+
+  // Keep V1 header size for backward compatibility
   static inline const size_t V1HeaderSize =
   MagicSize + VersionFieldSize + MethodFieldSize +
-  UncompressedSizeFieldSize + HashFieldSize;
+  UncompressedSizeFieldSizeV2 + HashFieldSize;
+
+  // Keep V2 header size for backward compatibility
   static inline const size_t V2HeaderSize =
-  MagicSize + VersionFieldSize + FileSizeFieldSize + MethodFieldSize +
-  UncompressedSizeFieldSize + HashFieldSize;
+  MagicSize + VersionFieldSize + FileSizeFieldSizeV2 + MethodFieldSize +
+  UncompressedSizeFieldSizeV2 + HashFieldSize;
+
+  // Add V3 header size with 64-bit fields
+  static inline const size_t V3HeaderSize =
+  MagicSize + VersionFieldSize + FileSizeFieldSizeV3 + MethodFieldSize +
+  UncompressedSizeFieldSizeV3 + HashFieldSize;
+
   static inline const llvm::StringRef MagicNumber = "CCOB";

[clang] [OffloadBundler] Compress bundles over 4GB (PR #122307)

2025-01-09 Thread Yaxun Liu via cfe-commits 

https://github.com/yxsamliu created 
https://github.com/llvm/llvm-project/pull/122307

Added initial support for version 3 of the compressed offload bundle format, 
which uses 64-bit fields for Total File Size and Uncompressed Binary Size. This 
enables support for files larger than 4GB. The support is currently 
experimental and can be enabled by setting the environment variable 
`COMPRESSED_BUNDLE_FORMAT_VERSION=3`.

>From b0feee1b1b6ea42535ba2345e9e506860653cdca Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" 
Date: Thu, 9 Jan 2025 09:19:35 -0500
Subject: [PATCH] [OffloadBundler] Compress bundles over 4GB

Added initial support for version 3 of the compressed offload bundle format,
which uses 64-bit fields for Total File Size and Uncompressed Binary Size.
This enables support for files larger than 4GB. The support is currently
experimental and can be enabled by setting the environment variable
`COMPRESSED_BUNDLE_FORMAT_VERSION=3`.
---
 clang/docs/ClangOffloadBundler.rst|   2 +
 clang/docs/ReleaseNotes.rst   |   1 +
 clang/include/clang/Driver/OffloadBundler.h   |  50 +--
 clang/lib/Driver/OffloadBundler.cpp   | 138 ++
 .../test/Driver/clang-offload-bundler-zlib.c  |  24 +++
 5 files changed, 172 insertions(+), 43 deletions(-)

diff --git a/clang/docs/ClangOffloadBundler.rst 
b/clang/docs/ClangOffloadBundler.rst
index 3c241027d405ca..bceb4060992fc8 100644
--- a/clang/docs/ClangOffloadBundler.rst
+++ b/clang/docs/ClangOffloadBundler.rst
@@ -542,3 +542,5 @@ The compressed offload bundle begins with a header followed 
by the compressed bi
 
 - **Compressed Data**:
 The actual compressed binary data follows the header. Its size can be 
inferred from the total size of the file minus the header size.
+
+> **Note**: Version 3 of the format is under development. It uses 64-bit 
fields for Total File Size and Uncompressed Binary Size to support files larger 
than 4GB. To experiment with version 3, set the environment variable 
`COMPRESSED_BUNDLE_FORMAT_VERSION=3`. This support is experimental and not 
recommended for production use.
\ No newline at end of file
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 93915e5db7d131..ea3c27bde40fb8 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -1053,6 +1053,7 @@ RISC-V Support
 
 CUDA/HIP Language Changes
 ^
+* Added initial support for version 3 of the compressed offload bundle format, 
which uses 64-bit fields for Total File Size and Uncompressed Binary Size. This 
enables support for files larger than 4GB. The support is currently 
experimental and can be enabled by setting the environment variable 
`COMPRESSED_BUNDLE_FORMAT_VERSION=3`.
 
 CUDA Support
 
diff --git a/clang/include/clang/Driver/OffloadBundler.h 
b/clang/include/clang/Driver/OffloadBundler.h
index 57ecbdcb7d040e..31c11e25ecd9f2 100644
--- a/clang/include/clang/Driver/OffloadBundler.h
+++ b/clang/include/clang/Driver/OffloadBundler.h
@@ -39,6 +39,7 @@ class OffloadBundlerConfig {
   bool Verbose = false;
   llvm::compression::Format CompressionFormat;
   int CompressionLevel;
+  uint16_t CompressedBundleVersion;
 
   unsigned BundleAlignment = 1;
   unsigned HostInputIndex = ~0u;
@@ -100,36 +101,63 @@ struct OffloadTargetInfo {
 // - Version (2 bytes)
 // - Compression Method (2 bytes) - Uses the values from
 // llvm::compression::Format.
-// - Total file size (4 bytes). Available in version 2 and above.
-// - Uncompressed Size (4 bytes).
+// - Total file size (4 bytes in V2, 8 bytes in V3).
+// - Uncompressed Size (4 bytes in V1/V2, 8 bytes in V3).
 // - Truncated MD5 Hash (8 bytes).
 // - Compressed Data (variable length).
-
 class CompressedOffloadBundle {
 private:
   static inline const size_t MagicSize = 4;
   static inline const size_t VersionFieldSize = sizeof(uint16_t);
   static inline const size_t MethodFieldSize = sizeof(uint16_t);
-  static inline const size_t FileSizeFieldSize = sizeof(uint32_t);
-  static inline const size_t UncompressedSizeFieldSize = sizeof(uint32_t);
+  // Legacy size fields for V1/V2
+  static inline const size_t FileSizeFieldSizeV2 = sizeof(uint32_t);
+  static inline const size_t UncompressedSizeFieldSizeV2 = sizeof(uint32_t);
+  // New size fields for V3
+  static inline const size_t FileSizeFieldSizeV3 = sizeof(uint64_t);
+  static inline const size_t UncompressedSizeFieldSizeV3 = sizeof(uint64_t);
   static inline const size_t HashFieldSize = sizeof(uint64_t);
+
+  // Keep V1 header size for backward compatibility
   static inline const size_t V1HeaderSize =
   MagicSize + VersionFieldSize + MethodFieldSize +
-  UncompressedSizeFieldSize + HashFieldSize;
+  UncompressedSizeFieldSizeV2 + HashFieldSize;
+
+  // Keep V2 header size for backward compatibility
   static inline const size_t V2HeaderSize =
-  MagicSize + VersionFieldSize + FileSizeFieldSize + MethodFieldSize +
-  UncompressedSi