undefined behaviour in troff

[Bruno Haible via discussion of the GNU roff typesetting system and related software]

Hi,

I tried building the groff-1.25.0.rc1 with clang's ASAN (usually better than
gcc's ASAN) on a GNU system. For reference, I use these environment variables:

CC="$HOME/inst-clang/22.1.0/bin/clang -Wl,-rpath,$HOME/inst-clang/22.1.0/lib 
-Wl,-rpath,$HOME/inst-clang/22.1.0/lib/x86_64-unknown-linux-gnu"
CXX="$HOME/inst-clang/22.1.0/bin/clang++ -I/usr/include/c++/14 
-I/usr/include/x86_64-linux-gnu/c++/11 -L/usr/lib/gcc/x86_64-linux-gnu/14 
-Wl,-rpath,$HOME/inst-clang/22.1.0/lib 
-Wl,-rpath,$HOME/inst-clang/22.1.0/lib/x86_64-unknown-linux-gnu"

CC="$CC 
-fsanitize=address,undefined,signed-integer-overflow,shift,integer-divide-by-zero
 -fno-sanitize-recover=undefined"; CXX="$CXX 
-fsanitize=address,undefined,signed-integer-overflow,shift,integer-divide-by-zero
 -fno-sanitize-recover=undefined"; export CC CXX
CFLAGS="-O0 -fno-omit-frame-pointer -ggdb"; CXXFLAGS="-O0 
-fno-omit-frame-pointer -ggdb"; export CFLAGS CXXFLAGS

ASAN_OPTIONS="detect_leaks=0 abort_on_error=1 allocator_may_return_null=1"; 
export ASAN_OPTIONS

The first finding is:

$ make -k
  GEN      doc/line-layout.eps
../src/roff/troff/dictionary.cpp:45:29: runtime error: left shift of 
72057594037927936 by 8 places cannot be represented in type 'ssize_t' (aka 
'long')
SUMMARY: UndefinedBehaviorSanitizer: undefined-behavior 
../src/roff/troff/dictionary.cpp:45:29 
/build/groff-1.25.0.rc1/build-64-clang/groff: error: troff: Aborted
make: *** [Makefile:20188: doc/line-layout.eps] Error 8

Signed integer overflow is undefined behaviour.

The attached patch fixes it for me.


diff -r -u groff-1.25.0.rc1.orig/src/roff/troff/dictionary.cpp groff-1.25.0.rc1/src/roff/troff/dictionary.cpp
--- groff-1.25.0.rc1.orig/src/roff/troff/dictionary.cpp	2026-07-14 10:02:39.000000000 +0200
+++ groff-1.25.0.rc1/src/roff/troff/dictionary.cpp	2026-07-25 00:56:48.222239730 +0200
@@ -26,12 +26,13 @@
 
 #include <stdio.h> // prerequisite of searchpath.h
 #include <sys/types.h> // ssize_t
+#include <limits.h> // SSIZE_MAX
 
 // libgroff
 #include "symbol.h" // prerequisite of dictionary.h
 #include "dictionary.h"
 #include "errarg.h" // prerequisite of error.h
-#include "error.h" // prerequisite of error.h
+#include "error.h"
 
 // is 'p' a good size for a hash table
 
@@ -42,9 +43,13 @@
   for (i = 2; i <= (p / 2); i++)
     if ((p % i) == 0)
       return false;
-  for (i = 0x100; i != 0; i <<= 8)
-    if ((i % p) <= SMALL || (i % p) > (p - SMALL))
-      return false;
+  for (i = 0x100; ; i <<= 8)
+    {
+      if ((i % p) <= SMALL || (i % p) > (p - SMALL))
+        return false;
+      if (i > (SSIZE_MAX >> 8))
+        break;
+    }
   return true;
 }