(trafficserver) branch master updated: Cap uncompressed length in TLS Certificate Compression (#13117)

[maskit]

This is an automated email from the ASF dual-hosted git repository.

maskit pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/trafficserver.git


The following commit(s) were added to refs/heads/master by this push:
     new a65725d423 Cap uncompressed length in TLS Certificate Compression 
(#13117)
a65725d423 is described below

commit a65725d423dfc32906e7acec6696ac9acbf91236
Author: Masakazu Kitajo <mas...@apache.org>
AuthorDate: Fri Apr 24 12:58:44 2026 -0600

    Cap uncompressed length in TLS Certificate Compression (#13117)
    
    * Cap uncompressed length in TLS Certificate Compression
    
    * Set a nullptr
    
    * Update the comment
---
 src/iocore/net/TLSCertCompression.h         | 6 ++++++
 src/iocore/net/TLSCertCompression_brotli.cc | 7 +++++++
 src/iocore/net/TLSCertCompression_zlib.cc   | 7 +++++++
 src/iocore/net/TLSCertCompression_zstd.cc   | 7 +++++++
 4 files changed, 27 insertions(+)

diff --git a/src/iocore/net/TLSCertCompression.h 
b/src/iocore/net/TLSCertCompression.h
index 891c2c5951..fa7bbc6ee5 100644
--- a/src/iocore/net/TLSCertCompression.h
+++ b/src/iocore/net/TLSCertCompression.h
@@ -27,6 +27,12 @@
 #include <vector>
 #include <string>
 
+// RFC 8879 uses uint24 for uncompressed_length, allowing up to ~16 MB.
+// Real certificate chains rarely exceed 10-30 KB even with large RSA
+// keys and multiple intermediates — 128 KB gives ample headroom while
+// preventing excessive memory allocation from a malicious peer.
+constexpr size_t MAX_CERT_UNCOMPRESSED_LEN = 128 * 1024;
+
 /**
  * Common function to set certificate compression preference
  *
diff --git a/src/iocore/net/TLSCertCompression_brotli.cc 
b/src/iocore/net/TLSCertCompression_brotli.cc
index b758e476e0..52cf22dada 100644
--- a/src/iocore/net/TLSCertCompression_brotli.cc
+++ b/src/iocore/net/TLSCertCompression_brotli.cc
@@ -22,6 +22,7 @@
  */
 
 #include "TLSCertCompression_brotli.h"
+#include "TLSCertCompression.h"
 #include "SSLStats.h"
 #include <openssl/ssl.h>
 #include <brotli/decode.h>
@@ -55,6 +56,12 @@ compression_func_brotli(SSL * /* ssl */, CBB *out, const 
uint8_t *in, size_t in_
 int
 decompression_func_brotli(SSL * /* ssl */, CRYPTO_BUFFER **out, size_t 
uncompressed_len, const uint8_t *in, size_t in_len)
 {
+  if (uncompressed_len > MAX_CERT_UNCOMPRESSED_LEN) {
+    *out = nullptr;
+    Metrics::Counter::increment(ssl_rsb.cert_decompress_brotli_failure);
+    return 0;
+  }
+
   uint8_t *buf;
 
   *out = CRYPTO_BUFFER_alloc(&buf, uncompressed_len);
diff --git a/src/iocore/net/TLSCertCompression_zlib.cc 
b/src/iocore/net/TLSCertCompression_zlib.cc
index 5416dc6986..1d724ef541 100644
--- a/src/iocore/net/TLSCertCompression_zlib.cc
+++ b/src/iocore/net/TLSCertCompression_zlib.cc
@@ -22,6 +22,7 @@
  */
 
 #include "TLSCertCompression_zlib.h"
+#include "TLSCertCompression.h"
 #include "SSLStats.h"
 #include <openssl/ssl.h>
 #include <zlib.h>
@@ -53,6 +54,12 @@ compression_func_zlib(SSL * /* ssl */, CBB *out, const 
uint8_t *in, size_t in_le
 int
 decompression_func_zlib(SSL * /* ssl */, CRYPTO_BUFFER **out, size_t 
uncompressed_len, const uint8_t *in, size_t in_len)
 {
+  if (uncompressed_len > MAX_CERT_UNCOMPRESSED_LEN) {
+    *out = nullptr;
+    Metrics::Counter::increment(ssl_rsb.cert_decompress_zlib_failure);
+    return 0;
+  }
+
   uint8_t *buf;
 
   *out = CRYPTO_BUFFER_alloc(&buf, uncompressed_len);
diff --git a/src/iocore/net/TLSCertCompression_zstd.cc 
b/src/iocore/net/TLSCertCompression_zstd.cc
index ab63633d0f..85cf2a94f9 100644
--- a/src/iocore/net/TLSCertCompression_zstd.cc
+++ b/src/iocore/net/TLSCertCompression_zstd.cc
@@ -22,6 +22,7 @@
  */
 
 #include "TLSCertCompression_zstd.h"
+#include "TLSCertCompression.h"
 #include "SSLStats.h"
 #include <openssl/ssl.h>
 #include <zstd.h>
@@ -60,6 +61,12 @@ compression_func_zstd(SSL * /* ssl */, CBB *out, const 
uint8_t *in, size_t in_le
 int
 decompression_func_zstd(SSL * /* ssl */, CRYPTO_BUFFER **out, size_t 
uncompressed_len, const uint8_t *in, size_t in_len)
 {
+  if (uncompressed_len > MAX_CERT_UNCOMPRESSED_LEN) {
+    *out = nullptr;
+    Metrics::Counter::increment(ssl_rsb.cert_decompress_zstd_failure);
+    return 0;
+  }
+
   uint8_t *buf;
 
   *out = CRYPTO_BUFFER_alloc(&buf, uncompressed_len);