[openssl.org #2602] [PATCH] DTLS bug prevents manual MTU setting

[Robin Seggelmann via RT]

Since patch #2506 the MTU stored in DTLS1_STATE is cleared with every initial 
handshake. This prevents setting an MTU manually when SSL_OP_NO_QUERY_MTU is 
set and causes OpenSSL to crash because an MTU of 0 is used therefore an memcpy 
with a negative length in the following. This patch prevents the MTU from being 
overwritten if SSL_OP_NO_QUERY_MTU is set, adds an assert to prevent invalid 
MTU values from being used and doesn't allow to manually set too low MTU values.

Thanks to Paul Witty for reporting this bug!

Best regards
Robin




--- ssl/d1_both.c       3 Apr 2011 17:15:08 -0000       1.14.2.24
+++ ssl/d1_both.c       13 Sep 2011 14:51:21 -0000
@@ -158,7 +158,6 @@
 /* XDTLS:  figure out the right values */
 static unsigned int g_probable_mtu[] = {1500 - 28, 512 - 28, 256 - 28};
 
-static unsigned int dtls1_min_mtu(void);
 static unsigned int dtls1_guess_mtu(unsigned int curr_mtu);
 static void dtls1_fix_message_header(SSL *s, unsigned long frag_off, 
        unsigned long frag_len);
@@ -264,11 +263,10 @@
                        return ret;
                mtu = s->d1->mtu - (DTLS1_HM_HEADER_LENGTH + 
DTLS1_RT_HEADER_LENGTH);
                }
-
-       OPENSSL_assert(mtu > 0);  /* should have something reasonable now */
-
 #endif
 
+       OPENSSL_assert(s->d1->mtu >= dtls1_min_mtu());  /* should have 
something reasonable now */
+
        if ( s->init_off == 0  && type == SSL3_RT_HANDSHAKE)
                OPENSSL_assert(s->init_num == 
                        (int)s->d1->w_msg_hdr.msg_len + DTLS1_HM_HEADER_LENGTH);
@@ -1367,7 +1365,7 @@
        return p;
        }
 
-static unsigned int 
+unsigned int 
 dtls1_min_mtu(void)
        {
        return (g_probable_mtu[(sizeof(g_probable_mtu) / 

--- ssl/d1_lib.c        22 Jun 2011 15:29:55 -0000      1.8.2.17
+++ ssl/d1_lib.c        13 Sep 2011 14:51:21 -0000
@@ -204,7 +204,8 @@
     pqueue buffered_messages;
        pqueue sent_messages;
        pqueue buffered_app_data;
-       
+       unsigned int mtu;
+
        if (s->d1)
                {
                unprocessed_rcds = s->d1->unprocessed_rcds.q;
@@ -212,6 +213,7 @@
                buffered_messages = s->d1->buffered_messages;
                sent_messages = s->d1->sent_messages;
                buffered_app_data = s->d1->buffered_app_data.q;
+               mtu = s->d1->mtu;
 
                dtls1_clear_queues(s);
 
@@ -222,6 +224,11 @@
                        s->d1->cookie_len = sizeof(s->d1->cookie);
                        }
 
+               if (SSL_get_options(s) & SSL_OP_NO_QUERY_MTU)
+                       {
+                       s->d1->mtu = mtu;
+                       }
+
                s->d1->unprocessed_rcds.q = unprocessed_rcds;
                s->d1->processed_rcds.q = processed_rcds;
                s->d1->buffered_messages = buffered_messages;

--- ssl/ssl_lib.c       14 Aug 2011 13:48:42 -0000      1.176.2.20
+++ ssl/ssl_lib.c       13 Sep 2011 14:51:21 -0000
@@ -1054,6 +1054,9 @@
                s->max_cert_list=larg;
                return(l);
        case SSL_CTRL_SET_MTU:
+               if (larg < dtls1_min_mtu())
+                       return 0;
+
                if (SSL_version(s) == DTLS1_VERSION ||
                    SSL_version(s) == DTLS1_BAD_VER)
                        {

--- ssl/ssl_locl.h      8 Dec 2009 11:38:18 -0000       1.100.2.10
+++ ssl/ssl_locl.h      13 Sep 2011 14:51:21 -0000
@@ -950,6 +950,7 @@
 int dtls1_is_timer_expired(SSL *s);
 void dtls1_double_timeout(SSL *s);
 int dtls1_send_newsession_ticket(SSL *s);
+unsigned int dtls1_min_mtu(void);
 
 /* some client-only functions */
 int ssl3_client_hello(SSL *s);

dtls-mtu-bug-1.0.1.patch
Description: Binary data

dtls-mtu-bug-1.0.0.patch
Description: Binary data

dtls-mtu-bug-0.9.8.patch
Description: Binary data