From 001b0511770c7ec83f921693d70e543f472ffc39 Mon Sep 17 00:00:00 2001 From: Frederic Koehler Date: Sun, 12 Aug 2012 02:12:28 -0400 Subject: [PATCH] tooltalk: Fix bad assumptions about sizeof(uid_t) In part of the tooltalk rpc code (mp_message.c), it was assumed that on the majority of platforms, sizeof(uid_t)=sizeof(gid_t)=sizeof(long). On Linux-x64, uid_t is an unsigned int, which makes the code fail: all tooltalk messages fail to send with an RPC_CANTENCODEARGS at the rpc-level, and TT_INTERNAL_ERR for the actual program. We instead change the code to explicitly examine sizeof(uid_t) to see whether it is int or long sized. This allows tooltalk-dependent functinoality like logout and multiple calls to dtfile to work. --- cde/lib/tt/lib/mp/mp_message.C | 45 ++++++++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/cde/lib/tt/lib/mp/mp_message.C b/cde/lib/tt/lib/mp/mp_message.C index 1c00fca..06afb24 100644 --- a/cde/lib/tt/lib/mp/mp_message.C +++ b/cde/lib/tt/lib/mp/mp_message.C @@ -47,7 +47,7 @@ #include #include #include - +#include // // Base constructor for a message. ::xdr() relies on this @@ -302,33 +302,44 @@ xdr(XDR *xdrs) // Even though in Solaris 2.x uid_t and gid_t are typedef'd as // u_long, the compiler complains if they are not explicitly // coerced. + + /* + * Obviously there is no xdr_u_uid_t, so we have to hack around this. + * At least by using sizeof checks, we aren't manually specifying + * the size of uid_t depending on the platform (which broke terribly + * when we tried to port to 64-bit linux at first). + * + * A side-effect of testing this way is that we have to spell out the pointer + * casts, since one of them is actually wrong for any given platform (but + * is also a dead code branch) + */ if (_ptr_guards&_TT_MSK_UID) { if (xdrs->x_op == XDR_DECODE) { _uid = 0; } -#ifdef __osf__ - if (! xdr_u_int(xdrs, &_uid)) { - return(0); - } -#else - if (! xdr_u_long(xdrs,(u_long *) &_uid)) { - return(0); + if (sizeof (int) == sizeof(uid_t)) { + if (! xdr_u_int(xdrs,(u_int *) &_uid)) return 0; + } else if (sizeof (long) == sizeof(uid_t)) { + if (! xdr_u_long(xdrs,(u_long *) &_gid)) return 0; + } else { + _tt_syslog( 0, LOG_ERR, "_Tt_message::xdr(XDR*): uid_t size is not equal to int or long; TOOLTALK RPC WILL NOT WORK!" ); } -#endif } if (_ptr_guards&_TT_MSK_GID) { if (xdrs->x_op == XDR_DECODE) { _gid = 0; } -#ifdef __osf__ - if (! xdr_u_int(xdrs, &_gid)) { - return(0); - } -#else - if (! xdr_u_long(xdrs,(u_long *) &_gid)) { - return(0); + if (sizeof (int) == sizeof(gid_t)) { + if (! xdr_u_int(xdrs,(u_int *) &_gid)) { + return(0); + } + } else if (sizeof (long) == sizeof(gid_t)) { + if (! xdr_u_long(xdrs,(u_long *) &_gid)) { + return(0); + } + } else { + _tt_syslog( 0, LOG_ERR, "_Tt_message::xdr(XDR*): gid_t size is not equal to int or long; TOOLTALK RPC WILL NOT WORK!" ); } -#endif } if (_ptr_guards&_TT_MSK_SESSION) { -- 1.7.11.2