After side by side comparisons between the MD5 routines from radclient and pam_radius_auth, I was able to determine the source of my problem. It appears that the MD5 code for pam_radius_auth was written for Linux/x86 and then ported to Solaris/SPARC. So the default little endian byte ordering in the MD5 routines is not compatible with big endian systems like HP's PA-RISC.
After changing the md5.c module to properly define the byteReverse function for HP compiles, the module worked fine. I have included all of the changes I had to make in order to get the code to compile and execute properly on HP-UX 11.0. It would be great if these changes could be rolled into the source tree. The output is from diff -u: Regards, Jim [EMAIL PROTECTED] ======================================================================= Begin output listing from diff -u ======================================================================= diff -u clean/pam_radius-1.3.16/Makefile pam_radius-1.3.16/Makefile --- clean/pam_radius-1.3.16/Makefile 2003-09-19 10:41:45.000000000 -0400 +++ pam_radius-1.3.16/Makefile 2003-12-23 11:21:26.000000000 -0500 @@ -15,7 +15,10 @@ # # If you're not using GCC, then you'll have to change the CFLAGS. # -CFLAGS = -Wall -fPIC +#CFLAGS = -Wall -fPIC +# Added by jl 12/09/2003 for HP-UX +CFLAGS = +DAportable +DSPA7100 +z +# End Add jl # # On Irix, use this with MIPSPRo C Compiler, and don't forget to export CC=cc # gcc on Irix does not work yet for pam_radius @@ -55,7 +58,10 @@ # gcc -shared pam_radius_auth.o md5.o -lpam -lc -o pam_radius_auth.so # pam_radius_auth.so: pam_radius_auth.o md5.o - ld -Bshareable pam_radius_auth.o md5.o -lpam -o pam_radius_auth.so +# ld -Bshareable pam_radius_auth.o md5.o -lpam -o pam_radius_auth.so +# Added by jl 12/09/2003 for HP-UX + ld -b pam_radius_auth.o md5.o -lpam -o pam_radius_auth.so +# End add jl ###################################################################### # diff -u clean/pam_radius-1.3.16/md5.c pam_radius-1.3.16/md5.c --- clean/pam_radius-1.3.16/md5.c 2002-06-28 02:29:21.000000000 -0400 +++ pam_radius-1.3.16/md5.c 2004-01-12 11:58:22.000000000 -0500 @@ -43,6 +43,12 @@ #define HIGHFIRST #endif +/* 01/12/2004 jl - Added for HPUX compiles */ +#ifdef _INCLUDE_HPUX_SOURCE +#define HIGHFIRST +#endif +/* END jl */ + #ifndef HIGHFIRST #define byteReverse(buf, len) /* Nothing */ #else diff -u clean/pam_radius-1.3.16/md5.h pam_radius-1.3.16/md5.h --- clean/pam_radius-1.3.16/md5.h 2003-04-29 16:19:16.000000000 -0400 +++ pam_radius-1.3.16/md5.h 2003-12-16 11:33:55.000000000 -0500 @@ -15,7 +15,10 @@ #define MD5Transform pra_MD5Transform #include <sys/types.h> -#define uint32 u_int32_t +/* Added by jl */ +/* #define u_int32_t unsigned int */ +/* #define uint32 u_int32_t */ +#define uint32 uint32_t struct MD5Context { uint32 buf[4]; Only in pam_radius-1.3.16: md5.o diff -u clean/pam_radius-1.3.16/pam_radius_auth.c pam_radius-1.3.16/pam_radius_auth.c --- clean/pam_radius-1.3.16/pam_radius_auth.c 2003-02-27 13:01:07.000000000 -0500 +++ pam_radius-1.3.16/pam_radius_auth.c 2004-01-12 12:00:52.000000000 -0500 @@ -58,6 +58,11 @@ #ifdef sun #include <security/pam_appl.h> #endif +/* Added by jl 12/09/2003 */ +#ifdef _INCLUDE_HPUX_SOURCE +#include <security/pam_appl.h> +#endif +/* End add jl */ #include <security/pam_modules.h> #include "pam_radius_auth.h" diff -u clean/pam_radius-1.3.16/pam_radius_auth.h pam_radius-1.3.16/pam_radius_auth.h --- clean/pam_radius-1.3.16/pam_radius_auth.h 2003-09-19 10:41:32.000000000 -0400 +++ pam_radius-1.3.16/pam_radius_auth.h 2003-12-09 10:20:29.000000000 -0500 @@ -66,6 +66,11 @@ */ #define CONST const #else +/* Added by jl 12/09/2003 */ +#ifdef _INCLUDE_HPUX_SOURCE +#define PAM_EXTERN extern +#endif +/* End add jl */ #define CONST const #endif ======================================================================= End output listing from diff -u ======================================================================= > -----Original Message----- > From: Lynch, James K [IT] > Sent: Wednesday, December 17, 2003 12:13 PM > To: '[EMAIL PROTECTED]' > Subject: problem with pam_radius_auth version 1.3.16 on HP-UX 11.11 > > > I am having a problem with the pam_radius_auth module running > under HP-UX. > > I compiled version 1.3.16 of pam_radius_auth on an HP-UX 11.0 > system with HP's Ansi C compiler. I had to #define u_int32_t > to be unsigned int. The code compiled ok and the shared > library was built successfully. > > My test environment is as follows: > > 1 HP-UX 11.11 system running pam_radius_auth version 1.3.16 > (systemA XXX.XXX.XXX.150) > 1 Solaris 2.8 system running pam_radius_auth version 1.3.16 (systemB) > 2 Red Hat AS 2.1 Linux servers running freeRADIUS server > version 0.9.3 (XXX.XXX.XXX.251 & XXX.XXX.XXX.238) > a local user account called "test" on the Linux systems, with > a valid password > a local user account called "test" with an invalid password > on both the HP-UX and Solaris systems. > > The /etc/raddb/server file on both clients systemA and > systemB contains: > XXX.XXX.XXX.251:1812 secret 5 > XXX.XXX.XXX.238:1812 secret 5 > > The problem is that pam_radius_auth module on the HP > (systemA) system fails to authenticate the user "test" on the > freeRADIUS server. The same "test" user will authenticate > fine when coming from the Solaris (systemB) system. The > messages produced by the debug on the failing client are: > ============================================================ > Dec 17 08:07:36 systemA login: pam_radius_auth: RADIUS server > XXX.XXX.XXX.251 failed to respond > Dec 17 08:07:37 systemA login: pam_radius_auth: packet from > RADIUS server XXX.XXX.XXX.238 fails verification: The shared > secret is probably incorrect. > Dec 17 08:07:37 systemA login: pam_radius_auth: All RADIUS > servers failed to respond. > Dec 17 08:07:37 systemA login: pam_radius_auth: authentication failed > Dec 17 08:07:37 systemA login: pam_authenticate: error Can > not retrieve authentication info > Dec 17 08:07:45 systemA login: pam_setcred: error Can not > retrieve authentication info > ============================================================ > > The Linux server XXX.XXX.XXX.238 is running with -X option > and produces the following messages: > ============================================================ > rad_recv: Access-Request packet from host > XXX.XXX.XXX.150:14570, id=39, length=101 > User-Name = "test" > User-Password = > "\311\260\020\\Q\245\306f}\025\224R\334?\016\275" > NAS-IP-Address = XXX.XXX.XXX.150 > NAS-Identifier = "login" > NAS-Port = 13545 > NAS-Port-Type = Virtual > Service-Type = Authenticate-Only > Calling-Station-Id = "clientA" > modcall: entering group authorize for request 0 > users: Matched DEFAULT at 5 > modcall[authorize]: module "files" returns ok for request 0 > modcall: group authorize returns ok for request 0 > Sending Access-Request of id 1 to XXX.XXX.XXX.72:1645 > User-Name = "test" > User-Password = > "\311\260\020\\Q\245\306f}\025\224R\334?\016\275" > NAS-IP-Address = XXX.XXX.XXX.150 > NAS-Identifier = "login" > NAS-Port = 13545 > NAS-Port-Type = Virtual > Service-Type = Authenticate-Only > Calling-Station-Id = "clientA" > Proxy-State = 0x3339 > --- Walking the entire request list --- > Re-sending Access-Request of id 1 to XXX.XXX.XXX.72:1645 > User-Name = "test" > User-Password = "\263\316+\025\312p\t\000\234\273l,\336)L~" > NAS-IP-Address = XXX.XXX.XXX.150 > NAS-Identifier = "login" > NAS-Port = 13545 > NAS-Port-Type = Virtual > Service-Type = Authenticate-Only > Calling-Station-Id = "clientA" > Realm = "realm1" > Proxy-State = 0x3339 > Waking up in 1 seconds... > --- Walking the entire request list --- > Server rejecting request 0. > Waking up in 1 seconds... > --- Walking the entire request list --- > Sending Access-Reject of id 39 to XXX.XXX.XXX.150:14570 > Cleaning up request 0 ID 39 with timestamp 3fe08909 > Nothing to do. Sleeping until we see a request. > ============================================================ > > > Now, I have triple checked the correctness of the shared > secret and I have also manually recreated this file twice, > typing the entries by hand, but I still continue to get the > "shared secret is incorrect" message. It appears that the > problem is related to MD5 hashing of the shared secret and > the user's password. Can anybody shed some light on this? > > > Regards, > Jim Lynch > [EMAIL PROTECTED] > - List info/subscribe/unsubscribe? See http://www.freeradius.org/list/users.html