Ran into 3 problems, all of which are addressed here:
1) dladdr() does not exist in AIX4 or AIX5.1, but the
DSO_pathbyaddr() function from crypto/dso/dso_dlfcn.c
is not called from anywhere so it's dead code...
Just #if'd it out.
2) The inline assembler in fips-1.0/fips_canister.c for
PPC doesn't work with GCC and AIX's assembler properly
(it creates an undefined symbol reference to '.', yeah
that's right a period is the symbol name). Since the
fips_canister.c cannot be modified, I had to define
INSTRUCTION_POINTER_IMPLEMENTED(ret), and I put that in
e_os.h. The actual implementation used is the same exact
implementation which the codepath actually uses on AIX
if you were to use IBMs XLC compiler.
3) There is no 64bit target in Configure for aix and gcc.
Attached is the patch. 'make test' succeeds in both 32bit
and 64bit mode using gcc 3.4.4 on AIX 5.1 ...
-Brad
diff -ruN openssl-fips-1.0.orig/Configure openssl-fips-1.0/Configure
--- openssl-fips-1.0.orig/Configure 2006-01-21 09:01:00.000000000 -0500
+++ openssl-fips-1.0/Configure 2006-04-11 16:48:15.000000000 -0400
@@ -465,6 +465,7 @@
# IBM's AIX.
"aix3-cc", "cc:-O -DB_ENDIAN -qmaxmem=16384::(unknown):AIX::BN_LLONG
RC4_CHAR:::",
"aix-gcc", "gcc:-O3 -DB_ENDIAN::-D_THREAD_SAFE:AIX::BN_LLONG
RC4_CHAR:asm/aix_ppc32.o:::::::::dlfcn:",
+"aix64-gcc", "gcc:-maix64 -O
-DB_ENDIAN::-D_THREAD_SAFE:AIX::SIXTY_FOUR_BIT_LONG
RC4_CHAR:asm/aix_ppc64.o:::::::::dlfcn::::::-X 64",
"aix-cc", "cc:-q32 -O -DB_ENDIAN -qmaxmem=16384::-qthreaded:AIX::BN_LLONG
RC4_CHAR:asm/aix_ppc32.o:::::::::dlfcn:aix-shared::-q32:.so.\$(SHLIB_MAJOR).\$(SHLIB_MINOR)::-X
32",
"aix64-cc", "cc:-q64 -O -DB_ENDIAN
-qmaxmem=16384::-qthreaded:AIX::SIXTY_FOUR_BIT_LONG
RC4_CHAR:asm/aix_ppc64.o:::::::::dlfcn:aix-shared::-q64:.so.\$(SHLIB_MAJOR).\$(SHLIB_MINOR)::-X
64",
diff -ruN openssl-fips-1.0.orig/crypto/dso/dso_dlfcn.c
openssl-fips-1.0/crypto/dso/dso_dlfcn.c
--- openssl-fips-1.0.orig/crypto/dso/dso_dlfcn.c 2005-06-09
17:37:30.000000000 -0400
+++ openssl-fips-1.0/crypto/dso/dso_dlfcn.c 2006-04-11 16:43:47.000000000
-0400
@@ -295,6 +295,7 @@
}
#ifdef OPENSSL_FIPS
+# if DEADBEEF_0
static void dlfcn_ref_point(){}
int DSO_pathbyaddr(void *addr,char *path,int sz)
@@ -321,5 +322,6 @@
ERR_add_error_data(4, "dlfcn_pathbyaddr(): ", dlerror());
return -1;
}
+# endif /* DEADBEEF_0 */
#endif
#endif /* DSO_DLFCN */
diff -ruN openssl-fips-1.0.orig/e_os.h openssl-fips-1.0/e_os.h
--- openssl-fips-1.0.orig/e_os.h 2005-06-19 16:20:29.000000000 -0400
+++ openssl-fips-1.0/e_os.h 2006-04-11 16:43:04.000000000 -0400
@@ -469,6 +469,11 @@
# endif
#endif
+/* Work-around for bug in fips_canister.c using GCC */
+#if defined(_AIX) && defined(__GNUC__) && __GNUC__>=2
+# define INSTRUCTION_POINTER_IMPLEMENTED(ret) struct { void *ip,*gp,*env; }
*p = (void *)instruction_pointer; ret=p->ip
+#endif
+
#if defined(sun) && !defined(__svr4__) && !defined(__SVR4)
/* include headers first, so our defines don't break it */
#include <stdlib.h>
