[Ada] Fix __gnat_backtrace for VxWorks7 on x86

[Pierre-Marie de Rodat]

A STORAGE ERROR is raised in __gnat_backtrace:

adainit: 0x00400DBC

Execution of ce.vxe terminated by unhandled exception
raised STORAGE_ERROR : SIGSEGV: possible stack overflow
Call stack traceback locations:
0x4082f1 0x408323 0x4080c9

It was passing with vxsim because the WRS_RTP_BASE is set to a different
place hence the (CURRENT) < (TOP_STACK) was stopping the backtrace at the
right time. So let's stop at the main symbol when RTS=rtp.

Tested on x86_64-pc-linux-gnu, committed on trunk

2018-05-31  Frederic Konrad  <kon...@adacore.com>

gcc/ada/

        * tracebak.c (STOP_FRAME): Harden condition.
        (is_return_from, EXTRA_STOP_CONDITION): New helpers for VxWorks in RTP
        mode.
--- gcc/ada/tracebak.c
+++ gcc/ada/tracebak.c
@@ -478,10 +478,11 @@ struct layout
 #define PC_ADJUST -2
 #define STOP_FRAME(CURRENT, TOP_STACK) \
   (IS_BAD_PTR((long)(CURRENT)) \
+   || (void *) (CURRENT) < (TOP_STACK) \
    || IS_BAD_PTR((long)(CURRENT)->return_address) \
    || (CURRENT)->return_address == 0 \
    || (void *) ((CURRENT)->next) < (TOP_STACK)  \
-   || (void *) (CURRENT) < (TOP_STACK))
+   || EXTRA_STOP_CONDITION(CURRENT))
 
 #define BASE_SKIP (1+FRAME_LEVEL)
 
@@ -504,6 +505,37 @@ struct layout
         || ((*((ptr) - 1) & 0xff) == 0xff) \
         || (((*(ptr) & 0xd0ff) == 0xd0ff))))
 
+#if defined (__vxworks) && defined (__RTP__)
+
+/* For VxWorks following backchains past the "main" frame gets us into the
+   kernel space, where it can't be dereferenced. So lets stop at the main
+   symbol.  */
+extern void main();
+
+static int
+is_return_from(void *symbol_addr, void *ret_addr)
+{
+  int ret = 0;
+  char *ptr = (char *)ret_addr;
+
+  if ((*(ptr - 5) & 0xff) == 0xe8)
+    {
+      /* call addr16  E8 xx xx xx xx  */
+      int32_t offset = *(int32_t *)(ptr - 4);
+      ret = (ptr + offset) == symbol_addr;
+    }
+
+  /* Others not implemented yet...  But it is very likely that call addr16
+     is used here.  */
+  return ret;
+}
+
+#define EXTRA_STOP_CONDITION(CURRENT) \
+  (is_return_from(&main, (CURRENT)->return_address))
+#else /* not (defined (__vxworks) && defined (__RTP__)) */
+#define EXTRA_STOP_CONDITION(CURRENT) (0)
+#endif /* not (defined (__vxworks) && defined (__RTP__)) */
+
 /*----------------------------- qnx ----------------------------------*/
 
 #elif defined (__QNX__)