OK, it's pretty much ready for proper testing now. If a few people are
happy with the results and think it's a good idea I'll chuck it in the
commitfest app.
As built, the crash dump handler works with a stock PostgreSQL 9.0
(release build) as shipped in EDB's installer. Just drop crashdump.dll
in lib\, optionally pop the dbghelp.dll redist in bin\, add 'crashdump'
to shared_preload_libraries, and crash some backends however you feel
like doing so.
The current build of crashdump.dll for release versions of PostgreSQL
9.0 on 32-bit Windows is here:
http://www.postnewspapers.com.au/~craig/webfiles/crashdump.dll
If folks are happy with how this works, all it needs is:
- Consideration of whether elog should be used or not. I'm inclined to
suggest using elog to tell the user about the dump, but only after
the crash dump has been written successfully.
- Comments on whether this should be left as a loadable module, or
integerated into core so it loads early in backend startup. The latter
will permit crash dumping of early backend startup problems, and will
have tiny overhead because there's no DLL to find and load. OTOH, it's
harder to pull out if somehow something breaks.
I'd want to start with loadable module in shared_preload_libraries
and if that proves useful, only then consider integrating in core.
I'm way too bad a programmer to want my code anywhere near Pg's core
without plenty of real world testing.
- (Maybe) a method to configure the crash dump type. I'm not too sure
it's necessary given the set of dump flags I've landed up with,
so I'd leave this be unless it proves to be necessary in real-world
testing.
Remember that with these crash dumps the user only has to email the
(~4MB in my tests) crash dump. They don't need debugging tools or the
ability to use them, only the recipient does.
I'm using a tweaked set of minidump flags now, to generate considerably
bigger dumps (around 4MB for the configuration I'm testing) that contain
pretty much everything except shared memory contents, the contents of
memory mapped files, and the loaded read-only code segments of the
executables and DLLs. You can see the results of using it to debug that
autovac crash at the end of this mail.
When testing the script provided by Andrea Peri, when the autovacuum
worker crashes, the message:
2010-10-05 22:23:44 WST 2040 WARNING: crashdump: wrote crash dump to
crashdumps\postgres-2040.mdmp
is emitted before the process resumes crashing out as it would've normally.
Opening and displaying that dump in windbg shows a useful stack trace
from the crashing process, and it's possible to examine the state of
local/global variables etc.
0:000> .ecxr
eax=90fffffe ebx=040af140 ecx=68f08610 edx=040af248 esi=011f64d4 edi=040b001c
eip=015d5267 esp=0055f1cc ebp=011f5bc8 iopl=0 nv up ei pl zr na pe nc
cs=001b ss=0023 ds=0023 es=0023 fs=003b gs=0000 efl=00010246
postgres!pfree+0x7:
015d5267 8b5004 mov edx,dword ptr [eax+4] ds:0023:91000002=????????
0:000> ~*k
. 0 Id: 7f8.7b8 Suspend: 0 Teb: 7ffdf000 Unfrozen
ChildEBP RetAddr
0055e210 75b4c1ee ntdll!KiFastSystemCallRet
0055e250 76e7100e KERNELBASE!FindClose+0x93
0055e514 679160c3 kernel32!_SEH_epilog4_GS+0xa
0055e6cc 779734e0 dbghelp!Win32LiveSystemProvider::OpenMapping+0x2c3
0055e7a8 7797353a ntdll!RtlpAllocateHeap+0xab2
0055e828 77965edc ntdll!RtlAllocateHeap+0x23a
0055e840 76e7123a ntdll!ZwWriteFile+0xc
0055e874 67916861 kernel32!WriteFileImplementation+0x76
0055e8ac 67916910 dbghelp!Win32LiveSystemProvider::ReadVirtual+0x71
0055e8d8 67908f09 dbghelp!Win32LiveSystemProvider::ReadAllVirtual+0x30
0055e910 679094b4 dbghelp!WriteMemoryFromProcess+0xa9
0055e9a8 6790d522 dbghelp!WriteThreadList+0x184
0055e9c0 6790e65a dbghelp!WriteDumpData+0x102
0055eb58 6790e9cb dbghelp!MiniDumpProvideDump+0x3fa
0055ebd0 73231353 dbghelp!MiniDumpWriteDump+0x1cb
0055ed14 76e82c4a crashdump!crashDumpHandler+0x183
[c:\users\craig\developer\postgresql-9.0.0\contrib\crashdump\crashdump.c @ 159]
0055ed9c 77995af4 kernel32!UnhandledExceptionFilter+0x127
0055eda4 7793d964 ntdll!__RtlUserThreadStart+0x62
0055edb8 7793d7fc ntdll!_EH4_CallFilterFunc+0x12
0055ede0 779665f9 ntdll!_except_handler4+0x8e
0055ee04 779665cb ntdll!ExecuteHandler2+0x26
0055eeb4 77966457 ntdll!ExecuteHandler+0x24
0055eeb4 015d5267 ntdll!KiUserExceptionDispatcher+0xf
0055f1c8 0139eee7 postgres!pfree+0x7
[c:\pginstaller-repo\postgres.windows\src\backend\utils\mmgr\mcxt.c @ 591]
0055f1e0 0139f14a postgres!examine_attribute+0x207
[c:\pginstaller-repo\postgres.windows\src\backend\commands\analyze.c @ 877]
0055f284 0139f94c postgres!do_analyze_rel+0x24a
[c:\pginstaller-repo\postgres.windows\src\backend\commands\analyze.c @ 357]
0055f2ac 013eb74a postgres!analyze_rel+0x1bc
[c:\pginstaller-repo\postgres.windows\src\backend\commands\analyze.c @ 232]
0055f330 014b30c6 postgres!vacuum+0x1ea
[c:\pginstaller-repo\postgres.windows\src\backend\commands\vacuum.c @ 248]
0055f368 014b3991 postgres!autovacuum_do_vac_analyze+0x86
[c:\pginstaller-repo\postgres.windows\src\backend\postmaster\autovacuum.c @
2651]
0055f4f4 014b41c5 postgres!do_autovacuum+0x811
[c:\pginstaller-repo\postgres.windows\src\backend\postmaster\autovacuum.c @
2231]
0055f588 014bed02 postgres!AutoVacWorkerMain+0x265
[c:\pginstaller-repo\postgres.windows\src\backend\postmaster\autovacuum.c @
1611]
0055f7d8 01423ce8 postgres!SubPostmasterMain+0x442
[c:\pginstaller-repo\postgres.windows\src\backend\postmaster\postmaster.c @
4093]
0055f7f0 015ee1ad postgres!main+0x1f8
[c:\pginstaller-repo\postgres.windows\src\backend\main\main.c @ 165]
0055f834 76e71194 postgres!__tmainCRTStartup+0x10f
[f:\dd\vctools\crt_bld\self_x86\crt\src\crtexe.c @ 586]
0055f840 7797b495 kernel32!BaseThreadInitThunk+0xe
0055f880 7797b468 ntdll!__RtlUserThreadStart+0x70
0055f898 00000000 ntdll!_RtlUserThreadStart+0x1b
/*-------------------------------------------------------------------------
*
* crashdump.c
* Automatic crash dump creation for PostgreSQL on Windows
*
* The crashdump module traps unhandled exceptions produced by the backend
* the module is loaded in, and tries to produce a Windows MiniDump crash
* dump for later debugging and analysis. The machine performing the dump
* doesn't need any special debugging tools; the user only needs to send
* the dump to somebody who has the same version of PostgreSQL and has debugging
* tools.
*
* crashdump module originally by Craig Ringer <ring...@ringerc.id.au>
*
* LIMITATIONS:
* ============
* This *won't* work in hard OOM situations or stack overflows.
*
* For those, it'd be necessary to take a much more complicated approach where
* the handler switches to a new stack (if it can) and forks a helper process
* to debug its self. That's in the too hard basket as far as I'm concerned;
* this approach will get 90% of the results with 10% of the work.
*
* POSSIBLE FUTURE WORK:
* =====================
* For bonus points, the crash dump format permits embedding of user-supplied
data.
* If there's anything else (postgresql.conf? Last few lines of a log file?)
that
* should always be supplied with a crash dump, it could potentially be added,
* though at the cost of a greater chance of the crash dump failing. Again,
* I'm not going to tackle that, but thought it worth mentioning in case
* someone wants it down the track.
*
*-------------------------------------------------------------------------
*/
#include "postgres.h"
#include "fmgr.h"
#define VC_EXTRALEAN
#include <windows.h>
#include <string.h>
#include <dbghelp.h>
PG_MODULE_MAGIC;
#if !(defined(_WIN32) || defined(_WIN64))
/* Don't add lots of ifdefs here - build different files for different
platforms instead,
* if adding crash dump support for additional platforms. */
#error crashdump_win32.c is only supported on MS Windows
#endif
/*
* Much of the following code is based on CodeProject and MSDN examples,
* particularly
http://www.codeproject.com/KB/debug/postmortemdebug_standalone1.aspx
*
* Useful MSDN articles:
*
* http://msdn.microsoft.com/en-us/library/ff805116(v=VS.85).aspx
* http://msdn.microsoft.com/en-us/library/ms679294(VS.85).aspx
* http://msdn.microsoft.com/en-us/library/ms679291(VS.85).aspx
* http://msdn.microsoft.com/en-us/library/ms680519(v=VS.85).aspx
* http://msdn.microsoft.com/en-us/library/ms680360(VS.85).aspx
*
* Other useful articles on working with minidumps:
* http://www.debuginfo.com/articles/effminidumps.html
*/
typedef BOOL (WINAPI *MINIDUMPWRITEDUMP)(HANDLE hProcess, DWORD dwPid, HANDLE
hFile, MINIDUMP_TYPE DumpType,
CONST
PMINIDUMP_EXCEPTION_INFORMATION ExceptionParam,
CONST
PMINIDUMP_USER_STREAM_INFORMATION UserStreamParam,
CONST
PMINIDUMP_CALLBACK_INFORMATION CallbackParam
);
/*
* To perform a crash dump, we need to load dbghelp.dll and find the
* MiniDumpWriteDump function. If possible, the copy of dbghelp.dll
* shipped with PostgreSQL is loaded; failing that, the system copy will
* be used.
*
* Called in crash handler context.
*
* If NULL is returned, loading failed and the crash dump handler should
* not try to continue.
*/
static MINIDUMPWRITEDUMP
loadDbgHelp()
{
// Use the dbghelp.dll shipped with Pg if we can find and load
// it, otherwise fall back to the copy installed in Windows.
HMODULE hDll = NULL;
MINIDUMPWRITEDUMP pDump = NULL;
char dbgHelpPath[_MAX_PATH];
if (GetModuleFileName( NULL, dbgHelpPath, _MAX_PATH ))
{
char *slash = strrchr( dbgHelpPath, '\\' );
if (slash)
{
strcpy( slash+1, "DBGHELP.DLL" );
hDll = LoadLibrary( dbgHelpPath );
}
}
if (hDll==NULL)
{
// Load any version we can
hDll = LoadLibrary( "DBGHELP.DLL" );
}
if (hDll!=NULL)
{
pDump = (MINIDUMPWRITEDUMP)GetProcAddress( hDll,
"MiniDumpWriteDump" );
}
return pDump;
}
/*
* This function is the exception handler passed to
SetUnhandledExceptionFilter. It's invoked
* only if there's an unhandled exception. The handler will use dbghelp.dll to
generate a crash
* dump, then resume the normal unhandled exception process, which will
generally exit with a
* an error message from the runtime.
*
* This function is run under the unhandled exception handler, effectively
* in a crash context, so it should be careful with memory and avoid using
* any PostgreSQL API or other functions that use PostgreSQL API.
*
*/
static LONG WINAPI
crashDumpHandler(struct _EXCEPTION_POINTERS *pExceptionInfo)
{
MINIDUMPWRITEDUMP pDump = NULL;
char dumpPath[_MAX_PATH];
// Dump pretty much everything except shared memory, code segments, and
memory mapped files
MINIDUMP_TYPE dumpType = MiniDumpNormal|\
MiniDumpWithIndirectlyReferencedMemory|\
MiniDumpWithHandleData|\
MiniDumpWithThreadInfo|\
MiniDumpWithPrivateReadWriteMemory|
MiniDumpWithDataSegs;
HANDLE selfProcHandle = GetCurrentProcess();
DWORD selfPid = GetProcessId(selfProcHandle);
HANDLE dumpFile;
struct _MINIDUMP_EXCEPTION_INFORMATION ExInfo;
ExInfo.ThreadId = GetCurrentThreadId();
ExInfo.ExceptionPointers = pExceptionInfo;
ExInfo.ClientPointers = FALSE;
pDump = loadDbgHelp();
if (pDump==NULL)
return EXCEPTION_CONTINUE_SEARCH;
snprintf(&dumpPath[0], _MAX_PATH, "crashdumps\\postgres-%i.mdmp",
selfPid);
dumpPath[_MAX_PATH-1] = '\0';
dumpFile = CreateFile( dumpPath, GENERIC_WRITE, FILE_SHARE_WRITE,\
NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL );
if (dumpFile==INVALID_HANDLE_VALUE)
{
elog(WARNING, "crashdump: Unable to open dump file %s for
writing (win32 error %i)",
&dumpPath[0], GetLastError());
elog(DEBUG1, "crashdump: is there a 'crashdump' directory
within the data dir, and is it writable by the postgres user?");
return EXCEPTION_CONTINUE_SEARCH;
}
if( (*pDump)( selfProcHandle, selfPid, dumpFile, dumpType, &ExInfo,
NULL, NULL ) )
{
elog(WARNING,"crashdump: wrote crash dump to %s", &dumpPath[0]);
}
else
{
elog(WARNING,"crashdump: failed to write dump file to %s (win32
error %i)",
&dumpPath[0], GetLastError());
}
CloseHandle(dumpFile);
return EXCEPTION_CONTINUE_SEARCH;
}
extern Datum crashdump_crashme(PG_FUNCTION_ARGS);
PG_FUNCTION_INFO_V1(crashdump_crashme);
Datum
crashdump_crashme(PG_FUNCTION_ARGS)
{
int * ptr = NULL;
*ptr = 1;
return NULL;
}
void
_PG_init()
{
// http://msdn.microsoft.com/en-us/library/ms680634(VS.85).aspx
SetUnhandledExceptionFilter( crashDumpHandler );
}
MODULE_big = crashdump
OBJS = crashdump.o
ifdef USE_PGXS
PG_CONFIG = pg_config
PGXS := $(shell $(PG_CONFIG) --pgxs)
include $(PGXS)
else
subdir = contrib/crashdump
top_builddir = ../..
include $(top_builddir)/src/Makefile.global
include $(top_srcdir)/contrib/contrib-global.mk
endif
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
