Hi,
There seems to be a few threads about zpool hang, do we have a
workaround to resolve the hang issue without rebooting ?
In my case, I have a pool with disks from external LUNs via a fiber
cable. When the cable is unplugged while there is IO in the pool,
All zpool related command hang (zpool status, zpool list, etc.), put the
cable back does not solve the problem.
Eventually, I cannot even open a new SSH session to the box, somehow
the system goes into half-locked state.
Any clue about resolving this problem would be appreciated.
Thanks
The followings are trace for zpoo list
#truss zpool list
execve("/sbin/zpool", 0x08047CB4, 0x08047CC0) argc = 2
sysinfo(SI_MACHINE, "i86pc", 257) = 6
mmap(0x00000000, 32, PROT_READ|PROT_WRITE|PROT_EXEC,
MAP_PRIVATE|MAP_ANON, -1, 0) = 0xFEFB0000
mmap(0x00000000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANON, -1,
0) = 0xFEFA0000
mmap(0x00000000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANON, -1,
0) = 0xFEF90000
mmap(0x00000000, 4096, PROT_READ|PROT_WRITE|PROT_EXEC,
MAP_PRIVATE|MAP_ANON, -1, 0) = 0xFEF80000
memcntl(0xFEFB7000, 32064, MC_ADVISE, MADV_WILLNEED, 0, 0) = 0
memcntl(0x08050000, 15416, MC_ADVISE, MADV_WILLNEED, 0, 0) = 0
resolvepath("/usr/lib/ld.so.1", "/lib/ld.so.1", 1023) = 12
resolvepath("/sbin/zpool", "/sbin/zpool", 1023) = 11
sysconfig(_CONFIG_PAGESIZE) = 4096
stat64("/sbin/zpool", 0x080478F8) = 0
open("/var/ld/ld.config", O_RDONLY) Err#2 ENOENT
stat64("/lib/libumem.so.1", 0x080470A8) = 0
resolvepath("/lib/libumem.so.1", "/lib/libumem.so.1", 1023) = 17
open("/lib/libumem.so.1", O_RDONLY) = 3
mmapobj(3, MMOBJ_INTERPRET, 0xFEF80AE0, 0x08047114, 0x00000000) = 0
close(3) = 0
memcntl(0xFE530000, 29304, MC_ADVISE, MADV_WILLNEED, 0, 0) = 0
mmap(0x00000000, 4096, PROT_READ|PROT_WRITE|PROT_EXEC,
MAP_PRIVATE|MAP_ANON, -1, 0) = 0xFEF70000
stat64("/lib/libc.so.1", 0x080470A8) = 0
resolvepath("/lib/libc.so.1", "/lib/libc.so.1", 1023) = 14
open("/lib/libc.so.1", O_RDONLY) = 3
mmapobj(3, MMOBJ_INTERPRET, 0xFEF70108, 0x08047114, 0x00000000) = 0
close(3) = 0
memcntl(0xFEE20000, 187236, MC_ADVISE, MADV_WILLNEED, 0, 0) = 0
stat64("/lib/libzfs.so.1", 0x08046DB8) = 0
resolvepath("/lib/libzfs.so.1", "/lib/libzfs.so.1", 1023) = 16
open("/lib/libzfs.so.1", O_RDONLY) = 3
mmapobj(3, MMOBJ_INTERPRET, 0xFEF70958, 0x08046E24, 0x00000000) = 0
close(3) = 0
memcntl(0xFED70000, 35024, MC_ADVISE, MADV_WILLNEED, 0, 0) = 0
mmap(0x00000000, 4096, PROT_READ|PROT_WRITE|PROT_EXEC,
MAP_PRIVATE|MAP_ANON, -1, 0) = 0xFED60000
mmap(0x00010000, 24576, PROT_READ|PROT_WRITE|PROT_EXEC,
MAP_PRIVATE|MAP_ANON|MAP_ALIGN, -1, 0) = 0xFED50000
getcontext(0x08047758)
getrlimit(RLIMIT_STACK, 0x08047750) = 0
getpid() = 1103 [1102]
lwp_private(0, 1, 0xFED52A40) = 0x000001C3
setustack(0xFED52AA0)
sysconfig(_CONFIG_PAGESIZE) = 4096
stat64("/usr/lib/fm//libtopo.so", 0x08046B24) = 0
resolvepath("/usr/lib/fm//libtopo.so", "/usr/lib/fm/libtopo.so.1", 1023)
= 24
open("/usr/lib/fm//libtopo.so", O_RDONLY) = 3
mmapobj(3, MMOBJ_INTERPRET, 0xFED607C0, 0x08046B90, 0x00000000) = 0
close(3) = 0
memcntl(0xFED00000, 40688, MC_ADVISE, MADV_WILLNEED, 0, 0) = 0
mmap(0x00000000, 4096, PROT_READ|PROT_WRITE|PROT_EXEC,
MAP_PRIVATE|MAP_ANON, -1, 0) = 0xFECF0000
stat64("/lib/libxml2.so.2", 0x08046714) = 0
resolvepath("/lib/libxml2.so.2", "/lib/libxml2.so.2", 1023) = 17
open("/lib/libxml2.so.2", O_RDONLY) = 3
mmapobj(3, MMOBJ_INTERPRET, 0xFECF01C8, 0x08046780, 0x00000000) = 0
close(3) = 0
memcntl(0xFE2A0000, 181964, MC_ADVISE, MADV_WILLNEED, 0, 0) = 0
stat64("/lib/libpthread.so.1", 0x08046654) = 0
resolvepath("/lib/libpthread.so.1", "/lib/libpthread.so.1", 1023) = 20
open("/lib/libpthread.so.1", O_RDONLY) = 3
mmapobj(3, MMOBJ_INTERPRET, 0xFECF0818, 0x080466C0, 0x00000000) = 0
close(3) = 0
stat64("/lib/libz.so.1", 0x08046654) = 0
resolvepath("/lib/libz.so.1", "/lib/libz.so.1", 1023) = 14
open("/lib/libz.so.1", O_RDONLY) = 3
mmapobj(3, MMOBJ_INTERPRET, 0xFECF0DB8, 0x080466C0, 0x00000000) = 0
close(3) = 0
mmap(0x00000000, 4096, PROT_READ|PROT_WRITE|PROT_EXEC,
MAP_PRIVATE|MAP_ANON, -1, 0) = 0xFECB0000
memcntl(0xFECC0000, 6984, MC_ADVISE, MADV_WILLNEED, 0, 0) = 0
stat64("/lib/libm.so.2", 0x08046654) = 0
resolvepath("/lib/libm.so.2", "/lib/libm.so.2", 1023) = 14
open("/lib/libm.so.2", O_RDONLY) = 3
mmapobj(3, MMOBJ_INTERPRET, 0xFECB0508, 0x080466C0, 0x00000000) = 0
close(3) = 0
memcntl(0xFEB90000, 39464, MC_ADVISE, MADV_WILLNEED, 0, 0) = 0
stat64("/lib/libsocket.so.1", 0x08046654) = 0
resolvepath("/lib/libsocket.so.1", "/lib/libsocket.so.1", 1023) = 19
open("/lib/libsocket.so.1", O_RDONLY) = 3
mmapobj(3, MMOBJ_INTERPRET, 0xFECB0B10, 0x080466C0, 0x00000000) = 0
close(3) = 0
memcntl(0xFEC90000, 16524, MC_ADVISE, MADV_WILLNEED, 0, 0) = 0
stat64("/lib/libnsl.so.1", 0x08046654) = 0
resolvepath("/lib/libnsl.so.1", "/lib/libnsl.so.1", 1023) = 16
open("/lib/libnsl.so.1", O_RDONLY) = 3
mmap(0x00000000, 4096, PROT_READ|PROT_WRITE|PROT_EXEC,
MAP_PRIVATE|MAP_ANON, -1, 0) = 0xFEC80000
mmapobj(3, MMOBJ_INTERPRET, 0xFEC80018, 0x080466C0, 0x00000000) = 0
close(3) = 0
memcntl(0xFE570000, 78408, MC_ADVISE, MADV_WILLNEED, 0, 0) = 0
sigfillset(0xFEF683A8) = 0
stat64("/usr/lib//libshare.so.1", 0x08046B24) = 0
resolvepath("/usr/lib//libshare.so.1", "/usr/lib/libshare.so.1", 1023) =
22
open("/usr/lib//libshare.so.1", O_RDONLY) = 3
mmapobj(3, MMOBJ_INTERPRET, 0xFEC80928, 0x08046B90, 0x00000000) = 0
close(3) = 0
memcntl(0xFEC50000, 24216, MC_ADVISE, MADV_WILLNEED, 0, 0) = 0
mmap(0x00000000, 4096, PROT_READ|PROT_WRITE|PROT_EXEC,
MAP_PRIVATE|MAP_ANON, -1, 0) = 0xFEC40000
sysi86(SI86FPSTART, 0xFEF68CD4, 0x0000133F, 0x00001F80) = 0x00000001
open("/usr/lib/locale/en_US.UTF-8/LC_MESSAGES/SUNW_OST_SGS.mo",
O_RDONLY) Err#2 ENOENT
sysconfig(_CONFIG_NPROC_ONLN) = 4
issetugid() = 0
open("/usr/lib/locale/en_US.UTF-8/LC_MESSAGES/SUNW_OST_OSLIB.mo",
O_RDONLY) Err#2 ENOENT
issetugid() = 0
brk(0x08089000) = 0
brk(0x08099000) = 0
stat64("/usr/lib/locale/en_US.UTF-8/en_US.UTF-8.so.3", 0x08042BA0) = 0
resolvepath("/usr/lib/locale/en_US.UTF-8/en_US.UTF-8.so.3",
"/usr/lib/locale/en_US.UTF-8/en_US.UTF-8.so.3", 1023) = 44
open("/usr/lib/locale/en_US.UTF-8/en_US.UTF-8.so.3", O_RDONLY) = 3
mmapobj(3, MMOBJ_INTERPRET, 0xFEC40560, 0x08042C0C, 0x00000000) = 0
close(3) = 0
memcntl(0xFE640000, 6780, MC_ADVISE, MADV_WILLNEED, 0, 0) = 0
stat64("/usr/lib/locale/en_US.UTF-8/libc.so.1", 0x08042A80) Err#2 ENOENT
stat64("/usr/lib/locale/en_US.UTF-8/methods_unicode.so.3", 0x08042A80) =
0
resolvepath("/usr/lib/locale/en_US.UTF-8/methods_unicode.so.3",
"/usr/lib/locale/common/methods_unicode.so.3", 1023) = 43
open("/usr/lib/locale/en_US.UTF-8/methods_unicode.so.3", O_RDONLY) = 3
mmapobj(3, MMOBJ_INTERPRET, 0xFEC40D30, 0x08042AEC, 0x00000000) = 0
close(3) = 0
mmap(0x00000000, 4096, PROT_READ|PROT_WRITE|PROT_EXEC,
MAP_PRIVATE|MAP_ANON, -1, 0) = 0xFEC30000
memcntl(0xFE620000, 3576, MC_ADVISE, MADV_WILLNEED, 0, 0) = 0
open("/dev/zfs", O_RDWR) = 3
open("/etc/mnttab", O_RDONLY) = 4
open("/etc/dfs/sharetab", O_RDONLY) = 5
stat64("/lib/libavl.so.1", 0x080431A8) = 0
resolvepath("/lib/libavl.so.1", "/lib/libavl.so.1", 1023) = 16
open("/lib/libavl.so.1", O_RDONLY) = 6
mmapobj(6, MMOBJ_INTERPRET, 0xFEC305E0, 0x08043214, 0x00000000) = 0
close(6) = 0
memcntl(0xFEC10000, 2416, MC_ADVISE, MADV_WILLNEED, 0, 0) = 0
sysconfig(_CONFIG_PAGESIZE) = 4096
stat64("/lib/libuutil.so.1", 0x080430F8) = 0
resolvepath("/lib/libuutil.so.1", "/lib/libuutil.so.1", 1023) = 18
open("/lib/libuutil.so.1", O_RDONLY) = 6
mmapobj(6, MMOBJ_INTERPRET, 0xFEC30B68, 0x08043164, 0x00000000) = 0
close(6) = 0
mmap(0x00000000, 4096, PROT_READ|PROT_WRITE|PROT_EXEC,
MAP_PRIVATE|MAP_ANON, -1, 0) = 0xFE610000
memcntl(0xFE510000, 9080, MC_ADVISE, MADV_WILLNEED, 0, 0) = 0
brk(0x080A9000) = 0
ioctl(3, ZFS_IOC_POOL_CONFIGS, 0x08042590) = 0
stat64("/lib/libnvpair.so.1", 0x08041AB8) = 0
resolvepath("/lib/libnvpair.so.1", "/lib/libnvpair.so.1", 1023) = 19
open("/lib/libnvpair.so.1", O_RDONLY) = 6
mmapobj(6, MMOBJ_INTERPRET, 0xFE6105D0, 0x08041B24, 0x00000000) = 0
close(6) = 0
memcntl(0xFE4F0000, 16180, MC_ADVISE, MADV_WILLNEED, 0, 0) = 0
brk(0x080BA000) = 0
< ==========stuck here ===================>
Tianhong Zhao | Senior Systems Architect
p. +1.514.683.1020 x145
tianhong.z...@nexsan.ca
www.nexsan.com <http://www.nexsan.com/>
This message is confidential and may also be legally privileged. It is
intended for the use of the individual to whom it is addressed.
If you are not the intended recipient, please notify the sender via
email and delete the message and you should not copy it or use
it for any purpose, nor disclose its contents to any other person. Thank
you.
<<image001.gif>>
_______________________________________________ zfs-discuss mailing list zfs-discuss@opensolaris.org http://mail.opensolaris.org/mailman/listinfo/zfs-discuss
