Any help with this will be greatly appreciated...my last backup was
four months ago so a lot of my thesis work will be lost. I mistakenly
thought a mirrored zpool on new drives would be good enough for a
while.
So here's what happened: we had a power outage one day and as soon as
I tried to boot the server again it enters an endless reboot cycle.
So I thought the OS drive became corrupted (not mirrored) and
reinstalled the OS. Then when I try zpool import it hangs forever. I
even left it going for a couple days in case it was trying to correct
corrupted data. The same thing happens no matter what version of
solaris I use. The symptoms and diagnostic results (see below) seem
to be very similar to the post above but the solution doesn't work.
Please let me know if you need any other information.
Thanks,
Brad
bash-3.2# zpool import
pool: tank
id: 4410438565134310480
state: ONLINE
status: The pool is formatted using an older on-disk version.
action: The pool can be imported using its name or numeric identifier, though
some features will not be available without an explicit 'zpool upgrade'.
config:
tank ONLINE
mirror ONLINE
c2d0p0 ONLINE
c1d0p0 ONLINE
bash-3.2# zpool import tank
cannot import 'tank': pool may be in use from other system
use '-f' to import anyway
bash-3.2# zpool import -f tank
(then it hangs here forever, can't be killed)
(the following commands were performed while this was running)
bash-3.2# fmdump -eV
TIME CLASS
May 27 2009 22:22:55.308533986 ereport.fs.zfs.checksum
nvlist version: 0
class = ereport.fs.zfs.checksum
ena = 0xd22e37db9000401
detector = (embedded nvlist)
nvlist version: 0
version = 0x0
scheme = zfs
pool = 0x3d350681ea839c50
vdev = 0x85cc302105002c5d
(end detector)
pool = tank
pool_guid = 0x3d350681ea839c50
pool_context = 0
pool_failmode = wait
vdev_guid = 0x85cc302105002c5d
vdev_type = disk
vdev_path = /dev/dsk/c2d0p0
vdev_devid = id1,c...@ast3750640as=____________5qd3myrh/q
parent_guid = 0x8fb729a008f16e65
parent_type = mirror
zio_err = 50
zio_offset = 0x2500407000
zio_size = 0x1000
zio_objset = 0x0
zio_object = 0x23
zio_level = 0
zio_blkid = 0x0
__ttl = 0x1
__tod = 0x4a1e038f 0x1263dae2
(and many others just like this)
bash-3.2# echo "0t3735::pid2proc|::walk thread|::findtsack -v" | mdb -k
stack pointer for thread df6ed220: e1ca8c54
e1ca8c94 swtch+0x188()
e1ca8ca4 cv_wait+0x53(e08442aa, e084426c, ffffffff, f9c305ac)
e1ca8ce4 txg_wait_synced+0x90(e0844100, 252b4c, 0, 0)
e1ca8d34 spa_config_update_common+0x88(d6c429c0, 0, 0, e1ca8d68)
e1ca8d84 spa_import_common+0x3cf()
e1ca8db4 spa_import+0x18(dbfcf000, e3c0d018, 0, f9c65810)
e1ca8de4 zfs_ioc_pool_import+0xcd(dbfcf000, 0, 0)
e1ca8e14 zfsdev_ioctl+0x124()
e1ca8e44 cdev_ioctl+0x31(2d80000, 5a02, 80418d0, 100003, dfde91f8, e1ca8f00)
e1ca8e74 spec_ioctl+0x6b(d7a593c0, 5a02, 80418d0, 100003, dfde91f8, e1ca8f00)
e1ca8ec4 fop_ioctl+0x49(d7a593c0, 5a02, 80418d0, 100003, dfde91f8, e1ca8f00)
e1ca8f84 ioctl+0x171()
e1ca8fac sys_sysenter+0x106()
bash-3.2# echo "::threadlist -v" | mdb -k
d4ed8dc0 fec1f558 0 0 60 d5033604
PC: _resume_from_idle+0xb1 THREAD: txg_sync_thread()
stack pointer for thread d4ed8dc0: d4ed8ba8
swtch+0x188()
cv_wait+0x53()
zio_wait+0x55()
vdev_uberblock_sync_list+0x19e()
vdev_config_sync+0x11c()
spa_sync+0x5a5()
txg_sync_thread+0x308()
thread_start+8()
(just chose one seemingly relevant thread from the long list)
bash-3.2# zdb -e -bb tank
Traversing all blocks to verify nothing leaked ...
Assertion failed: space_map_load(&msp->ms_map, &zdb_space_map_ops, 0x0, &msp->ms_smo, spa->spa_meta_objset) == 0, file ../zdb.c, line 1420, function
zdb_leak_init
Abort (core dumped)
bash-3.2# zdb -e -dddd tank
Dataset mos [META], ID 0, cr_txg 4, 10.3M, 137 objects, rootbp [L0 DMU objset] 400L/400P
DVA[0]=<0:2500005c00:400> DVA[1]=<0:4400005c00:400> DVA[2]=<0:6300003000:400>
fletcher4 uncompressed LE contiguous birth=2435914 fill=137
cksum=5224494b4:4524146f316:1d44c6f4690ea:84ef3bd0c105a0
Object lvl iblk dblk lsize asize type
0 2 16K 16K 80.0K 180K DMU dnode
Object lvl iblk dblk lsize asize type
1 1 16K 512 512 1K object directory
microzap: 512 bytes, 5 entries
history = 13
root_dataset = 2
sync_bplist = 12
config = 11
deflate = 1
Object lvl iblk dblk lsize asize type
2 1 16K 512 512 0 DSL directory
256 bonus DSL directory
creation_time = Wed Nov 26 02:07:49 2008
head_dataset_obj = 5
parent_dir_obj = 0
origin_obj = 0
child_dir_zapobj = 4
used_bytes = 402G
compressed_bytes = 401G
uncompressed_bytes = 401G
quota = 0
reserved = 0
props_zapobj = 3
deleg_zapobj = 0
flags = 0
used_breakdown[HEAD] = 0
used_breakdown[SNAP] = 0
used_breakdown[CHILD] = 0
used_breakdown[CHILD_RSRV] = 0
used_breakdown[REFRSRV] = 0
Object lvl iblk dblk lsize asize type
3 1 16K 512 512 1K DSL props
microzap: 512 bytes, 0 entries
Object lvl iblk dblk lsize asize type
4 1 16K 512 512 1K DSL directory child map
microzap: 512 bytes, 1 entries
$MOS = 8
Object lvl iblk dblk lsize asize type
5 1 16K 512 512 0 DSL dataset
320 bonus DSL dataset
dir_obj = 2
prev_snap_obj = 99
prev_snap_txg = 2088469
next_snap_obj = 0
snapnames_zapobj = 6
num_children = 0
creation_time = Wed Nov 26 02:07:49 2008
creation_txg = 4
deadlist_obj = 100
used_bytes = 392G
compressed_bytes = 392G
uncompressed_bytes = 392G
unique = 11.3G
fsid_guid = 43614162550022126
guid = 431952237769608890
flags = 0
next_clones_obj = 0
props_obj = 0
bp = [L0 DMU objset] 400L/200P DVA[0]=<0:2500756400:200>
DVA[1]=<0:44005dce00:200> DVA[2]=<0:630011fe00:200> fletcher4 lzjb LE contiguous
birth=2435911 fill=769038 cksum=b131ea18e:48f57187abe:f4e1282becf2:22e5d4ef0c414d
Object lvl iblk dblk lsize asize type
6 1 16K 512 512 1K DSL dataset snap map
microzap: 512 bytes, 2 entries
tank-2008-12-30 = 97
tank-2009-03-30 = 99
Object lvl iblk dblk lsize asize type
7 1 16K 128K 128K 0 bplist
32 bonus bplist header
Object lvl iblk dblk lsize asize type
8 1 16K 512 512 0 DSL directory
256 bonus DSL directory
creation_time = Wed Nov 26 02:07:49 2008
head_dataset_obj = 0
parent_dir_obj = 2
origin_obj = 0
child_dir_zapobj = 10
used_bytes = 10.3M
compressed_bytes = 5.01M
uncompressed_bytes = 5.01M
quota = 0
reserved = 0
props_zapobj = 9
deleg_zapobj = 0
flags = 0
used_breakdown[HEAD] = 0
used_breakdown[SNAP] = 0
used_breakdown[CHILD] = 0
used_breakdown[CHILD_RSRV] = 0
used_breakdown[REFRSRV] = 0
Object lvl iblk dblk lsize asize type
9 1 16K 512 512 1K DSL props
microzap: 512 bytes, 0 entries
Object lvl iblk dblk lsize asize type
10 1 16K 512 512 1K DSL directory child map
microzap: 512 bytes, 0 entries
Object lvl iblk dblk lsize asize type
11 1 16K 16K 16K 32K packed nvlist
8 bonus packed nvlist size
Assertion failed: 0 == dmu_read(os, object, 0, nvsize, packed), file ../zdb.c,
line 217, function dump_packed_nvlist
Abort (core dumped)