I tested pvfs2 on Debian lenny kernel 2.6.26-1-686 #1 SMP Sat Jan 10
18:29:31 UTC 2009 i686 GNU/Linux. When I ran pvfs2-fsck, these error
messages came up:
pvfs2-fsck -m /mnt/pvfs2/
# Current FSID is 1644169005.
[E 15:11:42.724517] Warning: msgpair failed to tcp://oyster.jatheon.com:3334,
will retry: Broken pipe
[E 15:11:44.730287] Warning: msgpair failed to tcp://oyster.jatheon.com:3334,
will retry: Connection refused
[E 15:11:46.736016] Warning: msgpair failed to tcp://oyster.jatheon.com:3334,
will retry: Connection refused
[E 15:11:48.739135] Warning: msgpair failed to tcp://oyster.jatheon.com:3334,
will retry: Connection refused
[E 15:11:50.744279] Warning: msgpair failed to tcp://oyster.jatheon.com:3334,
will retry: Connection refused
[E 15:11:52.750153] Warning: msgpair failed to tcp://oyster.jatheon.com:3334,
will retry: Connection refused
[E 15:11:52.750207] *** msgpairarray_completion_fn: msgpair to server tcp://
oyster.jatheon.com:3334 failed: Connection refused
[E 15:11:52.750228] *** Out of retries.
PVFS_mgmt_iterate_handles_list: Connection refused (error class: 128)
[E 15:11:52.751002] Warning: msgpair failed to tcp://oyster.jatheon.com:3334,
will retry: Connection refused
[E 15:11:54.756904] Warning: msgpair failed to tcp://oyster.jatheon.com:3334,
will retry: Connection refused
[E 15:11:56.763124] Warning: msgpair failed to tcp://oyster.jatheon.com:3334,
will retry: Connection refused
[E 15:11:58.769307] Warning: msgpair failed to tcp://oyster.jatheon.com:3334,
will retry: Connection refused
[E 15:12:00.775364] Warning: msgpair failed to tcp://oyster.jatheon.com:3334,
will retry: Connection refused
[E 15:12:02.782929] Warning: msgpair failed to tcp://oyster.jatheon.com:3334,
will retry: Connection refused
[E 15:12:02.782984] *** msgpairarray_completion_fn: msgpair to server tcp://
oyster.jatheon.com:3334 failed: Connection refused
[E 15:12:02.783008] *** Out of retries.
[E 15:12:02.783745] Warning: msgpair failed to tcp://oyster.jatheon.com:3334,
will retry: Connection refused
[E 15:12:04.790847] Warning: msgpair failed to tcp://oyster.jatheon.com:3334,
will retry: Connection refused
[E 15:12:06.797528] Warning: msgpair failed to tcp://oyster.jatheon.com:3334,
will retry: Connection refused
[E 15:12:08.804913] Warning: msgpair failed to tcp://oyster.jatheon.com:3334,
will retry: Connection refused
[E 15:12:10.807222] Warning: msgpair failed to tcp://oyster.jatheon.com:3334,
will retry: Connection refused
[E 15:12:12.816268] Warning: msgpair failed to tcp://oyster.jatheon.com:3334,
will retry: Connection refused
[E 15:12:12.816325] *** msgpairarray_completion_fn: msgpair to server tcp://
oyster.jatheon.com:3334 failed: Connection refused
[E 15:12:12.816348] *** Out of retries.
So I strace pvfs2-server and got this output when it crashed:
clock_gettime(CLOCK_REALTIME, {1236026198, 349308706}) = 0
futex(0x80ea6a4, FUTEX_WAIT_PRIVATE, 235, {0, 99942294}) = -1 ETIMEDOUT
(Connection timed out)
futex(0x80ea638, FUTEX_WAKE_PRIVATE, 1) = 0
gettimeofday({1236026198, 449537}, NULL) = 0
gettimeofday({1236026198, 449612}, NULL) = 0
clock_gettime(CLOCK_REALTIME, {1236026198, 449677827}) = 0
futex(0x80ea6a4, FUTEX_WAIT_PRIVATE, 237, {0, 99934173}) = -1 ETIMEDOUT
(Connection timed out)
futex(0x80ea638, FUTEX_WAKE_PRIVATE, 1) = 0
gettimeofday({1236026198, 549881}, NULL) = 0
gettimeofday({1236026198, 549942}, NULL) = 0
clock_gettime(CLOCK_REALTIME, {1236026198, 549997189}) = 0
futex(0x80ea6a4, FUTEX_WAIT_PRIVATE, 239, {0, 99944811}) = -1 ETIMEDOUT
(Connection timed out)
futex(0x80ea638, FUTEX_WAKE_PRIVATE, 1) = 0
gettimeofday({1236026198, 650129}, NULL) = 0
gettimeofday({1236026198, 650194}, NULL) = 0
gettimeofday({1236026198, 650252}, NULL) = 0
gettimeofday({1236026198, 650311}, NULL) = 0
gettimeofday({1236026198, 650369}, NULL) = 0
gettimeofday({1236026198, 650423}, NULL) = 0
gettimeofday({1236026198, 650474}, NULL) = 0
gettimeofday({1236026198, 650524}, NULL) = 0
clock_gettime(CLOCK_REALTIME, {1236026198, 650577709}) = 0
futex(0x80ea6a4, FUTEX_WAIT_PRIVATE, 241, {0, 99946291}) = -1 ETIMEDOUT
(Connection timed out)
futex(0x80ea638, FUTEX_WAKE_PRIVATE, 1) = 0
gettimeofday({1236026198, 750722}, NULL) = 0
gettimeofday({1236026198, 750774}, NULL) = 0
clock_gettime(CLOCK_REALTIME, {1236026198, 750831391}) = 0
futex(0x80ea6a4, FUTEX_WAIT_PRIVATE, 243, {0, 99942609}) = -1 ETIMEDOUT
(Connection timed out)
futex(0x80ea638, FUTEX_WAKE_PRIVATE, 1) = 0
gettimeofday({1236026198, 850951}, NULL) = 0
gettimeofday({1236026198, 851003}, NULL) = 0
clock_gettime(CLOCK_REALTIME, {1236026198, 851059553}) = 0
futex(0x80ea6a4, FUTEX_WAIT_PRIVATE, 245, {0, 99943447}) = -1 ETIMEDOUT
(Connection timed out)
futex(0x80ea638, FUTEX_WAKE_PRIVATE, 1) = 0
gettimeofday({1236026198, 951183}, NULL) = 0
gettimeofday({1236026198, 951240}, NULL) = 0
clock_gettime(CLOCK_REALTIME, {1236026198, 951292556}) = 0
futex(0x80ea6a4, FUTEX_WAIT_PRIVATE, 247, {0, 99947444}) = 0
futex(0x80ea638, FUTEX_WAKE_PRIVATE, 1) = 0
brk(0x8590000) = 0x8590000
writev(10, [{"\277\312\0\0\4\0\0\0\1\0\0\0\0\0\0\0\310\3\0\0\0\0\0\0"...,
24},
{"p\27\0\0\2\0\0\0\r\0\0\0\0\0\0\0\246\3\0\0\0\0\0\0\245\3\0\0<Defa"...,
968}], 2) = 992
gettimeofday({1236026199, 41736}, NULL) = 0
gettimeofday({1236026199, 41796}, NULL) = 0
clock_gettime(CLOCK_REALTIME, {1236026199, 41857591}) = 0
futex(0x80ea6a4, FUTEX_WAIT_PRIVATE, 249, {0, 99938409}) = 0
futex(0x80ea638, FUTEX_WAKE_PRIVATE, 1) = 0
futex(0x80f3464, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x80f3460, {FUTEX_OP_SET, 0,
FUTEX_OP_CMP_GT, 1}) = 1
gettimeofday({1236026199, 51424}, NULL) = 0
gettimeofday({1236026199, 51487}, NULL) = 0
clock_gettime(CLOCK_REALTIME, {1236026199, 51559118}) = 0
futex(0x80ea6a4, FUTEX_WAIT_PRIVATE, 251, {0, 99927882}) = 0
futex(0x80ea638, FUTEX_WAKE_PRIVATE, 1) = 0
futex(0x80f3464, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x80f3460, {FUTEX_OP_SET, 0,
FUTEX_OP_CMP_GT, 1}) = 1
gettimeofday({1236026199, 51915}, NULL) = 0
gettimeofday({1236026199, 51977}, NULL) = 0
clock_gettime(CLOCK_REALTIME, {1236026199, 52049314}) = 0
futex(0x80ea6a4, FUTEX_WAIT_PRIVATE, 253, {0, 99927686}) = 0
futex(0x80ea638, FUTEX_WAKE_PRIVATE, 1) = 0
futex(0x80f3464, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x80f3460, {FUTEX_OP_SET, 0,
FUTEX_OP_CMP_GT, 1}) = 1
gettimeofday({1236026199, 52359}, NULL) = 0
gettimeofday({1236026199, 52421}, NULL) = 0
clock_gettime(CLOCK_REALTIME, {1236026199, 52492391}) = 0
futex(0x80ea6a4, FUTEX_WAIT_PRIVATE, 255, {0, 99928609}) = 0
futex(0x80ea638, FUTEX_WAKE_PRIVATE, 1) = 0
futex(0x80f3464, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x80f3460, {FUTEX_OP_SET, 0,
FUTEX_OP_CMP_GT, 1}) = 1
gettimeofday({1236026199, 52817}, NULL) = 0
gettimeofday({1236026199, 52879}, NULL) = 0
clock_gettime(CLOCK_REALTIME, {1236026199, 52948988}) = 0
futex(0x80ea6a4, FUTEX_WAIT_PRIVATE, 257, {0, 99930012}) = 0
futex(0x80ea638, FUTEX_WAKE_PRIVATE, 1) = 0
writev(10, [{"\277\312\0\0\4\0\0\0\2\0\0\0\0\0\0\0X\0\0\0\0\0\0\0"..., 24},
{"p\27\0\0\2\0\0\0\6\0\0\0\0\0\0\0\0\0\0\0\1\0\0\0\1\0\20\0\0\0\0\0\0"...,
88}], 2) = 112
gettimeofday({1236026199, 53416}, NULL) = 0
gettimeofday({1236026199, 53479}, NULL) = 0
clock_gettime(CLOCK_REALTIME, {1236026199, 53550703}) = 0
futex(0x80ea6a4, FUTEX_WAIT_PRIVATE, 259, {0, 99928297}) = 0
futex(0x80ea638, FUTEX_WAKE_PRIVATE, 1) = 0
writev(10, [{"\277\312\0\0\4\0\0\0\3\0\0\0\0\0\0\0\20\0\0\0\0\0\0\0"...,
24}, {"p\27\0\0\2\0\0\0\20\0\0\0\0\0\0\0"..., 16}], 2) = 40
gettimeofday({1236026199, 54116}, NULL) = 0
gettimeofday({1236026199, 54179}, NULL) = 0
clock_gettime(CLOCK_REALTIME, {1236026199, 54248338}) = 0
futex(0x80ea6a4, FUTEX_WAIT_PRIVATE, 261, {0, 99930662}) = 0
futex(0x80ea638, FUTEX_WAKE_PRIVATE, 1) = 0
statfs64("//pvfs2-storage-space", 84, {f_type="EXT2_SUPER_MAGIC",
f_bsize=4096, f_blocks=12901535, f_bfree=10286937, f_bavail=9631577,
f_files=6553600, f_ffree=6240866, f_fsid={1892488181, 343772667},
f_namelen=255, f_frsize=4096}) = 0
sysinfo({uptime=1288517, loads=[6848, 18752, 21536] totalram=746405,
freeram=68070, sharedram=0, bufferram=86232} totalswap=524286,
freeswap=524286, procs=195}) = 0
writev(10, [{"\277\312\0\0\4\0\0\0\4\0\0\0\0\0\0\0h\0\0\0\0\0\0\0"..., 24},
{"p\27\0\0\2\0\0\0\22\0\0\0\0\0\0\0\0\0\0\0-\7\0b\0\220u/\t\0\0\0\0"...,
104}], 2) = 128
gettimeofday({1236026199, 55206}, NULL) = 0
gettimeofday({1236026199, 55267}, NULL) = 0
clock_gettime(CLOCK_REALTIME, {1236026199, 55328370}) = 0
futex(0x80ea6a4, FUTEX_WAIT_PRIVATE, 263, {0, 99938630}) = 0
futex(0x80ea638, FUTEX_WAKE_PRIVATE, 1) = 0
futex(0x80f3464, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x80f3460, {FUTEX_OP_SET, 0,
FUTEX_OP_CMP_GT, 1}) = 1
gettimeofday({1236026199, 60157}, NULL) = 0
gettimeofday({1236026199, 60216}, NULL) = 0
clock_gettime(CLOCK_REALTIME, {1236026199, 60277052}) = 0
futex(0x80ea6a4, FUTEX_WAIT_PRIVATE, 265, {0, 99938948}) = 0
futex(0x80ea638, FUTEX_WAKE_PRIVATE, 1) = 0
writev(10, [{"\277\312\0\0\4\0\0\0\5\0\0\0\0\0\...@\0\0\0\0\0\0\0"..., 24},
{"p\27\0\0\2\0\0\0\25\0\0\0\0\0\0\0\375\377\377\177\0\0\0\0\0\0\0\0\4\0\0\0\1"...,
64}], 2) = 88
gettimeofday({1236026199, 60777}, NULL) = 0
gettimeofday({1236026199, 60838}, NULL) = 0
clock_gettime(CLOCK_REALTIME, {1236026199, 60903807}) = 0
futex(0x80ea6a4, FUTEX_WAIT_PRIVATE, 267, {0, 99934193}) = 0
futex(0x80ea638, FUTEX_WAKE_PRIVATE, 1) = 0
write(2, "pvfs2-server: src/io/job/job.c:61"..., 96pvfs2-server:
src/io/job/job.c:6165: job_precreate_pool_iterate_handles: Assertion `fs'
failed.
) = 96
rt_sigprocmask(SIG_UNBLOCK, [ABRT], NULL, 8) = 0
tgkill(13557, 13557, SIGABRT) = 0
--- SIGABRT (Aborted) @ 0 (0) ---
+++ killed by SIGABRT +++
Any suggestion or help will be much appreciated.
_______________________________________________
Pvfs2-users mailing list
[email protected]
http://www.beowulf-underground.org/mailman/listinfo/pvfs2-users
