Hi,
I added the fourth monitor in the cluster, but it's always down even after I
restart the mon service. I attached the logs below, could you help?
[root@ceph1]# ceph health detail
HEALTH_WARN 1 mons down, quorum 0,1,2 1,3,0; clock skew detected on mon.3, mon.0
mon.8 (rank 3) addr 192.168.1.4:6789/0 is down (out of quorum)
[root@ceph1]# service ceph status mon.8
=== mon.8 ===
mon.8: running {"version":"0.80"}
[root@ceph1]# ceph -s
cluster bfc68f87-30b3-4189-83bc-895de3852064
health HEALTH_WARN 257 pgs stale; 1/25 in osds are down; 1 mons down,
quorum 0,1,2 1,3,0; clock skew detected on mon.3, mon.0
monmap e9: 4 mons at
{0=192.168.1.2:6789/0,1=192.168.0.3:6789/0,3=192.168.0.5:6789/0,8=192.168.1.4:6789/0},
election epoch 26, quorum 0,1,2 1,3,0
mdsmap e4: 1/1/1 up {0=0=up:active}
osdmap e832: 25 osds: 24 up, 25 in
pgmap v1589: 5217 pgs, 4 pools, 1900 bytes data, 23 objects
1631 MB used, 4888 GB / 4890 GB avail
257 stale+active+clean
4960 active+clean
[root@ceph1]# ceph mon_status
{"name":"1","rank":0,"state":"leader","election_epoch":26,"quorum":[0,1,2],"outside_quorum":[],"extra_probe_peers":[],"sync_provider":[],"monmap":{"epoch":9,"fsid":"bfc68f87-30b3-4189-83bc-895de3852064","modified":"2014-06-12
16:33:42.756814","created":"2014-06-12
15:37:48.802401","mons":[{"rank":0,"name":"1","addr":"192.168.0.3:6789\/0"},{"rank":1,"name":"3","addr":"192.168.0.5:6789\/0"},{"rank":2,"name":"0","addr":"192.168.1.2:6789\/0"},{"rank":3,"name":"8","addr":"192.168.1.4:6789\/0"}]}}
Ceph-osd.8.log -
2014-06-12 17:01:10.152999 7f1f644517a0 0 ceph version 0.80
(b78644e7dee100e48dfeca32c9270a6b210d3003), process ceph-mon, pid 12997
2014-06-12 17:01:10.259693 7f1f644517a0 0 mon.8 does not exist in monmap, will
attempt to join an existing cluster
2014-06-12 17:01:10.264868 7f1f644517a0 0 starting mon.8 rank -1 at
192.168.1.4:6789/0 mon_data /var/lib/ceph/mon/mon8 fsid
bfc68f87-30b3-4189-83bc-895de3852064
2014-06-12 17:01:10.265437 7f1f644517a0 1 mon.8@-1(probing) e0 preinit fsid
bfc68f87-30b3-4189-83bc-895de3852064
2014-06-12 17:02:10.265740 7f1f619fa700 0 mon.8@-1(probing).data_health(0)
update_stats avail 93% total 302379456 used 3150672 avail 283868784
2014-06-12 17:03:10.265946 7f1f619fa700 0 mon.8@-1(probing).data_health(0)
update_stats avail 93% total 302379456 used 3150696 avail 283868760
2014-06-12 17:04:10.266134 7f1f619fa700 0 mon.8@-1(probing).data_health(0)
update_stats avail 93% total 302379456 used 3150720 avail 283868736
2014-06-12 17:05:10.266296 7f1f619fa700 0 mon.8@-1(probing).data_health(0)
update_stats avail 93% total 302379456 used 3150708 avail 283868748
2014-06-12 17:06:10.266460 7f1f619fa700 0 mon.8@-1(probing).data_health(0)
update_stats avail 93% total 302379456 used 3150712 avail 283868744
2014-06-12 17:07:10.266633 7f1f619fa700 0 mon.8@-1(probing).data_health(0)
update_stats avail 93% total 302379456 used 3150732 avail 283868724
2014-06-12 17:08:10.266798 7f1f619fa700 0 mon.8@-1(probing).data_health(0)
update_stats avail 93% total 302379456 used 3150748 avail 283868708
2014-06-12 17:09:10.266959 7f1f619fa700 0 mon.8@-1(probing).data_health(0)
update_stats avail 93% total 302379456 used 3150756 avail 283868700
2014-06-12 17:10:10.267142 7f1f619fa700 0 mon.8@-1(probing).data_health(0)
update_stats avail 93% total 302379456 used 3150792 avail 283868664
2014-06-12 17:11:10.267286 7f1f619fa700 0 mon.8@-1(probing).data_health(0)
update_stats avail 93% total 302379456 used 3150804 avail 283868652
2014-06-12 17:12:10.267453 7f1f619fa700 0 mon.8@-1(probing).data_health(0)
update_stats avail 93% total 302379456 used 3150808 avail 283868648
2014-06-12 17:13:10.267622 7f1f619fa700 0 mon.8@-1(probing).data_health(0)
update_stats avail 93% total 302379456 used 3150840 avail 283868616
2014-06-12 17:14:10.267785 7f1f619fa700 0 mon.8@-1(probing).data_health(0)
update_stats avail 93% total 302379456 used 3150860 avail 283868596
2014-06-12 17:15:10.268075 7f1f619fa700 0 mon.8@-1(probing).data_health(0)
update_stats avail 93% total 302379456 used 3150872 avail 283868584
Wei Cao (Buddy)
_______________________________________________
ceph-users mailing list
ceph-users@lists.ceph.com
http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
