Hello,
I’m having a (strange) issue with OSD bucket persistence / affinity on my test
cluster..
The cluster is PoC / test, by no means production. Consists of a single OSD /
MON host + another MON running on a KVM VM.
Out of 12 OSDs I’m trying to get osd.10 and osd.11 to be part of the ssd bucket
in my CRUSH map. This works fine when either editing the CRUSH map by hand
(exporting, decompile, edit, compile, import), or via the ceph osd crush set
command:
"ceph osd crush set osd.11 0.140 root=ssd”
I’m able to verify that the OSD / MON host and another MON I have running see
the same CRUSH map.
After rebooting OSD / MON host, both osd.10 and osd.11 become part of the
default bucket. How can I ensure that ODSs persist in their configured
buckets?
Here’s my desired CRUSH map. This is a PoC and by no means production ready..
——
# begin crush map
tunable choose_local_tries 0
tunable choose_local_fallback_tries 0
tunable choose_total_tries 50
tunable chooseleaf_descend_once 1
tunable straw_calc_version 1
# devices
device 0 osd.0
device 1 osd.1
device 2 osd.2
device 3 osd.3
device 4 osd.4
device 5 osd.5
device 6 osd.6
device 7 osd.7
device 8 osd.8
device 9 osd.9
device 10 osd.10
device 11 osd.11
# types
type 0 osd
type 1 host
type 2 chassis
type 3 rack
type 4 row
type 5 pdu
type 6 pod
type 7 room
type 8 datacenter
type 9 region
type 10 root
# buckets
host osdhost {
id -2 # do not change unnecessarily
# weight 36.200
alg straw
hash 0 # rjenkins1
item osd.0 weight 3.620
item osd.1 weight 3.620
item osd.2 weight 3.620
item osd.3 weight 3.620
item osd.4 weight 3.620
item osd.5 weight 3.620
item osd.6 weight 3.620
item osd.7 weight 3.620
item osd.8 weight 3.620
item osd.9 weight 3.620
}
root default {
id -1 # do not change unnecessarily
# weight 36.200
alg straw
hash 0 # rjenkins1
item osdhost weight 36.200
}
host osdhost-ssd {
id -3 # do not change unnecessarily
# weight 0.280
alg straw
hash 0 # rjenkins1
item osd.10 weight 0.140
item osd.11 weight 0.140
}
root ssd {
id -4 # do not change unnecessarily
# weight 0.280
alg straw
hash 0 # rjenkins1
item osdhost-ssd weight 0.280
}
# rules
rule replicated_ruleset {
ruleset 0
type replicated
min_size 1
max_size 10
step take default
step chooseleaf firstn 0 type osd
step emit
}
rule ecpool {
ruleset 1
type erasure
min_size 3
max_size 7
step set_chooseleaf_tries 5
step set_choose_tries 100
step take default
step choose indep 0 type osd
step emit
}
rule cachetier {
ruleset 2
type replicated
min_size 1
max_size 10
step set_chooseleaf_tries 5
step set_choose_tries 100
step take ssd
step chooseleaf firstn 0 type osd
step emit
}
# end crush map
——
ceph osd tree (before reboot)
ID WEIGHT TYPE NAME UP/DOWN REWEIGHT PRIMARY-AFFINITY
-4 0.28000 root ssd
-3 0 host osdhost-ssd
10 0.14000 osd.10 up 1.00000 1.00000
11 0.14000 osd.11 up 1.00000 1.00000
-1 36.19995 root default
-2 36.19995 host osdhost
0 3.62000 osd.0 up 1.00000 1.00000
1 3.62000 osd.1 up 1.00000 1.00000
2 3.62000 osd.2 up 1.00000 1.00000
3 3.62000 osd.3 up 1.00000 1.00000
4 3.62000 osd.4 up 1.00000 1.00000
5 3.62000 osd.5 up 1.00000 1.00000
6 3.62000 osd.6 up 1.00000 1.00000
7 3.62000 osd.7 up 1.00000 1.00000
8 3.62000 osd.8 up 1.00000 1.00000
9 3.62000 osd.9 up 1.00000 1.00000
ceph osd tree (after reboot)
[root@osdhost tmp]# ceph osd tree
ID WEIGHT TYPE NAME UP/DOWN REWEIGHT PRIMARY-AFFINITY
-4 0 root ssd
-3 0 host osdhost-ssd
-1 36.47995 root default
-2 36.47995 host osdhost
0 3.62000 osd.0 up 1.00000 1.00000
1 3.62000 osd.1 up 1.00000 1.00000
2 3.62000 osd.2 up 1.00000 1.00000
3 3.62000 osd.3 up 1.00000 1.00000
4 3.62000 osd.4 up 1.00000 1.00000
5 3.62000 osd.5 up 1.00000 1.00000
6 3.62000 osd.6 up 1.00000 1.00000
7 3.62000 osd.7 up 1.00000 1.00000
8 3.62000 osd.8 up 1.00000 1.00000
9 3.62000 osd.9 up 1.00000 1.00000
10 0.14000 osd.10 up 1.00000 1.00000
11 0.14000 osd.11 up 1.00000 1.00000
Here’s the CRUSH map after reboot.
——
# begin crush map
tunable choose_local_tries 0
tunable choose_local_fallback_tries 0
tunable choose_total_tries 50
tunable chooseleaf_descend_once 1
tunable straw_calc_version 1
# devices
device 0 osd.0
device 1 osd.1
device 2 osd.2
device 3 osd.3
device 4 osd.4
device 5 osd.5
device 6 osd.6
device 7 osd.7
device 8 osd.8
device 9 osd.9
device 10 osd.10
device 11 osd.11
# types
type 0 osd
type 1 host
type 2 chassis
type 3 rack
type 4 row
type 5 pdu
type 6 pod
type 7 room
type 8 datacenter
type 9 region
type 10 root
# buckets
host osdhost {
id -2 # do not change unnecessarily
# weight 36.480
alg straw
hash 0 # rjenkins1
item osd.0 weight 3.620
item osd.1 weight 3.620
item osd.2 weight 3.620
item osd.3 weight 3.620
item osd.4 weight 3.620
item osd.5 weight 3.620
item osd.6 weight 3.620
item osd.7 weight 3.620
item osd.8 weight 3.620
item osd.9 weight 3.620
item osd.10 weight 0.140
item osd.11 weight 0.140
}
root default {
id -1 # do not change unnecessarily
# weight 36.480
alg straw
hash 0 # rjenkins1
item osdhost weight 36.480
}
host osdhost-ssd {
id -3 # do not change unnecessarily
# weight 0.000
alg straw
hash 0 # rjenkins1
}
root ssd {
id -4 # do not change unnecessarily
# weight 0.000
alg straw
hash 0 # rjenkins1
item osdhost-ssd weight 0.000
}
# rules
rule replicated_ruleset {
ruleset 0
type replicated
min_size 1
max_size 10
step take default
step chooseleaf firstn 0 type osd
step emit
}
rule ecpool {
ruleset 1
type erasure
min_size 3
max_size 7
step set_chooseleaf_tries 5
step set_choose_tries 100
step take default
step choose indep 0 type osd
step emit
}
rule cachetier {
ruleset 2
type replicated
min_size 1
max_size 10
step set_chooseleaf_tries 5
step set_choose_tries 100
step take ssd
step chooseleaf firstn 0 type osd
step emit
}
# end crush map
——
Thanks.
_______________________________________________
ceph-users mailing list
ceph-users@lists.ceph.com
http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
