Hi Ben, At first glance i would say the CPU's are a bit weak for this setup. Recommended is to have at least 1 core per OSD. Since you have 8 cores and 10 OSD's there isn't much left for other processes.
Furthermore, did you upgrade the firmware of those DC S4500's to the latest firmware? (SCV10142) If not, you can upgrade them via the Intel Data Center upgrade tool: https://downloadcenter.intel.com/download/27863?v=t Kind regards, Caspar 2018-07-04 10:26 GMT+02:00 Benjamin Naber <der-co...@coders-area.de>: > Hi @all, > > im currently in testing for setup an production environment based on the > following OSD Nodes: > > CEPH Version: luminous 12.2.5 > > 5x OSD Nodes with following specs: > > - 8 Core Intel Xeon 2,0 GHZ > > - 96GB Ram > > - 10x 1,92 TB Intel DC S4500 connectet via SATA > > - 4x 10 Gbit NIC 2 bonded via LACP for Backend Network 2 bonded via LACP > for Backend Network. > > if i run some fio benchmark via a VM that ist running on a RBD Device on a > KVM testing Host. the cluster always runs into slow request warning. Also > the performance goes heavy down. > > If i dump the osd that stucks, i get the following output: > > { > "ops": [ > { > "description": "osd_op(client.141944.0:359346834 13.1da > 13:5b8b7fd3:::rbd_data.170a3238e1f29.00000000000000be:head [write > 2097152~1048576] snapc 0=[] ondisk+write+known_if_redirected e2755)", > "initiated_at": "2018-07-04 10:00:49.475879", > "age": 287.180328, > "duration": 287.180355, > "type_data": { > "flag_point": "waiting for sub ops", > "client_info": { > "client": "client.141944", > "client_addr": "10.111.90.1:0/3532639465", > "tid": 359346834 > }, > "events": [ > { > "time": "2018-07-04 10:00:49.475879", > "event": "initiated" > }, > { > "time": "2018-07-04 10:00:49.476935", > "event": "queued_for_pg" > }, > { > "time": "2018-07-04 10:00:49.477547", > "event": "reached_pg" > }, > { > "time": "2018-07-04 10:00:49.477578", > "event": "started" > }, > { > "time": "2018-07-04 10:00:49.477614", > "event": "waiting for subops from 5,26" > }, > { > "time": "2018-07-04 10:00:49.484679", > "event": "op_commit" > }, > { > "time": "2018-07-04 10:00:49.484681", > "event": "op_applied" > }, > { > "time": "2018-07-04 10:00:49.485588", > "event": "sub_op_commit_rec from 5" > } > ] > } > }, > { > "description": "osd_op(client.141944.0:359346835 13.1da > 13:5b8b7fd3:::rbd_data.170a3238e1f29.00000000000000be:head [write > 3145728~1048576] snapc 0=[] ondisk+write+known_if_redirected e2755)", > "initiated_at": "2018-07-04 10:00:49.477065", > "age": 287.179143, > "duration": 287.179221, > "type_data": { > "flag_point": "waiting for sub ops", > "client_info": { > "client": "client.141944", > "client_addr": "10.111.90.1:0/3532639465", > "tid": 359346835 > }, > "events": [ > { > "time": "2018-07-04 10:00:49.477065", > "event": "initiated" > }, > { > "time": "2018-07-04 10:00:49.478116", > "event": "queued_for_pg" > }, > { > "time": "2018-07-04 10:00:49.478178", > "event": "reached_pg" > }, > { > "time": "2018-07-04 10:00:49.478201", > "event": "started" > }, > { > "time": "2018-07-04 10:00:49.478232", > "event": "waiting for subops from 5,26" > }, > { > "time": "2018-07-04 10:00:49.484695", > "event": "op_commit" > }, > { > "time": "2018-07-04 10:00:49.484696", > "event": "op_applied" > }, > { > "time": "2018-07-04 10:00:49.485621", > "event": "sub_op_commit_rec from 5" > } > ] > } > }, > { > "description": "osd_op(client.141944.0:359346440 13.11d > 13:b8afbe4a:::rbd_data.170a3238e1f29.000000000000005c:head [write > 0~1048576] snapc 0=[] ondisk+write+known_if_redirected e2755)", > "initiated_at": "2018-07-04 10:00:49.091127", > "age": 287.565080, > "duration": 287.565196, > "type_data": { > "flag_point": "waiting for sub ops", > "client_info": { > "client": "client.141944", > "client_addr": "10.111.90.1:0/3532639465", > "tid": 359346440 > }, > "events": [ > { > "time": "2018-07-04 10:00:49.091127", > "event": "initiated" > }, > { > "time": "2018-07-04 10:00:49.092764", > "event": "queued_for_pg" > }, > { > "time": "2018-07-04 10:00:49.092787", > "event": "reached_pg" > }, > { > "time": "2018-07-04 10:00:49.093062", > "event": "started" > }, > { > "time": "2018-07-04 10:00:49.093107", > "event": "waiting for subops from 21,26" > }, > { > "time": "2018-07-04 10:00:49.099496", > "event": "op_commit" > }, > { > "time": "2018-07-04 10:00:49.099502", > "event": "op_applied" > }, > { > "time": "2018-07-04 10:00:49.100772", > "event": "sub_op_commit_rec from 21" > } > ] > } > }, > { > "description": "osd_op(client.141944.0:359346441 13.11d > 13:b8afbe4a:::rbd_data.170a3238e1f29.000000000000005c:head [write > 1048576~1048576] snapc 0=[] ondisk+write+known_if_redirected e2755)", > "initiated_at": "2018-07-04 10:00:49.092772", > "age": 287.563436, > "duration": 287.563588, > "type_data": { > "flag_point": "waiting for sub ops", > "client_info": { > "client": "client.141944", > "client_addr": "10.111.90.1:0/3532639465", > "tid": 359346441 > }, > "events": [ > { > "time": "2018-07-04 10:00:49.092772", > "event": "initiated" > }, > { > "time": "2018-07-04 10:00:49.093766", > "event": "queued_for_pg" > }, > { > "time": "2018-07-04 10:00:49.093865", > "event": "reached_pg" > }, > { > "time": "2018-07-04 10:00:49.093895", > "event": "started" > }, > { > "time": "2018-07-04 10:00:49.093937", > "event": "waiting for subops from 21,26" > }, > { > "time": "2018-07-04 10:00:49.099505", > "event": "op_commit" > }, > { > "time": "2018-07-04 10:00:49.099506", > "event": "op_applied" > }, > { > "time": "2018-07-04 10:00:49.100788", > "event": "sub_op_commit_rec from 21" > } > ] > } > }, > { > "description": "osd_op(client.141944.0:359346442 13.11d > 13:b8afbe4a:::rbd_data.170a3238e1f29.000000000000005c:head [write > 2097152~1048576] snapc 0=[] ondisk+write+known_if_redirected e2755)", > "initiated_at": "2018-07-04 10:00:49.093773", > "age": 287.562434, > "duration": 287.562622, > "type_data": { > "flag_point": "waiting for sub ops", > "client_info": { > "client": "client.141944", > "client_addr": "10.111.90.1:0/3532639465", > "tid": 359346442 > }, > "events": [ > { > "time": "2018-07-04 10:00:49.093773", > "event": "initiated" > }, > { > "time": "2018-07-04 10:00:49.095388", > "event": "queued_for_pg" > }, > { > "time": "2018-07-04 10:00:49.095403", > "event": "reached_pg" > }, > { > "time": "2018-07-04 10:00:49.095428", > "event": "started" > }, > { > "time": "2018-07-04 10:00:49.095466", > "event": "waiting for subops from 21,26" > }, > { > "time": "2018-07-04 10:00:49.103411", > "event": "op_commit" > }, > { > "time": "2018-07-04 10:00:49.103413", > "event": "op_applied" > }, > { > "time": "2018-07-04 10:00:49.106365", > "event": "sub_op_commit_rec from 21" > } > ] > } > }, > { > "description": "osd_op(client.141944.0:359346443 13.11d > 13:b8afbe4a:::rbd_data.170a3238e1f29.000000000000005c:head [write > 3145728~1048576] snapc 0=[] ondisk+write+known_if_redirected e2755)", > "initiated_at": "2018-07-04 10:00:49.095395", > "age": 287.560813, > "duration": 287.561037, > "type_data": { > "flag_point": "waiting for sub ops", > "client_info": { > "client": "client.141944", > "client_addr": "10.111.90.1:0/3532639465", > "tid": 359346443 > }, > "events": [ > { > "time": "2018-07-04 10:00:49.095395", > "event": "initiated" > }, > { > "time": "2018-07-04 10:00:49.097194", > "event": "queued_for_pg" > }, > { > "time": "2018-07-04 10:00:49.098851", > "event": "reached_pg" > }, > { > "time": "2018-07-04 10:00:49.098878", > "event": "started" > }, > { > "time": "2018-07-04 10:00:49.098914", > "event": "waiting for subops from 21,26" > }, > { > "time": "2018-07-04 10:00:49.103704", > "event": "op_commit" > }, > { > "time": "2018-07-04 10:00:49.103706", > "event": "op_applied" > }, > { > "time": "2018-07-04 10:00:49.107064", > "event": "sub_op_commit_rec from 21" > } > ] > } > }, > { > "description": "osd_op(client.141944.0:359346172 13.11d > 13:b88a4b5e:::rbd_data.170a3238e1f29.000000000000001a:head > [set-alloc-hint object_size 4194304 write_size 4194304,write 0~1048576] > snapc 0=[] ondisk+write+known_if_redirected e2755)", > "initiated_at": "2018-07-04 10:00:48.836391", > "age": 287.819817, > "duration": 287.820078, > "type_data": { > "flag_point": "waiting for sub ops", > "client_info": { > "client": "client.141944", > "client_addr": "10.111.90.1:0/3532639465", > "tid": 359346172 > }, > "events": [ > { > "time": "2018-07-04 10:00:48.836391", > "event": "initiated" > }, > { > "time": "2018-07-04 10:00:48.837419", > "event": "queued_for_pg" > }, > { > "time": "2018-07-04 10:00:48.837438", > "event": "reached_pg" > }, > { > "time": "2018-07-04 10:00:48.837548", > "event": "started" > }, > { > "time": "2018-07-04 10:00:48.837605", > "event": "waiting for subops from 21,26" > }, > { > "time": "2018-07-04 10:00:48.847008", > "event": "op_commit" > }, > { > "time": "2018-07-04 10:00:48.847013", > "event": "op_applied" > }, > { > "time": "2018-07-04 10:00:48.862005", > "event": "sub_op_commit_rec from 21" > } > ] > } > }, > { > "description": "osd_op(client.141944.0:359346372 13.bc > 13:3d131f8b:::rbd_data.170a3238e1f29.000000000000004b:head [write > 0~1048576] snapc 0=[] ondisk+write+known_if_redirected e2755)", > "initiated_at": "2018-07-04 10:00:49.023513", > "age": 287.632694, > "duration": 287.632990, > "type_data": { > "flag_point": "waiting for sub ops", > "client_info": { > "client": "client.141944", > "client_addr": "10.111.90.1:0/3532639465", > "tid": 359346372 > }, > "events": [ > { > "time": "2018-07-04 10:00:49.023513", > "event": "initiated" > }, > { > "time": "2018-07-04 10:00:49.025442", > "event": "queued_for_pg" > }, > { > "time": "2018-07-04 10:00:49.025464", > "event": "reached_pg" > }, > { > "time": "2018-07-04 10:00:49.025796", > "event": "started" > }, > { > "time": "2018-07-04 10:00:49.025844", > "event": "waiting for subops from 21,26" > }, > { > "time": "2018-07-04 10:00:49.033594", > "event": "op_commit" > }, > { > "time": "2018-07-04 10:00:49.033600", > "event": "op_applied" > }, > { > "time": "2018-07-04 10:00:49.036086", > "event": "sub_op_commit_rec from 21" > } > ] > } > }, > { > "description": "osd_op(client.141944.0:359346173 13.11d > 13:b88a4b5e:::rbd_data.170a3238e1f29.000000000000001a:head > [set-alloc-hint object_size 4194304 write_size 4194304,write > 1048576~1048576] snapc 0=[] ondisk+write+known_if_redirected e2755)", > "initiated_at": "2018-07-04 10:00:48.837425", > "age": 287.818782, > "duration": 287.819113, > "type_data": { > "flag_point": "waiting for sub ops", > "client_info": { > "client": "client.141944", > "client_addr": "10.111.90.1:0/3532639465", > "tid": 359346173 > }, > "events": [ > { > "time": "2018-07-04 10:00:48.837425", > "event": "initiated" > }, > { > "time": "2018-07-04 10:00:48.838409", > "event": "queued_for_pg" > }, > { > "time": "2018-07-04 10:00:48.838427", > "event": "reached_pg" > }, > { > "time": "2018-07-04 10:00:48.838463", > "event": "started" > }, > { > "time": "2018-07-04 10:00:48.838505", > "event": "waiting for subops from 21,26" > }, > { > "time": "2018-07-04 10:00:48.847328", > "event": "op_commit" > }, > { > "time": "2018-07-04 10:00:48.847330", > "event": "op_applied" > }, > { > "time": "2018-07-04 10:00:48.862049", > "event": "sub_op_commit_rec from 21" > } > ] > } > }, > { > "description": "osd_op(client.141944.0:359346373 13.bc > 13:3d131f8b:::rbd_data.170a3238e1f29.000000000000004b:head [write > 1048576~1048576] snapc 0=[] ondisk+write+known_if_redirected e2755)", > "initiated_at": "2018-07-04 10:00:49.025449", > "age": 287.630759, > "duration": 287.631126, > "type_data": { > "flag_point": "waiting for sub ops", > "client_info": { > "client": "client.141944", > "client_addr": "10.111.90.1:0/3532639465", > "tid": 359346373 > }, > "events": [ > { > "time": "2018-07-04 10:00:49.025449", > "event": "initiated" > }, > { > "time": "2018-07-04 10:00:49.027010", > "event": "queued_for_pg" > }, > { > "time": "2018-07-04 10:00:49.027030", > "event": "reached_pg" > }, > { > "time": "2018-07-04 10:00:49.027071", > "event": "started" > }, > { > "time": "2018-07-04 10:00:49.027126", > "event": "waiting for subops from 21,26" > }, > { > "time": "2018-07-04 10:00:49.033603", > "event": "op_commit" > }, > { > "time": "2018-07-04 10:00:49.033605", > "event": "op_applied" > }, > { > "time": "2018-07-04 10:00:49.036143", > "event": "sub_op_commit_rec from 21" > } > ] > } > }, > { > "description": "osd_op(client.141944.0:359346174 13.11d > 13:b88a4b5e:::rbd_data.170a3238e1f29.000000000000001a:head > [set-alloc-hint object_size 4194304 write_size 4194304,write > 2097152~1048576] snapc 0=[] ondisk+write+known_if_redirected e2755)", > "initiated_at": "2018-07-04 10:00:48.838414", > "age": 287.817793, > "duration": 287.818196, > "type_data": { > "flag_point": "waiting for sub ops", > "client_info": { > "client": "client.141944", > "client_addr": "10.111.90.1:0/3532639465", > "tid": 359346174 > }, > "events": [ > { > "time": "2018-07-04 10:00:48.838414", > "event": "initiated" > }, > { > "time": "2018-07-04 10:00:48.839187", > "event": "queued_for_pg" > }, > { > "time": "2018-07-04 10:00:48.839203", > "event": "reached_pg" > }, > { > "time": "2018-07-04 10:00:48.839229", > "event": "started" > }, > { > "time": "2018-07-04 10:00:48.839269", > "event": "waiting for subops from 21,26" > }, > { > "time": "2018-07-04 10:00:48.847345", > "event": "op_commit" > }, > { > "time": "2018-07-04 10:00:48.847347", > "event": "op_applied" > }, > { > "time": "2018-07-04 10:00:48.862060", > "event": "sub_op_commit_rec from 21" > } > ] > } > }, > { > "description": "osd_op(client.141944.0:359346374 13.bc > 13:3d131f8b:::rbd_data.170a3238e1f29.000000000000004b:head [write > 2097152~1048576] snapc 0=[] ondisk+write+known_if_redirected e2755)", > "initiated_at": "2018-07-04 10:00:49.027016", > "age": 287.629192, > "duration": 287.629634, > "type_data": { > "flag_point": "waiting for sub ops", > "client_info": { > "client": "client.141944", > "client_addr": "10.111.90.1:0/3532639465", > "tid": 359346374 > }, > "events": [ > { > "time": "2018-07-04 10:00:49.027016", > "event": "initiated" > }, > { > "time": "2018-07-04 10:00:49.027972", > "event": "queued_for_pg" > }, > { > "time": "2018-07-04 10:00:49.027987", > "event": "reached_pg" > }, > { > "time": "2018-07-04 10:00:49.028017", > "event": "started" > }, > { > "time": "2018-07-04 10:00:49.028053", > "event": "waiting for subops from 21,26" > }, > { > "time": "2018-07-04 10:00:49.033606", > "event": "op_commit" > }, > { > "time": "2018-07-04 10:00:49.033609", > "event": "op_applied" > }, > { > "time": "2018-07-04 10:00:49.036159", > "event": "sub_op_commit_rec from 21" > } > ] > } > }, > { > "description": "osd_op(client.141944.0:359346175 13.11d > 13:b88a4b5e:::rbd_data.170a3238e1f29.000000000000001a:head > [set-alloc-hint object_size 4194304 write_size 4194304,write > 3145728~1048576] snapc 0=[] ondisk+write+known_if_redirected e2755)", > "initiated_at": "2018-07-04 10:00:48.839192", > "age": 287.817015, > "duration": 287.817491, > "type_data": { > "flag_point": "waiting for sub ops", > "client_info": { > "client": "client.141944", > "client_addr": "10.111.90.1:0/3532639465", > "tid": 359346175 > }, > "events": [ > { > "time": "2018-07-04 10:00:48.839192", > "event": "initiated" > }, > { > "time": "2018-07-04 10:00:48.840612", > "event": "queued_for_pg" > }, > { > "time": "2018-07-04 10:00:48.840626", > "event": "reached_pg" > }, > { > "time": "2018-07-04 10:00:48.840658", > "event": "started" > }, > { > "time": "2018-07-04 10:00:48.840693", > "event": "waiting for subops from 21,26" > }, > { > "time": "2018-07-04 10:00:48.847361", > "event": "op_commit" > }, > { > "time": "2018-07-04 10:00:48.847363", > "event": "op_applied" > }, > { > "time": "2018-07-04 10:00:48.862089", > "event": "sub_op_commit_rec from 21" > } > ] > } > }, > { > "description": "osd_op(client.141944.0:359346375 13.bc > 13:3d131f8b:::rbd_data.170a3238e1f29.000000000000004b:head [write > 3145728~1048576] snapc 0=[] ondisk+write+known_if_redirected e2755)", > "initiated_at": "2018-07-04 10:00:49.027978", > "age": 287.628229, > "duration": 287.628739, > "type_data": { > "flag_point": "waiting for sub ops", > "client_info": { > "client": "client.141944", > "client_addr": "10.111.90.1:0/3532639465", > "tid": 359346375 > }, > "events": [ > { > "time": "2018-07-04 10:00:49.027978", > "event": "initiated" > }, > { > "time": "2018-07-04 10:00:49.028846", > "event": "queued_for_pg" > }, > { > "time": "2018-07-04 10:00:49.028863", > "event": "reached_pg" > }, > { > "time": "2018-07-04 10:00:49.028889", > "event": "started" > }, > { > "time": "2018-07-04 10:00:49.028924", > "event": "waiting for subops from 21,26" > }, > { > "time": "2018-07-04 10:00:49.035948", > "event": "op_commit" > }, > { > "time": "2018-07-04 10:00:49.035950", > "event": "op_applied" > }, > { > "time": "2018-07-04 10:00:49.038542", > "event": "sub_op_commit_rec from 21" > } > ] > } > }, > { > "description": "osd_op(client.141944.0:359346792 13.bc > 13:3d313592:::rbd_data.170a3238e1f29.00000000000000b4:head [write > 0~1048576] snapc 0=[] ondisk+write+known_if_redirected e2755)", > "initiated_at": "2018-07-04 10:00:49.423998", > "age": 287.232209, > "duration": 287.232753, > "type_data": { > "flag_point": "waiting for sub ops", > "client_info": { > "client": "client.141944", > "client_addr": "10.111.90.1:0/3532639465", > "tid": 359346792 > }, > "events": [ > { > "time": "2018-07-04 10:00:49.423998", > "event": "initiated" > }, > { > "time": "2018-07-04 10:00:49.425331", > "event": "queued_for_pg" > }, > { > "time": "2018-07-04 10:00:49.425350", > "event": "reached_pg" > }, > { > "time": "2018-07-04 10:00:49.425655", > "event": "started" > }, > { > "time": "2018-07-04 10:00:49.425715", > "event": "waiting for subops from 21,26" > }, > { > "time": "2018-07-04 10:00:49.434856", > "event": "op_commit" > }, > { > "time": "2018-07-04 10:00:49.434869", > "event": "op_applied" > }, > { > "time": "2018-07-04 10:00:49.436410", > "event": "sub_op_commit_rec from 21" > } > ] > } > }, > { > "description": "osd_op(client.141944.0:359346793 13.bc > 13:3d313592:::rbd_data.170a3238e1f29.00000000000000b4:head [write > 1048576~1048576] snapc 0=[] ondisk+write+known_if_redirected e2755)", > "initiated_at": "2018-07-04 10:00:49.425349", > "age": 287.230859, > "duration": 287.231436, > "type_data": { > "flag_point": "waiting for sub ops", > "client_info": { > "client": "client.141944", > "client_addr": "10.111.90.1:0/3532639465", > "tid": 359346793 > }, > "events": [ > { > "time": "2018-07-04 10:00:49.425349", > "event": "initiated" > }, > { > "time": "2018-07-04 10:00:49.426349", > "event": "queued_for_pg" > }, > { > "time": "2018-07-04 10:00:49.426877", > "event": "reached_pg" > }, > { > "time": "2018-07-04 10:00:49.427022", > "event": "started" > }, > { > "time": "2018-07-04 10:00:49.427112", > "event": "waiting for subops from 21,26" > }, > { > "time": "2018-07-04 10:00:49.434876", > "event": "op_commit" > }, > { > "time": "2018-07-04 10:00:49.434877", > "event": "op_applied" > }, > { > "time": "2018-07-04 10:00:49.436661", > "event": "sub_op_commit_rec from 21" > } > ] > } > }, > { > "description": "osd_op(client.141944.0:359346794 13.bc > 13:3d313592:::rbd_data.170a3238e1f29.00000000000000b4:head [write > 2097152~1048576] snapc 0=[] ondisk+write+known_if_redirected e2755)", > "initiated_at": "2018-07-04 10:00:49.426356", > "age": 287.229852, > "duration": 287.230461, > "type_data": { > "flag_point": "waiting for sub ops", > "client_info": { > "client": "client.141944", > "client_addr": "10.111.90.1:0/3532639465", > "tid": 359346794 > }, > "events": [ > { > "time": "2018-07-04 10:00:49.426356", > "event": "initiated" > }, > { > "time": "2018-07-04 10:00:49.427126", > "event": "queued_for_pg" > }, > { > "time": "2018-07-04 10:00:49.427729", > "event": "reached_pg" > }, > { > "time": "2018-07-04 10:00:49.427771", > "event": "started" > }, > { > "time": "2018-07-04 10:00:49.427812", > "event": "waiting for subops from 21,26" > }, > { > "time": "2018-07-04 10:00:49.434878", > "event": "op_commit" > }, > { > "time": "2018-07-04 10:00:49.434879", > "event": "op_applied" > }, > { > "time": "2018-07-04 10:00:49.436686", > "event": "sub_op_commit_rec from 21" > } > ] > } > }, > { > "description": "osd_op(client.141944.0:359346795 13.bc > 13:3d313592:::rbd_data.170a3238e1f29.00000000000000b4:head [write > 3145728~1048576] snapc 0=[] ondisk+write+known_if_redirected e2755)", > "initiated_at": "2018-07-04 10:00:49.427141", > "age": 287.229066, > "duration": 287.229715, > "type_data": { > "flag_point": "waiting for sub ops", > "client_info": { > "client": "client.141944", > "client_addr": "10.111.90.1:0/3532639465", > "tid": 359346795 > }, > "events": [ > { > "time": "2018-07-04 10:00:49.427141", > "event": "initiated" > }, > { > "time": "2018-07-04 10:00:49.428212", > "event": "queued_for_pg" > }, > { > "time": "2018-07-04 10:00:49.431693", > "event": "reached_pg" > }, > { > "time": "2018-07-04 10:00:49.431730", > "event": "started" > }, > { > "time": "2018-07-04 10:00:49.431794", > "event": "waiting for subops from 21,26" > }, > { > "time": "2018-07-04 10:00:49.437279", > "event": "op_commit" > }, > { > "time": "2018-07-04 10:00:49.437282", > "event": "op_applied" > }, > { > "time": "2018-07-04 10:00:49.442448", > "event": "sub_op_commit_rec from 21" > } > ] > } > }, > { > "description": "osd_op(client.141944.0:359346832 13.1da > 13:5b8b7fd3:::rbd_data.170a3238e1f29.00000000000000be:head [write > 0~901120] snapc 0=[] ondisk+write+known_if_redirected e2755)", > "initiated_at": "2018-07-04 10:00:49.473234", > "age": 287.182974, > "duration": 287.183657, > "type_data": { > "flag_point": "waiting for sub ops", > "client_info": { > "client": "client.141944", > "client_addr": "10.111.90.1:0/3532639465", > "tid": 359346832 > }, > "events": [ > { > "time": "2018-07-04 10:00:49.473234", > "event": "initiated" > }, > { > "time": "2018-07-04 10:00:49.474116", > "event": "queued_for_pg" > }, > { > "time": "2018-07-04 10:00:49.474135", > "event": "reached_pg" > }, > { > "time": "2018-07-04 10:00:49.474444", > "event": "started" > }, > { > "time": "2018-07-04 10:00:49.474489", > "event": "waiting for subops from 5,26" > }, > { > "time": "2018-07-04 10:00:49.478163", > "event": "op_commit" > }, > { > "time": "2018-07-04 10:00:49.478170", > "event": "op_applied" > }, > { > "time": "2018-07-04 10:00:49.485418", > "event": "sub_op_commit_rec from 5" > } > ] > } > }, > { > "description": "osd_op(client.141944.0:359346833 13.1da > 13:5b8b7fd3:::rbd_data.170a3238e1f29.00000000000000be:head [write > 901120~1196032] snapc 0=[] ondisk+write+known_if_redirected e2755)", > "initiated_at": "2018-07-04 10:00:49.474651", > "age": 287.181557, > "duration": 287.182274, > "type_data": { > "flag_point": "waiting for sub ops", > "client_info": { > "client": "client.141944", > "client_addr": "10.111.90.1:0/3532639465", > "tid": 359346833 > }, > "events": [ > { > "time": "2018-07-04 10:00:49.474651", > "event": "initiated" > }, > { > "time": "2018-07-04 10:00:49.475829", > "event": "queued_for_pg" > }, > { > "time": "2018-07-04 10:00:49.475845", > "event": "reached_pg" > }, > { > "time": "2018-07-04 10:00:49.475881", > "event": "started" > }, > { > "time": "2018-07-04 10:00:49.475923", > "event": "waiting for subops from 5,26" > }, > { > "time": "2018-07-04 10:00:49.484417", > "event": "op_commit" > }, > { > "time": "2018-07-04 10:00:49.484420", > "event": "op_applied" > }, > { > "time": "2018-07-04 10:00:49.485539", > "event": "sub_op_commit_rec from 5" > } > ] > } > } > ], > "num_ops": 20 > } > > can someone explain me how i can find the source of the Problem ? > > kind regards > > Ben > _______________________________________________ > ceph-users mailing list > ceph-users@lists.ceph.com > http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com >
_______________________________________________ ceph-users mailing list ceph-users@lists.ceph.com http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
