Nir Soffer has uploaded a new change for review. Change subject: reppolot: Add mailbox dd commands graph ......................................................................
reppolot: Add mailbox dd commands graph Storage mailbox check for mail dd commands are performed only on the master domain, but since they are run every 3 seconds, they are good indicator for storage issues in the master domain. Checking logs for copy disk flow show that SPM check for mail commands become very slow (up to 6 seconds per command). This may cause extend operations slow down leading to pausing of a vm. Change-Id: I0fa6a0783b66959bc1a8dfb0abd4cae57b42c2af Relates-To: https://bugzilla.redhat.com/1295208 Signed-off-by: Nir Soffer <[email protected]> --- M contrib/repoplot 1 file changed, 29 insertions(+), 3 deletions(-) git pull ssh://gerrit.ovirt.org:29418/vdsm refs/changes/06/54106/1 diff --git a/contrib/repoplot b/contrib/repoplot index 2c3c6ae..3595282 100755 --- a/contrib/repoplot +++ b/contrib/repoplot @@ -61,10 +61,10 @@ parser.add_argument("--width", "-x", dest="width", type=int, help="output file width in inches (default 20)") parser.add_argument("--height", "-y", dest="height", type=int, - help="output file height in inches (default 15)") + help="output file height in inches (default 20)") parser.add_argument("files", nargs="+", help="vdsm log files to process") - parser.set_defaults(name=None, format="pdf", width=20, height=15) + parser.set_defaults(name=None, format="pdf", width=20, height=20) return parser.parse_args(args) @@ -75,6 +75,7 @@ def __init__(self): self.repostats = defaultdict(DomainStats) self.lvm_commands = defaultdict(CommandStats) + self.mailbox_commands = defaultdict(CommandStats) class DomainStats(object): @@ -114,6 +115,21 @@ # Thread-57::DEBUG::2016-02-17 # 19:26:33,933::lvm::286::Storage.Misc.excCmd::(cmd) SUCCESS: ... (add_command, "::Storage.Misc.excCmd::(cmd)", "lvm_commands"), + + # Match mailbox monitor check for mail dd commands (master domain + # only). + # + # Begin: + # mailbox.SPMMonitor::DEBUG::2016-02-17 + # 20:15:24,521::storage_mailbox::731::Storage.Misc.excCmd::(_checkForMail) + # /usr/bin/taskset --cpu-list 0-7 dd ... + # + # End: + # mailbox.SPMMonitor::DEBUG::2016-02-17 + # 20:15:24,547::storage_mailbox::731::Storage.Misc.excCmd::(_checkForMail) + # SUCCESS: ... + (add_command, "::Storage.Misc.excCmd::(_checkForMail)", + "mailbox_commands"), ] stats = Stats() @@ -186,7 +202,7 @@ def plot(stats, filename, size): - rows = 3 + rows = 4 columns = 1 pyplot.figure(figsize=size, dpi=300) @@ -231,6 +247,16 @@ for thread, cs in stats.lvm_commands.iteritems(): pyplot.plot(cs.timestamp, cs.runtime) + pyplot.subplot(rows, columns, 4) + pyplot.title("Mailbox commands") + pyplot.ylabel("runtime (seconds)") + pyplot.xlabel("time") + pyplot.grid(True) + pyplot.axis([lastcheck.index[0], lastcheck.index[-1], 0, 10]) + + for thread, cs in stats.mailbox_commands.iteritems(): + pyplot.plot(cs.timestamp, cs.runtime) + pyplot.savefig(filename, bbox_inches="tight") -- To view, visit https://gerrit.ovirt.org/54106 To unsubscribe, visit https://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I0fa6a0783b66959bc1a8dfb0abd4cae57b42c2af Gerrit-PatchSet: 1 Gerrit-Project: vdsm Gerrit-Branch: master Gerrit-Owner: Nir Soffer <[email protected]> _______________________________________________ vdsm-patches mailing list [email protected] https://lists.fedorahosted.org/mailman/listinfo/vdsm-patches
