Nir Soffer has uploaded a new change for review.
Change subject: utils: Add execCmd sync mode failing stress tests
......................................................................
utils: Add execCmd sync mode failing stress tests
This patch adds couple of tests that stress exceCmd sync mode under high
concurrency. I added the tests to ensure that the next patch, avoiding
AsyncProc in sync mode, does not introduce any regressions.
When run with the next patch:
- Python cpu usage reach 230%
- Load average in top show up to 60
- utilsTests.py completes in about 90 seconds.
- 10 of 10 runs passed
When run with the current execCmd (using AsyncProc):
- test_read_stderr pass (but slower)
- test_read_stdout_stderr cause extreme load and finally pass after
couple of minutes.
- test_write_stdin_read_stderr fail with OSError
- Python cpu usage reach 390%
- Load average in top show up to 190
- utilsTests.py completes in abut 450 seconds
- 3 of 3 runs failed
It seems that the current version of execCmd is not only more expensive
but less reliable than Python builtin Popen under these exetreme load.
Change-Id: I57ebd5faf028ccfc928a8b2e7ae47008f0970c23
Signed-off-by: Nir Soffer <nsof...@redhat.com>
---
M tests/utilsTests.py
1 file changed, 79 insertions(+), 0 deletions(-)
git pull ssh://gerrit.ovirt.org:29418/vdsm refs/changes/84/27784/1
diff --git a/tests/utilsTests.py b/tests/utilsTests.py
index 731a8f3..c62fc7b 100644
--- a/tests/utilsTests.py
+++ b/tests/utilsTests.py
@@ -22,10 +22,12 @@
import contextlib
import errno
import logging
+import threading
from testrunner import VdsmTestCase as TestCaseBase
from testrunner import permutations, expandPermutations
from testValidation import checkSudo
+from testValidation import stresstest
from vdsm import utils
from vdsm import constants
import time
@@ -484,3 +486,80 @@
sudo=True)
self.assertEquals(rc, 0)
self.assertEquals(int(out[0].split()[2]), 0)
+
+
+class ExecCmdStressTest(TestCaseBase):
+
+ CONCURRENCY = 200
+ START_DELAY = 2.0
+ FUNC_DELAY = 0.01
+ FUNC_CALLS = 10
+ BLOCK_SIZE = 4096
+ BLOCK_COUNT = 256
+
+ def setUp(self):
+ self.workers = []
+ self.start = threading.Event()
+ self.done = [False] * self.CONCURRENCY
+
+ @stresstest
+ def test_read_stderr(self):
+ self.check(self.read_stderr)
+
+ @stresstest
+ def test_read_stdout_stderr(self):
+ self.check(self.read_stdout_stderr)
+
+ @stresstest
+ def test_write_stdin_read_stderr(self):
+ data = 'x' * self.BLOCK_SIZE * self.BLOCK_COUNT
+ self.check(self.write_stdin_read_stderr, data)
+
+ def check(self, func, *args):
+ for i in xrange(self.CONCURRENCY):
+ t = threading.Thread(target=self.worker, args=(i, func, args))
+ t.daemon = True
+ self.workers.append(t)
+ t.start()
+ time.sleep(self.START_DELAY)
+ self.start.set()
+ for t in self.workers:
+ t.join()
+ self.assertTrue(all(self.done))
+
+ def worker(self, i, func, args):
+ self.start.wait()
+ for n in range(self.FUNC_CALLS):
+ func(*args)
+ time.sleep(self.FUNC_DELAY)
+ self.done[i] = True
+
+ def read_stderr(self):
+ cmd = [constants.EXT_DD, 'if=/dev/zero', 'of=/dev/null',
+ 'bs=%d' % self.BLOCK_SIZE, 'count=%d' % self.BLOCK_COUNT]
+ rc, _, err = utils.execCmd(cmd, raw=True)
+ if rc != 0:
+ raise Exception("Process failed with exit code %d", rc)
+ if err == '':
+ raise Exception("No data from stderr")
+
+ def read_stdout_stderr(self):
+ cmd = [constants.EXT_DD, 'if=/dev/zero', 'bs=%d' % self.BLOCK_SIZE,
+ 'count=%d' % self.BLOCK_COUNT]
+ rc, out, err = utils.execCmd(cmd, raw=True)
+ if rc != 0:
+ raise Exception("Process failed with exit code %d", rc)
+ if err == '':
+ raise Exception("No data from stderr")
+ size = 512 * 2048
+ if len(out) < size:
+ raise Exception("Partial read: %d/%d", len(out), size)
+
+ def write_stdin_read_stderr(self, data):
+ cmd = [constants.EXT_DD, 'of=/dev/null', 'bs=%d' % self.BLOCK_SIZE,
+ 'count=%d' % self.BLOCK_COUNT]
+ rc, _, err = utils.execCmd(cmd, raw=True, data=data)
+ if rc != 0:
+ raise Exception("Process failed with exit code %d", rc)
+ if err == '':
+ raise Exception("No data from stderr")
--
To view, visit http://gerrit.ovirt.org/27784
To unsubscribe, visit http://gerrit.ovirt.org/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I57ebd5faf028ccfc928a8b2e7ae47008f0970c23
Gerrit-PatchSet: 1
Gerrit-Project: vdsm
Gerrit-Branch: master
Gerrit-Owner: Nir Soffer <nsof...@redhat.com>
_______________________________________________
vdsm-patches mailing list
vdsm-patches@lists.fedorahosted.org
https://lists.fedorahosted.org/mailman/listinfo/vdsm-patches
