Yaniv Bronhaim has uploaded a new change for review. Change subject: itmap starts threads for each argument causes too many oop operations ......................................................................
itmap starts threads for each argument causes too many oop operations This patch starts the threads until we reach to maxthreads, than we wait for one thread to end and move on until we loaded all threads. Change-Id: If2877c28b916475aead16d47d37522c2bf44cc16 Bug-Id: https://bugzilla.redhat.com/show_bug.cgi?id=523011 Signed-off-by: Yaniv Bronhaim <[email protected]> --- M vdsm/storage/fileSD.py M vdsm/storage/misc.py 2 files changed, 22 insertions(+), 3 deletions(-) git pull ssh://gerrit.ovirt.org:29418/vdsm refs/changes/45/8745/1 diff --git a/vdsm/storage/fileSD.py b/vdsm/storage/fileSD.py index 054fadb..5e50dc7 100644 --- a/vdsm/storage/fileSD.py +++ b/vdsm/storage/fileSD.py @@ -34,6 +34,7 @@ from remoteFileHandler import Timeout from persistentDict import PersistentDict, DictValidator from vdsm import constants +from vdsm.config import config import time import supervdsm import mount @@ -486,7 +487,8 @@ log.warn("Could not collect metadata file for domain path %s", possibleDomain, exc_info=True) - for res in misc.itmap(collectMetaFiles, mntList): + for res in misc.itmap(collectMetaFiles, mntList, + config.getint("irs", "process_pool_max_slots_per_domain")): if res is None: continue diff --git a/vdsm/storage/misc.py b/vdsm/storage/misc.py index 8641312..67433e9 100644 --- a/vdsm/storage/misc.py +++ b/vdsm/storage/misc.py @@ -1250,7 +1250,7 @@ raise exception -def itmap(func, iterable): +def itmap(func, iterable, maxthreads): """ Make an iterator that computes the function using arguments from the iterable. It works similar to tmap @@ -1267,12 +1267,29 @@ except Exception, e: respQueue.put(e) + def nextArg(iterable): + for arg in iterable: + yield arg + n = 0 - for arg in iterable: + if maxthreads < 1: + raise Exception("wrong input to function itmap: %s", maxthreads) + + for arg in nextArg(iterable): + if not maxthreads: + if n: + yield respQueue.get() + # if yield returns one thread stopped, so we can run another + # simultaneosly thread + maxthreads += 1 + n -= 1 + t = threading.Thread(target=wrapper, args=(arg,)) t.start() n += 1 + maxthreads -= 1 + # waiting for rest threads to end for i in xrange(n): yield respQueue.get() -- To view, visit http://gerrit.ovirt.org/8745 To unsubscribe, visit http://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: If2877c28b916475aead16d47d37522c2bf44cc16 Gerrit-PatchSet: 1 Gerrit-Project: vdsm Gerrit-Branch: master Gerrit-Owner: Yaniv Bronhaim <[email protected]> _______________________________________________ vdsm-patches mailing list [email protected] https://lists.fedorahosted.org/mailman/listinfo/vdsm-patches
