patch. Testing it now. On 16 December 2014 at 09:22, Marc Leeman <marc.lee...@gmail.com> wrote: > Just to let you know that the current version upstream is not enough > to solve the problem. I copied their version in the jessie release, > backported it to wheezy and tried it. > > After less than 24h, the system is stuck again, The problem can be traced to: > > @wrap_exceptions > def threads(self): > > Where there is still an open that does not close properly: > > 2014-12-16 09:21:21,185 - com.barco.mgs.TeamMgrM56094 - INFO - start - > Received request to start a team sid T10621 - wid W90616 > 2014-12-16 09:21:21,185 - com.barco.mgs.TeamMgrM56094 - ERROR - start > - [' File "/usr/lib/pymodules/python2.7/Mgs2/Team/TeamMgr.py", line > 831, in start\n limit = status.cpu_percent\n', ' File > "/usr/lib/pymodules/python2.7/Mgs2/SystemStatusBase.py", line 346, in > cpu_percent\n', ' File > "/usr/lib/pymodules/python2.7/Mgs2/SystemStatusBase.py", line 349, in > get_cpu_percent\n', ' File > "/usr/lib/python2.7/dist-packages/psutil/__init__.py", line 1509, in > cpu_percent\n _last_cpu_times = cpu_times()\n', ' File > "/usr/lib/python2.7/dist-packages/psutil/__init__.py", line 1439, in > cpu_times\n return _psplatform.cpu_times()\n', ' File > "/usr/lib/python2.7/dist-packages/psutil/_pslinux.py", line 215, in > cpu_times\n with open(\'/proc/stat\', \'rb\') as f:\n'] > 2014-12-16 09:21:21,186 - com.barco.mgs.TeamMgrM56094 - ERROR - start > - [Errno 24] Too many open files: '/proc/stat' > 2014-12-16 09:21:21,186 - com.barco.mgs.TeamMgrM56094 - ERROR - start > - Error during worker creation [Errno 24] Too many open files: > '/proc/stat' > > > > <...snip...> > rtsp_mgs. 26609 29136 root txt unknown > /proc/26609/task/29136/exe (readlink: Permission denied) > rtsp_mgs. 26609 29136 root NOFD > /proc/26609/task/29136/fd (opendir: Permission denied) > rtsp_mgs. 26609 29144 root cwd unknown > /proc/26609/task/29144/cwd (readlink: Permission denied) > rtsp_mgs. 26609 29144 root rtd unknown > /proc/26609/task/29144/root (readlink: Permission denied) > rtsp_mgs. 26609 29144 root txt unknown > /proc/26609/task/29144/exe (readlink: Permission denied) > rtsp_mgs. 26609 29144 root NOFD > /proc/26609/task/29144/fd (opendir: Permission denied) > rtsp_mgs. 26609 29147 root cwd unknown > /proc/26609/task/29147/cwd (readlink: Permission denied) > rtsp_mgs. 26609 29147 root rtd unknown > /proc/26609/task/29147/root (readlink: Permission denied) > rtsp_mgs. 26609 29147 root txt unknown > /proc/26609/task/29147/exe (readlink: Permission denied) > rtsp_mgs. 26609 29147 root NOFD > /proc/26609/task/29147/fd (opendir: Permission denied) > rtsp_mgs. 26609 29148 root cwd unknown > /proc/26609/task/29148/cwd (readlink: Permission denied) > rtsp_mgs. 26609 29148 root rtd unknown > /proc/26609/task/29148/root (readlink: Permission denied) > rtsp_mgs. 26609 29148 root txt unknown > /proc/26609/task/29148/exe (readlink: Permission denied) > rtsp_mgs. 26609 29148 root NOFD > /proc/26609/task/29148/fd (opendir: Permission denied) > rtsp_mgs. 26609 29149 root cwd unknown > /proc/26609/task/29149/cwd (readlink: Permission denied) > rtsp_mgs. 26609 29149 root rtd unknown > /proc/26609/task/29149/root (readlink: Permission denied) > rtsp_mgs. 26609 29149 root txt unknown > /proc/26609/task/29149/exe (readlink: Permission denied) > rtsp_mgs. 26609 29149 root NOFD > /proc/26609/task/29149/fd (opendir: Permission denied) > rtsp_mgs. 26609 29152 root cwd unknown > /proc/26609/task/29152/cwd (readlink: Permission denied) > rtsp_mgs. 26609 29152 root rtd unknown > /proc/26609/task/29152/root (readlink: Permission denied) > rtsp_mgs. 26609 29152 root txt unknown > /proc/26609/task/29152/exe (readlink: Permission denied) > rtsp_mgs. 26609 29152 root NOFD > /proc/26609/task/29152/fd (opendir: Permission denied) > rtsp_mgs. 26609 29153 root cwd unknown > /proc/26609/task/29153/cwd (readlink: Permission denied) > rtsp_mgs. 26609 29153 root rtd unknown > /proc/26609/task/29153/root (readlink: Permission denied) > rtsp_mgs. 26609 29153 root txt unknown > /proc/26609/task/29153/exe (readlink: Permission denied) > rtsp_mgs. 26609 29153 root NOFD > /proc/26609/task/29153/fd (opendir: Permission denied) > rtsp_mgs. 26609 29155 root cwd unknown > /proc/26609/task/29155/cwd (readlink: Permission denied) > rtsp_mgs. 26609 29155 root rtd unknown > /proc/26609/task/29155/root (readlink: Permission denied) > rtsp_mgs. 26609 29155 root txt unknown > /proc/26609/task/29155/exe (readlink: Permission denied) > rtsp_mgs. 26609 29155 root NOFD > /proc/26609/task/29155/fd (opendir: Permission denied) > rtsp_mgs. 26609 29157 root cwd unknown > /proc/26609/task/29157/cwd (readlink: Permission denied) > rtsp_mgs. 26609 29157 root rtd unknown > /proc/26609/task/29157/root (readlink: Permission denied) > rtsp_mgs. 26609 29157 root txt unknown > /proc/26609/task/29157/exe (readlink: Permission denied) > rtsp_mgs. 26609 29157 root NOFD > /proc/26609/task/29157/fd (opendir: Permission denied) > rtsp_mgs. 26609 29161 root cwd unknown > /proc/26609/task/29161/cwd (readlink: Permission denied) > rtsp_mgs. 26609 29161 root rtd unknown > /proc/26609/task/29161/root (readlink: Permission denied) > rtsp_mgs. 26609 29161 root txt unknown > /proc/26609/task/29161/exe (readlink: Permission denied) > rtsp_mgs. 26609 29161 root NOFD > /proc/26609/task/29161/fd (opendir: Permission denied) > rtsp_mgs. 26609 29165 root cwd unknown > /proc/26609/task/29165/cwd (readlink: Permission denied) > <...snip...> > > On 8 December 2014 at 17:33, Sandro Tosi <mo...@debian.org> wrote: >> control: forward -1 https://github.com/giampaolo/psutil/issues/556 >> >> On Mon, Dec 8, 2014 at 4:28 PM, Marc Leeman <marc.lee...@gmail.com> wrote: >>>> it seems those were addressed in >>>> https://github.com/giampaolo/psutil/commit/3760d9593524e553e9e012c6b63cd802e3b25a46 >>>> but it was not released in the last upstream version - asked upstream >>>> for clarification on when they will be released. >>> >>> I don't think the commit solves all the problems: when looking at the >>> commit, I still see a lot of returns in a try block, while the file >>> handle is only closed in the finally block. This means that the >>> control flow returns before closing. It does seem to solve some open >>> file handles when something fails in the try. >>> >>> The problem here popped up after a weekend and there were no errors to >>> be seen during operation before hitting the file handle limit. >> >> yup agreed, reported upstream as referred above. >> >> Regards, >> -- >> Sandro Tosi (aka morph, morpheus, matrixhasu) >> My website: http://matrixhasu.altervista.org/ >> Me at Debian: http://wiki.debian.org/SandroTosi
Description: A lot of filehandles are left dangling when threads go missing . python-psutil (2.1.1-1+barco4) unstable; urgency=low . * more filehandles were not close properly in psutil Author: Marc Leeman <marc.lee...@barco.com>
--- Bug-Debian: http://bugs.debian.org/772543 Last-Update: 2014-12-16 Index: python-psutil-2.1.1/psutil/_pslinux.py =================================================================== --- python-psutil-2.1.1.orig/psutil/_pslinux.py 2014-12-16 09:04:44.000000000 +0000 +++ python-psutil-2.1.1/psutil/_pslinux.py 2014-12-16 09:08:37.692699822 +0000 @@ -669,16 +669,20 @@ self.pid = pid self._name = None + def __py3_name(self): + with open("/proc/%s/stat" % (self.pid), "rt", encoding=DEFAULT_ENCODING) as f: + return f.read().split(' ')[1].replace('(', '').replace(')', '') + + def __py2_name(self): + with open("/proc/%s/stat" % (self.pid), "rt") as f: + return f.read().split(' ')[1].replace('(', '').replace(')', '') + @wrap_exceptions def name(self): - fname = "/proc/%s/stat" % self.pid if PY3: - f = open(fname, "rt", encoding=DEFAULT_ENCODING) + return self.__py3_name() else: - f = open(fname, "rt") - with f: - # XXX - gets changed later and probably needs refactoring - return f.read().split(' ')[1].replace('(', '').replace(')', '') + return self.__py2_name() def exe(self): try: @@ -707,16 +711,20 @@ exe = exe[:-10] return exe + def __py3_cmdline(self): + with open("/proc/%s/cmdline" % (self.pid), "rt", encoding=DEFAULT_ENCODING) as f: + return [x for x in f.read().split('\x00') if x] + + def __py2_cmdline(self): + with open("/proc/%s/cmdline" % (self.pid), "rt") as f: + return [x for x in f.read().split('\x00') if x] + @wrap_exceptions def cmdline(self): - fname = "/proc/%s/cmdline" % self.pid if PY3: - f = open(fname, "rt", encoding=DEFAULT_ENCODING) + return self.__py3_cmdline() else: - f = open(fname, "rt") - with f: - # return the args as a list - return [x for x in f.read().split('\x00') if x] + return self.__py2_cmdline() @wrap_exceptions def terminal(self): @@ -820,73 +828,65 @@ Fields are explained in 'man proc'; here is an updated (Apr 2012) version: http://goo.gl/fmebo """ - f = None try: - f = open("/proc/%s/smaps" % self.pid, "rt") - first_line = f.readline() - current_block = [first_line] - - def get_blocks(): - data = {} - for line in f: - fields = line.split(None, 5) - if not fields[0].endswith(':'): - # new block section - yield (current_block.pop(), data) - current_block.append(line) - else: - try: - data[fields[0]] = int(fields[1]) * 1024 - except ValueError: - if fields[0].startswith('VmFlags:'): - # see issue #369 - continue - else: - raise ValueError("don't know how to inte" - "rpret line %r" % line) - yield (current_block.pop(), data) - - if first_line: # smaps file can be empty - for header, data in get_blocks(): - hfields = header.split(None, 5) - try: - addr, perms, offset, dev, inode, path = hfields - except ValueError: - addr, perms, offset, dev, inode, path = \ - hfields + [''] - if not path: - path = '[anon]' - else: - path = path.strip() - yield (addr, perms, path, - data['Rss:'], - data.get('Size:', 0), - data.get('Pss:', 0), - data.get('Shared_Clean:', 0), - data.get('Shared_Dirty:', 0), - data.get('Private_Clean:', 0), - data.get('Private_Dirty:', 0), - data.get('Referenced:', 0), - data.get('Anonymous:', 0), - data.get('Swap:', 0)) - f.close() - except EnvironmentError as err: - # XXX - Can't use wrap_exceptions decorator as we're - # returning a generator; this probably needs some - # refactoring in order to avoid this code duplication. - if f is not None: - f.close() - if err.errno in (errno.ENOENT, errno.ESRCH): - raise NoSuchProcess(self.pid, self._name) - if err.errno in (errno.EPERM, errno.EACCES): - raise AccessDenied(self.pid, self._name) - raise - except: - if f is not None: - f.close() - raise - f.close() - + with open("/proc/%s/smaps" % self.pid, "rt") as f: + first_line = f.readline() + current_block = [first_line] + + def get_blocks(): + data = {} + for line in f: + fields = line.split(None, 5) + if not fields[0].endswith(':'): + # new block section + yield (current_block.pop(), data) + current_block.append(line) + else: + try: + data[fields[0]] = int(fields[1]) * 1024 + except ValueError: + if fields[0].startswith('VmFlags:'): + # see issue #369 + continue + else: + raise ValueError("don't know how to inte" + "rpret line %r" % line) + yield (current_block.pop(), data) + + if first_line: # smaps file can be empty + for header, data in get_blocks(): + hfields = header.split(None, 5) + try: + addr, perms, offset, dev, inode, path = hfields + except ValueError: + addr, perms, offset, dev, inode, path = \ + hfields + [''] + if not path: + path = '[anon]' + else: + path = path.strip() + yield (addr, perms, path, + data['Rss:'], + data.get('Size:', 0), + data.get('Pss:', 0), + data.get('Shared_Clean:', 0), + data.get('Shared_Dirty:', 0), + data.get('Private_Clean:', 0), + data.get('Private_Dirty:', 0), + data.get('Referenced:', 0), + data.get('Anonymous:', 0), + data.get('Swap:', 0)) + except EnvironmentError as err: + # XXX - Can't use wrap_exceptions decorator as we're + # returning a generator; this probably needs some + # refactoring in order to avoid this code duplication. + if err.errno in (errno.ENOENT, errno.ESRCH): + raise NoSuchProcess(self.pid, self._name) + if err.errno in (errno.EPERM, errno.EACCES): + raise AccessDenied(self.pid, self._name) + raise + except: + raise else: def memory_maps(self): msg = "couldn't find /proc/%s/smaps; kernel < 2.6.14 or " \ @@ -934,7 +934,8 @@ hit_enoent = False for thread_id in thread_ids: try: - f = open("/proc/%s/task/%s/stat" % (self.pid, thread_id), 'rb') + with open("/proc/%s/task/%s/stat" % (self.pid, thread_id), 'rb') as f: + st = f.read().strip() except EnvironmentError as err: if err.errno == errno.ENOENT: # no such file or directory; it means thread @@ -942,8 +943,6 @@ hit_enoent = True continue raise - with f: - st = f.read().strip() # ignore the first two values ("pid (exe)") st = st[st.find(b')') + 2:] values = st.split(b' ')
