[arch-projects] [namcap] [PATCH] Add py_mtime rule
This is a patch inspired by https://www.archlinux.org/todo/fix-bad-mtime-pycpyo-files-for-python-packages/ It checks packages with py/pyc/pyo files for time conflicts. When a py file is newer than a pyc/pyo python locks up a little while figuring out what to do. Sometime it is our fault for copying files in the wrong order or doing a sed after the pyc was generated. Sometimes it is upstream's fault; many setup.py scripts copy them in the wrong order. The test checks both the tar file and the mtree logs. If there is a time error in the tar file, it will affect every installation. If the error only appears in the mtree file, then the package will be fine but there is a chance that you won't get as lucky the next time the package is built. The tar case generates an error and the mtree case generates a warning. My apologies for including the patch as an attachment. I tried to send this with 'git send-email' but every time Mailmain said "Message rejected by filter rule match" -Kyle http://kmkeen.com From 62d8b40985741ca2358f4de3b0fd97fd02b49625 Mon Sep 17 00:00:00 2001 From: Kyle Keen Date: Thu, 14 Nov 2013 21:46:36 -0500 Subject: [PATCH] Add py_mtime rule Signed-off-by: Kyle Keen --- Namcap/rules/__init__.py | 1 + Namcap/rules/py_mtime.py | 129 +++ Namcap/util.py | 28 ++ namcap-tags | 3 ++ 4 files changed, 161 insertions(+) create mode 100644 Namcap/rules/py_mtime.py diff --git a/Namcap/rules/__init__.py b/Namcap/rules/__init__.py index f7780d2..8dc4e68 100644 --- a/Namcap/rules/__init__.py +++ b/Namcap/rules/__init__.py @@ -42,6 +42,7 @@ from . import ( missingbackups, perllocal, permissions, + py_mtime, rpath, scrollkeeper, shebangdepends, diff --git a/Namcap/rules/py_mtime.py b/Namcap/rules/py_mtime.py new file mode 100644 index 000..aaff238 --- /dev/null +++ b/Namcap/rules/py_mtime.py @@ -0,0 +1,129 @@ +# +# namcap rules - py_mtime +# Copyright (C) 2013 Kyle Keen +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# + +""" +Check for py timestamps that are ahead of pyc/pyo timestamps +""" + +import os +from Namcap.util import load_mtree +from Namcap.ruleclass import * + +def _quick_filter(names): + "can this package be skipped outright" + if not names: + return True + found_py = any(n.endswith('.py') for n in names) + found_pyc = any(n.endswith('.pyc') for n in names) + found_pyo = any(n.endswith('.pyo') for n in names) + if found_py and found_pyc: + return False + if found_py and found_pyo: + return False + return True + +def _tar_timestamps(tar): + "takes a tar object" + return dict((m.name, m.mtime) for m in tar.getmembers()) + +def _mtree_timestamps(tar): + "takes a tar object" + return dict((h, a['time']) for h,a in load_mtree(tar) if 'time' in a) + +def _generic_timestamps(tar): + "works for mtree and tar" + if '.MTREE' in tar.getnames(): + return _mtree_timestamps(tar) + return _tar_timestamps(tar) + +def _try_mtree(tar): + "returns True if good, False if bad, None if N/A" + if '.MTREE' not in tar.getnames(): + return None + stamps = _mtree_timestamps(tar) + if _quick_filter(stamps.keys()): + return True + return not _mtime_filter(stamps) + +def _try_tar(tar): + "returns True if good, False if bad" + names = tar.getnames() + if _quick_filter(names): + return True + mtimes = _tar_timestamps(tar) + return not _mtime_filter(mtimes) + +def _split_all(path): + "like os.path.split but splits every directory" + p2 = path + dirs = [] + while p2 and p2 != '/': + p2,p3 = os.path.split(p2) + dirs.insert(0, p3) + #dirs.insert(0, '/') + return dirs + +def _source_py(path): + "given a pyc/pyo, return the source path" + if not path.endswith('.pyc') and not path.endswith('.pyo'): + return None + path = path[:-1] + # handle py2 + if '__pycache__' not in path: + return path + # handle py3 + splitup = _split_all(path) + if splitup[-2] != '__pycache__': + return None + splitup.pop(-2) + f = splitup[-1] + f = f.split('.') + f.pop(-2) + splitup[-1] = '.'.join(f) + return os.path.join(*splitup) + +def _mtime_filter(mtimes): + "return list of bad py file names" + bad = [] + for name, mt2 in mtimes.items(): + if not name.endswith('.pyc') and not name.endswith('.pyo'): + continue + source_name = _source_py(name
[arch-projects] [namcap] [PATCH] Add py_mtime rule
Signed-off-by: Kyle Keen --- Namcap/rules/__init__.py | 1 + Namcap/rules/py_mtime.py | 129 +++ Namcap/util.py | 28 ++ namcap-tags | 3 ++ 4 files changed, 161 insertions(+) create mode 100644 Namcap/rules/py_mtime.py diff --git a/Namcap/rules/__init__.py b/Namcap/rules/__init__.py index f7780d2..8dc4e68 100644 --- a/Namcap/rules/__init__.py +++ b/Namcap/rules/__init__.py @@ -42,6 +42,7 @@ from . import ( missingbackups, perllocal, permissions, + py_mtime, rpath, scrollkeeper, shebangdepends, diff --git a/Namcap/rules/py_mtime.py b/Namcap/rules/py_mtime.py new file mode 100644 index 000..aaff238 --- /dev/null +++ b/Namcap/rules/py_mtime.py @@ -0,0 +1,129 @@ +# +# namcap rules - py_mtime +# Copyright (C) 2013 Kyle Keen +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# + +""" +Check for py timestamps that are ahead of pyc/pyo timestamps +""" + +import os +from Namcap.util import load_mtree +from Namcap.ruleclass import * + +def _quick_filter(names): + "can this package be skipped outright" + if not names: + return True + found_py = any(n.endswith('.py') for n in names) + found_pyc = any(n.endswith('.pyc') for n in names) + found_pyo = any(n.endswith('.pyo') for n in names) + if found_py and found_pyc: + return False + if found_py and found_pyo: + return False + return True + +def _tar_timestamps(tar): + "takes a tar object" + return dict((m.name, m.mtime) for m in tar.getmembers()) + +def _mtree_timestamps(tar): + "takes a tar object" + return dict((h, a['time']) for h,a in load_mtree(tar) if 'time' in a) + +def _generic_timestamps(tar): + "works for mtree and tar" + if '.MTREE' in tar.getnames(): + return _mtree_timestamps(tar) + return _tar_timestamps(tar) + +def _try_mtree(tar): + "returns True if good, False if bad, None if N/A" + if '.MTREE' not in tar.getnames(): + return None + stamps = _mtree_timestamps(tar) + if _quick_filter(stamps.keys()): + return True + return not _mtime_filter(stamps) + +def _try_tar(tar): + "returns True if good, False if bad" + names = tar.getnames() + if _quick_filter(names): + return True + mtimes = _tar_timestamps(tar) + return not _mtime_filter(mtimes) + +def _split_all(path): + "like os.path.split but splits every directory" + p2 = path + dirs = [] + while p2 and p2 != '/': + p2,p3 = os.path.split(p2) + dirs.insert(0, p3) + #dirs.insert(0, '/') + return dirs + +def _source_py(path): + "given a pyc/pyo, return the source path" + if not path.endswith('.pyc') and not path.endswith('.pyo'): + return None + path = path[:-1] + # handle py2 + if '__pycache__' not in path: + return path + # handle py3 + splitup = _split_all(path) + if splitup[-2] != '__pycache__': + return None + splitup.pop(-2) + f = splitup[-1] + f = f.split('.') + f.pop(-2) + splitup[-1] = '.'.join(f) + return os.path.join(*splitup) + +def _mtime_filter(mtimes): + "return list of bad py file names" + bad = [] + for name, mt2 in mtimes.items(): + if not name.endswith('.pyc') and not name.endswith('.pyo'): + continue + source_name = _source_py(name) + if source_name not in mtimes: + continue + mt1 = mtimes[source_name] + if mt1 > mt2: + bad.append(source_name) + return bad + +class package(TarballRule): + name = "py_mtime" + description = "Check for py timestamps that are ahead of pyc/pyo timestamps" + def analyze(self, pkginfo, tar): + mtree_status = _try_mtree(tar) + tar_status = _try_tar(tar) + if mtree_status == False and tar_status: + # mtree only + self.warning = [('py-mtime-mtree-warning', ())] + elif not tar_stat
Re: [arch-projects] [namcap] [PATCH] Add py_mtime rule
On 08/27/14 at 01:18pm, Kyle Keen wrote: > Signed-off-by: Kyle Keen > --- > Namcap/rules/__init__.py | 1 + > Namcap/rules/py_mtime.py | 129 > +++ > Namcap/util.py | 28 ++ > namcap-tags | 3 ++ > 4 files changed, 161 insertions(+) > create mode 100644 Namcap/rules/py_mtime.py > > diff --git a/Namcap/rules/__init__.py b/Namcap/rules/__init__.py > index f7780d2..8dc4e68 100644 > --- a/Namcap/rules/__init__.py > +++ b/Namcap/rules/__init__.py > @@ -42,6 +42,7 @@ from . import ( >missingbackups, >perllocal, >permissions, > + py_mtime, >rpath, >scrollkeeper, >shebangdepends, > diff --git a/Namcap/rules/py_mtime.py b/Namcap/rules/py_mtime.py > new file mode 100644 > index 000..aaff238 > --- /dev/null > +++ b/Namcap/rules/py_mtime.py > @@ -0,0 +1,129 @@ > +# > +# namcap rules - py_mtime > +# Copyright (C) 2013 Kyle Keen > +# > +# This program is free software; you can redistribute it and/or modify > +# it under the terms of the GNU General Public License as published by > +# the Free Software Foundation; either version 2 of the License, or > +# (at your option) any later version. > +# > +# This program is distributed in the hope that it will be useful, > +# but WITHOUT ANY WARRANTY; without even the implied warranty of > +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > +# GNU General Public License for more details. > +# > +# You should have received a copy of the GNU General Public License > +# along with this program; if not, write to the Free Software > +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA > +# > + > +""" > +Check for py timestamps that are ahead of pyc/pyo timestamps > +""" > + > +import os > +from Namcap.util import load_mtree > +from Namcap.ruleclass import * > + > +def _quick_filter(names): > + "can this package be skipped outright" > + if not names: > + return True > + found_py = any(n.endswith('.py') for n in names) > + found_pyc = any(n.endswith('.pyc') for n in names) > + found_pyo = any(n.endswith('.pyo') for n in names) > + if found_py and found_pyc: > + return False > + if found_py and found_pyo: > + return False > + return True > + > +def _tar_timestamps(tar): > + "takes a tar object" > + return dict((m.name, m.mtime) for m in tar.getmembers()) Since pyalpm is a Python 3 library, we can use dictionary comprehension here and in the other cases where you use dict(). > + > +def _mtree_timestamps(tar): > + "takes a tar object" > + return dict((h, a['time']) for h,a in load_mtree(tar) if 'time' in a) > + > +def _generic_timestamps(tar): > + "works for mtree and tar" > + if '.MTREE' in tar.getnames(): > + return _mtree_timestamps(tar) > + return _tar_timestamps(tar) > + > +def _try_mtree(tar): > + "returns True if good, False if bad, None if N/A" > + if '.MTREE' not in tar.getnames(): > + return None > + stamps = _mtree_timestamps(tar) > + if _quick_filter(stamps.keys()): > + return True > + return not _mtime_filter(stamps) > + > +def _try_tar(tar): > + "returns True if good, False if bad" > + names = tar.getnames() > + if _quick_filter(names): > + return True > + mtimes = _tar_timestamps(tar) > + return not _mtime_filter(mtimes) > + > +def _split_all(path): > + "like os.path.split but splits every directory" > + p2 = path > + dirs = [] > + while p2 and p2 != '/': > + p2,p3 = os.path.split(p2) > + dirs.insert(0, p3) > + #dirs.insert(0, '/') > + return dirs > + > +def _source_py(path): > + "given a pyc/pyo, return the source path" > + if not path.endswith('.pyc') and not path.endswith('.pyo'): > + return None > + path = path[:-1] > + # handle py2 > + if '__pycache__' not in path: > + return path > + # handle py3 > + splitup = _split_all(path) > + if splitup[-2] != '__pycache__': > + return None > + splitup.pop(-2) > + f = splitup[-1] > + f = f.split('.') > + f.pop(-2) > + splitup[-1] = '.'.join(f) > + return os.path.join(*splitup) > + > +def _mtime_filter(mtimes): > + "return list of bad py file names" > + bad = [] > + for name, mt2 in mtimes.items(): > + if not name.endswith('.pyc') and not name.endswith('.pyo'): > + continue > + source_name = _source_py(name) > + if source_name not in mtimes: > + continue > + mt1 = mtimes[source_name] > + if mt1 > mt2: > + bad.append(source_name) > + return bad > + > +class package(TarballRule): > + name = "py_mtime" > + description = "Check for py timestamps that are ahead of pyc/pyo > timestamps" > + def an