[arch-projects] [namcap] [PATCH] Add py_mtime rule
This is a patch inspired by
https://www.archlinux.org/todo/fix-bad-mtime-pycpyo-files-for-python-packages/
It checks packages with py/pyc/pyo files for time conflicts. When a
py file is newer than a pyc/pyo python locks up a little while
figuring out what to do.
Sometime it is our fault for copying files in the wrong order or doing
a sed after the pyc was generated. Sometimes it is upstream's fault;
many setup.py scripts copy them in the wrong order.
The test checks both the tar file and the mtree logs. If there is a
time error in the tar file, it will affect every installation. If the
error only appears in the mtree file, then the package will be fine
but there is a chance that you won't get as lucky the next time the
package is built. The tar case generates an error and the mtree case
generates a warning.
My apologies for including the patch as an attachment. I tried to
send this with 'git send-email' but every time Mailmain said "Message
rejected by filter rule match"
-Kyle
http://kmkeen.com
From 62d8b40985741ca2358f4de3b0fd97fd02b49625 Mon Sep 17 00:00:00 2001
From: Kyle Keen
Date: Thu, 14 Nov 2013 21:46:36 -0500
Subject: [PATCH] Add py_mtime rule
Signed-off-by: Kyle Keen
---
Namcap/rules/__init__.py | 1 +
Namcap/rules/py_mtime.py | 129 +++
Namcap/util.py | 28 ++
namcap-tags | 3 ++
4 files changed, 161 insertions(+)
create mode 100644 Namcap/rules/py_mtime.py
diff --git a/Namcap/rules/__init__.py b/Namcap/rules/__init__.py
index f7780d2..8dc4e68 100644
--- a/Namcap/rules/__init__.py
+++ b/Namcap/rules/__init__.py
@@ -42,6 +42,7 @@ from . import (
missingbackups,
perllocal,
permissions,
+ py_mtime,
rpath,
scrollkeeper,
shebangdepends,
diff --git a/Namcap/rules/py_mtime.py b/Namcap/rules/py_mtime.py
new file mode 100644
index 000..aaff238
--- /dev/null
+++ b/Namcap/rules/py_mtime.py
@@ -0,0 +1,129 @@
+#
+# namcap rules - py_mtime
+# Copyright (C) 2013 Kyle Keen
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+#
+
+"""
+Check for py timestamps that are ahead of pyc/pyo timestamps
+"""
+
+import os
+from Namcap.util import load_mtree
+from Namcap.ruleclass import *
+
+def _quick_filter(names):
+ "can this package be skipped outright"
+ if not names:
+ return True
+ found_py = any(n.endswith('.py') for n in names)
+ found_pyc = any(n.endswith('.pyc') for n in names)
+ found_pyo = any(n.endswith('.pyo') for n in names)
+ if found_py and found_pyc:
+ return False
+ if found_py and found_pyo:
+ return False
+ return True
+
+def _tar_timestamps(tar):
+ "takes a tar object"
+ return dict((m.name, m.mtime) for m in tar.getmembers())
+
+def _mtree_timestamps(tar):
+ "takes a tar object"
+ return dict((h, a['time']) for h,a in load_mtree(tar) if 'time' in a)
+
+def _generic_timestamps(tar):
+ "works for mtree and tar"
+ if '.MTREE' in tar.getnames():
+ return _mtree_timestamps(tar)
+ return _tar_timestamps(tar)
+
+def _try_mtree(tar):
+ "returns True if good, False if bad, None if N/A"
+ if '.MTREE' not in tar.getnames():
+ return None
+ stamps = _mtree_timestamps(tar)
+ if _quick_filter(stamps.keys()):
+ return True
+ return not _mtime_filter(stamps)
+
+def _try_tar(tar):
+ "returns True if good, False if bad"
+ names = tar.getnames()
+ if _quick_filter(names):
+ return True
+ mtimes = _tar_timestamps(tar)
+ return not _mtime_filter(mtimes)
+
+def _split_all(path):
+ "like os.path.split but splits every directory"
+ p2 = path
+ dirs = []
+ while p2 and p2 != '/':
+ p2,p3 = os.path.split(p2)
+ dirs.insert(0, p3)
+ #dirs.insert(0, '/')
+ return dirs
+
+def _source_py(path):
+ "given a pyc/pyo, return the source path"
+ if not path.endswith('.pyc') and not path.endswith('.pyo'):
+ return None
+ path = path[:-1]
+ # handle py2
+ if '__pycache__' not in path:
+ return path
+ # handle py3
+ splitup = _split_all(path)
+ if splitup[-2] != '__pycache__':
+ return None
+ splitup.pop(-2)
+ f = splitup[-1]
+ f = f.split('.')
+ f.pop(-2)
+ splitup[-1] = '.'.join(f)
+ return os.path.join(*splitup)
+
+def _mtime_filter(mtimes):
+ "return list of bad py file names"
+ bad = []
+ for name, mt2 in mtimes.items():
+ if not name.endswith('.pyc') and not name.endswith('.pyo'):
+ continue
+ source_name = _source_py(name
[arch-projects] [namcap] [PATCH] Add py_mtime rule
Signed-off-by: Kyle Keen
---
Namcap/rules/__init__.py | 1 +
Namcap/rules/py_mtime.py | 129 +++
Namcap/util.py | 28 ++
namcap-tags | 3 ++
4 files changed, 161 insertions(+)
create mode 100644 Namcap/rules/py_mtime.py
diff --git a/Namcap/rules/__init__.py b/Namcap/rules/__init__.py
index f7780d2..8dc4e68 100644
--- a/Namcap/rules/__init__.py
+++ b/Namcap/rules/__init__.py
@@ -42,6 +42,7 @@ from . import (
missingbackups,
perllocal,
permissions,
+ py_mtime,
rpath,
scrollkeeper,
shebangdepends,
diff --git a/Namcap/rules/py_mtime.py b/Namcap/rules/py_mtime.py
new file mode 100644
index 000..aaff238
--- /dev/null
+++ b/Namcap/rules/py_mtime.py
@@ -0,0 +1,129 @@
+#
+# namcap rules - py_mtime
+# Copyright (C) 2013 Kyle Keen
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+#
+
+"""
+Check for py timestamps that are ahead of pyc/pyo timestamps
+"""
+
+import os
+from Namcap.util import load_mtree
+from Namcap.ruleclass import *
+
+def _quick_filter(names):
+ "can this package be skipped outright"
+ if not names:
+ return True
+ found_py = any(n.endswith('.py') for n in names)
+ found_pyc = any(n.endswith('.pyc') for n in names)
+ found_pyo = any(n.endswith('.pyo') for n in names)
+ if found_py and found_pyc:
+ return False
+ if found_py and found_pyo:
+ return False
+ return True
+
+def _tar_timestamps(tar):
+ "takes a tar object"
+ return dict((m.name, m.mtime) for m in tar.getmembers())
+
+def _mtree_timestamps(tar):
+ "takes a tar object"
+ return dict((h, a['time']) for h,a in load_mtree(tar) if 'time' in a)
+
+def _generic_timestamps(tar):
+ "works for mtree and tar"
+ if '.MTREE' in tar.getnames():
+ return _mtree_timestamps(tar)
+ return _tar_timestamps(tar)
+
+def _try_mtree(tar):
+ "returns True if good, False if bad, None if N/A"
+ if '.MTREE' not in tar.getnames():
+ return None
+ stamps = _mtree_timestamps(tar)
+ if _quick_filter(stamps.keys()):
+ return True
+ return not _mtime_filter(stamps)
+
+def _try_tar(tar):
+ "returns True if good, False if bad"
+ names = tar.getnames()
+ if _quick_filter(names):
+ return True
+ mtimes = _tar_timestamps(tar)
+ return not _mtime_filter(mtimes)
+
+def _split_all(path):
+ "like os.path.split but splits every directory"
+ p2 = path
+ dirs = []
+ while p2 and p2 != '/':
+ p2,p3 = os.path.split(p2)
+ dirs.insert(0, p3)
+ #dirs.insert(0, '/')
+ return dirs
+
+def _source_py(path):
+ "given a pyc/pyo, return the source path"
+ if not path.endswith('.pyc') and not path.endswith('.pyo'):
+ return None
+ path = path[:-1]
+ # handle py2
+ if '__pycache__' not in path:
+ return path
+ # handle py3
+ splitup = _split_all(path)
+ if splitup[-2] != '__pycache__':
+ return None
+ splitup.pop(-2)
+ f = splitup[-1]
+ f = f.split('.')
+ f.pop(-2)
+ splitup[-1] = '.'.join(f)
+ return os.path.join(*splitup)
+
+def _mtime_filter(mtimes):
+ "return list of bad py file names"
+ bad = []
+ for name, mt2 in mtimes.items():
+ if not name.endswith('.pyc') and not name.endswith('.pyo'):
+ continue
+ source_name = _source_py(name)
+ if source_name not in mtimes:
+ continue
+ mt1 = mtimes[source_name]
+ if mt1 > mt2:
+ bad.append(source_name)
+ return bad
+
+class package(TarballRule):
+ name = "py_mtime"
+ description = "Check for py timestamps that are ahead of pyc/pyo
timestamps"
+ def analyze(self, pkginfo, tar):
+ mtree_status = _try_mtree(tar)
+ tar_status = _try_tar(tar)
+ if mtree_status == False and tar_status:
+ # mtree only
+ self.warning = [('py-mtime-mtree-warning', ())]
+ elif not tar_stat
Re: [arch-projects] [namcap] [PATCH] Add py_mtime rule
On 08/27/14 at 01:18pm, Kyle Keen wrote:
> Signed-off-by: Kyle Keen
> ---
> Namcap/rules/__init__.py | 1 +
> Namcap/rules/py_mtime.py | 129
> +++
> Namcap/util.py | 28 ++
> namcap-tags | 3 ++
> 4 files changed, 161 insertions(+)
> create mode 100644 Namcap/rules/py_mtime.py
>
> diff --git a/Namcap/rules/__init__.py b/Namcap/rules/__init__.py
> index f7780d2..8dc4e68 100644
> --- a/Namcap/rules/__init__.py
> +++ b/Namcap/rules/__init__.py
> @@ -42,6 +42,7 @@ from . import (
>missingbackups,
>perllocal,
>permissions,
> + py_mtime,
>rpath,
>scrollkeeper,
>shebangdepends,
> diff --git a/Namcap/rules/py_mtime.py b/Namcap/rules/py_mtime.py
> new file mode 100644
> index 000..aaff238
> --- /dev/null
> +++ b/Namcap/rules/py_mtime.py
> @@ -0,0 +1,129 @@
> +#
> +# namcap rules - py_mtime
> +# Copyright (C) 2013 Kyle Keen
> +#
> +# This program is free software; you can redistribute it and/or modify
> +# it under the terms of the GNU General Public License as published by
> +# the Free Software Foundation; either version 2 of the License, or
> +# (at your option) any later version.
> +#
> +# This program is distributed in the hope that it will be useful,
> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
> +# GNU General Public License for more details.
> +#
> +# You should have received a copy of the GNU General Public License
> +# along with this program; if not, write to the Free Software
> +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
> +#
> +
> +"""
> +Check for py timestamps that are ahead of pyc/pyo timestamps
> +"""
> +
> +import os
> +from Namcap.util import load_mtree
> +from Namcap.ruleclass import *
> +
> +def _quick_filter(names):
> + "can this package be skipped outright"
> + if not names:
> + return True
> + found_py = any(n.endswith('.py') for n in names)
> + found_pyc = any(n.endswith('.pyc') for n in names)
> + found_pyo = any(n.endswith('.pyo') for n in names)
> + if found_py and found_pyc:
> + return False
> + if found_py and found_pyo:
> + return False
> + return True
> +
> +def _tar_timestamps(tar):
> + "takes a tar object"
> + return dict((m.name, m.mtime) for m in tar.getmembers())
Since pyalpm is a Python 3 library, we can use dictionary comprehension
here and in the other cases where you use dict().
> +
> +def _mtree_timestamps(tar):
> + "takes a tar object"
> + return dict((h, a['time']) for h,a in load_mtree(tar) if 'time' in a)
> +
> +def _generic_timestamps(tar):
> + "works for mtree and tar"
> + if '.MTREE' in tar.getnames():
> + return _mtree_timestamps(tar)
> + return _tar_timestamps(tar)
> +
> +def _try_mtree(tar):
> + "returns True if good, False if bad, None if N/A"
> + if '.MTREE' not in tar.getnames():
> + return None
> + stamps = _mtree_timestamps(tar)
> + if _quick_filter(stamps.keys()):
> + return True
> + return not _mtime_filter(stamps)
> +
> +def _try_tar(tar):
> + "returns True if good, False if bad"
> + names = tar.getnames()
> + if _quick_filter(names):
> + return True
> + mtimes = _tar_timestamps(tar)
> + return not _mtime_filter(mtimes)
> +
> +def _split_all(path):
> + "like os.path.split but splits every directory"
> + p2 = path
> + dirs = []
> + while p2 and p2 != '/':
> + p2,p3 = os.path.split(p2)
> + dirs.insert(0, p3)
> + #dirs.insert(0, '/')
> + return dirs
> +
> +def _source_py(path):
> + "given a pyc/pyo, return the source path"
> + if not path.endswith('.pyc') and not path.endswith('.pyo'):
> + return None
> + path = path[:-1]
> + # handle py2
> + if '__pycache__' not in path:
> + return path
> + # handle py3
> + splitup = _split_all(path)
> + if splitup[-2] != '__pycache__':
> + return None
> + splitup.pop(-2)
> + f = splitup[-1]
> + f = f.split('.')
> + f.pop(-2)
> + splitup[-1] = '.'.join(f)
> + return os.path.join(*splitup)
> +
> +def _mtime_filter(mtimes):
> + "return list of bad py file names"
> + bad = []
> + for name, mt2 in mtimes.items():
> + if not name.endswith('.pyc') and not name.endswith('.pyo'):
> + continue
> + source_name = _source_py(name)
> + if source_name not in mtimes:
> + continue
> + mt1 = mtimes[source_name]
> + if mt1 > mt2:
> + bad.append(source_name)
> + return bad
> +
> +class package(TarballRule):
> + name = "py_mtime"
> + description = "Check for py timestamps that are ahead of pyc/pyo
> timestamps"
> + def an
