[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 64bb8e17a5c050671765626bbe45174e10b5a3a4 Author: Brian Dolbec gentoo org> AuthorDate: Tue May 3 17:38:19 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sat May 14 18:29:40 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=64bb8e17 repoman/modules/.../pkgmetadata.py: Improve whole document validation Change to using assertValid() and add the error causing validation to fail to the qa tracker error. pym/repoman/modules/scan/metadata/pkgmetadata.py | 10 -- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index 4921b6f..1594b27 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -206,8 +206,14 @@ class PkgMetadata(ScanBase, USEFlagChecks): # Only carry out if in package directory or check forced if not metadata_bad: validator = etree.XMLSchema(file=self.metadata_xsd) - if not validator.validate(_metadata_xml): - self.qatracker.add_error("metadata.bad", xpkg + "/metadata.xml") + try: + validator.assertValid(_metadata_xml) + except etree.DocumentInvalid as error: + self.qatracker.add_error( + "metadata.bad", + xpkg + "/metadata.xml: %s" + % (str(error)) + ) del metadata_bad self.muselist = frozenset(self.musedict) return False
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: b78d61dd3e4281a43523aea1a24e395a2674c3ab Author: Dirkjan Ochtman ochtman nl> AuthorDate: Tue May 3 06:33:47 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sat May 14 18:29:40 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b78d61dd repoman/modules/.../pkgmetadata.py: Early return on metadata parse errors pym/repoman/modules/scan/metadata/pkgmetadata.py | 148 --- 1 file changed, 75 insertions(+), 73 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index 3c1c2d0..3b48b8e 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -102,89 +102,91 @@ class PkgMetadata(ScanBase, USEFlagChecks): metadata_bad = True self.qatracker.add_error("metadata.bad", "%s/metadata.xml: %s" % (xpkg, e)) del e + self.muselist = frozenset(self.musedict) + return False + + if not hasattr(xml_parser, 'parser') or \ + sys.hexversion < 0x207 or \ + (sys.hexversion > 0x300 and sys.hexversion < 0x302): + # doctype is not parsed with python 2.6 or 3.1 + pass else: - if not hasattr(xml_parser, 'parser') or \ - sys.hexversion < 0x207 or \ - (sys.hexversion > 0x300 and sys.hexversion < 0x302): - # doctype is not parsed with python 2.6 or 3.1 - pass + if "XML_DECLARATION" not in xml_info: + self.qatracker.add_error( + "metadata.bad", "%s/metadata.xml: " + "xml declaration is missing on first line, " + "should be '%s'" % (xpkg, metadata_xml_declaration)) else: - if "XML_DECLARATION" not in xml_info: + xml_version, xml_encoding, xml_standalone = \ + xml_info["XML_DECLARATION"] + if xml_encoding is None or \ + xml_encoding.upper() != metadata_xml_encoding: + if xml_encoding is None: + encoding_problem = "but it is undefined" + else: + encoding_problem = "not '%s'" % xml_encoding self.qatracker.add_error( "metadata.bad", "%s/metadata.xml: " - "xml declaration is missing on first line, " - "should be '%s'" % (xpkg, metadata_xml_declaration)) - else: - xml_version, xml_encoding, xml_standalone = \ - xml_info["XML_DECLARATION"] - if xml_encoding is None or \ - xml_encoding.upper() != metadata_xml_encoding: - if xml_encoding is None: - encoding_problem = "but it is undefined" - else: - encoding_problem = "not '%s'" % xml_encoding - self.qatracker.add_error( - "metadata.bad", "%s/metadata.xml: " - "xml declaration encoding should be '%s', %s" % - (xpkg, metadata_xml_encoding, encoding_problem)) + "xml declaration encoding should be '%s', %s" % + (xpkg, metadata_xml_encoding, encoding_problem)) - if "DOCTYPE" not in xml_info: - metadata_bad = True - self.qatracker.add_error( - "metadata.bad", - "%s/metadata.xml: %s" % (xpkg, "DOCTYPE is missing")) - else: - doctype_name, doctype_system, doctype_pubid = \ - xml_info["DOCTYPE"] -
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 42b531b2810ec3e8e81348ebc4519a56e01a5a47 Author: Dirkjan Ochtman ochtman nl> AuthorDate: Mon Jan 25 18:33:32 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sat May 14 18:29:40 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=42b531b2 repoman/modules/.../pkgmetadata.py: Early return in metadata check code for readability pym/repoman/modules/scan/metadata/pkgmetadata.py | 190 --- 1 file changed, 96 insertions(+), 94 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index 5c6452a..3c1c2d0 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -83,112 +83,114 @@ class PkgMetadata(ScanBase, USEFlagChecks): # metadata.xml file check if "metadata.xml" not in checkdirlist: self.qatracker.add_error("metadata.missing", xpkg + "/metadata.xml") + self.muselist = frozenset(self.musedict) + return False + # metadata.xml parse check + metadata_bad = False + xml_info = {} + xml_parser = _XMLParser(xml_info, target=_MetadataTreeBuilder()) + + # read metadata.xml into memory + try: + _metadata_xml = xml.etree.ElementTree.parse( + _unicode_encode( + os.path.join(checkdir, "metadata.xml"), + encoding=_encodings['fs'], errors='strict'), + parser=xml_parser) + except (ExpatError, SyntaxError, EnvironmentError) as e: + metadata_bad = True + self.qatracker.add_error("metadata.bad", "%s/metadata.xml: %s" % (xpkg, e)) + del e else: - metadata_bad = False - xml_info = {} - xml_parser = _XMLParser(xml_info, target=_MetadataTreeBuilder()) - - # read metadata.xml into memory - try: - _metadata_xml = xml.etree.ElementTree.parse( - _unicode_encode( - os.path.join(checkdir, "metadata.xml"), - encoding=_encodings['fs'], errors='strict'), - parser=xml_parser) - except (ExpatError, SyntaxError, EnvironmentError) as e: - metadata_bad = True - self.qatracker.add_error("metadata.bad", "%s/metadata.xml: %s" % (xpkg, e)) - del e + if not hasattr(xml_parser, 'parser') or \ + sys.hexversion < 0x207 or \ + (sys.hexversion > 0x300 and sys.hexversion < 0x302): + # doctype is not parsed with python 2.6 or 3.1 + pass else: - if not hasattr(xml_parser, 'parser') or \ - sys.hexversion < 0x207 or \ - (sys.hexversion > 0x300 and sys.hexversion < 0x302): - # doctype is not parsed with python 2.6 or 3.1 - pass + if "XML_DECLARATION" not in xml_info: + self.qatracker.add_error( + "metadata.bad", "%s/metadata.xml: " + "xml declaration is missing on first line, " + "should be '%s'" % (xpkg, metadata_xml_declaration)) else: - if "XML_DECLARATION" not in xml_info: + xml_version, xml_encoding, xml_standalone = \ + xml_info["XML_DECLARATION"] + if xml_encoding is None or \ + xml_encoding.upper() != metadata_xml_encoding: + if xml_encoding is None: + encoding_problem = "but it is undefined" + else: + encoding_problem = "not '%s'" % xml_encoding self.qatracker.add_error( "metadata.bad"
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 1f4d8908748dc59c362d655db3f6fe3b4c913476 Author: Brian Dolbec gentoo org> AuthorDate: Thu May 5 16:04:54 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sat May 14 18:29:40 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=1f4d8908 repoman/modules/.../pkgmetadata.py: Add code to remove mostly duplicate errors Some types of errors produce two error messages cluttering up the output. The first error message is clearer, listing the possible option values allowed. This filters out the second error message for that same line and attribute. pym/repoman/modules/scan/metadata/pkgmetadata.py | 13 + 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index d8344c2..433551a 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -221,11 +221,16 @@ class PkgMetadata(ScanBase, USEFlagChecks): return uselist def _add_validate_errors(self, xpkg, log): + listed = set() for error in log: - self.qatracker.add_error( - "metadata.bad", - "%s/metadata.xml: line: %s, %s" - % (xpkg, error.line, error.message)) + msg_prefix = error.message.split(":",1)[0] + info = "%s %s" % (error.line, msg_prefix) + if info not in listed: + listed.add(info) + self.qatracker.add_error( + "metadata.bad", + "%s/metadata.xml: line: %s, %s" + % (xpkg, error.line, error.message)) @property def runInPkgs(self):
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 633cabfb9215633c554fd967b9875310be3718bd Author: Dirkjan Ochtman ochtman nl> AuthorDate: Tue May 3 07:18:05 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sat May 14 18:29:40 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=633cabfb repoman: Use lxml for parsing of metadata Note that we no longer throw a QA error for a missing XML prolog, as long as the encoding matches the default ('UTF-8'; lowercase is also allowed). Update import error message to lxml pkg. pym/repoman/modules/scan/metadata/pkgmetadata.py | 46 +++- 1 file changed, 13 insertions(+), 33 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index bcddb3e..317ab56 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -7,14 +7,14 @@ import sys from itertools import chain try: - import xml.etree.ElementTree - from xml.parsers.expat import ExpatError + from lxml import etree + from lxml.etree import ParserError except (SystemExit, KeyboardInterrupt): raise except (ImportError, SystemError, RuntimeError, Exception): # broken or missing xml support # http://bugs.python.org/issue14988 - msg = ["Please enable python's \"xml\" USE flag in order to use repoman."] + msg = ["Please emerge dev-python/lxml in order to use repoman."] from portage.output import EOutput out = EOutput() for line in msg: @@ -26,12 +26,11 @@ from repoman._portage import portage from repoman.metadata import metadata_dtd_uri from repoman.checks.herds.herdbase import get_herd_base from repoman.checks.herds.metadata import check_metadata, UnknownHerdsError -from repoman._xml import _XMLParser, _MetadataTreeBuilder, XmlLint +from repoman._xml import XmlLint from repoman.modules.scan.scanbase import ScanBase from portage.exception import InvalidAtom from portage import os -from portage import _encodings, _unicode_encode from portage import exception from portage.dep import Atom @@ -145,50 +144,31 @@ class PkgMetadata(ScanBase, USEFlagChecks): # metadata.xml parse check metadata_bad = False - xml_info = {} - xml_parser = _XMLParser(xml_info, target=_MetadataTreeBuilder()) # read metadata.xml into memory try: - _metadata_xml = xml.etree.ElementTree.parse( - _unicode_encode( - os.path.join(checkdir, "metadata.xml"), - encoding=_encodings['fs'], errors='strict'), - parser=xml_parser) - except (ExpatError, SyntaxError, EnvironmentError) as e: + _metadata_xml = etree.parse(os.path.join(checkdir, 'metadata.xml')) + except (ParserError, SyntaxError, EnvironmentError) as e: metadata_bad = True self.qatracker.add_error("metadata.bad", "%s/metadata.xml: %s" % (xpkg, e)) del e self.muselist = frozenset(self.musedict) return False - if "XML_DECLARATION" not in xml_info: + xml_encoding = _metadata_xml.docinfo.encoding + if xml_encoding.upper() != metadata_xml_encoding: self.qatracker.add_error( "metadata.bad", "%s/metadata.xml: " - "xml declaration is missing on first line, " - "should be '%s'" % (xpkg, metadata_xml_declaration)) - else: - xml_version, xml_encoding, xml_standalone = \ - xml_info["XML_DECLARATION"] - if xml_encoding is None or \ - xml_encoding.upper() != metadata_xml_encoding: - if xml_encoding is None: - encoding_problem = "but it is undefined" - else: - encoding_problem = "not '%s'" % xml_encoding - self.qatracker.add_error( - "metadata.bad", "%s/metadata.xml: " - "xml declaration encoding should be '%s', %s" % - (xpkg, metadata_xml_encoding, encoding_problem)) + "xml declaration encoding should be '%s', not '%s'" % + (xpkg, metadata_xml_encoding, xml_encoding)) - if "DOCTYPE" not in xml_info: + if not _metadata_xml.docinfo: metadata_bad = True self
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 9c3e213142e0e39f69dc002bd15402a7a2ff22d1 Author: Dirkjan Ochtman ochtman nl> AuthorDate: Mon Jan 25 19:45:31 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sat May 14 18:29:40 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=9c3e2131 repoman: mandate use of python 2.7 or 3.2+ for simplicity pym/repoman/modules/scan/metadata/pkgmetadata.py | 78 +++- 1 file changed, 36 insertions(+), 42 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index af53f4b..bcddb3e 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -162,54 +162,48 @@ class PkgMetadata(ScanBase, USEFlagChecks): self.muselist = frozenset(self.musedict) return False - if not hasattr(xml_parser, 'parser') or \ - sys.hexversion < 0x207 or \ - (sys.hexversion > 0x300 and sys.hexversion < 0x302): - # doctype is not parsed with python 2.6 or 3.1 - pass + if "XML_DECLARATION" not in xml_info: + self.qatracker.add_error( + "metadata.bad", "%s/metadata.xml: " + "xml declaration is missing on first line, " + "should be '%s'" % (xpkg, metadata_xml_declaration)) else: - if "XML_DECLARATION" not in xml_info: + xml_version, xml_encoding, xml_standalone = \ + xml_info["XML_DECLARATION"] + if xml_encoding is None or \ + xml_encoding.upper() != metadata_xml_encoding: + if xml_encoding is None: + encoding_problem = "but it is undefined" + else: + encoding_problem = "not '%s'" % xml_encoding self.qatracker.add_error( "metadata.bad", "%s/metadata.xml: " - "xml declaration is missing on first line, " - "should be '%s'" % (xpkg, metadata_xml_declaration)) - else: - xml_version, xml_encoding, xml_standalone = \ - xml_info["XML_DECLARATION"] - if xml_encoding is None or \ - xml_encoding.upper() != metadata_xml_encoding: - if xml_encoding is None: - encoding_problem = "but it is undefined" - else: - encoding_problem = "not '%s'" % xml_encoding - self.qatracker.add_error( - "metadata.bad", "%s/metadata.xml: " - "xml declaration encoding should be '%s', %s" % - (xpkg, metadata_xml_encoding, encoding_problem)) + "xml declaration encoding should be '%s', %s" % + (xpkg, metadata_xml_encoding, encoding_problem)) - if "DOCTYPE" not in xml_info: - metadata_bad = True + if "DOCTYPE" not in xml_info: + metadata_bad = True + self.qatracker.add_error( + "metadata.bad", + "%s/metadata.xml: %s" % (xpkg, "DOCTYPE is missing")) + else: + doctype_name, doctype_system, doctype_pubid = \ + xml_info["DOCTYPE"] + if doctype_system != metadata_dtd_uri: + if doctype_system is None: + system_problem = "but it is undefined" + else: + system_problem = "not '%s'" % doctype_system self.qatracker.add_error( - "metadata.bad", - "%s/metadata.xml: %s" % (xpkg, "DOCTYPE is missing")) - else: - doctype_name, doctype_system, doctype_pubid = \ - xml_info["DOCTYPE"] - if doctype_system != metadata_dtd_uri: - if doctype_system is None: -
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 2eec213a7226380983f270276fef67f9c0337cab Author: Brian Dolbec gentoo org> AuthorDate: Tue May 3 07:39:09 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sat May 14 18:29:40 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=2eec213a repoman/modules/.../pkgmetadata.py: Remove obsolete herds checks This change based on original work done by Dirkjan Ochtman ochtman.nl> pym/repoman/modules/scan/metadata/pkgmetadata.py | 12 1 file changed, 12 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index 3ca7897..4921b6f 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -24,8 +24,6 @@ except (ImportError, SystemError, RuntimeError, Exception): # import our initialized portage instance from repoman._portage import portage from repoman.metadata import metadata_dtd_uri -from repoman.checks.herds.herdbase import get_herd_base -from repoman.checks.herds.metadata import check_metadata, UnknownHerdsError from repoman.modules.scan.scanbase import ScanBase from portage.exception import InvalidAtom @@ -205,16 +203,6 @@ class PkgMetadata(ScanBase, USEFlagChecks): "%s/metadata.xml: Atom contains " "unexpected cat/pn: %s" % (xpkg, atom)) - # Run other metadata.xml checkers - try: - check_metadata(_metadata_xml, get_herd_base( - self.repoman_settings)) - except (UnknownHerdsError, ) as e: - metadata_bad = True - self.qatracker.add_error( - "metadata.bad", "%s/metadata.xml: %s" % (xpkg, e)) - del e - # Only carry out if in package directory or check forced if not metadata_bad: validator = etree.XMLSchema(file=self.metadata_xsd)
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 57d1ef7f8f98275b066735c66d0da05ea103a9da Author: Brian Dolbec gentoo org> AuthorDate: Tue May 3 18:46:22 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sat May 14 18:29:40 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=57d1ef7f repoman/modules/.../pkgmetadata.py: Move parse_metadata_use into the PkgMetadata class This allows for a complete Q/A processing of all the use flags defined in the xml. pym/repoman/modules/scan/metadata/pkgmetadata.py | 138 +++ 1 file changed, 68 insertions(+), 70 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index 1594b27..7117e7d 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -28,7 +28,6 @@ from repoman.modules.scan.scanbase import ScanBase from portage.exception import InvalidAtom from portage import os -from portage import exception from portage.dep import Atom from .use_flags import USEFlagChecks @@ -43,55 +42,6 @@ metadata_xml_declaration = '' \ metadata_doctype_name = 'pkgmetadata' -def parse_metadata_use(xml_tree): - """ - Records are wrapped in XML as per GLEP 56 - returns a dict with keys constisting of USE flag names and values - containing their respective descriptions - """ - uselist = {} - - usetags = xml_tree.findall("use") - if not usetags: - return uselist - - # It's possible to have multiple 'use' elements. - for usetag in usetags: - flags = usetag.findall("flag") - if not flags: - # DTD allows use elements containing no flag elements. - continue - - for flag in flags: - pkg_flag = flag.get("name") - if pkg_flag is None: - raise exception.ParseError("missing 'name' attribute for 'flag' tag") - flag_restrict = flag.get("restrict") - - # emulate the Element.itertext() method from python-2.7 - inner_text = [] - stack = [] - stack.append(flag) - while stack: - obj = stack.pop() - if isinstance(obj, basestring): - inner_text.append(obj) - continue - if isinstance(obj.text, basestring): - inner_text.append(obj.text) - if isinstance(obj.tail, basestring): - stack.append(obj.tail) - stack.extend(reversed(obj)) - - if pkg_flag not in uselist: - uselist[pkg_flag] = {} - - # (flag_restrict can be None) - uselist[pkg_flag][flag_restrict] = " ".join("".join(inner_text).split()) - - return uselist - - class PkgMetadata(ScanBase, USEFlagChecks): '''Package metadata.xml checks''' @@ -180,28 +130,23 @@ class PkgMetadata(ScanBase, USEFlagChecks): (xpkg, metadata_doctype_name, doctype_name)) # load USE flags from metadata.xml - try: - self.musedict = parse_metadata_use(_metadata_xml) - except portage.exception.ParseError as e: - metadata_bad = True - self.qatracker.add_error( - "metadata.bad", "%s/metadata.xml: %s" % (xpkg, e)) - else: - for atom in chain(*self.musedict.values()): - if atom is None: - continue - try: - atom = Atom(atom) - except InvalidAtom as e: + self.musedict, metadata_bad = self._parse_metadata_use( + _metadata_xml, xpkg, metadata_bad) + for atom in chain(*self.musedict.values()): + if atom is None: + continue + try: + atom = Atom(atom) + except InvalidAtom as e: + self.qatracker.add_error( + "metadata.bad", + "%s/metadata.xml: Invalid atom: %s" % (xpkg, e)) + else: + if atom.cp != xpkg: self.qatracker.add_error( "metadata.bad", -
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: cfc31c04bf6e1d59b2756e3ac315a2040cdd162d Author: Brian Dolbec gentoo org> AuthorDate: Wed May 4 06:08:50 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sat May 14 18:29:40 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=cfc31c04 repoman/modules/.../pkgmetadata.py: Fix failed missing doctype detection The docinfo is always created, so is not valid to use like a boolean to detect a missing doctype. doctype is a null string or equivalent and as such the correct choice for the if statement. Failure discoverd by: Göktürk Yüksek binghamton.edu> Thank you pym/repoman/modules/scan/metadata/pkgmetadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index 83ca93f..d8344c2 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -106,7 +106,7 @@ class PkgMetadata(ScanBase, USEFlagChecks): "xml declaration encoding should be '%s', not '%s'" % (xpkg, metadata_xml_encoding, xml_encoding)) - if not _metadata_xml.docinfo: + if not _metadata_xml.docinfo.doctype: metadata_bad = True self.qatracker.add_error( "metadata.bad",
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: c6d6b003bc0545912f15c82a99c2e948a5c63e60 Author: Brian Dolbec gentoo org> AuthorDate: Tue May 3 20:54:33 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sat May 14 18:29:40 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c6d6b003 repoman/modules/.../pkgmetadata.py: Have xml validation log all qatracker errors Remove the use flag qatracker additions. Add all logged XMLSchema errors to the qatracker . This makes it a one run check to add all possible errors via the XMLSchema. pym/repoman/modules/scan/metadata/pkgmetadata.py | 80 +++- 1 file changed, 36 insertions(+), 44 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index 7117e7d..83ca93f 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -130,8 +130,7 @@ class PkgMetadata(ScanBase, USEFlagChecks): (xpkg, metadata_doctype_name, doctype_name)) # load USE flags from metadata.xml - self.musedict, metadata_bad = self._parse_metadata_use( - _metadata_xml, xpkg, metadata_bad) + self.musedict = self._parse_metadata_use(_metadata_xml, xpkg) for atom in chain(*self.musedict.values()): if atom is None: continue @@ -151,15 +150,8 @@ class PkgMetadata(ScanBase, USEFlagChecks): # Only carry out if in package directory or check forced if not metadata_bad: validator = etree.XMLSchema(file=self.metadata_xsd) - try: - validator.assertValid(_metadata_xml) - except etree.DocumentInvalid as error: - self.qatracker.add_error( - "metadata.bad", - xpkg + "/metadata.xml: %s" - % (str(error)) - ) - del metadata_bad + if not validator.validate(_metadata_xml): + self._add_validate_errors(xpkg, validator.error_log) self.muselist = frozenset(self.musedict) return False @@ -182,7 +174,7 @@ class PkgMetadata(ScanBase, USEFlagChecks): % (xpkg, myflag)) return False - def _parse_metadata_use(self, xml_tree, xpkg, metadata_bad): + def _parse_metadata_use(self, xml_tree, xpkg): """ Records are wrapped in XML as per GLEP 56 returns a dict with keys constisting of USE flag names and values @@ -192,7 +184,7 @@ class PkgMetadata(ScanBase, USEFlagChecks): usetags = xml_tree.findall("use") if not usetags: - return uselist, metadata_bad + return uselist # It's possible to have multiple 'use' elements. for usetag in usetags: @@ -203,37 +195,37 @@ class PkgMetadata(ScanBase, USEFlagChecks): for flag in flags: pkg_flag = flag.get("name") - if pkg_flag is None: - metadata_bad = True - self.qatracker.add_error( - "metadata.bad", - "%s/metadata.xml: line: %s, '%s', missing attribute: name" - % (xpkg, flag.sourceline, flag.text)) - continue - flag_restrict = flag.get("restrict") - - # emulate the Element.itertext() method from python-2.7 - inner_text = [] - stack = [] - stack.append(flag) - while stack: - obj = stack.pop() - if isinstance(obj, basestring): - inner_text.append(obj) - continue - if isinstance(obj.text, basestring): - inner_text.append(obj.text) - if isinstance(obj.tail, basestring): - stack.append(obj.tail) - stack.extend(reversed(obj)) - - if pkg_flag not in uselist: - uselist[pkg_flag] = {} - -
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 1fd217a14abd34e98c9bbee4ea5843b7460a6d4b Author: Brian Dolbec gentoo org> AuthorDate: Sun Jan 3 11:31:26 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Wed Jan 6 04:08:21 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=1fd217a1 scanner.py: Migrate another metadata check to ebuild_metadata pym/repoman/modules/scan/metadata/ebuild_metadata.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pym/repoman/modules/scan/metadata/ebuild_metadata.py b/pym/repoman/modules/scan/metadata/ebuild_metadata.py index 143a40e..2dc1db2 100644 --- a/pym/repoman/modules/scan/metadata/ebuild_metadata.py +++ b/pym/repoman/modules/scan/metadata/ebuild_metadata.py @@ -28,6 +28,9 @@ class EbuildMetadata(object): "%s: %s variable contains non-ASCII " "character at position %s" % (ebuild.relative_path, k, m.start() + 1)) + if ebuild.metadata.get("PROVIDE"): + self.qatracker.add_error("virtual.oldstyle", ebuild.relative_path) + return {'continue': False} @property
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 4c9cb5cb9e90e2a576f4d4a37bafbea9372c3ebe Author: Brian Dolbec gentoo org> AuthorDate: Tue Mar 15 18:45:45 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Tue Mar 15 18:45:45 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=4c9cb5cb repoman/modules.scan/metadata/unused.py: Add docstrings Remove un-needed override functions pym/repoman/modules/scan/metadata/unused.py | 21 + 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/unused.py b/pym/repoman/modules/scan/metadata/unused.py index b3dc863..6def48f 100644 --- a/pym/repoman/modules/scan/metadata/unused.py +++ b/pym/repoman/modules/scan/metadata/unused.py @@ -1,11 +1,23 @@ class UnusedCheck(object): + '''Checks and reports any un-used metadata.xml use flag descriptions''' def __init__(self, **kwargs): + '''UnusedCheck init function + + @param qatracker: QATracker instance + ''' self.qatracker = kwargs.get('qatracker') def check(self, **kwargs): + '''Reports on any unused metadata.xml use descriptions + + @param xpkg: the pacakge being checked + @param muselist: use flag list + @param used_useflags: use flag list + @param validity_fuse: Fuse instance + ''' xpkg = kwargs.get('xpkg') muselist = kwargs.get('muselist') used_useflags = kwargs.get('used_useflags') @@ -20,13 +32,6 @@ class UnusedCheck(object): return {'continue': False} @property - def runInPkgs(self): - return (False, []) - - @property - def runInEbuilds(self): - return (False, []) - - @property def runInFinal(self): + '''Final scans at the package level''' return (True, [self.check])
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 6ca71f4c66e45d86503f1cc199e1547e19b35a2e Author: Brian Dolbec gentoo org> AuthorDate: Tue May 3 07:39:09 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Tue May 3 09:12:55 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=6ca71f4c repoman/modules/.../pkgmetadata.py: Remove obsolete herds checks This change based on original work done by Dirkjan Ochtman ochtman.nl> pym/repoman/modules/scan/metadata/pkgmetadata.py | 12 1 file changed, 12 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index 22afddf..c744c13 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -24,8 +24,6 @@ except (ImportError, SystemError, RuntimeError, Exception): # import our initialized portage instance from repoman._portage import portage from repoman.metadata import metadata_dtd_uri -from repoman.checks.herds.herdbase import get_herd_base -from repoman.checks.herds.metadata import check_metadata, UnknownHerdsError from repoman._xml import XmlLint from repoman.modules.scan.scanbase import ScanBase @@ -205,16 +203,6 @@ class PkgMetadata(ScanBase, USEFlagChecks): "%s/metadata.xml: Atom contains " "unexpected cat/pn: %s" % (xpkg, atom)) - # Run other metadata.xml checkers - try: - check_metadata(_metadata_xml, get_herd_base( - self.repoman_settings)) - except (UnknownHerdsError, ) as e: - metadata_bad = True - self.qatracker.add_error( - "metadata.bad", "%s/metadata.xml: %s" % (xpkg, e)) - del e - # Only carry out if in package directory or check forced if not metadata_bad: if not self.xmllint.check(checkdir, repolevel):
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: e66ee7b35405a650c517939e503443948b660ca9 Author: Dirkjan Ochtman ochtman nl> AuthorDate: Mon Jan 25 18:33:32 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Tue May 3 06:25:53 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=e66ee7b3 repoman/modules/.../pkgmetadata.py: Early return in metadata check code for readability pym/repoman/modules/scan/metadata/pkgmetadata.py | 190 --- 1 file changed, 96 insertions(+), 94 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index 5c6452a..3c1c2d0 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -83,112 +83,114 @@ class PkgMetadata(ScanBase, USEFlagChecks): # metadata.xml file check if "metadata.xml" not in checkdirlist: self.qatracker.add_error("metadata.missing", xpkg + "/metadata.xml") + self.muselist = frozenset(self.musedict) + return False + # metadata.xml parse check + metadata_bad = False + xml_info = {} + xml_parser = _XMLParser(xml_info, target=_MetadataTreeBuilder()) + + # read metadata.xml into memory + try: + _metadata_xml = xml.etree.ElementTree.parse( + _unicode_encode( + os.path.join(checkdir, "metadata.xml"), + encoding=_encodings['fs'], errors='strict'), + parser=xml_parser) + except (ExpatError, SyntaxError, EnvironmentError) as e: + metadata_bad = True + self.qatracker.add_error("metadata.bad", "%s/metadata.xml: %s" % (xpkg, e)) + del e else: - metadata_bad = False - xml_info = {} - xml_parser = _XMLParser(xml_info, target=_MetadataTreeBuilder()) - - # read metadata.xml into memory - try: - _metadata_xml = xml.etree.ElementTree.parse( - _unicode_encode( - os.path.join(checkdir, "metadata.xml"), - encoding=_encodings['fs'], errors='strict'), - parser=xml_parser) - except (ExpatError, SyntaxError, EnvironmentError) as e: - metadata_bad = True - self.qatracker.add_error("metadata.bad", "%s/metadata.xml: %s" % (xpkg, e)) - del e + if not hasattr(xml_parser, 'parser') or \ + sys.hexversion < 0x207 or \ + (sys.hexversion > 0x300 and sys.hexversion < 0x302): + # doctype is not parsed with python 2.6 or 3.1 + pass else: - if not hasattr(xml_parser, 'parser') or \ - sys.hexversion < 0x207 or \ - (sys.hexversion > 0x300 and sys.hexversion < 0x302): - # doctype is not parsed with python 2.6 or 3.1 - pass + if "XML_DECLARATION" not in xml_info: + self.qatracker.add_error( + "metadata.bad", "%s/metadata.xml: " + "xml declaration is missing on first line, " + "should be '%s'" % (xpkg, metadata_xml_declaration)) else: - if "XML_DECLARATION" not in xml_info: + xml_version, xml_encoding, xml_standalone = \ + xml_info["XML_DECLARATION"] + if xml_encoding is None or \ + xml_encoding.upper() != metadata_xml_encoding: + if xml_encoding is None: + encoding_problem = "but it is undefined" + else: + encoding_problem = "not '%s'" % xml_encoding self.qatracker.add_error( "metadata.bad"
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 4ef6bc69a3fe62d980c285ac0b9bb5bbadeefe9b Author: Brian Dolbec gentoo org> AuthorDate: Tue May 3 06:33:47 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Tue May 3 06:33:47 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=4ef6bc69 repoman/modules/.../pkgmetadata.py: Early return on metadata parse errors pym/repoman/modules/scan/metadata/pkgmetadata.py | 148 --- 1 file changed, 75 insertions(+), 73 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index 3c1c2d0..3b48b8e 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -102,89 +102,91 @@ class PkgMetadata(ScanBase, USEFlagChecks): metadata_bad = True self.qatracker.add_error("metadata.bad", "%s/metadata.xml: %s" % (xpkg, e)) del e + self.muselist = frozenset(self.musedict) + return False + + if not hasattr(xml_parser, 'parser') or \ + sys.hexversion < 0x207 or \ + (sys.hexversion > 0x300 and sys.hexversion < 0x302): + # doctype is not parsed with python 2.6 or 3.1 + pass else: - if not hasattr(xml_parser, 'parser') or \ - sys.hexversion < 0x207 or \ - (sys.hexversion > 0x300 and sys.hexversion < 0x302): - # doctype is not parsed with python 2.6 or 3.1 - pass + if "XML_DECLARATION" not in xml_info: + self.qatracker.add_error( + "metadata.bad", "%s/metadata.xml: " + "xml declaration is missing on first line, " + "should be '%s'" % (xpkg, metadata_xml_declaration)) else: - if "XML_DECLARATION" not in xml_info: + xml_version, xml_encoding, xml_standalone = \ + xml_info["XML_DECLARATION"] + if xml_encoding is None or \ + xml_encoding.upper() != metadata_xml_encoding: + if xml_encoding is None: + encoding_problem = "but it is undefined" + else: + encoding_problem = "not '%s'" % xml_encoding self.qatracker.add_error( "metadata.bad", "%s/metadata.xml: " - "xml declaration is missing on first line, " - "should be '%s'" % (xpkg, metadata_xml_declaration)) - else: - xml_version, xml_encoding, xml_standalone = \ - xml_info["XML_DECLARATION"] - if xml_encoding is None or \ - xml_encoding.upper() != metadata_xml_encoding: - if xml_encoding is None: - encoding_problem = "but it is undefined" - else: - encoding_problem = "not '%s'" % xml_encoding - self.qatracker.add_error( - "metadata.bad", "%s/metadata.xml: " - "xml declaration encoding should be '%s', %s" % - (xpkg, metadata_xml_encoding, encoding_problem)) + "xml declaration encoding should be '%s', %s" % + (xpkg, metadata_xml_encoding, encoding_problem)) - if "DOCTYPE" not in xml_info: - metadata_bad = True - self.qatracker.add_error( - "metadata.bad", - "%s/metadata.xml: %s" % (xpkg, "DOCTYPE is missing")) - else: - doctype_name, doctype_system, doctype_pubid = \ - xml_info["DOCTYPE"] -
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 75b897d91874c505e14dc2c0808c4cad4268ad76 Author: Brian Dolbec gentoo org> AuthorDate: Tue May 3 07:18:05 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Tue May 3 09:12:30 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=75b897d9 repoman: Use lxml for parsing of metadata Note that we no longer throw a QA error for a missing XML prolog, as long as the encoding matches the default ('UTF-8'; lowercase is also allowed). pym/repoman/modules/scan/metadata/pkgmetadata.py | 44 +++- 1 file changed, 12 insertions(+), 32 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index e8db92f..22afddf 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -7,8 +7,8 @@ import sys from itertools import chain try: - import xml.etree.ElementTree - from xml.parsers.expat import ExpatError + from lxml import etree + from lxml.etree import ParserError except (SystemExit, KeyboardInterrupt): raise except (ImportError, SystemError, RuntimeError, Exception): @@ -26,12 +26,11 @@ from repoman._portage import portage from repoman.metadata import metadata_dtd_uri from repoman.checks.herds.herdbase import get_herd_base from repoman.checks.herds.metadata import check_metadata, UnknownHerdsError -from repoman._xml import _XMLParser, _MetadataTreeBuilder, XmlLint +from repoman._xml import XmlLint from repoman.modules.scan.scanbase import ScanBase from portage.exception import InvalidAtom from portage import os -from portage import _encodings, _unicode_encode from portage import exception from portage.dep import Atom @@ -141,50 +140,31 @@ class PkgMetadata(ScanBase, USEFlagChecks): # metadata.xml parse check metadata_bad = False - xml_info = {} - xml_parser = _XMLParser(xml_info, target=_MetadataTreeBuilder()) # read metadata.xml into memory try: - _metadata_xml = xml.etree.ElementTree.parse( - _unicode_encode( - os.path.join(checkdir, "metadata.xml"), - encoding=_encodings['fs'], errors='strict'), - parser=xml_parser) - except (ExpatError, SyntaxError, EnvironmentError) as e: + _metadata_xml = etree.parse(os.path.join(checkdir, 'metadata.xml')) + except (ParserError, SyntaxError, EnvironmentError) as e: metadata_bad = True self.qatracker.add_error("metadata.bad", "%s/metadata.xml: %s" % (xpkg, e)) del e self.muselist = frozenset(self.musedict) return False - if "XML_DECLARATION" not in xml_info: + xml_encoding = _metadata_xml.docinfo.encoding + if xml_encoding.upper() != metadata_xml_encoding: self.qatracker.add_error( "metadata.bad", "%s/metadata.xml: " - "xml declaration is missing on first line, " - "should be '%s'" % (xpkg, metadata_xml_declaration)) - else: - xml_version, xml_encoding, xml_standalone = \ - xml_info["XML_DECLARATION"] - if xml_encoding is None or \ - xml_encoding.upper() != metadata_xml_encoding: - if xml_encoding is None: - encoding_problem = "but it is undefined" - else: - encoding_problem = "not '%s'" % xml_encoding - self.qatracker.add_error( - "metadata.bad", "%s/metadata.xml: " - "xml declaration encoding should be '%s', %s" % - (xpkg, metadata_xml_encoding, encoding_problem)) + "xml declaration encoding should be '%s', not '%s'" % + (xpkg, metadata_xml_encoding, xml_encoding)) - if "DOCTYPE" not in xml_info: + if not _metadata_xml.docinfo: metadata_bad = True self.qatracker.add_error( "metadata.bad", "%s/metadata.xml: %s" % (xpkg, "DOCTYPE is missing")) else: - doctype_name, doctype_system, doctype_pubid = \ - xml_info["DOCTYPE"] + doctype_system = _metadata_xml.docinfo.system_url
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 57ad076b08d417bf5d2642131b76392a990f3c35 Author: Dirkjan Ochtman ochtman nl> AuthorDate: Mon Jan 25 19:45:31 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Tue May 3 06:55:53 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=57ad076b repoman: mandate use of python 2.7 or 3.2+ for simplicity pym/repoman/modules/scan/metadata/pkgmetadata.py | 78 +++- 1 file changed, 36 insertions(+), 42 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index cde2ba0..e8db92f 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -158,54 +158,48 @@ class PkgMetadata(ScanBase, USEFlagChecks): self.muselist = frozenset(self.musedict) return False - if not hasattr(xml_parser, 'parser') or \ - sys.hexversion < 0x207 or \ - (sys.hexversion > 0x300 and sys.hexversion < 0x302): - # doctype is not parsed with python 2.6 or 3.1 - pass + if "XML_DECLARATION" not in xml_info: + self.qatracker.add_error( + "metadata.bad", "%s/metadata.xml: " + "xml declaration is missing on first line, " + "should be '%s'" % (xpkg, metadata_xml_declaration)) else: - if "XML_DECLARATION" not in xml_info: + xml_version, xml_encoding, xml_standalone = \ + xml_info["XML_DECLARATION"] + if xml_encoding is None or \ + xml_encoding.upper() != metadata_xml_encoding: + if xml_encoding is None: + encoding_problem = "but it is undefined" + else: + encoding_problem = "not '%s'" % xml_encoding self.qatracker.add_error( "metadata.bad", "%s/metadata.xml: " - "xml declaration is missing on first line, " - "should be '%s'" % (xpkg, metadata_xml_declaration)) - else: - xml_version, xml_encoding, xml_standalone = \ - xml_info["XML_DECLARATION"] - if xml_encoding is None or \ - xml_encoding.upper() != metadata_xml_encoding: - if xml_encoding is None: - encoding_problem = "but it is undefined" - else: - encoding_problem = "not '%s'" % xml_encoding - self.qatracker.add_error( - "metadata.bad", "%s/metadata.xml: " - "xml declaration encoding should be '%s', %s" % - (xpkg, metadata_xml_encoding, encoding_problem)) + "xml declaration encoding should be '%s', %s" % + (xpkg, metadata_xml_encoding, encoding_problem)) - if "DOCTYPE" not in xml_info: - metadata_bad = True + if "DOCTYPE" not in xml_info: + metadata_bad = True + self.qatracker.add_error( + "metadata.bad", + "%s/metadata.xml: %s" % (xpkg, "DOCTYPE is missing")) + else: + doctype_name, doctype_system, doctype_pubid = \ + xml_info["DOCTYPE"] + if doctype_system != metadata_dtd_uri: + if doctype_system is None: + system_problem = "but it is undefined" + else: + system_problem = "not '%s'" % doctype_system self.qatracker.add_error( - "metadata.bad", - "%s/metadata.xml: %s" % (xpkg, "DOCTYPE is missing")) - else: - doctype_name, doctype_system, doctype_pubid = \ - xml_info["DOCTYPE"] - if doctype_system != metadata_dtd_uri: - if doctype_system is None: -
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 621a6852099848474ed2c0880eae52ebd85f1703 Author: Brian Dolbec gentoo org> AuthorDate: Tue May 3 17:38:19 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Tue May 3 17:38:19 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=621a6852 repoman/modules/.../pkgmetadata.py: Improve whole document validation Change to using assertValid() and add the error causing validation to fail to the qa tracker error. pym/repoman/modules/scan/metadata/pkgmetadata.py | 10 -- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index b231370..7390b5b 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -202,8 +202,14 @@ class PkgMetadata(ScanBase, USEFlagChecks): # Only carry out if in package directory or check forced if not metadata_bad: validator = etree.XMLSchema(file=self.metadata_xsd) - if not validator.validate(_metadata_xml): - self.qatracker.add_error("metadata.bad", xpkg + "/metadata.xml") + try: + validator.assertValid(_metadata_xml) + except etree.DocumentInvalid as error: + self.qatracker.add_error( + "metadata.bad", + xpkg + "/metadata.xml: %s" + % (str(error)) + ) del metadata_bad self.muselist = frozenset(self.musedict) return False
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 235235c69343e95793525ee2104a82977a5a6729 Author: Brian Dolbec gentoo org> AuthorDate: Tue May 3 17:42:22 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Tue May 3 17:42:22 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=235235c6 repoman/modules/.../pkgmetadata.py: Move parse_metadata_use into the PkgMetadata class This allows for a complete Q/A processing of all the use flags defined in the xml. pym/repoman/modules/scan/metadata/pkgmetadata.py | 139 +++ 1 file changed, 68 insertions(+), 71 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index 7390b5b..ef57a5d 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -28,7 +28,6 @@ from repoman.modules.scan.scanbase import ScanBase from portage.exception import InvalidAtom from portage import os -from portage import exception from portage.dep import Atom from .use_flags import USEFlagChecks @@ -39,55 +38,6 @@ metadata_xml_declaration = '' \ metadata_doctype_name = 'pkgmetadata' -def parse_metadata_use(xml_tree): - """ - Records are wrapped in XML as per GLEP 56 - returns a dict with keys constisting of USE flag names and values - containing their respective descriptions - """ - uselist = {} - - usetags = xml_tree.findall("use") - if not usetags: - return uselist - - # It's possible to have multiple 'use' elements. - for usetag in usetags: - flags = usetag.findall("flag") - if not flags: - # DTD allows use elements containing no flag elements. - continue - - for flag in flags: - pkg_flag = flag.get("name") - if pkg_flag is None: - raise exception.ParseError("missing 'name' attribute for 'flag' tag") - flag_restrict = flag.get("restrict") - - # emulate the Element.itertext() method from python-2.7 - inner_text = [] - stack = [] - stack.append(flag) - while stack: - obj = stack.pop() - if isinstance(obj, basestring): - inner_text.append(obj) - continue - if isinstance(obj.text, basestring): - inner_text.append(obj.text) - if isinstance(obj.tail, basestring): - stack.append(obj.tail) - stack.extend(reversed(obj)) - - if pkg_flag not in uselist: - uselist[pkg_flag] = {} - - # (flag_restrict can be None) - uselist[pkg_flag][flag_restrict] = " ".join("".join(inner_text).split()) - - return uselist - - class PkgMetadata(ScanBase, USEFlagChecks): '''Package metadata.xml checks''' @@ -176,32 +126,26 @@ class PkgMetadata(ScanBase, USEFlagChecks): (xpkg, metadata_doctype_name, doctype_name)) # load USE flags from metadata.xml - try: - self.musedict = parse_metadata_use(_metadata_xml) - except portage.exception.ParseError as e: - metadata_bad = True - self.qatracker.add_error( - "metadata.bad", "%s/metadata.xml: %s" % (xpkg, e)) - else: - for atom in chain(*self.musedict.values()): - if atom is None: - continue - try: - atom = Atom(atom) - except InvalidAtom as e: + self.musedict, metadata_bad = self._parse_metadata_use( + _metadata_xml, xpkg, metadata_bad) + for atom in chain(*self.musedict.values()): + if atom is None: + continue + try: + atom = Atom(atom) + except InvalidAtom as e: + self.qatracker.add_error( + "metadata.bad", + "%s/metadata.xml: Invalid atom: %s" % (xpkg, e)) + else: + if atom.cp != xpkg: self.qatracker.add_error( "metadata.bad", -
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 50de92c080ac8e5bc3f0d6cd2bc69fb919afc186 Author: Brian Dolbec gentoo org> AuthorDate: Tue May 3 18:46:22 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Tue May 3 18:46:22 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=50de92c0 repoman/modules/.../pkgmetadata.py: Move parse_metadata_use into the PkgMetadata class This allows for a complete Q/A processing of all the use flags defined in the xml. pym/repoman/modules/scan/metadata/pkgmetadata.py | 138 +++ 1 file changed, 68 insertions(+), 70 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index 7390b5b..d6d8557 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -28,7 +28,6 @@ from repoman.modules.scan.scanbase import ScanBase from portage.exception import InvalidAtom from portage import os -from portage import exception from portage.dep import Atom from .use_flags import USEFlagChecks @@ -39,55 +38,6 @@ metadata_xml_declaration = '' \ metadata_doctype_name = 'pkgmetadata' -def parse_metadata_use(xml_tree): - """ - Records are wrapped in XML as per GLEP 56 - returns a dict with keys constisting of USE flag names and values - containing their respective descriptions - """ - uselist = {} - - usetags = xml_tree.findall("use") - if not usetags: - return uselist - - # It's possible to have multiple 'use' elements. - for usetag in usetags: - flags = usetag.findall("flag") - if not flags: - # DTD allows use elements containing no flag elements. - continue - - for flag in flags: - pkg_flag = flag.get("name") - if pkg_flag is None: - raise exception.ParseError("missing 'name' attribute for 'flag' tag") - flag_restrict = flag.get("restrict") - - # emulate the Element.itertext() method from python-2.7 - inner_text = [] - stack = [] - stack.append(flag) - while stack: - obj = stack.pop() - if isinstance(obj, basestring): - inner_text.append(obj) - continue - if isinstance(obj.text, basestring): - inner_text.append(obj.text) - if isinstance(obj.tail, basestring): - stack.append(obj.tail) - stack.extend(reversed(obj)) - - if pkg_flag not in uselist: - uselist[pkg_flag] = {} - - # (flag_restrict can be None) - uselist[pkg_flag][flag_restrict] = " ".join("".join(inner_text).split()) - - return uselist - - class PkgMetadata(ScanBase, USEFlagChecks): '''Package metadata.xml checks''' @@ -176,28 +126,23 @@ class PkgMetadata(ScanBase, USEFlagChecks): (xpkg, metadata_doctype_name, doctype_name)) # load USE flags from metadata.xml - try: - self.musedict = parse_metadata_use(_metadata_xml) - except portage.exception.ParseError as e: - metadata_bad = True - self.qatracker.add_error( - "metadata.bad", "%s/metadata.xml: %s" % (xpkg, e)) - else: - for atom in chain(*self.musedict.values()): - if atom is None: - continue - try: - atom = Atom(atom) - except InvalidAtom as e: + self.musedict, metadata_bad = self._parse_metadata_use( + _metadata_xml, xpkg, metadata_bad) + for atom in chain(*self.musedict.values()): + if atom is None: + continue + try: + atom = Atom(atom) + except InvalidAtom as e: + self.qatracker.add_error( + "metadata.bad", + "%s/metadata.xml: Invalid atom: %s" % (xpkg, e)) + else: + if atom.cp != xpkg: self.qatracker.add_error( "metadata.bad", -
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: bf7dc3159db2c15ff7aa61c2c72b143bbd420be0 Author: Brian Dolbec gentoo org> AuthorDate: Tue May 3 20:54:33 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Tue May 3 20:54:33 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=bf7dc315 repoman/modules/.../pkgmetadata.py: Have xml validation log all qatracker errors Remove the use flag qatracker additions. Add all logged XMLSchema errors to the qatracker . This makes it a one run check to add all possible errors via the XMLSchema. pym/repoman/modules/scan/metadata/pkgmetadata.py | 80 +++- 1 file changed, 36 insertions(+), 44 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index d6d8557..44b5edd 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -126,8 +126,7 @@ class PkgMetadata(ScanBase, USEFlagChecks): (xpkg, metadata_doctype_name, doctype_name)) # load USE flags from metadata.xml - self.musedict, metadata_bad = self._parse_metadata_use( - _metadata_xml, xpkg, metadata_bad) + self.musedict = self._parse_metadata_use(_metadata_xml, xpkg) for atom in chain(*self.musedict.values()): if atom is None: continue @@ -147,15 +146,8 @@ class PkgMetadata(ScanBase, USEFlagChecks): # Only carry out if in package directory or check forced if not metadata_bad: validator = etree.XMLSchema(file=self.metadata_xsd) - try: - validator.assertValid(_metadata_xml) - except etree.DocumentInvalid as error: - self.qatracker.add_error( - "metadata.bad", - xpkg + "/metadata.xml: %s" - % (str(error)) - ) - del metadata_bad + if not validator.validate(_metadata_xml): + self._add_validate_errors(xpkg, validator.error_log) self.muselist = frozenset(self.musedict) return False @@ -178,7 +170,7 @@ class PkgMetadata(ScanBase, USEFlagChecks): % (xpkg, myflag)) return False - def _parse_metadata_use(self, xml_tree, xpkg, metadata_bad): + def _parse_metadata_use(self, xml_tree, xpkg): """ Records are wrapped in XML as per GLEP 56 returns a dict with keys constisting of USE flag names and values @@ -188,7 +180,7 @@ class PkgMetadata(ScanBase, USEFlagChecks): usetags = xml_tree.findall("use") if not usetags: - return uselist, metadata_bad + return uselist # It's possible to have multiple 'use' elements. for usetag in usetags: @@ -199,37 +191,37 @@ class PkgMetadata(ScanBase, USEFlagChecks): for flag in flags: pkg_flag = flag.get("name") - if pkg_flag is None: - metadata_bad = True - self.qatracker.add_error( - "metadata.bad", - "%s/metadata.xml: line: %s, '%s', missing attribute: name" - % (xpkg, flag.sourceline, flag.text)) - continue - flag_restrict = flag.get("restrict") - - # emulate the Element.itertext() method from python-2.7 - inner_text = [] - stack = [] - stack.append(flag) - while stack: - obj = stack.pop() - if isinstance(obj, basestring): - inner_text.append(obj) - continue - if isinstance(obj.text, basestring): - inner_text.append(obj.text) - if isinstance(obj.tail, basestring): - stack.append(obj.tail) - stack.extend(reversed(obj)) - - if pkg_flag not in uselist: - uselist[pkg_flag] = {} - -
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 378e5458a9a68231c8b7abc5df375eff8062a402 Author: Brian Dolbec gentoo org> AuthorDate: Wed May 4 03:44:17 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Wed May 4 03:44:17 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=378e5458 repoman/modules/.../pkgmetadata.py: Update import error message for lxml pkg pym/repoman/modules/scan/metadata/pkgmetadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index 44b5edd..c76922b 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -14,7 +14,7 @@ except (SystemExit, KeyboardInterrupt): except (ImportError, SystemError, RuntimeError, Exception): # broken or missing xml support # http://bugs.python.org/issue14988 - msg = ["Please enable python's \"xml\" USE flag in order to use repoman."] + msg = ["Please emerge dev-python/lxml in order to use repoman."] from portage.output import EOutput out = EOutput() for line in msg:
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 55943b69a42c292bc0922a22d71f9d992ea035e7 Author: Brian Dolbec gentoo org> AuthorDate: Wed May 4 06:08:50 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Wed May 4 06:08:50 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=55943b69 repoman/modules/.../pkgmetadata.py: Fix failed missing doctype detection The docinfo is always created, so is not valid to use like a boolean to detect a missing doctype. doctype is a null string or equivalent and as such the correct choice for the if statement. Failure discoverd by: Göktürk Yüksek binghamton.edu> Thank you pym/repoman/modules/scan/metadata/pkgmetadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index c76922b..8b31e3f 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -102,7 +102,7 @@ class PkgMetadata(ScanBase, USEFlagChecks): "xml declaration encoding should be '%s', not '%s'" % (xpkg, metadata_xml_encoding, xml_encoding)) - if not _metadata_xml.docinfo: + if not _metadata_xml.docinfo.doctype: metadata_bad = True self.qatracker.add_error( "metadata.bad",
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 9357614944df633df54871bc732d37abfe3a035f Author: Brian Dolbec gentoo org> AuthorDate: Wed May 4 06:23:05 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Wed May 4 06:23:05 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=93576149 repoman/modules/.../pkgmetadata.py: Add missing basestring compat code pym/repoman/modules/scan/metadata/pkgmetadata.py | 4 1 file changed, 4 insertions(+) diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index 8b31e3f..d8344c2 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -32,6 +32,10 @@ from portage.dep import Atom from .use_flags import USEFlagChecks +if sys.hexversion >= 0x300: + # pylint: disable=W0622 + basestring = str + metadata_xml_encoding = 'UTF-8' metadata_xml_declaration = '' \ % (metadata_xml_encoding,)
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: afefcbc188ea25540d72224034668207d9f4e024 Author: Brian Dolbec gentoo org> AuthorDate: Thu May 5 16:01:02 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Thu May 5 16:01:02 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=afefcbc1 repoman/modules/.../pkgmetadata.py: Add code to remove mostly duplicate errors Some types of errors produce two error messages cluttering up the output. The first error message is clearer, listing the possible option values allowed. This filters out the second error message for that same line and attribute. pym/repoman/modules/scan/metadata/pkgmetadata.py | 13 + 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index d8344c2..81e435e 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -221,11 +221,16 @@ class PkgMetadata(ScanBase, USEFlagChecks): return uselist def _add_validate_errors(self, xpkg, log): + listed = set() for error in log: - self.qatracker.add_error( - "metadata.bad", - "%s/metadata.xml: line: %s, %s" - % (xpkg, error.line, error.message)) + msg_prefix = error.message.split(":",1)[0] + info = "%s %s" % (error.line, msg_prefix) + if info not in listed: + listed.add(info) + self.qatracker.add_error( + "metadata.bad", + "%s/metadata.xml: line: %s, %s, %s" + % (xpkg, error.line, error.type, error.message)) @property def runInPkgs(self):
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 6a61a12b827a0fbcdf6c111f936d8f17f6b10a33 Author: Brian Dolbec gentoo org> AuthorDate: Thu May 5 16:04:54 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Thu May 5 16:04:54 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=6a61a12b repoman/modules/.../pkgmetadata.py: Add code to remove mostly duplicate errors Some types of errors produce two error messages cluttering up the output. The first error message is clearer, listing the possible option values allowed. This filters out the second error message for that same line and attribute. pym/repoman/modules/scan/metadata/pkgmetadata.py | 13 + 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index d8344c2..433551a 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -221,11 +221,16 @@ class PkgMetadata(ScanBase, USEFlagChecks): return uselist def _add_validate_errors(self, xpkg, log): + listed = set() for error in log: - self.qatracker.add_error( - "metadata.bad", - "%s/metadata.xml: line: %s, %s" - % (xpkg, error.line, error.message)) + msg_prefix = error.message.split(":",1)[0] + info = "%s %s" % (error.line, msg_prefix) + if info not in listed: + listed.add(info) + self.qatracker.add_error( + "metadata.bad", + "%s/metadata.xml: line: %s, %s" + % (xpkg, error.line, error.message)) @property def runInPkgs(self):
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 119a9e27ef5e3442205f2e9be8f379a10177d673 Author: Dirkjan Ochtman ochtman nl> AuthorDate: Mon Jan 25 19:45:31 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sun May 8 21:18:41 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=119a9e27 repoman: mandate use of python 2.7 or 3.2+ for simplicity pym/repoman/modules/scan/metadata/pkgmetadata.py | 78 +++- 1 file changed, 36 insertions(+), 42 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index af53f4b..bcddb3e 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -162,54 +162,48 @@ class PkgMetadata(ScanBase, USEFlagChecks): self.muselist = frozenset(self.musedict) return False - if not hasattr(xml_parser, 'parser') or \ - sys.hexversion < 0x207 or \ - (sys.hexversion > 0x300 and sys.hexversion < 0x302): - # doctype is not parsed with python 2.6 or 3.1 - pass + if "XML_DECLARATION" not in xml_info: + self.qatracker.add_error( + "metadata.bad", "%s/metadata.xml: " + "xml declaration is missing on first line, " + "should be '%s'" % (xpkg, metadata_xml_declaration)) else: - if "XML_DECLARATION" not in xml_info: + xml_version, xml_encoding, xml_standalone = \ + xml_info["XML_DECLARATION"] + if xml_encoding is None or \ + xml_encoding.upper() != metadata_xml_encoding: + if xml_encoding is None: + encoding_problem = "but it is undefined" + else: + encoding_problem = "not '%s'" % xml_encoding self.qatracker.add_error( "metadata.bad", "%s/metadata.xml: " - "xml declaration is missing on first line, " - "should be '%s'" % (xpkg, metadata_xml_declaration)) - else: - xml_version, xml_encoding, xml_standalone = \ - xml_info["XML_DECLARATION"] - if xml_encoding is None or \ - xml_encoding.upper() != metadata_xml_encoding: - if xml_encoding is None: - encoding_problem = "but it is undefined" - else: - encoding_problem = "not '%s'" % xml_encoding - self.qatracker.add_error( - "metadata.bad", "%s/metadata.xml: " - "xml declaration encoding should be '%s', %s" % - (xpkg, metadata_xml_encoding, encoding_problem)) + "xml declaration encoding should be '%s', %s" % + (xpkg, metadata_xml_encoding, encoding_problem)) - if "DOCTYPE" not in xml_info: - metadata_bad = True + if "DOCTYPE" not in xml_info: + metadata_bad = True + self.qatracker.add_error( + "metadata.bad", + "%s/metadata.xml: %s" % (xpkg, "DOCTYPE is missing")) + else: + doctype_name, doctype_system, doctype_pubid = \ + xml_info["DOCTYPE"] + if doctype_system != metadata_dtd_uri: + if doctype_system is None: + system_problem = "but it is undefined" + else: + system_problem = "not '%s'" % doctype_system self.qatracker.add_error( - "metadata.bad", - "%s/metadata.xml: %s" % (xpkg, "DOCTYPE is missing")) - else: - doctype_name, doctype_system, doctype_pubid = \ - xml_info["DOCTYPE"] - if doctype_system != metadata_dtd_uri: - if doctype_system is None: -
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: b1a831bffd28fc1bb889cc90bb7690c7e5e2b8d9 Author: Brian Dolbec gentoo org> AuthorDate: Wed May 4 06:08:50 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sun May 8 21:18:42 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b1a831bf repoman/modules/.../pkgmetadata.py: Fix failed missing doctype detection The docinfo is always created, so is not valid to use like a boolean to detect a missing doctype. doctype is a null string or equivalent and as such the correct choice for the if statement. Failure discoverd by: Göktürk Yüksek binghamton.edu> Thank you pym/repoman/modules/scan/metadata/pkgmetadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index 83ca93f..d8344c2 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -106,7 +106,7 @@ class PkgMetadata(ScanBase, USEFlagChecks): "xml declaration encoding should be '%s', not '%s'" % (xpkg, metadata_xml_encoding, xml_encoding)) - if not _metadata_xml.docinfo: + if not _metadata_xml.docinfo.doctype: metadata_bad = True self.qatracker.add_error( "metadata.bad",
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 5f15e81967f8dca211cbd87d8696059aeac9ada2 Author: Dirkjan Ochtman ochtman nl> AuthorDate: Tue May 3 06:33:47 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sun May 8 21:18:24 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=5f15e819 repoman/modules/.../pkgmetadata.py: Early return on metadata parse errors pym/repoman/modules/scan/metadata/pkgmetadata.py | 148 --- 1 file changed, 75 insertions(+), 73 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index 3c1c2d0..3b48b8e 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -102,89 +102,91 @@ class PkgMetadata(ScanBase, USEFlagChecks): metadata_bad = True self.qatracker.add_error("metadata.bad", "%s/metadata.xml: %s" % (xpkg, e)) del e + self.muselist = frozenset(self.musedict) + return False + + if not hasattr(xml_parser, 'parser') or \ + sys.hexversion < 0x207 or \ + (sys.hexversion > 0x300 and sys.hexversion < 0x302): + # doctype is not parsed with python 2.6 or 3.1 + pass else: - if not hasattr(xml_parser, 'parser') or \ - sys.hexversion < 0x207 or \ - (sys.hexversion > 0x300 and sys.hexversion < 0x302): - # doctype is not parsed with python 2.6 or 3.1 - pass + if "XML_DECLARATION" not in xml_info: + self.qatracker.add_error( + "metadata.bad", "%s/metadata.xml: " + "xml declaration is missing on first line, " + "should be '%s'" % (xpkg, metadata_xml_declaration)) else: - if "XML_DECLARATION" not in xml_info: + xml_version, xml_encoding, xml_standalone = \ + xml_info["XML_DECLARATION"] + if xml_encoding is None or \ + xml_encoding.upper() != metadata_xml_encoding: + if xml_encoding is None: + encoding_problem = "but it is undefined" + else: + encoding_problem = "not '%s'" % xml_encoding self.qatracker.add_error( "metadata.bad", "%s/metadata.xml: " - "xml declaration is missing on first line, " - "should be '%s'" % (xpkg, metadata_xml_declaration)) - else: - xml_version, xml_encoding, xml_standalone = \ - xml_info["XML_DECLARATION"] - if xml_encoding is None or \ - xml_encoding.upper() != metadata_xml_encoding: - if xml_encoding is None: - encoding_problem = "but it is undefined" - else: - encoding_problem = "not '%s'" % xml_encoding - self.qatracker.add_error( - "metadata.bad", "%s/metadata.xml: " - "xml declaration encoding should be '%s', %s" % - (xpkg, metadata_xml_encoding, encoding_problem)) + "xml declaration encoding should be '%s', %s" % + (xpkg, metadata_xml_encoding, encoding_problem)) - if "DOCTYPE" not in xml_info: - metadata_bad = True - self.qatracker.add_error( - "metadata.bad", - "%s/metadata.xml: %s" % (xpkg, "DOCTYPE is missing")) - else: - doctype_name, doctype_system, doctype_pubid = \ - xml_info["DOCTYPE"] -
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 82846cb35a214c51731fa701fab26119c1b3832d Author: Brian Dolbec gentoo org> AuthorDate: Tue May 3 07:39:09 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sun May 8 21:18:41 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=82846cb3 repoman/modules/.../pkgmetadata.py: Remove obsolete herds checks This change based on original work done by Dirkjan Ochtman ochtman.nl> pym/repoman/modules/scan/metadata/pkgmetadata.py | 12 1 file changed, 12 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index 3ca7897..4921b6f 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -24,8 +24,6 @@ except (ImportError, SystemError, RuntimeError, Exception): # import our initialized portage instance from repoman._portage import portage from repoman.metadata import metadata_dtd_uri -from repoman.checks.herds.herdbase import get_herd_base -from repoman.checks.herds.metadata import check_metadata, UnknownHerdsError from repoman.modules.scan.scanbase import ScanBase from portage.exception import InvalidAtom @@ -205,16 +203,6 @@ class PkgMetadata(ScanBase, USEFlagChecks): "%s/metadata.xml: Atom contains " "unexpected cat/pn: %s" % (xpkg, atom)) - # Run other metadata.xml checkers - try: - check_metadata(_metadata_xml, get_herd_base( - self.repoman_settings)) - except (UnknownHerdsError, ) as e: - metadata_bad = True - self.qatracker.add_error( - "metadata.bad", "%s/metadata.xml: %s" % (xpkg, e)) - del e - # Only carry out if in package directory or check forced if not metadata_bad: validator = etree.XMLSchema(file=self.metadata_xsd)
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 4d2cb398e48cdd90499dee48d52072e147c64e0f Author: Brian Dolbec gentoo org> AuthorDate: Tue May 3 17:38:19 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sun May 8 21:18:42 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=4d2cb398 repoman/modules/.../pkgmetadata.py: Improve whole document validation Change to using assertValid() and add the error causing validation to fail to the qa tracker error. pym/repoman/modules/scan/metadata/pkgmetadata.py | 10 -- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index 4921b6f..1594b27 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -206,8 +206,14 @@ class PkgMetadata(ScanBase, USEFlagChecks): # Only carry out if in package directory or check forced if not metadata_bad: validator = etree.XMLSchema(file=self.metadata_xsd) - if not validator.validate(_metadata_xml): - self.qatracker.add_error("metadata.bad", xpkg + "/metadata.xml") + try: + validator.assertValid(_metadata_xml) + except etree.DocumentInvalid as error: + self.qatracker.add_error( + "metadata.bad", + xpkg + "/metadata.xml: %s" + % (str(error)) + ) del metadata_bad self.muselist = frozenset(self.musedict) return False
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 278f13dfd0308f2d7d80fb332034611e0fa9f6ae Author: Brian Dolbec gentoo org> AuthorDate: Tue May 3 18:46:22 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sun May 8 21:18:42 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=278f13df repoman/modules/.../pkgmetadata.py: Move parse_metadata_use into the PkgMetadata class This allows for a complete Q/A processing of all the use flags defined in the xml. pym/repoman/modules/scan/metadata/pkgmetadata.py | 138 +++ 1 file changed, 68 insertions(+), 70 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index 1594b27..7117e7d 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -28,7 +28,6 @@ from repoman.modules.scan.scanbase import ScanBase from portage.exception import InvalidAtom from portage import os -from portage import exception from portage.dep import Atom from .use_flags import USEFlagChecks @@ -43,55 +42,6 @@ metadata_xml_declaration = '' \ metadata_doctype_name = 'pkgmetadata' -def parse_metadata_use(xml_tree): - """ - Records are wrapped in XML as per GLEP 56 - returns a dict with keys constisting of USE flag names and values - containing their respective descriptions - """ - uselist = {} - - usetags = xml_tree.findall("use") - if not usetags: - return uselist - - # It's possible to have multiple 'use' elements. - for usetag in usetags: - flags = usetag.findall("flag") - if not flags: - # DTD allows use elements containing no flag elements. - continue - - for flag in flags: - pkg_flag = flag.get("name") - if pkg_flag is None: - raise exception.ParseError("missing 'name' attribute for 'flag' tag") - flag_restrict = flag.get("restrict") - - # emulate the Element.itertext() method from python-2.7 - inner_text = [] - stack = [] - stack.append(flag) - while stack: - obj = stack.pop() - if isinstance(obj, basestring): - inner_text.append(obj) - continue - if isinstance(obj.text, basestring): - inner_text.append(obj.text) - if isinstance(obj.tail, basestring): - stack.append(obj.tail) - stack.extend(reversed(obj)) - - if pkg_flag not in uselist: - uselist[pkg_flag] = {} - - # (flag_restrict can be None) - uselist[pkg_flag][flag_restrict] = " ".join("".join(inner_text).split()) - - return uselist - - class PkgMetadata(ScanBase, USEFlagChecks): '''Package metadata.xml checks''' @@ -180,28 +130,23 @@ class PkgMetadata(ScanBase, USEFlagChecks): (xpkg, metadata_doctype_name, doctype_name)) # load USE flags from metadata.xml - try: - self.musedict = parse_metadata_use(_metadata_xml) - except portage.exception.ParseError as e: - metadata_bad = True - self.qatracker.add_error( - "metadata.bad", "%s/metadata.xml: %s" % (xpkg, e)) - else: - for atom in chain(*self.musedict.values()): - if atom is None: - continue - try: - atom = Atom(atom) - except InvalidAtom as e: + self.musedict, metadata_bad = self._parse_metadata_use( + _metadata_xml, xpkg, metadata_bad) + for atom in chain(*self.musedict.values()): + if atom is None: + continue + try: + atom = Atom(atom) + except InvalidAtom as e: + self.qatracker.add_error( + "metadata.bad", + "%s/metadata.xml: Invalid atom: %s" % (xpkg, e)) + else: + if atom.cp != xpkg: self.qatracker.add_error( "metadata.bad", -
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 0cd414979f4b5531fdd158b3c4766c436bd057eb Author: Brian Dolbec gentoo org> AuthorDate: Tue May 3 20:54:33 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sun May 8 21:18:42 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=0cd41497 repoman/modules/.../pkgmetadata.py: Have xml validation log all qatracker errors Remove the use flag qatracker additions. Add all logged XMLSchema errors to the qatracker . This makes it a one run check to add all possible errors via the XMLSchema. pym/repoman/modules/scan/metadata/pkgmetadata.py | 80 +++- 1 file changed, 36 insertions(+), 44 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index 7117e7d..83ca93f 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -130,8 +130,7 @@ class PkgMetadata(ScanBase, USEFlagChecks): (xpkg, metadata_doctype_name, doctype_name)) # load USE flags from metadata.xml - self.musedict, metadata_bad = self._parse_metadata_use( - _metadata_xml, xpkg, metadata_bad) + self.musedict = self._parse_metadata_use(_metadata_xml, xpkg) for atom in chain(*self.musedict.values()): if atom is None: continue @@ -151,15 +150,8 @@ class PkgMetadata(ScanBase, USEFlagChecks): # Only carry out if in package directory or check forced if not metadata_bad: validator = etree.XMLSchema(file=self.metadata_xsd) - try: - validator.assertValid(_metadata_xml) - except etree.DocumentInvalid as error: - self.qatracker.add_error( - "metadata.bad", - xpkg + "/metadata.xml: %s" - % (str(error)) - ) - del metadata_bad + if not validator.validate(_metadata_xml): + self._add_validate_errors(xpkg, validator.error_log) self.muselist = frozenset(self.musedict) return False @@ -182,7 +174,7 @@ class PkgMetadata(ScanBase, USEFlagChecks): % (xpkg, myflag)) return False - def _parse_metadata_use(self, xml_tree, xpkg, metadata_bad): + def _parse_metadata_use(self, xml_tree, xpkg): """ Records are wrapped in XML as per GLEP 56 returns a dict with keys constisting of USE flag names and values @@ -192,7 +184,7 @@ class PkgMetadata(ScanBase, USEFlagChecks): usetags = xml_tree.findall("use") if not usetags: - return uselist, metadata_bad + return uselist # It's possible to have multiple 'use' elements. for usetag in usetags: @@ -203,37 +195,37 @@ class PkgMetadata(ScanBase, USEFlagChecks): for flag in flags: pkg_flag = flag.get("name") - if pkg_flag is None: - metadata_bad = True - self.qatracker.add_error( - "metadata.bad", - "%s/metadata.xml: line: %s, '%s', missing attribute: name" - % (xpkg, flag.sourceline, flag.text)) - continue - flag_restrict = flag.get("restrict") - - # emulate the Element.itertext() method from python-2.7 - inner_text = [] - stack = [] - stack.append(flag) - while stack: - obj = stack.pop() - if isinstance(obj, basestring): - inner_text.append(obj) - continue - if isinstance(obj.text, basestring): - inner_text.append(obj.text) - if isinstance(obj.tail, basestring): - stack.append(obj.tail) - stack.extend(reversed(obj)) - - if pkg_flag not in uselist: - uselist[pkg_flag] = {} - -
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 8c8ddcfa78eee27a823bc83dc67b34fda77387d6 Author: Brian Dolbec gentoo org> AuthorDate: Thu May 5 16:04:54 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sun May 8 21:18:42 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8c8ddcfa repoman/modules/.../pkgmetadata.py: Add code to remove mostly duplicate errors Some types of errors produce two error messages cluttering up the output. The first error message is clearer, listing the possible option values allowed. This filters out the second error message for that same line and attribute. pym/repoman/modules/scan/metadata/pkgmetadata.py | 13 + 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index d8344c2..433551a 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -221,11 +221,16 @@ class PkgMetadata(ScanBase, USEFlagChecks): return uselist def _add_validate_errors(self, xpkg, log): + listed = set() for error in log: - self.qatracker.add_error( - "metadata.bad", - "%s/metadata.xml: line: %s, %s" - % (xpkg, error.line, error.message)) + msg_prefix = error.message.split(":",1)[0] + info = "%s %s" % (error.line, msg_prefix) + if info not in listed: + listed.add(info) + self.qatracker.add_error( + "metadata.bad", + "%s/metadata.xml: line: %s, %s" + % (xpkg, error.line, error.message)) @property def runInPkgs(self):
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 8075f0e91fc06125d6e2bcaaf655f66de694a568 Author: Dirkjan Ochtman ochtman nl> AuthorDate: Tue May 3 07:18:05 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sun May 8 21:18:41 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8075f0e9 repoman: Use lxml for parsing of metadata Note that we no longer throw a QA error for a missing XML prolog, as long as the encoding matches the default ('UTF-8'; lowercase is also allowed). Update import error message to lxml pkg. pym/repoman/modules/scan/metadata/pkgmetadata.py | 46 +++- 1 file changed, 13 insertions(+), 33 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index bcddb3e..317ab56 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -7,14 +7,14 @@ import sys from itertools import chain try: - import xml.etree.ElementTree - from xml.parsers.expat import ExpatError + from lxml import etree + from lxml.etree import ParserError except (SystemExit, KeyboardInterrupt): raise except (ImportError, SystemError, RuntimeError, Exception): # broken or missing xml support # http://bugs.python.org/issue14988 - msg = ["Please enable python's \"xml\" USE flag in order to use repoman."] + msg = ["Please emerge dev-python/lxml in order to use repoman."] from portage.output import EOutput out = EOutput() for line in msg: @@ -26,12 +26,11 @@ from repoman._portage import portage from repoman.metadata import metadata_dtd_uri from repoman.checks.herds.herdbase import get_herd_base from repoman.checks.herds.metadata import check_metadata, UnknownHerdsError -from repoman._xml import _XMLParser, _MetadataTreeBuilder, XmlLint +from repoman._xml import XmlLint from repoman.modules.scan.scanbase import ScanBase from portage.exception import InvalidAtom from portage import os -from portage import _encodings, _unicode_encode from portage import exception from portage.dep import Atom @@ -145,50 +144,31 @@ class PkgMetadata(ScanBase, USEFlagChecks): # metadata.xml parse check metadata_bad = False - xml_info = {} - xml_parser = _XMLParser(xml_info, target=_MetadataTreeBuilder()) # read metadata.xml into memory try: - _metadata_xml = xml.etree.ElementTree.parse( - _unicode_encode( - os.path.join(checkdir, "metadata.xml"), - encoding=_encodings['fs'], errors='strict'), - parser=xml_parser) - except (ExpatError, SyntaxError, EnvironmentError) as e: + _metadata_xml = etree.parse(os.path.join(checkdir, 'metadata.xml')) + except (ParserError, SyntaxError, EnvironmentError) as e: metadata_bad = True self.qatracker.add_error("metadata.bad", "%s/metadata.xml: %s" % (xpkg, e)) del e self.muselist = frozenset(self.musedict) return False - if "XML_DECLARATION" not in xml_info: + xml_encoding = _metadata_xml.docinfo.encoding + if xml_encoding.upper() != metadata_xml_encoding: self.qatracker.add_error( "metadata.bad", "%s/metadata.xml: " - "xml declaration is missing on first line, " - "should be '%s'" % (xpkg, metadata_xml_declaration)) - else: - xml_version, xml_encoding, xml_standalone = \ - xml_info["XML_DECLARATION"] - if xml_encoding is None or \ - xml_encoding.upper() != metadata_xml_encoding: - if xml_encoding is None: - encoding_problem = "but it is undefined" - else: - encoding_problem = "not '%s'" % xml_encoding - self.qatracker.add_error( - "metadata.bad", "%s/metadata.xml: " - "xml declaration encoding should be '%s', %s" % - (xpkg, metadata_xml_encoding, encoding_problem)) + "xml declaration encoding should be '%s', not '%s'" % + (xpkg, metadata_xml_encoding, xml_encoding)) - if "DOCTYPE" not in xml_info: + if not _metadata_xml.docinfo: metadata_bad = True self
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 01e09705312a9c94058a4ae12a6d01345dbc81d3 Author: Brian Dolbec gentoo org> AuthorDate: Tue Mar 15 18:45:45 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Thu Apr 21 16:49:29 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=01e09705 repoman/modules.scan/metadata/unused.py: Add docstrings Remove un-needed override functions pym/repoman/modules/scan/metadata/unused.py | 21 + 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/unused.py b/pym/repoman/modules/scan/metadata/unused.py index 9ff7e56..a58a614 100644 --- a/pym/repoman/modules/scan/metadata/unused.py +++ b/pym/repoman/modules/scan/metadata/unused.py @@ -3,11 +3,23 @@ from repoman.modules.scan.scanbase import ScanBase class UnusedCheck(ScanBase): + '''Checks and reports any un-used metadata.xml use flag descriptions''' def __init__(self, **kwargs): + '''UnusedCheck init function + + @param qatracker: QATracker instance + ''' self.qatracker = kwargs.get('qatracker') def check(self, **kwargs): + '''Reports on any unused metadata.xml use descriptions + + @param xpkg: the pacakge being checked + @param muselist: use flag list + @param used_useflags: use flag list + @param validity_fuse: Fuse instance + ''' xpkg = kwargs.get('xpkg') muselist = kwargs.get('muselist') used_useflags = kwargs.get('used_useflags') @@ -22,13 +34,6 @@ class UnusedCheck(ScanBase): return {'continue': False} @property - def runInPkgs(self): - return (False, []) - - @property - def runInEbuilds(self): - return (False, []) - - @property def runInFinal(self): + '''Final scans at the package level''' return (True, [self.check])
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/
commit: 926e2dc2975d1bea54454014cda714b426b3d5b9 Author: Brian Dolbec gentoo org> AuthorDate: Tue Mar 15 18:45:45 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Mon Apr 25 15:28:53 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=926e2dc2 repoman/modules.scan/metadata/unused.py: Add docstrings Remove un-needed override functions pym/repoman/modules/scan/metadata/unused.py | 21 + 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/unused.py b/pym/repoman/modules/scan/metadata/unused.py index 9ff7e56..a58a614 100644 --- a/pym/repoman/modules/scan/metadata/unused.py +++ b/pym/repoman/modules/scan/metadata/unused.py @@ -3,11 +3,23 @@ from repoman.modules.scan.scanbase import ScanBase class UnusedCheck(ScanBase): + '''Checks and reports any un-used metadata.xml use flag descriptions''' def __init__(self, **kwargs): + '''UnusedCheck init function + + @param qatracker: QATracker instance + ''' self.qatracker = kwargs.get('qatracker') def check(self, **kwargs): + '''Reports on any unused metadata.xml use descriptions + + @param xpkg: the pacakge being checked + @param muselist: use flag list + @param used_useflags: use flag list + @param validity_fuse: Fuse instance + ''' xpkg = kwargs.get('xpkg') muselist = kwargs.get('muselist') used_useflags = kwargs.get('used_useflags') @@ -22,13 +34,6 @@ class UnusedCheck(ScanBase): return {'continue': False} @property - def runInPkgs(self): - return (False, []) - - @property - def runInEbuilds(self): - return (False, []) - - @property def runInFinal(self): + '''Final scans at the package level''' return (True, [self.check])
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/
commit: 20d723057fe75d128debdb50705d18a3c85449f0 Author: Brian Dolbec gentoo org> AuthorDate: Sun Jan 3 10:35:49 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Thu Jan 21 00:35:22 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=20d72305 repoman: Migrate code from _scan_ebuilds to a new EbuildMetadata class and check pym/repoman/modules/scan/metadata/__init__.py | 10 +- .../modules/scan/metadata/ebuild_metadata.py | 39 ++ pym/repoman/scanner.py | 27 +-- 3 files changed, 49 insertions(+), 27 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py index 7327ec0..eba6565 100644 --- a/pym/repoman/modules/scan/metadata/__init__.py +++ b/pym/repoman/modules/scan/metadata/__init__.py @@ -10,7 +10,7 @@ module_spec = { 'name': 'metadata', 'description': doc, 'provides':{ - 'metadata-module': { + 'pkg-metadata': { 'name': "pkgmetadata", 'class': "PkgMetadata", 'description': doc, @@ -18,6 +18,14 @@ module_spec = { 'func_desc': { }, }, + 'ebuild-metadata': { + 'name': "ebuild_metadata", + 'class': "EbuildMetadata", + 'description': doc, + 'functions': ['check'], + 'func_desc': { + }, + }, } } diff --git a/pym/repoman/modules/scan/metadata/ebuild_metadata.py b/pym/repoman/modules/scan/metadata/ebuild_metadata.py new file mode 100644 index 000..143a40e --- /dev/null +++ b/pym/repoman/modules/scan/metadata/ebuild_metadata.py @@ -0,0 +1,39 @@ +# -*- coding:utf-8 -*- + +'''Ebuild Metadata Checks''' + +import re +import sys + +if sys.hexversion >= 0x300: + basestring = str + +NON_ASCII_RE = re.compile(r'[^\x00-\x7f]') + + +class EbuildMetadata(object): + + def __init__(self, **kwargs): + self.qatracker = kwargs.get('qatracker') + + def check(self, **kwargs): + ebuild = kwargs.get('ebuild') + for k, v in ebuild.metadata.items(): + if not isinstance(v, basestring): + continue + m = NON_ASCII_RE.search(v) + if m is not None: + self.qatracker.add_error( + "variable.invalidchar", + "%s: %s variable contains non-ASCII " + "character at position %s" % + (ebuild.relative_path, k, m.start() + 1)) + return {'continue': False} + + @property + def runInPkgs(self): + return (False, []) + + @property + def runInEbuilds(self): + return (True, [self.check]) diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py index a8aa2f3..6f3fb53 100644 --- a/pym/repoman/scanner.py +++ b/pym/repoman/scanner.py @@ -5,8 +5,6 @@ from __future__ import print_function, unicode_literals import copy import io import logging -import re -import sys from itertools import chain from pprint import pformat @@ -47,18 +45,10 @@ MODULE_CONTROLLER = Modules(path=MODULES_PATH, namepath="repoman.modules.scan") MODULE_NAMES = MODULE_CONTROLLER.module_names[:] - -if sys.hexversion >= 0x300: - basestring = str - -NON_ASCII_RE = re.compile(r'[^\x00-\x7f]') - - def sort_key(item): return item[2].sub_path - class Scanner(object): '''Primary scan class. Operates all the small Q/A tests and checks''' @@ -311,7 +301,7 @@ class Scanner(object): # initialize per ebuild plugin checks here # need to set it up for ==> self.modules_list or some other ordered list for mod in [('ebuild', 'Ebuild'), ('live', 'LiveEclassChecks'), - ('eapi', 'EAPIChecks')]: + ('eapi', 'EAPIChecks'), ('ebuild_metadata', 'EbuildMetadata')]: if mod[0]: mod_class = MODULE_CONTROLLER.get_class(mod[0]) logging.debug("Initializing class name: %s", mod_class.__name__) @@ -338,21 +328,6 @@ class Scanner(object): if y_ebuild_continue: continue - - for k, v in dynamic_data['ebuild'].metadata.items(): - if not isinstance(v, basestring): - continue - m = NON_ASCII_RE.search(v) -
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/
commit: 04cc3461557369a1ba864280925282771232c7fe Author: Brian Dolbec gentoo org> AuthorDate: Sun Jan 3 11:31:26 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Thu Jan 21 00:35:22 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=04cc3461 scanner.py: Migrate another metadata check to ebuild_metadata pym/repoman/modules/scan/metadata/ebuild_metadata.py | 3 +++ pym/repoman/scanner.py | 2 -- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/ebuild_metadata.py b/pym/repoman/modules/scan/metadata/ebuild_metadata.py index 143a40e..2dc1db2 100644 --- a/pym/repoman/modules/scan/metadata/ebuild_metadata.py +++ b/pym/repoman/modules/scan/metadata/ebuild_metadata.py @@ -28,6 +28,9 @@ class EbuildMetadata(object): "%s: %s variable contains non-ASCII " "character at position %s" % (ebuild.relative_path, k, m.start() + 1)) + if ebuild.metadata.get("PROVIDE"): + self.qatracker.add_error("virtual.oldstyle", ebuild.relative_path) + return {'continue': False} @property diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py index e6a17cd..46f46f5 100644 --- a/pym/repoman/scanner.py +++ b/pym/repoman/scanner.py @@ -324,8 +324,6 @@ class Scanner(object): if y_ebuild_continue: continue - if dynamic_data['ebuild'].metadata.get("PROVIDE"): - self.qatracker.add_error("virtual.oldstyle", dynamic_data['ebuild'].relative_path) for pos, missing_var in enumerate(missingvars): if not dynamic_data['ebuild'].metadata.get(missing_var):
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/
commit: 02395b0dea31388acbc5ae58ebc5d90f64b2d10a Author: Brian Dolbec gentoo org> AuthorDate: Mon Jan 4 08:37:22 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Thu Jan 21 02:53:44 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=02395b0d repoman: Create a metadata UnusedCheck and final pkg checks Create a plugin loop for any final pkg checks. Create the one plugin for the unused use-descriptions in mteadata.xml pym/repoman/modules/scan/metadata/__init__.py | 8 ++ pym/repoman/modules/scan/metadata/unused.py | 32 pym/repoman/scanner.py| 36 --- 3 files changed, 67 insertions(+), 9 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py index 6ab44f6..ed4a967 100644 --- a/pym/repoman/modules/scan/metadata/__init__.py +++ b/pym/repoman/modules/scan/metadata/__init__.py @@ -50,6 +50,14 @@ module_spec = { 'func_desc': { }, }, + 'unused-metadata': { + 'name': "unused", + 'class': "UnusedCheck", + 'description': doc, + 'functions': ['check'], + 'func_desc': { + }, + }, } } diff --git a/pym/repoman/modules/scan/metadata/unused.py b/pym/repoman/modules/scan/metadata/unused.py new file mode 100644 index 000..5eb6716 --- /dev/null +++ b/pym/repoman/modules/scan/metadata/unused.py @@ -0,0 +1,32 @@ + + +class UnusedCheck(object): + + def __init__(self, **kwargs): + self.qatracker = kwargs.get('qatracker') + + def check(self, **kwargs): + xpkg = kwargs.get('xpkg') + muselist = kwargs.get('muselist') + used_useflags = kwargs.get('used_useflags') + # check if there are unused local USE-descriptions in metadata.xml + # (unless there are any invalids, to avoid noise) + if kwargs.get('allvalid'): + for myflag in muselist.difference(used_useflags): + self.qatracker.add_error( + "metadata.warning", + "%s/metadata.xml: unused local USE-description: '%s'" + % (xpkg, myflag)) + return {'continue': False} + + @property + def runInPkgs(self): + return (False, []) + + @property + def runInEbuilds(self): + return (False, []) + + @property + def runInFinal(self): + return (True, [self.check]) diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py index 1cd37d0..4cc2e67 100644 --- a/pym/repoman/scanner.py +++ b/pym/repoman/scanner.py @@ -269,7 +269,6 @@ class Scanner(object): def _scan_ebuilds(self, ebuildlist, dynamic_data): - xpkg = dynamic_data['xpkg'] # detect unused local USE-descriptions dynamic_data['used_useflags'] = set() @@ -317,11 +316,30 @@ class Scanner(object): if y_ebuild_continue: continue - # check if there are unused local USE-descriptions in metadata.xml - # (unless there are any invalids, to avoid noise) - if dynamic_data['allvalid']: - for myflag in dynamic_data['muselist'].difference(dynamic_data['used_useflags']): - self.qatracker.add_error( - "metadata.warning", - "%s/metadata.xml: unused local USE-description: '%s'" - % (xpkg, myflag)) + # Final checks + # initialize per pkg plugin final checks here + # need to set it up for ==> self.modules_list or some other ordered list + xpkg_complete = False + for mod in [('unused', 'UnusedChecks')]: + if mod[0]: + mod_class = MODULE_CONTROLLER.get_class(mod[0]) + print("Initializing class name:", mod_class.__name__) + self.modules[mod[1]] = mod_class(**self.kwargs) + print("scan_ebuilds final checks: module:", mod[1]) + do_it, functions = self.modules[mod[1]].runInFinal + # print("do_it", do_it, "functions", functions) + if do_it: + for func in functions: + print("\tRunning function:", func) + rdata = func(**dynamic_data) + if rdata.g
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/
commit: 18d5f07ffda73e32a083baf5b9f4944bcf733718 Author: Brian Dolbec gentoo org> AuthorDate: Sun Jan 3 11:56:25 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Thu Jan 21 00:35:22 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=18d5f07f repoman: Migrate more metadata checks to ebuild_metadata.py .../modules/scan/metadata/ebuild_metadata.py | 32 -- pym/repoman/scanner.py | 17 2 files changed, 30 insertions(+), 19 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/ebuild_metadata.py b/pym/repoman/modules/scan/metadata/ebuild_metadata.py index 2dc1db2..77c947e 100644 --- a/pym/repoman/modules/scan/metadata/ebuild_metadata.py +++ b/pym/repoman/modules/scan/metadata/ebuild_metadata.py @@ -5,6 +5,8 @@ import re import sys +from repoman.qa_data import missingvars + if sys.hexversion >= 0x300: basestring = str @@ -16,7 +18,7 @@ class EbuildMetadata(object): def __init__(self, **kwargs): self.qatracker = kwargs.get('qatracker') - def check(self, **kwargs): + def invalidchar(self, **kwargs): ebuild = kwargs.get('ebuild') for k, v in ebuild.metadata.items(): if not isinstance(v, basestring): @@ -28,9 +30,35 @@ class EbuildMetadata(object): "%s: %s variable contains non-ASCII " "character at position %s" % (ebuild.relative_path, k, m.start() + 1)) + return {'continue': False} + + def missing(self, **kwargs): + ebuild = kwargs.get('ebuild') + for pos, missing_var in enumerate(missingvars): + if not ebuild.metadata.get(missing_var): + if kwargs.get('catdir') == "virtual" and \ + missing_var in ("HOMEPAGE", "LICENSE"): + continue + if kwargs.get('live_ebuild') and missing_var == "KEYWORDS": + continue + myqakey = missingvars[pos] + ".missing" + self.qatracker.add_error(myqakey, '%s/%s.ebuild' + % (kwargs.get('xpkg'), kwargs.get('y_ebuild'))) + return {'continue': False} + + def old_virtual(self, **kwargs): + ebuild = kwargs.get('ebuild') if ebuild.metadata.get("PROVIDE"): self.qatracker.add_error("virtual.oldstyle", ebuild.relative_path) + return {'continue': False} + def virtual(self, **kwargs): + ebuild = kwargs.get('ebuild') + if kwargs.get('catdir') == "virtual": + for var in ("HOMEPAGE", "LICENSE"): + if ebuild.metadata.get(var): + myqakey = var + ".virtual" + self.qatracker.add_error(myqakey, ebuild.relative_path) return {'continue': False} @property @@ -39,4 +67,4 @@ class EbuildMetadata(object): @property def runInEbuilds(self): - return (True, [self.check]) + return (True, [self.invalidchar, self.missing, self.old_virtual, self.virtual]) diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py index 46f46f5..d42fd33 100644 --- a/pym/repoman/scanner.py +++ b/pym/repoman/scanner.py @@ -324,23 +324,6 @@ class Scanner(object): if y_ebuild_continue: continue - - for pos, missing_var in enumerate(missingvars): - if not dynamic_data['ebuild'].metadata.get(missing_var): - if dynamic_data['catdir'] == "virtual" and \ - missing_var in ("HOMEPAGE", "LICENSE"): - continue - if dynamic_data['live_ebuild'] and missing_var == "KEYWORDS": - continue - myqakey = missingvars[pos] + ".missing" - self.qatracker.add_error(myqakey, xpkg + "/" + y_ebuild + ".ebuild") - - if dynamic_data['catdir'] == "virtual": - for var in ("HOMEPAGE", "LICENSE"): - if dynamic_data['ebuild'].metadata.get(var): - myqakey = var + ".virtual" - self.qatracker.add_error(myqakey, dynamic_data['ebuild'].relative_path) - if dynami
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/
commit: fd22acf104e1a2a501ea8ae940de11f62dbd58a6 Author: Brian Dolbec gentoo org> AuthorDate: Thu Jan 21 18:29:07 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Thu Jan 21 19:28:20 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=fd22acf1 repoman: Fix a traceback due to xmlint not being installed for a manifest generation Mike helped find teh fact that self.binary was None, causing the misleading traceback. Tighten up the logic in XmlLint. Bypass the PkgMetadata check for manifest mode. floppym@naomi btrfs-progs % repoman manifest >>> Creating Manifest for /home/floppym/repos/gentoo/sys-fs/btrfs-progs Traceback (most recent call last): File "/home/floppym/bin/repoman", line 37, in repoman_main(sys.argv[1:]) File "/home/floppym/src/portage/pym/repoman/main.py", line 111, in repoman_main can_force = scanner.scan_pkgs(can_force) File "/home/floppym/src/portage/pym/repoman/scanner.py", line 242, in scan_pkgs rdata = func(**dynamic_data) File "/home/floppym/src/portage/pym/repoman/modules/scan/metadata/pkgmetadata.py", line 180, in check if not self.xmllint.check(checkdir, repolevel): File "/home/floppym/src/portage/pym/repoman/_xml.py", line 98, in check os.path.join(checkdir, "metadata.xml" TypeError: unsupported operand type(s) for +: 'NoneType' and 'str' pym/repoman/_xml.py | 2 +- pym/repoman/modules/scan/metadata/pkgmetadata.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/pym/repoman/_xml.py b/pym/repoman/_xml.py index 43fc930..334377d 100644 --- a/pym/repoman/_xml.py +++ b/pym/repoman/_xml.py @@ -58,6 +58,7 @@ class XmlLint(object): self.repoman_settings = repoman_settings self._is_capable = metadata_dtd is not None self.binary = None + self._is_capable = False self._check_capable() def _check_capable(self): @@ -66,7 +67,6 @@ class XmlLint(object): self.binary = find_binary('xmllint') if not self.binary: print(red("!!! xmllint not found. Can't check metadata.xml.\n")) - self._is_capable = False elif not self._is_capable: if not fetch_metadata_dtd(self.metadata_dtd, self.repoman_settings): sys.exit(1) diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index 220fd23..8e93457 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -71,6 +71,9 @@ class PkgMetadata(ScanBase): repolevel = kwargs.get('repolevel') self.musedict = {} + if self.options.mode in ['manifest']: + return {'continue': False, 'muselist': frozenset(self.musedict)} + # metadata.xml file check if "metadata.xml" not in checkdirlist: self.qatracker.add_error("metadata.missing", xpkg + "/metadata.xml")
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/
commit: 4602b7a0c896840ddfa53e92462050aca89e8574 Author: Brian Dolbec gentoo org> AuthorDate: Mon Jan 4 08:37:22 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Thu Jan 21 19:28:19 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=4602b7a0 repoman: Create a metadata UnusedCheck and final pkg checks Create a plugin loop for any final pkg checks. Create the one plugin for the unused use-descriptions in mteadata.xml pym/repoman/modules/scan/metadata/__init__.py | 8 ++ pym/repoman/modules/scan/metadata/unused.py | 32 pym/repoman/scanner.py| 36 --- 3 files changed, 67 insertions(+), 9 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py index 6ab44f6..ed4a967 100644 --- a/pym/repoman/modules/scan/metadata/__init__.py +++ b/pym/repoman/modules/scan/metadata/__init__.py @@ -50,6 +50,14 @@ module_spec = { 'func_desc': { }, }, + 'unused-metadata': { + 'name': "unused", + 'class': "UnusedCheck", + 'description': doc, + 'functions': ['check'], + 'func_desc': { + }, + }, } } diff --git a/pym/repoman/modules/scan/metadata/unused.py b/pym/repoman/modules/scan/metadata/unused.py new file mode 100644 index 000..5eb6716 --- /dev/null +++ b/pym/repoman/modules/scan/metadata/unused.py @@ -0,0 +1,32 @@ + + +class UnusedCheck(object): + + def __init__(self, **kwargs): + self.qatracker = kwargs.get('qatracker') + + def check(self, **kwargs): + xpkg = kwargs.get('xpkg') + muselist = kwargs.get('muselist') + used_useflags = kwargs.get('used_useflags') + # check if there are unused local USE-descriptions in metadata.xml + # (unless there are any invalids, to avoid noise) + if kwargs.get('allvalid'): + for myflag in muselist.difference(used_useflags): + self.qatracker.add_error( + "metadata.warning", + "%s/metadata.xml: unused local USE-description: '%s'" + % (xpkg, myflag)) + return {'continue': False} + + @property + def runInPkgs(self): + return (False, []) + + @property + def runInEbuilds(self): + return (False, []) + + @property + def runInFinal(self): + return (True, [self.check]) diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py index 1cd37d0..4cc2e67 100644 --- a/pym/repoman/scanner.py +++ b/pym/repoman/scanner.py @@ -269,7 +269,6 @@ class Scanner(object): def _scan_ebuilds(self, ebuildlist, dynamic_data): - xpkg = dynamic_data['xpkg'] # detect unused local USE-descriptions dynamic_data['used_useflags'] = set() @@ -317,11 +316,30 @@ class Scanner(object): if y_ebuild_continue: continue - # check if there are unused local USE-descriptions in metadata.xml - # (unless there are any invalids, to avoid noise) - if dynamic_data['allvalid']: - for myflag in dynamic_data['muselist'].difference(dynamic_data['used_useflags']): - self.qatracker.add_error( - "metadata.warning", - "%s/metadata.xml: unused local USE-description: '%s'" - % (xpkg, myflag)) + # Final checks + # initialize per pkg plugin final checks here + # need to set it up for ==> self.modules_list or some other ordered list + xpkg_complete = False + for mod in [('unused', 'UnusedChecks')]: + if mod[0]: + mod_class = MODULE_CONTROLLER.get_class(mod[0]) + print("Initializing class name:", mod_class.__name__) + self.modules[mod[1]] = mod_class(**self.kwargs) + print("scan_ebuilds final checks: module:", mod[1]) + do_it, functions = self.modules[mod[1]].runInFinal + # print("do_it", do_it, "functions", functions) + if do_it: + for func in functions: + print("\tRunning function:", func) + rdata = func(**dynamic_data) + if rdata.g
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/, ...
commit: a475cee940d23a530b533545505c07cf8500e03d Author: Brian Dolbec gentoo org> AuthorDate: Sun Jan 3 23:10:48 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Fri Jan 22 18:44:13 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a475cee9 repoman: Migrate RestrictChecks to a plugin module pym/repoman/modules/scan/metadata/__init__.py | 8 ++ .../scan/metadata}/restrict.py | 29 +++--- pym/repoman/scanner.py | 7 +- 3 files changed, 29 insertions(+), 15 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py index ed0c59d..6ab44f6 100644 --- a/pym/repoman/modules/scan/metadata/__init__.py +++ b/pym/repoman/modules/scan/metadata/__init__.py @@ -42,6 +42,14 @@ module_spec = { 'func_desc': { }, }, + 'restrict-metadata': { + 'name': "restrict", + 'class': "RestrictChecks", + 'description': doc, + 'functions': ['check'], + 'func_desc': { + }, + }, } } diff --git a/pym/repoman/checks/ebuilds/variables/restrict.py b/pym/repoman/modules/scan/metadata/restrict.py similarity index 54% rename from pym/repoman/checks/ebuilds/variables/restrict.py rename to pym/repoman/modules/scan/metadata/restrict.py index 215b792..93ca298 100644 --- a/pym/repoman/checks/ebuilds/variables/restrict.py +++ b/pym/repoman/modules/scan/metadata/restrict.py @@ -12,21 +12,23 @@ from repoman.qa_data import valid_restrict class RestrictChecks(object): '''Perform checks on the RESTRICT variable.''' - def __init__(self, qatracker): + def __init__(self, **kwargs): ''' @param qatracker: QATracker instance ''' - self.qatracker = qatracker + self.qatracker = kwargs.get('qatracker') - def check(self, pkg, package, ebuild, y_ebuild): + def check(self, **kwargs): + xpkg = kwargs.get('xpkg') + ebuild = kwargs.get('ebuild') + y_ebuild = kwargs.get('y_ebuild') myrestrict = None try: myrestrict = portage.dep.use_reduce( - pkg._metadata["RESTRICT"], matchall=1, flat=True) + ebuild.metadata["RESTRICT"], matchall=1, flat=True) except portage.exception.InvalidDependString as e: - self. qatracker.add_error( - "RESTRICT.syntax", + self.qatracker.add_error("RESTRICT.syntax", "%s: RESTRICT: %s" % (ebuild.relative_path, e)) del e @@ -36,6 +38,15 @@ class RestrictChecks(object): if mybadrestrict: for mybad in mybadrestrict: - self.qatracker.add_error( - "RESTRICT.invalid", - package + "/" + y_ebuild + ".ebuild: %s" % mybad) + self.qatracker.add_error("RESTRICT.invalid", + "%s/%s.ebuild: %s" % (xpkg, y_ebuild, mybad)) + return {'continue': False} + + @property + def runInPkgs(self): + return (False, []) + + @property + def runInEbuilds(self): + return (True, [self.check]) + diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py index 1f792bd..8657c73 100644 --- a/pym/repoman/scanner.py +++ b/pym/repoman/scanner.py @@ -18,7 +18,6 @@ from portage import _unicode_encode from portage.dep import Atom from portage.output import green from repoman.checks.ebuilds.checks import run_checks -from repoman.checks.ebuilds.variables.restrict import RestrictChecks from repoman.modules.commit import repochecks from repoman.profile import check_profiles, dev_profile_keywords, setup_profile from repoman.repos import repo_metadata @@ -209,8 +208,6 @@ class Scanner(object): self.modules[mod_class.__name__] = mod_class(**self.kwargs) # initialize our checks classes here before the big xpkg loop - self.restrictcheck = RestrictChecks(self.qatracker) - def scan_pkgs(self, can_force): dynamic_data = {'can_force': can_force} @@ -295,7 +292,7 @@ class Scanner(object): ('description', 'DescriptionChecks'), (None, 'KeywordChecks'), ('arches', 'ArchChecks'), ('depend', 'DependChecks'), ('use_flags', 'USEFlagChecks'), ('ruby', '
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/
commit: 009529b32d753e53e43673c53db8004703a53c17 Author: Brian Dolbec gentoo org> AuthorDate: Sun Jan 3 10:35:49 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Fri Jan 22 18:44:10 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=009529b3 repoman: Migrate code from _scan_ebuilds to a new EbuildMetadata class and check pym/repoman/modules/scan/metadata/__init__.py | 10 +- .../modules/scan/metadata/ebuild_metadata.py | 39 ++ pym/repoman/scanner.py | 27 +-- 3 files changed, 49 insertions(+), 27 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py index 7327ec0..eba6565 100644 --- a/pym/repoman/modules/scan/metadata/__init__.py +++ b/pym/repoman/modules/scan/metadata/__init__.py @@ -10,7 +10,7 @@ module_spec = { 'name': 'metadata', 'description': doc, 'provides':{ - 'metadata-module': { + 'pkg-metadata': { 'name': "pkgmetadata", 'class': "PkgMetadata", 'description': doc, @@ -18,6 +18,14 @@ module_spec = { 'func_desc': { }, }, + 'ebuild-metadata': { + 'name': "ebuild_metadata", + 'class': "EbuildMetadata", + 'description': doc, + 'functions': ['check'], + 'func_desc': { + }, + }, } } diff --git a/pym/repoman/modules/scan/metadata/ebuild_metadata.py b/pym/repoman/modules/scan/metadata/ebuild_metadata.py new file mode 100644 index 000..143a40e --- /dev/null +++ b/pym/repoman/modules/scan/metadata/ebuild_metadata.py @@ -0,0 +1,39 @@ +# -*- coding:utf-8 -*- + +'''Ebuild Metadata Checks''' + +import re +import sys + +if sys.hexversion >= 0x300: + basestring = str + +NON_ASCII_RE = re.compile(r'[^\x00-\x7f]') + + +class EbuildMetadata(object): + + def __init__(self, **kwargs): + self.qatracker = kwargs.get('qatracker') + + def check(self, **kwargs): + ebuild = kwargs.get('ebuild') + for k, v in ebuild.metadata.items(): + if not isinstance(v, basestring): + continue + m = NON_ASCII_RE.search(v) + if m is not None: + self.qatracker.add_error( + "variable.invalidchar", + "%s: %s variable contains non-ASCII " + "character at position %s" % + (ebuild.relative_path, k, m.start() + 1)) + return {'continue': False} + + @property + def runInPkgs(self): + return (False, []) + + @property + def runInEbuilds(self): + return (True, [self.check]) diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py index a8aa2f3..6f3fb53 100644 --- a/pym/repoman/scanner.py +++ b/pym/repoman/scanner.py @@ -5,8 +5,6 @@ from __future__ import print_function, unicode_literals import copy import io import logging -import re -import sys from itertools import chain from pprint import pformat @@ -47,18 +45,10 @@ MODULE_CONTROLLER = Modules(path=MODULES_PATH, namepath="repoman.modules.scan") MODULE_NAMES = MODULE_CONTROLLER.module_names[:] - -if sys.hexversion >= 0x300: - basestring = str - -NON_ASCII_RE = re.compile(r'[^\x00-\x7f]') - - def sort_key(item): return item[2].sub_path - class Scanner(object): '''Primary scan class. Operates all the small Q/A tests and checks''' @@ -311,7 +301,7 @@ class Scanner(object): # initialize per ebuild plugin checks here # need to set it up for ==> self.modules_list or some other ordered list for mod in [('ebuild', 'Ebuild'), ('live', 'LiveEclassChecks'), - ('eapi', 'EAPIChecks')]: + ('eapi', 'EAPIChecks'), ('ebuild_metadata', 'EbuildMetadata')]: if mod[0]: mod_class = MODULE_CONTROLLER.get_class(mod[0]) logging.debug("Initializing class name: %s", mod_class.__name__) @@ -338,21 +328,6 @@ class Scanner(object): if y_ebuild_continue: continue - - for k, v in dynamic_data['ebuild'].metadata.items(): - if not isinstance(v, basestring): - continue - m = NON_ASCII_RE.search(v) -
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/
commit: 2f2c6d6d06c42dff98bdf9950b1d1c14efb819c5 Author: Brian Dolbec gentoo org> AuthorDate: Sun Jan 3 10:35:49 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sat Jan 23 01:23:14 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=2f2c6d6d repoman: Migrate code from _scan_ebuilds to a new EbuildMetadata class and check pym/repoman/modules/scan/metadata/__init__.py | 11 +- .../modules/scan/metadata/ebuild_metadata.py | 39 ++ pym/repoman/scanner.py | 27 +-- 3 files changed, 50 insertions(+), 27 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py index ae9b18b..bacedf5 100644 --- a/pym/repoman/modules/scan/metadata/__init__.py +++ b/pym/repoman/modules/scan/metadata/__init__.py @@ -10,7 +10,7 @@ module_spec = { 'name': 'metadata', 'description': doc, 'provides':{ - 'metadata-module': { + 'pkg-metadata': { 'name': "pkgmetadata", 'sourcefile': "pkgmetadata", 'class': "PkgMetadata", @@ -19,6 +19,15 @@ module_spec = { 'func_desc': { }, }, + 'ebuild-metadata': { + 'name': "ebuild_metadata", + 'sourcefile': "ebuild_metadata", + 'class': "EbuildMetadata", + 'description': doc, + 'functions': ['check'], + 'func_desc': { + }, + }, } } diff --git a/pym/repoman/modules/scan/metadata/ebuild_metadata.py b/pym/repoman/modules/scan/metadata/ebuild_metadata.py new file mode 100644 index 000..143a40e --- /dev/null +++ b/pym/repoman/modules/scan/metadata/ebuild_metadata.py @@ -0,0 +1,39 @@ +# -*- coding:utf-8 -*- + +'''Ebuild Metadata Checks''' + +import re +import sys + +if sys.hexversion >= 0x300: + basestring = str + +NON_ASCII_RE = re.compile(r'[^\x00-\x7f]') + + +class EbuildMetadata(object): + + def __init__(self, **kwargs): + self.qatracker = kwargs.get('qatracker') + + def check(self, **kwargs): + ebuild = kwargs.get('ebuild') + for k, v in ebuild.metadata.items(): + if not isinstance(v, basestring): + continue + m = NON_ASCII_RE.search(v) + if m is not None: + self.qatracker.add_error( + "variable.invalidchar", + "%s: %s variable contains non-ASCII " + "character at position %s" % + (ebuild.relative_path, k, m.start() + 1)) + return {'continue': False} + + @property + def runInPkgs(self): + return (False, []) + + @property + def runInEbuilds(self): + return (True, [self.check]) diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py index a8aa2f3..6f3fb53 100644 --- a/pym/repoman/scanner.py +++ b/pym/repoman/scanner.py @@ -5,8 +5,6 @@ from __future__ import print_function, unicode_literals import copy import io import logging -import re -import sys from itertools import chain from pprint import pformat @@ -47,18 +45,10 @@ MODULE_CONTROLLER = Modules(path=MODULES_PATH, namepath="repoman.modules.scan") MODULE_NAMES = MODULE_CONTROLLER.module_names[:] - -if sys.hexversion >= 0x300: - basestring = str - -NON_ASCII_RE = re.compile(r'[^\x00-\x7f]') - - def sort_key(item): return item[2].sub_path - class Scanner(object): '''Primary scan class. Operates all the small Q/A tests and checks''' @@ -311,7 +301,7 @@ class Scanner(object): # initialize per ebuild plugin checks here # need to set it up for ==> self.modules_list or some other ordered list for mod in [('ebuild', 'Ebuild'), ('live', 'LiveEclassChecks'), - ('eapi', 'EAPIChecks')]: + ('eapi', 'EAPIChecks'), ('ebuild_metadata', 'EbuildMetadata')]: if mod[0]: mod_class = MODULE_CONTROLLER.get_class(mod[0]) logging.debug("Initializing class name: %s", mod_class.__name__) @@ -338,21 +328,6 @@ class Scanner(object): if y_ebuild_continue: continue - - for k, v in dynamic_data['ebuild'].metadata.items(): - if not isinstance(v, basestring): - continue -
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/
commit: 13e0d777ca6dbb32f75e7600a3260ebb4d3803d5 Author: Brian Dolbec gentoo org> AuthorDate: Thu Jan 21 18:29:07 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sat Jan 23 01:38:03 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=13e0d777 repoman: Fix a traceback due to xmlint not being installed for a manifest generation Mike helped find teh fact that self.binary was None, causing the misleading traceback. Tighten up the logic in XmlLint. Bypass the PkgMetadata check for manifest mode. floppym@naomi btrfs-progs % repoman manifest >>> Creating Manifest for /home/floppym/repos/gentoo/sys-fs/btrfs-progs Traceback (most recent call last): File "/home/floppym/bin/repoman", line 37, in repoman_main(sys.argv[1:]) File "/home/floppym/src/portage/pym/repoman/main.py", line 111, in repoman_main can_force = scanner.scan_pkgs(can_force) File "/home/floppym/src/portage/pym/repoman/scanner.py", line 242, in scan_pkgs rdata = func(**dynamic_data) File "/home/floppym/src/portage/pym/repoman/modules/scan/metadata/pkgmetadata.py", line 180, in check if not self.xmllint.check(checkdir, repolevel): File "/home/floppym/src/portage/pym/repoman/_xml.py", line 98, in check os.path.join(checkdir, "metadata.xml" TypeError: unsupported operand type(s) for +: 'NoneType' and 'str' pym/repoman/_xml.py | 2 +- pym/repoman/modules/scan/metadata/pkgmetadata.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/pym/repoman/_xml.py b/pym/repoman/_xml.py index 2661f14..f7ff9fb 100644 --- a/pym/repoman/_xml.py +++ b/pym/repoman/_xml.py @@ -60,6 +60,7 @@ class XmlLint(object): self.repoman_settings = repoman_settings self._is_capable = metadata_dtd is not None self.binary = None + self._is_capable = False self._check_capable() def _check_capable(self): @@ -68,7 +69,6 @@ class XmlLint(object): self.binary = find_binary('xmllint') if not self.binary: print(red("!!! xmllint not found. Can't check metadata.xml.\n")) - self._is_capable = False elif not self._is_capable: if not fetch_metadata_dtd(self.metadata_dtd, self.repoman_settings): sys.exit(1) diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index 220fd23..8e93457 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -71,6 +71,9 @@ class PkgMetadata(ScanBase): repolevel = kwargs.get('repolevel') self.musedict = {} + if self.options.mode in ['manifest']: + return {'continue': False, 'muselist': frozenset(self.musedict)} + # metadata.xml file check if "metadata.xml" not in checkdirlist: self.qatracker.add_error("metadata.missing", xpkg + "/metadata.xml")
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/, ...
commit: 7dfb4ff8da3870591cd28bea9380f9dbb5bd0e9a Author: Brian Dolbec gentoo org> AuthorDate: Sun Jan 3 23:10:48 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sat Jan 23 01:32:22 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=7dfb4ff8 repoman: Migrate RestrictChecks to a plugin module pym/repoman/modules/scan/metadata/__init__.py | 9 +++ .../scan/metadata}/restrict.py | 29 +++--- pym/repoman/scanner.py | 7 +- 3 files changed, 30 insertions(+), 15 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py index c8f3609..4f376e1 100644 --- a/pym/repoman/modules/scan/metadata/__init__.py +++ b/pym/repoman/modules/scan/metadata/__init__.py @@ -46,6 +46,15 @@ module_spec = { 'func_desc': { }, }, + 'restrict-metadata': { + 'name': "restrict", + 'sourcefile': "restrict", + 'class': "RestrictChecks", + 'description': doc, + 'functions': ['check'], + 'func_desc': { + }, + }, } } diff --git a/pym/repoman/checks/ebuilds/variables/restrict.py b/pym/repoman/modules/scan/metadata/restrict.py similarity index 54% rename from pym/repoman/checks/ebuilds/variables/restrict.py rename to pym/repoman/modules/scan/metadata/restrict.py index 215b792..93ca298 100644 --- a/pym/repoman/checks/ebuilds/variables/restrict.py +++ b/pym/repoman/modules/scan/metadata/restrict.py @@ -12,21 +12,23 @@ from repoman.qa_data import valid_restrict class RestrictChecks(object): '''Perform checks on the RESTRICT variable.''' - def __init__(self, qatracker): + def __init__(self, **kwargs): ''' @param qatracker: QATracker instance ''' - self.qatracker = qatracker + self.qatracker = kwargs.get('qatracker') - def check(self, pkg, package, ebuild, y_ebuild): + def check(self, **kwargs): + xpkg = kwargs.get('xpkg') + ebuild = kwargs.get('ebuild') + y_ebuild = kwargs.get('y_ebuild') myrestrict = None try: myrestrict = portage.dep.use_reduce( - pkg._metadata["RESTRICT"], matchall=1, flat=True) + ebuild.metadata["RESTRICT"], matchall=1, flat=True) except portage.exception.InvalidDependString as e: - self. qatracker.add_error( - "RESTRICT.syntax", + self.qatracker.add_error("RESTRICT.syntax", "%s: RESTRICT: %s" % (ebuild.relative_path, e)) del e @@ -36,6 +38,15 @@ class RestrictChecks(object): if mybadrestrict: for mybad in mybadrestrict: - self.qatracker.add_error( - "RESTRICT.invalid", - package + "/" + y_ebuild + ".ebuild: %s" % mybad) + self.qatracker.add_error("RESTRICT.invalid", + "%s/%s.ebuild: %s" % (xpkg, y_ebuild, mybad)) + return {'continue': False} + + @property + def runInPkgs(self): + return (False, []) + + @property + def runInEbuilds(self): + return (True, [self.check]) + diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py index 1f792bd..8657c73 100644 --- a/pym/repoman/scanner.py +++ b/pym/repoman/scanner.py @@ -18,7 +18,6 @@ from portage import _unicode_encode from portage.dep import Atom from portage.output import green from repoman.checks.ebuilds.checks import run_checks -from repoman.checks.ebuilds.variables.restrict import RestrictChecks from repoman.modules.commit import repochecks from repoman.profile import check_profiles, dev_profile_keywords, setup_profile from repoman.repos import repo_metadata @@ -209,8 +208,6 @@ class Scanner(object): self.modules[mod_class.__name__] = mod_class(**self.kwargs) # initialize our checks classes here before the big xpkg loop - self.restrictcheck = RestrictChecks(self.qatracker) - def scan_pkgs(self, can_force): dynamic_data = {'can_force': can_force} @@ -295,7 +292,7 @@ class Scanner(object): ('description', 'DescriptionChecks'), (None, 'KeywordChecks'), ('arches', 'ArchChecks'), ('depend', 'DependChecks'),
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/
commit: 24aa36db0fcd5cddd2ed58e0c5735413a535e1bc Author: Brian Dolbec gentoo org> AuthorDate: Mon Jan 4 08:37:22 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sat Jan 23 01:37:59 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=24aa36db repoman: Create a metadata UnusedCheck and final pkg checks Create a plugin loop for any final pkg checks. Create the one plugin for the unused use-descriptions in mteadata.xml pym/repoman/modules/scan/metadata/__init__.py | 9 +++ pym/repoman/modules/scan/metadata/unused.py | 32 pym/repoman/scanner.py| 36 --- 3 files changed, 68 insertions(+), 9 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py index 4f376e1..f619764 100644 --- a/pym/repoman/modules/scan/metadata/__init__.py +++ b/pym/repoman/modules/scan/metadata/__init__.py @@ -55,6 +55,15 @@ module_spec = { 'func_desc': { }, }, + 'unused-metadata': { + 'name': "unused", + 'sourcefile': "unused", + 'class': "UnusedCheck", + 'description': doc, + 'functions': ['check'], + 'func_desc': { + }, + }, } } diff --git a/pym/repoman/modules/scan/metadata/unused.py b/pym/repoman/modules/scan/metadata/unused.py new file mode 100644 index 000..5eb6716 --- /dev/null +++ b/pym/repoman/modules/scan/metadata/unused.py @@ -0,0 +1,32 @@ + + +class UnusedCheck(object): + + def __init__(self, **kwargs): + self.qatracker = kwargs.get('qatracker') + + def check(self, **kwargs): + xpkg = kwargs.get('xpkg') + muselist = kwargs.get('muselist') + used_useflags = kwargs.get('used_useflags') + # check if there are unused local USE-descriptions in metadata.xml + # (unless there are any invalids, to avoid noise) + if kwargs.get('allvalid'): + for myflag in muselist.difference(used_useflags): + self.qatracker.add_error( + "metadata.warning", + "%s/metadata.xml: unused local USE-description: '%s'" + % (xpkg, myflag)) + return {'continue': False} + + @property + def runInPkgs(self): + return (False, []) + + @property + def runInEbuilds(self): + return (False, []) + + @property + def runInFinal(self): + return (True, [self.check]) diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py index 1cd37d0..4cc2e67 100644 --- a/pym/repoman/scanner.py +++ b/pym/repoman/scanner.py @@ -269,7 +269,6 @@ class Scanner(object): def _scan_ebuilds(self, ebuildlist, dynamic_data): - xpkg = dynamic_data['xpkg'] # detect unused local USE-descriptions dynamic_data['used_useflags'] = set() @@ -317,11 +316,30 @@ class Scanner(object): if y_ebuild_continue: continue - # check if there are unused local USE-descriptions in metadata.xml - # (unless there are any invalids, to avoid noise) - if dynamic_data['allvalid']: - for myflag in dynamic_data['muselist'].difference(dynamic_data['used_useflags']): - self.qatracker.add_error( - "metadata.warning", - "%s/metadata.xml: unused local USE-description: '%s'" - % (xpkg, myflag)) + # Final checks + # initialize per pkg plugin final checks here + # need to set it up for ==> self.modules_list or some other ordered list + xpkg_complete = False + for mod in [('unused', 'UnusedChecks')]: + if mod[0]: + mod_class = MODULE_CONTROLLER.get_class(mod[0]) + print("Initializing class name:", mod_class.__name__) + self.modules[mod[1]] = mod_class(**self.kwargs) + print("scan_ebuilds final checks: module:", mod[1]) + do_it, functions = self.modules[mod[1]].runInFinal + # print("do_it", do_it, "functions", functions) + if do_it: + for func in functions: + print("\tRunning function:", func) + rdata = func(**dynamic_data) +
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/, ...
commit: 7fd7f5ac90f7f6cea917f8c617c47866bf0efcee Author: Brian Dolbec gentoo org> AuthorDate: Sun Jan 3 17:36:26 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Wed Jan 27 22:44:21 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=7fd7f5ac repoman: Migrate DescriptionChecks to the plugin system pym/repoman/modules/scan/metadata/__init__.py| 9 + .../scan/metadata}/description.py| 20 ++-- pym/repoman/scanner.py | 5 + 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py index bacedf5..83aac7f 100644 --- a/pym/repoman/modules/scan/metadata/__init__.py +++ b/pym/repoman/modules/scan/metadata/__init__.py @@ -28,6 +28,15 @@ module_spec = { 'func_desc': { }, }, + 'description-metadata': { + 'name': "description", + 'sourcefile': "description", + 'class': "DescriptionChecks", + 'description': doc, + 'functions': ['check'], + 'func_desc': { + }, + }, } } diff --git a/pym/repoman/checks/ebuilds/variables/description.py b/pym/repoman/modules/scan/metadata/description.py similarity index 66% rename from pym/repoman/checks/ebuilds/variables/description.py rename to pym/repoman/modules/scan/metadata/description.py index a2b1057..3570607 100644 --- a/pym/repoman/checks/ebuilds/variables/description.py +++ b/pym/repoman/modules/scan/metadata/description.py @@ -9,20 +9,19 @@ from repoman.qa_data import max_desc_len class DescriptionChecks(object): '''Perform checks on the DESCRIPTION variable.''' - def __init__(self, qatracker): + def __init__(self, **kwargs): ''' @param qatracker: QATracker instance ''' - self.qatracker = qatracker + self.qatracker = kwargs.get('qatracker') - def check(self, pkg, ebuild): + def checkTooLong(self, **kwargs): ''' @param pkg: Package in which we check (object). @param ebuild: Ebuild which we check (object). ''' - self._checkTooLong(pkg, ebuild) - - def _checkTooLong(self, pkg, ebuild): + ebuild = kwargs.get('ebuild') + pkg = kwargs.get('pkg') # 14 is the length of DESCRIPTION="" if len(pkg._metadata['DESCRIPTION']) > max_desc_len: self.qatracker.add_error( @@ -30,3 +29,12 @@ class DescriptionChecks(object): "%s: DESCRIPTION is %d characters (max %d)" % (ebuild.relative_path, len( pkg._metadata['DESCRIPTION']), max_desc_len)) + return {'continue': False} + + @property + def runInPkgs(self): + return (False, []) + + @property + def runInEbuilds(self): + return (True, [self.checkTooLong]) diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py index 29edcd8..b9e80ff 100644 --- a/pym/repoman/scanner.py +++ b/pym/repoman/scanner.py @@ -21,7 +21,6 @@ from repoman.checks.ebuilds.checks import run_checks from repoman.checks.ebuilds.eclasses.ruby import RubyEclassChecks from repoman.check_missingslot import check_missingslot from repoman.checks.ebuilds.use_flags import USEFlagChecks -from repoman.checks.ebuilds.variables.description import DescriptionChecks from repoman.checks.ebuilds.variables.license import LicenseChecks from repoman.checks.ebuilds.variables.restrict import RestrictChecks from repoman.modules.commit import repochecks @@ -219,7 +218,6 @@ class Scanner(object): # initialize our checks classes here before the big xpkg loop self.use_flag_checks = USEFlagChecks(self.qatracker, uselist) self.rubyeclasscheck = RubyEclassChecks(self.qatracker) - self.descriptioncheck = DescriptionChecks(self.qatracker) self.licensecheck = LicenseChecks(self.qatracker, liclist, liclist_deprecated) self.restrictcheck = RestrictChecks(self.qatracker) @@ -304,6 +302,7 @@ class Scanner(object): for mod in [('ebuild', 'Ebuild'), ('live', 'LiveEclassChecks'), ('eapi', 'EAPIChecks'), ('ebuild_metadata', 'EbuildMetadata'), ('thirdpartymirrors', 'ThirdPartyMirrors'), + ('description', 'DescriptionChecks'), ]: if mod[0]: mod_class = MODULE_CONTR
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/
commit: 20b86b331841e540b9fca8b22c0e615ed1fb7c77 Author: Brian Dolbec gentoo org> AuthorDate: Mon Jan 4 08:37:22 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Wed Jan 27 22:44:25 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=20b86b33 repoman: Create a metadata UnusedCheck and final pkg checks Create a plugin loop for any final pkg checks. Create the one plugin for the unused use-descriptions in mteadata.xml pym/repoman/modules/scan/metadata/__init__.py | 9 +++ pym/repoman/modules/scan/metadata/unused.py | 32 pym/repoman/scanner.py| 36 --- 3 files changed, 68 insertions(+), 9 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py index 4f376e1..f619764 100644 --- a/pym/repoman/modules/scan/metadata/__init__.py +++ b/pym/repoman/modules/scan/metadata/__init__.py @@ -55,6 +55,15 @@ module_spec = { 'func_desc': { }, }, + 'unused-metadata': { + 'name': "unused", + 'sourcefile': "unused", + 'class': "UnusedCheck", + 'description': doc, + 'functions': ['check'], + 'func_desc': { + }, + }, } } diff --git a/pym/repoman/modules/scan/metadata/unused.py b/pym/repoman/modules/scan/metadata/unused.py new file mode 100644 index 000..5eb6716 --- /dev/null +++ b/pym/repoman/modules/scan/metadata/unused.py @@ -0,0 +1,32 @@ + + +class UnusedCheck(object): + + def __init__(self, **kwargs): + self.qatracker = kwargs.get('qatracker') + + def check(self, **kwargs): + xpkg = kwargs.get('xpkg') + muselist = kwargs.get('muselist') + used_useflags = kwargs.get('used_useflags') + # check if there are unused local USE-descriptions in metadata.xml + # (unless there are any invalids, to avoid noise) + if kwargs.get('allvalid'): + for myflag in muselist.difference(used_useflags): + self.qatracker.add_error( + "metadata.warning", + "%s/metadata.xml: unused local USE-description: '%s'" + % (xpkg, myflag)) + return {'continue': False} + + @property + def runInPkgs(self): + return (False, []) + + @property + def runInEbuilds(self): + return (False, []) + + @property + def runInFinal(self): + return (True, [self.check]) diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py index 4472f88..e486282 100644 --- a/pym/repoman/scanner.py +++ b/pym/repoman/scanner.py @@ -272,7 +272,6 @@ class Scanner(object): def _scan_ebuilds(self, ebuildlist, dynamic_data): - xpkg = dynamic_data['xpkg'] # detect unused local USE-descriptions dynamic_data['used_useflags'] = set() @@ -320,11 +319,30 @@ class Scanner(object): if y_ebuild_continue: continue - # check if there are unused local USE-descriptions in metadata.xml - # (unless there are any invalids, to avoid noise) - if dynamic_data['allvalid']: - for myflag in dynamic_data['muselist'].difference(dynamic_data['used_useflags']): - self.qatracker.add_error( - "metadata.warning", - "%s/metadata.xml: unused local USE-description: '%s'" - % (xpkg, myflag)) + # Final checks + # initialize per pkg plugin final checks here + # need to set it up for ==> self.modules_list or some other ordered list + xpkg_complete = False + for mod in [('unused', 'UnusedChecks')]: + if mod[0]: + mod_class = MODULE_CONTROLLER.get_class(mod[0]) + print("Initializing class name:", mod_class.__name__) + self.modules[mod[1]] = mod_class(**self.kwargs) + print("scan_ebuilds final checks: module:", mod[1]) + do_it, functions = self.modules[mod[1]].runInFinal + # print("do_it", do_it, "functions", functions) + if do_it: + for func in functions: + print("\tRunning function:", func) + rdata = func(**dynamic_data) +
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/
commit: 0dcc20356b9c4e482c0e898d4a16fe63fc13e14d Author: Brian Dolbec gentoo org> AuthorDate: Sun Jan 3 11:56:25 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Wed Jan 27 22:44:23 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=0dcc2035 repoman: Migrate more metadata checks to ebuild_metadata.py .../modules/scan/metadata/ebuild_metadata.py | 32 -- pym/repoman/scanner.py | 17 2 files changed, 30 insertions(+), 19 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/ebuild_metadata.py b/pym/repoman/modules/scan/metadata/ebuild_metadata.py index 2dc1db2..77c947e 100644 --- a/pym/repoman/modules/scan/metadata/ebuild_metadata.py +++ b/pym/repoman/modules/scan/metadata/ebuild_metadata.py @@ -5,6 +5,8 @@ import re import sys +from repoman.qa_data import missingvars + if sys.hexversion >= 0x300: basestring = str @@ -16,7 +18,7 @@ class EbuildMetadata(object): def __init__(self, **kwargs): self.qatracker = kwargs.get('qatracker') - def check(self, **kwargs): + def invalidchar(self, **kwargs): ebuild = kwargs.get('ebuild') for k, v in ebuild.metadata.items(): if not isinstance(v, basestring): @@ -28,9 +30,35 @@ class EbuildMetadata(object): "%s: %s variable contains non-ASCII " "character at position %s" % (ebuild.relative_path, k, m.start() + 1)) + return {'continue': False} + + def missing(self, **kwargs): + ebuild = kwargs.get('ebuild') + for pos, missing_var in enumerate(missingvars): + if not ebuild.metadata.get(missing_var): + if kwargs.get('catdir') == "virtual" and \ + missing_var in ("HOMEPAGE", "LICENSE"): + continue + if kwargs.get('live_ebuild') and missing_var == "KEYWORDS": + continue + myqakey = missingvars[pos] + ".missing" + self.qatracker.add_error(myqakey, '%s/%s.ebuild' + % (kwargs.get('xpkg'), kwargs.get('y_ebuild'))) + return {'continue': False} + + def old_virtual(self, **kwargs): + ebuild = kwargs.get('ebuild') if ebuild.metadata.get("PROVIDE"): self.qatracker.add_error("virtual.oldstyle", ebuild.relative_path) + return {'continue': False} + def virtual(self, **kwargs): + ebuild = kwargs.get('ebuild') + if kwargs.get('catdir') == "virtual": + for var in ("HOMEPAGE", "LICENSE"): + if ebuild.metadata.get(var): + myqakey = var + ".virtual" + self.qatracker.add_error(myqakey, ebuild.relative_path) return {'continue': False} @property @@ -39,4 +67,4 @@ class EbuildMetadata(object): @property def runInEbuilds(self): - return (True, [self.check]) + return (True, [self.invalidchar, self.missing, self.old_virtual, self.virtual]) diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py index 3ea27b6..335590b 100644 --- a/pym/repoman/scanner.py +++ b/pym/repoman/scanner.py @@ -327,23 +327,6 @@ class Scanner(object): if y_ebuild_continue: continue - - for pos, missing_var in enumerate(missingvars): - if not dynamic_data['ebuild'].metadata.get(missing_var): - if dynamic_data['catdir'] == "virtual" and \ - missing_var in ("HOMEPAGE", "LICENSE"): - continue - if dynamic_data['live_ebuild'] and missing_var == "KEYWORDS": - continue - myqakey = missingvars[pos] + ".missing" - self.qatracker.add_error(myqakey, xpkg + "/" + y_ebuild + ".ebuild") - - if dynamic_data['catdir'] == "virtual": - for var in ("HOMEPAGE", "LICENSE"): - if dynamic_data['ebuild'].metadata.get(var): - myqakey = var + ".virtual" - self.qatracker.add_error(myqakey, dynamic_data['ebuild'].relative_path) - if dynami
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/
commit: 563f65e4b167f213981ec030c71ebd35a2ca9780 Author: Brian Dolbec gentoo org> AuthorDate: Thu Jan 21 18:29:07 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Wed Jan 27 22:44:26 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=563f65e4 repoman: Fix a traceback due to xmlint not being installed for a manifest generation Mike helped find teh fact that self.binary was None, causing the misleading traceback. Tighten up the logic in XmlLint. Bypass the PkgMetadata check for manifest mode. floppym@naomi btrfs-progs % repoman manifest >>> Creating Manifest for /home/floppym/repos/gentoo/sys-fs/btrfs-progs Traceback (most recent call last): File "/home/floppym/bin/repoman", line 37, in repoman_main(sys.argv[1:]) File "/home/floppym/src/portage/pym/repoman/main.py", line 111, in repoman_main can_force = scanner.scan_pkgs(can_force) File "/home/floppym/src/portage/pym/repoman/scanner.py", line 242, in scan_pkgs rdata = func(**dynamic_data) File "/home/floppym/src/portage/pym/repoman/modules/scan/metadata/pkgmetadata.py", line 180, in check if not self.xmllint.check(checkdir, repolevel): File "/home/floppym/src/portage/pym/repoman/_xml.py", line 98, in check os.path.join(checkdir, "metadata.xml" TypeError: unsupported operand type(s) for +: 'NoneType' and 'str' pym/repoman/_xml.py | 2 +- pym/repoman/modules/scan/metadata/pkgmetadata.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/pym/repoman/_xml.py b/pym/repoman/_xml.py index 2661f14..f7ff9fb 100644 --- a/pym/repoman/_xml.py +++ b/pym/repoman/_xml.py @@ -60,6 +60,7 @@ class XmlLint(object): self.repoman_settings = repoman_settings self._is_capable = metadata_dtd is not None self.binary = None + self._is_capable = False self._check_capable() def _check_capable(self): @@ -68,7 +69,6 @@ class XmlLint(object): self.binary = find_binary('xmllint') if not self.binary: print(red("!!! xmllint not found. Can't check metadata.xml.\n")) - self._is_capable = False elif not self._is_capable: if not fetch_metadata_dtd(self.metadata_dtd, self.repoman_settings): sys.exit(1) diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index 220fd23..8e93457 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -71,6 +71,9 @@ class PkgMetadata(ScanBase): repolevel = kwargs.get('repolevel') self.musedict = {} + if self.options.mode in ['manifest']: + return {'continue': False, 'muselist': frozenset(self.musedict)} + # metadata.xml file check if "metadata.xml" not in checkdirlist: self.qatracker.add_error("metadata.missing", xpkg + "/metadata.xml")
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/, ...
commit: 12189a5924aefef99a9cf9684b6aed7c0720b38a Author: Brian Dolbec gentoo org> AuthorDate: Sun Jan 3 23:10:48 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Wed Jan 27 22:44:24 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=12189a59 repoman: Migrate RestrictChecks to a plugin module pym/repoman/modules/scan/metadata/__init__.py | 9 +++ .../scan/metadata}/restrict.py | 29 +++--- pym/repoman/scanner.py | 7 +- 3 files changed, 30 insertions(+), 15 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py index c8f3609..4f376e1 100644 --- a/pym/repoman/modules/scan/metadata/__init__.py +++ b/pym/repoman/modules/scan/metadata/__init__.py @@ -46,6 +46,15 @@ module_spec = { 'func_desc': { }, }, + 'restrict-metadata': { + 'name': "restrict", + 'sourcefile': "restrict", + 'class': "RestrictChecks", + 'description': doc, + 'functions': ['check'], + 'func_desc': { + }, + }, } } diff --git a/pym/repoman/checks/ebuilds/variables/restrict.py b/pym/repoman/modules/scan/metadata/restrict.py similarity index 54% rename from pym/repoman/checks/ebuilds/variables/restrict.py rename to pym/repoman/modules/scan/metadata/restrict.py index 215b792..93ca298 100644 --- a/pym/repoman/checks/ebuilds/variables/restrict.py +++ b/pym/repoman/modules/scan/metadata/restrict.py @@ -12,21 +12,23 @@ from repoman.qa_data import valid_restrict class RestrictChecks(object): '''Perform checks on the RESTRICT variable.''' - def __init__(self, qatracker): + def __init__(self, **kwargs): ''' @param qatracker: QATracker instance ''' - self.qatracker = qatracker + self.qatracker = kwargs.get('qatracker') - def check(self, pkg, package, ebuild, y_ebuild): + def check(self, **kwargs): + xpkg = kwargs.get('xpkg') + ebuild = kwargs.get('ebuild') + y_ebuild = kwargs.get('y_ebuild') myrestrict = None try: myrestrict = portage.dep.use_reduce( - pkg._metadata["RESTRICT"], matchall=1, flat=True) + ebuild.metadata["RESTRICT"], matchall=1, flat=True) except portage.exception.InvalidDependString as e: - self. qatracker.add_error( - "RESTRICT.syntax", + self.qatracker.add_error("RESTRICT.syntax", "%s: RESTRICT: %s" % (ebuild.relative_path, e)) del e @@ -36,6 +38,15 @@ class RestrictChecks(object): if mybadrestrict: for mybad in mybadrestrict: - self.qatracker.add_error( - "RESTRICT.invalid", - package + "/" + y_ebuild + ".ebuild: %s" % mybad) + self.qatracker.add_error("RESTRICT.invalid", + "%s/%s.ebuild: %s" % (xpkg, y_ebuild, mybad)) + return {'continue': False} + + @property + def runInPkgs(self): + return (False, []) + + @property + def runInEbuilds(self): + return (True, [self.check]) + diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py index 697724f..1b6bbb4 100644 --- a/pym/repoman/scanner.py +++ b/pym/repoman/scanner.py @@ -18,7 +18,6 @@ from portage import _unicode_encode from portage.dep import Atom from portage.output import green from repoman.checks.ebuilds.checks import run_checks -from repoman.checks.ebuilds.variables.restrict import RestrictChecks from repoman.modules.commit import repochecks from repoman.profile import check_profiles, dev_profile_keywords, setup_profile from repoman.repos import repo_metadata @@ -212,8 +211,6 @@ class Scanner(object): self.modules[mod_class.__name__] = mod_class(**self.kwargs) # initialize our checks classes here before the big xpkg loop - self.restrictcheck = RestrictChecks(self.qatracker) - def scan_pkgs(self, can_force): dynamic_data = {'can_force': can_force} @@ -298,7 +295,7 @@ class Scanner(object): ('description', 'DescriptionChecks'), (None, 'KeywordChecks'), ('arches', 'ArchChecks'), ('depend', 'DependChecks'),
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/
commit: 3c7b49d26de283239a1789832972353a623ec676 Author: Brian Dolbec gentoo org> AuthorDate: Thu Jan 21 18:29:07 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Fri Jan 29 04:53:00 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3c7b49d2 repoman: Fix a traceback due to xmlint not being installed for a manifest generation Mike helped find teh fact that self.binary was None, causing the misleading traceback. Tighten up the logic in XmlLint. Bypass the PkgMetadata check for manifest mode. floppym@naomi btrfs-progs % repoman manifest >>> Creating Manifest for /home/floppym/repos/gentoo/sys-fs/btrfs-progs Traceback (most recent call last): File "/home/floppym/bin/repoman", line 37, in repoman_main(sys.argv[1:]) File "/home/floppym/src/portage/pym/repoman/main.py", line 111, in repoman_main can_force = scanner.scan_pkgs(can_force) File "/home/floppym/src/portage/pym/repoman/scanner.py", line 242, in scan_pkgs rdata = func(**dynamic_data) File "/home/floppym/src/portage/pym/repoman/modules/scan/metadata/pkgmetadata.py", line 180, in check if not self.xmllint.check(checkdir, repolevel): File "/home/floppym/src/portage/pym/repoman/_xml.py", line 98, in check os.path.join(checkdir, "metadata.xml" TypeError: unsupported operand type(s) for +: 'NoneType' and 'str' pym/repoman/_xml.py | 2 +- pym/repoman/modules/scan/metadata/pkgmetadata.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/pym/repoman/_xml.py b/pym/repoman/_xml.py index 2661f14..f7ff9fb 100644 --- a/pym/repoman/_xml.py +++ b/pym/repoman/_xml.py @@ -60,6 +60,7 @@ class XmlLint(object): self.repoman_settings = repoman_settings self._is_capable = metadata_dtd is not None self.binary = None + self._is_capable = False self._check_capable() def _check_capable(self): @@ -68,7 +69,6 @@ class XmlLint(object): self.binary = find_binary('xmllint') if not self.binary: print(red("!!! xmllint not found. Can't check metadata.xml.\n")) - self._is_capable = False elif not self._is_capable: if not fetch_metadata_dtd(self.metadata_dtd, self.repoman_settings): sys.exit(1) diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index 220fd23..8e93457 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -71,6 +71,9 @@ class PkgMetadata(ScanBase): repolevel = kwargs.get('repolevel') self.musedict = {} + if self.options.mode in ['manifest']: + return {'continue': False, 'muselist': frozenset(self.musedict)} + # metadata.xml file check if "metadata.xml" not in checkdirlist: self.qatracker.add_error("metadata.missing", xpkg + "/metadata.xml")
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/, ...
commit: b94feb66d48bf58770c84ee70154faa21dbb5174 Author: Brian Dolbec gentoo org> AuthorDate: Sun Jan 3 23:10:48 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Fri Jan 29 04:52:58 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b94feb66 repoman: Migrate RestrictChecks to a plugin module pym/repoman/modules/scan/metadata/__init__.py | 9 +++ .../scan/metadata}/restrict.py | 29 +++--- pym/repoman/scanner.py | 7 +- 3 files changed, 30 insertions(+), 15 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py index c8f3609..4f376e1 100644 --- a/pym/repoman/modules/scan/metadata/__init__.py +++ b/pym/repoman/modules/scan/metadata/__init__.py @@ -46,6 +46,15 @@ module_spec = { 'func_desc': { }, }, + 'restrict-metadata': { + 'name': "restrict", + 'sourcefile': "restrict", + 'class': "RestrictChecks", + 'description': doc, + 'functions': ['check'], + 'func_desc': { + }, + }, } } diff --git a/pym/repoman/checks/ebuilds/variables/restrict.py b/pym/repoman/modules/scan/metadata/restrict.py similarity index 54% rename from pym/repoman/checks/ebuilds/variables/restrict.py rename to pym/repoman/modules/scan/metadata/restrict.py index 215b792..93ca298 100644 --- a/pym/repoman/checks/ebuilds/variables/restrict.py +++ b/pym/repoman/modules/scan/metadata/restrict.py @@ -12,21 +12,23 @@ from repoman.qa_data import valid_restrict class RestrictChecks(object): '''Perform checks on the RESTRICT variable.''' - def __init__(self, qatracker): + def __init__(self, **kwargs): ''' @param qatracker: QATracker instance ''' - self.qatracker = qatracker + self.qatracker = kwargs.get('qatracker') - def check(self, pkg, package, ebuild, y_ebuild): + def check(self, **kwargs): + xpkg = kwargs.get('xpkg') + ebuild = kwargs.get('ebuild') + y_ebuild = kwargs.get('y_ebuild') myrestrict = None try: myrestrict = portage.dep.use_reduce( - pkg._metadata["RESTRICT"], matchall=1, flat=True) + ebuild.metadata["RESTRICT"], matchall=1, flat=True) except portage.exception.InvalidDependString as e: - self. qatracker.add_error( - "RESTRICT.syntax", + self.qatracker.add_error("RESTRICT.syntax", "%s: RESTRICT: %s" % (ebuild.relative_path, e)) del e @@ -36,6 +38,15 @@ class RestrictChecks(object): if mybadrestrict: for mybad in mybadrestrict: - self.qatracker.add_error( - "RESTRICT.invalid", - package + "/" + y_ebuild + ".ebuild: %s" % mybad) + self.qatracker.add_error("RESTRICT.invalid", + "%s/%s.ebuild: %s" % (xpkg, y_ebuild, mybad)) + return {'continue': False} + + @property + def runInPkgs(self): + return (False, []) + + @property + def runInEbuilds(self): + return (True, [self.check]) + diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py index 697724f..1b6bbb4 100644 --- a/pym/repoman/scanner.py +++ b/pym/repoman/scanner.py @@ -18,7 +18,6 @@ from portage import _unicode_encode from portage.dep import Atom from portage.output import green from repoman.checks.ebuilds.checks import run_checks -from repoman.checks.ebuilds.variables.restrict import RestrictChecks from repoman.modules.commit import repochecks from repoman.profile import check_profiles, dev_profile_keywords, setup_profile from repoman.repos import repo_metadata @@ -212,8 +211,6 @@ class Scanner(object): self.modules[mod_class.__name__] = mod_class(**self.kwargs) # initialize our checks classes here before the big xpkg loop - self.restrictcheck = RestrictChecks(self.qatracker) - def scan_pkgs(self, can_force): dynamic_data = {'can_force': can_force} @@ -298,7 +295,7 @@ class Scanner(object): ('description', 'DescriptionChecks'), (None, 'KeywordChecks'), ('arches', 'ArchChecks'), ('depend', 'DependChecks'),
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/
commit: b1711bafa09ca63c8d256c4a57dd49d25d049cd7 Author: Brian Dolbec gentoo org> AuthorDate: Mon Jan 4 08:37:22 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sat Jan 30 06:51:58 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b1711baf repoman: Create a metadata UnusedCheck and final pkg checks Create a plugin loop for any final pkg checks. Create the one plugin for the unused use-descriptions in mteadata.xml. pym/repoman/modules/scan/metadata/__init__.py | 9 +++ pym/repoman/modules/scan/metadata/unused.py | 32 pym/repoman/scanner.py| 36 --- 3 files changed, 68 insertions(+), 9 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py index 4f376e1..f619764 100644 --- a/pym/repoman/modules/scan/metadata/__init__.py +++ b/pym/repoman/modules/scan/metadata/__init__.py @@ -55,6 +55,15 @@ module_spec = { 'func_desc': { }, }, + 'unused-metadata': { + 'name': "unused", + 'sourcefile': "unused", + 'class': "UnusedCheck", + 'description': doc, + 'functions': ['check'], + 'func_desc': { + }, + }, } } diff --git a/pym/repoman/modules/scan/metadata/unused.py b/pym/repoman/modules/scan/metadata/unused.py new file mode 100644 index 000..5eb6716 --- /dev/null +++ b/pym/repoman/modules/scan/metadata/unused.py @@ -0,0 +1,32 @@ + + +class UnusedCheck(object): + + def __init__(self, **kwargs): + self.qatracker = kwargs.get('qatracker') + + def check(self, **kwargs): + xpkg = kwargs.get('xpkg') + muselist = kwargs.get('muselist') + used_useflags = kwargs.get('used_useflags') + # check if there are unused local USE-descriptions in metadata.xml + # (unless there are any invalids, to avoid noise) + if kwargs.get('allvalid'): + for myflag in muselist.difference(used_useflags): + self.qatracker.add_error( + "metadata.warning", + "%s/metadata.xml: unused local USE-description: '%s'" + % (xpkg, myflag)) + return {'continue': False} + + @property + def runInPkgs(self): + return (False, []) + + @property + def runInEbuilds(self): + return (False, []) + + @property + def runInFinal(self): + return (True, [self.check]) diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py index 1e61eb6..4dd5408 100644 --- a/pym/repoman/scanner.py +++ b/pym/repoman/scanner.py @@ -270,7 +270,6 @@ class Scanner(object): def _scan_ebuilds(self, ebuildlist, dynamic_data): - xpkg = dynamic_data['xpkg'] # detect unused local USE-descriptions dynamic_data['used_useflags'] = set() @@ -318,11 +317,30 @@ class Scanner(object): if y_ebuild_continue: continue - # check if there are unused local USE-descriptions in metadata.xml - # (unless there are any invalids, to avoid noise) - if dynamic_data['allvalid']: - for myflag in dynamic_data['muselist'].difference(dynamic_data['used_useflags']): - self.qatracker.add_error( - "metadata.warning", - "%s/metadata.xml: unused local USE-description: '%s'" - % (xpkg, myflag)) + # Final checks + # initialize per pkg plugin final checks here + # need to set it up for ==> self.modules_list or some other ordered list + xpkg_complete = False + for mod in [('unused', 'UnusedChecks')]: + if mod[0]: + mod_class = MODULE_CONTROLLER.get_class(mod[0]) + print("Initializing class name:", mod_class.__name__) + self.modules[mod[1]] = mod_class(**self.kwargs) + print("scan_ebuilds final checks: module:", mod[1]) + do_it, functions = self.modules[mod[1]].runInFinal + # print("do_it", do_it, "functions", functions) + if do_it: + for func in functions: + print("\tRunning function:", func) + rdata = func(**dynamic_data)
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/
commit: 75d44c473dfe67a828c33a6ce3f097ab3c483f13 Author: Brian Dolbec gentoo org> AuthorDate: Sun Jan 3 11:31:26 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sat Jan 30 07:50:18 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=75d44c47 scanner.py: Migrate another metadata check to ebuild_metadata pym/repoman/modules/scan/metadata/ebuild_metadata.py | 3 +++ pym/repoman/scanner.py | 2 -- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/ebuild_metadata.py b/pym/repoman/modules/scan/metadata/ebuild_metadata.py index 143a40e..2dc1db2 100644 --- a/pym/repoman/modules/scan/metadata/ebuild_metadata.py +++ b/pym/repoman/modules/scan/metadata/ebuild_metadata.py @@ -28,6 +28,9 @@ class EbuildMetadata(object): "%s: %s variable contains non-ASCII " "character at position %s" % (ebuild.relative_path, k, m.start() + 1)) + if ebuild.metadata.get("PROVIDE"): + self.qatracker.add_error("virtual.oldstyle", ebuild.relative_path) + return {'continue': False} @property diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py index 006c3f7..803e5a3 100644 --- a/pym/repoman/scanner.py +++ b/pym/repoman/scanner.py @@ -327,8 +327,6 @@ class Scanner(object): if y_ebuild_continue: continue - if dynamic_data['ebuild'].metadata.get("PROVIDE"): - self.qatracker.add_error("virtual.oldstyle", dynamic_data['ebuild'].relative_path) for pos, missing_var in enumerate(missingvars): if not dynamic_data['ebuild'].metadata.get(missing_var):
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/
commit: 96c4b3db3f34c1e5f0996fd7975a73c846cd7ff5 Author: Brian Dolbec gentoo org> AuthorDate: Mon Jan 4 08:37:22 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sat Jan 30 07:50:20 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=96c4b3db repoman: Create a metadata UnusedCheck and final pkg checks Create a plugin loop for any final pkg checks. Create the one plugin for the unused use-descriptions in mteadata.xml. pym/repoman/modules/scan/metadata/__init__.py | 9 +++ pym/repoman/modules/scan/metadata/unused.py | 32 pym/repoman/scanner.py| 36 --- 3 files changed, 68 insertions(+), 9 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py index 4f376e1..f619764 100644 --- a/pym/repoman/modules/scan/metadata/__init__.py +++ b/pym/repoman/modules/scan/metadata/__init__.py @@ -55,6 +55,15 @@ module_spec = { 'func_desc': { }, }, + 'unused-metadata': { + 'name': "unused", + 'sourcefile': "unused", + 'class': "UnusedCheck", + 'description': doc, + 'functions': ['check'], + 'func_desc': { + }, + }, } } diff --git a/pym/repoman/modules/scan/metadata/unused.py b/pym/repoman/modules/scan/metadata/unused.py new file mode 100644 index 000..5eb6716 --- /dev/null +++ b/pym/repoman/modules/scan/metadata/unused.py @@ -0,0 +1,32 @@ + + +class UnusedCheck(object): + + def __init__(self, **kwargs): + self.qatracker = kwargs.get('qatracker') + + def check(self, **kwargs): + xpkg = kwargs.get('xpkg') + muselist = kwargs.get('muselist') + used_useflags = kwargs.get('used_useflags') + # check if there are unused local USE-descriptions in metadata.xml + # (unless there are any invalids, to avoid noise) + if kwargs.get('allvalid'): + for myflag in muselist.difference(used_useflags): + self.qatracker.add_error( + "metadata.warning", + "%s/metadata.xml: unused local USE-description: '%s'" + % (xpkg, myflag)) + return {'continue': False} + + @property + def runInPkgs(self): + return (False, []) + + @property + def runInEbuilds(self): + return (False, []) + + @property + def runInFinal(self): + return (True, [self.check]) diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py index 1e61eb6..4dd5408 100644 --- a/pym/repoman/scanner.py +++ b/pym/repoman/scanner.py @@ -270,7 +270,6 @@ class Scanner(object): def _scan_ebuilds(self, ebuildlist, dynamic_data): - xpkg = dynamic_data['xpkg'] # detect unused local USE-descriptions dynamic_data['used_useflags'] = set() @@ -318,11 +317,30 @@ class Scanner(object): if y_ebuild_continue: continue - # check if there are unused local USE-descriptions in metadata.xml - # (unless there are any invalids, to avoid noise) - if dynamic_data['allvalid']: - for myflag in dynamic_data['muselist'].difference(dynamic_data['used_useflags']): - self.qatracker.add_error( - "metadata.warning", - "%s/metadata.xml: unused local USE-description: '%s'" - % (xpkg, myflag)) + # Final checks + # initialize per pkg plugin final checks here + # need to set it up for ==> self.modules_list or some other ordered list + xpkg_complete = False + for mod in [('unused', 'UnusedChecks')]: + if mod[0]: + mod_class = MODULE_CONTROLLER.get_class(mod[0]) + print("Initializing class name:", mod_class.__name__) + self.modules[mod[1]] = mod_class(**self.kwargs) + print("scan_ebuilds final checks: module:", mod[1]) + do_it, functions = self.modules[mod[1]].runInFinal + # print("do_it", do_it, "functions", functions) + if do_it: + for func in functions: + print("\tRunning function:", func) + rdata = func(**dynamic_data)
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/
commit: ecb12dd210c1452c3690702640161d252e8baf00 Author: Brian Dolbec gentoo org> AuthorDate: Sun Jan 3 11:56:25 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sat Jan 30 07:50:18 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=ecb12dd2 repoman: Migrate more metadata checks to ebuild_metadata.py .../modules/scan/metadata/ebuild_metadata.py | 32 -- pym/repoman/scanner.py | 17 2 files changed, 30 insertions(+), 19 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/ebuild_metadata.py b/pym/repoman/modules/scan/metadata/ebuild_metadata.py index 2dc1db2..77c947e 100644 --- a/pym/repoman/modules/scan/metadata/ebuild_metadata.py +++ b/pym/repoman/modules/scan/metadata/ebuild_metadata.py @@ -5,6 +5,8 @@ import re import sys +from repoman.qa_data import missingvars + if sys.hexversion >= 0x300: basestring = str @@ -16,7 +18,7 @@ class EbuildMetadata(object): def __init__(self, **kwargs): self.qatracker = kwargs.get('qatracker') - def check(self, **kwargs): + def invalidchar(self, **kwargs): ebuild = kwargs.get('ebuild') for k, v in ebuild.metadata.items(): if not isinstance(v, basestring): @@ -28,9 +30,35 @@ class EbuildMetadata(object): "%s: %s variable contains non-ASCII " "character at position %s" % (ebuild.relative_path, k, m.start() + 1)) + return {'continue': False} + + def missing(self, **kwargs): + ebuild = kwargs.get('ebuild') + for pos, missing_var in enumerate(missingvars): + if not ebuild.metadata.get(missing_var): + if kwargs.get('catdir') == "virtual" and \ + missing_var in ("HOMEPAGE", "LICENSE"): + continue + if kwargs.get('live_ebuild') and missing_var == "KEYWORDS": + continue + myqakey = missingvars[pos] + ".missing" + self.qatracker.add_error(myqakey, '%s/%s.ebuild' + % (kwargs.get('xpkg'), kwargs.get('y_ebuild'))) + return {'continue': False} + + def old_virtual(self, **kwargs): + ebuild = kwargs.get('ebuild') if ebuild.metadata.get("PROVIDE"): self.qatracker.add_error("virtual.oldstyle", ebuild.relative_path) + return {'continue': False} + def virtual(self, **kwargs): + ebuild = kwargs.get('ebuild') + if kwargs.get('catdir') == "virtual": + for var in ("HOMEPAGE", "LICENSE"): + if ebuild.metadata.get(var): + myqakey = var + ".virtual" + self.qatracker.add_error(myqakey, ebuild.relative_path) return {'continue': False} @property @@ -39,4 +67,4 @@ class EbuildMetadata(object): @property def runInEbuilds(self): - return (True, [self.check]) + return (True, [self.invalidchar, self.missing, self.old_virtual, self.virtual]) diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py index 803e5a3..5c35519 100644 --- a/pym/repoman/scanner.py +++ b/pym/repoman/scanner.py @@ -327,23 +327,6 @@ class Scanner(object): if y_ebuild_continue: continue - - for pos, missing_var in enumerate(missingvars): - if not dynamic_data['ebuild'].metadata.get(missing_var): - if dynamic_data['catdir'] == "virtual" and \ - missing_var in ("HOMEPAGE", "LICENSE"): - continue - if dynamic_data['live_ebuild'] and missing_var == "KEYWORDS": - continue - myqakey = missingvars[pos] + ".missing" - self.qatracker.add_error(myqakey, xpkg + "/" + y_ebuild + ".ebuild") - - if dynamic_data['catdir'] == "virtual": - for var in ("HOMEPAGE", "LICENSE"): - if dynamic_data['ebuild'].metadata.get(var): - myqakey = var + ".virtual" - self.qatracker.add_error(myqakey, dynamic_data['ebuild'].relative_path) - if dynami
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/
commit: 0b5a15fbbd8d012f82f9014d6ef013fb00efcd0c Author: Brian Dolbec gentoo org> AuthorDate: Sun Jan 3 11:56:25 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sat Jan 30 06:33:56 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=0b5a15fb repoman: Migrate more metadata checks to ebuild_metadata.py .../modules/scan/metadata/ebuild_metadata.py | 32 -- pym/repoman/scanner.py | 17 2 files changed, 30 insertions(+), 19 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/ebuild_metadata.py b/pym/repoman/modules/scan/metadata/ebuild_metadata.py index 2dc1db2..77c947e 100644 --- a/pym/repoman/modules/scan/metadata/ebuild_metadata.py +++ b/pym/repoman/modules/scan/metadata/ebuild_metadata.py @@ -5,6 +5,8 @@ import re import sys +from repoman.qa_data import missingvars + if sys.hexversion >= 0x300: basestring = str @@ -16,7 +18,7 @@ class EbuildMetadata(object): def __init__(self, **kwargs): self.qatracker = kwargs.get('qatracker') - def check(self, **kwargs): + def invalidchar(self, **kwargs): ebuild = kwargs.get('ebuild') for k, v in ebuild.metadata.items(): if not isinstance(v, basestring): @@ -28,9 +30,35 @@ class EbuildMetadata(object): "%s: %s variable contains non-ASCII " "character at position %s" % (ebuild.relative_path, k, m.start() + 1)) + return {'continue': False} + + def missing(self, **kwargs): + ebuild = kwargs.get('ebuild') + for pos, missing_var in enumerate(missingvars): + if not ebuild.metadata.get(missing_var): + if kwargs.get('catdir') == "virtual" and \ + missing_var in ("HOMEPAGE", "LICENSE"): + continue + if kwargs.get('live_ebuild') and missing_var == "KEYWORDS": + continue + myqakey = missingvars[pos] + ".missing" + self.qatracker.add_error(myqakey, '%s/%s.ebuild' + % (kwargs.get('xpkg'), kwargs.get('y_ebuild'))) + return {'continue': False} + + def old_virtual(self, **kwargs): + ebuild = kwargs.get('ebuild') if ebuild.metadata.get("PROVIDE"): self.qatracker.add_error("virtual.oldstyle", ebuild.relative_path) + return {'continue': False} + def virtual(self, **kwargs): + ebuild = kwargs.get('ebuild') + if kwargs.get('catdir') == "virtual": + for var in ("HOMEPAGE", "LICENSE"): + if ebuild.metadata.get(var): + myqakey = var + ".virtual" + self.qatracker.add_error(myqakey, ebuild.relative_path) return {'continue': False} @property @@ -39,4 +67,4 @@ class EbuildMetadata(object): @property def runInEbuilds(self): - return (True, [self.check]) + return (True, [self.invalidchar, self.missing, self.old_virtual, self.virtual]) diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py index 803e5a3..5c35519 100644 --- a/pym/repoman/scanner.py +++ b/pym/repoman/scanner.py @@ -327,23 +327,6 @@ class Scanner(object): if y_ebuild_continue: continue - - for pos, missing_var in enumerate(missingvars): - if not dynamic_data['ebuild'].metadata.get(missing_var): - if dynamic_data['catdir'] == "virtual" and \ - missing_var in ("HOMEPAGE", "LICENSE"): - continue - if dynamic_data['live_ebuild'] and missing_var == "KEYWORDS": - continue - myqakey = missingvars[pos] + ".missing" - self.qatracker.add_error(myqakey, xpkg + "/" + y_ebuild + ".ebuild") - - if dynamic_data['catdir'] == "virtual": - for var in ("HOMEPAGE", "LICENSE"): - if dynamic_data['ebuild'].metadata.get(var): - myqakey = var + ".virtual" - self.qatracker.add_error(myqakey, dynamic_data['ebuild'].relative_path) - if dynami
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/, ...
commit: 47f880cf6ffa44309dd85e4e54ee5b0cf3043498 Author: Brian Dolbec gentoo org> AuthorDate: Sun Jan 3 23:09:27 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sat Jan 30 06:33:57 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=47f880cf repoman: Migrate license checks to a plugin module pym/repoman/checks/ebuilds/variables/license.py | 47 -- pym/repoman/modules/scan/metadata/__init__.py | 9 + pym/repoman/modules/scan/metadata/license.py| 53 + pym/repoman/scanner.py | 7 +--- 4 files changed, 63 insertions(+), 53 deletions(-) diff --git a/pym/repoman/checks/ebuilds/variables/license.py b/pym/repoman/checks/ebuilds/variables/license.py deleted file mode 100644 index bdc859c..000 --- a/pym/repoman/checks/ebuilds/variables/license.py +++ /dev/null @@ -1,47 +0,0 @@ - -'''description.py -Perform checks on the LICENSE variable. -''' - -# import our initialized portage instance -from repoman._portage import portage - - -class LicenseChecks(object): - '''Perform checks on the LICENSE variable.''' - - def __init__(self, qatracker, liclist, liclist_deprecated): - ''' - @param qatracker: QATracker instance - @param liclist: List of licenses. - @param liclist: List of deprecated licenses. - ''' - self.qatracker = qatracker - self.liclist = liclist - self.liclist_deprecated = liclist_deprecated - - def check( - self, pkg, package, ebuild, y_ebuild): - ''' - @param pkg: Package in which we check (object). - @param package: Package in which we check (string). - @param ebuild: Ebuild which we check (object). - @param y_ebuild: Ebuild which we check (string). - ''' - - # Parse the LICENSE variable, remove USE conditions and flatten it. - licenses = portage.dep.use_reduce( - pkg._metadata["LICENSE"], matchall=1, flat=True) - - # Check each entry to ensure that it exists in ${PORTDIR}/licenses/. - for lic in licenses: - # Need to check for "||" manually as no portage - # function will remove it without removing values. - if lic not in self.liclist and lic != "||": - self.qatracker.add_error( - "LICENSE.invalid", - package + "/" + y_ebuild + ".ebuild: %s" % lic) - elif lic in self.liclist_deprecated: - self.qatracker.add_error( - "LICENSE.deprecated", - "%s: %s" % (ebuild.relative_path, lic)) diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py index 83aac7f..c8f3609 100644 --- a/pym/repoman/modules/scan/metadata/__init__.py +++ b/pym/repoman/modules/scan/metadata/__init__.py @@ -37,6 +37,15 @@ module_spec = { 'func_desc': { }, }, + 'license-metadata': { + 'name': "license", + 'sourcefile': "license", + 'class': "LicenseChecks", + 'description': doc, + 'functions': ['check'], + 'func_desc': { + }, + }, } } diff --git a/pym/repoman/modules/scan/metadata/license.py b/pym/repoman/modules/scan/metadata/license.py new file mode 100644 index 000..b022b20 --- /dev/null +++ b/pym/repoman/modules/scan/metadata/license.py @@ -0,0 +1,53 @@ + +'''license.py +Perform checks on the LICENSE variable. +''' + +# import our initialized portage instance +from repoman._portage import portage + + +class LicenseChecks(object): + '''Perform checks on the LICENSE variable.''' + + def __init__(self, **kwargs): + ''' + @param qatracker: QATracker instance + @param repo_metadata: dictionary of various repository items. + ''' + self.qatracker = kwargs.get('qatracker') + self.repo_metadata = kwargs.get('repo_metadata') + + def check(self, **kwargs): + ''' + @param xpkg: Package in which we check (string). + @param ebuild: Ebuild which we check (object). + @param y_ebuild: Ebuild which we check (string). + ''' + xpkg = kwargs.get('xpkg') + ebuild = kwargs.get('ebuild') + y_ebuild = kwargs.get('y_ebuild') + if not kwargs.get('badlicsyntax'): + # Parse the L
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/
commit: 048c07d97b0a813ab0547ab23e27f6667751d166 Author: Brian Dolbec gentoo org> AuthorDate: Mon Jan 4 08:37:22 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Mon Mar 7 21:21:33 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=048c07d9 repoman: Create a metadata UnusedCheck and final pkg checks Create a plugin loop for any final pkg checks. Create the one plugin for the unused use-descriptions in mteadata.xml. pym/repoman/modules/scan/metadata/__init__.py | 9 +++ pym/repoman/modules/scan/metadata/unused.py | 32 pym/repoman/scanner.py| 36 --- 3 files changed, 68 insertions(+), 9 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py index 4f376e1..f619764 100644 --- a/pym/repoman/modules/scan/metadata/__init__.py +++ b/pym/repoman/modules/scan/metadata/__init__.py @@ -55,6 +55,15 @@ module_spec = { 'func_desc': { }, }, + 'unused-metadata': { + 'name': "unused", + 'sourcefile': "unused", + 'class': "UnusedCheck", + 'description': doc, + 'functions': ['check'], + 'func_desc': { + }, + }, } } diff --git a/pym/repoman/modules/scan/metadata/unused.py b/pym/repoman/modules/scan/metadata/unused.py new file mode 100644 index 000..5eb6716 --- /dev/null +++ b/pym/repoman/modules/scan/metadata/unused.py @@ -0,0 +1,32 @@ + + +class UnusedCheck(object): + + def __init__(self, **kwargs): + self.qatracker = kwargs.get('qatracker') + + def check(self, **kwargs): + xpkg = kwargs.get('xpkg') + muselist = kwargs.get('muselist') + used_useflags = kwargs.get('used_useflags') + # check if there are unused local USE-descriptions in metadata.xml + # (unless there are any invalids, to avoid noise) + if kwargs.get('allvalid'): + for myflag in muselist.difference(used_useflags): + self.qatracker.add_error( + "metadata.warning", + "%s/metadata.xml: unused local USE-description: '%s'" + % (xpkg, myflag)) + return {'continue': False} + + @property + def runInPkgs(self): + return (False, []) + + @property + def runInEbuilds(self): + return (False, []) + + @property + def runInFinal(self): + return (True, [self.check]) diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py index ca1de5f..da9dcac 100644 --- a/pym/repoman/scanner.py +++ b/pym/repoman/scanner.py @@ -270,7 +270,6 @@ class Scanner(object): def _scan_ebuilds(self, ebuildlist, dynamic_data): - xpkg = dynamic_data['xpkg'] # detect unused local USE-descriptions dynamic_data['used_useflags'] = set() @@ -318,11 +317,30 @@ class Scanner(object): if y_ebuild_continue: continue - # check if there are unused local USE-descriptions in metadata.xml - # (unless there are any invalids, to avoid noise) - if dynamic_data['allvalid']: - for myflag in dynamic_data['muselist'].difference(dynamic_data['used_useflags']): - self.qatracker.add_error( - "metadata.warning", - "%s/metadata.xml: unused local USE-description: '%s'" - % (xpkg, myflag)) + # Final checks + # initialize per pkg plugin final checks here + # need to set it up for ==> self.modules_list or some other ordered list + xpkg_complete = False + for mod in [('unused', 'UnusedChecks')]: + if mod[0]: + mod_class = MODULE_CONTROLLER.get_class(mod[0]) + print("Initializing class name:", mod_class.__name__) + self.modules[mod[1]] = mod_class(**self.kwargs) + print("scan_ebuilds final checks: module:", mod[1]) + do_it, functions = self.modules[mod[1]].runInFinal + # print("do_it", do_it, "functions", functions) + if do_it: + for func in functions: + print("\tRunning function:", func) + rdata = func(**dynamic_data)
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/
commit: f70c87e8f0704e25550a9401ab1785e692efe3c3 Author: Brian Dolbec gentoo org> AuthorDate: Mon Jan 4 08:37:22 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Thu Mar 10 23:47:35 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=f70c87e8 repoman: Create a metadata UnusedCheck and final pkg checks Create a plugin loop for any final pkg checks. Create the one plugin for the unused use-descriptions in mteadata.xml. pym/repoman/modules/scan/metadata/__init__.py | 9 +++ pym/repoman/modules/scan/metadata/unused.py | 32 pym/repoman/scanner.py| 36 --- 3 files changed, 68 insertions(+), 9 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py index 4f376e1..f619764 100644 --- a/pym/repoman/modules/scan/metadata/__init__.py +++ b/pym/repoman/modules/scan/metadata/__init__.py @@ -55,6 +55,15 @@ module_spec = { 'func_desc': { }, }, + 'unused-metadata': { + 'name': "unused", + 'sourcefile': "unused", + 'class': "UnusedCheck", + 'description': doc, + 'functions': ['check'], + 'func_desc': { + }, + }, } } diff --git a/pym/repoman/modules/scan/metadata/unused.py b/pym/repoman/modules/scan/metadata/unused.py new file mode 100644 index 000..5eb6716 --- /dev/null +++ b/pym/repoman/modules/scan/metadata/unused.py @@ -0,0 +1,32 @@ + + +class UnusedCheck(object): + + def __init__(self, **kwargs): + self.qatracker = kwargs.get('qatracker') + + def check(self, **kwargs): + xpkg = kwargs.get('xpkg') + muselist = kwargs.get('muselist') + used_useflags = kwargs.get('used_useflags') + # check if there are unused local USE-descriptions in metadata.xml + # (unless there are any invalids, to avoid noise) + if kwargs.get('allvalid'): + for myflag in muselist.difference(used_useflags): + self.qatracker.add_error( + "metadata.warning", + "%s/metadata.xml: unused local USE-description: '%s'" + % (xpkg, myflag)) + return {'continue': False} + + @property + def runInPkgs(self): + return (False, []) + + @property + def runInEbuilds(self): + return (False, []) + + @property + def runInFinal(self): + return (True, [self.check]) diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py index 4f4f78b..2f717e2 100644 --- a/pym/repoman/scanner.py +++ b/pym/repoman/scanner.py @@ -270,7 +270,6 @@ class Scanner(object): def _scan_ebuilds(self, ebuildlist, dynamic_data): - xpkg = dynamic_data['xpkg'] # detect unused local USE-descriptions dynamic_data['used_useflags'] = set() @@ -318,11 +317,30 @@ class Scanner(object): if y_ebuild_continue: continue - # check if there are unused local USE-descriptions in metadata.xml - # (unless there are any invalids, to avoid noise) - if dynamic_data['allvalid']: - for myflag in dynamic_data['muselist'].difference(dynamic_data['used_useflags']): - self.qatracker.add_error( - "metadata.warning", - "%s/metadata.xml: unused local USE-description: '%s'" - % (xpkg, myflag)) + # Final checks + # initialize per pkg plugin final checks here + # need to set it up for ==> self.modules_list or some other ordered list + xpkg_complete = False + for mod in [('unused', 'UnusedChecks')]: + if mod[0]: + mod_class = MODULE_CONTROLLER.get_class(mod[0]) + print("Initializing class name:", mod_class.__name__) + self.modules[mod[1]] = mod_class(**self.kwargs) + print("scan_ebuilds final checks: module:", mod[1]) + do_it, functions = self.modules[mod[1]].runInFinal + # print("do_it", do_it, "functions", functions) + if do_it: + for func in functions: + print("\tRunning function:", func) + rdata = func(**dynamic_data)
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/
commit: f2e20a96e9f1d9073bb6c8722170667ba96915a0 Author: Brian Dolbec gentoo org> AuthorDate: Sun Jan 3 10:35:49 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sun Jan 10 03:23:49 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=f2e20a96 repoman: Migrate code from _scan_ebuilds to a new EbuildMetadata class and check pym/repoman/modules/scan/metadata/__init__.py | 10 +- .../modules/scan/metadata/ebuild_metadata.py | 39 ++ pym/repoman/scanner.py | 27 +-- 3 files changed, 49 insertions(+), 27 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py index 7327ec0..eba6565 100644 --- a/pym/repoman/modules/scan/metadata/__init__.py +++ b/pym/repoman/modules/scan/metadata/__init__.py @@ -10,7 +10,7 @@ module_spec = { 'name': 'metadata', 'description': doc, 'provides':{ - 'metadata-module': { + 'pkg-metadata': { 'name': "pkgmetadata", 'class': "PkgMetadata", 'description': doc, @@ -18,6 +18,14 @@ module_spec = { 'func_desc': { }, }, + 'ebuild-metadata': { + 'name': "ebuild_metadata", + 'class': "EbuildMetadata", + 'description': doc, + 'functions': ['check'], + 'func_desc': { + }, + }, } } diff --git a/pym/repoman/modules/scan/metadata/ebuild_metadata.py b/pym/repoman/modules/scan/metadata/ebuild_metadata.py new file mode 100644 index 000..143a40e --- /dev/null +++ b/pym/repoman/modules/scan/metadata/ebuild_metadata.py @@ -0,0 +1,39 @@ +# -*- coding:utf-8 -*- + +'''Ebuild Metadata Checks''' + +import re +import sys + +if sys.hexversion >= 0x300: + basestring = str + +NON_ASCII_RE = re.compile(r'[^\x00-\x7f]') + + +class EbuildMetadata(object): + + def __init__(self, **kwargs): + self.qatracker = kwargs.get('qatracker') + + def check(self, **kwargs): + ebuild = kwargs.get('ebuild') + for k, v in ebuild.metadata.items(): + if not isinstance(v, basestring): + continue + m = NON_ASCII_RE.search(v) + if m is not None: + self.qatracker.add_error( + "variable.invalidchar", + "%s: %s variable contains non-ASCII " + "character at position %s" % + (ebuild.relative_path, k, m.start() + 1)) + return {'continue': False} + + @property + def runInPkgs(self): + return (False, []) + + @property + def runInEbuilds(self): + return (True, [self.check]) diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py index a8aa2f3..6f3fb53 100644 --- a/pym/repoman/scanner.py +++ b/pym/repoman/scanner.py @@ -5,8 +5,6 @@ from __future__ import print_function, unicode_literals import copy import io import logging -import re -import sys from itertools import chain from pprint import pformat @@ -47,18 +45,10 @@ MODULE_CONTROLLER = Modules(path=MODULES_PATH, namepath="repoman.modules.scan") MODULE_NAMES = MODULE_CONTROLLER.module_names[:] - -if sys.hexversion >= 0x300: - basestring = str - -NON_ASCII_RE = re.compile(r'[^\x00-\x7f]') - - def sort_key(item): return item[2].sub_path - class Scanner(object): '''Primary scan class. Operates all the small Q/A tests and checks''' @@ -311,7 +301,7 @@ class Scanner(object): # initialize per ebuild plugin checks here # need to set it up for ==> self.modules_list or some other ordered list for mod in [('ebuild', 'Ebuild'), ('live', 'LiveEclassChecks'), - ('eapi', 'EAPIChecks')]: + ('eapi', 'EAPIChecks'), ('ebuild_metadata', 'EbuildMetadata')]: if mod[0]: mod_class = MODULE_CONTROLLER.get_class(mod[0]) logging.debug("Initializing class name: %s", mod_class.__name__) @@ -338,21 +328,6 @@ class Scanner(object): if y_ebuild_continue: continue - - for k, v in dynamic_data['ebuild'].metadata.items(): - if not isinstance(v, basestring): - continue - m = NON_ASCII_RE.search(v) -
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/
commit: 5e273483758272413387c1dadd403a8f1bffdc4a Author: Brian Dolbec gentoo org> AuthorDate: Mon Jan 4 08:37:22 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sun Jan 10 03:23:53 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=5e273483 repoman: Create a metadata UnusedCheck and final pkg checks Create a plugin loop for any final pkg checks. Create the one plugin for the unused use-descriptions in mteadata.xml pym/repoman/modules/scan/metadata/__init__.py | 8 ++ pym/repoman/modules/scan/metadata/unused.py | 32 pym/repoman/scanner.py| 36 --- 3 files changed, 67 insertions(+), 9 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py index 6ab44f6..ed4a967 100644 --- a/pym/repoman/modules/scan/metadata/__init__.py +++ b/pym/repoman/modules/scan/metadata/__init__.py @@ -50,6 +50,14 @@ module_spec = { 'func_desc': { }, }, + 'unused-metadata': { + 'name': "unused", + 'class': "UnusedCheck", + 'description': doc, + 'functions': ['check'], + 'func_desc': { + }, + }, } } diff --git a/pym/repoman/modules/scan/metadata/unused.py b/pym/repoman/modules/scan/metadata/unused.py new file mode 100644 index 000..5eb6716 --- /dev/null +++ b/pym/repoman/modules/scan/metadata/unused.py @@ -0,0 +1,32 @@ + + +class UnusedCheck(object): + + def __init__(self, **kwargs): + self.qatracker = kwargs.get('qatracker') + + def check(self, **kwargs): + xpkg = kwargs.get('xpkg') + muselist = kwargs.get('muselist') + used_useflags = kwargs.get('used_useflags') + # check if there are unused local USE-descriptions in metadata.xml + # (unless there are any invalids, to avoid noise) + if kwargs.get('allvalid'): + for myflag in muselist.difference(used_useflags): + self.qatracker.add_error( + "metadata.warning", + "%s/metadata.xml: unused local USE-description: '%s'" + % (xpkg, myflag)) + return {'continue': False} + + @property + def runInPkgs(self): + return (False, []) + + @property + def runInEbuilds(self): + return (False, []) + + @property + def runInFinal(self): + return (True, [self.check]) diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py index 89eaa57..50dd259 100644 --- a/pym/repoman/scanner.py +++ b/pym/repoman/scanner.py @@ -269,7 +269,6 @@ class Scanner(object): def _scan_ebuilds(self, ebuildlist, dynamic_data): - xpkg = dynamic_data['xpkg'] # detect unused local USE-descriptions dynamic_data['used_useflags'] = set() @@ -317,11 +316,30 @@ class Scanner(object): if y_ebuild_continue: continue - # check if there are unused local USE-descriptions in metadata.xml - # (unless there are any invalids, to avoid noise) - if dynamic_data['allvalid']: - for myflag in dynamic_data['muselist'].difference(dynamic_data['used_useflags']): - self.qatracker.add_error( - "metadata.warning", - "%s/metadata.xml: unused local USE-description: '%s'" - % (xpkg, myflag)) + # Final checks + # initialize per pkg plugin final checks here + # need to set it up for ==> self.modules_list or some other ordered list + xpkg_complete = False + for mod in [('unused', 'UnusedChecks')]: + if mod[0]: + mod_class = MODULE_CONTROLLER.get_class(mod[0]) + print("Initializing class name:", mod_class.__name__) + self.modules[mod[1]] = mod_class(**self.kwargs) + print("scan_ebuilds final checks: module:", mod[1]) + do_it, functions = self.modules[mod[1]].runInFinal + # print("do_it", do_it, "functions", functions) + if do_it: + for func in functions: + print("\tRunning function:", func) + rdata = func(**dynamic_data) + if rdata.g
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/
commit: ae123d5af1e2c33e22e1d501b252ea9954b384dc Author: Brian Dolbec gentoo org> AuthorDate: Sun Jan 3 11:56:25 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sun Jan 10 20:15:08 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=ae123d5a repoman: Migrate more metadata checks to ebuild_metadata.py .../modules/scan/metadata/ebuild_metadata.py | 32 -- pym/repoman/scanner.py | 17 2 files changed, 30 insertions(+), 19 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/ebuild_metadata.py b/pym/repoman/modules/scan/metadata/ebuild_metadata.py index 2dc1db2..77c947e 100644 --- a/pym/repoman/modules/scan/metadata/ebuild_metadata.py +++ b/pym/repoman/modules/scan/metadata/ebuild_metadata.py @@ -5,6 +5,8 @@ import re import sys +from repoman.qa_data import missingvars + if sys.hexversion >= 0x300: basestring = str @@ -16,7 +18,7 @@ class EbuildMetadata(object): def __init__(self, **kwargs): self.qatracker = kwargs.get('qatracker') - def check(self, **kwargs): + def invalidchar(self, **kwargs): ebuild = kwargs.get('ebuild') for k, v in ebuild.metadata.items(): if not isinstance(v, basestring): @@ -28,9 +30,35 @@ class EbuildMetadata(object): "%s: %s variable contains non-ASCII " "character at position %s" % (ebuild.relative_path, k, m.start() + 1)) + return {'continue': False} + + def missing(self, **kwargs): + ebuild = kwargs.get('ebuild') + for pos, missing_var in enumerate(missingvars): + if not ebuild.metadata.get(missing_var): + if kwargs.get('catdir') == "virtual" and \ + missing_var in ("HOMEPAGE", "LICENSE"): + continue + if kwargs.get('live_ebuild') and missing_var == "KEYWORDS": + continue + myqakey = missingvars[pos] + ".missing" + self.qatracker.add_error(myqakey, '%s/%s.ebuild' + % (kwargs.get('xpkg'), kwargs.get('y_ebuild'))) + return {'continue': False} + + def old_virtual(self, **kwargs): + ebuild = kwargs.get('ebuild') if ebuild.metadata.get("PROVIDE"): self.qatracker.add_error("virtual.oldstyle", ebuild.relative_path) + return {'continue': False} + def virtual(self, **kwargs): + ebuild = kwargs.get('ebuild') + if kwargs.get('catdir') == "virtual": + for var in ("HOMEPAGE", "LICENSE"): + if ebuild.metadata.get(var): + myqakey = var + ".virtual" + self.qatracker.add_error(myqakey, ebuild.relative_path) return {'continue': False} @property @@ -39,4 +67,4 @@ class EbuildMetadata(object): @property def runInEbuilds(self): - return (True, [self.check]) + return (True, [self.invalidchar, self.missing, self.old_virtual, self.virtual]) diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py index 46f46f5..d42fd33 100644 --- a/pym/repoman/scanner.py +++ b/pym/repoman/scanner.py @@ -324,23 +324,6 @@ class Scanner(object): if y_ebuild_continue: continue - - for pos, missing_var in enumerate(missingvars): - if not dynamic_data['ebuild'].metadata.get(missing_var): - if dynamic_data['catdir'] == "virtual" and \ - missing_var in ("HOMEPAGE", "LICENSE"): - continue - if dynamic_data['live_ebuild'] and missing_var == "KEYWORDS": - continue - myqakey = missingvars[pos] + ".missing" - self.qatracker.add_error(myqakey, xpkg + "/" + y_ebuild + ".ebuild") - - if dynamic_data['catdir'] == "virtual": - for var in ("HOMEPAGE", "LICENSE"): - if dynamic_data['ebuild'].metadata.get(var): - myqakey = var + ".virtual" - self.qatracker.add_error(myqakey, dynamic_data['ebuild'].relative_path) - if dynami
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/
commit: b273922a63f9b18b86d3dcdcaac8492c1e60ac69 Author: Brian Dolbec gentoo org> AuthorDate: Sun Jan 3 11:31:26 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sun Jan 10 22:59:35 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b273922a scanner.py: Migrate another metadata check to ebuild_metadata pym/repoman/modules/scan/metadata/ebuild_metadata.py | 3 +++ pym/repoman/scanner.py | 2 -- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/ebuild_metadata.py b/pym/repoman/modules/scan/metadata/ebuild_metadata.py index 143a40e..2dc1db2 100644 --- a/pym/repoman/modules/scan/metadata/ebuild_metadata.py +++ b/pym/repoman/modules/scan/metadata/ebuild_metadata.py @@ -28,6 +28,9 @@ class EbuildMetadata(object): "%s: %s variable contains non-ASCII " "character at position %s" % (ebuild.relative_path, k, m.start() + 1)) + if ebuild.metadata.get("PROVIDE"): + self.qatracker.add_error("virtual.oldstyle", ebuild.relative_path) + return {'continue': False} @property diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py index e6a17cd..46f46f5 100644 --- a/pym/repoman/scanner.py +++ b/pym/repoman/scanner.py @@ -324,8 +324,6 @@ class Scanner(object): if y_ebuild_continue: continue - if dynamic_data['ebuild'].metadata.get("PROVIDE"): - self.qatracker.add_error("virtual.oldstyle", dynamic_data['ebuild'].relative_path) for pos, missing_var in enumerate(missingvars): if not dynamic_data['ebuild'].metadata.get(missing_var):
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/, ...
commit: 7beb8bb246c84563e4caaabee5eb9e53cdc6 Author: Brian Dolbec gentoo org> AuthorDate: Sun Jan 3 23:10:48 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sun Jan 10 22:59:36 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=7beb8bb2 repoman: Migrate RestrictChecks to a plugin module pym/repoman/modules/scan/metadata/__init__.py | 8 ++ .../scan/metadata}/restrict.py | 29 +++--- pym/repoman/scanner.py | 7 +- 3 files changed, 29 insertions(+), 15 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py index ed0c59d..6ab44f6 100644 --- a/pym/repoman/modules/scan/metadata/__init__.py +++ b/pym/repoman/modules/scan/metadata/__init__.py @@ -42,6 +42,14 @@ module_spec = { 'func_desc': { }, }, + 'restrict-metadata': { + 'name': "restrict", + 'class': "RestrictChecks", + 'description': doc, + 'functions': ['check'], + 'func_desc': { + }, + }, } } diff --git a/pym/repoman/checks/ebuilds/variables/restrict.py b/pym/repoman/modules/scan/metadata/restrict.py similarity index 54% rename from pym/repoman/checks/ebuilds/variables/restrict.py rename to pym/repoman/modules/scan/metadata/restrict.py index 215b792..93ca298 100644 --- a/pym/repoman/checks/ebuilds/variables/restrict.py +++ b/pym/repoman/modules/scan/metadata/restrict.py @@ -12,21 +12,23 @@ from repoman.qa_data import valid_restrict class RestrictChecks(object): '''Perform checks on the RESTRICT variable.''' - def __init__(self, qatracker): + def __init__(self, **kwargs): ''' @param qatracker: QATracker instance ''' - self.qatracker = qatracker + self.qatracker = kwargs.get('qatracker') - def check(self, pkg, package, ebuild, y_ebuild): + def check(self, **kwargs): + xpkg = kwargs.get('xpkg') + ebuild = kwargs.get('ebuild') + y_ebuild = kwargs.get('y_ebuild') myrestrict = None try: myrestrict = portage.dep.use_reduce( - pkg._metadata["RESTRICT"], matchall=1, flat=True) + ebuild.metadata["RESTRICT"], matchall=1, flat=True) except portage.exception.InvalidDependString as e: - self. qatracker.add_error( - "RESTRICT.syntax", + self.qatracker.add_error("RESTRICT.syntax", "%s: RESTRICT: %s" % (ebuild.relative_path, e)) del e @@ -36,6 +38,15 @@ class RestrictChecks(object): if mybadrestrict: for mybad in mybadrestrict: - self.qatracker.add_error( - "RESTRICT.invalid", - package + "/" + y_ebuild + ".ebuild: %s" % mybad) + self.qatracker.add_error("RESTRICT.invalid", + "%s/%s.ebuild: %s" % (xpkg, y_ebuild, mybad)) + return {'continue': False} + + @property + def runInPkgs(self): + return (False, []) + + @property + def runInEbuilds(self): + return (True, [self.check]) + diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py index 1f792bd..8657c73 100644 --- a/pym/repoman/scanner.py +++ b/pym/repoman/scanner.py @@ -18,7 +18,6 @@ from portage import _unicode_encode from portage.dep import Atom from portage.output import green from repoman.checks.ebuilds.checks import run_checks -from repoman.checks.ebuilds.variables.restrict import RestrictChecks from repoman.modules.commit import repochecks from repoman.profile import check_profiles, dev_profile_keywords, setup_profile from repoman.repos import repo_metadata @@ -209,8 +208,6 @@ class Scanner(object): self.modules[mod_class.__name__] = mod_class(**self.kwargs) # initialize our checks classes here before the big xpkg loop - self.restrictcheck = RestrictChecks(self.qatracker) - def scan_pkgs(self, can_force): dynamic_data = {'can_force': can_force} @@ -295,7 +292,7 @@ class Scanner(object): ('description', 'DescriptionChecks'), (None, 'KeywordChecks'), ('arches', 'ArchChecks'), ('depend', 'DependChecks'), ('use_flags', 'USEFlagChecks'), ('ruby', '
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/
commit: 91883ecc397925167ce18cd6f0baed863b775837 Author: Brian Dolbec gentoo org> AuthorDate: Mon Jan 4 08:37:22 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sun Jan 10 22:59:38 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=91883ecc repoman: Create a metadata UnusedCheck and final pkg checks Create a plugin loop for any final pkg checks. Create the one plugin for the unused use-descriptions in mteadata.xml pym/repoman/modules/scan/metadata/__init__.py | 8 ++ pym/repoman/modules/scan/metadata/unused.py | 32 pym/repoman/scanner.py| 36 --- 3 files changed, 67 insertions(+), 9 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py index 6ab44f6..ed4a967 100644 --- a/pym/repoman/modules/scan/metadata/__init__.py +++ b/pym/repoman/modules/scan/metadata/__init__.py @@ -50,6 +50,14 @@ module_spec = { 'func_desc': { }, }, + 'unused-metadata': { + 'name': "unused", + 'class': "UnusedCheck", + 'description': doc, + 'functions': ['check'], + 'func_desc': { + }, + }, } } diff --git a/pym/repoman/modules/scan/metadata/unused.py b/pym/repoman/modules/scan/metadata/unused.py new file mode 100644 index 000..5eb6716 --- /dev/null +++ b/pym/repoman/modules/scan/metadata/unused.py @@ -0,0 +1,32 @@ + + +class UnusedCheck(object): + + def __init__(self, **kwargs): + self.qatracker = kwargs.get('qatracker') + + def check(self, **kwargs): + xpkg = kwargs.get('xpkg') + muselist = kwargs.get('muselist') + used_useflags = kwargs.get('used_useflags') + # check if there are unused local USE-descriptions in metadata.xml + # (unless there are any invalids, to avoid noise) + if kwargs.get('allvalid'): + for myflag in muselist.difference(used_useflags): + self.qatracker.add_error( + "metadata.warning", + "%s/metadata.xml: unused local USE-description: '%s'" + % (xpkg, myflag)) + return {'continue': False} + + @property + def runInPkgs(self): + return (False, []) + + @property + def runInEbuilds(self): + return (False, []) + + @property + def runInFinal(self): + return (True, [self.check]) diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py index 89eaa57..50dd259 100644 --- a/pym/repoman/scanner.py +++ b/pym/repoman/scanner.py @@ -269,7 +269,6 @@ class Scanner(object): def _scan_ebuilds(self, ebuildlist, dynamic_data): - xpkg = dynamic_data['xpkg'] # detect unused local USE-descriptions dynamic_data['used_useflags'] = set() @@ -317,11 +316,30 @@ class Scanner(object): if y_ebuild_continue: continue - # check if there are unused local USE-descriptions in metadata.xml - # (unless there are any invalids, to avoid noise) - if dynamic_data['allvalid']: - for myflag in dynamic_data['muselist'].difference(dynamic_data['used_useflags']): - self.qatracker.add_error( - "metadata.warning", - "%s/metadata.xml: unused local USE-description: '%s'" - % (xpkg, myflag)) + # Final checks + # initialize per pkg plugin final checks here + # need to set it up for ==> self.modules_list or some other ordered list + xpkg_complete = False + for mod in [('unused', 'UnusedChecks')]: + if mod[0]: + mod_class = MODULE_CONTROLLER.get_class(mod[0]) + print("Initializing class name:", mod_class.__name__) + self.modules[mod[1]] = mod_class(**self.kwargs) + print("scan_ebuilds final checks: module:", mod[1]) + do_it, functions = self.modules[mod[1]].runInFinal + # print("do_it", do_it, "functions", functions) + if do_it: + for func in functions: + print("\tRunning function:", func) + rdata = func(**dynamic_data) + if rdata.g
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/
commit: 891dd1e6bda1a5368562629c2864ec043ee9ec76 Author: Brian Dolbec gentoo org> AuthorDate: Sun Jan 3 10:35:49 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sun Jan 10 22:59:32 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=891dd1e6 repoman: Migrate code from _scan_ebuilds to a new EbuildMetadata class and check pym/repoman/modules/scan/metadata/__init__.py | 10 +- .../modules/scan/metadata/ebuild_metadata.py | 39 ++ pym/repoman/scanner.py | 27 +-- 3 files changed, 49 insertions(+), 27 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py index 7327ec0..eba6565 100644 --- a/pym/repoman/modules/scan/metadata/__init__.py +++ b/pym/repoman/modules/scan/metadata/__init__.py @@ -10,7 +10,7 @@ module_spec = { 'name': 'metadata', 'description': doc, 'provides':{ - 'metadata-module': { + 'pkg-metadata': { 'name': "pkgmetadata", 'class': "PkgMetadata", 'description': doc, @@ -18,6 +18,14 @@ module_spec = { 'func_desc': { }, }, + 'ebuild-metadata': { + 'name': "ebuild_metadata", + 'class': "EbuildMetadata", + 'description': doc, + 'functions': ['check'], + 'func_desc': { + }, + }, } } diff --git a/pym/repoman/modules/scan/metadata/ebuild_metadata.py b/pym/repoman/modules/scan/metadata/ebuild_metadata.py new file mode 100644 index 000..143a40e --- /dev/null +++ b/pym/repoman/modules/scan/metadata/ebuild_metadata.py @@ -0,0 +1,39 @@ +# -*- coding:utf-8 -*- + +'''Ebuild Metadata Checks''' + +import re +import sys + +if sys.hexversion >= 0x300: + basestring = str + +NON_ASCII_RE = re.compile(r'[^\x00-\x7f]') + + +class EbuildMetadata(object): + + def __init__(self, **kwargs): + self.qatracker = kwargs.get('qatracker') + + def check(self, **kwargs): + ebuild = kwargs.get('ebuild') + for k, v in ebuild.metadata.items(): + if not isinstance(v, basestring): + continue + m = NON_ASCII_RE.search(v) + if m is not None: + self.qatracker.add_error( + "variable.invalidchar", + "%s: %s variable contains non-ASCII " + "character at position %s" % + (ebuild.relative_path, k, m.start() + 1)) + return {'continue': False} + + @property + def runInPkgs(self): + return (False, []) + + @property + def runInEbuilds(self): + return (True, [self.check]) diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py index a8aa2f3..6f3fb53 100644 --- a/pym/repoman/scanner.py +++ b/pym/repoman/scanner.py @@ -5,8 +5,6 @@ from __future__ import print_function, unicode_literals import copy import io import logging -import re -import sys from itertools import chain from pprint import pformat @@ -47,18 +45,10 @@ MODULE_CONTROLLER = Modules(path=MODULES_PATH, namepath="repoman.modules.scan") MODULE_NAMES = MODULE_CONTROLLER.module_names[:] - -if sys.hexversion >= 0x300: - basestring = str - -NON_ASCII_RE = re.compile(r'[^\x00-\x7f]') - - def sort_key(item): return item[2].sub_path - class Scanner(object): '''Primary scan class. Operates all the small Q/A tests and checks''' @@ -311,7 +301,7 @@ class Scanner(object): # initialize per ebuild plugin checks here # need to set it up for ==> self.modules_list or some other ordered list for mod in [('ebuild', 'Ebuild'), ('live', 'LiveEclassChecks'), - ('eapi', 'EAPIChecks')]: + ('eapi', 'EAPIChecks'), ('ebuild_metadata', 'EbuildMetadata')]: if mod[0]: mod_class = MODULE_CONTROLLER.get_class(mod[0]) logging.debug("Initializing class name: %s", mod_class.__name__) @@ -338,21 +328,6 @@ class Scanner(object): if y_ebuild_continue: continue - - for k, v in dynamic_data['ebuild'].metadata.items(): - if not isinstance(v, basestring): - continue - m = NON_ASCII_RE.search(v) -
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/
commit: 63d8ca7ebc4c93b16640ce3c7ca65032520457f1 Author: Brian Dolbec gentoo org> AuthorDate: Sun Jan 3 10:35:49 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Mon Jan 11 08:00:15 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=63d8ca7e repoman: Migrate code from _scan_ebuilds to a new EbuildMetadata class and check pym/repoman/modules/scan/metadata/__init__.py | 10 +- .../modules/scan/metadata/ebuild_metadata.py | 39 ++ pym/repoman/scanner.py | 27 +-- 3 files changed, 49 insertions(+), 27 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py index 7327ec0..eba6565 100644 --- a/pym/repoman/modules/scan/metadata/__init__.py +++ b/pym/repoman/modules/scan/metadata/__init__.py @@ -10,7 +10,7 @@ module_spec = { 'name': 'metadata', 'description': doc, 'provides':{ - 'metadata-module': { + 'pkg-metadata': { 'name': "pkgmetadata", 'class': "PkgMetadata", 'description': doc, @@ -18,6 +18,14 @@ module_spec = { 'func_desc': { }, }, + 'ebuild-metadata': { + 'name': "ebuild_metadata", + 'class': "EbuildMetadata", + 'description': doc, + 'functions': ['check'], + 'func_desc': { + }, + }, } } diff --git a/pym/repoman/modules/scan/metadata/ebuild_metadata.py b/pym/repoman/modules/scan/metadata/ebuild_metadata.py new file mode 100644 index 000..143a40e --- /dev/null +++ b/pym/repoman/modules/scan/metadata/ebuild_metadata.py @@ -0,0 +1,39 @@ +# -*- coding:utf-8 -*- + +'''Ebuild Metadata Checks''' + +import re +import sys + +if sys.hexversion >= 0x300: + basestring = str + +NON_ASCII_RE = re.compile(r'[^\x00-\x7f]') + + +class EbuildMetadata(object): + + def __init__(self, **kwargs): + self.qatracker = kwargs.get('qatracker') + + def check(self, **kwargs): + ebuild = kwargs.get('ebuild') + for k, v in ebuild.metadata.items(): + if not isinstance(v, basestring): + continue + m = NON_ASCII_RE.search(v) + if m is not None: + self.qatracker.add_error( + "variable.invalidchar", + "%s: %s variable contains non-ASCII " + "character at position %s" % + (ebuild.relative_path, k, m.start() + 1)) + return {'continue': False} + + @property + def runInPkgs(self): + return (False, []) + + @property + def runInEbuilds(self): + return (True, [self.check]) diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py index a8aa2f3..6f3fb53 100644 --- a/pym/repoman/scanner.py +++ b/pym/repoman/scanner.py @@ -5,8 +5,6 @@ from __future__ import print_function, unicode_literals import copy import io import logging -import re -import sys from itertools import chain from pprint import pformat @@ -47,18 +45,10 @@ MODULE_CONTROLLER = Modules(path=MODULES_PATH, namepath="repoman.modules.scan") MODULE_NAMES = MODULE_CONTROLLER.module_names[:] - -if sys.hexversion >= 0x300: - basestring = str - -NON_ASCII_RE = re.compile(r'[^\x00-\x7f]') - - def sort_key(item): return item[2].sub_path - class Scanner(object): '''Primary scan class. Operates all the small Q/A tests and checks''' @@ -311,7 +301,7 @@ class Scanner(object): # initialize per ebuild plugin checks here # need to set it up for ==> self.modules_list or some other ordered list for mod in [('ebuild', 'Ebuild'), ('live', 'LiveEclassChecks'), - ('eapi', 'EAPIChecks')]: + ('eapi', 'EAPIChecks'), ('ebuild_metadata', 'EbuildMetadata')]: if mod[0]: mod_class = MODULE_CONTROLLER.get_class(mod[0]) logging.debug("Initializing class name: %s", mod_class.__name__) @@ -338,21 +328,6 @@ class Scanner(object): if y_ebuild_continue: continue - - for k, v in dynamic_data['ebuild'].metadata.items(): - if not isinstance(v, basestring): - continue - m = NON_ASCII_RE.search(v) -
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/
commit: e0d88104006627f24165ac88ef121025dc844952 Author: Brian Dolbec gentoo org> AuthorDate: Sun Jan 3 10:35:49 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Mon Jan 18 19:20:03 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=e0d88104 repoman: Migrate code from _scan_ebuilds to a new EbuildMetadata class and check pym/repoman/modules/scan/metadata/__init__.py | 10 +- .../modules/scan/metadata/ebuild_metadata.py | 39 ++ pym/repoman/scanner.py | 27 +-- 3 files changed, 49 insertions(+), 27 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py index 7327ec0..eba6565 100644 --- a/pym/repoman/modules/scan/metadata/__init__.py +++ b/pym/repoman/modules/scan/metadata/__init__.py @@ -10,7 +10,7 @@ module_spec = { 'name': 'metadata', 'description': doc, 'provides':{ - 'metadata-module': { + 'pkg-metadata': { 'name': "pkgmetadata", 'class': "PkgMetadata", 'description': doc, @@ -18,6 +18,14 @@ module_spec = { 'func_desc': { }, }, + 'ebuild-metadata': { + 'name': "ebuild_metadata", + 'class': "EbuildMetadata", + 'description': doc, + 'functions': ['check'], + 'func_desc': { + }, + }, } } diff --git a/pym/repoman/modules/scan/metadata/ebuild_metadata.py b/pym/repoman/modules/scan/metadata/ebuild_metadata.py new file mode 100644 index 000..143a40e --- /dev/null +++ b/pym/repoman/modules/scan/metadata/ebuild_metadata.py @@ -0,0 +1,39 @@ +# -*- coding:utf-8 -*- + +'''Ebuild Metadata Checks''' + +import re +import sys + +if sys.hexversion >= 0x300: + basestring = str + +NON_ASCII_RE = re.compile(r'[^\x00-\x7f]') + + +class EbuildMetadata(object): + + def __init__(self, **kwargs): + self.qatracker = kwargs.get('qatracker') + + def check(self, **kwargs): + ebuild = kwargs.get('ebuild') + for k, v in ebuild.metadata.items(): + if not isinstance(v, basestring): + continue + m = NON_ASCII_RE.search(v) + if m is not None: + self.qatracker.add_error( + "variable.invalidchar", + "%s: %s variable contains non-ASCII " + "character at position %s" % + (ebuild.relative_path, k, m.start() + 1)) + return {'continue': False} + + @property + def runInPkgs(self): + return (False, []) + + @property + def runInEbuilds(self): + return (True, [self.check]) diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py index a8aa2f3..6f3fb53 100644 --- a/pym/repoman/scanner.py +++ b/pym/repoman/scanner.py @@ -5,8 +5,6 @@ from __future__ import print_function, unicode_literals import copy import io import logging -import re -import sys from itertools import chain from pprint import pformat @@ -47,18 +45,10 @@ MODULE_CONTROLLER = Modules(path=MODULES_PATH, namepath="repoman.modules.scan") MODULE_NAMES = MODULE_CONTROLLER.module_names[:] - -if sys.hexversion >= 0x300: - basestring = str - -NON_ASCII_RE = re.compile(r'[^\x00-\x7f]') - - def sort_key(item): return item[2].sub_path - class Scanner(object): '''Primary scan class. Operates all the small Q/A tests and checks''' @@ -311,7 +301,7 @@ class Scanner(object): # initialize per ebuild plugin checks here # need to set it up for ==> self.modules_list or some other ordered list for mod in [('ebuild', 'Ebuild'), ('live', 'LiveEclassChecks'), - ('eapi', 'EAPIChecks')]: + ('eapi', 'EAPIChecks'), ('ebuild_metadata', 'EbuildMetadata')]: if mod[0]: mod_class = MODULE_CONTROLLER.get_class(mod[0]) logging.debug("Initializing class name: %s", mod_class.__name__) @@ -338,21 +328,6 @@ class Scanner(object): if y_ebuild_continue: continue - - for k, v in dynamic_data['ebuild'].metadata.items(): - if not isinstance(v, basestring): - continue - m = NON_ASCII_RE.search(v) -
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/
commit: 07b87ced7b378bace32904c6322cc3d00d610a01 Author: Brian Dolbec gentoo org> AuthorDate: Sun Jan 3 11:56:25 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sat Jan 30 20:25:21 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=07b87ced repoman: Migrate more metadata checks to ebuild_metadata.py .../modules/scan/metadata/ebuild_metadata.py | 32 -- pym/repoman/scanner.py | 17 2 files changed, 30 insertions(+), 19 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/ebuild_metadata.py b/pym/repoman/modules/scan/metadata/ebuild_metadata.py index 2dc1db2..77c947e 100644 --- a/pym/repoman/modules/scan/metadata/ebuild_metadata.py +++ b/pym/repoman/modules/scan/metadata/ebuild_metadata.py @@ -5,6 +5,8 @@ import re import sys +from repoman.qa_data import missingvars + if sys.hexversion >= 0x300: basestring = str @@ -16,7 +18,7 @@ class EbuildMetadata(object): def __init__(self, **kwargs): self.qatracker = kwargs.get('qatracker') - def check(self, **kwargs): + def invalidchar(self, **kwargs): ebuild = kwargs.get('ebuild') for k, v in ebuild.metadata.items(): if not isinstance(v, basestring): @@ -28,9 +30,35 @@ class EbuildMetadata(object): "%s: %s variable contains non-ASCII " "character at position %s" % (ebuild.relative_path, k, m.start() + 1)) + return {'continue': False} + + def missing(self, **kwargs): + ebuild = kwargs.get('ebuild') + for pos, missing_var in enumerate(missingvars): + if not ebuild.metadata.get(missing_var): + if kwargs.get('catdir') == "virtual" and \ + missing_var in ("HOMEPAGE", "LICENSE"): + continue + if kwargs.get('live_ebuild') and missing_var == "KEYWORDS": + continue + myqakey = missingvars[pos] + ".missing" + self.qatracker.add_error(myqakey, '%s/%s.ebuild' + % (kwargs.get('xpkg'), kwargs.get('y_ebuild'))) + return {'continue': False} + + def old_virtual(self, **kwargs): + ebuild = kwargs.get('ebuild') if ebuild.metadata.get("PROVIDE"): self.qatracker.add_error("virtual.oldstyle", ebuild.relative_path) + return {'continue': False} + def virtual(self, **kwargs): + ebuild = kwargs.get('ebuild') + if kwargs.get('catdir') == "virtual": + for var in ("HOMEPAGE", "LICENSE"): + if ebuild.metadata.get(var): + myqakey = var + ".virtual" + self.qatracker.add_error(myqakey, ebuild.relative_path) return {'continue': False} @property @@ -39,4 +67,4 @@ class EbuildMetadata(object): @property def runInEbuilds(self): - return (True, [self.check]) + return (True, [self.invalidchar, self.missing, self.old_virtual, self.virtual]) diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py index ec126e6..1a2de05 100644 --- a/pym/repoman/scanner.py +++ b/pym/repoman/scanner.py @@ -327,23 +327,6 @@ class Scanner(object): if y_ebuild_continue: continue - - for pos, missing_var in enumerate(missingvars): - if not dynamic_data['ebuild'].metadata.get(missing_var): - if dynamic_data['catdir'] == "virtual" and \ - missing_var in ("HOMEPAGE", "LICENSE"): - continue - if dynamic_data['live_ebuild'] and missing_var == "KEYWORDS": - continue - myqakey = missingvars[pos] + ".missing" - self.qatracker.add_error(myqakey, xpkg + "/" + y_ebuild + ".ebuild") - - if dynamic_data['catdir'] == "virtual": - for var in ("HOMEPAGE", "LICENSE"): - if dynamic_data['ebuild'].metadata.get(var): - myqakey = var + ".virtual" - self.qatracker.add_error(myqakey, dynamic_data['ebuild'].relative_path) - if dynami
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/
commit: de3ef8986c2607fb4741ab664b31c29e1a3c936e Author: Brian Dolbec gentoo org> AuthorDate: Sun Jan 3 11:31:26 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sat Jan 30 20:25:21 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=de3ef898 scanner.py: Migrate another metadata check to ebuild_metadata pym/repoman/modules/scan/metadata/ebuild_metadata.py | 3 +++ pym/repoman/scanner.py | 2 -- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/ebuild_metadata.py b/pym/repoman/modules/scan/metadata/ebuild_metadata.py index 143a40e..2dc1db2 100644 --- a/pym/repoman/modules/scan/metadata/ebuild_metadata.py +++ b/pym/repoman/modules/scan/metadata/ebuild_metadata.py @@ -28,6 +28,9 @@ class EbuildMetadata(object): "%s: %s variable contains non-ASCII " "character at position %s" % (ebuild.relative_path, k, m.start() + 1)) + if ebuild.metadata.get("PROVIDE"): + self.qatracker.add_error("virtual.oldstyle", ebuild.relative_path) + return {'continue': False} @property diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py index c5601dd..ec126e6 100644 --- a/pym/repoman/scanner.py +++ b/pym/repoman/scanner.py @@ -327,8 +327,6 @@ class Scanner(object): if y_ebuild_continue: continue - if dynamic_data['ebuild'].metadata.get("PROVIDE"): - self.qatracker.add_error("virtual.oldstyle", dynamic_data['ebuild'].relative_path) for pos, missing_var in enumerate(missingvars): if not dynamic_data['ebuild'].metadata.get(missing_var):
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/
commit: 4a4f67beab47c0adcc8048afb4159794a486faaf Author: Brian Dolbec gentoo org> AuthorDate: Mon Jan 4 08:37:22 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sat Jan 30 20:25:24 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=4a4f67be repoman: Create a metadata UnusedCheck and final pkg checks Create a plugin loop for any final pkg checks. Create the one plugin for the unused use-descriptions in mteadata.xml. pym/repoman/modules/scan/metadata/__init__.py | 9 +++ pym/repoman/modules/scan/metadata/unused.py | 32 pym/repoman/scanner.py| 36 --- 3 files changed, 68 insertions(+), 9 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py index 4f376e1..f619764 100644 --- a/pym/repoman/modules/scan/metadata/__init__.py +++ b/pym/repoman/modules/scan/metadata/__init__.py @@ -55,6 +55,15 @@ module_spec = { 'func_desc': { }, }, + 'unused-metadata': { + 'name': "unused", + 'sourcefile': "unused", + 'class': "UnusedCheck", + 'description': doc, + 'functions': ['check'], + 'func_desc': { + }, + }, } } diff --git a/pym/repoman/modules/scan/metadata/unused.py b/pym/repoman/modules/scan/metadata/unused.py new file mode 100644 index 000..5eb6716 --- /dev/null +++ b/pym/repoman/modules/scan/metadata/unused.py @@ -0,0 +1,32 @@ + + +class UnusedCheck(object): + + def __init__(self, **kwargs): + self.qatracker = kwargs.get('qatracker') + + def check(self, **kwargs): + xpkg = kwargs.get('xpkg') + muselist = kwargs.get('muselist') + used_useflags = kwargs.get('used_useflags') + # check if there are unused local USE-descriptions in metadata.xml + # (unless there are any invalids, to avoid noise) + if kwargs.get('allvalid'): + for myflag in muselist.difference(used_useflags): + self.qatracker.add_error( + "metadata.warning", + "%s/metadata.xml: unused local USE-description: '%s'" + % (xpkg, myflag)) + return {'continue': False} + + @property + def runInPkgs(self): + return (False, []) + + @property + def runInEbuilds(self): + return (False, []) + + @property + def runInFinal(self): + return (True, [self.check]) diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py index ca1de5f..da9dcac 100644 --- a/pym/repoman/scanner.py +++ b/pym/repoman/scanner.py @@ -270,7 +270,6 @@ class Scanner(object): def _scan_ebuilds(self, ebuildlist, dynamic_data): - xpkg = dynamic_data['xpkg'] # detect unused local USE-descriptions dynamic_data['used_useflags'] = set() @@ -318,11 +317,30 @@ class Scanner(object): if y_ebuild_continue: continue - # check if there are unused local USE-descriptions in metadata.xml - # (unless there are any invalids, to avoid noise) - if dynamic_data['allvalid']: - for myflag in dynamic_data['muselist'].difference(dynamic_data['used_useflags']): - self.qatracker.add_error( - "metadata.warning", - "%s/metadata.xml: unused local USE-description: '%s'" - % (xpkg, myflag)) + # Final checks + # initialize per pkg plugin final checks here + # need to set it up for ==> self.modules_list or some other ordered list + xpkg_complete = False + for mod in [('unused', 'UnusedChecks')]: + if mod[0]: + mod_class = MODULE_CONTROLLER.get_class(mod[0]) + print("Initializing class name:", mod_class.__name__) + self.modules[mod[1]] = mod_class(**self.kwargs) + print("scan_ebuilds final checks: module:", mod[1]) + do_it, functions = self.modules[mod[1]].runInFinal + # print("do_it", do_it, "functions", functions) + if do_it: + for func in functions: + print("\tRunning function:", func) + rdata = func(**dynamic_data)
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/
commit: 2d2b99917ed054bea083a0e47f30aac34f0fefdd Author: Dirkjan Ochtman ochtman nl> AuthorDate: Tue May 3 09:01:29 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sat May 14 18:29:40 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=2d2b9991 repoman: Migrate from XmlLint to etree.XMLSchema for validation Remove No longer used repoman._xml module This change based on work by Dirkjan Ochtman gentoo.org> Updated the change from XML.DTD to XMLSchema. Additionally: Move the metadata.xsd path determination code to metadata.py. Add the DISTDIR backup location and fetching of the file if missing or stale. pym/repoman/_xml.py | 105 --- pym/repoman/metadata.py | 21 + pym/repoman/modules/scan/metadata/pkgmetadata.py | 9 +- pym/repoman/scanner.py | 10 +-- 4 files changed, 26 insertions(+), 119 deletions(-) diff --git a/pym/repoman/_xml.py b/pym/repoman/_xml.py deleted file mode 100644 index 33a536a..000 --- a/pym/repoman/_xml.py +++ /dev/null @@ -1,105 +0,0 @@ -# -*- coding:utf-8 -*- - -from __future__ import print_function, unicode_literals - -import sys -import xml - -# import our initialized portage instance -from repoman._portage import portage - -from portage import os -from portage.output import red -from portage.process import find_binary - -from repoman.metadata import fetch_metadata_xsd -from repoman._subprocess import repoman_getstatusoutput - - -class _XMLParser(xml.etree.ElementTree.XMLParser): - - def __init__(self, data, **kwargs): - xml.etree.ElementTree.XMLParser.__init__(self, **kwargs) - self._portage_data = data - if hasattr(self, 'parser'): - self._base_XmlDeclHandler = self.parser.XmlDeclHandler - self.parser.XmlDeclHandler = self._portage_XmlDeclHandler - self._base_StartDoctypeDeclHandler = \ - self.parser.StartDoctypeDeclHandler - self.parser.StartDoctypeDeclHandler = \ - self._portage_StartDoctypeDeclHandler - - def _portage_XmlDeclHandler(self, version, encoding, standalone): - if self._base_XmlDeclHandler is not None: - self._base_XmlDeclHandler(version, encoding, standalone) - self._portage_data["XML_DECLARATION"] = (version, encoding, standalone) - - def _portage_StartDoctypeDeclHandler( - self, doctypeName, systemId, publicId, has_internal_subset): - if self._base_StartDoctypeDeclHandler is not None: - self._base_StartDoctypeDeclHandler( - doctypeName, systemId, publicId, has_internal_subset) - self._portage_data["DOCTYPE"] = (doctypeName, systemId, publicId) - - -class _MetadataTreeBuilder(xml.etree.ElementTree.TreeBuilder): - """ - Implements doctype() as required to avoid deprecation warnings with - >=python-2.7. - """ - def doctype(self, name, pubid, system): - pass - - -class XmlLint(object): - - def __init__(self, options, repoman_settings, metadata_xsd=None): - self.metadata_xsd = (metadata_xsd or - os.path.join(repoman_settings["DISTDIR"], 'metadata.xsd')) - self.options = options - self.repoman_settings = repoman_settings - self._is_capable = metadata_xsd is not None - self.binary = None - self._check_capable() - - def _check_capable(self): - if self.options.mode == "manifest": - return - self.binary = find_binary('xmllint') - if not self.binary: - print(red("!!! xmllint not found. Can't check metadata.xml.\n")) - elif not self._is_capable: - if not fetch_metadata_xsd(self.metadata_xsd, self.repoman_settings): - sys.exit(1) - # this can be problematic if xmllint changes their output - self._is_capable = True - - @property - def capable(self): - return self._is_capable - - def check(self, checkdir, repolevel): - '''Runs checks on the package metadata.xml file - - @param checkdir: string, path - @param repolevel: integer - @return boolean, False == bad metadata - ''' - if not self.capable: - if self.options.xml_parse or repolevel == 3: - print("%s sorry, xmllint is needed. failing\n" % red("!!!")) - sys.exit(1) - return True - # xmlint can produce garbage output eve
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/
commit: 22a9d9df146499ebf740eb61964633ba7d6a10ef Author: Dirkjan Ochtman ochtman nl> AuthorDate: Mon Jan 25 18:42:41 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sat May 14 18:29:40 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=22a9d9df repoman: Move some metdata_* variables to module it's used pym/repoman/metadata.py | 4 pym/repoman/modules/scan/metadata/pkgmetadata.py | 9 ++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/pym/repoman/metadata.py b/pym/repoman/metadata.py index 7a514dc..a980184 100644 --- a/pym/repoman/metadata.py +++ b/pym/repoman/metadata.py @@ -28,10 +28,6 @@ if sys.hexversion >= 0x300: if sys.hexversion >= 0x300: basestring = str -metadata_xml_encoding = 'UTF-8' -metadata_xml_declaration = '' \ - % (metadata_xml_encoding,) -metadata_doctype_name = 'pkgmetadata' metadata_dtd_uri = 'http://www.gentoo.org/dtd/metadata.dtd' metadata_xsd_uri = 'http://www.gentoo.org/xml-schema/metadata.xsd' # force refetch if the local copy creation time is older than this diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index 3b48b8e..a7150a9 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -23,9 +23,7 @@ except (ImportError, SystemError, RuntimeError, Exception): # import our initialized portage instance from repoman._portage import portage -from repoman.metadata import ( - metadata_xml_encoding, metadata_doctype_name, - metadata_dtd_uri, metadata_xml_declaration, parse_metadata_use) +from repoman.metadata import metadata_dtd_uri, parse_metadata_use from repoman.checks.herds.herdbase import get_herd_base from repoman.checks.herds.metadata import check_metadata, UnknownHerdsError from repoman._xml import _XMLParser, _MetadataTreeBuilder, XmlLint @@ -38,6 +36,11 @@ from portage.dep import Atom from .use_flags import USEFlagChecks +metadata_xml_encoding = 'UTF-8' +metadata_xml_declaration = '' \ + % (metadata_xml_encoding,) +metadata_doctype_name = 'pkgmetadata' + class PkgMetadata(ScanBase, USEFlagChecks): '''Package metadata.xml checks'''
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/
commit: aebf5722a1bc262b8bd843348c947d06efa6713c Author: Brian Dolbec gentoo org> AuthorDate: Sun Jan 3 11:56:25 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Wed Jan 6 04:08:21 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=aebf5722 repoman: Migrate more metadata checks to ebuild_metadata.py .../modules/scan/metadata/ebuild_metadata.py | 32 -- pym/repoman/scanner.py | 19 - 2 files changed, 30 insertions(+), 21 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/ebuild_metadata.py b/pym/repoman/modules/scan/metadata/ebuild_metadata.py index 2dc1db2..77c947e 100644 --- a/pym/repoman/modules/scan/metadata/ebuild_metadata.py +++ b/pym/repoman/modules/scan/metadata/ebuild_metadata.py @@ -5,6 +5,8 @@ import re import sys +from repoman.qa_data import missingvars + if sys.hexversion >= 0x300: basestring = str @@ -16,7 +18,7 @@ class EbuildMetadata(object): def __init__(self, **kwargs): self.qatracker = kwargs.get('qatracker') - def check(self, **kwargs): + def invalidchar(self, **kwargs): ebuild = kwargs.get('ebuild') for k, v in ebuild.metadata.items(): if not isinstance(v, basestring): @@ -28,9 +30,35 @@ class EbuildMetadata(object): "%s: %s variable contains non-ASCII " "character at position %s" % (ebuild.relative_path, k, m.start() + 1)) + return {'continue': False} + + def missing(self, **kwargs): + ebuild = kwargs.get('ebuild') + for pos, missing_var in enumerate(missingvars): + if not ebuild.metadata.get(missing_var): + if kwargs.get('catdir') == "virtual" and \ + missing_var in ("HOMEPAGE", "LICENSE"): + continue + if kwargs.get('live_ebuild') and missing_var == "KEYWORDS": + continue + myqakey = missingvars[pos] + ".missing" + self.qatracker.add_error(myqakey, '%s/%s.ebuild' + % (kwargs.get('xpkg'), kwargs.get('y_ebuild'))) + return {'continue': False} + + def old_virtual(self, **kwargs): + ebuild = kwargs.get('ebuild') if ebuild.metadata.get("PROVIDE"): self.qatracker.add_error("virtual.oldstyle", ebuild.relative_path) + return {'continue': False} + def virtual(self, **kwargs): + ebuild = kwargs.get('ebuild') + if kwargs.get('catdir') == "virtual": + for var in ("HOMEPAGE", "LICENSE"): + if ebuild.metadata.get(var): + myqakey = var + ".virtual" + self.qatracker.add_error(myqakey, ebuild.relative_path) return {'continue': False} @property @@ -39,4 +67,4 @@ class EbuildMetadata(object): @property def runInEbuilds(self): - return (True, [self.check]) + return (True, [self.invalidchar, self.missing, self.old_virtual, self.virtual]) diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py index ca540a7..20c6460 100644 --- a/pym/repoman/scanner.py +++ b/pym/repoman/scanner.py @@ -330,25 +330,6 @@ class Scanner(object): print(" finished plugin loop, continuing...") - if dynamic_data['ebuild'].metadata.get("PROVIDE"): - self.qatracker.add_error("virtual.oldstyle", dynamic_data['ebuild'].relative_path) - - for pos, missing_var in enumerate(missingvars): - if not dynamic_data['ebuild'].metadata.get(missing_var): - if dynamic_data['catdir'] == "virtual" and \ - missing_var in ("HOMEPAGE", "LICENSE"): - continue - if dynamic_data['live_ebuild'] and missing_var == "KEYWORDS": - continue - myqakey = missingvars[pos] + ".missing" - self.qatracker.add_error(myqakey, xpkg + "/" + y_ebuild + ".ebuild") - - if dynamic_data['catdir'] == "virtual": - for var in ("HOMEPAGE", "LICENSE"): - if dynamic_data['ebuild'].metadata.get(var): - my
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/
commit: 0acb069f91d0e400f2182c3965462850f7d02da0 Author: Brian Dolbec gentoo org> AuthorDate: Mon Jan 4 08:37:22 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Wed Jan 6 04:08:25 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=0acb069f repoman: Create an metadata UnusedCheck and final pkg checks Create a plugin loop for any final pkg checks. Create the one plugin for the unused use-descriptions in mteadata.xml pym/repoman/modules/scan/metadata/__init__.py | 8 ++ pym/repoman/modules/scan/metadata/unused.py | 32 ++ pym/repoman/scanner.py| 39 --- 3 files changed, 69 insertions(+), 10 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py index 6ab44f6..ed4a967 100644 --- a/pym/repoman/modules/scan/metadata/__init__.py +++ b/pym/repoman/modules/scan/metadata/__init__.py @@ -50,6 +50,14 @@ module_spec = { 'func_desc': { }, }, + 'unused-metadata': { + 'name': "unused", + 'class': "UnusedCheck", + 'description': doc, + 'functions': ['check'], + 'func_desc': { + }, + }, } } diff --git a/pym/repoman/modules/scan/metadata/unused.py b/pym/repoman/modules/scan/metadata/unused.py new file mode 100644 index 000..5eb6716 --- /dev/null +++ b/pym/repoman/modules/scan/metadata/unused.py @@ -0,0 +1,32 @@ + + +class UnusedCheck(object): + + def __init__(self, **kwargs): + self.qatracker = kwargs.get('qatracker') + + def check(self, **kwargs): + xpkg = kwargs.get('xpkg') + muselist = kwargs.get('muselist') + used_useflags = kwargs.get('used_useflags') + # check if there are unused local USE-descriptions in metadata.xml + # (unless there are any invalids, to avoid noise) + if kwargs.get('allvalid'): + for myflag in muselist.difference(used_useflags): + self.qatracker.add_error( + "metadata.warning", + "%s/metadata.xml: unused local USE-description: '%s'" + % (xpkg, myflag)) + return {'continue': False} + + @property + def runInPkgs(self): + return (False, []) + + @property + def runInEbuilds(self): + return (False, []) + + @property + def runInFinal(self): + return (True, [self.check]) diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py index 08e53ac..cfc6802 100644 --- a/pym/repoman/scanner.py +++ b/pym/repoman/scanner.py @@ -271,7 +271,6 @@ class Scanner(object): def _scan_ebuilds(self, ebuildlist, dynamic_data): - xpkg = dynamic_data['xpkg'] # detect unused local USE-descriptions dynamic_data['used_useflags'] = set() @@ -320,13 +319,33 @@ class Scanner(object): if y_ebuild_continue: continue - print(" finished plugin loop, continuing...") + print(" finished ebuild plugin loop, continuing...") + + # Final checks + # initialize per pkg plugin final checks here + # need to set it up for ==> self.modules_list or some other ordered list + xpkg_complete = False + for mod in [('unused', 'UnusedChecks')]: + if mod[0]: + mod_class = MODULE_CONTROLLER.get_class(mod[0]) + print("Initializing class name:", mod_class.__name__) + self.modules[mod[1]] = mod_class(**self.kwargs) + print("scan_ebuilds final checks: module:", mod[1]) + do_it, functions = self.modules[mod[1]].runInFinal + # print("do_it", do_it, "functions", functions) + if do_it: + for func in functions: + print("\tRunning function:", func) + rdata = func(**dynamic_data) + if rdata['continue']: + xpkg_complete = True + print("\t>>> Continuing") + break + #print("rdata:", rdata) + dynamic_data.update(rdata) + #print("dynamic_data", dynamic_data)
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/, ...
commit: 4ee48de5477bbbe299cdb43f8779a9c9a0388eac Author: Brian Dolbec gentoo org> AuthorDate: Tue Mar 15 18:40:06 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Tue Mar 15 18:40:06 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=4ee48de5 repoman: Create a new boolean Fuse type Create a Fuse type which implememts a boolean as a one time fuse. The Fuse is initialized True, then is pop()'d to become False. Once the Fuse is blown, it can not be reset to True. Convert the use of the dynamic_data variable 'allvalid' to a Fuse instance. pym/repoman/fuse.py | 68 + pym/repoman/modules/scan/ebuild/ebuild.py | 7 ++- pym/repoman/modules/scan/ebuild/isebuild.py | 13 +++--- pym/repoman/modules/scan/metadata/unused.py | 2 +- pym/repoman/scanner.py | 2 + 5 files changed, 83 insertions(+), 9 deletions(-) diff --git a/pym/repoman/fuse.py b/pym/repoman/fuse.py new file mode 100644 index 000..ac864fd --- /dev/null +++ b/pym/repoman/fuse.py @@ -0,0 +1,68 @@ + +''' +fuse.py + +A tiny one-time-fuse class that uses a boolean to mimic the property of +an electrical fuse. IT's good (True) until it is popped (bad, False). +It is not resetable. +''' + + +class Fuse(object): + '''A One time fuse style boolean instance''' + + __slots__ = ('_state') + + def __init__(self): + self._state = True + + def pop(self): + '''Blow's the fuse state (makes it False)''' + self._state = False + + def __repr__(self): + '''x.__repr__() <==> repr(x)''' + return repr(self._state>0) + + def __str__(self): + '''x.__str__() <==> str(x)''' + return ['False', 'True'][self._state] + + def __bool__(self): + '''self != 0''' + return self._state != 0 + + def __nonzero__(self): + '''self != 0''' + return self._state != 0 + + def __abs__(self): + '''x.__abs__() <==> abs(x)''' + return [0, 1] [self._state] + + def __int__(self): + '''int(self)''' + return [0, 1][self._state] + + def __eq__(self, value): + '''Return self==value.''' + return self._state == value + + def __ne__(self, value): + '''Return self!=value.''' + return self._state != value + + def __ge__(self, value): + '''Return self>=value.''' + return self._state >= value + + def __gt__(self, value): + return self._state > value + + def __le__(self, value): + '''Return self<=value.''' + return self._state <= value + + def __lt__(self, value): + '''Return self self.modules or some other ordered list for mod in ['Manifests', 'IsEbuild', 'KeywordChecks', 'FileChecks',
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/
commit: fdb0fd395a582996cf88df2819bf246a2ddcf914 Author: Dirkjan Ochtman ochtman nl> AuthorDate: Mon Jan 25 19:40:29 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Tue May 3 06:54:24 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=fdb0fd39 repoman: Move parse_metadata_use function to the pkgmetadata.py pym/repoman/metadata.py | 50 --- pym/repoman/modules/scan/metadata/pkgmetadata.py | 52 +++- 2 files changed, 51 insertions(+), 51 deletions(-) diff --git a/pym/repoman/metadata.py b/pym/repoman/metadata.py index a980184..7c64c8e 100644 --- a/pym/repoman/metadata.py +++ b/pym/repoman/metadata.py @@ -17,7 +17,6 @@ except ImportError: # import our initialized portage instance from repoman._portage import portage -from portage import exception from portage import os from portage import shutil from portage.output import green @@ -34,55 +33,6 @@ metadata_xsd_uri = 'http://www.gentoo.org/xml-schema/metadata.xsd' metadata_xsd_ctime_interval = 60 * 60 * 24 * 7 # 7 days -def parse_metadata_use(xml_tree): - """ - Records are wrapped in XML as per GLEP 56 - returns a dict with keys constisting of USE flag names and values - containing their respective descriptions - """ - uselist = {} - - usetags = xml_tree.findall("use") - if not usetags: - return uselist - - # It's possible to have multiple 'use' elements. - for usetag in usetags: - flags = usetag.findall("flag") - if not flags: - # DTD allows use elements containing no flag elements. - continue - - for flag in flags: - pkg_flag = flag.get("name") - if pkg_flag is None: - raise exception.ParseError("missing 'name' attribute for 'flag' tag") - flag_restrict = flag.get("restrict") - - # emulate the Element.itertext() method from python-2.7 - inner_text = [] - stack = [] - stack.append(flag) - while stack: - obj = stack.pop() - if isinstance(obj, basestring): - inner_text.append(obj) - continue - if isinstance(obj.text, basestring): - inner_text.append(obj.text) - if isinstance(obj.tail, basestring): - stack.append(obj.tail) - stack.extend(reversed(obj)) - - if pkg_flag not in uselist: - uselist[pkg_flag] = {} - - # (flag_restrict can be None) - uselist[pkg_flag][flag_restrict] = " ".join("".join(inner_text).split()) - - return uselist - - def fetch_metadata_xsd(metadata_xsd, repoman_settings): """ Fetch metadata.xsd if it doesn't exist or the ctime is older than diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index a7150a9..cde2ba0 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -23,7 +23,7 @@ except (ImportError, SystemError, RuntimeError, Exception): # import our initialized portage instance from repoman._portage import portage -from repoman.metadata import metadata_dtd_uri, parse_metadata_use +from repoman.metadata import metadata_dtd_uri from repoman.checks.herds.herdbase import get_herd_base from repoman.checks.herds.metadata import check_metadata, UnknownHerdsError from repoman._xml import _XMLParser, _MetadataTreeBuilder, XmlLint @@ -32,6 +32,7 @@ from repoman.modules.scan.scanbase import ScanBase from portage.exception import InvalidAtom from portage import os from portage import _encodings, _unicode_encode +from portage import exception from portage.dep import Atom from .use_flags import USEFlagChecks @@ -42,6 +43,55 @@ metadata_xml_declaration = '' \ metadata_doctype_name = 'pkgmetadata' +def parse_metadata_use(xml_tree): + """ + Records are wrapped in XML as per GLEP 56 + returns a dict with keys constisting of USE flag names and values + containing their respective descriptions + """ + uselist = {} + + usetags = xml_tree.findall("use") + if not usetags: + return uselist + + # It's possible to have multiple 'use' elements. + for usetag in usetags: + flags = usetag.findall("flag") + if not flags: + # DTD allows use elements containing no flag elements. +
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/
commit: b7eec0cd3bb59c2b21e91a71619212dbff8a5b0b Author: Dirkjan Ochtman ochtman nl> AuthorDate: Tue May 3 09:01:29 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sun May 8 21:18:41 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b7eec0cd repoman: Migrate from XmlLint to etree.XMLSchema for validation Remove No longer used repoman._xml module This change based on work by Dirkjan Ochtman gentoo.org> Updated the change from XML.DTD to XMLSchema. Additionally: Move the metadata.xsd path determination code to metadata.py. Add the DISTDIR backup location and fetching of the file if missing or stale. pym/repoman/_xml.py | 105 --- pym/repoman/metadata.py | 21 + pym/repoman/modules/scan/metadata/pkgmetadata.py | 9 +- pym/repoman/scanner.py | 10 +-- 4 files changed, 26 insertions(+), 119 deletions(-) diff --git a/pym/repoman/_xml.py b/pym/repoman/_xml.py deleted file mode 100644 index 33a536a..000 --- a/pym/repoman/_xml.py +++ /dev/null @@ -1,105 +0,0 @@ -# -*- coding:utf-8 -*- - -from __future__ import print_function, unicode_literals - -import sys -import xml - -# import our initialized portage instance -from repoman._portage import portage - -from portage import os -from portage.output import red -from portage.process import find_binary - -from repoman.metadata import fetch_metadata_xsd -from repoman._subprocess import repoman_getstatusoutput - - -class _XMLParser(xml.etree.ElementTree.XMLParser): - - def __init__(self, data, **kwargs): - xml.etree.ElementTree.XMLParser.__init__(self, **kwargs) - self._portage_data = data - if hasattr(self, 'parser'): - self._base_XmlDeclHandler = self.parser.XmlDeclHandler - self.parser.XmlDeclHandler = self._portage_XmlDeclHandler - self._base_StartDoctypeDeclHandler = \ - self.parser.StartDoctypeDeclHandler - self.parser.StartDoctypeDeclHandler = \ - self._portage_StartDoctypeDeclHandler - - def _portage_XmlDeclHandler(self, version, encoding, standalone): - if self._base_XmlDeclHandler is not None: - self._base_XmlDeclHandler(version, encoding, standalone) - self._portage_data["XML_DECLARATION"] = (version, encoding, standalone) - - def _portage_StartDoctypeDeclHandler( - self, doctypeName, systemId, publicId, has_internal_subset): - if self._base_StartDoctypeDeclHandler is not None: - self._base_StartDoctypeDeclHandler( - doctypeName, systemId, publicId, has_internal_subset) - self._portage_data["DOCTYPE"] = (doctypeName, systemId, publicId) - - -class _MetadataTreeBuilder(xml.etree.ElementTree.TreeBuilder): - """ - Implements doctype() as required to avoid deprecation warnings with - >=python-2.7. - """ - def doctype(self, name, pubid, system): - pass - - -class XmlLint(object): - - def __init__(self, options, repoman_settings, metadata_xsd=None): - self.metadata_xsd = (metadata_xsd or - os.path.join(repoman_settings["DISTDIR"], 'metadata.xsd')) - self.options = options - self.repoman_settings = repoman_settings - self._is_capable = metadata_xsd is not None - self.binary = None - self._check_capable() - - def _check_capable(self): - if self.options.mode == "manifest": - return - self.binary = find_binary('xmllint') - if not self.binary: - print(red("!!! xmllint not found. Can't check metadata.xml.\n")) - elif not self._is_capable: - if not fetch_metadata_xsd(self.metadata_xsd, self.repoman_settings): - sys.exit(1) - # this can be problematic if xmllint changes their output - self._is_capable = True - - @property - def capable(self): - return self._is_capable - - def check(self, checkdir, repolevel): - '''Runs checks on the package metadata.xml file - - @param checkdir: string, path - @param repolevel: integer - @return boolean, False == bad metadata - ''' - if not self.capable: - if self.options.xml_parse or repolevel == 3: - print("%s sorry, xmllint is needed. failing\n" % red("!!!")) - sys.exit(1) - return True - # xmlint can produce garbage output eve
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/
commit: 20fd37b3ddb901f15135d05518a7d8a7bfb7c320 Author: Dirkjan Ochtman ochtman nl> AuthorDate: Mon Jan 25 19:40:29 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sun May 8 21:18:41 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=20fd37b3 repoman: Move parse_metadata_use function to the pkgmetadata.py Add missing basestring compat code pym/repoman/metadata.py | 50 - pym/repoman/modules/scan/metadata/pkgmetadata.py | 56 +++- 2 files changed, 55 insertions(+), 51 deletions(-) diff --git a/pym/repoman/metadata.py b/pym/repoman/metadata.py index a980184..7c64c8e 100644 --- a/pym/repoman/metadata.py +++ b/pym/repoman/metadata.py @@ -17,7 +17,6 @@ except ImportError: # import our initialized portage instance from repoman._portage import portage -from portage import exception from portage import os from portage import shutil from portage.output import green @@ -34,55 +33,6 @@ metadata_xsd_uri = 'http://www.gentoo.org/xml-schema/metadata.xsd' metadata_xsd_ctime_interval = 60 * 60 * 24 * 7 # 7 days -def parse_metadata_use(xml_tree): - """ - Records are wrapped in XML as per GLEP 56 - returns a dict with keys constisting of USE flag names and values - containing their respective descriptions - """ - uselist = {} - - usetags = xml_tree.findall("use") - if not usetags: - return uselist - - # It's possible to have multiple 'use' elements. - for usetag in usetags: - flags = usetag.findall("flag") - if not flags: - # DTD allows use elements containing no flag elements. - continue - - for flag in flags: - pkg_flag = flag.get("name") - if pkg_flag is None: - raise exception.ParseError("missing 'name' attribute for 'flag' tag") - flag_restrict = flag.get("restrict") - - # emulate the Element.itertext() method from python-2.7 - inner_text = [] - stack = [] - stack.append(flag) - while stack: - obj = stack.pop() - if isinstance(obj, basestring): - inner_text.append(obj) - continue - if isinstance(obj.text, basestring): - inner_text.append(obj.text) - if isinstance(obj.tail, basestring): - stack.append(obj.tail) - stack.extend(reversed(obj)) - - if pkg_flag not in uselist: - uselist[pkg_flag] = {} - - # (flag_restrict can be None) - uselist[pkg_flag][flag_restrict] = " ".join("".join(inner_text).split()) - - return uselist - - def fetch_metadata_xsd(metadata_xsd, repoman_settings): """ Fetch metadata.xsd if it doesn't exist or the ctime is older than diff --git a/pym/repoman/modules/scan/metadata/pkgmetadata.py b/pym/repoman/modules/scan/metadata/pkgmetadata.py index a7150a9..af53f4b 100644 --- a/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ b/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -23,7 +23,7 @@ except (ImportError, SystemError, RuntimeError, Exception): # import our initialized portage instance from repoman._portage import portage -from repoman.metadata import metadata_dtd_uri, parse_metadata_use +from repoman.metadata import metadata_dtd_uri from repoman.checks.herds.herdbase import get_herd_base from repoman.checks.herds.metadata import check_metadata, UnknownHerdsError from repoman._xml import _XMLParser, _MetadataTreeBuilder, XmlLint @@ -32,16 +32,70 @@ from repoman.modules.scan.scanbase import ScanBase from portage.exception import InvalidAtom from portage import os from portage import _encodings, _unicode_encode +from portage import exception from portage.dep import Atom from .use_flags import USEFlagChecks +if sys.hexversion >= 0x300: + # pylint: disable=W0622 + basestring = str + metadata_xml_encoding = 'UTF-8' metadata_xml_declaration = '' \ % (metadata_xml_encoding,) metadata_doctype_name = 'pkgmetadata' +def parse_metadata_use(xml_tree): + """ + Records are wrapped in XML as per GLEP 56 + returns a dict with keys constisting of USE flag names and values + containing their respective descriptions + """ + uselist = {} + + usetags = xml_tree.findall("use") + if not usetags: + return uselist + + # It's possible to have multiple 'use' elements. + for usetag in useta
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/, ...
commit: d7bacba37b93eaf77a0669cda344a41c99a64658 Author: Zac Medico gentoo org> AuthorDate: Sat Apr 16 20:24:06 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Thu Apr 21 16:51:34 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d7bacba3 repoman: replace Fuse with Future Replace Fuse with Future, which is similar more generic. The code ends up being slightly more verbose, but more flexible. The Future class will be useful elsewhere, including the EventLoop class. pym/portage/util/futures.py | 118 pym/repoman/fuse.py | 68 pym/repoman/main.py | 12 ++- pym/repoman/modules/scan/ebuild/ebuild.py | 10 ++- pym/repoman/modules/scan/ebuild/isebuild.py | 25 -- pym/repoman/modules/scan/metadata/unused.py | 10 ++- pym/repoman/scanner.py | 4 +- 7 files changed, 164 insertions(+), 83 deletions(-) diff --git a/pym/portage/util/futures.py b/pym/portage/util/futures.py new file mode 100644 index 000..c648f10 --- /dev/null +++ b/pym/portage/util/futures.py @@ -0,0 +1,118 @@ +# Copyright 2016 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# +# For compatibility with python versions which do not have the +# asyncio module (Python 3.3 and earlier), this module provides a +# subset of the asyncio.futures.Futures interface. + +from __future__ import unicode_literals + +__all__ = ( + 'CancelledError', + 'Future', + 'InvalidStateError', +) + +try: + from asyncio import ( + CancelledError, + Future, + InvalidStateError, + ) +except ImportError: + + from portage.exception import PortageException + + _PENDING = 'PENDING' + _CANCELLED = 'CANCELLED' + _FINISHED = 'FINISHED' + + class Error(PortageException): + pass + + class CancelledError(Error): + def __init__(self): + Error.__init__(self, "cancelled") + + class InvalidStateError(Error): + pass + + class Future(object): + + # Class variables serving as defaults for instance variables. + _state = _PENDING + _result = None + _exception = None + + def cancel(self): + """Cancel the future and schedule callbacks. + + If the future is already done or cancelled, return False. Otherwise, + change the future's state to cancelled, schedule the callbacks and + return True. + """ + if self._state != _PENDING: + return False + self._state = _CANCELLED + return True + + def done(self): + """Return True if the future is done. + + Done means either that a result / exception are available, or that the + future was cancelled. + """ + return self._state != _PENDING + + def result(self): + """Return the result this future represents. + + If the future has been cancelled, raises CancelledError. If the + future's result isn't yet available, raises InvalidStateError. If + the future is done and has an exception set, this exception is raised. + """ + if self._state == _CANCELLED: + raise CancelledError() + if self._state != _FINISHED: + raise InvalidStateError('Result is not ready.') + if self._exception is not None: + raise self._exception + return self._result + + def exception(self): + """Return the exception that was set on this future. + + The exception (or None if no exception was set) is returned only if + the future is done. If the future has been cancelled, raises + CancelledError. If the future isn't done yet, raises + InvalidStateError. + """ + if self._state == _CANCELLED: + raise CancelledError + if self._state != _FINISHED: + raise InvalidStateError('Exception is not set.') + return self._exception + + def set_result(self, result): + """Mark the future done and set its result. + + If the future is already done when this method is called, ra
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/
commit: b275450533245b5097b5f6c08a84df999ed2c989 Author: Michał Górny gentoo org> AuthorDate: Sun Apr 17 08:06:28 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Mon Apr 25 15:03:37 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b2754505 repoman: Use XML Schema for metadata.xml validation pym/repoman/_xml.py | 16 +- pym/repoman/metadata.py | 39 pym/repoman/modules/scan/metadata/__init__.py| 2 +- pym/repoman/modules/scan/metadata/pkgmetadata.py | 6 ++-- pym/repoman/scanner.py | 8 ++--- 5 files changed, 36 insertions(+), 35 deletions(-) diff --git a/pym/repoman/_xml.py b/pym/repoman/_xml.py index d55dda5..33a536a 100644 --- a/pym/repoman/_xml.py +++ b/pym/repoman/_xml.py @@ -12,7 +12,7 @@ from portage import os from portage.output import red from portage.process import find_binary -from repoman.metadata import fetch_metadata_dtd +from repoman.metadata import fetch_metadata_xsd from repoman._subprocess import repoman_getstatusoutput @@ -53,12 +53,12 @@ class _MetadataTreeBuilder(xml.etree.ElementTree.TreeBuilder): class XmlLint(object): - def __init__(self, options, repoman_settings, metadata_dtd=None): - self.metadata_dtd = (metadata_dtd or - os.path.join(repoman_settings["DISTDIR"], 'metadata.dtd')) + def __init__(self, options, repoman_settings, metadata_xsd=None): + self.metadata_xsd = (metadata_xsd or + os.path.join(repoman_settings["DISTDIR"], 'metadata.xsd')) self.options = options self.repoman_settings = repoman_settings - self._is_capable = metadata_dtd is not None + self._is_capable = metadata_xsd is not None self.binary = None self._check_capable() @@ -69,7 +69,7 @@ class XmlLint(object): if not self.binary: print(red("!!! xmllint not found. Can't check metadata.xml.\n")) elif not self._is_capable: - if not fetch_metadata_dtd(self.metadata_dtd, self.repoman_settings): + if not fetch_metadata_xsd(self.metadata_xsd, self.repoman_settings): sys.exit(1) # this can be problematic if xmllint changes their output self._is_capable = True @@ -93,8 +93,8 @@ class XmlLint(object): # xmlint can produce garbage output even on success, so only dump # the ouput when it fails. st, out = repoman_getstatusoutput( - self.binary + " --nonet --noout --dtdvalid %s %s" % ( - portage._shell_quote(self.metadata_dtd), + self.binary + " --nonet --noout --schema %s %s" % ( + portage._shell_quote(self.metadata_xsd), portage._shell_quote( os.path.join(checkdir, "metadata.xml" if st != os.EX_OK: diff --git a/pym/repoman/metadata.py b/pym/repoman/metadata.py index e95ad41..7a514dc 100644 --- a/pym/repoman/metadata.py +++ b/pym/repoman/metadata.py @@ -33,8 +33,9 @@ metadata_xml_declaration = '' \ % (metadata_xml_encoding,) metadata_doctype_name = 'pkgmetadata' metadata_dtd_uri = 'http://www.gentoo.org/dtd/metadata.dtd' +metadata_xsd_uri = 'http://www.gentoo.org/xml-schema/metadata.xsd' # force refetch if the local copy creation time is older than this -metadata_dtd_ctime_interval = 60 * 60 * 24 * 7 # 7 days +metadata_xsd_ctime_interval = 60 * 60 * 24 * 7 # 7 days def parse_metadata_use(xml_tree): @@ -86,36 +87,36 @@ def parse_metadata_use(xml_tree): return uselist -def fetch_metadata_dtd(metadata_dtd, repoman_settings): +def fetch_metadata_xsd(metadata_xsd, repoman_settings): """ - Fetch metadata.dtd if it doesn't exist or the ctime is older than - metadata_dtd_ctime_interval. + Fetch metadata.xsd if it doesn't exist or the ctime is older than + metadata_xsd_ctime_interval. @rtype: bool @return: True if successful, otherwise False """ must_fetch = True - metadata_dtd_st = None + metadata_xsd_st = None current_time = int(time.time()) try: - metadata_dtd_st = os.stat(metadata_dtd) + metadata_xsd_st = os.stat(metadata_xsd) except EnvironmentError as e: if e.errno not in (errno.ENOENT, errno.ESTALE): raise del e else: - # Trigger fetch if metadata.dtd mtime is old or clock is wrong. - if abs(current_time - metadata_dtd_st.st_ctime) \ - < metadata_dtd_ctime_interval: +
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/, ...
commit: 3dc78ff91eccd81c972ceadf59d059aabe9ccdbc Author: Brian Dolbec gentoo org> AuthorDate: Tue Mar 15 18:40:06 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Mon Apr 25 15:28:53 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3dc78ff9 repoman: Create a new boolean Fuse type Create a Fuse type which implememts a boolean as a one time fuse. The Fuse is initialized True, then is pop()'d to become False. Once the Fuse is blown, it can not be reset to True. Convert the use of the dynamic_data variable 'allvalid' to a Fuse instance. pym/repoman/fuse.py | 68 + pym/repoman/modules/scan/ebuild/ebuild.py | 7 ++- pym/repoman/modules/scan/ebuild/isebuild.py | 13 +++--- pym/repoman/modules/scan/metadata/unused.py | 2 +- pym/repoman/scanner.py | 2 + 5 files changed, 83 insertions(+), 9 deletions(-) diff --git a/pym/repoman/fuse.py b/pym/repoman/fuse.py new file mode 100644 index 000..ac864fd --- /dev/null +++ b/pym/repoman/fuse.py @@ -0,0 +1,68 @@ + +''' +fuse.py + +A tiny one-time-fuse class that uses a boolean to mimic the property of +an electrical fuse. IT's good (True) until it is popped (bad, False). +It is not resetable. +''' + + +class Fuse(object): + '''A One time fuse style boolean instance''' + + __slots__ = ('_state') + + def __init__(self): + self._state = True + + def pop(self): + '''Blow's the fuse state (makes it False)''' + self._state = False + + def __repr__(self): + '''x.__repr__() <==> repr(x)''' + return repr(self._state>0) + + def __str__(self): + '''x.__str__() <==> str(x)''' + return ['False', 'True'][self._state] + + def __bool__(self): + '''self != 0''' + return self._state != 0 + + def __nonzero__(self): + '''self != 0''' + return self._state != 0 + + def __abs__(self): + '''x.__abs__() <==> abs(x)''' + return [0, 1] [self._state] + + def __int__(self): + '''int(self)''' + return [0, 1][self._state] + + def __eq__(self, value): + '''Return self==value.''' + return self._state == value + + def __ne__(self, value): + '''Return self!=value.''' + return self._state != value + + def __ge__(self, value): + '''Return self>=value.''' + return self._state >= value + + def __gt__(self, value): + return self._state > value + + def __le__(self, value): + '''Return self<=value.''' + return self._state <= value + + def __lt__(self, value): + '''Return self self.modules or some other ordered list for mod in ['Manifests', 'IsEbuild', 'KeywordChecks', 'FileChecks',
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/, ...
commit: 836cc3ffc55376348315577958e7212fefa38a2a Author: Zac Medico gentoo org> AuthorDate: Mon Apr 25 16:50:59 2016 + Commit: Zac Medico gentoo org> CommitDate: Mon Apr 25 16:58:53 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=836cc3ff ProfileDependsChecks: encapsulate DependChecks and LicenseChecks pym/repoman/modules/scan/depend/__init__.py | 38 +- pym/repoman/modules/scan/depend/_depend_checks.py | 150 pym/repoman/modules/scan/depend/depend.py | 158 -- pym/repoman/modules/scan/depend/profile.py| 17 ++- pym/repoman/modules/scan/metadata/__init__.py | 17 --- pym/repoman/modules/scan/metadata/license.py | 54 pym/repoman/scanner.py| 4 +- 7 files changed, 167 insertions(+), 271 deletions(-) diff --git a/pym/repoman/modules/scan/depend/__init__.py b/pym/repoman/modules/scan/depend/__init__.py index 01bd116..6d12286 100644 --- a/pym/repoman/modules/scan/depend/__init__.py +++ b/pym/repoman/modules/scan/depend/__init__.py @@ -10,24 +10,6 @@ module_spec = { 'name': 'depend', 'description': doc, 'provides':{ - 'depend-module': { - 'name': "depend", - 'sourcefile': "depend", - 'class': "DependChecks", - 'description': doc, - 'functions': ['check'], - 'func_desc': { - }, - 'mod_kwargs': ['qatracker', 'portdb' - ], - 'func_kwargs': { - 'baddepsyntax': ('Future', False), - 'badlicsyntax': ('Future', False), - 'ebuild': (None, None), - 'pkg': (None, None), - 'unknown_pkgs': ('Future', 'UNSET'), - }, - }, 'profile-module': { 'name': "profile", 'sourcefile': "profile", @@ -37,30 +19,12 @@ module_spec = { 'func_desc': { }, 'mod_kwargs': ['qatracker', 'portdb', 'profiles', 'options', - 'repo_settings', 'include_arches', 'caches', + 'repo_metadata', 'repo_settings', 'include_arches', 'caches', 'repoman_incrementals', 'env', 'have', 'dev_keywords' ], 'func_kwargs': { - 'baddepsyntax': (None, None), 'ebuild': (None, None), 'pkg': (None, None), - 'unknown_pkgs': (None, None), - }, - }, - 'unknown-module': { - 'name': "unknown", - 'sourcefile': "unknown", - 'class': "DependUnknown", - 'description': doc, - 'functions': ['check'], - 'func_desc': { - }, - 'mod_kwargs': ['qatracker', - ], - 'func_kwargs': { - 'baddepsyntax': (None, None), - 'ebuild': (None, None), - 'unknown_pkgs': ('Future', 'UNSET'), }, }, } diff --git a/pym/repoman/modules/scan/depend/_depend_checks.py b/pym/repoman/modules/scan/depend/_depend_checks.py new file mode 100644 index 000..4e1d216 --- /dev/null +++ b/pym/repoman/modules/scan/depend/_depend_checks.py @@ -0,0 +1,150 @@ +# -*- coding:utf-8 -*- + + +from _emerge.Package import Package + +from repoman.check_missingslot import check_missingslot +# import our initialized portage instance +from repoman._portage import portage +from repoman.qa_data import suspect_virtual, suspect_rdepend + + +def _depend_checks(ebuild, pkg, portdb, qatracker, repo_metadata): + '''Checks the ebuild dependencies for errors + + @param pkg: Package in which we check (object). + @param ebuild: Ebuild which we check (object). + @param portdb: portdb instance + @param qatracker: QATracker instance + @param repo_metadata: dictionary of various repository items. + @returns: (unknown_pkgs, badlicsyntax) + ''' + + unknown_pkgs = set() + + inherited_java_eclass = "java-pkg-2" in ebuild.inherited or \ + "java-pkg-opt-2" in ebuild.inherited, + inherited_wxwidgets_eclass = "wxwidgets" in ebuild.inherited + # operator_tokens = set(["||", "(", ")"]) + type_list, badsyntax = [], [] + for mytype in Package._dep_keys + ("LICENSE",
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/checks/ebuilds/variables/, ...
commit: 40b9ed5989fd3793ef23a6a23b529362e8327c8d Author: Brian Dolbec gentoo org> AuthorDate: Sun Jan 3 17:36:26 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sat Jan 30 07:50:17 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=40b9ed59 repoman: Migrate DescriptionChecks to the plugin system pym/repoman/modules/scan/metadata/__init__.py| 9 + .../scan/metadata}/description.py| 20 ++-- pym/repoman/scanner.py | 5 + 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py index bacedf5..83aac7f 100644 --- a/pym/repoman/modules/scan/metadata/__init__.py +++ b/pym/repoman/modules/scan/metadata/__init__.py @@ -28,6 +28,15 @@ module_spec = { 'func_desc': { }, }, + 'description-metadata': { + 'name': "description", + 'sourcefile': "description", + 'class': "DescriptionChecks", + 'description': doc, + 'functions': ['check'], + 'func_desc': { + }, + }, } } diff --git a/pym/repoman/checks/ebuilds/variables/description.py b/pym/repoman/modules/scan/metadata/description.py similarity index 66% rename from pym/repoman/checks/ebuilds/variables/description.py rename to pym/repoman/modules/scan/metadata/description.py index a2b1057..3570607 100644 --- a/pym/repoman/checks/ebuilds/variables/description.py +++ b/pym/repoman/modules/scan/metadata/description.py @@ -9,20 +9,19 @@ from repoman.qa_data import max_desc_len class DescriptionChecks(object): '''Perform checks on the DESCRIPTION variable.''' - def __init__(self, qatracker): + def __init__(self, **kwargs): ''' @param qatracker: QATracker instance ''' - self.qatracker = qatracker + self.qatracker = kwargs.get('qatracker') - def check(self, pkg, ebuild): + def checkTooLong(self, **kwargs): ''' @param pkg: Package in which we check (object). @param ebuild: Ebuild which we check (object). ''' - self._checkTooLong(pkg, ebuild) - - def _checkTooLong(self, pkg, ebuild): + ebuild = kwargs.get('ebuild') + pkg = kwargs.get('pkg') # 14 is the length of DESCRIPTION="" if len(pkg._metadata['DESCRIPTION']) > max_desc_len: self.qatracker.add_error( @@ -30,3 +29,12 @@ class DescriptionChecks(object): "%s: DESCRIPTION is %d characters (max %d)" % (ebuild.relative_path, len( pkg._metadata['DESCRIPTION']), max_desc_len)) + return {'continue': False} + + @property + def runInPkgs(self): + return (False, []) + + @property + def runInEbuilds(self): + return (True, [self.checkTooLong]) diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py index 28142ae..27c1be9 100644 --- a/pym/repoman/scanner.py +++ b/pym/repoman/scanner.py @@ -21,7 +21,6 @@ from repoman.checks.ebuilds.checks import run_checks from repoman.checks.ebuilds.eclasses.ruby import RubyEclassChecks from repoman.check_missingslot import check_missingslot from repoman.checks.ebuilds.use_flags import USEFlagChecks -from repoman.checks.ebuilds.variables.description import DescriptionChecks from repoman.checks.ebuilds.variables.license import LicenseChecks from repoman.checks.ebuilds.variables.restrict import RestrictChecks from repoman.modules.commit import repochecks @@ -219,7 +218,6 @@ class Scanner(object): # initialize our checks classes here before the big xpkg loop self.use_flag_checks = USEFlagChecks(self.qatracker, uselist) self.rubyeclasscheck = RubyEclassChecks(self.qatracker) - self.descriptioncheck = DescriptionChecks(self.qatracker) self.licensecheck = LicenseChecks(self.qatracker, liclist, liclist_deprecated) self.restrictcheck = RestrictChecks(self.qatracker) @@ -304,6 +302,7 @@ class Scanner(object): for mod in [('ebuild', 'Ebuild'), ('live', 'LiveEclassChecks'), ('eapi', 'EAPIChecks'), ('ebuild_metadata', 'EbuildMetadata'), ('thirdpartymirrors', 'ThirdPartyMirrors'), + ('description', 'DescriptionChecks'), ]: if mod[0]: mod_class = MODULE_CONTR
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/checks/ebuilds/variables/, ...
commit: 0172019530d7a71d04f4ded5cda8008edbaab739 Author: Brian Dolbec gentoo org> AuthorDate: Sun Jan 3 23:10:48 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sat Jan 30 07:50:19 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=01720195 repoman: Migrate RestrictChecks to a plugin module pym/repoman/modules/scan/metadata/__init__.py | 9 +++ .../scan/metadata}/restrict.py | 29 +++--- pym/repoman/scanner.py | 9 +-- 3 files changed, 30 insertions(+), 17 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py index c8f3609..4f376e1 100644 --- a/pym/repoman/modules/scan/metadata/__init__.py +++ b/pym/repoman/modules/scan/metadata/__init__.py @@ -46,6 +46,15 @@ module_spec = { 'func_desc': { }, }, + 'restrict-metadata': { + 'name': "restrict", + 'sourcefile': "restrict", + 'class': "RestrictChecks", + 'description': doc, + 'functions': ['check'], + 'func_desc': { + }, + }, } } diff --git a/pym/repoman/checks/ebuilds/variables/restrict.py b/pym/repoman/modules/scan/metadata/restrict.py similarity index 54% rename from pym/repoman/checks/ebuilds/variables/restrict.py rename to pym/repoman/modules/scan/metadata/restrict.py index 215b792..93ca298 100644 --- a/pym/repoman/checks/ebuilds/variables/restrict.py +++ b/pym/repoman/modules/scan/metadata/restrict.py @@ -12,21 +12,23 @@ from repoman.qa_data import valid_restrict class RestrictChecks(object): '''Perform checks on the RESTRICT variable.''' - def __init__(self, qatracker): + def __init__(self, **kwargs): ''' @param qatracker: QATracker instance ''' - self.qatracker = qatracker + self.qatracker = kwargs.get('qatracker') - def check(self, pkg, package, ebuild, y_ebuild): + def check(self, **kwargs): + xpkg = kwargs.get('xpkg') + ebuild = kwargs.get('ebuild') + y_ebuild = kwargs.get('y_ebuild') myrestrict = None try: myrestrict = portage.dep.use_reduce( - pkg._metadata["RESTRICT"], matchall=1, flat=True) + ebuild.metadata["RESTRICT"], matchall=1, flat=True) except portage.exception.InvalidDependString as e: - self. qatracker.add_error( - "RESTRICT.syntax", + self.qatracker.add_error("RESTRICT.syntax", "%s: RESTRICT: %s" % (ebuild.relative_path, e)) del e @@ -36,6 +38,15 @@ class RestrictChecks(object): if mybadrestrict: for mybad in mybadrestrict: - self.qatracker.add_error( - "RESTRICT.invalid", - package + "/" + y_ebuild + ".ebuild: %s" % mybad) + self.qatracker.add_error("RESTRICT.invalid", + "%s/%s.ebuild: %s" % (xpkg, y_ebuild, mybad)) + return {'continue': False} + + @property + def runInPkgs(self): + return (False, []) + + @property + def runInEbuilds(self): + return (True, [self.check]) + diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py index 55f3d56..dcb955c 100644 --- a/pym/repoman/scanner.py +++ b/pym/repoman/scanner.py @@ -18,7 +18,6 @@ from portage import _unicode_encode from portage.dep import Atom from portage.output import green from repoman.checks.ebuilds.checks import run_checks -from repoman.checks.ebuilds.variables.restrict import RestrictChecks from repoman.modules.commit import repochecks from repoman.profile import check_profiles, dev_profile_keywords, setup_profile from repoman.repos import repo_metadata @@ -211,10 +210,6 @@ class Scanner(object): print("Initializing class name:", mod_class.__name__) self.modules[mod_class.__name__] = mod_class(**self.kwargs) - # initialize our checks classes here before the big xpkg loop - self.restrictcheck = RestrictChecks(self.qatracker) - - def scan_pkgs(self, can_force): dynamic_data = {'can_force': can_force} for xpkg in self.effective_scanlist: @@ -298,7 +293,7 @@ class Scanner(object): ('description', 'DescriptionChecks'), (Non
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/checks/ebuilds/variables/, ...
commit: 9e70c49fc67178c884765ec86ae2c07d44c9fd6f Author: Brian Dolbec gentoo org> AuthorDate: Sun Jan 3 23:09:27 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sat Mar 12 17:57:37 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=9e70c49f repoman: Migrate license checks to a plugin module pym/repoman/checks/ebuilds/variables/license.py | 47 -- pym/repoman/modules/scan/metadata/__init__.py | 9 + pym/repoman/modules/scan/metadata/license.py| 53 + pym/repoman/scanner.py | 7 +--- 4 files changed, 63 insertions(+), 53 deletions(-) diff --git a/pym/repoman/checks/ebuilds/variables/license.py b/pym/repoman/checks/ebuilds/variables/license.py deleted file mode 100644 index bdc859c..000 --- a/pym/repoman/checks/ebuilds/variables/license.py +++ /dev/null @@ -1,47 +0,0 @@ - -'''description.py -Perform checks on the LICENSE variable. -''' - -# import our initialized portage instance -from repoman._portage import portage - - -class LicenseChecks(object): - '''Perform checks on the LICENSE variable.''' - - def __init__(self, qatracker, liclist, liclist_deprecated): - ''' - @param qatracker: QATracker instance - @param liclist: List of licenses. - @param liclist: List of deprecated licenses. - ''' - self.qatracker = qatracker - self.liclist = liclist - self.liclist_deprecated = liclist_deprecated - - def check( - self, pkg, package, ebuild, y_ebuild): - ''' - @param pkg: Package in which we check (object). - @param package: Package in which we check (string). - @param ebuild: Ebuild which we check (object). - @param y_ebuild: Ebuild which we check (string). - ''' - - # Parse the LICENSE variable, remove USE conditions and flatten it. - licenses = portage.dep.use_reduce( - pkg._metadata["LICENSE"], matchall=1, flat=True) - - # Check each entry to ensure that it exists in ${PORTDIR}/licenses/. - for lic in licenses: - # Need to check for "||" manually as no portage - # function will remove it without removing values. - if lic not in self.liclist and lic != "||": - self.qatracker.add_error( - "LICENSE.invalid", - package + "/" + y_ebuild + ".ebuild: %s" % lic) - elif lic in self.liclist_deprecated: - self.qatracker.add_error( - "LICENSE.deprecated", - "%s: %s" % (ebuild.relative_path, lic)) diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py index 83aac7f..c8f3609 100644 --- a/pym/repoman/modules/scan/metadata/__init__.py +++ b/pym/repoman/modules/scan/metadata/__init__.py @@ -37,6 +37,15 @@ module_spec = { 'func_desc': { }, }, + 'license-metadata': { + 'name': "license", + 'sourcefile': "license", + 'class': "LicenseChecks", + 'description': doc, + 'functions': ['check'], + 'func_desc': { + }, + }, } } diff --git a/pym/repoman/modules/scan/metadata/license.py b/pym/repoman/modules/scan/metadata/license.py new file mode 100644 index 000..b022b20 --- /dev/null +++ b/pym/repoman/modules/scan/metadata/license.py @@ -0,0 +1,53 @@ + +'''license.py +Perform checks on the LICENSE variable. +''' + +# import our initialized portage instance +from repoman._portage import portage + + +class LicenseChecks(object): + '''Perform checks on the LICENSE variable.''' + + def __init__(self, **kwargs): + ''' + @param qatracker: QATracker instance + @param repo_metadata: dictionary of various repository items. + ''' + self.qatracker = kwargs.get('qatracker') + self.repo_metadata = kwargs.get('repo_metadata') + + def check(self, **kwargs): + ''' + @param xpkg: Package in which we check (string). + @param ebuild: Ebuild which we check (object). + @param y_ebuild: Ebuild which we check (string). + ''' + xpkg = kwargs.get('xpkg') + ebuild = kwargs.get('ebuild') + y_ebuild = kwargs.get('y_ebuild') + if not kwargs.get('badlicsyntax'): + # Parse the L
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/checks/ebuilds/variables/, ...
commit: 50b8e2d62f8e6312706e51f55e72720637a31d74 Author: Brian Dolbec gentoo org> AuthorDate: Sun Jan 3 23:10:48 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sun Jan 10 20:15:08 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=50b8e2d6 repoman: Migrate RestrictChecks to a plugin module pym/repoman/modules/scan/metadata/__init__.py | 8 ++ .../scan/metadata}/restrict.py | 29 +++--- pym/repoman/scanner.py | 7 +- 3 files changed, 29 insertions(+), 15 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py index ed0c59d..6ab44f6 100644 --- a/pym/repoman/modules/scan/metadata/__init__.py +++ b/pym/repoman/modules/scan/metadata/__init__.py @@ -42,6 +42,14 @@ module_spec = { 'func_desc': { }, }, + 'restrict-metadata': { + 'name': "restrict", + 'class': "RestrictChecks", + 'description': doc, + 'functions': ['check'], + 'func_desc': { + }, + }, } } diff --git a/pym/repoman/checks/ebuilds/variables/restrict.py b/pym/repoman/modules/scan/metadata/restrict.py similarity index 54% rename from pym/repoman/checks/ebuilds/variables/restrict.py rename to pym/repoman/modules/scan/metadata/restrict.py index 215b792..93ca298 100644 --- a/pym/repoman/checks/ebuilds/variables/restrict.py +++ b/pym/repoman/modules/scan/metadata/restrict.py @@ -12,21 +12,23 @@ from repoman.qa_data import valid_restrict class RestrictChecks(object): '''Perform checks on the RESTRICT variable.''' - def __init__(self, qatracker): + def __init__(self, **kwargs): ''' @param qatracker: QATracker instance ''' - self.qatracker = qatracker + self.qatracker = kwargs.get('qatracker') - def check(self, pkg, package, ebuild, y_ebuild): + def check(self, **kwargs): + xpkg = kwargs.get('xpkg') + ebuild = kwargs.get('ebuild') + y_ebuild = kwargs.get('y_ebuild') myrestrict = None try: myrestrict = portage.dep.use_reduce( - pkg._metadata["RESTRICT"], matchall=1, flat=True) + ebuild.metadata["RESTRICT"], matchall=1, flat=True) except portage.exception.InvalidDependString as e: - self. qatracker.add_error( - "RESTRICT.syntax", + self.qatracker.add_error("RESTRICT.syntax", "%s: RESTRICT: %s" % (ebuild.relative_path, e)) del e @@ -36,6 +38,15 @@ class RestrictChecks(object): if mybadrestrict: for mybad in mybadrestrict: - self.qatracker.add_error( - "RESTRICT.invalid", - package + "/" + y_ebuild + ".ebuild: %s" % mybad) + self.qatracker.add_error("RESTRICT.invalid", + "%s/%s.ebuild: %s" % (xpkg, y_ebuild, mybad)) + return {'continue': False} + + @property + def runInPkgs(self): + return (False, []) + + @property + def runInEbuilds(self): + return (True, [self.check]) + diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py index 1f792bd..8657c73 100644 --- a/pym/repoman/scanner.py +++ b/pym/repoman/scanner.py @@ -18,7 +18,6 @@ from portage import _unicode_encode from portage.dep import Atom from portage.output import green from repoman.checks.ebuilds.checks import run_checks -from repoman.checks.ebuilds.variables.restrict import RestrictChecks from repoman.modules.commit import repochecks from repoman.profile import check_profiles, dev_profile_keywords, setup_profile from repoman.repos import repo_metadata @@ -209,8 +208,6 @@ class Scanner(object): self.modules[mod_class.__name__] = mod_class(**self.kwargs) # initialize our checks classes here before the big xpkg loop - self.restrictcheck = RestrictChecks(self.qatracker) - def scan_pkgs(self, can_force): dynamic_data = {'can_force': can_force} @@ -295,7 +292,7 @@ class Scanner(object): ('description', 'DescriptionChecks'), (None, 'KeywordChecks'), ('arches', 'ArchChecks'), ('depend', 'DependChecks'), ('use_flags', 'USEFlagChecks'), ('ruby', '
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/checks/ebuilds/variables/, ...
commit: 76e5b13788076339bbf816ba876602cf5fbf64d9 Author: Brian Dolbec gentoo org> AuthorDate: Sun Jan 3 23:09:27 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Sun Jan 10 20:15:08 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=76e5b137 repoman: Migrate license checks to a plugin module pym/repoman/checks/ebuilds/variables/license.py | 47 -- pym/repoman/modules/scan/metadata/__init__.py | 8 pym/repoman/modules/scan/metadata/license.py| 53 + pym/repoman/scanner.py | 7 +--- 4 files changed, 62 insertions(+), 53 deletions(-) diff --git a/pym/repoman/checks/ebuilds/variables/license.py b/pym/repoman/checks/ebuilds/variables/license.py deleted file mode 100644 index bdc859c..000 --- a/pym/repoman/checks/ebuilds/variables/license.py +++ /dev/null @@ -1,47 +0,0 @@ - -'''description.py -Perform checks on the LICENSE variable. -''' - -# import our initialized portage instance -from repoman._portage import portage - - -class LicenseChecks(object): - '''Perform checks on the LICENSE variable.''' - - def __init__(self, qatracker, liclist, liclist_deprecated): - ''' - @param qatracker: QATracker instance - @param liclist: List of licenses. - @param liclist: List of deprecated licenses. - ''' - self.qatracker = qatracker - self.liclist = liclist - self.liclist_deprecated = liclist_deprecated - - def check( - self, pkg, package, ebuild, y_ebuild): - ''' - @param pkg: Package in which we check (object). - @param package: Package in which we check (string). - @param ebuild: Ebuild which we check (object). - @param y_ebuild: Ebuild which we check (string). - ''' - - # Parse the LICENSE variable, remove USE conditions and flatten it. - licenses = portage.dep.use_reduce( - pkg._metadata["LICENSE"], matchall=1, flat=True) - - # Check each entry to ensure that it exists in ${PORTDIR}/licenses/. - for lic in licenses: - # Need to check for "||" manually as no portage - # function will remove it without removing values. - if lic not in self.liclist and lic != "||": - self.qatracker.add_error( - "LICENSE.invalid", - package + "/" + y_ebuild + ".ebuild: %s" % lic) - elif lic in self.liclist_deprecated: - self.qatracker.add_error( - "LICENSE.deprecated", - "%s: %s" % (ebuild.relative_path, lic)) diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py index 2506521..ed0c59d 100644 --- a/pym/repoman/modules/scan/metadata/__init__.py +++ b/pym/repoman/modules/scan/metadata/__init__.py @@ -34,6 +34,14 @@ module_spec = { 'func_desc': { }, }, + 'license-metadata': { + 'name': "license", + 'class': "LicenseChecks", + 'description': doc, + 'functions': ['check'], + 'func_desc': { + }, + }, } } diff --git a/pym/repoman/modules/scan/metadata/license.py b/pym/repoman/modules/scan/metadata/license.py new file mode 100644 index 000..b022b20 --- /dev/null +++ b/pym/repoman/modules/scan/metadata/license.py @@ -0,0 +1,53 @@ + +'''license.py +Perform checks on the LICENSE variable. +''' + +# import our initialized portage instance +from repoman._portage import portage + + +class LicenseChecks(object): + '''Perform checks on the LICENSE variable.''' + + def __init__(self, **kwargs): + ''' + @param qatracker: QATracker instance + @param repo_metadata: dictionary of various repository items. + ''' + self.qatracker = kwargs.get('qatracker') + self.repo_metadata = kwargs.get('repo_metadata') + + def check(self, **kwargs): + ''' + @param xpkg: Package in which we check (string). + @param ebuild: Ebuild which we check (object). + @param y_ebuild: Ebuild which we check (string). + ''' + xpkg = kwargs.get('xpkg') + ebuild = kwargs.get('ebuild') + y_ebuild = kwargs.get('y_ebuild') + if not kwargs.get('badlicsyntax'): + # Parse the LICENSE variable, remove USE conditions and flatte
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/checks/ebuilds/variables/, ...
commit: 0187e2e6559e0d2e66b9c27c11228400024199a6 Author: Brian Dolbec gentoo org> AuthorDate: Sun Jan 3 23:10:48 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Wed Jan 6 04:08:23 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=0187e2e6 repoman: Migrate RestrictChecks to a plugin module pym/repoman/modules/scan/metadata/__init__.py | 8 ++ .../scan/metadata}/restrict.py | 29 +++--- pym/repoman/scanner.py | 9 +-- 3 files changed, 29 insertions(+), 17 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py index ed0c59d..6ab44f6 100644 --- a/pym/repoman/modules/scan/metadata/__init__.py +++ b/pym/repoman/modules/scan/metadata/__init__.py @@ -42,6 +42,14 @@ module_spec = { 'func_desc': { }, }, + 'restrict-metadata': { + 'name': "restrict", + 'class': "RestrictChecks", + 'description': doc, + 'functions': ['check'], + 'func_desc': { + }, + }, } } diff --git a/pym/repoman/checks/ebuilds/variables/restrict.py b/pym/repoman/modules/scan/metadata/restrict.py similarity index 54% rename from pym/repoman/checks/ebuilds/variables/restrict.py rename to pym/repoman/modules/scan/metadata/restrict.py index 215b792..93ca298 100644 --- a/pym/repoman/checks/ebuilds/variables/restrict.py +++ b/pym/repoman/modules/scan/metadata/restrict.py @@ -12,21 +12,23 @@ from repoman.qa_data import valid_restrict class RestrictChecks(object): '''Perform checks on the RESTRICT variable.''' - def __init__(self, qatracker): + def __init__(self, **kwargs): ''' @param qatracker: QATracker instance ''' - self.qatracker = qatracker + self.qatracker = kwargs.get('qatracker') - def check(self, pkg, package, ebuild, y_ebuild): + def check(self, **kwargs): + xpkg = kwargs.get('xpkg') + ebuild = kwargs.get('ebuild') + y_ebuild = kwargs.get('y_ebuild') myrestrict = None try: myrestrict = portage.dep.use_reduce( - pkg._metadata["RESTRICT"], matchall=1, flat=True) + ebuild.metadata["RESTRICT"], matchall=1, flat=True) except portage.exception.InvalidDependString as e: - self. qatracker.add_error( - "RESTRICT.syntax", + self.qatracker.add_error("RESTRICT.syntax", "%s: RESTRICT: %s" % (ebuild.relative_path, e)) del e @@ -36,6 +38,15 @@ class RestrictChecks(object): if mybadrestrict: for mybad in mybadrestrict: - self.qatracker.add_error( - "RESTRICT.invalid", - package + "/" + y_ebuild + ".ebuild: %s" % mybad) + self.qatracker.add_error("RESTRICT.invalid", + "%s/%s.ebuild: %s" % (xpkg, y_ebuild, mybad)) + return {'continue': False} + + @property + def runInPkgs(self): + return (False, []) + + @property + def runInEbuilds(self): + return (True, [self.check]) + diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py index ea1869f..457bdcb 100644 --- a/pym/repoman/scanner.py +++ b/pym/repoman/scanner.py @@ -18,7 +18,6 @@ from portage import _unicode_encode from portage.dep import Atom from portage.output import green from repoman.checks.ebuilds.checks import run_checks -from repoman.checks.ebuilds.variables.restrict import RestrictChecks from repoman.modules.commit import repochecks from repoman.profile import check_profiles, dev_profile_keywords, setup_profile from repoman.repos import repo_metadata @@ -210,10 +209,6 @@ class Scanner(object): print("Initializing class name:", mod_class.__name__) self.modules[mod_class.__name__] = mod_class(**self.kwargs) - # initialize our checks classes here before the big xpkg loop - self.restrictcheck = RestrictChecks(self.qatracker) - - def scan_pkgs(self, can_force): dynamic_data = {'can_force': can_force} for xpkg in self.effective_scanlist: @@ -299,7 +294,7 @@ class Scanner(object): ('description', 'DescriptionChecks'), (None, 'KeywordChecks'),
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/checks/ebuilds/variables/, ...
commit: 9fedd97887e6af9e54e02ea6ea10d6673a909dde Author: Brian Dolbec gentoo org> AuthorDate: Sun Jan 3 17:36:26 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Wed Jan 6 04:08:21 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=9fedd978 repoman: Migrate DescriptionChecks to the plugin system pym/repoman/modules/scan/metadata/__init__.py| 8 .../scan/metadata}/description.py| 20 ++-- pym/repoman/scanner.py | 5 + 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py index eba6565..2506521 100644 --- a/pym/repoman/modules/scan/metadata/__init__.py +++ b/pym/repoman/modules/scan/metadata/__init__.py @@ -26,6 +26,14 @@ module_spec = { 'func_desc': { }, }, + 'description-metadata': { + 'name': "description", + 'class': "DescriptionChecks", + 'description': doc, + 'functions': ['check'], + 'func_desc': { + }, + }, } } diff --git a/pym/repoman/checks/ebuilds/variables/description.py b/pym/repoman/modules/scan/metadata/description.py similarity index 66% rename from pym/repoman/checks/ebuilds/variables/description.py rename to pym/repoman/modules/scan/metadata/description.py index a2b1057..3570607 100644 --- a/pym/repoman/checks/ebuilds/variables/description.py +++ b/pym/repoman/modules/scan/metadata/description.py @@ -9,20 +9,19 @@ from repoman.qa_data import max_desc_len class DescriptionChecks(object): '''Perform checks on the DESCRIPTION variable.''' - def __init__(self, qatracker): + def __init__(self, **kwargs): ''' @param qatracker: QATracker instance ''' - self.qatracker = qatracker + self.qatracker = kwargs.get('qatracker') - def check(self, pkg, ebuild): + def checkTooLong(self, **kwargs): ''' @param pkg: Package in which we check (object). @param ebuild: Ebuild which we check (object). ''' - self._checkTooLong(pkg, ebuild) - - def _checkTooLong(self, pkg, ebuild): + ebuild = kwargs.get('ebuild') + pkg = kwargs.get('pkg') # 14 is the length of DESCRIPTION="" if len(pkg._metadata['DESCRIPTION']) > max_desc_len: self.qatracker.add_error( @@ -30,3 +29,12 @@ class DescriptionChecks(object): "%s: DESCRIPTION is %d characters (max %d)" % (ebuild.relative_path, len( pkg._metadata['DESCRIPTION']), max_desc_len)) + return {'continue': False} + + @property + def runInPkgs(self): + return (False, []) + + @property + def runInEbuilds(self): + return (True, [self.checkTooLong]) diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py index 1f56c5c..295d35e 100644 --- a/pym/repoman/scanner.py +++ b/pym/repoman/scanner.py @@ -19,7 +19,6 @@ from portage.dep import Atom from portage.output import green from repoman.checks.ebuilds.checks import run_checks from repoman.check_missingslot import check_missingslot -from repoman.checks.ebuilds.variables.description import DescriptionChecks from repoman.checks.ebuilds.variables.license import LicenseChecks from repoman.checks.ebuilds.variables.restrict import RestrictChecks from repoman.modules.commit import repochecks @@ -214,7 +213,6 @@ class Scanner(object): self.modules[mod_class.__name__] = mod_class(**self.kwargs) # initialize our checks classes here before the big xpkg loop - self.descriptioncheck = DescriptionChecks(self.qatracker) self.licensecheck = LicenseChecks(self.qatracker, liclist, liclist_deprecated) self.restrictcheck = RestrictChecks(self.qatracker) @@ -301,6 +299,7 @@ class Scanner(object): for mod in [('ebuild', 'Ebuild'), ('live', 'LiveEclassChecks'), ('eapi', 'EAPIChecks'), ('ebuild_metadata', 'EbuildMetadata'), ('thirdpartymirrors', 'ThirdPartyMirrors'), + ('description', 'DescriptionChecks'), ]: mod_class = MODULE_CONTROLLER.get_class(mod[0]) print("Initializing class name:", mod_class.__name__) @@ -330,8 +329,6 @@ class Scanner(object): print(" finished plugin loop, continuing...")
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/modules/scan/eclasses/, ...
commit: 1521249252e81eb4eff9c37f498b51c52199592d Author: Brian Dolbec gentoo org> AuthorDate: Thu Apr 28 05:11:57 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Thu Apr 28 05:11:57 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=15212492 repoman: Move the LiveEclasses.is_live() to the Ebuild class This removes another dynaic_data variable. It also belongs with the ebuild eclass as a primary source for that type of data. pym/repoman/modules/scan/ebuild/ebuild.py | 9 + pym/repoman/modules/scan/eclasses/__init__.py | 1 - pym/repoman/modules/scan/eclasses/live.py | 22 +++--- pym/repoman/modules/scan/keywords/__init__.py | 1 - pym/repoman/modules/scan/keywords/keywords.py | 8 +++- pym/repoman/modules/scan/metadata/__init__.py | 1 - .../modules/scan/metadata/ebuild_metadata.py | 3 +-- 7 files changed, 16 insertions(+), 29 deletions(-) diff --git a/pym/repoman/modules/scan/ebuild/ebuild.py b/pym/repoman/modules/scan/ebuild/ebuild.py index c247a7f..0277aa9 100644 --- a/pym/repoman/modules/scan/ebuild/ebuild.py +++ b/pym/repoman/modules/scan/ebuild/ebuild.py @@ -13,6 +13,7 @@ from repoman.qa_data import no_exec, allvars # import our initialized portage instance from repoman._portage import portage from portage import os +from portage.const import LIVE_ECLASSES pv_toolong_re = re.compile(r'[0-9]{19,}') @@ -220,6 +221,14 @@ class Ebuild(ScanBase): return self.continue_ @property + def is_live(self): + '''Test if the ebuild inherits a live eclass + + @returns: boolean + ''' + return set(LIVE_ECLASSES.intersection(self.inherited)) + + @property def runInPkgs(self): '''Package level scans''' return (True, [self.check_isebuild]) diff --git a/pym/repoman/modules/scan/eclasses/__init__.py b/pym/repoman/modules/scan/eclasses/__init__.py index 63bb86f..78d46e4 100644 --- a/pym/repoman/modules/scan/eclasses/__init__.py +++ b/pym/repoman/modules/scan/eclasses/__init__.py @@ -22,7 +22,6 @@ module_spec = { ], 'func_kwargs': { 'ebuild': (None, None), - 'live_ebuild': ('Future', 'UNSET'), 'pkg': (None, None), 'xpkg': (None, None), 'y_ebuild': (None, None), diff --git a/pym/repoman/modules/scan/eclasses/live.py b/pym/repoman/modules/scan/eclasses/live.py index 842cbab..1ce33c0 100644 --- a/pym/repoman/modules/scan/eclasses/live.py +++ b/pym/repoman/modules/scan/eclasses/live.py @@ -6,8 +6,6 @@ Performs Live eclass checks from repoman._portage import portage from repoman.modules.scan.scanbase import ScanBase -from portage.const import LIVE_ECLASSES - class LiveEclassChecks(ScanBase): '''Performs checks for the usage of Live eclasses in ebuilds''' @@ -20,27 +18,14 @@ class LiveEclassChecks(ScanBase): self.pmaskdict = kwargs.get('repo_metadata')['pmaskdict'] self.repo_settings = kwargs.get('repo_settings') - def is_live(self, **kwargs): - '''Test if the ebuild inherits a live eclass - - @returns: dictionary, including {live_ebuild} - ''' - ebuild = kwargs.get('ebuild').get() - # update the dynamic data - dyn_live = kwargs.get('live_ebuild') - dyn_live.set(LIVE_ECLASSES.intersection(ebuild.inherited)) - return False - def check(self, **kwargs): '''Ebuilds that inherit a "Live" eclass (darcs, subversion, git, cvs, etc..) should not be allowed to be marked stable @param pkg: Package in which we check (object). - @param package: Package in which we check (string). + @param xpkg: Package in which we check (string). @param ebuild: Ebuild which we check (object). @param y_ebuild: Ebuild which we check (string). - @param keywords: The keywords of the ebuild. - @param global_pmaskdict: A global dictionary of all the masks. @returns: dictionary ''' pkg = kwargs.get("pkg").result() @@ -48,9 +33,8 @@ class LiveEclassChecks(ScanBase): ebuild = kwargs.get('ebuild').get() y_ebuild = kwargs.get('y_ebuild') keywords = ebuild.keywords - live_ebuild = kwargs.get('live_ebuild').get() - if not live_ebuild and self.repo_settings.repo_config.name == "gentoo": + if not ebuild.is_live and self.repo_settings.repo_config.name == "gentoo": return False is_stable = lambda kw: not kw.sta
[gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/metadata/, pym/repoman/modules/scan/keywords/, ...
commit: c65d9bc746b4c6774eb71332ab851ec3dc6f3675 Author: Brian Dolbec gentoo org> AuthorDate: Thu Apr 28 05:11:57 2016 + Commit: Brian Dolbec gentoo org> CommitDate: Thu Apr 28 14:50:42 2016 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c65d9bc7 repoman: Move the LiveEclasses.is_live() to the Ebuild class This removes another dynaic_data variable. It also belongs with the ebuild eclass as a primary source for that type of data. pym/repoman/modules/scan/ebuild/ebuild.py | 9 + pym/repoman/modules/scan/eclasses/__init__.py | 1 - pym/repoman/modules/scan/eclasses/live.py | 22 +++--- pym/repoman/modules/scan/keywords/__init__.py | 1 - pym/repoman/modules/scan/keywords/keywords.py | 8 +++- pym/repoman/modules/scan/metadata/__init__.py | 1 - .../modules/scan/metadata/ebuild_metadata.py | 3 +-- 7 files changed, 16 insertions(+), 29 deletions(-) diff --git a/pym/repoman/modules/scan/ebuild/ebuild.py b/pym/repoman/modules/scan/ebuild/ebuild.py index c247a7f..0277aa9 100644 --- a/pym/repoman/modules/scan/ebuild/ebuild.py +++ b/pym/repoman/modules/scan/ebuild/ebuild.py @@ -13,6 +13,7 @@ from repoman.qa_data import no_exec, allvars # import our initialized portage instance from repoman._portage import portage from portage import os +from portage.const import LIVE_ECLASSES pv_toolong_re = re.compile(r'[0-9]{19,}') @@ -220,6 +221,14 @@ class Ebuild(ScanBase): return self.continue_ @property + def is_live(self): + '''Test if the ebuild inherits a live eclass + + @returns: boolean + ''' + return set(LIVE_ECLASSES.intersection(self.inherited)) + + @property def runInPkgs(self): '''Package level scans''' return (True, [self.check_isebuild]) diff --git a/pym/repoman/modules/scan/eclasses/__init__.py b/pym/repoman/modules/scan/eclasses/__init__.py index 63bb86f..78d46e4 100644 --- a/pym/repoman/modules/scan/eclasses/__init__.py +++ b/pym/repoman/modules/scan/eclasses/__init__.py @@ -22,7 +22,6 @@ module_spec = { ], 'func_kwargs': { 'ebuild': (None, None), - 'live_ebuild': ('Future', 'UNSET'), 'pkg': (None, None), 'xpkg': (None, None), 'y_ebuild': (None, None), diff --git a/pym/repoman/modules/scan/eclasses/live.py b/pym/repoman/modules/scan/eclasses/live.py index 842cbab..1ce33c0 100644 --- a/pym/repoman/modules/scan/eclasses/live.py +++ b/pym/repoman/modules/scan/eclasses/live.py @@ -6,8 +6,6 @@ Performs Live eclass checks from repoman._portage import portage from repoman.modules.scan.scanbase import ScanBase -from portage.const import LIVE_ECLASSES - class LiveEclassChecks(ScanBase): '''Performs checks for the usage of Live eclasses in ebuilds''' @@ -20,27 +18,14 @@ class LiveEclassChecks(ScanBase): self.pmaskdict = kwargs.get('repo_metadata')['pmaskdict'] self.repo_settings = kwargs.get('repo_settings') - def is_live(self, **kwargs): - '''Test if the ebuild inherits a live eclass - - @returns: dictionary, including {live_ebuild} - ''' - ebuild = kwargs.get('ebuild').get() - # update the dynamic data - dyn_live = kwargs.get('live_ebuild') - dyn_live.set(LIVE_ECLASSES.intersection(ebuild.inherited)) - return False - def check(self, **kwargs): '''Ebuilds that inherit a "Live" eclass (darcs, subversion, git, cvs, etc..) should not be allowed to be marked stable @param pkg: Package in which we check (object). - @param package: Package in which we check (string). + @param xpkg: Package in which we check (string). @param ebuild: Ebuild which we check (object). @param y_ebuild: Ebuild which we check (string). - @param keywords: The keywords of the ebuild. - @param global_pmaskdict: A global dictionary of all the masks. @returns: dictionary ''' pkg = kwargs.get("pkg").result() @@ -48,9 +33,8 @@ class LiveEclassChecks(ScanBase): ebuild = kwargs.get('ebuild').get() y_ebuild = kwargs.get('y_ebuild') keywords = ebuild.keywords - live_ebuild = kwargs.get('live_ebuild').get() - if not live_ebuild and self.repo_settings.repo_config.name == "gentoo": + if not ebuild.is_live and self.repo_settings.repo_config.name == "gentoo": return False is_stable = lambda kw: not kw.sta