Hi,
I guess the change in pyflakes is intended, please see [1][2].
The FTBFS for src:python-apt 1.7.0 downloaded from shanpshot.d.o [3]
vanishes after rebuilding with this commit reverted [4] (attached
for saving your time and for double-checking purpose).
Kind Regards
[1] https://github.com/PyCQA/pyflakes/issues/747
[2] https://data.safetycli.com/packages/pypi/pyflakes/changelog
[3] https://snapshot.debian.org/package/python-apt/2.7.0/
[4] https://github.com/PyCQA/pyflakes/pull/684
--- a/pyflakes/api.py
+++ b/pyflakes/api.py
@@ -44,7 +44,8 @@
reporter.unexpectedError(filename, 'problem decoding source')
return 1
# Okay, it's syntactically valid. Now check it.
- w = checker.Checker(tree, filename=filename)
+ file_tokens = checker.make_tokens(codeString)
+ w = checker.Checker(tree, file_tokens=file_tokens, filename=filename)
w.messages.sort(key=lambda m: m.lineno)
for warning in w.messages:
reporter.flake(warning)
--- a/pyflakes/checker.py
+++ b/pyflakes/checker.py
@@ -15,6 +15,9 @@
import re
import string
import sys
+import bisect
+import collections
+import tokenize
import warnings
from pyflakes import messages
@@ -66,6 +69,14 @@
(isinstance(node, ast.Attribute) and node.attr == name)
)
+# https://github.com/python/typed_ast/blob/1.4.0/ast27/Parser/tokenizer.c#L102-L104
+TYPE_COMMENT_RE = re.compile(r'^#\s*type:\s*')
+# https://github.com/python/typed_ast/blob/1.4.0/ast27/Parser/tokenizer.c#L1408-L1413
+ASCII_NON_ALNUM = ''.join([chr(i) for i in range(128) if not chr(i).isalnum()])
+TYPE_IGNORE_RE = re.compile(
+ TYPE_COMMENT_RE.pattern + fr'ignore([{ASCII_NON_ALNUM}]|$)')
+# https://github.com/python/typed_ast/blob/1.4.0/ast27/Grammar/Grammar#L147
+TYPE_FUNC_RE = re.compile(r'^(\(.*?\))\s*->\s*(.*)$')
MAPPING_KEY_RE = re.compile(r'\(([^()]*)\)')
CONVERSION_FLAG_RE = re.compile('[#0+ -]*')
@@ -594,6 +605,11 @@
class DoctestScope(ModuleScope):
"""Scope for a doctest."""
+class DummyNode:
+ """Used in place of an `ast.AST` to set error message positions"""
+ def __init__(self, lineno, col_offset):
+ self.lineno = lineno
+ self.col_offset = col_offset
class DetectClassScopedMagic:
names = dir()
@@ -713,6 +729,60 @@
return func(self, *args, **kwargs)
return in_annotation_func
+def make_tokens(code):
+ # PY3: tokenize.tokenize requires readline of bytes
+ if not isinstance(code, bytes):
+ code = code.encode('UTF-8')
+ lines = iter(code.splitlines(True))
+ # next(lines, b'') is to prevent an error in pypy3
+ return tuple(tokenize.tokenize(lambda: next(lines, b'')))
+
+
+class _TypeableVisitor(ast.NodeVisitor):
+ """Collect the line number and nodes which are deemed typeable by
+ PEP 484
+ https://www.python.org/dev/peps/pep-0484/#type-comments
+ """
+ def __init__(self):
+ self.typeable_lines = []
+ self.typeable_nodes = {}
+
+ def _typeable(self, node):
+ # if there is more than one typeable thing on a line last one wins
+ self.typeable_lines.append(node.lineno)
+ self.typeable_nodes[node.lineno] = node
+
+ self.generic_visit(node)
+
+ visit_Assign = visit_For = visit_FunctionDef = visit_With = _typeable
+ visit_AsyncFor = visit_AsyncFunctionDef = visit_AsyncWith = _typeable
+
+
+def _collect_type_comments(tree, tokens):
+ visitor = _TypeableVisitor()
+ visitor.visit(tree)
+
+ type_comments = collections.defaultdict(list)
+ for tp, text, start, _, _ in tokens:
+ if (
+ tp != tokenize.COMMENT or # skip non comments
+ not TYPE_COMMENT_RE.match(text) or # skip non-type comments
+ TYPE_IGNORE_RE.match(text) # skip ignores
+ ):
+ continue
+
+ # search for the typeable node at or before the line number of the
+ # type comment.
+ # if the bisection insertion point is before any nodes this is an
+ # invalid type comment which is ignored.
+ lineno, _ = start
+ idx = bisect.bisect_right(visitor.typeable_lines, lineno)
+ if idx == 0:
+ continue
+ node = visitor.typeable_nodes[visitor.typeable_lines[idx - 1]]
+ type_comments[node].append((start, text))
+
+ return type_comments
class Checker:
"""I check the cleanliness and sanity of Python code."""
@@ -751,6 +821,7 @@
self.withDoctest = withDoctest
self.exceptHandlers = [()]
self.root = tree
+ self._type_comments = _collect_type_comments(tree, file_tokens)
self.scopeStack = []
try:
@@ -783,6 +854,25 @@
"""
self._deferred.append((callable, self.scopeStack[:], self.offset))
+ def _handle_type_comments(self, node):
+ for (lineno, col_offset), comment in self._type_comments.get(node, ()):
+ comment = comment.split(':', 1)[1].strip()
+ func_match = TYPE_FUNC_RE.match(comment)
+ if func_match:
+ parts = (
+ func_match.group(1).replace('*', ''),
+ func_match.group(2).strip(),
+ )
+ else:
+ parts = (comment,)
+
+ for part in parts:
+ self.deferFunction(functools.partial(
+ self.handleStringAnnotation,
+ part, DummyNode(lineno, col_offset), lineno, col_offset,
+ messages.CommentAnnotationSyntaxError,
+ ))
+
def _run_deferred(self):
orig = (self.scopeStack, self.offset)
@@ -1210,6 +1300,7 @@
)
def handleChildren(self, tree, omit=None):
+ self._handle_type_comments(tree)
for node in iter_child_nodes(tree, omit=omit):
self.handleNode(node, tree)
--- a/pyflakes/messages.py
+++ b/pyflakes/messages.py
@@ -240,6 +240,12 @@
Message.__init__(self, filename, loc)
self.message_args = (annotation,)
+class CommentAnnotationSyntaxError(Message):
+ message = 'syntax error in type comment %r'
+
+ def __init__(self, filename, loc, annotation):
+ Message.__init__(self, filename, loc)
+ self.message_args = (annotation,)
class RaiseNotImplemented(Message):
message = "'raise NotImplemented' should be 'raise NotImplementedError'"
