Author: Benjamin Peterson <benja...@python.org>
Branch: py3k
Changeset: r53610:5d2f6423dabb
Date: 2012-03-14 16:37 -0500
http://bitbucket.org/pypy/pypy/changeset/5d2f6423dabb/
Log: allow ... anywhere and make it its own token
diff --git a/pypy/interpreter/astcompiler/ast.py
b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -1637,6 +1637,25 @@
pass
+class Ellipsis(expr):
+
+ def __init__(self, lineno, col_offset):
+ expr.__init__(self, lineno, col_offset)
+ self.initialization_state = 3
+
+ def walkabout(self, visitor):
+ visitor.visit_Ellipsis(self)
+
+ def mutate_over(self, visitor):
+ return visitor.visit_Ellipsis(self)
+
+ def sync_app_attrs(self, space):
+ if (self.initialization_state & ~0) ^ 3:
+ self.missing_field(space, ['lineno', 'col_offset'], 'Ellipsis')
+ else:
+ pass
+
+
class Attribute(expr):
def __init__(self, value, attr, ctx, lineno, col_offset):
@@ -1874,24 +1893,6 @@
class slice(AST):
pass
-class Ellipsis(slice):
-
- def __init__(self):
- self.initialization_state = 0
-
- def walkabout(self, visitor):
- visitor.visit_Ellipsis(self)
-
- def mutate_over(self, visitor):
- return visitor.visit_Ellipsis(self)
-
- def sync_app_attrs(self, space):
- if (self.initialization_state & ~0) ^ 0:
- self.missing_field(space, [], 'Ellipsis')
- else:
- pass
-
-
class Slice(slice):
def __init__(self, lower, upper, step):
@@ -2562,6 +2563,8 @@
return self.default_visitor(node)
def visit_Bytes(self, node):
return self.default_visitor(node)
+ def visit_Ellipsis(self, node):
+ return self.default_visitor(node)
def visit_Attribute(self, node):
return self.default_visitor(node)
def visit_Subscript(self, node):
@@ -2576,8 +2579,6 @@
return self.default_visitor(node)
def visit_Const(self, node):
return self.default_visitor(node)
- def visit_Ellipsis(self, node):
- return self.default_visitor(node)
def visit_Slice(self, node):
return self.default_visitor(node)
def visit_ExtSlice(self, node):
@@ -2778,6 +2779,9 @@
def visit_Bytes(self, node):
pass
+ def visit_Ellipsis(self, node):
+ pass
+
def visit_Attribute(self, node):
node.value.walkabout(self)
@@ -2800,9 +2804,6 @@
def visit_Const(self, node):
pass
- def visit_Ellipsis(self, node):
- pass
-
def visit_Slice(self, node):
if node.lower:
node.lower.walkabout(self)
@@ -5830,6 +5831,23 @@
__init__=interp2app(Bytes_init),
)
+def Ellipsis_init(space, w_self, __args__):
+ w_self = space.descr_self_interp_w(Ellipsis, w_self)
+ args_w, kwargs_w = __args__.unpack()
+ if args_w:
+ w_err = space.wrap("Ellipsis constructor takes no arguments")
+ raise OperationError(space.w_TypeError, w_err)
+ for field, w_value in kwargs_w.iteritems():
+ space.setattr(w_self, space.wrap(field), w_value)
+
+Ellipsis.typedef = typedef.TypeDef("Ellipsis",
+ expr.typedef,
+ __module__='_ast',
+ _fields=_FieldsWrapper([]),
+ __new__=interp2app(get_AST_new(Ellipsis)),
+ __init__=interp2app(Ellipsis_init),
+)
+
def Attribute_get_value(space, w_self):
if w_self.w_dict is not None:
w_obj = w_self.getdictvalue(space, 'value')
@@ -6391,23 +6409,6 @@
__new__=interp2app(get_AST_new(slice)),
)
-def Ellipsis_init(space, w_self, __args__):
- w_self = space.descr_self_interp_w(Ellipsis, w_self)
- args_w, kwargs_w = __args__.unpack()
- if args_w:
- w_err = space.wrap("Ellipsis constructor takes no arguments")
- raise OperationError(space.w_TypeError, w_err)
- for field, w_value in kwargs_w.iteritems():
- space.setattr(w_self, space.wrap(field), w_value)
-
-Ellipsis.typedef = typedef.TypeDef("Ellipsis",
- slice.typedef,
- __module__='_ast',
- _fields=_FieldsWrapper([]),
- __new__=interp2app(get_AST_new(Ellipsis)),
- __init__=interp2app(Ellipsis_init),
-)
-
def Slice_get_lower(space, w_self):
if w_self.w_dict is not None:
w_obj = w_self.getdictvalue(space, 'lower')
diff --git a/pypy/interpreter/astcompiler/astbuilder.py
b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -211,11 +211,15 @@
dot_count = 0
while i < child_count:
child = import_node.children[i]
- if child.type == syms.dotted_name:
+ child_type = child.type
+ if child_type == syms.dotted_name:
module = self.alias_for_import_name(child, False)
i += 1
break
- elif child.type != tokens.DOT:
+ elif child_type == tokens.ELLIPSIS:
+ # Special case for tokenization.
+ dot_count += 2
+ elif child_type != tokens.DOT:
break
i += 1
dot_count += 1
@@ -923,8 +927,6 @@
def handle_slice(self, slice_node):
first_child = slice_node.children[0]
- if first_child.type == tokens.DOT:
- return ast.Ellipsis()
if len(slice_node.children) == 1 and first_child.type == syms.test:
index = self.handle_expr(first_child)
return ast.Index(index)
@@ -1138,6 +1140,8 @@
elif first_child_type == tokens.NUMBER:
num_value = self.parse_number(first_child.value)
return ast.Num(num_value, atom_node.lineno, atom_node.column)
+ elif first_child_type == tokens.ELLIPSIS:
+ return ast.Ellipsis(atom_node.lineno, atom_node.column)
elif first_child_type == tokens.LPAR:
second_child = atom_node.children[1]
if second_child.type == tokens.RPAR:
diff --git a/pypy/interpreter/astcompiler/asthelpers.py
b/pypy/interpreter/astcompiler/asthelpers.py
--- a/pypy/interpreter/astcompiler/asthelpers.py
+++ b/pypy/interpreter/astcompiler/asthelpers.py
@@ -168,3 +168,9 @@
class __extend__(ast.Num):
constant = True
+
+
+class __extend__(ast.Ellipsis):
+
+ _description = "Ellipsis"
+ constant = True
diff --git a/pypy/interpreter/astcompiler/codegen.py
b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -825,6 +825,9 @@
space = self.space
self.load_const(const.value)
+ def visit_Ellipsis(self, e):
+ self.load_const(self.space.w_Ellipsis)
+
def visit_UnaryOp(self, op):
self.update_position(op.lineno)
op.operand.walkabout(self)
@@ -1126,9 +1129,7 @@
self.emit_op_arg(ops.BUILD_SLICE, arg)
def _nested_slice(self, slc, ctx):
- if isinstance(slc, ast.Ellipsis):
- self.load_const(self.space.w_Ellipsis)
- elif isinstance(slc, ast.Slice):
+ if isinstance(slc, ast.Slice):
self._complex_slice(slc, ctx)
elif isinstance(slc, ast.Index):
slc.value.walkabout(self)
@@ -1140,10 +1141,6 @@
kind = "index"
if ctx != ast.AugStore:
slc.value.walkabout(self)
- elif isinstance(slc, ast.Ellipsis):
- kind = "ellipsis"
- if ctx != ast.AugStore:
- self.load_const(self.space.w_Ellipsis)
elif isinstance(slc, ast.Slice):
kind = "slice"
if ctx != ast.AugStore:
diff --git a/pypy/interpreter/astcompiler/optimize.py
b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -62,6 +62,12 @@
return self.s
+class __extend__(ast.Ellipsis):
+
+ def as_constant_truth(self, space):
+ return True
+
+
class __extend__(ast.Const):
def as_constant(self):
diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py
b/pypy/interpreter/astcompiler/test/test_astbuilder.py
--- a/pypy/interpreter/astcompiler/test/test_astbuilder.py
+++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py
@@ -765,6 +765,7 @@
("{1, 2, 3}", "literal"),
("(x > 4)", "comparison"),
("(x if y else a)", "conditional expression"),
+ ("...", "Ellipsis"),
)
test_contexts = (
("assign to", "%s = 23"),
@@ -1076,8 +1077,6 @@
slc = self.get_first_expr("x[1:2:3]").slice
for field in (slc.lower, slc.upper, slc.step):
assert isinstance(field, ast.Num)
- sub = self.get_first_expr("x[...]")
- assert isinstance(sub.slice, ast.Ellipsis)
sub = self.get_first_expr("x[1,2,3]")
slc = sub.slice
assert isinstance(slc, ast.Index)
@@ -1093,6 +1092,12 @@
assert isinstance(complex_slc.upper, ast.Num)
assert complex_slc.step is None
+ def test_ellipsis(self):
+ e = self.get_first_expr("...")
+ assert isinstance(e, ast.Ellipsis)
+ sub = self.get_first_expr("x[...]")
+ assert isinstance(sub.slice.value, ast.Ellipsis)
+
def test_string(self):
space = self.space
s = self.get_first_expr("'hi'")
diff --git a/pypy/interpreter/astcompiler/tools/Python.asdl
b/pypy/interpreter/astcompiler/tools/Python.asdl
--- a/pypy/interpreter/astcompiler/tools/Python.asdl
+++ b/pypy/interpreter/astcompiler/tools/Python.asdl
@@ -69,6 +69,7 @@
| Num(object n) -- a number as a PyObject.
| Str(string s) -- need to specify raw, unicode, etc?
| Bytes(string s)
+ | Ellipsis
-- other literals? bools?
-- the following expression can appear in assignment context
@@ -87,7 +88,7 @@
expr_context = Load | Store | Del | AugLoad | AugStore | Param
- slice = Ellipsis | Slice(expr? lower, expr? upper, expr? step)
+ slice = Slice(expr? lower, expr? upper, expr? step)
| ExtSlice(slice* dims)
| Index(expr value)
diff --git a/pypy/interpreter/pyparser/data/Grammar3.2
b/pypy/interpreter/pyparser/data/Grammar3.2
--- a/pypy/interpreter/pyparser/data/Grammar3.2
+++ b/pypy/interpreter/pyparser/data/Grammar3.2
@@ -53,7 +53,8 @@
raise_stmt: 'raise' [test ['from' test]]
import_stmt: import_name | import_from
import_name: 'import' dotted_as_names
-import_from: ('from' ('.'* dotted_name | '.'+)
+# note below: the ('.' | '...') is necessary because '...' is tokenized as
ELLIPSIS
+import_from: ('from' (('.' | '...')* dotted_name | ('.' | '...')+)
'import' ('*' | '(' import_as_names ')' | import_as_names))
import_as_name: NAME ['as' NAME]
dotted_as_name: dotted_name ['as' NAME]
@@ -106,12 +107,12 @@
atom: ('(' [yield_expr|testlist_comp] ')' |
'[' [testlist_comp] ']' |
'{' [dictorsetmaker] '}' |
- NAME | NUMBER | STRING+)
+ NAME | NUMBER | STRING+ | '...')
testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )
lambdef: 'lambda' [varargslist] ':' test
trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
subscriptlist: subscript (',' subscript)* [',']
-subscript: '.' '.' '.' | test | [test] ':' [test] [sliceop]
+subscript: test | [test] ':' [test] [sliceop]
sliceop: ':' [test]
exprlist: expr (',' expr)* [',']
testlist: test (',' test)* [',']
diff --git a/pypy/interpreter/pyparser/dfa_generated.py
b/pypy/interpreter/pyparser/dfa_generated.py
--- a/pypy/interpreter/pyparser/dfa_generated.py
+++ b/pypy/interpreter/pyparser/dfa_generated.py
@@ -7,10 +7,10 @@
accepts = [True, True, True, True, True, True, True, True,
True, True, False, True, True, True, True, True,
False, False, False, True, False, False, False,
- True, False, True, False, True, False, True,
- False, False, True, False, False, True, True,
- True, False, False, True, False, False, False,
- True]
+ True, False, True, False, True, False, False,
+ True, False, False, True, False, False, True,
+ True, True, False, False, True, False, False,
+ False, True]
states = [
# 0
{'\t': 0, '\n': 14, '\x0c': 0,
@@ -104,9 +104,9 @@
'7': 5, '8': 5, '9': 5, 'E': 26,
'J': 14, 'e': 26, 'j': 14},
# 6
- {'0': 27, '1': 27, '2': 27, '3': 27,
- '4': 27, '5': 27, '6': 27, '7': 27,
- '8': 27, '9': 27},
+ {'.': 28, '0': 27, '1': 27, '2': 27,
+ '3': 27, '4': 27, '5': 27, '6': 27,
+ '7': 27, '8': 27, '9': 27},
# 7
{'*': 13, '=': 14},
# 8
@@ -126,27 +126,27 @@
# 15
{'\n': 14},
# 16
- {automata.DEFAULT: 31, '\n': 28,
- '\r': 28, "'": 29, '\\': 30},
+ {automata.DEFAULT: 32, '\n': 29,
+ '\r': 29, "'": 30, '\\': 31},
# 17
- {automata.DEFAULT: 34, '\n': 28,
- '\r': 28, '"': 32, '\\': 33},
+ {automata.DEFAULT: 35, '\n': 29,
+ '\r': 29, '"': 33, '\\': 34},
# 18
{'\n': 14, '\r': 15},
# 19
- {automata.DEFAULT: 19, '\n': 28, '\r': 28},
+ {automata.DEFAULT: 19, '\n': 29, '\r': 29},
# 20
- {'0': 35, '1': 35, '2': 35, '3': 35,
- '4': 35, '5': 35, '6': 35, '7': 35,
- '8': 35, '9': 35, 'A': 35, 'B': 35,
- 'C': 35, 'D': 35, 'E': 35, 'F': 35,
- 'a': 35, 'b': 35, 'c': 35, 'd': 35,
- 'e': 35, 'f': 35},
+ {'0': 36, '1': 36, '2': 36, '3': 36,
+ '4': 36, '5': 36, '6': 36, '7': 36,
+ '8': 36, '9': 36, 'A': 36, 'B': 36,
+ 'C': 36, 'D': 36, 'E': 36, 'F': 36,
+ 'a': 36, 'b': 36, 'c': 36, 'd': 36,
+ 'e': 36, 'f': 36},
# 21
- {'0': 36, '1': 36, '2': 36, '3': 36,
- '4': 36, '5': 36, '6': 36, '7': 36},
+ {'0': 37, '1': 37, '2': 37, '3': 37,
+ '4': 37, '5': 37, '6': 37, '7': 37},
# 22
- {'0': 37, '1': 37},
+ {'0': 38, '1': 38},
# 23
{'.': 25, '0': 23, '1': 24, '2': 24,
'3': 24, '4': 24, '5': 24, '6': 24,
@@ -160,71 +160,73 @@
# 25
{'0': 25, '1': 25, '2': 25, '3': 25,
'4': 25, '5': 25, '6': 25, '7': 25,
- '8': 25, '9': 25, 'E': 38, 'J': 14,
- 'e': 38, 'j': 14},
+ '8': 25, '9': 25, 'E': 39, 'J': 14,
+ 'e': 39, 'j': 14},
# 26
- {'+': 39, '-': 39, '0': 40, '1': 40,
- '2': 40, '3': 40, '4': 40, '5': 40,
- '6': 40, '7': 40, '8': 40, '9': 40},
+ {'+': 40, '-': 40, '0': 41, '1': 41,
+ '2': 41, '3': 41, '4': 41, '5': 41,
+ '6': 41, '7': 41, '8': 41, '9': 41},
# 27
{'0': 27, '1': 27, '2': 27, '3': 27,
'4': 27, '5': 27, '6': 27, '7': 27,
- '8': 27, '9': 27, 'E': 38, 'J': 14,
- 'e': 38, 'j': 14},
+ '8': 27, '9': 27, 'E': 39, 'J': 14,
+ 'e': 39, 'j': 14},
# 28
+ {'.': 14},
+ # 29
{},
- # 29
+ # 30
{"'": 14},
- # 30
- {automata.DEFAULT: 41, '\n': 14, '\r': 15},
# 31
- {automata.DEFAULT: 31, '\n': 28,
- '\r': 28, "'": 14, '\\': 30},
+ {automata.DEFAULT: 42, '\n': 14, '\r': 15},
# 32
+ {automata.DEFAULT: 32, '\n': 29,
+ '\r': 29, "'": 14, '\\': 31},
+ # 33
{'"': 14},
- # 33
- {automata.DEFAULT: 42, '\n': 14, '\r': 15},
# 34
- {automata.DEFAULT: 34, '\n': 28,
- '\r': 28, '"': 14, '\\': 33},
+ {automata.DEFAULT: 43, '\n': 14, '\r': 15},
# 35
- {'0': 35, '1': 35, '2': 35, '3': 35,
- '4': 35, '5': 35, '6': 35, '7': 35,
- '8': 35, '9': 35, 'A': 35, 'B': 35,
- 'C': 35, 'D': 35, 'E': 35, 'F': 35,
- 'a': 35, 'b': 35, 'c': 35, 'd': 35,
- 'e': 35, 'f': 35},
+ {automata.DEFAULT: 35, '\n': 29,
+ '\r': 29, '"': 14, '\\': 34},
# 36
{'0': 36, '1': 36, '2': 36, '3': 36,
- '4': 36, '5': 36, '6': 36, '7': 36},
+ '4': 36, '5': 36, '6': 36, '7': 36,
+ '8': 36, '9': 36, 'A': 36, 'B': 36,
+ 'C': 36, 'D': 36, 'E': 36, 'F': 36,
+ 'a': 36, 'b': 36, 'c': 36, 'd': 36,
+ 'e': 36, 'f': 36},
# 37
- {'0': 37, '1': 37},
+ {'0': 37, '1': 37, '2': 37, '3': 37,
+ '4': 37, '5': 37, '6': 37, '7': 37},
# 38
- {'+': 43, '-': 43, '0': 44, '1': 44,
- '2': 44, '3': 44, '4': 44, '5': 44,
- '6': 44, '7': 44, '8': 44, '9': 44},
+ {'0': 38, '1': 38},
# 39
- {'0': 40, '1': 40, '2': 40, '3': 40,
- '4': 40, '5': 40, '6': 40, '7': 40,
- '8': 40, '9': 40},
+ {'+': 44, '-': 44, '0': 45, '1': 45,
+ '2': 45, '3': 45, '4': 45, '5': 45,
+ '6': 45, '7': 45, '8': 45, '9': 45},
# 40
- {'0': 40, '1': 40, '2': 40, '3': 40,
- '4': 40, '5': 40, '6': 40, '7': 40,
- '8': 40, '9': 40, 'J': 14, 'j': 14},
+ {'0': 41, '1': 41, '2': 41, '3': 41,
+ '4': 41, '5': 41, '6': 41, '7': 41,
+ '8': 41, '9': 41},
# 41
- {automata.DEFAULT: 41, '\n': 28,
- '\r': 28, "'": 14, '\\': 30},
+ {'0': 41, '1': 41, '2': 41, '3': 41,
+ '4': 41, '5': 41, '6': 41, '7': 41,
+ '8': 41, '9': 41, 'J': 14, 'j': 14},
# 42
- {automata.DEFAULT: 42, '\n': 28,
- '\r': 28, '"': 14, '\\': 33},
+ {automata.DEFAULT: 42, '\n': 29,
+ '\r': 29, "'": 14, '\\': 31},
# 43
- {'0': 44, '1': 44, '2': 44, '3': 44,
- '4': 44, '5': 44, '6': 44, '7': 44,
- '8': 44, '9': 44},
+ {automata.DEFAULT: 43, '\n': 29,
+ '\r': 29, '"': 14, '\\': 34},
# 44
- {'0': 44, '1': 44, '2': 44, '3': 44,
- '4': 44, '5': 44, '6': 44, '7': 44,
- '8': 44, '9': 44, 'J': 14, 'j': 14},
+ {'0': 45, '1': 45, '2': 45, '3': 45,
+ '4': 45, '5': 45, '6': 45, '7': 45,
+ '8': 45, '9': 45},
+ # 45
+ {'0': 45, '1': 45, '2': 45, '3': 45,
+ '4': 45, '5': 45, '6': 45, '7': 45,
+ '8': 45, '9': 45, 'J': 14, 'j': 14},
]
pseudoDFA = automata.DFA(states, accepts)
diff --git a/pypy/interpreter/pyparser/gendfa.py
b/pypy/interpreter/pyparser/gendfa.py
--- a/pypy/interpreter/pyparser/gendfa.py
+++ b/pypy/interpreter/pyparser/gendfa.py
@@ -142,6 +142,7 @@
bracket = groupStr(states, "[](){}")
special = group(states,
makeEOL(),
+ chainStr(states, "..."),
groupStr(states, "@:;.,`"))
funny = group(states, operator, bracket, special)
# ____________________________________________________________
diff --git a/pypy/interpreter/pyparser/pytoken.py
b/pypy/interpreter/pyparser/pytoken.py
--- a/pypy/interpreter/pyparser/pytoken.py
+++ b/pypy/interpreter/pyparser/pytoken.py
@@ -62,6 +62,7 @@
_add_tok('DOUBLESLASHEQUAL',"//=" )
_add_tok('AT', "@" )
_add_tok('RARROW', "->")
+_add_tok('ELLIPSIS', "...")
_add_tok('OP')
_add_tok('ERRORTOKEN')
diff --git a/pypy/interpreter/pyparser/pytokenizer.py
b/pypy/interpreter/pyparser/pytokenizer.py
--- a/pypy/interpreter/pyparser/pytokenizer.py
+++ b/pypy/interpreter/pyparser/pytokenizer.py
@@ -172,8 +172,9 @@
pos = end
token, initial = line[start:end], line[start]
- if initial in numchars or \
- (initial == '.' and token != '.'): # ordinary number
+ if (initial in numchars or \
+ (initial == '.' and token != '.' and token != '...')):
+ # ordinary number
token_list.append((tokens.NUMBER, token, lnum, start,
line))
last_comment = ''
elif initial in '\r\n':
diff --git a/pypy/interpreter/test/test_compiler.py
b/pypy/interpreter/test/test_compiler.py
--- a/pypy/interpreter/test/test_compiler.py
+++ b/pypy/interpreter/test/test_compiler.py
@@ -449,6 +449,9 @@
space = self.space
w_d = space.newdict()
space.exec_(code, w_d, w_d)
+ snip = "d[. . .]"
+ space.raises_w(space.w_SyntaxError, self.compiler.compile,
+ snip, '<test>', 'exec', 0)
def test_chained_access_augassign(self):
snippet = str(py.code.Source(r'''
@@ -757,6 +760,12 @@
assert math.copysign(1., ns['c'][0]) == -1.0
assert math.copysign(1., ns['c'][1]) == -1.0
+ def test_ellipsis_anywhere(self):
+ """
+ x = ...
+ assert x is Ellipsis
+ """
+
class AppTestOptimizer:
@@ -793,6 +802,10 @@
assert isinstance(ns["x"][0], int)
assert isinstance(ns["y"][0], int)
+ def test_ellipsis_truth(self):
+ co = compile("if ...: x + 3\nelse: x + 4", "<test>", "exec")
+ assert 4 not in co.co_consts
+
def test_division_folding(self):
def code(source):
return compile(source, "<test>", "exec")
_______________________________________________
pypy-commit mailing list
pypy-commit@python.org
http://mail.python.org/mailman/listinfo/pypy-commit
