http://git-wip-us.apache.org/repos/asf/ambari/blob/570de228/ambari-common/src/main/python/ambari_jinja2/ambari_jinja2/nodes.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/ambari_jinja2/ambari_jinja2/nodes.py b/ambari-common/src/main/python/ambari_jinja2/ambari_jinja2/nodes.py deleted file mode 100644 index 804070c..0000000 --- a/ambari-common/src/main/python/ambari_jinja2/ambari_jinja2/nodes.py +++ /dev/null @@ -1,901 +0,0 @@ -# -*- coding: utf-8 -*- -""" - ambari_jinja2.nodes - ~~~~~~~~~~~~ - - This module implements additional nodes derived from the ast base node. - - It also provides some node tree helper functions like `in_lineno` and - `get_nodes` used by the parser and translator in order to normalize - python and jinja nodes. - - :copyright: (c) 2010 by the Jinja Team. - :license: BSD, see LICENSE for more details. -""" -import operator -from itertools import chain, izip -from collections import deque -from ambari_jinja2.utils import Markup, MethodType, FunctionType - - -#: the types we support for context functions -_context_function_types = (FunctionType, MethodType) - - -_binop_to_func = { - '*': operator.mul, - '/': operator.truediv, - '//': operator.floordiv, - '**': operator.pow, - '%': operator.mod, - '+': operator.add, - '-': operator.sub -} - -_uaop_to_func = { - 'not': operator.not_, - '+': operator.pos, - '-': operator.neg -} - -_cmpop_to_func = { - 'eq': operator.eq, - 'ne': operator.ne, - 'gt': operator.gt, - 'gteq': operator.ge, - 'lt': operator.lt, - 'lteq': operator.le, - 'in': lambda a, b: a in b, - 'notin': lambda a, b: a not in b -} - - -class Impossible(Exception): - """Raised if the node could not perform a requested action.""" - - -class NodeType(type): - """A metaclass for nodes that handles the field and attribute - inheritance. fields and attributes from the parent class are - automatically forwarded to the child.""" - - def __new__(cls, name, bases, d): - for attr in 'fields', 'attributes': - storage = [] - storage.extend(getattr(bases[0], attr, ())) - storage.extend(d.get(attr, ())) - assert len(bases) == 1, 'multiple inheritance not allowed' - assert len(storage) == len(set(storage)), 'layout conflict' - d[attr] = tuple(storage) - d.setdefault('abstract', False) - return type.__new__(cls, name, bases, d) - - -class EvalContext(object): - """Holds evaluation time information. Custom attributes can be attached - to it in extensions. - """ - - def __init__(self, environment, template_name=None): - if callable(environment.autoescape): - self.autoescape = environment.autoescape(template_name) - else: - self.autoescape = environment.autoescape - self.volatile = False - - def save(self): - return self.__dict__.copy() - - def revert(self, old): - self.__dict__.clear() - self.__dict__.update(old) - - -def get_eval_context(node, ctx): - if ctx is None: - if node.environment is None: - raise RuntimeError('if no eval context is passed, the ' - 'node must have an attached ' - 'environment.') - return EvalContext(node.environment) - return ctx - - -class Node(object): - """Baseclass for all Jinja2 nodes. There are a number of nodes available - of different types. There are three major types: - - - :class:`Stmt`: statements - - :class:`Expr`: expressions - - :class:`Helper`: helper nodes - - :class:`Template`: the outermost wrapper node - - All nodes have fields and attributes. Fields may be other nodes, lists, - or arbitrary values. Fields are passed to the constructor as regular - positional arguments, attributes as keyword arguments. Each node has - two attributes: `lineno` (the line number of the node) and `environment`. - The `environment` attribute is set at the end of the parsing process for - all nodes automatically. - """ - __metaclass__ = NodeType - fields = () - attributes = ('lineno', 'environment') - abstract = True - - def __init__(self, *fields, **attributes): - if self.abstract: - raise TypeError('abstract nodes are not instanciable') - if fields: - if len(fields) != len(self.fields): - if not self.fields: - raise TypeError('%r takes 0 arguments' % - self.__class__.__name__) - raise TypeError('%r takes 0 or %d argument%s' % ( - self.__class__.__name__, - len(self.fields), - len(self.fields) != 1 and 's' or '' - )) - for name, arg in izip(self.fields, fields): - setattr(self, name, arg) - for attr in self.attributes: - setattr(self, attr, attributes.pop(attr, None)) - if attributes: - raise TypeError('unknown attribute %r' % - iter(attributes).next()) - - def iter_fields(self, exclude=None, only=None): - """This method iterates over all fields that are defined and yields - ``(key, value)`` tuples. Per default all fields are returned, but - it's possible to limit that to some fields by providing the `only` - parameter or to exclude some using the `exclude` parameter. Both - should be sets or tuples of field names. - """ - for name in self.fields: - if (exclude is only is None) or \ - (exclude is not None and name not in exclude) or \ - (only is not None and name in only): - try: - yield name, getattr(self, name) - except AttributeError: - pass - - def iter_child_nodes(self, exclude=None, only=None): - """Iterates over all direct child nodes of the node. This iterates - over all fields and yields the values of they are nodes. If the value - of a field is a list all the nodes in that list are returned. - """ - for field, item in self.iter_fields(exclude, only): - if isinstance(item, list): - for n in item: - if isinstance(n, Node): - yield n - elif isinstance(item, Node): - yield item - - def find(self, node_type): - """Find the first node of a given type. If no such node exists the - return value is `None`. - """ - for result in self.find_all(node_type): - return result - - def find_all(self, node_type): - """Find all the nodes of a given type. If the type is a tuple, - the check is performed for any of the tuple items. - """ - for child in self.iter_child_nodes(): - if isinstance(child, node_type): - yield child - for result in child.find_all(node_type): - yield result - - def set_ctx(self, ctx): - """Reset the context of a node and all child nodes. Per default the - parser will all generate nodes that have a 'load' context as it's the - most common one. This method is used in the parser to set assignment - targets and other nodes to a store context. - """ - todo = deque([self]) - while todo: - node = todo.popleft() - if 'ctx' in node.fields: - node.ctx = ctx - todo.extend(node.iter_child_nodes()) - return self - - def set_lineno(self, lineno, override=False): - """Set the line numbers of the node and children.""" - todo = deque([self]) - while todo: - node = todo.popleft() - if 'lineno' in node.attributes: - if node.lineno is None or override: - node.lineno = lineno - todo.extend(node.iter_child_nodes()) - return self - - def set_environment(self, environment): - """Set the environment for all nodes.""" - todo = deque([self]) - while todo: - node = todo.popleft() - node.environment = environment - todo.extend(node.iter_child_nodes()) - return self - - def __eq__(self, other): - return type(self) is type(other) and \ - tuple(self.iter_fields()) == tuple(other.iter_fields()) - - def __ne__(self, other): - return not self.__eq__(other) - - def __repr__(self): - return '%s(%s)' % ( - self.__class__.__name__, - ', '.join('%s=%r' % (arg, getattr(self, arg, None)) for - arg in self.fields) - ) - - -class Stmt(Node): - """Base node for all statements.""" - abstract = True - - -class Helper(Node): - """Nodes that exist in a specific context only.""" - abstract = True - - -class Template(Node): - """Node that represents a template. This must be the outermost node that - is passed to the compiler. - """ - fields = ('body',) - - -class Output(Stmt): - """A node that holds multiple expressions which are then printed out. - This is used both for the `print` statement and the regular template data. - """ - fields = ('nodes',) - - -class Extends(Stmt): - """Represents an extends statement.""" - fields = ('template',) - - -class For(Stmt): - """The for loop. `target` is the target for the iteration (usually a - :class:`Name` or :class:`Tuple`), `iter` the iterable. `body` is a list - of nodes that are used as loop-body, and `else_` a list of nodes for the - `else` block. If no else node exists it has to be an empty list. - - For filtered nodes an expression can be stored as `test`, otherwise `None`. - """ - fields = ('target', 'iter', 'body', 'else_', 'test', 'recursive') - - -class If(Stmt): - """If `test` is true, `body` is rendered, else `else_`.""" - fields = ('test', 'body', 'else_') - - -class Macro(Stmt): - """A macro definition. `name` is the name of the macro, `args` a list of - arguments and `defaults` a list of defaults if there are any. `body` is - a list of nodes for the macro body. - """ - fields = ('name', 'args', 'defaults', 'body') - - -class CallBlock(Stmt): - """Like a macro without a name but a call instead. `call` is called with - the unnamed macro as `caller` argument this node holds. - """ - fields = ('call', 'args', 'defaults', 'body') - - -class FilterBlock(Stmt): - """Node for filter sections.""" - fields = ('body', 'filter') - - -class Block(Stmt): - """A node that represents a block.""" - fields = ('name', 'body', 'scoped') - - -class Include(Stmt): - """A node that represents the include tag.""" - fields = ('template', 'with_context', 'ignore_missing') - - -class Import(Stmt): - """A node that represents the import tag.""" - fields = ('template', 'target', 'with_context') - - -class FromImport(Stmt): - """A node that represents the from import tag. It's important to not - pass unsafe names to the name attribute. The compiler translates the - attribute lookups directly into getattr calls and does *not* use the - subscript callback of the interface. As exported variables may not - start with double underscores (which the parser asserts) this is not a - problem for regular Jinja code, but if this node is used in an extension - extra care must be taken. - - The list of names may contain tuples if aliases are wanted. - """ - fields = ('template', 'names', 'with_context') - - -class ExprStmt(Stmt): - """A statement that evaluates an expression and discards the result.""" - fields = ('node',) - - -class Assign(Stmt): - """Assigns an expression to a target.""" - fields = ('target', 'node') - - -class Expr(Node): - """Baseclass for all expressions.""" - abstract = True - - def as_const(self, eval_ctx=None): - """Return the value of the expression as constant or raise - :exc:`Impossible` if this was not possible. - - An :class:`EvalContext` can be provided, if none is given - a default context is created which requires the nodes to have - an attached environment. - - .. versionchanged:: 2.4 - the `eval_ctx` parameter was added. - """ - raise Impossible() - - def can_assign(self): - """Check if it's possible to assign something to this node.""" - return False - - -class BinExpr(Expr): - """Baseclass for all binary expressions.""" - fields = ('left', 'right') - operator = None - abstract = True - - def as_const(self, eval_ctx=None): - eval_ctx = get_eval_context(self, eval_ctx) - f = _binop_to_func[self.operator] - try: - return f(self.left.as_const(eval_ctx), self.right.as_const(eval_ctx)) - except: - raise Impossible() - - -class UnaryExpr(Expr): - """Baseclass for all unary expressions.""" - fields = ('node',) - operator = None - abstract = True - - def as_const(self, eval_ctx=None): - eval_ctx = get_eval_context(self, eval_ctx) - f = _uaop_to_func[self.operator] - try: - return f(self.node.as_const(eval_ctx)) - except: - raise Impossible() - - -class Name(Expr): - """Looks up a name or stores a value in a name. - The `ctx` of the node can be one of the following values: - - - `store`: store a value in the name - - `load`: load that name - - `param`: like `store` but if the name was defined as function parameter. - """ - fields = ('name', 'ctx') - - def can_assign(self): - return self.name not in ('true', 'false', 'none', - 'True', 'False', 'None') - - -class Literal(Expr): - """Baseclass for literals.""" - abstract = True - - -class Const(Literal): - """All constant values. The parser will return this node for simple - constants such as ``42`` or ``"foo"`` but it can be used to store more - complex values such as lists too. Only constants with a safe - representation (objects where ``eval(repr(x)) == x`` is true). - """ - fields = ('value',) - - def as_const(self, eval_ctx=None): - return self.value - - @classmethod - def from_untrusted(cls, value, lineno=None, environment=None): - """Return a const object if the value is representable as - constant value in the generated code, otherwise it will raise - an `Impossible` exception. - """ - from compiler import has_safe_repr - if not has_safe_repr(value): - raise Impossible() - return cls(value, lineno=lineno, environment=environment) - - -class TemplateData(Literal): - """A constant template string.""" - fields = ('data',) - - def as_const(self, eval_ctx=None): - eval_ctx = get_eval_context(self, eval_ctx) - if eval_ctx.volatile: - raise Impossible() - if eval_ctx.autoescape: - return Markup(self.data) - return self.data - - -class Tuple(Literal): - """For loop unpacking and some other things like multiple arguments - for subscripts. Like for :class:`Name` `ctx` specifies if the tuple - is used for loading the names or storing. - """ - fields = ('items', 'ctx') - - def as_const(self, eval_ctx=None): - eval_ctx = get_eval_context(self, eval_ctx) - return tuple(x.as_const(eval_ctx) for x in self.items) - - def can_assign(self): - for item in self.items: - if not item.can_assign(): - return False - return True - - -class List(Literal): - """Any list literal such as ``[1, 2, 3]``""" - fields = ('items',) - - def as_const(self, eval_ctx=None): - eval_ctx = get_eval_context(self, eval_ctx) - return [x.as_const(eval_ctx) for x in self.items] - - -class Dict(Literal): - """Any dict literal such as ``{1: 2, 3: 4}``. The items must be a list of - :class:`Pair` nodes. - """ - fields = ('items',) - - def as_const(self, eval_ctx=None): - eval_ctx = get_eval_context(self, eval_ctx) - return dict(x.as_const(eval_ctx) for x in self.items) - - -class Pair(Helper): - """A key, value pair for dicts.""" - fields = ('key', 'value') - - def as_const(self, eval_ctx=None): - eval_ctx = get_eval_context(self, eval_ctx) - return self.key.as_const(eval_ctx), self.value.as_const(eval_ctx) - - -class Keyword(Helper): - """A key, value pair for keyword arguments where key is a string.""" - fields = ('key', 'value') - - def as_const(self, eval_ctx=None): - eval_ctx = get_eval_context(self, eval_ctx) - return self.key, self.value.as_const(eval_ctx) - - -class CondExpr(Expr): - """A conditional expression (inline if expression). (``{{ - foo if bar else baz }}``) - """ - fields = ('test', 'expr1', 'expr2') - - def as_const(self, eval_ctx=None): - eval_ctx = get_eval_context(self, eval_ctx) - if self.test.as_const(eval_ctx): - return self.expr1.as_const(eval_ctx) - - # if we evaluate to an undefined object, we better do that at runtime - if self.expr2 is None: - raise Impossible() - - return self.expr2.as_const(eval_ctx) - - -class Filter(Expr): - """This node applies a filter on an expression. `name` is the name of - the filter, the rest of the fields are the same as for :class:`Call`. - - If the `node` of a filter is `None` the contents of the last buffer are - filtered. Buffers are created by macros and filter blocks. - """ - fields = ('node', 'name', 'args', 'kwargs', 'dyn_args', 'dyn_kwargs') - - def as_const(self, eval_ctx=None): - eval_ctx = get_eval_context(self, eval_ctx) - if eval_ctx.volatile or self.node is None: - raise Impossible() - # we have to be careful here because we call filter_ below. - # if this variable would be called filter, 2to3 would wrap the - # call in a list beause it is assuming we are talking about the - # builtin filter function here which no longer returns a list in - # python 3. because of that, do not rename filter_ to filter! - filter_ = self.environment.filters.get(self.name) - if filter_ is None or getattr(filter_, 'contextfilter', False): - raise Impossible() - obj = self.node.as_const(eval_ctx) - args = [x.as_const(eval_ctx) for x in self.args] - if getattr(filter_, 'evalcontextfilter', False): - args.insert(0, eval_ctx) - elif getattr(filter_, 'environmentfilter', False): - args.insert(0, self.environment) - kwargs = dict(x.as_const(eval_ctx) for x in self.kwargs) - if self.dyn_args is not None: - try: - args.extend(self.dyn_args.as_const(eval_ctx)) - except: - raise Impossible() - if self.dyn_kwargs is not None: - try: - kwargs.update(self.dyn_kwargs.as_const(eval_ctx)) - except: - raise Impossible() - try: - return filter_(obj, *args, **kwargs) - except: - raise Impossible() - - -class Test(Expr): - """Applies a test on an expression. `name` is the name of the test, the - rest of the fields are the same as for :class:`Call`. - """ - fields = ('node', 'name', 'args', 'kwargs', 'dyn_args', 'dyn_kwargs') - - -class Call(Expr): - """Calls an expression. `args` is a list of arguments, `kwargs` a list - of keyword arguments (list of :class:`Keyword` nodes), and `dyn_args` - and `dyn_kwargs` has to be either `None` or a node that is used as - node for dynamic positional (``*args``) or keyword (``**kwargs``) - arguments. - """ - fields = ('node', 'args', 'kwargs', 'dyn_args', 'dyn_kwargs') - - def as_const(self, eval_ctx=None): - eval_ctx = get_eval_context(self, eval_ctx) - if eval_ctx.volatile: - raise Impossible() - obj = self.node.as_const(eval_ctx) - - # don't evaluate context functions - args = [x.as_const(eval_ctx) for x in self.args] - if isinstance(obj, _context_function_types): - if getattr(obj, 'contextfunction', False): - raise Impossible() - elif getattr(obj, 'evalcontextfunction', False): - args.insert(0, eval_ctx) - elif getattr(obj, 'environmentfunction', False): - args.insert(0, self.environment) - - kwargs = dict(x.as_const(eval_ctx) for x in self.kwargs) - if self.dyn_args is not None: - try: - args.extend(self.dyn_args.as_const(eval_ctx)) - except: - raise Impossible() - if self.dyn_kwargs is not None: - try: - kwargs.update(self.dyn_kwargs.as_const(eval_ctx)) - except: - raise Impossible() - try: - return obj(*args, **kwargs) - except: - raise Impossible() - - -class Getitem(Expr): - """Get an attribute or item from an expression and prefer the item.""" - fields = ('node', 'arg', 'ctx') - - def as_const(self, eval_ctx=None): - eval_ctx = get_eval_context(self, eval_ctx) - if self.ctx != 'load': - raise Impossible() - try: - return self.environment.getitem(self.node.as_const(eval_ctx), - self.arg.as_const(eval_ctx)) - except: - raise Impossible() - - def can_assign(self): - return False - - -class Getattr(Expr): - """Get an attribute or item from an expression that is a ascii-only - bytestring and prefer the attribute. - """ - fields = ('node', 'attr', 'ctx') - - def as_const(self, eval_ctx=None): - if self.ctx != 'load': - raise Impossible() - try: - eval_ctx = get_eval_context(self, eval_ctx) - return self.environment.getattr(self.node.as_const(eval_ctx), - self.attr) - except: - raise Impossible() - - def can_assign(self): - return False - - -class Slice(Expr): - """Represents a slice object. This must only be used as argument for - :class:`Subscript`. - """ - fields = ('start', 'stop', 'step') - - def as_const(self, eval_ctx=None): - eval_ctx = get_eval_context(self, eval_ctx) - def const(obj): - if obj is None: - return None - return obj.as_const(eval_ctx) - return slice(const(self.start), const(self.stop), const(self.step)) - - -class Concat(Expr): - """Concatenates the list of expressions provided after converting them to - unicode. - """ - fields = ('nodes',) - - def as_const(self, eval_ctx=None): - eval_ctx = get_eval_context(self, eval_ctx) - return ''.join(unicode(x.as_const(eval_ctx)) for x in self.nodes) - - -class Compare(Expr): - """Compares an expression with some other expressions. `ops` must be a - list of :class:`Operand`\s. - """ - fields = ('expr', 'ops') - - def as_const(self, eval_ctx=None): - eval_ctx = get_eval_context(self, eval_ctx) - result = value = self.expr.as_const(eval_ctx) - try: - for op in self.ops: - new_value = op.expr.as_const(eval_ctx) - result = _cmpop_to_func[op.op](value, new_value) - value = new_value - except: - raise Impossible() - return result - - -class Operand(Helper): - """Holds an operator and an expression.""" - fields = ('op', 'expr') - -if __debug__: - Operand.__doc__ += '\nThe following operators are available: ' + \ - ', '.join(sorted('``%s``' % x for x in set(_binop_to_func) | - set(_uaop_to_func) | set(_cmpop_to_func))) - - -class Mul(BinExpr): - """Multiplies the left with the right node.""" - operator = '*' - - -class Div(BinExpr): - """Divides the left by the right node.""" - operator = '/' - - -class FloorDiv(BinExpr): - """Divides the left by the right node and truncates conver the - result into an integer by truncating. - """ - operator = '//' - - -class Add(BinExpr): - """Add the left to the right node.""" - operator = '+' - - -class Sub(BinExpr): - """Substract the right from the left node.""" - operator = '-' - - -class Mod(BinExpr): - """Left modulo right.""" - operator = '%' - - -class Pow(BinExpr): - """Left to the power of right.""" - operator = '**' - - -class And(BinExpr): - """Short circuited AND.""" - operator = 'and' - - def as_const(self, eval_ctx=None): - eval_ctx = get_eval_context(self, eval_ctx) - return self.left.as_const(eval_ctx) and self.right.as_const(eval_ctx) - - -class Or(BinExpr): - """Short circuited OR.""" - operator = 'or' - - def as_const(self, eval_ctx=None): - eval_ctx = get_eval_context(self, eval_ctx) - return self.left.as_const(eval_ctx) or self.right.as_const(eval_ctx) - - -class Not(UnaryExpr): - """Negate the expression.""" - operator = 'not' - - -class Neg(UnaryExpr): - """Make the expression negative.""" - operator = '-' - - -class Pos(UnaryExpr): - """Make the expression positive (noop for most expressions)""" - operator = '+' - - -# Helpers for extensions - - -class EnvironmentAttribute(Expr): - """Loads an attribute from the environment object. This is useful for - extensions that want to call a callback stored on the environment. - """ - fields = ('name',) - - -class ExtensionAttribute(Expr): - """Returns the attribute of an extension bound to the environment. - The identifier is the identifier of the :class:`Extension`. - - This node is usually constructed by calling the - :meth:`~ambari_jinja2.ext.Extension.attr` method on an extension. - """ - fields = ('identifier', 'name') - - -class ImportedName(Expr): - """If created with an import name the import name is returned on node - access. For example ``ImportedName('cgi.escape')`` returns the `escape` - function from the cgi module on evaluation. Imports are optimized by the - compiler so there is no need to assign them to local variables. - """ - fields = ('importname',) - - -class InternalName(Expr): - """An internal name in the compiler. You cannot create these nodes - yourself but the parser provides a - :meth:`~ambari_jinja2.parser.Parser.free_identifier` method that creates - a new identifier for you. This identifier is not available from the - template and is not threated specially by the compiler. - """ - fields = ('name',) - - def __init__(self): - raise TypeError('Can\'t create internal names. Use the ' - '`free_identifier` method on a parser.') - - -class MarkSafe(Expr): - """Mark the wrapped expression as safe (wrap it as `Markup`).""" - fields = ('expr',) - - def as_const(self, eval_ctx=None): - eval_ctx = get_eval_context(self, eval_ctx) - return Markup(self.expr.as_const(eval_ctx)) - - -class MarkSafeIfAutoescape(Expr): - """Mark the wrapped expression as safe (wrap it as `Markup`) but - only if autoescaping is active. - - .. versionadded:: 2.5 - """ - fields = ('expr',) - - def as_const(self, eval_ctx=None): - eval_ctx = get_eval_context(self, eval_ctx) - if eval_ctx.volatile: - raise Impossible() - expr = self.expr.as_const(eval_ctx) - if eval_ctx.autoescape: - return Markup(expr) - return expr - - -class ContextReference(Expr): - """Returns the current template context. It can be used like a - :class:`Name` node, with a ``'load'`` ctx and will return the - current :class:`~ambari_jinja2.runtime.Context` object. - - Here an example that assigns the current template name to a - variable named `foo`:: - - Assign(Name('foo', ctx='store'), - Getattr(ContextReference(), 'name')) - """ - - -class Continue(Stmt): - """Continue a loop.""" - - -class Break(Stmt): - """Break a loop.""" - - -class Scope(Stmt): - """An artificial scope.""" - fields = ('body',) - - -class EvalContextModifier(Stmt): - """Modifies the eval context. For each option that should be modified, - a :class:`Keyword` has to be added to the :attr:`options` list. - - Example to change the `autoescape` setting:: - - EvalContextModifier(options=[Keyword('autoescape', Const(True))]) - """ - fields = ('options',) - - -class ScopedEvalContextModifier(EvalContextModifier): - """Modifies the eval context and reverts it later. Works exactly like - :class:`EvalContextModifier` but will only modify the - :class:`~ambari_jinja2.nodes.EvalContext` for nodes in the :attr:`body`. - """ - fields = ('body',) - - -# make sure nobody creates custom nodes -def _failing_new(*args, **kwargs): - raise TypeError('can\'t create custom node types') -NodeType.__new__ = staticmethod(_failing_new); del _failing_new
http://git-wip-us.apache.org/repos/asf/ambari/blob/570de228/ambari-common/src/main/python/ambari_jinja2/ambari_jinja2/optimizer.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/ambari_jinja2/ambari_jinja2/optimizer.py b/ambari-common/src/main/python/ambari_jinja2/ambari_jinja2/optimizer.py deleted file mode 100644 index b014396..0000000 --- a/ambari-common/src/main/python/ambari_jinja2/ambari_jinja2/optimizer.py +++ /dev/null @@ -1,68 +0,0 @@ -# -*- coding: utf-8 -*- -""" - ambari_jinja2.optimizer - ~~~~~~~~~~~~~~~~ - - The jinja optimizer is currently trying to constant fold a few expressions - and modify the AST in place so that it should be easier to evaluate it. - - Because the AST does not contain all the scoping information and the - compiler has to find that out, we cannot do all the optimizations we - want. For example loop unrolling doesn't work because unrolled loops would - have a different scoping. - - The solution would be a second syntax tree that has the scoping rules stored. - - :copyright: (c) 2010 by the Jinja Team. - :license: BSD. -""" -from ambari_jinja2 import nodes -from ambari_jinja2.visitor import NodeTransformer - - -def optimize(node, environment): - """The context hint can be used to perform an static optimization - based on the context given.""" - optimizer = Optimizer(environment) - return optimizer.visit(node) - - -class Optimizer(NodeTransformer): - - def __init__(self, environment): - self.environment = environment - - def visit_If(self, node): - """Eliminate dead code.""" - # do not optimize ifs that have a block inside so that it doesn't - # break super(). - if node.find(nodes.Block) is not None: - return self.generic_visit(node) - try: - val = self.visit(node.test).as_const() - except nodes.Impossible: - return self.generic_visit(node) - if val: - body = node.body - else: - body = node.else_ - result = [] - for node in body: - result.extend(self.visit_list(node)) - return result - - def fold(self, node): - """Do constant folding.""" - node = self.generic_visit(node) - try: - return nodes.Const.from_untrusted(node.as_const(), - lineno=node.lineno, - environment=self.environment) - except nodes.Impossible: - return node - - visit_Add = visit_Sub = visit_Mul = visit_Div = visit_FloorDiv = \ - visit_Pow = visit_Mod = visit_And = visit_Or = visit_Pos = visit_Neg = \ - visit_Not = visit_Compare = visit_Getitem = visit_Getattr = visit_Call = \ - visit_Filter = visit_Test = visit_CondExpr = fold - del fold http://git-wip-us.apache.org/repos/asf/ambari/blob/570de228/ambari-common/src/main/python/ambari_jinja2/ambari_jinja2/parser.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/ambari_jinja2/ambari_jinja2/parser.py b/ambari-common/src/main/python/ambari_jinja2/ambari_jinja2/parser.py deleted file mode 100644 index a14c055..0000000 --- a/ambari-common/src/main/python/ambari_jinja2/ambari_jinja2/parser.py +++ /dev/null @@ -1,896 +0,0 @@ -# -*- coding: utf-8 -*- -""" - ambari_jinja2.parser - ~~~~~~~~~~~~~ - - Implements the template parser. - - :copyright: (c) 2010 by the Jinja Team. - :license: BSD, see LICENSE for more details. -""" -from ambari_jinja2 import nodes -from ambari_jinja2.exceptions import TemplateSyntaxError, TemplateAssertionError -from ambari_jinja2.utils import next -from ambari_jinja2.lexer import describe_token, describe_token_expr - - -#: statements that callinto -_statement_keywords = frozenset(['for', 'if', 'block', 'extends', 'print', - 'macro', 'include', 'from', 'import', - 'set']) -_compare_operators = frozenset(['eq', 'ne', 'lt', 'lteq', 'gt', 'gteq']) - - -class Parser(object): - """This is the central parsing class Jinja2 uses. It's passed to - extensions and can be used to parse expressions or statements. - """ - - def __init__(self, environment, source, name=None, filename=None, - state=None): - self.environment = environment - self.stream = environment._tokenize(source, name, filename, state) - self.name = name - self.filename = filename - self.closed = False - self.extensions = {} - for extension in environment.iter_extensions(): - for tag in extension.tags: - self.extensions[tag] = extension.parse - self._last_identifier = 0 - self._tag_stack = [] - self._end_token_stack = [] - - def fail(self, msg, lineno=None, exc=TemplateSyntaxError): - """Convenience method that raises `exc` with the message, passed - line number or last line number as well as the current name and - filename. - """ - if lineno is None: - lineno = self.stream.current.lineno - raise exc(msg, lineno, self.name, self.filename) - - def _fail_ut_eof(self, name, end_token_stack, lineno): - expected = [] - for exprs in end_token_stack: - expected.extend(map(describe_token_expr, exprs)) - if end_token_stack: - currently_looking = ' or '.join( - "'%s'" % describe_token_expr(expr) - for expr in end_token_stack[-1]) - else: - currently_looking = None - - if name is None: - message = ['Unexpected end of template.'] - else: - message = ['Encountered unknown tag \'%s\'.' % name] - - if currently_looking: - if name is not None and name in expected: - message.append('You probably made a nesting mistake. Jinja ' - 'is expecting this tag, but currently looking ' - 'for %s.' % currently_looking) - else: - message.append('Jinja was looking for the following tags: ' - '%s.' % currently_looking) - - if self._tag_stack: - message.append('The innermost block that needs to be ' - 'closed is \'%s\'.' % self._tag_stack[-1]) - - self.fail(' '.join(message), lineno) - - def fail_unknown_tag(self, name, lineno=None): - """Called if the parser encounters an unknown tag. Tries to fail - with a human readable error message that could help to identify - the problem. - """ - return self._fail_ut_eof(name, self._end_token_stack, lineno) - - def fail_eof(self, end_tokens=None, lineno=None): - """Like fail_unknown_tag but for end of template situations.""" - stack = list(self._end_token_stack) - if end_tokens is not None: - stack.append(end_tokens) - return self._fail_ut_eof(None, stack, lineno) - - def is_tuple_end(self, extra_end_rules=None): - """Are we at the end of a tuple?""" - if self.stream.current.type in ('variable_end', 'block_end', 'rparen'): - return True - elif extra_end_rules is not None: - return self.stream.current.test_any(extra_end_rules) - return False - - def free_identifier(self, lineno=None): - """Return a new free identifier as :class:`~ambari_jinja2.nodes.InternalName`.""" - self._last_identifier += 1 - rv = object.__new__(nodes.InternalName) - nodes.Node.__init__(rv, 'fi%d' % self._last_identifier, lineno=lineno) - return rv - - def parse_statement(self): - """Parse a single statement.""" - token = self.stream.current - if token.type != 'name': - self.fail('tag name expected', token.lineno) - self._tag_stack.append(token.value) - pop_tag = True - try: - if token.value in _statement_keywords: - return getattr(self, 'parse_' + self.stream.current.value)() - if token.value == 'call': - return self.parse_call_block() - if token.value == 'filter': - return self.parse_filter_block() - ext = self.extensions.get(token.value) - if ext is not None: - return ext(self) - - # did not work out, remove the token we pushed by accident - # from the stack so that the unknown tag fail function can - # produce a proper error message. - self._tag_stack.pop() - pop_tag = False - self.fail_unknown_tag(token.value, token.lineno) - finally: - if pop_tag: - self._tag_stack.pop() - - def parse_statements(self, end_tokens, drop_needle=False): - """Parse multiple statements into a list until one of the end tokens - is reached. This is used to parse the body of statements as it also - parses template data if appropriate. The parser checks first if the - current token is a colon and skips it if there is one. Then it checks - for the block end and parses until if one of the `end_tokens` is - reached. Per default the active token in the stream at the end of - the call is the matched end token. If this is not wanted `drop_needle` - can be set to `True` and the end token is removed. - """ - # the first token may be a colon for python compatibility - self.stream.skip_if('colon') - - # in the future it would be possible to add whole code sections - # by adding some sort of end of statement token and parsing those here. - self.stream.expect('block_end') - result = self.subparse(end_tokens) - - # we reached the end of the template too early, the subparser - # does not check for this, so we do that now - if self.stream.current.type == 'eof': - self.fail_eof(end_tokens) - - if drop_needle: - next(self.stream) - return result - - def parse_set(self): - """Parse an assign statement.""" - lineno = next(self.stream).lineno - target = self.parse_assign_target() - self.stream.expect('assign') - expr = self.parse_tuple() - return nodes.Assign(target, expr, lineno=lineno) - - def parse_for(self): - """Parse a for loop.""" - lineno = self.stream.expect('name:for').lineno - target = self.parse_assign_target(extra_end_rules=('name:in',)) - self.stream.expect('name:in') - iter = self.parse_tuple(with_condexpr=False, - extra_end_rules=('name:recursive',)) - test = None - if self.stream.skip_if('name:if'): - test = self.parse_expression() - recursive = self.stream.skip_if('name:recursive') - body = self.parse_statements(('name:endfor', 'name:else')) - if next(self.stream).value == 'endfor': - else_ = [] - else: - else_ = self.parse_statements(('name:endfor',), drop_needle=True) - return nodes.For(target, iter, body, else_, test, - recursive, lineno=lineno) - - def parse_if(self): - """Parse an if construct.""" - node = result = nodes.If(lineno=self.stream.expect('name:if').lineno) - while 1: - node.test = self.parse_tuple(with_condexpr=False) - node.body = self.parse_statements(('name:elif', 'name:else', - 'name:endif')) - token = next(self.stream) - if token.test('name:elif'): - new_node = nodes.If(lineno=self.stream.current.lineno) - node.else_ = [new_node] - node = new_node - continue - elif token.test('name:else'): - node.else_ = self.parse_statements(('name:endif',), - drop_needle=True) - else: - node.else_ = [] - break - return result - - def parse_block(self): - node = nodes.Block(lineno=next(self.stream).lineno) - node.name = self.stream.expect('name').value - node.scoped = self.stream.skip_if('name:scoped') - - # common problem people encounter when switching from django - # to jinja. we do not support hyphens in block names, so let's - # raise a nicer error message in that case. - if self.stream.current.type == 'sub': - self.fail('Block names in Jinja have to be valid Python ' - 'identifiers and may not contain hypens, use an ' - 'underscore instead.') - - node.body = self.parse_statements(('name:endblock',), drop_needle=True) - self.stream.skip_if('name:' + node.name) - return node - - def parse_extends(self): - node = nodes.Extends(lineno=next(self.stream).lineno) - node.template = self.parse_expression() - return node - - def parse_import_context(self, node, default): - if self.stream.current.test_any('name:with', 'name:without') and \ - self.stream.look().test('name:context'): - node.with_context = next(self.stream).value == 'with' - self.stream.skip() - else: - node.with_context = default - return node - - def parse_include(self): - node = nodes.Include(lineno=next(self.stream).lineno) - node.template = self.parse_expression() - if self.stream.current.test('name:ignore') and \ - self.stream.look().test('name:missing'): - node.ignore_missing = True - self.stream.skip(2) - else: - node.ignore_missing = False - return self.parse_import_context(node, True) - - def parse_import(self): - node = nodes.Import(lineno=next(self.stream).lineno) - node.template = self.parse_expression() - self.stream.expect('name:as') - node.target = self.parse_assign_target(name_only=True).name - return self.parse_import_context(node, False) - - def parse_from(self): - node = nodes.FromImport(lineno=next(self.stream).lineno) - node.template = self.parse_expression() - self.stream.expect('name:import') - node.names = [] - - def parse_context(): - if self.stream.current.value in ('with', 'without') and \ - self.stream.look().test('name:context'): - node.with_context = next(self.stream).value == 'with' - self.stream.skip() - return True - return False - - while 1: - if node.names: - self.stream.expect('comma') - if self.stream.current.type == 'name': - if parse_context(): - break - target = self.parse_assign_target(name_only=True) - if target.name.startswith('_'): - self.fail('names starting with an underline can not ' - 'be imported', target.lineno, - exc=TemplateAssertionError) - if self.stream.skip_if('name:as'): - alias = self.parse_assign_target(name_only=True) - node.names.append((target.name, alias.name)) - else: - node.names.append(target.name) - if parse_context() or self.stream.current.type != 'comma': - break - else: - break - if not hasattr(node, 'with_context'): - node.with_context = False - self.stream.skip_if('comma') - return node - - def parse_signature(self, node): - node.args = args = [] - node.defaults = defaults = [] - self.stream.expect('lparen') - while self.stream.current.type != 'rparen': - if args: - self.stream.expect('comma') - arg = self.parse_assign_target(name_only=True) - arg.set_ctx('param') - if self.stream.skip_if('assign'): - defaults.append(self.parse_expression()) - args.append(arg) - self.stream.expect('rparen') - - def parse_call_block(self): - node = nodes.CallBlock(lineno=next(self.stream).lineno) - if self.stream.current.type == 'lparen': - self.parse_signature(node) - else: - node.args = [] - node.defaults = [] - - node.call = self.parse_expression() - if not isinstance(node.call, nodes.Call): - self.fail('expected call', node.lineno) - node.body = self.parse_statements(('name:endcall',), drop_needle=True) - return node - - def parse_filter_block(self): - node = nodes.FilterBlock(lineno=next(self.stream).lineno) - node.filter = self.parse_filter(None, start_inline=True) - node.body = self.parse_statements(('name:endfilter',), - drop_needle=True) - return node - - def parse_macro(self): - node = nodes.Macro(lineno=next(self.stream).lineno) - node.name = self.parse_assign_target(name_only=True).name - self.parse_signature(node) - node.body = self.parse_statements(('name:endmacro',), - drop_needle=True) - return node - - def parse_print(self): - node = nodes.Output(lineno=next(self.stream).lineno) - node.nodes = [] - while self.stream.current.type != 'block_end': - if node.nodes: - self.stream.expect('comma') - node.nodes.append(self.parse_expression()) - return node - - def parse_assign_target(self, with_tuple=True, name_only=False, - extra_end_rules=None): - """Parse an assignment target. As Jinja2 allows assignments to - tuples, this function can parse all allowed assignment targets. Per - default assignments to tuples are parsed, that can be disable however - by setting `with_tuple` to `False`. If only assignments to names are - wanted `name_only` can be set to `True`. The `extra_end_rules` - parameter is forwarded to the tuple parsing function. - """ - if name_only: - token = self.stream.expect('name') - target = nodes.Name(token.value, 'store', lineno=token.lineno) - else: - if with_tuple: - target = self.parse_tuple(simplified=True, - extra_end_rules=extra_end_rules) - else: - target = self.parse_primary() - target.set_ctx('store') - if not target.can_assign(): - self.fail('can\'t assign to %r' % target.__class__. - __name__.lower(), target.lineno) - return target - - def parse_expression(self, with_condexpr=True): - """Parse an expression. Per default all expressions are parsed, if - the optional `with_condexpr` parameter is set to `False` conditional - expressions are not parsed. - """ - if with_condexpr: - return self.parse_condexpr() - return self.parse_or() - - def parse_condexpr(self): - lineno = self.stream.current.lineno - expr1 = self.parse_or() - while self.stream.skip_if('name:if'): - expr2 = self.parse_or() - if self.stream.skip_if('name:else'): - expr3 = self.parse_condexpr() - else: - expr3 = None - expr1 = nodes.CondExpr(expr2, expr1, expr3, lineno=lineno) - lineno = self.stream.current.lineno - return expr1 - - def parse_or(self): - lineno = self.stream.current.lineno - left = self.parse_and() - while self.stream.skip_if('name:or'): - right = self.parse_and() - left = nodes.Or(left, right, lineno=lineno) - lineno = self.stream.current.lineno - return left - - def parse_and(self): - lineno = self.stream.current.lineno - left = self.parse_not() - while self.stream.skip_if('name:and'): - right = self.parse_not() - left = nodes.And(left, right, lineno=lineno) - lineno = self.stream.current.lineno - return left - - def parse_not(self): - if self.stream.current.test('name:not'): - lineno = next(self.stream).lineno - return nodes.Not(self.parse_not(), lineno=lineno) - return self.parse_compare() - - def parse_compare(self): - lineno = self.stream.current.lineno - expr = self.parse_add() - ops = [] - while 1: - token_type = self.stream.current.type - if token_type in _compare_operators: - next(self.stream) - ops.append(nodes.Operand(token_type, self.parse_add())) - elif self.stream.skip_if('name:in'): - ops.append(nodes.Operand('in', self.parse_add())) - elif self.stream.current.test('name:not') and \ - self.stream.look().test('name:in'): - self.stream.skip(2) - ops.append(nodes.Operand('notin', self.parse_add())) - else: - break - lineno = self.stream.current.lineno - if not ops: - return expr - return nodes.Compare(expr, ops, lineno=lineno) - - def parse_add(self): - lineno = self.stream.current.lineno - left = self.parse_sub() - while self.stream.current.type == 'add': - next(self.stream) - right = self.parse_sub() - left = nodes.Add(left, right, lineno=lineno) - lineno = self.stream.current.lineno - return left - - def parse_sub(self): - lineno = self.stream.current.lineno - left = self.parse_concat() - while self.stream.current.type == 'sub': - next(self.stream) - right = self.parse_concat() - left = nodes.Sub(left, right, lineno=lineno) - lineno = self.stream.current.lineno - return left - - def parse_concat(self): - lineno = self.stream.current.lineno - args = [self.parse_mul()] - while self.stream.current.type == 'tilde': - next(self.stream) - args.append(self.parse_mul()) - if len(args) == 1: - return args[0] - return nodes.Concat(args, lineno=lineno) - - def parse_mul(self): - lineno = self.stream.current.lineno - left = self.parse_div() - while self.stream.current.type == 'mul': - next(self.stream) - right = self.parse_div() - left = nodes.Mul(left, right, lineno=lineno) - lineno = self.stream.current.lineno - return left - - def parse_div(self): - lineno = self.stream.current.lineno - left = self.parse_floordiv() - while self.stream.current.type == 'div': - next(self.stream) - right = self.parse_floordiv() - left = nodes.Div(left, right, lineno=lineno) - lineno = self.stream.current.lineno - return left - - def parse_floordiv(self): - lineno = self.stream.current.lineno - left = self.parse_mod() - while self.stream.current.type == 'floordiv': - next(self.stream) - right = self.parse_mod() - left = nodes.FloorDiv(left, right, lineno=lineno) - lineno = self.stream.current.lineno - return left - - def parse_mod(self): - lineno = self.stream.current.lineno - left = self.parse_pow() - while self.stream.current.type == 'mod': - next(self.stream) - right = self.parse_pow() - left = nodes.Mod(left, right, lineno=lineno) - lineno = self.stream.current.lineno - return left - - def parse_pow(self): - lineno = self.stream.current.lineno - left = self.parse_unary() - while self.stream.current.type == 'pow': - next(self.stream) - right = self.parse_unary() - left = nodes.Pow(left, right, lineno=lineno) - lineno = self.stream.current.lineno - return left - - def parse_unary(self, with_filter=True): - token_type = self.stream.current.type - lineno = self.stream.current.lineno - if token_type == 'sub': - next(self.stream) - node = nodes.Neg(self.parse_unary(False), lineno=lineno) - elif token_type == 'add': - next(self.stream) - node = nodes.Pos(self.parse_unary(False), lineno=lineno) - else: - node = self.parse_primary() - node = self.parse_postfix(node) - if with_filter: - node = self.parse_filter_expr(node) - return node - - def parse_primary(self): - token = self.stream.current - if token.type == 'name': - if token.value in ('true', 'false', 'True', 'False'): - node = nodes.Const(token.value in ('true', 'True'), - lineno=token.lineno) - elif token.value in ('none', 'None'): - node = nodes.Const(None, lineno=token.lineno) - else: - node = nodes.Name(token.value, 'load', lineno=token.lineno) - next(self.stream) - elif token.type == 'string': - next(self.stream) - buf = [token.value] - lineno = token.lineno - while self.stream.current.type == 'string': - buf.append(self.stream.current.value) - next(self.stream) - node = nodes.Const(''.join(buf), lineno=lineno) - elif token.type in ('integer', 'float'): - next(self.stream) - node = nodes.Const(token.value, lineno=token.lineno) - elif token.type == 'lparen': - next(self.stream) - node = self.parse_tuple(explicit_parentheses=True) - self.stream.expect('rparen') - elif token.type == 'lbracket': - node = self.parse_list() - elif token.type == 'lbrace': - node = self.parse_dict() - else: - self.fail("unexpected '%s'" % describe_token(token), token.lineno) - return node - - def parse_tuple(self, simplified=False, with_condexpr=True, - extra_end_rules=None, explicit_parentheses=False): - """Works like `parse_expression` but if multiple expressions are - delimited by a comma a :class:`~ambari_jinja2.nodes.Tuple` node is created. - This method could also return a regular expression instead of a tuple - if no commas where found. - - The default parsing mode is a full tuple. If `simplified` is `True` - only names and literals are parsed. The `no_condexpr` parameter is - forwarded to :meth:`parse_expression`. - - Because tuples do not require delimiters and may end in a bogus comma - an extra hint is needed that marks the end of a tuple. For example - for loops support tuples between `for` and `in`. In that case the - `extra_end_rules` is set to ``['name:in']``. - - `explicit_parentheses` is true if the parsing was triggered by an - expression in parentheses. This is used to figure out if an empty - tuple is a valid expression or not. - """ - lineno = self.stream.current.lineno - if simplified: - parse = self.parse_primary - elif with_condexpr: - parse = self.parse_expression - else: - parse = lambda: self.parse_expression(with_condexpr=False) - args = [] - is_tuple = False - while 1: - if args: - self.stream.expect('comma') - if self.is_tuple_end(extra_end_rules): - break - args.append(parse()) - if self.stream.current.type == 'comma': - is_tuple = True - else: - break - lineno = self.stream.current.lineno - - if not is_tuple: - if args: - return args[0] - - # if we don't have explicit parentheses, an empty tuple is - # not a valid expression. This would mean nothing (literally - # nothing) in the spot of an expression would be an empty - # tuple. - if not explicit_parentheses: - self.fail('Expected an expression, got \'%s\'' % - describe_token(self.stream.current)) - - return nodes.Tuple(args, 'load', lineno=lineno) - - def parse_list(self): - token = self.stream.expect('lbracket') - items = [] - while self.stream.current.type != 'rbracket': - if items: - self.stream.expect('comma') - if self.stream.current.type == 'rbracket': - break - items.append(self.parse_expression()) - self.stream.expect('rbracket') - return nodes.List(items, lineno=token.lineno) - - def parse_dict(self): - token = self.stream.expect('lbrace') - items = [] - while self.stream.current.type != 'rbrace': - if items: - self.stream.expect('comma') - if self.stream.current.type == 'rbrace': - break - key = self.parse_expression() - self.stream.expect('colon') - value = self.parse_expression() - items.append(nodes.Pair(key, value, lineno=key.lineno)) - self.stream.expect('rbrace') - return nodes.Dict(items, lineno=token.lineno) - - def parse_postfix(self, node): - while 1: - token_type = self.stream.current.type - if token_type == 'dot' or token_type == 'lbracket': - node = self.parse_subscript(node) - # calls are valid both after postfix expressions (getattr - # and getitem) as well as filters and tests - elif token_type == 'lparen': - node = self.parse_call(node) - else: - break - return node - - def parse_filter_expr(self, node): - while 1: - token_type = self.stream.current.type - if token_type == 'pipe': - node = self.parse_filter(node) - elif token_type == 'name' and self.stream.current.value == 'is': - node = self.parse_test(node) - # calls are valid both after postfix expressions (getattr - # and getitem) as well as filters and tests - elif token_type == 'lparen': - node = self.parse_call(node) - else: - break - return node - - def parse_subscript(self, node): - token = next(self.stream) - if token.type == 'dot': - attr_token = self.stream.current - next(self.stream) - if attr_token.type == 'name': - return nodes.Getattr(node, attr_token.value, 'load', - lineno=token.lineno) - elif attr_token.type != 'integer': - self.fail('expected name or number', attr_token.lineno) - arg = nodes.Const(attr_token.value, lineno=attr_token.lineno) - return nodes.Getitem(node, arg, 'load', lineno=token.lineno) - if token.type == 'lbracket': - priority_on_attribute = False - args = [] - while self.stream.current.type != 'rbracket': - if args: - self.stream.expect('comma') - args.append(self.parse_subscribed()) - self.stream.expect('rbracket') - if len(args) == 1: - arg = args[0] - else: - arg = nodes.Tuple(args, 'load', lineno=token.lineno) - return nodes.Getitem(node, arg, 'load', lineno=token.lineno) - self.fail('expected subscript expression', self.lineno) - - def parse_subscribed(self): - lineno = self.stream.current.lineno - - if self.stream.current.type == 'colon': - next(self.stream) - args = [None] - else: - node = self.parse_expression() - if self.stream.current.type != 'colon': - return node - next(self.stream) - args = [node] - - if self.stream.current.type == 'colon': - args.append(None) - elif self.stream.current.type not in ('rbracket', 'comma'): - args.append(self.parse_expression()) - else: - args.append(None) - - if self.stream.current.type == 'colon': - next(self.stream) - if self.stream.current.type not in ('rbracket', 'comma'): - args.append(self.parse_expression()) - else: - args.append(None) - else: - args.append(None) - - return nodes.Slice(lineno=lineno, *args) - - def parse_call(self, node): - token = self.stream.expect('lparen') - args = [] - kwargs = [] - dyn_args = dyn_kwargs = None - require_comma = False - - def ensure(expr): - if not expr: - self.fail('invalid syntax for function call expression', - token.lineno) - - while self.stream.current.type != 'rparen': - if require_comma: - self.stream.expect('comma') - # support for trailing comma - if self.stream.current.type == 'rparen': - break - if self.stream.current.type == 'mul': - ensure(dyn_args is None and dyn_kwargs is None) - next(self.stream) - dyn_args = self.parse_expression() - elif self.stream.current.type == 'pow': - ensure(dyn_kwargs is None) - next(self.stream) - dyn_kwargs = self.parse_expression() - else: - ensure(dyn_args is None and dyn_kwargs is None) - if self.stream.current.type == 'name' and \ - self.stream.look().type == 'assign': - key = self.stream.current.value - self.stream.skip(2) - value = self.parse_expression() - kwargs.append(nodes.Keyword(key, value, - lineno=value.lineno)) - else: - ensure(not kwargs) - args.append(self.parse_expression()) - - require_comma = True - self.stream.expect('rparen') - - if node is None: - return args, kwargs, dyn_args, dyn_kwargs - return nodes.Call(node, args, kwargs, dyn_args, dyn_kwargs, - lineno=token.lineno) - - def parse_filter(self, node, start_inline=False): - while self.stream.current.type == 'pipe' or start_inline: - if not start_inline: - next(self.stream) - token = self.stream.expect('name') - name = token.value - while self.stream.current.type == 'dot': - next(self.stream) - name += '.' + self.stream.expect('name').value - if self.stream.current.type == 'lparen': - args, kwargs, dyn_args, dyn_kwargs = self.parse_call(None) - else: - args = [] - kwargs = [] - dyn_args = dyn_kwargs = None - node = nodes.Filter(node, name, args, kwargs, dyn_args, - dyn_kwargs, lineno=token.lineno) - start_inline = False - return node - - def parse_test(self, node): - token = next(self.stream) - if self.stream.current.test('name:not'): - next(self.stream) - negated = True - else: - negated = False - name = self.stream.expect('name').value - while self.stream.current.type == 'dot': - next(self.stream) - name += '.' + self.stream.expect('name').value - dyn_args = dyn_kwargs = None - kwargs = [] - if self.stream.current.type == 'lparen': - args, kwargs, dyn_args, dyn_kwargs = self.parse_call(None) - elif self.stream.current.type in ('name', 'string', 'integer', - 'float', 'lparen', 'lbracket', - 'lbrace') and not \ - self.stream.current.test_any('name:else', 'name:or', - 'name:and'): - if self.stream.current.test('name:is'): - self.fail('You cannot chain multiple tests with is') - args = [self.parse_expression()] - else: - args = [] - node = nodes.Test(node, name, args, kwargs, dyn_args, - dyn_kwargs, lineno=token.lineno) - if negated: - node = nodes.Not(node, lineno=token.lineno) - return node - - def subparse(self, end_tokens=None): - body = [] - data_buffer = [] - add_data = data_buffer.append - - if end_tokens is not None: - self._end_token_stack.append(end_tokens) - - def flush_data(): - if data_buffer: - lineno = data_buffer[0].lineno - body.append(nodes.Output(data_buffer[:], lineno=lineno)) - del data_buffer[:] - - try: - while self.stream: - token = self.stream.current - if token.type == 'data': - if token.value: - add_data(nodes.TemplateData(token.value, - lineno=token.lineno)) - next(self.stream) - elif token.type == 'variable_begin': - next(self.stream) - add_data(self.parse_tuple(with_condexpr=True)) - self.stream.expect('variable_end') - elif token.type == 'block_begin': - flush_data() - next(self.stream) - if end_tokens is not None and \ - self.stream.current.test_any(*end_tokens): - return body - rv = self.parse_statement() - if isinstance(rv, list): - body.extend(rv) - else: - body.append(rv) - self.stream.expect('block_end') - else: - raise AssertionError('internal parsing error') - - flush_data() - finally: - if end_tokens is not None: - self._end_token_stack.pop() - - return body - - def parse(self): - """Parse the whole template into a `Template` node.""" - result = nodes.Template(self.subparse(), lineno=1) - result.set_environment(self.environment) - return result http://git-wip-us.apache.org/repos/asf/ambari/blob/570de228/ambari-common/src/main/python/ambari_jinja2/ambari_jinja2/runtime.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/ambari_jinja2/ambari_jinja2/runtime.py b/ambari-common/src/main/python/ambari_jinja2/ambari_jinja2/runtime.py deleted file mode 100644 index 0830132..0000000 --- a/ambari-common/src/main/python/ambari_jinja2/ambari_jinja2/runtime.py +++ /dev/null @@ -1,544 +0,0 @@ -# -*- coding: utf-8 -*- -""" - ambari_jinja2.runtime - ~~~~~~~~~~~~~~ - - Runtime helpers. - - :copyright: (c) 2010 by the Jinja Team. - :license: BSD. -""" -import sys -from itertools import chain, imap -from ambari_jinja2.nodes import EvalContext, _context_function_types -from ambari_jinja2.utils import Markup, partial, soft_unicode, escape, missing, \ - concat, internalcode, next, object_type_repr -from ambari_jinja2.exceptions import UndefinedError, TemplateRuntimeError, \ - TemplateNotFound - - -# these variables are exported to the template runtime -__all__ = ['LoopContext', 'TemplateReference', 'Macro', 'Markup', - 'TemplateRuntimeError', 'missing', 'concat', 'escape', - 'markup_join', 'unicode_join', 'to_string', 'identity', - 'TemplateNotFound'] - -#: the name of the function that is used to convert something into -#: a string. 2to3 will adopt that automatically and the generated -#: code can take advantage of it. -to_string = unicode - -#: the identity function. Useful for certain things in the environment -identity = lambda x: x - - -def markup_join(seq): - """Concatenation that escapes if necessary and converts to unicode.""" - buf = [] - iterator = imap(soft_unicode, seq) - for arg in iterator: - buf.append(arg) - if hasattr(arg, '__html__'): - return Markup(u'').join(chain(buf, iterator)) - return concat(buf) - - -def unicode_join(seq): - """Simple args to unicode conversion and concatenation.""" - return concat(imap(unicode, seq)) - - -def new_context(environment, template_name, blocks, vars=None, - shared=None, globals=None, locals=None): - """Internal helper to for context creation.""" - if vars is None: - vars = {} - if shared: - parent = vars - else: - parent = dict(globals or (), **vars) - if locals: - # if the parent is shared a copy should be created because - # we don't want to modify the dict passed - if shared: - parent = dict(parent) - for key, value in locals.iteritems(): - if key[:2] == 'l_' and value is not missing: - parent[key[2:]] = value - return Context(environment, parent, template_name, blocks) - - -class TemplateReference(object): - """The `self` in templates.""" - - def __init__(self, context): - self.__context = context - - def __getitem__(self, name): - blocks = self.__context.blocks[name] - wrap = self.__context.eval_ctx.autoescape and \ - Markup or (lambda x: x) - return BlockReference(name, self.__context, blocks, 0) - - def __repr__(self): - return '<%s %r>' % ( - self.__class__.__name__, - self.__context.name - ) - - -class Context(object): - """The template context holds the variables of a template. It stores the - values passed to the template and also the names the template exports. - Creating instances is neither supported nor useful as it's created - automatically at various stages of the template evaluation and should not - be created by hand. - - The context is immutable. Modifications on :attr:`parent` **must not** - happen and modifications on :attr:`vars` are allowed from generated - template code only. Template filters and global functions marked as - :func:`contextfunction`\s get the active context passed as first argument - and are allowed to access the context read-only. - - The template context supports read only dict operations (`get`, - `keys`, `values`, `items`, `iterkeys`, `itervalues`, `iteritems`, - `__getitem__`, `__contains__`). Additionally there is a :meth:`resolve` - method that doesn't fail with a `KeyError` but returns an - :class:`Undefined` object for missing variables. - """ - __slots__ = ('parent', 'vars', 'environment', 'eval_ctx', 'exported_vars', - 'name', 'blocks', '__weakref__') - - def __init__(self, environment, parent, name, blocks): - self.parent = parent - self.vars = {} - self.environment = environment - self.eval_ctx = EvalContext(self.environment, name) - self.exported_vars = set() - self.name = name - - # create the initial mapping of blocks. Whenever template inheritance - # takes place the runtime will update this mapping with the new blocks - # from the template. - self.blocks = dict((k, [v]) for k, v in blocks.iteritems()) - - def super(self, name, current): - """Render a parent block.""" - try: - blocks = self.blocks[name] - index = blocks.index(current) + 1 - blocks[index] - except LookupError: - return self.environment.undefined('there is no parent block ' - 'called %r.' % name, - name='super') - return BlockReference(name, self, blocks, index) - - def get(self, key, default=None): - """Returns an item from the template context, if it doesn't exist - `default` is returned. - """ - try: - return self[key] - except KeyError: - return default - - def resolve(self, key): - """Looks up a variable like `__getitem__` or `get` but returns an - :class:`Undefined` object with the name of the name looked up. - """ - if key in self.vars: - return self.vars[key] - if key in self.parent: - return self.parent[key] - return self.environment.undefined(name=key) - - def get_exported(self): - """Get a new dict with the exported variables.""" - return dict((k, self.vars[k]) for k in self.exported_vars) - - def get_all(self): - """Return a copy of the complete context as dict including the - exported variables. - """ - return dict(self.parent, **self.vars) - - @internalcode - def call(__self, __obj, *args, **kwargs): - """Call the callable with the arguments and keyword arguments - provided but inject the active context or environment as first - argument if the callable is a :func:`contextfunction` or - :func:`environmentfunction`. - """ - if __debug__: - __traceback_hide__ = True - if isinstance(__obj, _context_function_types): - if getattr(__obj, 'contextfunction', 0): - args = (__self,) + args - elif getattr(__obj, 'evalcontextfunction', 0): - args = (__self.eval_ctx,) + args - elif getattr(__obj, 'environmentfunction', 0): - args = (__self.environment,) + args - try: - return __obj(*args, **kwargs) - except StopIteration: - return __self.environment.undefined('value was undefined because ' - 'a callable raised a ' - 'StopIteration exception') - - def derived(self, locals=None): - """Internal helper function to create a derived context.""" - context = new_context(self.environment, self.name, {}, - self.parent, True, None, locals) - context.eval_ctx = self.eval_ctx - context.blocks.update((k, list(v)) for k, v in self.blocks.iteritems()) - return context - - def _all(meth): - proxy = lambda self: getattr(self.get_all(), meth)() - proxy.__doc__ = getattr(dict, meth).__doc__ - proxy.__name__ = meth - return proxy - - keys = _all('keys') - values = _all('values') - items = _all('items') - - # not available on python 3 - if hasattr(dict, 'iterkeys'): - iterkeys = _all('iterkeys') - itervalues = _all('itervalues') - iteritems = _all('iteritems') - del _all - - def __contains__(self, name): - return name in self.vars or name in self.parent - - def __getitem__(self, key): - """Lookup a variable or raise `KeyError` if the variable is - undefined. - """ - item = self.resolve(key) - if isinstance(item, Undefined): - raise KeyError(key) - return item - - def __repr__(self): - return '<%s %s of %r>' % ( - self.__class__.__name__, - repr(self.get_all()), - self.name - ) - - -# register the context as mapping if possible -try: - from collections import Mapping - Mapping.register(Context) -except ImportError: - pass - - -class BlockReference(object): - """One block on a template reference.""" - - def __init__(self, name, context, stack, depth): - self.name = name - self._context = context - self._stack = stack - self._depth = depth - - @property - def super(self): - """Super the block.""" - if self._depth + 1 >= len(self._stack): - return self._context.environment. \ - undefined('there is no parent block called %r.' % - self.name, name='super') - return BlockReference(self.name, self._context, self._stack, - self._depth + 1) - - @internalcode - def __call__(self): - rv = concat(self._stack[self._depth](self._context)) - if self._context.eval_ctx.autoescape: - rv = Markup(rv) - return rv - - -class LoopContext(object): - """A loop context for dynamic iteration.""" - - def __init__(self, iterable, recurse=None): - self._iterator = iter(iterable) - self._recurse = recurse - self.index0 = -1 - - # try to get the length of the iterable early. This must be done - # here because there are some broken iterators around where there - # __len__ is the number of iterations left (i'm looking at your - # listreverseiterator!). - try: - self._length = len(iterable) - except (TypeError, AttributeError): - self._length = None - - def cycle(self, *args): - """Cycles among the arguments with the current loop index.""" - if not args: - raise TypeError('no items for cycling given') - return args[self.index0 % len(args)] - - first = property(lambda x: x.index0 == 0) - last = property(lambda x: x.index0 + 1 == x.length) - index = property(lambda x: x.index0 + 1) - revindex = property(lambda x: x.length - x.index0) - revindex0 = property(lambda x: x.length - x.index) - - def __len__(self): - return self.length - - def __iter__(self): - return LoopContextIterator(self) - - @internalcode - def loop(self, iterable): - if self._recurse is None: - raise TypeError('Tried to call non recursive loop. Maybe you ' - "forgot the 'recursive' modifier.") - return self._recurse(iterable, self._recurse) - - # a nifty trick to enhance the error message if someone tried to call - # the the loop without or with too many arguments. - __call__ = loop - del loop - - @property - def length(self): - if self._length is None: - # if was not possible to get the length of the iterator when - # the loop context was created (ie: iterating over a generator) - # we have to convert the iterable into a sequence and use the - # length of that. - iterable = tuple(self._iterator) - self._iterator = iter(iterable) - self._length = len(iterable) + self.index0 + 1 - return self._length - - def __repr__(self): - return '<%s %r/%r>' % ( - self.__class__.__name__, - self.index, - self.length - ) - - -class LoopContextIterator(object): - """The iterator for a loop context.""" - __slots__ = ('context',) - - def __init__(self, context): - self.context = context - - def __iter__(self): - return self - - def next(self): - ctx = self.context - ctx.index0 += 1 - return next(ctx._iterator), ctx - - -class Macro(object): - """Wraps a macro function.""" - - def __init__(self, environment, func, name, arguments, defaults, - catch_kwargs, catch_varargs, caller): - self._environment = environment - self._func = func - self._argument_count = len(arguments) - self.name = name - self.arguments = arguments - self.defaults = defaults - self.catch_kwargs = catch_kwargs - self.catch_varargs = catch_varargs - self.caller = caller - - @internalcode - def __call__(self, *args, **kwargs): - # try to consume the positional arguments - arguments = list(args[:self._argument_count]) - off = len(arguments) - - # if the number of arguments consumed is not the number of - # arguments expected we start filling in keyword arguments - # and defaults. - if off != self._argument_count: - for idx, name in enumerate(self.arguments[len(arguments):]): - try: - value = kwargs.pop(name) - except KeyError: - try: - value = self.defaults[idx - self._argument_count + off] - except IndexError: - value = self._environment.undefined( - 'parameter %r was not provided' % name, name=name) - arguments.append(value) - - # it's important that the order of these arguments does not change - # if not also changed in the compiler's `function_scoping` method. - # the order is caller, keyword arguments, positional arguments! - if self.caller: - caller = kwargs.pop('caller', None) - if caller is None: - caller = self._environment.undefined('No caller defined', - name='caller') - arguments.append(caller) - if self.catch_kwargs: - arguments.append(kwargs) - elif kwargs: - raise TypeError('macro %r takes no keyword argument %r' % - (self.name, next(iter(kwargs)))) - if self.catch_varargs: - arguments.append(args[self._argument_count:]) - elif len(args) > self._argument_count: - raise TypeError('macro %r takes not more than %d argument(s)' % - (self.name, len(self.arguments))) - return self._func(*arguments) - - def __repr__(self): - return '<%s %s>' % ( - self.__class__.__name__, - self.name is None and 'anonymous' or repr(self.name) - ) - - -class Undefined(object): - """The default undefined type. This undefined type can be printed and - iterated over, but every other access will raise an :exc:`UndefinedError`: - - >>> foo = Undefined(name='foo') - >>> str(foo) - '' - >>> not foo - True - >>> foo + 42 - Traceback (most recent call last): - ... - UndefinedError: 'foo' is undefined - """ - __slots__ = ('_undefined_hint', '_undefined_obj', '_undefined_name', - '_undefined_exception') - - def __init__(self, hint=None, obj=missing, name=None, exc=UndefinedError): - self._undefined_hint = hint - self._undefined_obj = obj - self._undefined_name = name - self._undefined_exception = exc - - @internalcode - def _fail_with_undefined_error(self, *args, **kwargs): - """Regular callback function for undefined objects that raises an - `UndefinedError` on call. - """ - if self._undefined_hint is None: - if self._undefined_obj is missing: - hint = '%r is undefined' % self._undefined_name - elif not isinstance(self._undefined_name, basestring): - hint = '%s has no element %r' % ( - object_type_repr(self._undefined_obj), - self._undefined_name - ) - else: - hint = '%r has no attribute %r' % ( - object_type_repr(self._undefined_obj), - self._undefined_name - ) - else: - hint = self._undefined_hint - raise self._undefined_exception(hint) - - __add__ = __radd__ = __mul__ = __rmul__ = __div__ = __rdiv__ = \ - __truediv__ = __rtruediv__ = __floordiv__ = __rfloordiv__ = \ - __mod__ = __rmod__ = __pos__ = __neg__ = __call__ = \ - __getattr__ = __getitem__ = __lt__ = __le__ = __gt__ = __ge__ = \ - __int__ = __float__ = __complex__ = __pow__ = __rpow__ = \ - _fail_with_undefined_error - - def __str__(self): - return unicode(self).encode('utf-8') - - # unicode goes after __str__ because we configured 2to3 to rename - # __unicode__ to __str__. because the 2to3 tree is not designed to - # remove nodes from it, we leave the above __str__ around and let - # it override at runtime. - def __unicode__(self): - return u'' - - def __len__(self): - return 0 - - def __iter__(self): - if 0: - yield None - - def __nonzero__(self): - return False - - def __repr__(self): - return 'Undefined' - - -class DebugUndefined(Undefined): - """An undefined that returns the debug info when printed. - - >>> foo = DebugUndefined(name='foo') - >>> str(foo) - '{{ foo }}' - >>> not foo - True - >>> foo + 42 - Traceback (most recent call last): - ... - UndefinedError: 'foo' is undefined - """ - __slots__ = () - - def __unicode__(self): - if self._undefined_hint is None: - if self._undefined_obj is missing: - return u'{{ %s }}' % self._undefined_name - return '{{ no such element: %s[%r] }}' % ( - object_type_repr(self._undefined_obj), - self._undefined_name - ) - return u'{{ undefined value printed: %s }}' % self._undefined_hint - - -class StrictUndefined(Undefined): - """An undefined that barks on print and iteration as well as boolean - tests and all kinds of comparisons. In other words: you can do nothing - with it except checking if it's defined using the `defined` test. - - >>> foo = StrictUndefined(name='foo') - >>> str(foo) - Traceback (most recent call last): - ... - UndefinedError: 'foo' is undefined - >>> not foo - Traceback (most recent call last): - ... - UndefinedError: 'foo' is undefined - >>> foo + 42 - Traceback (most recent call last): - ... - UndefinedError: 'foo' is undefined - """ - __slots__ = () - __iter__ = __unicode__ = __str__ = __len__ = __nonzero__ = __eq__ = \ - __ne__ = __bool__ = Undefined._fail_with_undefined_error - - -# remove remaining slots attributes, after the metaclass did the magic they -# are unneeded and irritating as they contain wrong data for the subclasses. -del Undefined.__slots__, DebugUndefined.__slots__, StrictUndefined.__slots__