Christopher Armstrong has proposed merging lp:~radix/txaws/parameter-enrichment into lp:txaws.
Requested reviews: txAWS Technical List (txaws-tech) txAWS Technical List (txaws-tech) Related bugs: Bug #984660 in txAWS: "Enrich schema declarations to allow for describing all the details of an API" https://bugs.launchpad.net/txaws/+bug/984660 For more details, see: https://code.launchpad.net/~radix/txaws/parameter-enrichment/+merge/103592 This is one part of schema enrichment. It adds List and Structure Parameter types, and significantly refactors the way that paramater parsing and formatting is done. -- https://code.launchpad.net/~radix/txaws/parameter-enrichment/+merge/103592 Your team txAWS Technical List is requested to review the proposed merge of lp:~radix/txaws/parameter-enrichment into lp:txaws.
=== modified file 'txaws/server/schema.py' --- txaws/server/schema.py 2012-03-27 11:51:09 +0000 +++ txaws/server/schema.py 2012-04-26 01:41:19 +0000 @@ -26,6 +26,12 @@ super(MissingParameterError, self).__init__(message) +class InconsistentParameterError(SchemaError): + def __init__(self, name): + message = "Parameter %s is used inconsistently" % (name,) + super(InconsistentParameterError, self).__init__(message) + + class InvalidParameterValueError(SchemaError): """Raised when the value of a parameter is invalid.""" @@ -51,6 +57,20 @@ super(UnknownParameterError, self).__init__(message) +class UnknownParametersError(Exception): + """ + Raised when extra unknown fields are passed to L{Structure.parse}. + + @ivar result: The already coerced result representing the known parameters. + @ivar unknown: The unknown parameters. + """ + def __init__(self, result, unknown): + self.result = result + self.unknown = unknown + message = "The parameters %s are not recognized" % (unknown,) + super(UnknownParametersError, self).__init__(message) + + class Parameter(object): """A single parameter in an HTTP request. @@ -67,7 +87,9 @@ @param validator: A callable to validate the parameter, returning a bool. """ - def __init__(self, name, optional=False, default=None, + supports_multiple = False + + def __init__(self, name=None, optional=False, default=None, min=None, max=None, allow_none=False, validator=None): self.name = name self.optional = optional @@ -182,7 +204,7 @@ lower_than_min_template = "Value must be at least %s." greater_than_max_template = "Value exceeds maximum of %s." - def __init__(self, name, optional=False, default=None, + def __init__(self, name=None, optional=False, default=None, min=0, max=None, allow_none=False, validator=None): super(Integer, self).__init__(name, optional, default, min, max, allow_none, validator) @@ -228,8 +250,10 @@ kind = "enum" - def __init__(self, name, mapping, optional=False, default=None): + def __init__(self, name=None, mapping=None, optional=False, default=None): super(Enum, self).__init__(name, optional=optional, default=default) + if mapping is None: + raise TypeError("Must provide mapping") self.mapping = mapping self.reverse = dict((value, key) for key, value in mapping.iteritems()) @@ -260,18 +284,145 @@ return datetime.strftime(utc_value, "%Y-%m-%dT%H:%M:%SZ") +class List(Parameter): + """ + A homogenous list of instances of a parameterized type. + + There is a strange behavior that lists can have any starting index and any + gaps are ignored. Conventionally they are 1-based, and so indexes proceed + like 1, 2, 3... However, any non-negative index can be used and the + ordering will be used to determine the true index. So:: + + {5: 'a', 7: 'b', 9: 'c'} + + becomes:: + + ['a', 'b', 'c'] + """ + + kind = "list" + supports_multiple = True + + def __init__(self, name=None, item=None, optional=False, default=None): + """ + @param item: A L{Parameter} instance which will be used to parse and + format the values in the list. + """ + if item is None: + raise TypeError("Must provide item") + super(List, self).__init__(name, optional=optional, default=default) + self.item = item + + def parse(self, value): + """ + Convert a dictionary of {relative index: value} to a list of parsed + C{value}s. + """ + indices = [] + if not isinstance(value, dict): + raise InvalidParameterValueError("%r should be a dict." % (value,)) + for index in value.keys(): + try: + indices.append(int(index)) + except ValueError: + raise UnknownParameterError(index) + result = [None] * len(value) + for index_index, index in enumerate(sorted(indices)): + v = value[str(index)] + if index < 0: + raise UnknownParameterError(index) + result[index_index] = self.item.coerce(v) + return result + + def format(self, value): + """ + Convert a list like:: + + ["a", "b", "c"] + + to: + + {"1": "a", "2": "b", "3": "c"} + + C{value} may also be an L{Arguments} instance, mapping indices to + values. Who knows why. + """ + if isinstance(value, Arguments): + return dict((str(i), self.item.format(v)) for i, v in value) + return dict((str(i + 1), self.item.format(v)) + for i, v in enumerate(value)) + + +class Structure(Parameter): + """ + A structure with named fields of parameterized types. + """ + + kind = "structure" + supports_multiple = True + + def __init__(self, name=None, fields=None, optional=False, default=None): + """ + @param fields: A mapping of field name to field L{Parameter} instance. + """ + if fields is None: + raise TypeError("Must provide fields") + super(Structure, self).__init__(name, optional=optional, + default=default) + self.fields = fields + + def parse(self, value): + """ + Convert a dictionary of raw values to a dictionary of processed values. + """ + result = {} + rest = {} + for k, v in value.iteritems(): + if k in self.fields: + if (isinstance(v, dict) + and not self.fields[k].supports_multiple): + if len(v) == 1: + # We support "foo.1" as "foo" as long as there is only + # one "foo.#" parameter provided.... -_- + v = v.values()[0] + else: + raise InvalidParameterCombinationError(k) + result[k] = self.fields[k].coerce(v) + else: + rest[k] = v + for k, v in self.fields.iteritems(): + if k not in result: + result[k] = v.coerce(None) + if rest: + raise UnknownParametersError(result, rest) + return result + + def format(self, value): + """ + Convert a dictionary of processed values to a dictionary of raw values. + """ + if not isinstance(value, Arguments): + value = value.iteritems() + return dict((k, self.fields[k].format(v)) for k, v in value) + + class Arguments(object): """Arguments parsed from a request.""" def __init__(self, tree): """Initialize a new L{Arguments} instance. - @param tree: The C{dict}-based structure of the L{Argument}instance + @param tree: The C{dict}-based structure of the L{Argument} instance to create. """ for key, value in tree.iteritems(): self.__dict__[key] = self._wrap(value) + def __str__(self): + return "Arguments(%s)" % (self.__dict__,) + + __repr__ = __str__ + def __iter__(self): """Returns an iterator yielding C{(name, value)} tuples.""" return self.__dict__.iteritems() @@ -288,7 +439,7 @@ """Wrap the given L{tree} with L{Arguments} as necessary. @param tree: A {dict}, containing L{dict}s and/or leaf values, nested - arbitrarily deep. + arbitrarily deep. """ if isinstance(value, dict): if any(isinstance(name, int) for name in value.keys()): @@ -299,6 +450,8 @@ return [self._wrap(value) for (name, value) in items] else: return Arguments(value) + elif isinstance(value, list): + return [self._wrap(x) for x in value] else: return value @@ -308,7 +461,7 @@ The schema that the arguments of an HTTP request must be compliant with. """ - def __init__(self, *parameters): + def __init__(self, *_parameters, **kwargs): """Initialize a new L{Schema} instance. Any number of L{Parameter} instances can be passed. The parameter path @@ -323,60 +476,34 @@ A more complex example:: - schema = Schema(Unicode('Name.#')) + schema = Schema(List('Names', item=Unicode())) - means that the result of L{Schema.extract} would have a C{Name} + means that the result of L{Schema.extract} would have a C{Names} attribute, which would itself contain a list of names. Similarly, - L{Schema.bundle} would look for a C{Name} attribute. + L{Schema.bundle} would look for a C{Names} attribute. """ - self._parameters = dict( - (self._get_template(parameter.name), parameter) - for parameter in parameters) + if 'parameters' in kwargs: + if len(_parameters) > 0: + raise TypeError("parameters= must only be passed " + "without positional arguments") + self._parameters = kwargs['parameters'] + else: + self._parameters = self._convert_old_schema(_parameters) def extract(self, params): """Extract parameters from a raw C{dict} according to this schema. @param params: The raw parameters to parse. - @return: An L{Arguments} object holding the extracted arguments. - - @raises UnknownParameterError: If C{params} contains keys that this - schema doesn't know about. + @return: A tuple of an L{Arguments} object holding the extracted + arguments and any unparsed arguments. """ - tree = {} - rest = {} - - # Extract from the given arguments and parse according to the - # corresponding parameters. - for name, value in params.iteritems(): - template = self._get_template(name) - parameter = self._parameters.get(template) - - if template.endswith(".#") and parameter is None: - # If we were unable to find a direct match for a template that - # allows multiple values. Let's attempt to find it without the - # multiple value marker which Amazon allows. For example if the - # template is 'PublicIp', then a single key 'PublicIp.1' is - # allowed. - parameter = self._parameters.get(template[:-2]) - if parameter is not None: - name = name[:-2] - - # At this point, we have a template that doesn't have the .# - # marker to indicate multiple values. We don't allow multiple - # "single" values for the same element. - if name in tree.keys(): - raise InvalidParameterCombinationError(name) - - if parameter is None: - rest[name] = value - else: - self._set_value(tree, name, parameter.coerce(value)) - - # Ensure that the tree arguments are consistent with constraints - # defined in the schema. - for template, parameter in self._parameters.iteritems(): - self._ensure_tree(tree, parameter, *template.split(".")) - + structure = Structure(fields=self._parameters) + try: + tree = structure.coerce(self._convert_flat_to_nest(params)) + rest = {} + except UnknownParametersError, error: + tree = error.result + rest = error.unknown return Arguments(tree), rest def bundle(self, *arguments, **extra): @@ -390,117 +517,86 @@ params = {} for argument in arguments: - self._flatten(params, argument) - self._flatten(params, extra) + params.update(argument) + params.update(extra) + result = {} for name, value in params.iteritems(): - parameter = self._parameters.get(self._get_template(name)) + if value is None: + continue + segments = name.split('.') + first = segments[0] + parameter = self._parameters.get(first) if parameter is None: raise RuntimeError("Parameter '%s' not in schema" % name) else: if value is None: - params[name] = "" - else: - params[name] = parameter.format(value) - - return params - - def _get_template(self, key): - """Return the canonical template for a given parameter key. - - For example:: - - 'Child.1.Name.2' - - becomes:: - - 'Child.#.Name.#' - - """ - parts = key.split(".") - for index, part in enumerate(parts[1::2]): - parts[index * 2 + 1] = "#" - return ".".join(parts) - - def _set_value(self, tree, path, value): - """Set C{value} at C{path} in the given C{tree}. - - For example:: - - tree = {} - _set_value(tree, 'foo.1.bar.2', True) - - results in C{tree} becoming:: - - {'foo': {1: {'bar': {2: True}}}} - - @param tree: A L{dict}. - @param path: A L{str}. - @param value: The value to set. Can be anything. - """ - nodes = [] - for index, node in enumerate(path.split(".")): - if index % 2: - # Nodes with odd indexes must be non-negative integers - try: - node = int(node) - except ValueError: - raise UnknownParameterError(path) - if node < 0: - raise UnknownParameterError(path) - nodes.append(node) - for node in nodes[:-1]: - tree = tree.setdefault(node, {}) - tree[nodes[-1]] = value - - def _ensure_tree(self, tree, parameter, node, *nodes): - """Check that C{node} exists in C{tree} and is followed by C{nodes}. - - C{node} and C{nodes} should correspond to a template path (i.e. where - there are no absolute indexes, but C{#} instead). - """ - if node == "#": - if len(nodes) == 0: - if len(tree.keys()) == 0 and not parameter.optional: - raise MissingParameterError(parameter.name) - else: - for subtree in tree.itervalues(): - self._ensure_tree(subtree, parameter, *nodes) - else: - if len(nodes) == 0: - if node not in tree.keys(): - # No value for this parameter is present, if it's not - # optional nor allow_none is set, the call below will - # raise a MissingParameterError - tree[node] = parameter.coerce(None) - else: - if node not in tree.keys(): - tree[node] = {} - self._ensure_tree(tree[node], parameter, *nodes) - - def _flatten(self, params, tree, path=""): - """ - For every element in L{tree}, set C{path} to C{value} in the given - L{params} dictionary. - - @param params: A L{dict} which will be populated. - @param tree: A structure made up of L{Argument}s, L{list}s, L{dict}s - and leaf values. - """ - if isinstance(tree, Arguments): - for name, value in tree: - self._flatten(params, value, "%s.%s" % (path, name)) - elif isinstance(tree, dict): - for name, value in tree.iteritems(): - self._flatten(params, value, "%s.%s" % (path, name)) - elif isinstance(tree, list): - for index, value in enumerate(tree): - self._flatten(params, value, "%s.%d" % (path, index + 1)) - elif tree is not None: - params[path.lstrip(".")] = tree - else: - # None is discarded. - pass + result[name] = "" + else: + result[name] = parameter.format(value) + + return self._convert_nest_to_flat(result) + + def _convert_flat_to_nest(self, params): + """ + Convert a structure in the form of:: + + {'foo.1.bar': 'value', + 'foo.2.baz': 'value'} + + to:: + + {'foo': {'1': {'bar': 'value'}, + '2': {'baz': 'value'}}} + + This is intended for use both during parsing of HTTP arguments like + 'foo.1.bar=value' and when dealing with schema declarations that look + like 'foo.n.bar'. + + This is the inverse of L{_convert_nest_to_flat}. + """ + result = {} + for k, v in params.iteritems(): + last = result + segments = k.split('.') + for index, item in enumerate(segments): + if index == len(segments) - 1: + newd = v + else: + newd = {} + if not isinstance(last, dict): + raise InconsistentParameterError(k) + if type(last.get(item)) is dict and type(newd) is not dict: + raise InconsistentParameterError(k) + last = last.setdefault(item, newd) + return result + + def _convert_nest_to_flat(self, params, _result=None, _prefix=None): + """ + Convert a data structure that looks like:: + + {"foo": {"bar": "baz", "shimmy": "sham"}} + + to:: + + {"foo.bar": "baz", + "foo.shimmy": "sham"} + + This is the inverse of L{_convert_flat_to_nest}. + """ + if _result is None: + _result = {} + for k, v in params.iteritems(): + if _prefix is None: + path = k + else: + path = _prefix + '.' + k + if isinstance(v, dict): + return self._convert_nest_to_flat(v, _result=_result, + _prefix=path) + else: + _result[path] = v + return _result def extend(self, *schema_items): """ @@ -513,3 +609,49 @@ else: raise TypeError("Illegal argument %s" % item) return Schema(*parameters) + + def _convert_old_schema(self, parameters): + """ + Convert an ugly old schema, using dotted names, to the hot new schema, + using List and Structure. + + The old schema assumes that every other dot implies an array. So a list + of two parameters, + + [Integer("foo.bar.baz.quux"), Integer("foo.bar.shimmy")] + + becomes:: + + {"foo": List( + item=Structure( + fields={"baz": List(item=Integer()), + "shimmy": Integer()}))} + + By design, the old schema syntax ignored the names "bar" and "quux". + """ + crap = {} + for parameter in parameters: + crap[parameter.name] = parameter + nest = self._convert_flat_to_nest(crap) + return self._secret_convert_old_schema(nest, 0).fields + + def _secret_convert_old_schema(self, mapping, depth): + """ + Internal recursion helper for L{_convert_old_schema}. + """ + if not isinstance(mapping, dict): + return mapping + if depth % 2 == 0: + fields = {} + for k, v in mapping.iteritems(): + fields[k] = self._secret_convert_old_schema(v, depth + 1) + return Structure(fields=fields) + else: + if not isinstance(mapping, dict): + raise TypeError("mapping %r must be a dict" % (mapping,)) + if not len(mapping) == 1: + raise ValueError("mapping %r must only have one element" + % (mapping,)) + item = mapping.values()[0] + item = self._secret_convert_old_schema(item, depth + 1) + return List(item=item) === modified file 'txaws/server/tests/test_schema.py' --- txaws/server/tests/test_schema.py 2012-03-27 12:01:45 +0000 +++ txaws/server/tests/test_schema.py 2012-04-26 01:41:19 +0000 @@ -8,7 +8,9 @@ from txaws.server.exception import APIError from txaws.server.schema import ( - Arguments, Bool, Date, Enum, Integer, Parameter, RawStr, Schema, Unicode) + Arguments, Bool, Date, Enum, Integer, Parameter, RawStr, Schema, Unicode, + List, Structure, + InconsistentParameterError, InvalidParameterValueError) class ArgumentsTestCase(TestCase): @@ -395,6 +397,26 @@ self.assertEqual(u"value", arguments.name) self.assertEqual(None, arguments.count) + def test_extract_with_optional_default(self): + """ + The value of C{default} on a parameter is used as the value when it is + not provided as an argument and the parameter is C{optional}. + """ + schema = Schema(Unicode("name"), + Integer("count", optional=True, default=5)) + arguments, _ = schema.extract({"name": "value"}) + self.assertEqual(u"value", arguments.name) + self.assertEqual(5, arguments.count) + + def test_extract_structure_with_optional(self): + """L{Schema.extract} can handle optional parameters.""" + schema = Schema( + Structure( + "struct", + fields={"name": Unicode(optional=True, default="radix")})) + arguments, _ = schema.extract({"struct": {}}) + self.assertEqual(u"radix", arguments.struct.name) + def test_extract_with_numbered(self): """ L{Schema.extract} can handle parameters with numbered values. @@ -404,6 +426,16 @@ self.assertEqual("Joe", arguments.name[0]) self.assertEqual("Tom", arguments.name[1]) + def test_extract_with_goofy_numbered(self): + """ + L{Schema.extract} only uses the relative values of indices to determine + the index in the resultant list. + """ + schema = Schema(Unicode("name.n")) + arguments, _ = schema.extract({"name.5": "Joe", "name.10": "Tom"}) + self.assertEqual("Joe", arguments.name[0]) + self.assertEqual("Tom", arguments.name[1]) + def test_extract_with_single_numbered(self): """ L{Schema.extract} can handle a single parameter with a numbered value. @@ -458,8 +490,8 @@ given without an index. """ schema = Schema(Unicode("name.n")) - _, rest = schema.extract({"name": "foo", "name.1": "bar"}) - self.assertEqual(rest, {"name": "foo"}) + self.assertRaises(InconsistentParameterError, + schema.extract, {"name": "foo", "name.1": "bar"}) def test_extract_with_non_numbered_template(self): """ @@ -480,7 +512,7 @@ error = self.assertRaises(APIError, schema.extract, params) self.assertEqual(400, error.status) self.assertEqual("UnknownParameter", error.code) - self.assertEqual("The parameter name.one is not recognized", + self.assertEqual("The parameter one is not recognized", error.message) def test_extract_with_negative_index(self): @@ -493,7 +525,7 @@ error = self.assertRaises(APIError, schema.extract, params) self.assertEqual(400, error.status) self.assertEqual("UnknownParameter", error.code) - self.assertEqual("The parameter name.-1 is not recognized", + self.assertEqual("The parameter -1 is not recognized", error.message) def test_bundle(self): @@ -524,7 +556,7 @@ L{Schema.bundle} correctly handles an empty numbered arguments list. """ schema = Schema(Unicode("name.n")) - params = schema.bundle(names=[]) + params = schema.bundle(name=[]) self.assertEqual({}, params) def test_bundle_with_numbered_not_supplied(self): @@ -544,6 +576,41 @@ self.assertEqual({"name.1": "Foo", "name.2": "Bar", "count": "123"}, params) + def test_bundle_with_structure(self): + """L{Schema.bundle} can bundle L{Structure}s.""" + schema = Schema( + parameters={ + "struct": Structure(fields={"field1": Unicode(), + "field2": Integer()})}) + params = schema.bundle(struct={"field1": "hi", "field2": 59}) + self.assertEqual({"struct.field1": "hi", "struct.field2": "59"}, + params) + + def test_bundle_with_list(self): + """L{Schema.bundle} can bundle L{List}s.""" + schema = Schema(parameters={"things": List(item=Unicode())}) + params = schema.bundle(things=["foo", "bar"]) + self.assertEqual({"things.1": "foo", "things.2": "bar"}, params) + + def test_bundle_with_structure_with_arguments(self): + """ + L{Schema.bundle} can bundle L{Structure}s (specified as L{Arguments}). + """ + schema = Schema( + parameters={ + "struct": Structure(fields={"field1": Unicode(), + "field2": Integer()})}) + params = schema.bundle(struct=Arguments({"field1": "hi", + "field2": 59})) + self.assertEqual({"struct.field1": "hi", "struct.field2": "59"}, + params) + + def test_bundle_with_list_with_arguments(self): + """L{Schema.bundle} can bundle L{List}s (specified as L{Arguments}).""" + schema = Schema(parameters={"things": List(item=Unicode())}) + params = schema.bundle(things=Arguments({1: "foo", 2: "bar"})) + self.assertEqual({"things.1": "foo", "things.2": "bar"}, params) + def test_bundle_with_arguments(self): """L{Schema.bundle} can bundle L{Arguments} too.""" schema = Schema(Unicode("name.n"), Integer("count")) @@ -590,3 +657,77 @@ self.assertEqual(u"value", arguments.name) self.assertEqual("testing", arguments.computer) self.assertEqual(5, arguments.count) + + def test_list(self): + """L{List}s can be extracted.""" + schema = Schema(List("foo", Integer())) + arguments, _ = schema.extract({"foo.1": "1", "foo.2": "2"}) + self.assertEqual([1, 2], arguments.foo) + + def test_non_list(self): + """ + When a non-list argument is passed to a L{List} parameter, a + L{InvalidParameterValueError} is raised. + """ + schema = Schema(List("name", Unicode())) + self.assertRaises(InvalidParameterValueError, + schema.extract, {"name": "foo"}) + + def test_list_of_list(self): + """L{List}s can be nested.""" + schema = Schema(List("foo", List(item=Unicode()))) + arguments, _ = schema.extract( + {"foo.1.1": "first-first", "foo.1.2": "first-second", + "foo.2.1": "second-first", "foo.2.2": "second-second"}) + self.assertEqual([["first-first", "first-second"], + ["second-first", "second-second"]], + arguments.foo) + + def test_structure(self): + """ + L{Schema}s with L{Structure} parameters can have arguments extracted. + """ + schema = Schema(Structure("foo", {"a": Integer(), "b": Integer()})) + arguments, _ = schema.extract({"foo.a": "1", "foo.b": "2"}) + self.assertEqual(1, arguments.foo.a) + self.assertEqual(2, arguments.foo.b) + + def test_structure_of_structures(self): + """L{Structure}s can be nested.""" + sub_struct = Structure(fields={"a": Unicode(), "b": Unicode()}) + schema = Schema(Structure("foo", fields={"a": sub_struct, + "b": sub_struct})) + arguments, _ = schema.extract({"foo.a.a": "a-a", "foo.a.b": "a-b", + "foo.b.a": "b-a", "foo.b.b": "b-b"}) + self.assertEqual("a-a", arguments.foo.a.a) + self.assertEqual("a-b", arguments.foo.a.b) + self.assertEqual("b-a", arguments.foo.b.a) + self.assertEqual("b-b", arguments.foo.b.b) + + def test_list_of_structures(self): + """L{List}s of L{Structure}s are extracted properly.""" + schema = Schema( + List("foo", Structure(fields={"a": Integer(), "b": Integer()}))) + arguments, _ = schema.extract({"foo.1.a": "1", "foo.1.b": "2", + "foo.2.a": "3", "foo.2.b": "4"}) + self.assertEqual(1, arguments.foo[0]['a']) + self.assertEqual(2, arguments.foo[0]['b']) + self.assertEqual(3, arguments.foo[1]['a']) + self.assertEqual(4, arguments.foo[1]['b']) + + def test_structure_of_list(self): + """L{Structure}s of L{List}s are extracted properly.""" + schema = Schema(Structure("foo", fields={"l": List(item=Integer())})) + arguments, _ = schema.extract({"foo.l.1": "1", "foo.l.2": "2"}) + self.assertEqual([1, 2], arguments.foo.l) + + def test_new_parameters(self): + """ + L{Schema} accepts a C{parameters} parameter to specify parameters in a + {name: field} format. + """ + schema = Schema( + parameters={"foo": Structure( + fields={"l": List(item=Integer())})}) + arguments, _ = schema.extract({"foo.l.1": "1", "foo.l.2": "2"}) + self.assertEqual([1, 2], arguments.foo.l)
_______________________________________________ Mailing list: https://launchpad.net/~txaws-dev Post to : txaws-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~txaws-dev More help : https://help.launchpad.net/ListHelp