Various changes to support further high-level automation efforts. * added a RESTful interface for TKO. right now there's only a single, simple resource for accessing test attributes. * extended the REST server library in a few ways, most notably to support * querying on keyvals, with something like ?has_keyval=mykey=myvalue&... * operators, delimited by a colon, like ?hostname:in=host1,host2,host3 * loading relationships over many items efficiently (see InstanceEntry.prepare_for_full_representation()). this is used to fill in keyvals when requesting a job listing, but it can (and should) be used in other places, such as listing labels for a host collection. * loading a collection with inlined full representations, by passing full_representations=true * added various features to the AFE RESTful interface as necessary. * various fixes to the rest_client library, most notably * changed HTTP client in rest_client.py to use DI rather than singleton, easing testability. the same should be done for _get_request_headers(), to be honest. * better support for query params, including accepting a MultiValueDict and supporting URIs that already have query args * basic support for redirects * builtin support for requesting a full collection (get_full()), when clients explicitly expect the result not to be paged. i'm still considering alternative approaches to this -- it may make sense to have something like this be the default, and have clients set a default page size limit rather than passing it every time. * minor change to mock.py to provide better debugging output.
Signed-off-by: Steve Howard <[email protected]> --- autotest/client/common_lib/test_utils/mock.py 2010-04-08 13:32:10.000000000 -0700 +++ autotest/client/common_lib/test_utils/mock.py 2010-04-08 13:32:10.000000000 -0700 @@ -262,6 +262,10 @@ setattr(self, symbol, orig_symbol) + def __repr__(self): + return '<mock_class: %s>' % self.__name + + class mock_god: NONEXISTENT_ATTRIBUTE = object() --- autotest/frontend/afe/models.py 2010-04-08 13:32:10.000000000 -0700 +++ autotest/frontend/afe/models.py 2010-04-08 13:32:10.000000000 -0700 @@ -396,6 +396,7 @@ run_verify: Whether or not the scheduler should run the verify stage """ TestTime = enum.Enum('SHORT', 'MEDIUM', 'LONG', start_value=1) + TestTypes = model_attributes.TestTypes # TODO(showard) - this should be merged with Job.ControlType (but right # now they use opposite values) @@ -409,8 +410,7 @@ run_verify = dbmodels.BooleanField(default=True) test_time = dbmodels.SmallIntegerField(choices=TestTime.choices(), default=TestTime.MEDIUM) - test_type = dbmodels.SmallIntegerField( - choices=model_attributes.TestTypes.choices()) + test_type = dbmodels.SmallIntegerField(choices=TestTypes.choices()) sync_count = dbmodels.IntegerField(default=1) path = dbmodels.CharField(max_length=255, unique=True) --- autotest/frontend/afe/resources.py 2010-04-08 13:32:10.000000000 -0700 +++ autotest/frontend/afe/resources.py 2010-04-08 13:32:10.000000000 -0700 @@ -109,7 +109,9 @@ def update(self, input_dict): # TODO update atomic group - raise NotImplementedError + if 'is_platform' in input_dict: + self.instance.platform = input_dict['is_platform'] + self.instance.save() class LabelCollection(resource_lib.Collection): @@ -352,6 +354,11 @@ @classmethod + def add_query_selectors(cls, query_processor): + query_processor.add_field_selector('name') + + + @classmethod def from_uri_args(cls, request, test_name, **kwargs): return cls(request, models.Test.objects.get(name=test_name)) @@ -374,6 +381,7 @@ model_attributes.TestTypes.get_string( self.instance.test_type), 'control_file_path': self.instance.path, + 'sync_count': self.instance.sync_count, 'dependencies': TestDependencyCollection(fixed_entry=self).link(), }) @@ -599,6 +607,8 @@ query_lib.Selector('status', doc='One of queued, active or complete'), Job._StatusConstraint()) + query_processor.add_keyval_selector('has_keyval', models.JobKeyval, + 'key', 'value') @classmethod @@ -610,6 +620,12 @@ return {'job_id': self.instance.id} + @classmethod + def _do_prepare_for_full_representation(cls, instances): + models.Job.objects.populate_relationships(instances, models.JobKeyval, + 'keyvals') + + def short_representation(self): rep = super(Job, self).short_representation() rep.update({'id': self.instance.id, @@ -633,6 +649,8 @@ 'execution_info': ExecutionInfo.execution_info_from_job(self.instance), 'queue_entries': queue_entries.link(), + 'keyvals': dict((keyval.key, keyval.value) + for keyval in self.instance.keyvals) }) return rep --- autotest/frontend/afe/resources_test.py 2010-04-08 13:32:10.000000000 -0700 +++ autotest/frontend/afe/resources_test.py 2010-04-08 13:32:10.000000000 -0700 @@ -1,37 +1,21 @@ #!/usr/bin/python import common -import operator, unittest -import simplejson +import unittest from autotest_lib.frontend import setup_django_environment from autotest_lib.frontend import setup_test_environment from django.test import client -from autotest_lib.frontend.afe import control_file, frontend_test_utils, models -from autotest_lib.frontend.afe import model_attributes +from autotest_lib.frontend.shared import resource_test_utils +from autotest_lib.frontend.afe import control_file, models, model_attributes -class ResourceTestCase(unittest.TestCase, - frontend_test_utils.FrontendTestMixin): +class AfeResourceTestCase(resource_test_utils.ResourceTestCase): URI_PREFIX = 'http://testserver/afe/server/resources' CONTROL_FILE_CONTENTS = 'my control file contents' def setUp(self): - super(ResourceTestCase, self).setUp() - self._frontend_common_setup() - self._setup_debug_user() + super(AfeResourceTestCase, self).setUp() self._add_additional_data() - self.client = client.Client() - - - def tearDown(self): - super(ResourceTestCase, self).tearDown() - self._frontend_common_teardown() - - - def _setup_debug_user(self): - user = models.User.objects.create(login='debug_user') - acl = models.AclGroup.objects.get(name='my_acl') - user.aclgroup_set.add(acl) def _add_additional_data(self): @@ -40,110 +24,7 @@ path='/path/to/mytest') - def _expected_status(self, method): - if method == 'post': - return 201 - if method == 'delete': - return 204 - return 200 - - - def request(self, method, uri, **kwargs): - expected_status = self._expected_status(method) - - if method == 'put': - # the put() implementation in Django's test client is poorly - # implemented and only supports url-encoded keyvals for the data. - # the post() implementation is correct, though, so use that, with a - # trick to override the method. - method = 'post' - kwargs['REQUEST_METHOD'] = 'PUT' - if 'data' in kwargs: - kwargs.setdefault('content_type', 'application/json') - if kwargs['content_type'] == 'application/json': - kwargs['data'] = simplejson.dumps(kwargs['data']) - - client_method = getattr(self.client, method) - if uri.startswith('http://'): - full_uri = uri - else: - full_uri = self.URI_PREFIX + '/' + uri - response = client_method(full_uri, **kwargs) - self.assertEquals( - response.status_code, expected_status, - 'Requesting %s\nExpected %s, got %s: %s' - % (full_uri, expected_status, response.status_code, - response.content)) - - if response['content-type'] != 'application/json': - return response.content - - try: - return simplejson.loads(response.content) - except ValueError: - self.fail('Invalid reponse body: %s' % response.content) - - - def sorted_by(self, collection, attribute): - return sorted(collection, key=operator.itemgetter(attribute)) - - - def _read_attribute(self, item, attribute_or_list): - if isinstance(attribute_or_list, basestring): - attribute_or_list = [attribute_or_list] - for attribute in attribute_or_list: - item = item[attribute] - return item - - - def check_collection(self, collection, attribute_or_list, expected_list, - length=None, check_number=None): - """Check the members of a collection of dicts. - - @param collection: an iterable of dicts - @param attribute_or_list: an attribute or list of attributes to read. - the results will be sorted and compared with expected_list. if - a list of attributes is given, the attributes will be read - hierarchically, i.e. item[attribute1][attribute2]... - @param expected_list: list of expected values - @param check_number: if given, only check this number of entries - @param length: expected length of list, only necessary if check_number - is given - """ - actual_list = sorted(self._read_attribute(item, attribute_or_list) - for item in collection['members']) - if length is None and check_number is None: - length = len(expected_list) - if length is not None: - self.assertEquals(len(actual_list), length, - 'Expected %s, got %s: %s' - % (length, len(actual_list), - ', '.join(str(item) for item in actual_list))) - if check_number: - actual_list = actual_list[:check_number] - self.assertEquals(actual_list, expected_list) - - - def check_relationship(self, resource_uri, relationship_name, - other_entry_name, field, expected_values, - length=None, check_number=None): - """Check the members of a relationship collection. - - @param resource_uri: URI of base resource - @param relationship_name: name of relationship attribute on base - resource - @param other_entry_name: name of other entry in relationship - @param field: name of field to grab on other entry - @param expected values: list of expected values for the given field - """ - response = self.request('get', resource_uri) - relationship_uri = response[relationship_name]['href'] - relationships = self.request('get', relationship_uri) - self.check_collection(relationships, [other_entry_name, field], - expected_values, length, check_number) - - -class FilteringPagingTest(ResourceTestCase): +class FilteringPagingTest(AfeResourceTestCase): # we'll arbitarily choose to use hosts for this def setUp(self): @@ -159,6 +40,11 @@ self.check_collection(response, 'hostname', ['host1', 'host2']) + def test_in_filtering(self): + response = self.request('get', 'hosts?hostname:in=host1,host2') + self.check_collection(response, 'hostname', ['host1', 'host2']) + + def test_paging(self): response = self.request('get', 'hosts?start_index=1&items_per_page=2') self.check_collection(response, 'hostname', ['host2', 'host3']) @@ -167,13 +53,22 @@ self.assertEquals(response['start_index'], 1) -class MiscellaneousTest(ResourceTestCase): + def test_full_representations(self): + response = self.request( + 'get', 'hosts?hostname=host1&full_representations=true') + self.check_collection(response, 'hostname', ['host1']) + host = response['members'][0] + # invalid only included in full representation + self.assertEquals(host['invalid'], False) + + +class MiscellaneousTest(AfeResourceTestCase): def test_trailing_slash(self): response = self.request('get', 'hosts/host1/') self.assertEquals(response['hostname'], 'host1') -class AtomicGroupClassTest(ResourceTestCase): +class AtomicGroupClassTest(AfeResourceTestCase): def test_collection(self): response = self.request('get', 'atomic_group_classes') self.check_collection(response, 'name', ['atomic1', 'atomic2'], @@ -191,7 +86,7 @@ 'label', 'name', ['label4', 'label5']) -class LabelTest(ResourceTestCase): +class LabelTest(AfeResourceTestCase): def test_collection(self): response = self.request('get', 'labels') self.check_collection(response, 'name', ['label1', 'label2'], length=9, @@ -212,7 +107,7 @@ ['host1']) -class UserTest(ResourceTestCase): +class UserTest(AfeResourceTestCase): def test_collection(self): response = self.request('get', 'users') self.check_collection(response, 'username', @@ -242,7 +137,7 @@ self.check_collection(response, 'hostname', ['host1']) -class AclTest(ResourceTestCase): +class AclTest(AfeResourceTestCase): def test_collection(self): response = self.request('get', 'acls') self.check_collection(response, 'name', ['Everyone', 'my_acl']) @@ -263,7 +158,7 @@ ['host1', 'host2'], length=9, check_number=2) -class HostTest(ResourceTestCase): +class HostTest(AfeResourceTestCase): def test_collection(self): response = self.request('get', 'hosts') self.check_collection(response, 'hostname', ['host1', 'host2'], @@ -351,7 +246,7 @@ self.assertEquals(len(hosts), 0) -class TestTest(ResourceTestCase): # yes, we're testing the "tests" resource +class TestTest(AfeResourceTestCase): # yes, we're testing the "tests" resource def test_collection(self): response = self.request('get', 'tests') self.check_collection(response, 'name', ['mytest']) @@ -370,7 +265,7 @@ ['label3']) -class ExecutionInfoTest(ResourceTestCase): +class ExecutionInfoTest(AfeResourceTestCase): def setUp(self): super(ExecutionInfoTest, self).setUp() @@ -386,7 +281,7 @@ self.assertEquals(info['machines_per_execution'], 1) -class QueueEntriesRequestTest(ResourceTestCase): +class QueueEntriesRequestTest(AfeResourceTestCase): def test_get(self): response = self.request( 'get', @@ -406,7 +301,7 @@ self.assertEquals(entries, expected) -class JobTest(ResourceTestCase): +class JobTest(AfeResourceTestCase): def setUp(self): super(JobTest, self).setUp() @@ -417,16 +312,24 @@ job.control_file = self.CONTROL_FILE_CONTENTS job.save() + models.JobKeyval.objects.create(job=job, key='mykey', value='myvalue') + def test_collection(self): response = self.request('get', 'jobs') self.check_collection(response, 'id', [1, 2]) + def test_keyval_filtering(self): + response = self.request('get', 'jobs?has_keyval=mykey=myvalue') + self.check_collection(response, 'id', [1]) + + def test_entry(self): response = self.request('get', 'jobs/1') self.assertEquals(response['id'], 1) self.assertEquals(response['name'], 'test') + self.assertEquals(response['keyvals'], {'mykey': 'myvalue'}) info = response['execution_info'] self.assertEquals(info['control_file'], self.CONTROL_FILE_CONTENTS) self.assertEquals(info['is_server'], False) @@ -460,7 +363,7 @@ self.assertEquals(entries[1].host.hostname, 'host2') -class DirectoryTest(ResourceTestCase): +class DirectoryTest(AfeResourceTestCase): def test_get(self): response = self.request('get', '') for key in ('atomic_group_classes', 'labels', 'users', 'acl_groups', --- autotest/frontend/shared/query_lib.py 2010-04-08 13:32:10.000000000 -0700 +++ autotest/frontend/shared/query_lib.py 2010-04-08 13:32:10.000000000 -0700 @@ -22,7 +22,16 @@ def add_related_existence_selector(self, name, model, field, doc=None): self.add_selector( Selector(name, doc), - _RelatedExistenceConstraint(model, field, self.make_alias)) + _RelatedExistenceConstraint(model, field, + make_alias_fn=self.make_alias)) + + + def add_keyval_selector(self, name, model, key_field, value_field, + doc=None): + self.add_selector( + Selector(name, doc), + _KeyvalConstraint(model, key_field, value_field, + make_alias_fn=self.make_alias)) def add_selector(self, selector, constraint): @@ -46,7 +55,9 @@ def apply_selector(self, queryset, selector_name, value, - comparison_type='equals', is_inverse=False): + comparison_type=None, is_inverse=False): + if comparison_type is None: + comparison_type = 'equals' _, constraint = self._selectors[selector_name] try: return constraint.apply_constraint(queryset, value, comparison_type, @@ -103,6 +114,8 @@ kwarg_name = str(self._field + '__' + self._COMPARISON_MAP[comparison_type]) + if comparison_type == 'in': + value = value.split(',') if is_inverse: return queryset.exclude(**{kwarg_name: value}) @@ -138,3 +151,38 @@ queryset = queryset.model.objects.add_where(queryset, condition) return queryset + + +class _KeyvalConstraint(Constraint): + def __init__(self, model, key_field, value_field, make_alias_fn): + self._model = model + self._key_field = key_field + self._value_field = value_field + self._make_alias_fn = make_alias_fn + + + def apply_constraint(self, queryset, value, comparison_type, is_inverse): + if comparison_type not in (None, 'equals'): + raise ConstraintError('Can only use equals or not equals with ' + 'this selector') + if '=' not in value: + raise ConstraintError('You must specify a key=value pair for this ' + 'selector') + + key, actual_value = value.split('=', 1) + related_query = self._model.objects.filter( + **{self._key_field: key, self._value_field: actual_value}) + alias = self._make_alias_fn() + queryset = queryset.model.objects.join_custom_field(queryset, + related_query, + alias) + if is_inverse: + condition = '%s.%s IS NULL' + else: + condition = '%s.%s IS NOT NULL' + condition %= (alias, + queryset.model.objects.key_on_joined_table(related_query)) + + queryset = queryset.model.objects.add_where(queryset, condition) + + return queryset --- autotest/frontend/shared/resource_lib.py 2010-04-08 13:32:10.000000000 -0700 +++ autotest/frontend/shared/resource_lib.py 2010-04-08 13:32:10.000000000 -0700 @@ -131,10 +131,12 @@ def read_query_parameters(self, parameters): """Read relevant query parameters from a Django MultiValueDict.""" - for param_name, _ in self._query_parameters_accepted(): - if param_name in parameters: - self._query_params.setlist(param_name, - parameters.getlist(param_name)) + params_acccepted = set(param_name for param_name, _ + in self._query_parameters_accepted()) + for name, values in parameters.iterlists(): + base_name = name.split(':', 1)[0] + if base_name in params_acccepted: + self._query_params.setlist(name, values) def set_query_parameters(self, **parameters): @@ -216,6 +218,9 @@ except ValueError, exc: raise exceptions.BadRequest('Error decoding request body: ' '%s\n%r' % (exc, raw_data)) + if not isinstance(raw_dict, dict): + raise exceptions.BadRequest('Expected dict input, got %s: %r' % + (type(raw_dict), raw_dict)) elif content_type == 'application/x-www-form-urlencoded': cgi_dict = cgi.parse_qs(raw_data) # django won't do this for PUT raw_dict = {} @@ -319,6 +324,7 @@ assert self.model is not None super(Entry, self).__init__(request) self.instance = instance + self._is_prepared_for_full_representation = False @classmethod @@ -332,6 +338,41 @@ self.instance.delete() + def full_representation(self): + self.prepare_for_full_representation([self]) + return super(InstanceEntry, self).full_representation() + + + @classmethod + def prepare_for_full_representation(cls, entries): + """ + Prepare the given list of entries to generate full representations. + + This method delegates to _do_prepare_for_full_representation(), which + subclasses may override as necessary to do the actual processing. This + method also marks the instance as prepared, so it's safe to call this + multiple times with the same instance(s) without wasting work. + """ + not_prepared = [entry for entry in entries + if not entry._is_prepared_for_full_representation] + cls._do_prepare_for_full_representation([entry.instance + for entry in not_prepared]) + for entry in not_prepared: + entry._is_prepared_for_full_representation = True + + + @classmethod + def _do_prepare_for_full_representation(cls, instances): + """ + Subclasses may override this to gather data as needed for full + representations of the given model instances. Typically, this involves + querying over related objects, and this method offers a chance to query + for many instances at once, which can provide a great performance + benefit. + """ + pass + + class Collection(Resource): _DEFAULT_ITEMS_PER_PAGE = 50 @@ -354,7 +395,9 @@ def _query_parameters_accepted(self): params = [('start_index', 'Index of first member to include'), - ('items_per_page', 'Number of members to include')] + ('items_per_page', 'Number of members to include'), + ('full_representations', + 'True to include full representations of members')] for selector in self._query_processor.selectors(): params.append((selector.name, selector.doc)) return params @@ -371,16 +414,33 @@ def _representation(self, entry_instances): + entries = [self._entry_from_instance(instance) + for instance in entry_instances] + + want_full_representation = self._read_bool_parameter( + 'full_representations') + if want_full_representation: + self.entry_class.prepare_for_full_representation(entries) + members = [] - for instance in entry_instances: - entry = self._entry_from_instance(instance) - members.append(entry.short_representation()) + for entry in entries: + if want_full_representation: + rep = entry.full_representation() + else: + rep = entry.short_representation() + members.append(rep) rep = self.link() rep.update({'members': members}) return rep + def _read_bool_parameter(self, name): + if name not in self._query_params: + return False + return (self._query_params[name].lower() == 'true') + + def _read_int_parameter(self, name, default): if name not in self._query_params: return default @@ -395,12 +455,17 @@ def _apply_form_query(self, queryset): """Apply any query selectors passed as form variables.""" for parameter, values in self._query_params.lists(): + if ':' in parameter: + parameter, comparison_type = parameter.split(':', 1) + else: + comparison_type = None + if not self._query_processor.has_selector(parameter): continue for value in values: # forms keys can have multiple values - queryset = self._query_processor.apply_selector(queryset, - parameter, - value) + queryset = self._query_processor.apply_selector( + queryset, parameter, value, + comparison_type=comparison_type) return queryset --- /dev/null 2009-12-17 12:29:38.000000000 -0800 +++ autotest/frontend/shared/resource_test_utils.py 2010-04-08 13:32:10.000000000 -0700 @@ -0,0 +1,136 @@ +import operator, unittest +import simplejson +from django.test import client +from autotest_lib.frontend.afe import frontend_test_utils, models as afe_models + +class ResourceTestCase(unittest.TestCase, + frontend_test_utils.FrontendTestMixin): + URI_PREFIX = None # subclasses may override this to use partial URIs + + def setUp(self): + super(ResourceTestCase, self).setUp() + self._frontend_common_setup() + self._setup_debug_user() + self.client = client.Client() + + + def tearDown(self): + super(ResourceTestCase, self).tearDown() + self._frontend_common_teardown() + + + def _setup_debug_user(self): + user = afe_models.User.objects.create(login='debug_user') + acl = afe_models.AclGroup.objects.get(name='my_acl') + user.aclgroup_set.add(acl) + + + def _expected_status(self, method): + if method == 'post': + return 201 + if method == 'delete': + return 204 + return 200 + + + def raw_request(self, method, uri, **kwargs): + method = method.lower() + if method == 'put': + # the put() implementation in Django's test client is poorly + # implemented and only supports url-encoded keyvals for the data. + # the post() implementation is correct, though, so use that, with a + # trick to override the method. + method = 'post' + kwargs['REQUEST_METHOD'] = 'PUT' + + client_method = getattr(self.client, method) + return client_method(uri, **kwargs) + + + def request(self, method, uri, encode_body=True, **kwargs): + expected_status = self._expected_status(method) + + if 'data' in kwargs: + kwargs.setdefault('content_type', 'application/json') + if kwargs['content_type'] == 'application/json': + kwargs['data'] = simplejson.dumps(kwargs['data']) + + if uri.startswith('http://'): + full_uri = uri + else: + assert self.URI_PREFIX + full_uri = self.URI_PREFIX + '/' + uri + + response = self.raw_request(method, full_uri, **kwargs) + self.assertEquals( + response.status_code, expected_status, + 'Requesting %s\nExpected %s, got %s: %s (headers: %s)' + % (full_uri, expected_status, response.status_code, + response.content, response._headers)) + + if response['content-type'] != 'application/json': + return response.content + + try: + return simplejson.loads(response.content) + except ValueError: + self.fail('Invalid reponse body: %s' % response.content) + + + def sorted_by(self, collection, attribute): + return sorted(collection, key=operator.itemgetter(attribute)) + + + def _read_attribute(self, item, attribute_or_list): + if isinstance(attribute_or_list, basestring): + attribute_or_list = [attribute_or_list] + for attribute in attribute_or_list: + item = item[attribute] + return item + + + def check_collection(self, collection, attribute_or_list, expected_list, + length=None, check_number=None): + """Check the members of a collection of dicts. + + @param collection: an iterable of dicts + @param attribute_or_list: an attribute or list of attributes to read. + the results will be sorted and compared with expected_list. if + a list of attributes is given, the attributes will be read + hierarchically, i.e. item[attribute1][attribute2]... + @param expected_list: list of expected values + @param check_number: if given, only check this number of entries + @param length: expected length of list, only necessary if check_number + is given + """ + actual_list = sorted(self._read_attribute(item, attribute_or_list) + for item in collection['members']) + if length is None and check_number is None: + length = len(expected_list) + if length is not None: + self.assertEquals(len(actual_list), length, + 'Expected %s, got %s: %s' + % (length, len(actual_list), + ', '.join(str(item) for item in actual_list))) + if check_number: + actual_list = actual_list[:check_number] + self.assertEquals(actual_list, expected_list) + + + def check_relationship(self, resource_uri, relationship_name, + other_entry_name, field, expected_values, + length=None, check_number=None): + """Check the members of a relationship collection. + + @param resource_uri: URI of base resource + @param relationship_name: name of relationship attribute on base + resource + @param other_entry_name: name of other entry in relationship + @param field: name of field to grab on other entry + @param expected values: list of expected values for the given field + """ + response = self.request('get', resource_uri) + relationship_uri = response[relationship_name]['href'] + relationships = self.request('get', relationship_uri) + self.check_collection(relationships, [other_entry_name, field], + expected_values, length, check_number) --- autotest/frontend/shared/rest_client.py 2010-04-08 11:51:30.000000000 -0700 +++ autotest/frontend/shared/rest_client.py 2010-04-08 13:32:10.000000000 -0700 @@ -1,11 +1,10 @@ -import logging, pprint, re, urllib, getpass, urlparse +import copy, getpass, logging, pprint, re, urllib, urlparse import httplib2 -from django.utils import simplejson +from django.utils import datastructures, simplejson from autotest_lib.frontend.afe import rpc_client_lib from autotest_lib.client.common_lib import utils -_http = httplib2.Http() _request_headers = {} @@ -59,7 +58,8 @@ class Resource(object): - def __init__(self, representation_dict): + def __init__(self, representation_dict, http): + self._http = http assert 'href' in representation_dict for key, value in representation_dict.iteritems(): setattr(self, str(key), value) @@ -75,8 +75,10 @@ @classmethod - def load(cls, uri): - directory = cls({'href': uri}) + def load(cls, uri, http=None): + if not http: + http = httplib2.Http() + directory = cls({'href': uri}, http) return directory.get() @@ -88,7 +90,7 @@ converted_dict = dict((key, self._read_representation(sub_value)) for key, sub_value in value.iteritems()) if 'href' in converted_dict: - return type(self)(converted_dict) + return type(self)(converted_dict, http=self._http) return converted_dict return value @@ -113,11 +115,14 @@ def _do_request(self, method, uri, query_parameters, encoded_body): + uri_parts = [uri] if query_parameters: - query_string = '?' + urllib.urlencode(query_parameters) - else: - query_string = '' - full_uri = uri + query_string + if '?' in uri: + uri_parts += '&' + else: + uri_parts += '?' + uri_parts += urllib.urlencode(query_parameters, doseq=True) + full_uri = ''.join(uri_parts) if encoded_body: entity_body = simplejson.dumps(encoded_body) @@ -131,7 +136,7 @@ site_verify = utils.import_site_function( __file__, 'autotest_lib.frontend.shared.site_rest_client', 'site_verify_response', _site_verify_response_default) - headers, response_body = _http.request( + headers, response_body = self._http.request( full_uri, method, body=entity_body, headers=_get_request_headers(uri)) if not site_verify(headers, response_body): @@ -155,7 +160,8 @@ encoded_body) if 300 <= response.status < 400: # redirection - raise NotImplementedError(str(response)) # TODO + return self._do_request(method, response.headers['location'], + query_parameters, encoded_body) if 400 <= response.status < 500: raise ClientError(str(response)) if 500 <= response.status < 600: @@ -165,19 +171,59 @@ def _stringify_query_parameter(self, value): if isinstance(value, (list, tuple)): - return ','.join(value) + return ','.join(self._stringify_query_parameter(item) + for item in value) return str(value) - def get(self, **query_parameters): - string_parameters = dict((key, self._stringify_query_parameter(value)) - for key, value in query_parameters.iteritems() - if value is not None) - response = self._request('GET', query_parameters=string_parameters) + def _iterlists(self, mapping): + """This effectively lets us treat dicts as MultiValueDicts.""" + if hasattr(mapping, 'iterlists'): # mapping is already a MultiValueDict + return mapping.iterlists() + return ((key, (value,)) for key, value in mapping.iteritems()) + + + def get(self, query_parameters=None, **kwarg_query_parameters): + """ + @param query_parameters: a dict or MultiValueDict + """ + query_parameters = copy.copy(query_parameters) # avoid mutating original + if query_parameters is None: + query_parameters = {} + query_parameters.update(kwarg_query_parameters) + + string_parameters = datastructures.MultiValueDict() + for key, values in self._iterlists(query_parameters): + string_parameters.setlist( + key, [self._stringify_query_parameter(value) + for value in values]) + + response = self._request('GET', + query_parameters=string_parameters.lists()) assert response.status == 200 return self._read_representation(response.decoded_body()) + def get_full(self, results_limit, query_parameters=None, + **kwarg_query_parameters): + """ + Like get() for collections, when the full collection is expected. + + @param results_limit: maxmimum number of results to allow + @raises ClientError if there are more than results_limit results. + """ + result = self.get(query_parameters=query_parameters, + items_per_page=results_limit, + **kwarg_query_parameters) + if result.total_results > results_limit: + raise ClientError( + 'Too many results (%s > %s) for request %s (%s %s)' + % (result.total_results, results_limit, self.href, + query_parameters, kwarg_query_parameters)) + return result + + + def put(self): response = self._request('PUT', encoded_body=self._representation()) assert response.status == 200 --- /dev/null 2009-12-17 12:29:38.000000000 -0800 +++ autotest/frontend/tko/resources.py 2010-04-08 13:32:10.000000000 -0700 @@ -0,0 +1,57 @@ +from autotest_lib.frontend.shared import query_lib, resource_lib +from autotest_lib.frontend.tko import models + +class TestResult(resource_lib.InstanceEntry): + model = models.Test + + + @classmethod + def add_query_selectors(cls, query_processor): + query_processor.add_field_selector('afe_job_id', + field='job__afe_job_id') + query_processor.add_keyval_selector('has_keyval', models.TestAttribute, + 'attribute', 'value') + + + @classmethod + def from_uri_args(cls, request, test_id, **kwargs): + return cls(request, models.Test.objects.get(pk=test_id)) + + + def _uri_args(self): + return {'test_id': self.instance.pk} + + + def short_representation(self): + rep = super(TestResult, self).short_representation() + rep.update(id=self.instance.test_idx, + test_name=self.instance.test, + status=self.instance.status.word, + reason=self.instance.reason, + afe_job_id=self.instance.job.afe_job_id, + ) + return rep + + + def full_representation(self): + rep = super(TestResult, self).full_representation() + rep['keyvals'] = dict((keyval.attribute, keyval.value) + for keyval + in self.instance.testattribute_set.all()) + return rep + + +class TestResultCollection(resource_lib.Collection): + queryset = models.Test.objects.order_by('-test_idx') + entry_class = TestResult + + +class ResourceDirectory(resource_lib.Resource): + _permitted_methods = ('GET',) + + def handle_request(self): + result = self.link() + result.update({ + 'test_results': TestResultCollection(self._request).link(), + }) + return self._basic_response(result) --- /dev/null 2009-12-17 12:29:38.000000000 -0800 +++ autotest/frontend/tko/resources_test.py 2010-04-08 13:32:10.000000000 -0700 @@ -0,0 +1,48 @@ +#!/usr/bin/python + +import common +import unittest +from autotest_lib.frontend import setup_django_environment +from autotest_lib.frontend import setup_test_environment +from autotest_lib.client.common_lib.test_utils import mock +from autotest_lib.frontend.shared import resource_test_utils +from autotest_lib.frontend.tko import models, rpc_interface_unittest + + +class TkoResourceTestCase(resource_test_utils.ResourceTestCase, + rpc_interface_unittest.TkoTestMixin): + URI_PREFIX = 'http://testserver/new_tko/server/resources' + + def setUp(self): + super(TkoResourceTestCase, self).setUp() + self.god = mock.mock_god() + self._patch_sqlite_stuff() + self._create_initial_data() + + + def tearDown(self): + super(TkoResourceTestCase, self).tearDown() + self.god.unstub_all() + + +class TestResultTest(TkoResourceTestCase): + def test_collection(self): + response = self.request('get', 'test_results') + self.check_collection(response, 'test_name', + ['kernbench', 'mytest1', 'mytest2']) + + + def test_filter_afe_job_id(self): + response = self.request('get', 'test_results?afe_job_id=1') + self.check_collection(response, 'test_name', ['mytest1', 'mytest2']) + + + def test_entry(self): + response = self.request('get', 'test_results/1') + self.assertEquals(response['test_name'], 'mytest1') + self.assertEquals(response['status'], 'GOOD') + self.assertEquals(response['reason'], '') + + +if __name__ == '__main__': + unittest.main() --- autotest/frontend/tko/rpc_interface_unittest.py 2010-04-08 13:32:10.000000000 -0700 +++ autotest/frontend/tko/rpc_interface_unittest.py 2010-04-08 13:32:10.000000000 -0700 @@ -87,13 +87,12 @@ cursor.execute(_CREATE_ITERATION_RESULTS) -class RpcInterfaceTest(unittest.TestCase): - def setUp(self): - self._god = mock.mock_god() - self._god.stub_with(models.TempManager, '_get_column_names', - self._get_column_names_for_sqlite3) - self._god.stub_with(models.TempManager, '_cursor_rowcount', - self._cursor_rowcount_for_sqlite3) +class TkoTestMixin(object): + def _patch_sqlite_stuff(self): + self.god.stub_with(models.TempManager, '_get_column_names', + self._get_column_names_for_sqlite3) + self.god.stub_with(models.TempManager, '_cursor_rowcount', + self._cursor_rowcount_for_sqlite3) # add some functions to SQLite for MySQL compatibility connection.cursor() # ensure connection is alive @@ -101,10 +100,7 @@ connection.connection.create_function('find_in_set', 2, self._sqlite_find_in_set) - setup_test_environment.set_up() fix_iteration_tables() - setup_test_view() - self._create_initial_data() def _cursor_rowcount_for_sqlite3(self, cursor): @@ -139,11 +135,6 @@ return names - def tearDown(self): - setup_test_environment.tear_down() - self._god.unstub_all() - - def _create_initial_data(self): machine = models.Machine.objects.create(hostname='myhost') @@ -162,9 +153,11 @@ failed_status = models.Status.objects.create(word='FAILED') job1 = models.Job.objects.create(tag='1-myjobtag1', label='myjob1', - username='myuser', machine=machine) + username='myuser', machine=machine, + afe_job_id=1) job2 = models.Job.objects.create(tag='2-myjobtag2', label='myjob2', - username='myuser', machine=machine) + username='myuser', machine=machine, + afe_job_id=2) job1_test1 = models.Test.objects.create(job=job1, test='mytest1', kernel=kernel1, @@ -216,6 +209,21 @@ (test.test_idx, iteration, attribute, value)) +class RpcInterfaceTest(unittest.TestCase, TkoTestMixin): + def setUp(self): + self.god = mock.mock_god() + + setup_test_environment.set_up() + self._patch_sqlite_stuff() + setup_test_view() + self._create_initial_data() + + + def tearDown(self): + setup_test_environment.tear_down() + self.god.unstub_all() + + def _check_for_get_test_views(self, test): self.assertEquals(test['test_name'], 'mytest1') self.assertEquals(test['job_tag'], '1-myjobtag1') --- autotest/frontend/tko/urls.py 2010-04-08 13:32:10.000000000 -0700 +++ autotest/frontend/tko/urls.py 2010-04-08 13:32:10.000000000 -0700 @@ -1,15 +1,26 @@ from django.conf.urls import defaults import common from autotest_lib.frontend import settings, urls_common +from autotest_lib.frontend.tko import resources urlpatterns, debug_patterns = ( urls_common.generate_patterns('frontend.tko', 'TkoClient')) +resource_patterns = defaults.patterns( + '', + (r'^/?$', resources.ResourceDirectory.dispatch_request), + (r'^test_results/?$', resources.TestResultCollection.dispatch_request), + (r'^test_results/(?P<test_id>\d+)/?$', + resources.TestResult.dispatch_request), + ) + urlpatterns += defaults.patterns( '', (r'^jsonp_rpc/', 'frontend.tko.views.handle_jsonp_rpc'), (r'^csv/', 'frontend.tko.views.handle_csv'), - (r'^plot/', 'frontend.tko.views.handle_plot')) + (r'^plot/', 'frontend.tko.views.handle_plot'), + + (r'^resources/', defaults.include(resource_patterns))) if settings.DEBUG: urlpatterns += debug_patterns --- autotest/utils/unittest_suite.py 2010-04-05 12:38:58.000000000 -0700 +++ autotest/utils/unittest_suite.py 2010-04-08 13:32:10.000000000 -0700 @@ -29,6 +29,7 @@ 'metahost_scheduler_unittest.py', 'site_metahost_scheduler_unittest.py', 'rpc_utils_unittest.py', + 'service_proxy_lib_test.py', )) REQUIRES_MYSQLDB = set(( _______________________________________________ Autotest mailing list [email protected] http://test.kernel.org/cgi-bin/mailman/listinfo/autotest
