On Mon, Jul 7, 2014 at 10:36 AM, Raphael Hertzog <hert...@debian.org> wrote: >> > On Sun, 06 Jul 2014, Andrew Starr-Bochicchio wrote: >> >> +@python_2_unicode_compatible >> >> +class DebciStatus(models.Model): >> >> + """ >> >> + Model for debci status of packages. >> >> + """ >> >> + package = models.OneToOneField(PackageName, >> >> related_name='debci_status') >> >> + status = JSONField() >> > >> > Please don't add a new model for this. Rather use a new "key" in >> > PackageExtractedInfo. >> > >> > PackageExtractedInfo model should probably be renamed into something else >> > but it's the better place to store this as we can then query most of the >> > relevant information for a single package in a single query... >> >> Right. After I sent this off, I wondered if it needs to store this >> information at all. Is enough to just store the information in the >> action item? > > Possibly, yes. It really depends on whether you expect other parts of the > package tracker to build upon this information or not. > > But we can always move it to a more appropriate place later on if we > we have something that needs this information.
I went ahead and dropped it for now. It's all just stored in the ActionItem. Easy enough to add it if we need it. Updated patch attached. This time with some tests added as well. Thanks! -- Andrew Starr-Bochicchio Ubuntu Developer <https://launchpad.net/~andrewsomething> Debian Developer <http://qa.debian.org/developer.php?login=asb> PGP/GPG Key ID: D53FDCB1
From 2d23d7110111ca0cf449c4465c88ba4d91d037c2 Mon Sep 17 00:00:00 2001 From: Andrew Starr-Bochicchio <a.star...@gmail.com> Date: Mon, 7 Jul 2014 23:18:20 -0400 Subject: [PATCH] Add support for showing Debci test failures as an action item. Closes: #753910 To: 753...@bugs.debian.org --- .../debian/templates/debian/debci-action-item.html | 10 +++ distro_tracker/vendor/debian/tests.py | 86 +++++++++++++++++++++ distro_tracker/vendor/debian/tracker_tasks.py | 87 ++++++++++++++++++++++ 3 files changed, 183 insertions(+) create mode 100644 distro_tracker/vendor/debian/templates/debian/debci-action-item.html diff --git a/distro_tracker/vendor/debian/templates/debian/debci-action-item.html b/distro_tracker/vendor/debian/templates/debian/debci-action-item.html new file mode 100644 index 0000000..7142e9f --- /dev/null +++ b/distro_tracker/vendor/debian/templates/debian/debci-action-item.html @@ -0,0 +1,10 @@ +{% with duration=item.extra_data.duration %} +{% with previous_status=item.extra_data.previous_status %} +{% with date=item.extra_data.date %} +<p><a href="http://ci.debian.net/">Debci</a> reports +<a href="{{ item.extra_data.url }}"> test failures </a> for this package +(<a href="{{ item.extra_data.log }}">log</a>).</p> +<p>The tests ran in {{duration}}</p> +<p>Last run: {{date}}</p> +<p>Previous status: <b>{{previous_status}}</b></p> +{% endwith %}{% endwith %}{% endwith %} diff --git a/distro_tracker/vendor/debian/tests.py b/distro_tracker/vendor/debian/tests.py index cd9abce..ca456b8 100644 --- a/distro_tracker/vendor/debian/tests.py +++ b/distro_tracker/vendor/debian/tests.py @@ -60,6 +60,7 @@ from distro_tracker.vendor.debian.tracker_tasks import RetrieveDebianMaintainers from distro_tracker.vendor.debian.tracker_tasks import RetrieveLowThresholdNmuTask from distro_tracker.vendor.debian.tracker_tasks import DebianWatchFileScannerUpdate from distro_tracker.vendor.debian.tracker_tasks import UpdateExcusesTask +from distro_tracker.vendor.debian.tracker_tasks import UpdateDebciStatusTask from distro_tracker.vendor.debian.models import DebianContributor from distro_tracker.vendor.debian.models import UbuntuPackage from distro_tracker.vendor.debian.tracker_tasks import UpdateLintianStatsTask @@ -4496,3 +4497,88 @@ class DebianSsoLoginTests(TestCase): self.client.get('/') self.assert_no_user_logged_in() + + +class UpdateDebciStatusTaskTest(TestCase): + """ + Tests for the :class:`distro_tracker.vendor.debian.tracker_tasks.UpdateDebciStatusTask` task. + """ + def setUp(self): + self.dummy_package = SourcePackageName.objects.create(name='dummy-package') + self.other_package = SourcePackageName.objects.create(name='other-package') + + def run_task(self): + """ + Runs the debci status update task. + """ + task = UpdateDebciStatusTask() + task.execute() + + @mock.patch('distro_tracker.core.utils.http.requests') + def test_debci_status(self, mock_requests): + """ + Tests that an ActionItem isn't created for a psssing debici status. + """ + json_data = """[ + { + "run_id": "20140705_145427", + "package": "dummy-package", + "version": "1.0-1", + "date": "2014-07-05 14:55:57", + "status": "pass", + "blame": [ ], + "previous_status": "pass", + "duration_seconds": "91", + "duration_human": "0h 1m 31s", + "message": "All tests passed" + }, + { + "run_id": "20140705_212616", + "package": "other-package", + "version": "2.0-2", + "date": "2014-07-05 21:34:22", + "status": "fail", + "blame": [ ], + "previous_status": "fail", + "duration_seconds": "488", + "duration_human": "0h 8m 8s", + "message": "Tests failed" + }, + { + "run_id": "20140705_143518", + "package": "another-package", + "version": "3.0-3", + "date": "2014-07-05 17:33:08", + "status": "fail", + "blame": [ ], + "previous_status": "fail", + "duration_seconds": "222", + "duration_human": "0h 3m 42s", + "message": "Tests failed" + }] + """ + set_mock_response(mock_requests, text=json_data) + + # Sanity check: there are no action items and two packages to start + self.assertEqual(0, ActionItem.objects.count()) + self.assertEqual(2, SourcePackageName.objects.count()) + + self.run_task() + + # The ActionItem is only added for failed tests for known packages. + self.assertEqual(1, ActionItem.objects.count()) + self.assertEqual(0, self.dummy_package.action_items.count()) + self.assertEqual(1, self.other_package.action_items.count()) + + # Check that the ActionItem contains the correct contents. + action_item = self.other_package.action_items.all()[0] + url = "http://ci.debian.net/#package/other-package" + log = "http://ci.debian.net/data/packages/unstable/amd64/o/other-package/latest-autopkgtest/log" + desc = ('Debci reports <a href="{url}"> failed tests</a> (<a href="{log}">log</a>)' + ).format(url=url, log=log) + self.assertEqual(action_item.short_description, desc) + self.assertEqual(action_item.extra_data['duration'], "0h 8m 8s") + self.assertEqual(action_item.extra_data['previous_status'], "fail") + self.assertEqual(action_item.extra_data['date'], "2014-07-05 21:34:22") + self.assertEqual(action_item.extra_data['url'], url) + self.assertEqual(action_item.extra_data['log'], log) diff --git a/distro_tracker/vendor/debian/tracker_tasks.py b/distro_tracker/vendor/debian/tracker_tasks.py index 0ea38d9..da17041 100644 --- a/distro_tracker/vendor/debian/tracker_tasks.py +++ b/distro_tracker/vendor/debian/tracker_tasks.py @@ -43,6 +43,7 @@ from distro_tracker import vendor import re import SOAPpy import yaml +import json from debian import deb822 from debian.debian_support import AptPkgVersion from debian import debian_support @@ -1938,3 +1939,89 @@ class UpdateNewQueuePackages(BaseTask): }) new_queue_info.save() + + +class UpdateDebciStatusTask(BaseTask): + """ + Updates packages' debci status. + """ + ACTION_ITEM_TYPE_NAME = 'debci-failed-tests' + ITEM_DESCRIPTION = ( + 'Debci reports <a href="{debci_url}"> failed tests</a> (<a href="{log_url}">log</a>)') + ITEM_FULL_DESCRIPTION_TEMPLATE = 'debian/debci-action-item.html' + + def __init__(self, force_update=False, *args, **kwargs): + super(UpdateDebciStatusTask, self).__init__(*args, **kwargs) + self.force_update = force_update + self.debci_action_item_type = ActionItemType.objects.create_or_update( + type_name=self.ACTION_ITEM_TYPE_NAME, + full_description_template=self.ITEM_FULL_DESCRIPTION_TEMPLATE) + + def set_parameters(self, parameters): + if 'force_update' in parameters: + self.force_update = parameters['force_update'] + + def get_debci_status(self): + url = 'http://ci.debian.net/data/status/unstable/amd64/packages.json' + cache = HttpCache(settings.DISTRO_TRACKER_CACHE_DIRECTORY) + response, updated = cache.update(url, force=self.force_update) + response.raise_for_status() + if not updated: + return + debci_status = json.loads(response.text) + return debci_status + + def update_action_item(self, package, debci_status): + """ + Updates the :class:`ActionItem` for the given package based on the + :class:`DebciStatus <distro_tracker.vendor.debian.models.DebciStatus` + If the package has test failures an :class:`ActionItem` is created. + """ + debci_action_item = package.get_action_item_for_type( + self.debci_action_item_type.type_name) + if debci_status.get('status') == 'pass': + debci_action_item.delete() + return + + if debci_action_item is None: + debci_action_item = ActionItem( + package=package, + item_type=self.debci_action_item_type) + + package_name = debci_status.get('package') + url = 'http://ci.debian.net/#package/' + package_name + log = 'http://ci.debian.net/data/packages/unstable/amd64/' + \ + package_name[:1] + "/" + package_name + '/latest-autopkgtest/log' + debci_action_item.short_description = self.ITEM_DESCRIPTION.format( + debci_url=url, + log_url=log) + + debci_action_item.extra_data = { + 'duration': debci_status.get('duration_human'), + 'previous_status': debci_status.get('previous_status'), + 'date': debci_status.get('date'), + 'url': url, + 'log': log, + } + + debci_action_item.save() + + def execute(self): + all_debci_status = self.get_debci_status() + if not all_debci_status: + return + + with transaction.atomic(): + packages = [] + for result in all_debci_status: + if result['status'] == 'fail': + try: + package = SourcePackageName.objects.get(name=result['package']) + packages.append(package) + self.update_action_item(package, result) + except SourcePackageName.DoesNotExist: + pass + + # Remove action items for packages without failing tests. + ActionItem.objects.delete_obsolete_items( + [self.debci_action_item_type], packages) -- 1.9.1