jenkins-bot has submitted this change and it was merged.
Change subject: Delete PENDING reports and increase Timeout
......................................................................
Delete PENDING reports and increase Timeout
If recurrent report children are in the PENDING state for more than a
few seconds, it means they should be cleaned up. This is because delay
is called on their task and the first step there is to change the status
to STARTED. Therefore, we're deleting any PENDING children before we
figure out which days are left to run.
Also, related, the timeout on the daily scheduled task is increased to
three times the default. This is to accomodate the fact that the
scheduler task won't execute as quickly since celery is swamped by
handling all the new workers that the scheduler is putting to work.
Bug: 70663
Change-Id: I9e8ef523833c9395ef250e12757c449e2fb32b6b
---
M tests/test_models/test_run_report.py
M wikimetrics/models/report_nodes/run_report.py
M wikimetrics/schedules/daily.py
3 files changed, 62 insertions(+), 1 deletion(-)
Approvals:
Nuria: Looks good to me, approved
jenkins-bot: Verified
diff --git a/tests/test_models/test_run_report.py
b/tests/test_models/test_run_report.py
index 5138da7..8af207c 100644
--- a/tests/test_models/test_run_report.py
+++ b/tests/test_models/test_run_report.py
@@ -16,6 +16,10 @@
from wikimetrics.schedules.daily import recurring_reports
+def make_pending(report):
+ report.status = 'PENDING'
+
+
class RunReportClassMethodsTest(DatabaseTest):
def tearDown(self):
# re-enable the scheduler after these tests
@@ -114,6 +118,45 @@
for m in self.missed_by_index[2]
]))
+ def test_days_missed_0_with_cleanup(self):
+ self.add_runs_to_report(0)
+
+ # change some reports back to pending to make sure they're cleaned up
+ map(make_pending, self.report_runs[0::2])
+ self.session.commit()
+
+ missed_days = RunReport.days_missed(self.reports[0], self.session)
+ assert_equals(set(missed_days), set([
+ self.today - timedelta(days=m)
+ for m in self.missed_by_index[0]
+ ] + [r.created for r in self.report_runs[0::2]]))
+
+ def test_days_missed_1_with_cleanup(self):
+ self.add_runs_to_report(1)
+
+ # change some reports back to pending to make sure they're cleaned up
+ map(make_pending, self.report_runs[0::2])
+ self.session.commit()
+
+ missed_days = RunReport.days_missed(self.reports[1], self.session)
+ assert_equals(set(missed_days), set([
+ self.today - timedelta(days=m)
+ for m in self.missed_by_index[1]
+ ] + [r.created for r in self.report_runs[0::2]]))
+
+ def test_days_missed_2_with_cleanup(self):
+ self.add_runs_to_report(2)
+
+ # change some reports back to pending to make sure they're cleaned up
+ map(make_pending, self.report_runs[0::2])
+ self.session.commit()
+
+ missed_days = RunReport.days_missed(self.reports[2], self.session)
+ assert_equals(set(missed_days), set([
+ self.today - timedelta(days=m)
+ for m in self.missed_by_index[2]
+ ] + [r.created for r in self.report_runs[0::2]]))
+
def test_create_reports_for_missed_days_0(self):
self.add_runs_to_report(0)
new_runs = list(RunReport.create_reports_for_missed_days(
diff --git a/wikimetrics/models/report_nodes/run_report.py
b/wikimetrics/models/report_nodes/run_report.py
index 481b667..b988cb2 100644
--- a/wikimetrics/models/report_nodes/run_report.py
+++ b/wikimetrics/models/report_nodes/run_report.py
@@ -226,6 +226,13 @@
"""
search_from = strip_time(report.created)
+ # if a report is pending by this point, it means that it should be
re-tried
+ session.query(ReportStore) \
+ .filter(ReportStore.recurrent_parent_id == report.id) \
+ .filter(ReportStore.created >= search_from) \
+ .filter(ReportStore.status == celery.states.PENDING) \
+ .delete()
+
completed_days = [pr[0] for pr in session.query(ReportStore.created)
.filter(ReportStore.recurrent_parent_id == report.id)
.filter(ReportStore.created >= search_from)
diff --git a/wikimetrics/schedules/daily.py b/wikimetrics/schedules/daily.py
index 3119ed8..06c8306 100644
--- a/wikimetrics/schedules/daily.py
+++ b/wikimetrics/schedules/daily.py
@@ -10,7 +10,18 @@
task_logger = get_task_logger(__name__)
[email protected]()
+# NOTE: We found an interesting problem leading to the default timeouts
expiring.
+# The scheduler would run, create the generator of child reports, and
delay each one
+# This works but causes the scheduler to execute more slowly (we think
because the
+# workers handling the report tasks are taking up resources). So in a
situation
+# where reports are running quickly, this task will need to be allowed
to live
+# for the entire life of all the other tasks. Therefore, extending
timeout to 3
+# times the amount allotted to normal tasks.
+configured_soft_limit = queue.conf.get('CELERYD_TASK_SOFT_TIME_LIMIT', 3600)
+new_limit = 3 * configured_soft_limit
+
+
[email protected](time_limit=new_limit, soft_time_limit=new_limit)
def recurring_reports(report_id=None):
from wikimetrics.configurables import db
from wikimetrics.models import ReportStore, RunReport
--
To view, visit https://gerrit.wikimedia.org/r/159503
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I9e8ef523833c9395ef250e12757c449e2fb32b6b
Gerrit-PatchSet: 2
Gerrit-Project: analytics/wikimetrics
Gerrit-Branch: master
Gerrit-Owner: Milimetric <[email protected]>
Gerrit-Reviewer: Nuria <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
