Milimetric has submitted this change and it was merged.
Change subject: fixed csv output and errant user_id column
......................................................................
fixed csv output and errant user_id column
Change-Id: I1653a1dbb9908a64347074dbc7bde6cb8c98b8e1
---
M tests/test_models/test_report_response.py
M wikimetrics/config/celery_config.yaml
M wikimetrics/controllers/reports.py
M wikimetrics/models/report_nodes/aggregate_report.py
4 files changed, 52 insertions(+), 17 deletions(-)
Approvals:
Milimetric: Verified; Looks good to me, approved
diff --git a/tests/test_models/test_report_response.py
b/tests/test_models/test_report_response.py
index 89fe9b0..503109e 100644
--- a/tests/test_models/test_report_response.py
+++ b/tests/test_models/test_report_response.py
@@ -1,4 +1,4 @@
-from wikimetrics.models import Cohort, RunReport, MultiProjectMetricReport
+from wikimetrics.models import Cohort, RunReport, MultiProjectMetricReport,
Aggregation
from wikimetrics.metrics import NamespaceEdits
from ..fixtures import QueueDatabaseTest
from nose.tools import assert_equals, assert_true
@@ -32,7 +32,7 @@
results = jr.task.delay(jr).get()[0]
# TODO: figure out why one of the resulting wiki_user_ids is None here
assert_equals(
- results['individual
results'][0][self.test_mediawiki_user_id]['edits'],
+ results[Aggregation.IND][0][self.test_mediawiki_user_id]['edits'],
2,
)
@@ -55,12 +55,12 @@
jr = RunReport(desired_responses, user_id=self.test_user_id)
results = jr.task.delay(jr).get()[0]
assert_equals(
- results['individual
results'][0][self.test_mediawiki_user_id]['edits'],
+ results[Aggregation.IND][0][self.test_mediawiki_user_id]['edits'],
2,
)
assert_equals(
- results['Sum']['edits'],
+ results[Aggregation.SUM]['edits'],
5,
)
@@ -83,12 +83,12 @@
jr = RunReport(desired_responses, user_id=self.test_user_id)
results = jr.task.delay(jr).get()[0]
assert_equals(
- results['individual
results'][0][self.test_mediawiki_user_id]['net_sum'],
+
results[Aggregation.IND][0][self.test_mediawiki_user_id]['net_sum'],
10,
)
assert_equals(
- results['Sum']['positive_only_sum'],
+ results[Aggregation.SUM]['positive_only_sum'],
50,
)
@@ -119,7 +119,7 @@
for report in reports:
try:
results = report.get()[0]
- if results['Sum']['positive_only_sum'] == 50:
+ if results[Aggregation.SUM]['positive_only_sum'] == 50:
successes += 1
except:
print('timeout expired for this task')
diff --git a/wikimetrics/config/celery_config.yaml
b/wikimetrics/config/celery_config.yaml
index 36a3398..0b28098 100644
--- a/wikimetrics/config/celery_config.yaml
+++ b/wikimetrics/config/celery_config.yaml
@@ -3,7 +3,7 @@
CELERY_TASK_RESULT_EXPIRES : 3600
CELERY_DISABLE_RATE_LIMITS : True
CELERY_STORE_ERRORS_EVEN_IF_IGNORED : True
-CELERYD_CONCURRENCY : 8
-CELERYD_TASK_TIME_LIMIT : 10
-CELERYD_TASK_SOFT_TIME_LIMIT : 5
+CELERYD_CONCURRENCY : 16
+CELERYD_TASK_TIME_LIMIT : 60
+CELERYD_TASK_SOFT_TIME_LIMIT : 30
DEBUG : True
diff --git a/wikimetrics/controllers/reports.py
b/wikimetrics/controllers/reports.py
index 77253db..ccce0a0 100644
--- a/wikimetrics/controllers/reports.py
+++ b/wikimetrics/controllers/reports.py
@@ -4,6 +4,7 @@
from celery.task.control import revoke
from ..configurables import app, db
from ..models import Report, RunReport, PersistentReport
+from ..models.report_nodes import Aggregation
from ..metrics import metric_classes
from ..utils import json_response, json_error, json_redirect, thirty_days_ago
import json
@@ -78,17 +79,50 @@
csv_io = StringIO()
if task_result:
+ columns = []
+
+ if Aggregation.IND in task_result:
+ columns = task_result[Aggregation.IND][0].values()[0].keys()
+ elif Aggregation.SUM in task_result:
+ columns = task_result[Aggregation.SUM].keys()
+ elif Aggregation.AVG in task_result:
+ columns = task_result[Aggregation.AVG].keys()
+ elif Aggregation.STD in task_result:
+ columns = task_result[Aggregation.STD].keys()
+
# if task_result is not empty find header in first row
- fieldnames = ['user_id'] + sorted(task_result.values()[0].keys())
+ fieldnames = ['user_id'] + columns
else:
fieldnames = ['user_id']
writer = DictWriter(csv_io, fieldnames)
+ # collect rows to output in CSV
task_rows = []
- # fold user_id into dict so we can use DictWriter to escape things
- for user_id, row in task_result.iteritems():
- row['user_id'] = user_id
- task_rows.append(row)
+
+ # Individual Results
+ if Aggregation.IND in task_result:
+ # fold user_id into dict so we can use DictWriter to escape things
+ for user_id, row in task_result[Aggregation.IND][0].iteritems():
+ task_row = row.copy()
+ task_row['user_id'] = user_id
+ task_rows.append(task_row)
+
+ # Aggregate Results
+ if Aggregation.SUM in task_result:
+ task_row = task_result[Aggregation.SUM].copy()
+ task_row['user_id'] = Aggregation.SUM
+ task_rows.append(task_row)
+
+ if Aggregation.AVG in task_result:
+ task_row = task_result[Aggregation.AVG].copy()
+ task_row['user_id'] = Aggregation.AVG
+ task_rows.append(task_row)
+
+ if Aggregation.STD in task_result:
+ task_row = task_result[Aggregation.STD].copy()
+ task_row['user_id'] = Aggregation.STD
+ task_rows.append(task_row)
+
writer.writeheader()
writer.writerows(task_rows)
return Response(csv_io.getvalue(), mimetype='text/csv')
diff --git a/wikimetrics/models/report_nodes/aggregate_report.py
b/wikimetrics/models/report_nodes/aggregate_report.py
index fef5cf4..09a08f1 100644
--- a/wikimetrics/models/report_nodes/aggregate_report.py
+++ b/wikimetrics/models/report_nodes/aggregate_report.py
@@ -4,12 +4,13 @@
from celery.utils.log import get_task_logger
-__all__ = ['AggregateReport']
+__all__ = ['AggregateReport', 'Aggregation']
task_logger = get_task_logger(__name__)
class Aggregation(object):
+ IND = 'Individual Results'
SUM = 'Sum'
AVG = 'Average'
STD = 'Standard Deviation'
@@ -81,7 +82,7 @@
)
if self.individual:
- aggregated_results['individual results'] = multi_project_results
+ aggregated_results[Aggregation.IND] = multi_project_results
return aggregated_results
--
To view, visit https://gerrit.wikimedia.org/r/76930
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I1653a1dbb9908a64347074dbc7bde6cb8c98b8e1
Gerrit-PatchSet: 1
Gerrit-Project: analytics/wikimetrics
Gerrit-Branch: master
Gerrit-Owner: Milimetric <dandree...@wikimedia.org>
Gerrit-Reviewer: Milimetric <dandree...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
