Mforns has uploaded a new change for review.
https://gerrit.wikimedia.org/r/223031
Change subject: Add agrgegation across projects
......................................................................
Add agrgegation across projects
With this change, a new file 'all.csv' will be created
for the raw_daily data as well as for each additional
aggregator (daily, weekly_rescaled, etc.). This file
will contain an aggregation across all projects.
Bug: T95339
Change-Id: Ieee012e60d2286e450779cabf253d6a1f11578b4
---
M aggregator/projectcounts.py
M aggregator/util.py
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-010000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-020000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-030000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-040000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-050000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-060000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-070000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-080000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-090000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-100000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-110000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-120000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-130000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-140000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-150000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-160000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-170000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-180000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-190000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-200000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-210000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-220000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-230000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-000000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-010000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-020000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-030000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-040000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-050000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-060000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-070000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-080000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-090000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-100000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-110000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-120000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-130000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-140000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-150000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-160000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-170000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-180000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-190000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-200000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-210000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-220000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-230000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-000000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-010000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-020000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-030000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-040000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-050000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-060000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-070000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-080000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-090000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-100000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-110000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-120000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-130000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-140000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-150000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-160000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-170000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-180000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-190000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-200000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-210000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-220000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-230000
A
tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141104-000000
M tests/test_projectcounts/test_per_project_aggregation.py
M tests/test_util.py
76 files changed, 387 insertions(+), 24 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/analytics/aggregator
refs/changes/31/223031/1
diff --git a/aggregator/projectcounts.py b/aggregator/projectcounts.py
index 75c7f5e..32f391a 100644
--- a/aggregator/projectcounts.py
+++ b/aggregator/projectcounts.py
@@ -497,6 +497,9 @@
:param force_recomputation: If True, recompute data for the given days,
even if it is already in the CSV. (Default: False)
"""
+ # Contains the aggregation of all data across projects indexed by date.
+ all_projects_data = {}
+
for csv_file_abs in sorted(glob.glob(os.path.join(
target_dir_abs, 'daily_raw', '*.csv'))):
logging.info("Updating csv '%s'" % (csv_file_abs))
@@ -545,20 +548,69 @@
count_mobile if date >= DATE_MOBILE_ADDED else None,
count_zero if date >= DATE_MOBILE_ADDED else None)
- util.write_dict_values_sorted_to_csv(
- csv_file_abs,
+ _write_raw_and_aggregated_csv_data(
+ target_dir_abs,
+ dbname,
csv_data,
- header=CSV_HEADER)
+ first_date,
+ last_date,
+ additional_aggregators,
+ bad_dates,
+ force_recomputation)
- for additional_aggregator in additional_aggregators:
- additional_aggregator(
- target_dir_abs,
- dbname,
- csv_data,
- first_date,
- last_date,
- bad_dates=bad_dates,
- force_recomputation=force_recomputation)
+ # Aggregates values across all projects
+ util.merge_sum_csv_data_dict(all_projects_data, csv_data)
+
+ # Writes aggregations across all projects
+ _write_raw_and_aggregated_csv_data(
+ target_dir_abs,
+ 'all',
+ all_projects_data,
+ first_date,
+ last_date,
+ additional_aggregators,
+ bad_dates,
+ force_recomputation)
+
+
+def _write_raw_and_aggregated_csv_data(
+ target_dir_abs, dbname, csv_data, first_date, last_date,
+ additional_aggregators, bad_dates, force_recomputation):
+ """
+ Writes the data passed in the csv_data dict to various destinations:
+
+ 1. Writes the raw data (as it comes in the csv_data dict), to:
+ <target_dir_abs>/daily_raw/<dbname>.csv
+
+ 2. Uses each aggregator in additional_aggregators to write the data
+ to the aggregator's specific location. Note: Some of this method's
+ parameters are just forwarded to these aggregators.
+
+ :param target_dir_abs: Absolute directory of the per project CSVs.
+ :param dbname: The database name of the wiki to consider (E.g.: 'enwiki')
+ :param csv_data: The dict containing the data to be written.
+ :param first_date: The first date to write non-existing data for.
+ :param last_date: The last date to write non-existing data for.
+ :param additional_aggregators: See update_per_project_csvs_for_dates.
+ :param bad_dates: List of dates considered having bad data.
+ :param force_recomputation: If True, recompute data for the given days.
+ """
+ csv_file_abs = os.path.join(target_dir_abs, 'daily_raw', dbname + '.csv')
+
+ util.write_dict_values_sorted_to_csv(
+ csv_file_abs,
+ csv_data,
+ header=CSV_HEADER)
+
+ for additional_aggregator in additional_aggregators:
+ additional_aggregator(
+ target_dir_abs,
+ dbname,
+ csv_data,
+ first_date,
+ last_date,
+ bad_dates=bad_dates,
+ force_recomputation=force_recomputation)
def _get_validity_issues_for_aggregated_projectcounts_generic(
diff --git a/aggregator/util.py b/aggregator/util.py
index 9527b6d..6853521 100644
--- a/aggregator/util.py
+++ b/aggregator/util.py
@@ -21,6 +21,7 @@
import datetime
import os
+from operator import add
WEBSTATSCOLLECTOR_WHITELISTED_WIKIMEDIA_WIKIS = [
'commons',
@@ -205,3 +206,36 @@
line += ',%s' % ('' if column is None else column)
csv_data[first_column] = line
return csv_data
+
+
+def merge_sum_csv_data_dict(csv_data_1, csv_data_2):
+ """
+ Merge csv_data_2 into csv_data_1, accumulating values.
+
+ The two dicts have to be of the form dict<key: str, value: str>
+ and their value format should be 'date,int,int,int,int'.
+ If any of those is incorrect, a ValueError is raised.
+
+ The merged values will be the cross sum of all integer operators,
+ with the exception of the first integer column, that will hold
+ the sum of the other integer columns:
+ 'date,int1a,int1b,int1c,int1d' + 'date,int2a,int2b,int2c,int2d' =
+ 'date,<sumOfTheOtherInts>,int1b+int2b,int1c+int2c,int1d+int2d'
+ Note that the date will always be the same on both sides,
+ and will remain unchanged.
+
+ :param csv_data_1: The data dict to be merged on.
+ :param csv_data_2: The data dict to be added from.
+ """
+ parse = lambda vs: [int(v.strip()) for v in vs.split(',')[2:5]]
+ build = lambda d, vs: ','.join([d, str(sum(vs))] + [str(v) for v in vs])
+
+ for date in csv_data_2:
+
+ value_1 = parse(csv_data_1[date]) if date in csv_data_1 else [0, 0, 0]
+ value_2 = parse(csv_data_2[date])
+ if len(value_1) != 3 or len(value_2) != 3:
+ raise ValueError('Cannot parse CSV data dict value.')
+
+ summed_value = map(add, value_1, value_2)
+ csv_data_1[date] = build(date, summed_value)
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-010000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-010000
new file mode 100644
index 0000000..20a5299
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-010000
@@ -0,0 +1,3 @@
+en - 3 100
+de - 2 100
+fr - 5 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-020000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-020000
new file mode 100644
index 0000000..db54213
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-020000
@@ -0,0 +1,2 @@
+en - 8 100
+fr - 9 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-030000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-030000
new file mode 100644
index 0000000..2ba29f4
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-030000
@@ -0,0 +1 @@
+de - 9 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-040000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-040000
new file mode 100644
index 0000000..f7e2efc
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-040000
@@ -0,0 +1,2 @@
+de - 7 100
+fr - 4 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-050000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-050000
new file mode 100644
index 0000000..f0f4ceb
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-050000
@@ -0,0 +1,3 @@
+en - 10 100
+de - 3 100
+fr - 4 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-060000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-060000
new file mode 100644
index 0000000..550d23f
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-060000
@@ -0,0 +1,2 @@
+de - 5 100
+fr - 1 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-070000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-070000
new file mode 100644
index 0000000..8cd13be
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-070000
@@ -0,0 +1,3 @@
+en - 3 100
+de - 10 100
+fr - 10 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-080000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-080000
new file mode 100644
index 0000000..143f271
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-080000
@@ -0,0 +1,3 @@
+en - 6 100
+de - 5 100
+fr - 6 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-090000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-090000
new file mode 100644
index 0000000..4bdbca3
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-090000
@@ -0,0 +1,3 @@
+en - 2 100
+de - 4 100
+fr - 5 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-100000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-100000
new file mode 100644
index 0000000..d7441c7
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-100000
@@ -0,0 +1,2 @@
+en - 9 100
+fr - 4 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-110000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-110000
new file mode 100644
index 0000000..28320c1
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-110000
@@ -0,0 +1,3 @@
+en - 8 100
+de - 7 100
+fr - 1 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-120000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-120000
new file mode 100644
index 0000000..a411fc0
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-120000
@@ -0,0 +1,2 @@
+en - 6 100
+de - 7 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-130000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-130000
new file mode 100644
index 0000000..79d9344
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-130000
@@ -0,0 +1,3 @@
+en - 1 100
+de - 10 100
+fr - 1 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-140000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-140000
new file mode 100644
index 0000000..0cc611c
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-140000
@@ -0,0 +1,3 @@
+en - 7 100
+de - 6 100
+fr - 6 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-150000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-150000
new file mode 100644
index 0000000..5c990f9
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-150000
@@ -0,0 +1 @@
+de - 10 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-160000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-160000
new file mode 100644
index 0000000..5cbbd36
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-160000
@@ -0,0 +1,2 @@
+de - 7 100
+fr - 7 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-170000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-170000
new file mode 100644
index 0000000..ab12bb5
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-170000
@@ -0,0 +1,2 @@
+en - 10 100
+fr - 5 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-180000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-180000
new file mode 100644
index 0000000..fa98519
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-180000
@@ -0,0 +1,3 @@
+en - 2 100
+de - 4 100
+fr - 2 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-190000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-190000
new file mode 100644
index 0000000..5d9eeee
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-190000
@@ -0,0 +1,2 @@
+de - 4 100
+fr - 3 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-200000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-200000
new file mode 100644
index 0000000..f61ad79
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-200000
@@ -0,0 +1,3 @@
+en - 9 100
+de - 5 100
+fr - 1 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-210000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-210000
new file mode 100644
index 0000000..42cd30b
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-210000
@@ -0,0 +1,3 @@
+en - 4 100
+de - 9 100
+fr - 4 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-220000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-220000
new file mode 100644
index 0000000..f599cc7
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-220000
@@ -0,0 +1,3 @@
+en - 7 100
+de - 1 100
+fr - 4 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-230000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-230000
new file mode 100644
index 0000000..07ec149
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141101-230000
@@ -0,0 +1,2 @@
+en - 8 100
+fr - 7 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-000000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-000000
new file mode 100644
index 0000000..ee33a23
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-000000
@@ -0,0 +1,2 @@
+de - 6 100
+fr - 10 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-010000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-010000
new file mode 100644
index 0000000..b6aee19
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-010000
@@ -0,0 +1,2 @@
+de - 9 100
+fr - 2 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-020000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-020000
new file mode 100644
index 0000000..342f8bb
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-020000
@@ -0,0 +1,2 @@
+en - 3 100
+fr - 10 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-030000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-030000
new file mode 100644
index 0000000..b5d3008
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-030000
@@ -0,0 +1,3 @@
+en - 2 100
+de - 7 100
+fr - 1 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-040000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-040000
new file mode 100644
index 0000000..9b19dea
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-040000
@@ -0,0 +1,3 @@
+en - 5 100
+de - 10 100
+fr - 3 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-050000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-050000
new file mode 100644
index 0000000..1126eaa
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-050000
@@ -0,0 +1,3 @@
+en - 5 100
+de - 5 100
+fr - 3 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-060000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-060000
new file mode 100644
index 0000000..4b01cb0
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-060000
@@ -0,0 +1,2 @@
+de - 1 100
+fr - 6 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-070000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-070000
new file mode 100644
index 0000000..c49e31f
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-070000
@@ -0,0 +1,3 @@
+en - 10 100
+de - 10 100
+fr - 7 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-080000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-080000
new file mode 100644
index 0000000..fe39ef4
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-080000
@@ -0,0 +1,3 @@
+en - 10 100
+de - 6 100
+fr - 5 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-090000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-090000
new file mode 100644
index 0000000..4032e72
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-090000
@@ -0,0 +1,2 @@
+en - 2 100
+fr - 10 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-100000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-100000
new file mode 100644
index 0000000..f1b8aaf
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-100000
@@ -0,0 +1,3 @@
+en - 10 100
+de - 2 100
+fr - 3 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-110000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-110000
new file mode 100644
index 0000000..5d9eeee
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-110000
@@ -0,0 +1,2 @@
+de - 4 100
+fr - 3 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-120000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-120000
new file mode 100644
index 0000000..a3122e3
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-120000
@@ -0,0 +1,3 @@
+en - 9 100
+de - 8 100
+fr - 9 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-130000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-130000
new file mode 100644
index 0000000..a9d5d52
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-130000
@@ -0,0 +1,3 @@
+en - 7 100
+de - 10 100
+fr - 2 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-140000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-140000
new file mode 100644
index 0000000..5ae14a5
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-140000
@@ -0,0 +1 @@
+de - 2 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-150000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-150000
new file mode 100644
index 0000000..d9f9e99
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-150000
@@ -0,0 +1,2 @@
+en - 1 100
+fr - 6 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-160000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-160000
new file mode 100644
index 0000000..5d280f6
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-160000
@@ -0,0 +1,3 @@
+en - 8 100
+de - 4 100
+fr - 8 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-170000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-170000
new file mode 100644
index 0000000..f3b4231
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-170000
@@ -0,0 +1 @@
+en - 6 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-180000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-180000
new file mode 100644
index 0000000..3015d99
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-180000
@@ -0,0 +1,3 @@
+en - 8 100
+de - 1 100
+fr - 7 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-190000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-190000
new file mode 100644
index 0000000..3481f50
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-190000
@@ -0,0 +1,2 @@
+de - 4 100
+fr - 9 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-200000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-200000
new file mode 100644
index 0000000..5b84afe
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-200000
@@ -0,0 +1,2 @@
+en - 2 100
+de - 6 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-210000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-210000
new file mode 100644
index 0000000..fb80dc3
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-210000
@@ -0,0 +1,2 @@
+en - 6 100
+fr - 10 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-220000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-220000
new file mode 100644
index 0000000..fca1e7a
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-220000
@@ -0,0 +1,3 @@
+en - 6 100
+de - 2 100
+fr - 8 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-230000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-230000
new file mode 100644
index 0000000..07dfda7
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141102-230000
@@ -0,0 +1,2 @@
+en - 3 100
+de - 2 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-000000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-000000
new file mode 100644
index 0000000..b9a25ff
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-000000
@@ -0,0 +1,3 @@
+en - 5 100
+de - 6 100
+fr - 2 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-010000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-010000
new file mode 100644
index 0000000..7a7cc98
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-010000
@@ -0,0 +1,3 @@
+en - 4 100
+de - 7 100
+fr - 1 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-020000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-020000
new file mode 100644
index 0000000..e2bfebb
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-020000
@@ -0,0 +1,2 @@
+en - 4 100
+fr - 3 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-030000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-030000
new file mode 100644
index 0000000..769a819
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-030000
@@ -0,0 +1,2 @@
+en - 6 100
+de - 1 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-040000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-040000
new file mode 100644
index 0000000..6f8555c
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-040000
@@ -0,0 +1,2 @@
+en - 4 100
+de - 6 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-050000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-050000
new file mode 100644
index 0000000..7882d2a
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-050000
@@ -0,0 +1,3 @@
+en - 10 100
+de - 9 100
+fr - 6 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-060000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-060000
new file mode 100644
index 0000000..d606e2f
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-060000
@@ -0,0 +1 @@
+de - 3 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-070000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-070000
new file mode 100644
index 0000000..c74e5be
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-070000
@@ -0,0 +1,2 @@
+en - 9 100
+de - 1 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-080000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-080000
new file mode 100644
index 0000000..2466e50
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-080000
@@ -0,0 +1,3 @@
+en - 8 100
+de - 4 100
+fr - 6 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-090000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-090000
new file mode 100644
index 0000000..e37c329
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-090000
@@ -0,0 +1,3 @@
+en - 3 100
+de - 10 100
+fr - 7 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-100000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-100000
new file mode 100644
index 0000000..8d2fa32
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-100000
@@ -0,0 +1,3 @@
+en - 6 100
+de - 1 100
+fr - 7 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-110000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-110000
new file mode 100644
index 0000000..a411fc0
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-110000
@@ -0,0 +1,2 @@
+en - 6 100
+de - 7 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-120000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-120000
new file mode 100644
index 0000000..97476cb
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-120000
@@ -0,0 +1,3 @@
+en - 10 100
+de - 4 100
+fr - 7 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-130000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-130000
new file mode 100644
index 0000000..f193e51
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-130000
@@ -0,0 +1,3 @@
+en - 3 100
+de - 3 100
+fr - 9 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-140000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-140000
new file mode 100644
index 0000000..eaef622
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-140000
@@ -0,0 +1,3 @@
+en - 5 100
+de - 3 100
+fr - 5 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-150000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-150000
new file mode 100644
index 0000000..6ead5be
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-150000
@@ -0,0 +1,3 @@
+en - 6 100
+de - 5 100
+fr - 9 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-160000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-160000
new file mode 100644
index 0000000..88e3715
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-160000
@@ -0,0 +1,2 @@
+en - 1 100
+de - 5 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-170000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-170000
new file mode 100644
index 0000000..25b81aa
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-170000
@@ -0,0 +1,2 @@
+en - 1 100
+de - 7 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-180000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-180000
new file mode 100644
index 0000000..fd852b1
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-180000
@@ -0,0 +1,3 @@
+en - 6 100
+de - 7 100
+fr - 1 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-190000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-190000
new file mode 100644
index 0000000..47a0c2d
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-190000
@@ -0,0 +1 @@
+en - 2 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-200000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-200000
new file mode 100644
index 0000000..57c813d
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-200000
@@ -0,0 +1,2 @@
+de - 5 100
+fr - 4 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-210000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-210000
new file mode 100644
index 0000000..a5e80dd
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-210000
@@ -0,0 +1,3 @@
+en - 2 100
+de - 2 100
+fr - 7 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-220000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-220000
new file mode 100644
index 0000000..bbb3bab
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-220000
@@ -0,0 +1,3 @@
+en - 5 100
+de - 8 100
+fr - 9 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-230000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-230000
new file mode 100644
index 0000000..5f1c8ef
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141103-230000
@@ -0,0 +1,3 @@
+en - 8 100
+de - 5 100
+fr - 8 100
diff --git
a/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141104-000000
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141104-000000
new file mode 100644
index 0000000..4792cdf
--- /dev/null
+++
b/tests/test_projectcounts/fixtures/2014-11-3projects-for-aggregation/2014/2014-11/projectcounts-20141104-000000
@@ -0,0 +1,2 @@
+de - 2 100
+fr - 7 100
diff --git a/tests/test_projectcounts/test_per_project_aggregation.py
b/tests/test_projectcounts/test_per_project_aggregation.py
index c75f400..7b3558e 100644
--- a/tests/test_projectcounts/test_per_project_aggregation.py
+++ b/tests/test_projectcounts/test_per_project_aggregation.py
@@ -273,19 +273,28 @@
])
# Checking calls to additional aggregators
+ expected_dict = {
+ '2014-11-01': '2014-11-01,24276,24276,0,0',
+ '2014-11-02': '2014-11-02,48276,48276,0,0',
+ '2014-11-03': '2014-11-03,72276,72276,0,0'
+ }
self.assertEquals(funcMockParams, [[
self.data_dir_abs,
'enwiki',
- {
- '2014-11-01': '2014-11-01,24276,24276,0,0',
- '2014-11-02': '2014-11-02,48276,48276,0,0',
- '2014-11-03': '2014-11-03,72276,72276,0,0'
- },
+ expected_dict,
first_date,
last_date,
[],
False
- ]])
+ ], [
+ self.data_dir_abs,
+ 'all',
+ expected_dict,
+ first_date,
+ last_date,
+ [],
+ False
+ ]])
def test_update_per_project_additional_aggregators_with_parameters(self):
fixture = self.get_fixture_dir_abs(
@@ -325,16 +334,52 @@
])
# Checking calls to additional aggregators
+ expected_dict = {
+ '2014-11-01': '2014-11-01,24276,24276,0,0',
+ '2014-11-02': '2014-11-02,48276,48276,0,0',
+ '2014-11-03': '2014-11-03,72276,72276,0,0'
+ }
self.assertEquals(funcMockParams, [[
self.data_dir_abs,
'enwiki',
- {
- '2014-11-01': '2014-11-01,24276,24276,0,0',
- '2014-11-02': '2014-11-02,48276,48276,0,0',
- '2014-11-03': '2014-11-03,72276,72276,0,0'
- },
+ expected_dict,
first_date,
last_date,
bad_dates,
True
- ]])
+ ], [
+ self.data_dir_abs,
+ 'all',
+ expected_dict,
+ first_date,
+ last_date,
+ bad_dates,
+ True
+ ]])
+
+ def test_update_per_project_all_projects_aggregation(self):
+ fixture = self.get_fixture_dir_abs(
+ '2014-11-3projects-for-aggregation')
+
+ first_date = datetime.date(2014, 11, 1)
+ last_date = datetime.date(2014, 11, 3)
+
+ enwiki_file_abs = os.path.join(self.daily_raw_dir_abs, 'enwiki.csv')
+ dewiki_file_abs = os.path.join(self.daily_raw_dir_abs, 'dewiki.csv')
+ frwiki_file_abs = os.path.join(self.daily_raw_dir_abs, 'frwiki.csv')
+ self.create_empty_file(enwiki_file_abs)
+ self.create_empty_file(dewiki_file_abs)
+ self.create_empty_file(frwiki_file_abs)
+
+ aggregator.update_per_project_csvs_for_dates(
+ fixture,
+ self.data_dir_abs,
+ first_date,
+ last_date)
+
+ all_file_abs = os.path.join(self.daily_raw_dir_abs, 'all.csv')
+ self.assert_file_content_equals(all_file_abs, [
+ '2014-11-01,323,323,0,0',
+ '2014-11-02,321,321,0,0',
+ '2014-11-03,310,310,0,0',
+ ])
diff --git a/tests/test_util.py b/tests/test_util.py
index 75e4011..34ba5b9 100644
--- a/tests/test_util.py
+++ b/tests/test_util.py
@@ -408,3 +408,60 @@
"2014-05-12,1,2",
"2014-05-13,3,4",
])
+
+ def test_merge_sum_csv_data_dict_with_empty_add_dict(self):
+ dict_1 = {
+ '2014-01-01': '2014-01-01,6,3,2,1',
+ '2014-01-02': '2014-01-02,9,4,3,2',
+ }
+ dict_2 = {}
+ aggregator.merge_sum_csv_data_dict(dict_1, dict_2)
+ self.assertEqual(dict_1, dict_1)
+
+ def test_merge_sum_csv_data_dict_with_empty_base_dict(self):
+ dict_1 = {}
+ dict_2 = {
+ '2014-01-01': '2014-01-01,6,3,2,1',
+ '2014-01-02': '2014-01-02,9,4,3,2',
+ }
+ aggregator.merge_sum_csv_data_dict(dict_1, dict_2)
+ self.assertEqual(dict_1, dict_2)
+
+ def test_merge_sum_csv_data_dict_with_update(self):
+ dict_1 = {
+ '2014-01-01': '2014-01-01,6,3,2,1',
+ '2014-01-03': '2014-01-03,12,5,4,3',
+ }
+ dict_2 = {
+ '2014-01-01': '2014-01-01,6,2,3,1',
+ '2014-01-02': '2014-01-02,6,4,2,3',
+ }
+ aggregator.merge_sum_csv_data_dict(dict_1, dict_2)
+ expected = {
+ '2014-01-01': '2014-01-01,12,5,5,2',
+ '2014-01-02': '2014-01-02,9,4,2,3',
+ '2014-01-03': '2014-01-03,12,5,4,3',
+ }
+ self.assertEqual(dict_1, expected)
+
+ def test_merge_sum_csv_data_dict_with_wrong_value(self):
+ dict_1 = {}
+ dict_2 = {
+ '2014-01-01': 'wrong value',
+ }
+ nose.tools.assert_raises(
+ ValueError,
+ aggregator.merge_sum_csv_data_dict,
+ dict_1,
+ dict_2)
+
+ def test_merge_sum_csv_data_dict_with_non_integers(self):
+ dict_1 = {}
+ dict_2 = {
+ '2014-01-01': '2014-01-01,not,an,integer,value',
+ }
+ nose.tools.assert_raises(
+ ValueError,
+ aggregator.merge_sum_csv_data_dict,
+ dict_1,
+ dict_2)
--
To view, visit https://gerrit.wikimedia.org/r/223031
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ieee012e60d2286e450779cabf253d6a1f11578b4
Gerrit-PatchSet: 1
Gerrit-Project: analytics/aggregator
Gerrit-Branch: master
Gerrit-Owner: Mforns <mfo...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
