Milimetric has uploaded a new change for review. https://gerrit.wikimedia.org/r/97753
Change subject: Config files can be specified in environment ...................................................................... Config files can be specified in environment Change-Id: I9c386af13a93f53a428e0d304b2ea262421e5282 Card: analytics 1270 --- M README.md M tests/__init__.py M tests/fixtures.py R wikimetrics/config/queue_config.yaml M wikimetrics/configurables.py M wikimetrics/run.py 6 files changed, 41 insertions(+), 24 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/analytics/wikimetrics refs/changes/53/97753/1 diff --git a/README.md b/README.md index 2c7fde7..5807e65 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ ### Development Environment -Wikimetrics consists of a website that runs on Flask and an asynchronous queue implemented with Celery. The Celery queue stores its results in Redis and the Flask website stores metadata in MySQL. To set up your dev environment, run the following or change it slightly to use whatever package manager you're using (brew, yum, etc.): +Wikimetrics consists of a website that runs on Flask and an asynchronous queue implemented with Celery. The Celery queue stores its results in Redis and the Flask website stores metadata in MySQL. To set up your dev environment, run the following or change it slightly to use whatever package manager you're using (brew, yum, etc.). A fairly recent version of pip is required. Version 1.4.1 is known working, 1.0.2 is known not working: ```` $ sudo apt-get install libmysqlclient-dev python-dev redis-server @@ -16,20 +16,26 @@ $ sudo pip install -e . ```` -A fairly recent version of pip is required. Version 1.4.1 is known working, 1.0.2 is known not working. +Now you need to set up your mysql databases. You just need empty databases because sqlalchemy will create the tables it needs: + +```` +$ sudo scripts/00_create_wikimetrics_db +$ sudo scripts/01_create_enwiki_db +$ sudo scripts/02_create_dewiki_db +```` Wikimetrics has over 90% unit test coverage. We use Nose to write unit and integration tests to achieve this, and you should too. ```` $ sudo pip install nose -$ nosetests +$ scripts/test ```` -OK, so the only thing is to run the tool. In two separate command lines, start the dev web server and the celery queue to process report requests. +OK, so now you can run the tool. In two separate command lines, start the dev web server and the celery queue to process report requests. ```` $ wikimetrics --mode web -$ wikimetrics --mode celery +$ wikimetrics --mode queue ```` go to [localhost:5000](http://localhost:5000) diff --git a/tests/__init__.py b/tests/__init__.py index 0131b3e..b239005 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -27,7 +27,7 @@ app.config['TESTING'] = True celery_out = open(devnull, "w") - celery_cmd = ['wikimetrics', '--mode', 'celery'] + celery_cmd = ['wikimetrics', '--mode', 'queue'] global celery_proc celery_proc = Popen(celery_cmd, stdout=celery_out, stderr=celery_out) diff --git a/tests/fixtures.py b/tests/fixtures.py index 2da2d11..01be389 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -1,5 +1,4 @@ import unittest -import celery import sys from itertools import product from datetime import datetime diff --git a/wikimetrics/config/celery_config.yaml b/wikimetrics/config/queue_config.yaml similarity index 100% rename from wikimetrics/config/celery_config.yaml rename to wikimetrics/config/queue_config.yaml diff --git a/wikimetrics/configurables.py b/wikimetrics/configurables.py index a48ff0e..a7a7157 100644 --- a/wikimetrics/configurables.py +++ b/wikimetrics/configurables.py @@ -80,15 +80,14 @@ db.config.__dict__.update(config_override) -def config_celery(args): +def config_queue(args): from celery import Celery # create and configure celery app - global queue queue = Celery('wikimetrics', include=['wikimetrics']) - celery_config = create_dict_from_text_config_file(args.celery_config) - queue.config_from_object(celery_config) + queue_config = create_dict_from_text_config_file(args.queue_config) + queue.config_from_object(queue_config) if args.override_config: queue.config_from_object(args.override_config) diff --git a/wikimetrics/run.py b/wikimetrics/run.py index 5044dcb..21e3907 100644 --- a/wikimetrics/run.py +++ b/wikimetrics/run.py @@ -3,7 +3,8 @@ import sys import logging import pprint -from .configurables import config_web, config_db, config_celery +from os import environ as env +from .configurables import config_web, config_db, config_queue logger = logging.getLogger(__name__) @@ -20,7 +21,7 @@ nose.run(module='tests') -def run_celery(): +def run_queue(): from configurables import queue queue.start(argv=['celery', 'worker', '-l', queue.conf['LOG_LEVEL']]) @@ -44,33 +45,45 @@ 'import', 'web', 'test', - 'celery', + 'queue', ], # NOTE: flake made me format the strings this way, nothing could be uglier help=''' web : runs flask webserver... test : run nosetests... - celery : runs celery worker... + queue : runs celery worker... import : configures everything and runs nothing... ''', ) + # get defaults from environment variables + web_config = 'wikimetrics/config/web_config.yaml' + if 'WIKIMETRICS_WEB_CONFIG' in env: + web_config = env['WIKIMETRICS_WEB_CONFIG'] + db_config = 'wikimetrics/config/db_config.yaml' + if 'WIKIMETRICS_DB_CONFIG' in env: + db_config = env['WIKIMETRICS_DB_CONFIG'] + queue_config = 'wikimetrics/config/queue_config.yaml' + if 'WIKIMETRICS_QUEUE_CONFIG' in env: + queue_config = env['WIKIMETRICS_QUEUE_CONFIG'] + + # add parser arguments with those defaults parser.add_argument( '--web-config', '-w', - default='wikimetrics/config/web_config.yaml', + default=web_config, help='Flask config file', dest='web_config', ) parser.add_argument( '--db-config', '-d', - default='wikimetrics/config/db_config.yaml', + default=db_config, help='Database config file', dest='db_config', ) parser.add_argument( - '--celery-config', '-c', - default='wikimetrics/config/celery_config.yaml', + '--queue-config', '-c', + default=queue_config, help='Celery config file', - dest='celery_config', + dest='queue_config', ) return parser @@ -83,10 +96,10 @@ args, others = parser.parse_known_args() logger.info('running with arguments:\n%s', pprint.pformat(vars(args))) -# runs the appropriate config function (web, celery, test) +# runs the appropriate config function (web, queue, test) config_web(args) config_db(args) -config_celery(args) +config_queue(args) def main(): @@ -95,8 +108,8 @@ run_web() elif args.mode == 'test': run_test() - elif args.mode == 'celery': - run_celery() + elif args.mode == 'queue': + run_queue() elif args.mode == 'import': pass -- To view, visit https://gerrit.wikimedia.org/r/97753 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I9c386af13a93f53a428e0d304b2ea262421e5282 Gerrit-PatchSet: 1 Gerrit-Project: analytics/wikimetrics Gerrit-Branch: master Gerrit-Owner: Milimetric <dandree...@wikimedia.org> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
