Ottomata has uploaded a new change for review. https://gerrit.wikimedia.org/r/60744
Change subject: Adding user_metrics dev env module ...................................................................... Adding user_metrics dev env module Change-Id: Ifc7ef061005cea20ee1eb98aec7805ba6ccb60e8 --- A puppet/modules/user_metrics/README A puppet/modules/user_metrics/manifests/init.pp A puppet/modules/user_metrics/templates/settings.py.erb A puppet/modules/user_metrics/templates/virtual_host.erb 4 files changed, 207 insertions(+), 0 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/vagrant refs/changes/44/60744/1 diff --git a/puppet/modules/user_metrics/README b/puppet/modules/user_metrics/README new file mode 100644 index 0000000..e10bd1d --- /dev/null +++ b/puppet/modules/user_metrics/README @@ -0,0 +1,21 @@ + + +user_metrics puppet module for MediaWiki vagrant VM. + +This uses the MediaWiki database set up by the MediaWiki +module to host a user_metrics dev instance. See: +https://metrics.wikimedia.org for the production version. + +Usage: + +In your site.pp: + + class { 'user_metrics': } + +Then run + + vagrant provision + +That should set up a user_metrics flask web app running in +WSGI in apache. It will be accessible at http://10.11.12.13:8121. + diff --git a/puppet/modules/user_metrics/manifests/init.pp b/puppet/modules/user_metrics/manifests/init.pp new file mode 100644 index 0000000..23f4b0e --- /dev/null +++ b/puppet/modules/user_metrics/manifests/init.pp @@ -0,0 +1,83 @@ +# Dev site for https://metrics.wikimedia.org +# Include this class in your site.pp and run vagrant provision. +# The site will be available at http://10.11.12.13:8182 +# +class user_metrics { + require mysql + require misc::wikimedia + + $site_name = 'metrics.vagrant' + $user_metrics_path = '/vagrant/user_metrics' + $document_root = '/vagrant/user_metrics/user_metrics/api' + $metrics_user = 'vagrant' + + $secret_key = 'nonyabiznass' + + $user_metrics_db_user = 'root' + $user_metrics_db_pass = 'vagrant' + $user_metrics_db_host = '127.0.0.1' + $user_metrics_db_port = 3306 + $user_metrics_db_name = 'user_metrics' + + # connetions will be rendered into settings.py. + $mysql_connections = { + 'cohorts' => { + 'user' => $user_metrics_db_user, + 'passwd' => $user_metrics_db_pass, + 'host' => $user_metrics_db_host, + 'port' => $user_metrics_db_port, + 'db' => $user_metrics_db_name, + }, + 's1' => { + 'user' => $user_metrics_db_user, + 'passwd' => $user_metrics_db_pass, + 'host' => $user_metrics_db_host, + 'port' => $user_metrics_db_port, + 'db' => 'wiki', + } + } + + package { ['python-flask', 'python-flask-login', 'python-mysqldb', 'python-numpy']: + ensure => 'installed', + } + + # # clone the E3 Analysis repository + # git::clone { 'user_metrics': + # directory => $user_metrics_path, + # origin => 'https://gerrit.wikimedia.org/r/p/analytics/E3Analysis.git', + # require => [Package['python-flask']], + # ensure => 'latest', + # } + + git::clone { 'analytics/E3Analysis': + directory => $user_metrics_path, + require => [Package['python-flask']], + } + + # create the user_metrics cohorts database + exec { 'user_metrics_mysql_create_database': + command => "/usr/bin/mysql -pvagrant -e \"CREATE DATABASE ${user_metrics_db_name};\" &&/usr/bin/mysql -pvagrant ${user_metrics_db_name} < ${user_metrics_path}/scripts/user_metrics.sql;", + unless => "/usr/bin/mysql -pvagrant -e 'SHOW DATABASES' | /bin/grep -q ${user_metrics_db_name}", + user => 'root', + logoutput => true, + require => [Git::Clone['analytics/E3Analysis'], Service['mysql']] + } + + # Need settings.py to configure metrics-api python application + file { "${user_metrics_path}/user_metrics/config/settings.py": + content => template('user_metrics/settings.py.erb'), + require => Git::Clone['analytics/E3Analysis'], + } + + include apache + + package { 'libapache2-mod-wsgi': + ensure => installed, + } + apache::mod { "wsgi": require => Package['libapache2-mod-wsgi'] } + + apache::site { $site_name: + content => template("user_metrics/virtual_host.erb"), + require => [Git::Clone['analytics/E3Analysis'], Apache::Mod['wsgi'], Apache::Mod['alias']], + } +} diff --git a/puppet/modules/user_metrics/templates/settings.py.erb b/puppet/modules/user_metrics/templates/settings.py.erb new file mode 100644 index 0000000..ca18389 --- /dev/null +++ b/puppet/modules/user_metrics/templates/settings.py.erb @@ -0,0 +1,73 @@ + +from user_metrics.config import logging +from pkg_resources import WorkingSet , DistributionNotFound, VersionConflict +from os.path import exists +from urllib2 import urlopen +import json + +# Get working set of Python modules +working_set = WorkingSet() + + +# Project settings +# ================ + +__instance_host__ = '<%= site_name %>' +__instance_port__ = 5000 + +__project_home__ = '<%= user_metrics_path %>' +__web_home__ = ''.join([__project_home__, 'user_metrics/api/']) +__data_file_dir__ = ''.join([__project_home__, 'data/']) + +__query_module__ = 'user_metrics.query.query_calls_sql' +__user_thread_max__ = 50 +__rev_thread_max__ = 20 +__time_series_thread_max__ = 6 + +__cohort_data_instance__ = 'cohorts' +__cohort_db__ = 'usertags' +__cohort_meta_db__ = 'usertags_meta' +__cohort_meta_instance__ = 'prod' + +__secret_key__ = '<%= secret_key %>' + +try: + working_set.require('Flask-Login>=0.1.2') + __flask_login_exists__ = True + logging.debug(__name__ + ' :: Using module flask.ext.login...') +except (DistributionNotFound, VersionConflict): + __flask_login_exists__ = False + logging.debug(__name__ + ' :: Can\'t find module flask.ext.login...') + + +# Database connection settings +# ============================ + +connections = { +<% mysql_connections.keys.sort.each do |name| -%> + <%= "'#{name}'" %>: { +<% mysql_connections[name].keys.sort.each do |key| -%> + <%= "'#{key}'" %>: <%= (mysql_connections[name][key] =~ /^\d+$/) ? mysql_connections[name][key] : "'#{mysql_connections[name][key]}'" %>, +<% end # connection..each -%> + }, +<% end # mysql_connections..each -%> +} + +def get_project_host_map(usecache=True): + cache_name = 'project_host_map.json' + if not exists(cache_name) or not usecache: + cluster_url_fmt = 'http://noc.wikimedia.org/conf/s%d.dblist' + host_fmt = 's%d' + project_host_map = {} + for i in range(1,8): + host = host_fmt % i + url = cluster_url_fmt % i + projects = urlopen(url).read().splitlines() + for project in projects: + project_host_map[project] = host + json.dump(project_host_map, open(cache_name, 'w')) + else: + project_host_map = json.load(open(cache_name)) + return project_host_map + +PROJECT_DB_MAP = get_project_host_map() diff --git a/puppet/modules/user_metrics/templates/virtual_host.erb b/puppet/modules/user_metrics/templates/virtual_host.erb new file mode 100644 index 0000000..bf08175 --- /dev/null +++ b/puppet/modules/user_metrics/templates/virtual_host.erb @@ -0,0 +1,30 @@ +# Note: This file is managed by Puppet. +<% +# ERb template variables: +# +# site_name +# document_root +# user_metrics_path +# metrics_user +# +-%> + +Listen 8182 +<VirtualHost *:8182> + ServerName <%= site_name %> + ServerAdmin http://www.mediawiki.org/wiki/Vagrant + + WSGIDaemonProcess api user=<%= metrics_user %> group=www-data threads=5 python-path=<%= user_metrics_path %> + WSGIScriptAlias / <%= document_root %>/api.wsgi + + <Directory <%= document_root %>> + WSGIProcessGroup api + WSGIApplicationGroup %{GLOBAL} + + Options Indexes FollowSymLinks MultiViews + AllowOverride None + Order deny,allow + Allow from all + </Directory> +</VirtualHost> + -- To view, visit https://gerrit.wikimedia.org/r/60744 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Ifc7ef061005cea20ee1eb98aec7805ba6ccb60e8 Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/vagrant Gerrit-Branch: master Gerrit-Owner: Ottomata <o...@wikimedia.org> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
