Adamw has submitted this change and it was merged.
Change subject: python libraries are available in common
......................................................................
python libraries are available in common
Deprecate virtualenv jail, it had no walls.
Other than refactoring, the config semantics changed
slightly. I'm unhappy with the import-by-value issue,
"from process.globals import config" will not update
config during initialization, so you need to call
load_config before importing any other tools/ files.
TODO:
* moar docs
Change-Id: I5bc6741852b95b67c6354c4634369ab7c7423fd8
---
M .gitignore
D audit/globalcollect/db.py
M audit/globalcollect/history.py
M audit/paypal/history.py
D banner_screenshot/mediawiki/__init__.py
D banner_screenshot/mediawiki/centralnotice.py
D banner_screenshot/mediawiki/time_util.py
M banner_screenshot/shoot_banners
A civicrm/__init__.py
R civicrm/civicrm.py
A database/__init__.py
R database/db.py
R fundraising_ab_tests/__init__.py
R fundraising_ab_tests/campaign_log.py
R fundraising_ab_tests/confidence.py
R fundraising_ab_tests/results.py
R fundraising_ab_tests/results_gdoc.py
R fundraising_ab_tests/spec.py
R fundraising_ab_tests/spec_gdoc.py
A google/__init__.py
R google/gdocs.py
D junkyard/.gitignore
M live_analysis/README
M live_analysis/dump_tests.py
D live_analysis/jail.py
M live_analysis/publish_results
M live_analysis/update_spec_sheet
A mediawiki/__init__.py
R mediawiki/api.py
R mediawiki/centralnotice/__init__.py
R mediawiki/centralnotice/api.py
R mediawiki/centralnotice/contributions.py
R mediawiki/centralnotice/impressions.py
R mediawiki/centralnotice/time_util.py
R mediawiki/centralnotice/top_ten.py
A process/__init__.py
A process/globals.py
R process/lock.py
A stats/__init__.py
R stats/stats_abba.py
40 files changed, 104 insertions(+), 224 deletions(-)
Approvals:
Adamw: Verified; Looks good to me, approved
diff --git a/.gitignore b/.gitignore
index d4a8918..8711254 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,5 @@
+*.pyc
+
# IDE Files
.idea
diff --git a/audit/globalcollect/db.py b/audit/globalcollect/db.py
deleted file mode 100644
index 30a2e6c..0000000
--- a/audit/globalcollect/db.py
+++ /dev/null
@@ -1,17 +0,0 @@
-import MySQLdb as Dbi
-
-class Connection(object):
- def __init__(self, host=None, user=None, passwd=None, db=None,
debug=False, **ignore):
- self.db_conn = Dbi.connect(host=host, user=user, passwd=passwd, db=db)
- self.debug = debug
-
- def close(self):
- self.db_conn.commit()
- pass
-
- def execute(self, sql, params=()):
- cursor = self.db_conn.cursor(cursorclass=Dbi.cursors.DictCursor)
- if self.debug:
- print(sql % params)
- cursor.execute(sql, params)
- cursor.close()
diff --git a/audit/globalcollect/history.py b/audit/globalcollect/history.py
index 38b5a08..b4f53fe 100755
--- a/audit/globalcollect/history.py
+++ b/audit/globalcollect/history.py
@@ -11,7 +11,7 @@
import csv
import atexit
-from db import Connection as DbConnection
+from database.db import Connection as DbConnection
def main():
global config, options, db
diff --git a/audit/paypal/history.py b/audit/paypal/history.py
index 93d1802..061e72e 100755
--- a/audit/paypal/history.py
+++ b/audit/paypal/history.py
@@ -11,7 +11,7 @@
import gzip
import locale
import dateutil.parser
-import civicrm
+from civicrm.civicrm import Civicrm
def main():
global config, messaging, options, civi
@@ -36,7 +36,7 @@
log("*** Dummy mode! Not injecting stomp messages ***")
messaging = Stomp(config)
- civi = civicrm.Civicrm(config)
+ civi = Civicrm(config)
locale.setlocale(locale.LC_NUMERIC, "")
diff --git a/banner_screenshot/mediawiki/__init__.py
b/banner_screenshot/mediawiki/__init__.py
deleted file mode 100644
index 247139a..0000000
--- a/banner_screenshot/mediawiki/__init__.py
+++ /dev/null
@@ -1,22 +0,0 @@
-'''
-Dumb interface to the MediaWiki api.
-'''
-
-import config
-
-import json
-
-def mw_call( args ):
- import simplemediawiki
-
- wiki = simplemediawiki.MediaWiki(
- config.centralnotice_mw_api,
- user_agent='bot: fr-screenshots'
- )
- result = wiki.call( args )
- if 'error' in result:
- raise RuntimeError(json.dumps(result, indent=4).replace('\\n', '\n'))
- val = result[ args['action'] ]
- if 'list' in args:
- val = val[ args['list'] ]
- return val
diff --git a/banner_screenshot/mediawiki/centralnotice.py
b/banner_screenshot/mediawiki/centralnotice.py
deleted file mode 100644
index 4133d29..0000000
--- a/banner_screenshot/mediawiki/centralnotice.py
+++ /dev/null
@@ -1,56 +0,0 @@
-'''
-Interface to the MediaWiki CentralNotice api
-'''
-
-from mediawiki import mw_call
-
-cached_campaigns = {}
-
-def get_banners( **kw ):
- if 'campaign' in kw:
- campaign = get_campaign( kw['campaign'] )
- return campaign['banners'].keys()
- return get_allocations( **kw )
-
-def get_campaign( campaign ):
- #TODO: push caching down into mediawiki.mw_call, with optional invalidation
- global cached_campaigns
- if campaign in cached_campaigns:
- return cached_campaigns[campaign]
-
- #if '__iter__' in campaign: return get_campaigns
- result = mw_call( {
- 'action': 'centralnoticequerycampaign',
- 'campaign': campaign,
- } )
-
- if campaign in result:
- cached_campaigns[campaign] = result[campaign]
- return cached_campaigns[campaign]
-
-def get_campaigns( campaigns ):
- #FIXME cache
- return mw_call( {
- 'action': 'centralnoticequerycampaign',
- 'campaign': '|'.join( campaigns ),
- } )
-
-def get_allocations( project=None, language=None, country=None,
anonymous=True, bucket='0' ):
- result = mw_call( {
- 'action': 'centralnoticeallocations',
- 'project': project,
- 'language': language,
- 'country': country,
- 'anonymous': anonymous,
- 'bucket': bucket,
- 'minimal': 'false'
- } )
- return result['banners']
-
-def get_campaign_logs( since=None ):
- result = mw_call( {
- 'action': 'query',
- 'list': 'centralnoticelogs',
- 'start': since,
- } )
- return result['logs']
diff --git a/banner_screenshot/mediawiki/time_util.py
b/banner_screenshot/mediawiki/time_util.py
deleted file mode 100644
index b02c2a6..0000000
--- a/banner_screenshot/mediawiki/time_util.py
+++ /dev/null
@@ -1,17 +0,0 @@
-from datetime import datetime, timedelta
-
-def str_time_offset(str_time=None, **delta_args):
- if not str_time:
- str_time = str_now()
- time_time = datetime.strptime( str_time, '%Y%m%d%H%M%S' )
- str_time = ( time_time + timedelta( **delta_args )).strftime(
'%Y%m%d%H%M%S' )
- return(str_time)
-
-def str_now():
- return( datetime.utcnow().strftime('%Y%m%d%H%M%S') )
-
-def datetimefunix( unix_timestamp ):
- return datetime.fromtimestamp(unix_timestamp)
-
-def strfunix( unix_timestamp ):
- return datetime.fromtimestamp(unix_timestamp).strftime('%Y-%m-%d %H:%M')
diff --git a/banner_screenshot/shoot_banners b/banner_screenshot/shoot_banners
index 17c78c4..a700e6c 100755
--- a/banner_screenshot/shoot_banners
+++ b/banner_screenshot/shoot_banners
@@ -6,11 +6,13 @@
import re
import sys
-import config
-from mediawiki.centralnotice import get_campaign_logs
-from mediawiki.time_util import str_time_offset
+from process.globals import load_config
+load_config("config.py")
-JS_RENDER_SCRIPT = os.path.join(os.path.dirname(os.path.abspath(__file__)),
"rasterize.js")
+from process.globals import config
+from mediawiki.centralnotice.api import get_campaign_logs
+from mediawiki.centralnotice.time_util import str_time_offset
+from process.lock import begin, end
def reduce_banners(campaign_logs):
'''Return a map from banner names to most recent campaign settings.'''
@@ -47,6 +49,8 @@
return os.path.exists(get_screenshot_path(name, lang))
def render(name, lang, country):
+ global JS_RENDER_SCRIPT
+
url = config.article_url % { "banner": name, "lang": lang, "country":
country }
path = get_screenshot_path(name, lang)
dir = os.path.dirname(path)
@@ -75,8 +79,15 @@
if __name__ == "__main__":
- if len(sys.argv) > 1:
- for name in sys.argv[1:]:
- screenshot_banner(name)
- else:
- process_banners()
+ try:
+ begin()
+
+ JS_RENDER_SCRIPT =
os.path.join(os.path.dirname(os.path.abspath(__file__)), "rasterize.js")
+
+ if len(sys.argv) > 1:
+ for name in sys.argv[1:]:
+ screenshot_banner(name)
+ else:
+ process_banners()
+ finally:
+ end()
diff --git a/civicrm/__init__.py b/civicrm/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/civicrm/__init__.py
diff --git a/audit/paypal/civicrm.py b/civicrm/civicrm.py
similarity index 100%
rename from audit/paypal/civicrm.py
rename to civicrm/civicrm.py
diff --git a/database/__init__.py b/database/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/database/__init__.py
diff --git a/live_analysis/db.py b/database/db.py
similarity index 97%
rename from live_analysis/db.py
rename to database/db.py
index d454465..3ea43cd 100644
--- a/live_analysis/db.py
+++ b/database/db.py
@@ -3,6 +3,8 @@
'''
import MySQLdb as Dbi
+from process.globals import config
+
class Connection(object):
def __init__(self, host=None, user=None, passwd=None, db=None,
debug=False):
self.db_conn = Dbi.connect(host=host, user=user, passwd=passwd, db=db)
@@ -56,8 +58,8 @@
def get_db():
'''Convienience'''
- import config
global db_conn
+
if not db_conn:
db_conn = Connection(**config.db_params)
return db_conn
diff --git a/live_analysis/fr/tests/__init__.py
b/fundraising_ab_tests/__init__.py
similarity index 94%
rename from live_analysis/fr/tests/__init__.py
rename to fundraising_ab_tests/__init__.py
index f0e5cc3..01a2efe 100644
--- a/live_analysis/fr/tests/__init__.py
+++ b/fundraising_ab_tests/__init__.py
@@ -4,14 +4,14 @@
These are not unit tests ;) they are WMF Fundraising A/B tests.
'''
-from fr.centralnotice import get_campaign
-from fr.tests.results import get_banner_results
+import mediawiki.centralnotice.api
+from results import get_banner_results
class FrTest(object):
def __init__(self, label=None, type="", campaign=None, banners=None,
start=None, end=None, disabled=False, **ignore):
print "Warning: ignoring columns: %s" % (", ".join(ignore.keys()), )
- self.campaign = get_campaign(campaign)
+ self.campaign = mediawiki.centralnotice.api.get_campaign(campaign)
if not self.campaign:
print "Warning: no such campaign '%s'" % campaign
diff --git a/live_analysis/fr/tests/campaign_log.py
b/fundraising_ab_tests/campaign_log.py
similarity index 86%
rename from live_analysis/fr/tests/campaign_log.py
rename to fundraising_ab_tests/campaign_log.py
index 2b0dc50..e1a7bca 100644
--- a/live_analysis/fr/tests/campaign_log.py
+++ b/fundraising_ab_tests/campaign_log.py
@@ -1,7 +1,4 @@
-'''
-'''
-
-from fr.tests import FrTest
+from fundraising_ab_tests import FrTest
def tests_from_entry(entry):
'''
@@ -40,8 +37,8 @@
return (test_from_entry('begin'), test_from_entry('end'), )
def get_relevant_events():
- from fr.centralnotice import get_campaign_logs
- import fr.time_util
+ from mediawiki.centralnotice.api import get_campaign_logs
+ from mediawiki.centralnotice import time_util
def is_relevant(entry):
'''
@@ -50,5 +47,5 @@
if 'enabled' in entry['added'] or entry['begin']['enabled'] is 1:
return True
- logs = get_campaign_logs(since=fr.time_util.str_time_offset(days=-1))
+ logs = get_campaign_logs(since=time_util.str_time_offset(days=-1))
return [ tests_from_entry(e) for e in reversed(logs) if is_relevant(e) ]
diff --git a/live_analysis/fr/tests/confidence.py
b/fundraising_ab_tests/confidence.py
similarity index 96%
rename from live_analysis/fr/tests/confidence.py
rename to fundraising_ab_tests/confidence.py
index 2c02f31..69e2771 100644
--- a/live_analysis/fr/tests/confidence.py
+++ b/fundraising_ab_tests/confidence.py
@@ -1,5 +1,5 @@
-import config
-from stats_abba import Experiment
+from process.globals import config
+from stats.stats_abba import Experiment
def add_confidence(results, name_column, successes_column):
confidence = get_confidence(results, name_column, successes_column)
diff --git a/live_analysis/fr/tests/results.py b/fundraising_ab_tests/results.py
similarity index 87%
rename from live_analysis/fr/tests/results.py
rename to fundraising_ab_tests/results.py
index 8179732..d9704da 100644
--- a/live_analysis/fr/tests/results.py
+++ b/fundraising_ab_tests/results.py
@@ -1,10 +1,10 @@
import re
import json
-import config
-from fr.contributions import get_totals
-from fr.impressions import get_impressions
-from fr.tests.confidence import add_confidence
+from process.globals import config
+from mediawiki.centralnotice.contributions import get_totals
+from mediawiki.centralnotice.impressions import get_impressions
+from fundraising_ab_tests.confidence import add_confidence
class TestResult(object):
def __init__(self, criteria=None, results={}):
diff --git a/live_analysis/fr/tests/results_gdoc.py
b/fundraising_ab_tests/results_gdoc.py
similarity index 97%
rename from live_analysis/fr/tests/results_gdoc.py
rename to fundraising_ab_tests/results_gdoc.py
index 64bf0a6..c52b7d7 100644
--- a/live_analysis/fr/tests/results_gdoc.py
+++ b/fundraising_ab_tests/results_gdoc.py
@@ -1,4 +1,4 @@
-from gdocs import Spreadsheet
+from google.gdocs import Spreadsheet
def write_gdoc_results(doc=None, results=[]):
print "Writing test results to %s" % doc
diff --git a/live_analysis/fr/tests/spec.py b/fundraising_ab_tests/spec.py
similarity index 96%
rename from live_analysis/fr/tests/spec.py
rename to fundraising_ab_tests/spec.py
index e121617..099ec3d 100644
--- a/live_analysis/fr/tests/spec.py
+++ b/fundraising_ab_tests/spec.py
@@ -5,7 +5,7 @@
import re
-from fr.tests import FrTest
+from fundraising_ab_tests import FrTest
import campaign_log
def parse_spec(spec):
diff --git a/live_analysis/fr/tests/spec_gdoc.py
b/fundraising_ab_tests/spec_gdoc.py
similarity index 96%
rename from live_analysis/fr/tests/spec_gdoc.py
rename to fundraising_ab_tests/spec_gdoc.py
index 6d130da..3c56201 100644
--- a/live_analysis/fr/tests/spec_gdoc.py
+++ b/fundraising_ab_tests/spec_gdoc.py
@@ -1,6 +1,5 @@
-#import jail
from spec import FrTestSpec, parse_spec
-from gdocs import Spreadsheet
+from google.gdocs import Spreadsheet
def read_gdoc_spec(doc=None):
return
FrTestSpec(spec=list(parse_spec(Spreadsheet(doc=doc).get_all_rows())))
diff --git a/google/__init__.py b/google/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/google/__init__.py
diff --git a/live_analysis/gdocs.py b/google/gdocs.py
similarity index 98%
rename from live_analysis/gdocs.py
rename to google/gdocs.py
index abaa31d..edad349 100644
--- a/live_analysis/gdocs.py
+++ b/google/gdocs.py
@@ -6,7 +6,7 @@
import gdata.docs.data
import gdata.spreadsheet.service
-import config
+from process.globals import config
# TODO: ExecuteBatch; 2-leg oauth
# TODO: cache rows locally, operate and then flush
diff --git a/junkyard/.gitignore b/junkyard/.gitignore
deleted file mode 100644
index 72e8ffc..0000000
--- a/junkyard/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-*
diff --git a/live_analysis/README b/live_analysis/README
index 13d7df1..dea3ecb 100644
--- a/live_analysis/README
+++ b/live_analysis/README
@@ -42,11 +42,8 @@
As root:
- pip install virtualenv MySQL-python
+ pip install MySQL-python simplemediawiki kitchen gdata
In this directory:
-
- virtualenv jailenv
- PATH=jailenv/bin pip install simplemediawiki kitchen gdata
cp config.py.example config.py # and edit
diff --git a/live_analysis/dump_tests.py b/live_analysis/dump_tests.py
index 3a905cd..e18eede 100755
--- a/live_analysis/dump_tests.py
+++ b/live_analysis/dump_tests.py
@@ -5,8 +5,9 @@
import csv
import sys
-from fr.tests.spec import FrTestSpec, parse_spec
-from fr.centralnotice import get_campaign_logs
+import mediawiki.centralnotice.api
+
+import process.globals
def is_relevant(entry):
'''
@@ -33,7 +34,7 @@
pagesize = 500
while True:
- logs = get_campaign_logs(limit=pagesize, offset=cur)
+ logs = mediawiki.centralnotice.api.get_campaign_logs(limit=pagesize,
offset=cur)
for test in logs:
if is_relevant(test):
@@ -50,4 +51,6 @@
cur = cur + pagesize
-fetch()
+if __name__ == "__main__":
+ process.globals.load_config("config.py")
+ fetch()
diff --git a/live_analysis/jail.py b/live_analysis/jail.py
deleted file mode 100644
index f665022..0000000
--- a/live_analysis/jail.py
+++ /dev/null
@@ -1,14 +0,0 @@
-'''
-Phony "jail" provides inline "virtualenv" execution
-'''
-import os
-import subprocess
-
-jail_path = os.path.dirname(os.path.abspath(__file__)) + "/jailenv"
-
-def run(source):
- env = os.environ
- env['PATH'] = jail_path + "/bin:" + env['PATH']
- ret = subprocess.call( [ "python", "-c", source ], env=env )
- if ret is not 0:
- raise Exception("Failed to run jail command, result=%d" % ret)
diff --git a/live_analysis/publish_results b/live_analysis/publish_results
index 159a4b4..c24457e 100755
--- a/live_analysis/publish_results
+++ b/live_analysis/publish_results
@@ -1,32 +1,18 @@
#!/usr/bin/env python
-import json
-import cPickle as pickle
-import os
+from process.globals import load_config
+load_config("config.py")
-import config
-import jail
-import lock
+from process.lock import begin, end
+from fundraising_ab_tests.spec_gdoc import read_gdoc_spec
+from fundraising_ab_tests.results_gdoc import update_gdoc_results
-unique = os.environ['LOGNAME']
-spec_tmp = "/tmp/%s-tests.pickle" % unique
-results_tmp = "/tmp/%s-test_results.json" % unique
-lockfile = "/tmp/%s-franal-publish-proc.lock" % unique
+from process.globals import config
-lock.begin(filename=lockfile)
+begin()
-# do gdocs and simplemediawiki queries in an insolated environment
-#FIXME: security is illusory
-jail.run("""
-from cPickle import dump
-from fr.tests.spec_gdoc import read_gdoc_spec
-
-print "Reading test specifications from %(url)s"
-tests = read_gdoc_spec(doc="%(url)s")
-dump(tests, open("%(out)s", "w"))
-""" % {'url': config.test_spec_url, 'out': spec_tmp})
-
-tests = pickle.load(open(spec_tmp, "r"))
+print "Reading test specifications from %s" % (config.test_spec_url, )
+tests = read_gdoc_spec(doc=config.test_spec_url)
# Compile statistics from the database
results = []
@@ -39,15 +25,7 @@
results.extend([r.__dict__ for r in test.results])
#results.extend(test.results)
-open(results_tmp, "w").write(json.dumps(results, indent=4))
-
# store in gdocs spreadsheet
-jail.run("""
-from json import loads
-from fr.tests.results_gdoc import update_gdoc_results
+update_gdoc_results(doc=config.test_results_url, results=results)
-results = loads(open("%(results)s", "r").read())
-update_gdoc_results(doc="%(url)s", results=results)
-""" % {'results': results_tmp, 'url': config.test_results_url})
-
-lock.end()
+end()
diff --git a/live_analysis/update_spec_sheet b/live_analysis/update_spec_sheet
index 453ba38..d114378 100755
--- a/live_analysis/update_spec_sheet
+++ b/live_analysis/update_spec_sheet
@@ -1,21 +1,15 @@
#!/usr/bin/env python
-import os
+from process.globals import load_config
+load_config("config.py")
-import config
-import jail
-import lock
+from process.lock import begin, end
+from fundraising_ab_tests.spec_gdoc import update_gdoc_spec
-unique = os.environ['LOGNAME']
-lockfile = "/tmp/%s-franal-update-proc.lock" % unique
+from process.globals import config
-lock.begin(filename=lockfile)
+begin()
-# do gdocs and simplemediawiki queries in an insolated environment
-#FIXME: security is illusory
-jail.run("""
-from fr.tests.spec_gdoc import update_gdoc_spec
-update_gdoc_spec(doc="%(url)s")
-""" % {'url': config.test_spec_url})
+update_gdoc_spec(doc=config.test_spec_url)
-lock.end()
+end()
diff --git a/mediawiki/__init__.py b/mediawiki/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/mediawiki/__init__.py
diff --git a/live_analysis/fr/mediawiki.py b/mediawiki/api.py
similarity index 92%
rename from live_analysis/fr/mediawiki.py
rename to mediawiki/api.py
index ed7f0b5..8c6bd87 100644
--- a/live_analysis/fr/mediawiki.py
+++ b/mediawiki/api.py
@@ -2,7 +2,7 @@
Dumb interface to the MediaWiki api.
'''
-import config
+from process.globals import config
import json
diff --git a/live_analysis/fr/__init__.py b/mediawiki/centralnotice/__init__.py
similarity index 100%
rename from live_analysis/fr/__init__.py
rename to mediawiki/centralnotice/__init__.py
diff --git a/live_analysis/fr/centralnotice.py b/mediawiki/centralnotice/api.py
similarity index 97%
rename from live_analysis/fr/centralnotice.py
rename to mediawiki/centralnotice/api.py
index b661cf5..9589a4a 100644
--- a/live_analysis/fr/centralnotice.py
+++ b/mediawiki/centralnotice/api.py
@@ -2,7 +2,7 @@
Interface to the MediaWiki CentralNotice api
'''
-from mediawiki import mw_call
+from mediawiki.api import mw_call
cached_campaigns = {}
diff --git a/live_analysis/fr/contributions.py
b/mediawiki/centralnotice/contributions.py
similarity index 97%
rename from live_analysis/fr/contributions.py
rename to mediawiki/centralnotice/contributions.py
index 6f8d141..1c8e47d 100644
--- a/live_analysis/fr/contributions.py
+++ b/mediawiki/centralnotice/contributions.py
@@ -7,9 +7,11 @@
import os
import decimal
+from process.globals import config
+from database import db
+
import time_util
import top_ten
-import config
ct_banner_clause = "LEFT(SUBSTRING_INDEX(SUBSTRING_INDEX(utm_source, '.',
2),'.',1), LENGTH(SUBSTRING_INDEX(SUBSTRING_INDEX(utm_source, '.', 2),'.',1)))"
@@ -36,8 +38,6 @@
'''
Note that the column names must match a heading in the results spreadsheet.
'''
- import db
-
if not query:
query = db.Query()
query.columns.append('SUM(total_amount) AS total')
diff --git a/live_analysis/fr/impressions.py
b/mediawiki/centralnotice/impressions.py
similarity index 94%
rename from live_analysis/fr/impressions.py
rename to mediawiki/centralnotice/impressions.py
index 232df64..3d54392 100644
--- a/live_analysis/fr/impressions.py
+++ b/mediawiki/centralnotice/impressions.py
@@ -1,6 +1,6 @@
-def get_impressions(campaign=None, banner=None, **ignore):
- import db
+from database import db
+def get_impressions(campaign=None, banner=None, **ignore):
query = db.Query()
query.columns.append("SUM(count) AS count")
query.tables.append("pgehres.bannerimpressions")
diff --git a/live_analysis/fr/time_util.py
b/mediawiki/centralnotice/time_util.py
similarity index 100%
rename from live_analysis/fr/time_util.py
rename to mediawiki/centralnotice/time_util.py
diff --git a/live_analysis/fr/top_ten.py b/mediawiki/centralnotice/top_ten.py
similarity index 100%
rename from live_analysis/fr/top_ten.py
rename to mediawiki/centralnotice/top_ten.py
diff --git a/process/__init__.py b/process/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/process/__init__.py
diff --git a/process/globals.py b/process/globals.py
new file mode 100644
index 0000000..060fbb6
--- /dev/null
+++ b/process/globals.py
@@ -0,0 +1,16 @@
+import re
+from importlib import import_module
+from yaml import safe_load as load_yaml
+
+# n.b. Careful not to import `config` by value
+config = dict()
+
+def load_config(filename):
+ global config
+
+ if re.search(r'[.]py$', filename):
+ config = import_module(filename[:-3])
+ elif re.search(r'[.]ya?ml$', filename):
+ config = load_yaml(file(filename, 'r'))
+ else:
+ raise Exception("No config found.")
diff --git a/live_analysis/lock.py b/process/lock.py
similarity index 81%
rename from live_analysis/lock.py
rename to process/lock.py
index d7528f4..1e69ead 100644
--- a/live_analysis/lock.py
+++ b/process/lock.py
@@ -9,6 +9,11 @@
lockfile = None
def begin(filename=None, failopen=False):
+ if not filename:
+ unique = os.environ['LOGNAME']
+ cmd = os.path.basename(sys.argv[0])
+ filename = "/tmp/%s-%s.lock" % (unique, cmd)
+
if os.path.exists(filename):
print "Lockfile found!"
f = open(filename, "r")
@@ -42,4 +47,7 @@
def end():
global lockfile
- os.unlink(lockfile)
+ if lockfile:
+ os.unlink(lockfile)
+ else:
+ raise RuntimeError("Already unlocked!")
diff --git a/stats/__init__.py b/stats/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/stats/__init__.py
diff --git a/live_analysis/stats_abba.py b/stats/stats_abba.py
similarity index 100%
rename from live_analysis/stats_abba.py
rename to stats/stats_abba.py
--
To view, visit https://gerrit.wikimedia.org/r/81436
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I5bc6741852b95b67c6354c4634369ab7c7423fd8
Gerrit-PatchSet: 5
Gerrit-Project: wikimedia/fundraising/tools
Gerrit-Branch: master
Gerrit-Owner: Adamw <awi...@wikimedia.org>
Gerrit-Reviewer: Adamw <awi...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
