Dzahn has submitted this change and it was merged.
Change subject: toollabs: pep8 fixes for pretty code :)
......................................................................
toollabs: pep8 fixes for pretty code :)
Change-Id: I7701ed0848b3a5132ff4ad2de899a1769749746c
---
M modules/toollabs/files/gridscripts/runninggridtasks.py
M modules/toollabs/files/kube2dynproxy.py
M modules/toollabs/files/portgrabber
M modules/toollabs/files/portgrabber.py
M modules/toollabs/files/proxylistener.py
M modules/toollabs/files/tool-nodejs
M modules/toollabs/files/tool-uwsgi-python
M modules/toollabs/files/toolschecker.py
M modules/toollabs/manifests/genpp/python.py
9 files changed, 70 insertions(+), 45 deletions(-)
Approvals:
Yuvipanda: Looks good to me, but someone else must approve
jenkins-bot: Verified
Dzahn: Looks good to me, approved
diff --git a/modules/toollabs/files/gridscripts/runninggridtasks.py
b/modules/toollabs/files/gridscripts/runninggridtasks.py
index dffa62a..55cbc28 100755
--- a/modules/toollabs/files/gridscripts/runninggridtasks.py
+++ b/modules/toollabs/files/gridscripts/runninggridtasks.py
@@ -38,8 +38,9 @@
job[jobvalue.get('name')] = jobvalue.text
yield job
-groupkey = lambda x: x['job_owner']
-sortkey = lambda x: (groupkey(x), x['start_time'])
+
+def groupkey(x):
+ return x['job_owner']
proc = subprocess.Popen(
['qhost', '-j', '-xml', '-h'] + sys.argv[1:],
@@ -47,10 +48,11 @@
)
jobs = [job for job in get_jobs(proc.stdout)
- if not job['queue_name'].startswith('continuous')
- and not job['queue_name'].startswith('webgrid')]
+ if not job['queue_name'].startswith('continuous') and
+ not job['queue_name'].startswith('webgrid')]
-jobs = sorted(jobs, key=sortkey)
+jobs = sorted(jobs, key=lambda x: (groupkey(x), x['start_time']))
-data = {owner: list(jobs) for owner, jobs in itertools.groupby(jobs,
key=groupkey)}
+data = {owner: list(jobs) for owner, jobs in itertools.groupby(
+ jobs, key=groupkey)}
print(yaml.dump(data))
diff --git a/modules/toollabs/files/kube2dynproxy.py
b/modules/toollabs/files/kube2dynproxy.py
index fcc43ad..b651434 100755
--- a/modules/toollabs/files/kube2dynproxy.py
+++ b/modules/toollabs/files/kube2dynproxy.py
@@ -41,7 +41,8 @@
services = []
try:
servicelist = resp.json()
- self.resourceVersion =
int(servicelist['metadata']['resourceVersion'])
+ self.resourceVersion = int(
+ servicelist['metadata']['resourceVersion'])
log.debug("global resourceVersion now at %s", self.resourceVersion)
for servicedata in servicelist['items']:
services.append(KubeClient._resp_to_service(servicedata))
@@ -62,7 +63,8 @@
"""Does a full sync of the services, returns a list
of the active ones."""
services = self.get_services()
- registered_services = set([s.decode('utf-8') for s in
self.conn.smembers(services_registry)])
+ registered_services = set(
+ [s.decode('utf-8') for s in self.conn.smembers(services_registry)])
actual_services = set([str(s) for s in services])
services_to_delete = registered_services - actual_services
for service in services_to_delete:
diff --git a/modules/toollabs/files/portgrabber
b/modules/toollabs/files/portgrabber
index 1cb78b3..9c1da0c 100755
--- a/modules/toollabs/files/portgrabber
+++ b/modules/toollabs/files/portgrabber
@@ -11,7 +11,8 @@
if len(sys.argv) <= 2:
sys.stderr.write('Usage: portgrabber TOOLNAME EXECUTABLE [ARGUMENTS...]\n')
sys.stderr.write('\n')
- sys.stderr.write('portgrabber requests a free port number for TOOLNAME and
then\n')
+ sys.stderr.write(
+ 'portgrabber requests a free port number for TOOLNAME and then\n')
sys.stderr.write('calls EXECUTABLE [ARGUMENTS...] PORT.\n')
sys.exit(1)
diff --git a/modules/toollabs/files/portgrabber.py
b/modules/toollabs/files/portgrabber.py
index d4fc953..8d32a08 100644
--- a/modules/toollabs/files/portgrabber.py
+++ b/modules/toollabs/files/portgrabber.py
@@ -11,9 +11,9 @@
def get_open_port():
"""Tries to get a random open port to listen on
- It does this by starting to listen on a socket, letting the kernel
determine
- the open port. It then immediately closes it and returns the port.
- """
+ It does this by starting to listen on a socket, letting the kernel
+ determine the open port. It then immediately closes it and returns the
+ port."""
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(('', 0))
port = s.getsockname()[1]
diff --git a/modules/toollabs/files/proxylistener.py
b/modules/toollabs/files/proxylistener.py
index fc2cdc2..e71712c 100644
--- a/modules/toollabs/files/proxylistener.py
+++ b/modules/toollabs/files/proxylistener.py
@@ -17,16 +17,16 @@
"""
Simple TCP server that keeps routes in the Redis db for authenticated requests.
-The routes are kept as long as the socket making the request is open, and
cleaned
-up right afterwards. identd is used for authentication - while normally that is
-a terrible idea, this is okay in the toollabs environment because we only have
-a limited number of trusted admins. This also allows routes to be added
-only for URLs that are under the URL prefix allocated for the tool making the
-request. For example, a tool named 'testtool' can ask only for URLs that
+The routes are kept as long as the socket making the request is open, and
+cleaned up right afterwards. identd is used for authentication - while normally
+that is a terrible idea, this is okay in the toollabs environment because we
+only have a limited number of trusted admins. This also allows routes to be
+added only for URLs that are under the URL prefix allocated for the tool making
+the request. For example, a tool named 'testtool' can ask only for URLs that
start with /testtool/ to be routed to where it wants.
-The socket server is a threaded implementation. Python can not be truly
parallel
-(hello, GIL!), but for our purposes it is good enough.
+The socket server is a threaded implementation. Python can not be truly
+parallel (hello, GIL!), but for our purposes it is good enough.
"""
import logging
import socket
@@ -65,7 +65,8 @@
resp_parts = [r.strip() for r in resp.split(":")]
if "USERID" not in resp_parts:
# Some auth error has occured. Abort!
- logging.log(logging.INFO, "Identd auth failed, sent %s got back %s" %
(request.strip(), resp.strip()))
+ logging.log(logging.INFO, "Identd auth failed, sent %s got back %s" %
+ (request.strip(), resp.strip()))
return None
return resp_parts[-1]
@@ -77,11 +78,14 @@
"""
def handle(self):
- user = get_remote_user(self.client_address[0], self.client_address[1],
PORT)
- # For some reason the identd response gave us an error, or failed
otherwise
- # This should usually not happen, so we'll just ask folks to 'Contact
an administrator'
+ user = get_remote_user(self.client_address[0],
+ self.client_address[1], PORT)
+ # For some reason the identd response gave an error or failed otherwise
+ # This should usually not happen, so we'll just ask folks to 'Contact
+ # an administrator'
if user is None:
- self.request.send("Identd authentication failed. Please contact an
administrator")
+ self.request.send("Identd authentication failed. " +
+ "Please contact an administrator")
self.request.close()
return
@@ -101,17 +105,21 @@
if command == 'register':
destination = self.rfile.readline().strip()
- logging.log(logging.INFO, "Received request from %s for %s to %s",
toolname, route, destination)
+ logging.log(logging.INFO, "Received request from %s for %s to %s",
+ toolname, route, destination)
red.hset(redis_key, route, destination)
- logging.log(logging.DEBUG, "Set redis key %s with key/value
%s:%s", redis_key, route, destination)
+ logging.log(logging.DEBUG, "Set redis key %s with key/value %s:%s",
+ redis_key, route, destination)
self.request.send('ok')
elif command == 'unregister':
- logging.log(logging.INFO, "Cleaning up request from %s for %s",
toolname, route)
+ logging.log(logging.INFO, "Cleaning up request from %s for %s",
+ toolname, route)
red.hdel(redis_key, route)
- logging.log(logging.DEBUG, "Removed redis key %s with key %s",
redis_key, route)
+ logging.log(logging.DEBUG, "Removed redis key %s with key %s",
+ redis_key, route)
self.request.send('ok')
else:
logging.log(logging.ERROR, "Unknown command received: %s", command)
diff --git a/modules/toollabs/files/tool-nodejs
b/modules/toollabs/files/tool-nodejs
index 55d7c55..0808113 100644
--- a/modules/toollabs/files/tool-nodejs
+++ b/modules/toollabs/files/tool-nodejs
@@ -18,7 +18,7 @@
# Pass in all of our parent environment too
# npm doesn't like it too much if you give it an almost empty environment
os.environ['TOOL_WEB_PORT'] = str(port)
-os.environ['PORT'] = str(port) # Switch things to PORT, same as heroku
+os.environ['PORT'] = str(port) # Switch things to PORT, same as heroku
os.environ['MYSQL_CREDS_PATH'] = os.path.expanduser('~/replica.my.cnf')
diff --git a/modules/toollabs/files/tool-uwsgi-python
b/modules/toollabs/files/tool-uwsgi-python
index 04005f7..d66e82e 100755
--- a/modules/toollabs/files/tool-uwsgi-python
+++ b/modules/toollabs/files/tool-uwsgi-python
@@ -26,7 +26,8 @@
'--callable', 'app',
'--manage-script-name',
'--workers', '4',
- '--mount', '/%s=%s' % (TOOL,
os.path.expanduser('~/www/python/src/app.py')),
+ '--mount', '/%s=%s' % (TOOL,
+ os.path.expanduser('~/www/python/src/app.py')),
'--die-on-term',
'--strict',
'--master'
diff --git a/modules/toollabs/files/toolschecker.py
b/modules/toollabs/files/toolschecker.py
index bfd7cf6..6b20828 100644
--- a/modules/toollabs/files/toolschecker.py
+++ b/modules/toollabs/files/toolschecker.py
@@ -106,7 +106,8 @@
def db_query_check(host):
# Run a simple known query, verify the db returns.
- connection = pymysql.connect(host,
read_default_file=os.path.expanduser('~/replica.my.cnf'))
+ connection = pymysql.connect(
+ host, read_default_file=os.path.expanduser('~/replica.my.cnf'))
cur = connection.cursor()
cur.execute('select * from meta_p.wiki limit 1')
result = cur.fetchone()
@@ -132,7 +133,10 @@
@check('/labsdb/labsdb1005')
def labsdb_check_labsdb1005():
- connection = pymysql.connect('labsdb1005.eqiad.wmnet',
read_default_file=os.path.expanduser('~/replica.my.cnf'))
+ connection = pymysql.connect(
+ 'labsdb1005.eqiad.wmnet',
+ read_default_file=os.path.expanduser('~/replica.my.cnf')
+ )
cur = connection.cursor()
cur.execute('select * from toolserverdb_p.wiki limit 1')
result = cur.fetchone()
@@ -157,7 +161,8 @@
"""Check if a job with given name is running"""
try:
with open(os.devnull, 'w') as devnull:
- subprocess.check_call(['/usr/bin/qstat', '-j', name],
stderr=devnull)
+ subprocess.check_call(
+ ['/usr/bin/qstat', '-j', name], stderr=devnull)
return True
except subprocess.CalledProcessError:
return False
@@ -216,7 +221,11 @@
a table named "test" with one field, also named "test" '''
success = False
try:
- connection = pymysql.connect(host,
read_default_file=os.path.expanduser('~/replica.my.cnf'), db=db)
+ connection = pymysql.connect(
+ host,
+ read_default_file=os.path.expanduser('~/replica.my.cnf'),
+ db=db
+ )
cur = connection.cursor()
magicnumber = int(time.time())
cur.execute("INSERT INTO test (test) VALUES (%s)" % magicnumber)
@@ -243,8 +252,8 @@
try:
connection = psycopg2.connect(
- "host=labsdb1004.eqiad.wmnet dbname=%s_rwtest user=%s password=%s"
%
- (user, user, password))
+ "host=labsdb1004.eqiad.wmnet dbname=%s_rwtest user=%s password=%s"
+ % (user, user, password))
cur = connection.cursor()
cur.execute("INSERT INTO test (test) VALUES (%s)" % magicnumber)
connection.commit()
@@ -328,8 +337,9 @@
# So far so good -- now, test wsgi
success = False
with open(os.devnull, 'w') as devnull:
- subprocess.check_call(['/usr/local/bin/webservice2', 'uwsgi-python',
'start'],
- stderr=devnull, stdout=devnull)
+ subprocess.check_call(
+ ['/usr/local/bin/webservice2', 'uwsgi-python', 'start'],
+ stderr=devnull, stdout=devnull)
for i in range(0, 10):
request = requests.get(url)
@@ -339,8 +349,9 @@
time.sleep(1)
with open(os.devnull, 'w') as devnull:
- subprocess.check_call(['/usr/local/bin/webservice2', 'uwsgi-python',
'stop'],
- stderr=devnull, stdout=devnull)
+ subprocess.check_call(
+ ['/usr/local/bin/webservice2', 'uwsgi-python', 'stop'],
+ stderr=devnull, stdout=devnull)
# Make sure it really stopped
success = False
diff --git a/modules/toollabs/manifests/genpp/python.py
b/modules/toollabs/manifests/genpp/python.py
index 5fa6d82..f56dff3 100755
--- a/modules/toollabs/manifests/genpp/python.py
+++ b/modules/toollabs/manifests/genpp/python.py
@@ -4,6 +4,10 @@
# python 2 and 3, and for all genpp-supported
# distributions
+import genpp
+import logging
+import pprint
+
environ_packages = {
'dev': ['coverage', 'dev', 'stdeb'],
'exec': [
@@ -66,10 +70,6 @@
'zmq',
],
}
-
-import genpp
-import logging
-import pprint
logging.basicConfig(level=logging.DEBUG)
--
To view, visit https://gerrit.wikimedia.org/r/257002
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I7701ed0848b3a5132ff4ad2de899a1769749746c
Gerrit-PatchSet: 2
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Chad <[email protected]>
Gerrit-Reviewer: Dzahn <[email protected]>
Gerrit-Reviewer: Merlijn van Deen <[email protected]>
Gerrit-Reviewer: Yuvipanda <[email protected]>
Gerrit-Reviewer: coren <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
