Volans has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/405879 )
Change subject: Migrate the server side to Python3
......................................................................
Migrate the server side to Python3
Change-Id: I9c330ad93aa93c2873f87f8127c5123b79e5db79
---
M debian/changelog
M debian/control
M debian/debdeploy-server.dirs
M debian/debdeploy-server.install
M docs/INSTALL
M server/debdeploy
M server/debdeploy_conf.py
M server/debdeploy_updatespec.py
M server/generate-debdeploy-spec
M server/rollback-update
10 files changed, 67 insertions(+), 107 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/debs/debdeploy
refs/changes/79/405879/1
diff --git a/debian/changelog b/debian/changelog
index 8997c7c..1e5cafd 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+debdeploy (0.0.99.3-1+deb8u1) stretch-wikimedia; urgency=medium
+
+ * Migrate the server side of Debdeploy to Python3
+
+ -- Riccardo Coccioli <rcocci...@wikimedia.org> Tue, 23 Jan 2018 10:21:58
+0100
+
debdeploy (0.0.99.2-1+deb9u1) stretch-wikimedia; urgency=medium
* Remove a stray import of salt.client
diff --git a/debian/control b/debian/control
index 788deb6..92fc6fa 100644
--- a/debian/control
+++ b/debian/control
@@ -13,6 +13,6 @@
Package: debdeploy-server
Architecture: all
-Depends: lsof, python-debian, cumin, ${misc:Depends}
+Depends: lsof, python3-debian, cumin, ${misc:Depends}
Description: Debdeploy central package management (server)
This is the server package of Debdeploy.
diff --git a/debian/debdeploy-server.dirs b/debian/debdeploy-server.dirs
index 6802d02..ac96e39 100644
--- a/debian/debdeploy-server.dirs
+++ b/debian/debdeploy-server.dirs
@@ -1,4 +1,3 @@
-/usr/lib/python2.7/dist-packages/
+/usr/lib/python3/dist-packages/
/usr/sbin/
/usr/bin/
-
diff --git a/debian/debdeploy-server.install b/debian/debdeploy-server.install
index b798abe..bc962fb 100644
--- a/debian/debdeploy-server.install
+++ b/debian/debdeploy-server.install
@@ -1,5 +1,5 @@
-server/debdeploy_updatespec.py /usr/lib/python2.7/dist-packages/
-server/debdeploy_conf.py /usr/lib/python2.7/dist-packages/
+server/debdeploy_updatespec.py /usr/lib/python3/dist-packages/
+server/debdeploy_conf.py /usr/lib/python3/dist-packages/
server/debdeploy /usr/sbin
server/generate-debdeploy-spec /usr/bin
server/rollback-update /usr/bin
diff --git a/docs/INSTALL b/docs/INSTALL
index e620da4..8fcba25 100644
--- a/docs/INSTALL
+++ b/docs/INSTALL
@@ -1,18 +1,7 @@
-On the salt master run
+On the master(s) run:
- apt-get install debdeploy-master
+ apt-get install debdeploy-server
-The Salt master needs to be restarted manually to re-read the Salt modules
-provided by debdeploy:
+On all other systems run:
- systemctl restart salt-master.service
-
-On all other systems run
-
- apt-get install debdeploy-minion
-
-All Salt minions needs to reload their modules list (since debdeploy provides
-new modules):
-
- salt '*' saltutil.refresh_modules
-
+ apt-get install debdeploy-client
diff --git a/server/debdeploy b/server/debdeploy
index fe467dc..16ed4da 100755
--- a/server/debdeploy
+++ b/server/debdeploy
@@ -1,32 +1,27 @@
-#! /usr/bin/python
+#! /usr/bin/python3
# -*- coding: utf-8 -*-
-from __future__ import print_function
-
import argparse
-import code
-import cumin
-import datetime
import json
import logging
import os
-import pkgutil
import pydoc
-import signal
import sys
import textwrap
-from ClusterShell.NodeSet import NodeSet
from collections import defaultdict
+import cumin
+
from cumin import query, transport, transports
+
from debdeploy_conf import *
from debdeploy_updatespec import *
+
if os.geteuid() != 0:
print("debdeploy needs to be run as root")
sys.exit(1)
-
cumin_config = cumin.Config()
conf = DebDeployConfig("/etc/debdeploy.conf")
@@ -40,28 +35,6 @@
level=logging_level)
-class logpager:
- threshold = 20 # if pager buffer contains more than <threshold> lines,
use the pager
-
- def __init__(self):
- self.buf = ""
-
- def add(self, *args):
- for i in args:
- self.buf += str(i)
- self.buf += "\n"
-
- def add_nb(self, *args):
- for i in args:
- self.buf += str(i)
-
- def show(self):
- if self.buf.count("\n") > self.threshold:
- pydoc.pager(self.buf)
- else:
- print(self.buf)
-
-
def show_unreachable_hosts(worker):
unreachable_hosts = []
for node in worker._handler_instance.nodes.itervalues():
@@ -72,6 +45,7 @@
print("\nThe following hosts were unreachable:")
for host in unreachable_hosts:
print(host)
+
def run_cumin(cmd, alias):
'''
@@ -125,7 +99,7 @@
update_desc["daemon-disrupt"] = "Daemon update with service availability
impact"
update_desc["library"] = "Library update, several services might need to
be restarted"
- print("Rolling out", source, ": ")
+ print("Rolling out {}:".format(source))
print(update_desc[update_type])
print()
@@ -157,7 +131,8 @@
elif result['success'] and result['status'] == 'UPDATES':
updates = result['data']
for package in updates.keys():
- print(package, "was updated: ", updates[package][0], "->",
updates[package][1])
+ print("{} was updated: {} -> {}".format(
+ package, updates[package][0], updates[package][1]))
print(status_print.fill(str(nodeset) + " (" +
str(len(nodeset)) + " hosts)"))
print()
else:
@@ -173,7 +148,7 @@
print(e)
except KeyError as e:
- print("Failed to read expected value", e, ":")
+ print("Failed to read expected value {}:".format(e))
print(nodeset)
show_unreachable_hosts(worker)
@@ -192,7 +167,7 @@
worker = run_cumin(cmd, servergroup)
- restarts_per_lib = defaultdict(lambda: defaultdict(NodeSet))
+ restarts_per_lib = defaultdict(lambda: defaultdict(cumin.NodeSet))
for nodeset, output in worker.get_results():
msg = str(output)
@@ -214,9 +189,9 @@
print(e)
for lib in restarts_per_lib:
- print("Restarts for", lib, ":")
+ print("Restarts for {}:".format(lib))
for program in restarts_per_lib[lib]:
- print(" ", program)
+ print(" " + program)
print(" " + str(restarts_per_lib[lib][program]) + " (" +
str(len(restarts_per_lib[lib][program])) + " hosts)")
show_unreachable_hosts(worker)
@@ -264,7 +239,7 @@
print(e)
except KeyError as e:
- print("Failed to read expected value", e, ":")
+ print("Failed to read expected value {}:".format(e))
print(nodeset)
show_unreachable_hosts(worker)
@@ -304,7 +279,7 @@
print(e)
except KeyError as e:
- print("Failed to read expected value", e, ":")
+ print("Failed to read expected value {}:".format(e))
print(nodeset)
show_unreachable_hosts(worker)
@@ -370,6 +345,7 @@
elif opt.command == "query_version":
detect_package_versions(opt.packages, opt.serverlist)
+
if __name__ == '__main__':
main()
diff --git a/server/debdeploy_conf.py b/server/debdeploy_conf.py
index 6870476..f2a736d 100644
--- a/server/debdeploy_conf.py
+++ b/server/debdeploy_conf.py
@@ -1,8 +1,7 @@
# -*- coding: utf-8 -*-
+import configparser
+import sys
-from __future__ import print_function
-
-import ConfigParser, sys
class DebDeployConfig(object):
'''
@@ -16,19 +15,21 @@
library_hints = {}
def __init__(self, configfile):
- config = ConfigParser.ConfigParser()
+ config = configparser.ConfigParser()
if len(config.read(configfile)) == 0:
print("/etc/debdeploy.conf doesn't exist, you need to create it.")
print("See
/usr/share/doc/debdeploy-master/examples/debdeploy.conf")
sys.exit(1)
if not config.has_section("distros"):
- print("Could not read list of supported distributions, make sure",
configfile, "contains a section [distros]")
+ print(("Could not read list of supported distributions, make sure
{} contains a "
+ "section [distros]".format(configfile)))
sys.exit(1)
for distro in config.options("distros"):
self.supported_distros[distro] = []
- self.supported_distros[distro].append([x.strip() for x in
config.get("distros", distro).split(",")])
+ self.supported_distros[distro].append(
+ [x.strip() for x in config.get("distros", distro).split(",")])
if len(self.supported_distros) < 1:
print("You need to specify at least one supported distribution in
/etc/debdeploy.conf")
@@ -44,13 +45,9 @@
if config.getboolean("logging", "debug"):
self.debug = True
+
conf = DebDeployConfig("/etc/debdeploy.conf")
# Local variables:
# mode: python
# End:
-
-
-
-
-
diff --git a/server/debdeploy_updatespec.py b/server/debdeploy_updatespec.py
index c36fdb4..718413b 100644
--- a/server/debdeploy_updatespec.py
+++ b/server/debdeploy_updatespec.py
@@ -1,9 +1,8 @@
# -*- coding: utf-8 -*-
-
-from __future__ import print_function
+import sys
import yaml
-import sys
+
class DebDeployUpdateSpec(object):
'''
@@ -16,16 +15,17 @@
update_type = ""
fixes = {}
libraries = []
- legit_type = ['tool', 'daemon-direct', 'daemon-disrupt', 'daemon-cluster',
'reboot', 'reboot-cluster', 'library']
+ legit_type = ['tool', 'daemon-direct', 'daemon-disrupt', 'daemon-cluster',
'reboot',
+ 'reboot-cluster', 'library']
downgrade = False
-
def __init__(self, updatespec, supported_distros):
'''
Parse an update spec file.
updatespec : Filename of the update spec file (string)
- supported_distros : These are the distro codenames for which a fixed
version can be provided (list of strings)
+ supported_distros : These are the distro codenames for which a fixed
version can be provided
+ (list of strings)
'''
try:
@@ -33,22 +33,24 @@
updatefile = yaml.load(stream)
except IOError:
- print("Error: Could not open", updatespec)
+ print("Error: Could not open {}".format(updatespec))
sys.exit(1)
- except yaml.scanner.ScannerError, e:
+ except yaml.scanner.ScannerError as e:
print("Invalid YAML file:")
print(e)
sys.exit(1)
- if not updatefile.has_key("source"):
- print("Invalid YAML file, you need to specify the source package
using the 'source' stanza, see the annotated example file for details")
+ if "source" not in updatefile:
+ print(("Invalid YAML file, you need to specify the source package
using the 'source' "
+ "stanza, see the annotated example file for details"))
sys.exit(1)
else:
self.source = updatefile["source"]
- if not updatefile.has_key("update_type"):
- print("Invalid YAML file, you need to specify the type of update
using the 'update_type' stanza, see the annotated example file for details")
+ if "update_type" not in updatefile:
+ print(("Invalid YAML file, you need to specify the type of update
using the "
+ "'update_type' stanza, see the annotated example file for
details"))
sys.exit(1)
else:
if updatefile["update_type"] not in self.legit_type:
@@ -56,24 +58,26 @@
sys.exit(1)
self.update_type = updatefile["update_type"]
- if updatefile.has_key("comment"):
+ if "comment" in updatefile:
self.comment = updatefile["comment"]
- if updatefile.has_key("libraries"):
+ if "libraries" in updatefile:
self.libraries = updatefile["libraries"]
if "downgrade" in updatefile:
self.downgrade = updatefile["downgrade"]
- if not updatefile.has_key("fixes"):
- print("Invalid YAML file, you need to specify at least one fixed
version using the 'fixes' stanza, see the annotated example file for details")
+ if "fixes" not in updatefile:
+ print(("Invalid YAML file, you need to specify at least one fixed
version using the "
+ "'fixes' stanza, see the annotated example file for
details"))
sys.exit(1)
else:
for i in updatefile["fixes"]:
- if len(supported_distros.keys()) >= 1:
+ if len(list(supported_distros.keys())) >= 1:
self.fixes[i] = updatefile["fixes"].get(i)
else:
- print("Invalid YAML file,", i, "is not a supported
distribution. You need to activate it in /deb/debdeploy.conf")
+ print(("Invalid YAML file, {} is not a supported
distribution. You need to "
+ "activate it in /deb/debdeploy.conf".format(i)))
sys.exit(1)
# Local variables:
diff --git a/server/generate-debdeploy-spec b/server/generate-debdeploy-spec
index e90a55a..3b3c300 100755
--- a/server/generate-debdeploy-spec
+++ b/server/generate-debdeploy-spec
@@ -1,11 +1,7 @@
-#! /usr/bin/python
+#! /usr/bin/python3
# -*- coding: utf-8 -*-
-from __future__ import print_function
-
import datetime
-import logging
-import optparse
import os
import readline
import sys
@@ -99,15 +95,10 @@
for i in libraries:
yamlfile.write(" - " + i + "\n")
except IOError, e:
- print("Error:", e)
+ print("Error: {}".format(e))
sys.exit(1)
# Local variables:
# mode: python
# End:
-
-
-
-
-
diff --git a/server/rollback-update b/server/rollback-update
index d4bf3cd..6bd5695 100755
--- a/server/rollback-update
+++ b/server/rollback-update
@@ -1,7 +1,5 @@
-#! /usr/bin/python
+#! /usr/bin/python3
# -*- coding: utf-8 -*-
-
-from __future__ import print_function
import argparse
import readline
@@ -9,6 +7,7 @@
import debdeploy_conf
import debdeploy_updatespec
+
def parse_args():
p = argparse.ArgumentParser(
@@ -28,7 +27,7 @@
old_update = debdeploy_updatespec.DebDeployUpdateSpec(args.update_spec,
conf.supported_distros)
yamlfilename = "rollback-" + args.update_spec
- print("Generating a spec file to rollback", args.update_spec, "\n")
+ print("Generating a spec file to rollback {}\n".format(args.update_spec))
for distro in conf.supported_distros:
if old_update.fixes[distro]:
@@ -42,8 +41,7 @@
comment: Rollback of update {spec}
update_type: {updatetype}
downgrade: true
-fixes:\n'''.format(src=old_update.source, spec=args.update_spec,
- updatetype=old_update.update_type)
+fixes:\n'''.format(src=old_update.source, spec=args.update_spec,
updatetype=old_update.update_type)
yamlfile.write(yamltext)
for i in downgrade_fixes:
yamltext = " {distro}: {fix} \n".format(distro=i,
fix=downgrade_fixes[i])
@@ -54,7 +52,7 @@
yamltext = " - {lib} \n".format(lib=i)
yamlfile.write(yamltext)
except IOError as e:
- print("Error:", e)
+ print("Error: {}".format(e))
sys.exit(1)
--
To view, visit https://gerrit.wikimedia.org/r/405879
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I9c330ad93aa93c2873f87f8127c5123b79e5db79
Gerrit-PatchSet: 1
Gerrit-Project: operations/debs/debdeploy
Gerrit-Branch: master
Gerrit-Owner: Volans <rcocci...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
