Volans has uploaded a new change for review. ( https://gerrit.wikimedia.org/r/405879 )
Change subject: Migrate the server side to Python3 ...................................................................... Migrate the server side to Python3 Change-Id: I9c330ad93aa93c2873f87f8127c5123b79e5db79 --- M debian/changelog M debian/control M debian/debdeploy-server.dirs M debian/debdeploy-server.install M docs/INSTALL M server/debdeploy M server/debdeploy_conf.py M server/debdeploy_updatespec.py M server/generate-debdeploy-spec M server/rollback-update 10 files changed, 67 insertions(+), 107 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/operations/debs/debdeploy refs/changes/79/405879/1 diff --git a/debian/changelog b/debian/changelog index 8997c7c..1e5cafd 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +debdeploy (0.0.99.3-1+deb8u1) stretch-wikimedia; urgency=medium + + * Migrate the server side of Debdeploy to Python3 + + -- Riccardo Coccioli <rcocci...@wikimedia.org> Tue, 23 Jan 2018 10:21:58 +0100 + debdeploy (0.0.99.2-1+deb9u1) stretch-wikimedia; urgency=medium * Remove a stray import of salt.client diff --git a/debian/control b/debian/control index 788deb6..92fc6fa 100644 --- a/debian/control +++ b/debian/control @@ -13,6 +13,6 @@ Package: debdeploy-server Architecture: all -Depends: lsof, python-debian, cumin, ${misc:Depends} +Depends: lsof, python3-debian, cumin, ${misc:Depends} Description: Debdeploy central package management (server) This is the server package of Debdeploy. diff --git a/debian/debdeploy-server.dirs b/debian/debdeploy-server.dirs index 6802d02..ac96e39 100644 --- a/debian/debdeploy-server.dirs +++ b/debian/debdeploy-server.dirs @@ -1,4 +1,3 @@ -/usr/lib/python2.7/dist-packages/ +/usr/lib/python3/dist-packages/ /usr/sbin/ /usr/bin/ - diff --git a/debian/debdeploy-server.install b/debian/debdeploy-server.install index b798abe..bc962fb 100644 --- a/debian/debdeploy-server.install +++ b/debian/debdeploy-server.install @@ -1,5 +1,5 @@ -server/debdeploy_updatespec.py /usr/lib/python2.7/dist-packages/ -server/debdeploy_conf.py /usr/lib/python2.7/dist-packages/ +server/debdeploy_updatespec.py /usr/lib/python3/dist-packages/ +server/debdeploy_conf.py /usr/lib/python3/dist-packages/ server/debdeploy /usr/sbin server/generate-debdeploy-spec /usr/bin server/rollback-update /usr/bin diff --git a/docs/INSTALL b/docs/INSTALL index e620da4..8fcba25 100644 --- a/docs/INSTALL +++ b/docs/INSTALL @@ -1,18 +1,7 @@ -On the salt master run +On the master(s) run: - apt-get install debdeploy-master + apt-get install debdeploy-server -The Salt master needs to be restarted manually to re-read the Salt modules -provided by debdeploy: +On all other systems run: - systemctl restart salt-master.service - -On all other systems run - - apt-get install debdeploy-minion - -All Salt minions needs to reload their modules list (since debdeploy provides -new modules): - - salt '*' saltutil.refresh_modules - + apt-get install debdeploy-client diff --git a/server/debdeploy b/server/debdeploy index fe467dc..16ed4da 100755 --- a/server/debdeploy +++ b/server/debdeploy @@ -1,32 +1,27 @@ -#! /usr/bin/python +#! /usr/bin/python3 # -*- coding: utf-8 -*- -from __future__ import print_function - import argparse -import code -import cumin -import datetime import json import logging import os -import pkgutil import pydoc -import signal import sys import textwrap -from ClusterShell.NodeSet import NodeSet from collections import defaultdict +import cumin + from cumin import query, transport, transports + from debdeploy_conf import * from debdeploy_updatespec import * + if os.geteuid() != 0: print("debdeploy needs to be run as root") sys.exit(1) - cumin_config = cumin.Config() conf = DebDeployConfig("/etc/debdeploy.conf") @@ -40,28 +35,6 @@ level=logging_level) -class logpager: - threshold = 20 # if pager buffer contains more than <threshold> lines, use the pager - - def __init__(self): - self.buf = "" - - def add(self, *args): - for i in args: - self.buf += str(i) - self.buf += "\n" - - def add_nb(self, *args): - for i in args: - self.buf += str(i) - - def show(self): - if self.buf.count("\n") > self.threshold: - pydoc.pager(self.buf) - else: - print(self.buf) - - def show_unreachable_hosts(worker): unreachable_hosts = [] for node in worker._handler_instance.nodes.itervalues(): @@ -72,6 +45,7 @@ print("\nThe following hosts were unreachable:") for host in unreachable_hosts: print(host) + def run_cumin(cmd, alias): ''' @@ -125,7 +99,7 @@ update_desc["daemon-disrupt"] = "Daemon update with service availability impact" update_desc["library"] = "Library update, several services might need to be restarted" - print("Rolling out", source, ": ") + print("Rolling out {}:".format(source)) print(update_desc[update_type]) print() @@ -157,7 +131,8 @@ elif result['success'] and result['status'] == 'UPDATES': updates = result['data'] for package in updates.keys(): - print(package, "was updated: ", updates[package][0], "->", updates[package][1]) + print("{} was updated: {} -> {}".format( + package, updates[package][0], updates[package][1])) print(status_print.fill(str(nodeset) + " (" + str(len(nodeset)) + " hosts)")) print() else: @@ -173,7 +148,7 @@ print(e) except KeyError as e: - print("Failed to read expected value", e, ":") + print("Failed to read expected value {}:".format(e)) print(nodeset) show_unreachable_hosts(worker) @@ -192,7 +167,7 @@ worker = run_cumin(cmd, servergroup) - restarts_per_lib = defaultdict(lambda: defaultdict(NodeSet)) + restarts_per_lib = defaultdict(lambda: defaultdict(cumin.NodeSet)) for nodeset, output in worker.get_results(): msg = str(output) @@ -214,9 +189,9 @@ print(e) for lib in restarts_per_lib: - print("Restarts for", lib, ":") + print("Restarts for {}:".format(lib)) for program in restarts_per_lib[lib]: - print(" ", program) + print(" " + program) print(" " + str(restarts_per_lib[lib][program]) + " (" + str(len(restarts_per_lib[lib][program])) + " hosts)") show_unreachable_hosts(worker) @@ -264,7 +239,7 @@ print(e) except KeyError as e: - print("Failed to read expected value", e, ":") + print("Failed to read expected value {}:".format(e)) print(nodeset) show_unreachable_hosts(worker) @@ -304,7 +279,7 @@ print(e) except KeyError as e: - print("Failed to read expected value", e, ":") + print("Failed to read expected value {}:".format(e)) print(nodeset) show_unreachable_hosts(worker) @@ -370,6 +345,7 @@ elif opt.command == "query_version": detect_package_versions(opt.packages, opt.serverlist) + if __name__ == '__main__': main() diff --git a/server/debdeploy_conf.py b/server/debdeploy_conf.py index 6870476..f2a736d 100644 --- a/server/debdeploy_conf.py +++ b/server/debdeploy_conf.py @@ -1,8 +1,7 @@ # -*- coding: utf-8 -*- +import configparser +import sys -from __future__ import print_function - -import ConfigParser, sys class DebDeployConfig(object): ''' @@ -16,19 +15,21 @@ library_hints = {} def __init__(self, configfile): - config = ConfigParser.ConfigParser() + config = configparser.ConfigParser() if len(config.read(configfile)) == 0: print("/etc/debdeploy.conf doesn't exist, you need to create it.") print("See /usr/share/doc/debdeploy-master/examples/debdeploy.conf") sys.exit(1) if not config.has_section("distros"): - print("Could not read list of supported distributions, make sure", configfile, "contains a section [distros]") + print(("Could not read list of supported distributions, make sure {} contains a " + "section [distros]".format(configfile))) sys.exit(1) for distro in config.options("distros"): self.supported_distros[distro] = [] - self.supported_distros[distro].append([x.strip() for x in config.get("distros", distro).split(",")]) + self.supported_distros[distro].append( + [x.strip() for x in config.get("distros", distro).split(",")]) if len(self.supported_distros) < 1: print("You need to specify at least one supported distribution in /etc/debdeploy.conf") @@ -44,13 +45,9 @@ if config.getboolean("logging", "debug"): self.debug = True + conf = DebDeployConfig("/etc/debdeploy.conf") # Local variables: # mode: python # End: - - - - - diff --git a/server/debdeploy_updatespec.py b/server/debdeploy_updatespec.py index c36fdb4..718413b 100644 --- a/server/debdeploy_updatespec.py +++ b/server/debdeploy_updatespec.py @@ -1,9 +1,8 @@ # -*- coding: utf-8 -*- - -from __future__ import print_function +import sys import yaml -import sys + class DebDeployUpdateSpec(object): ''' @@ -16,16 +15,17 @@ update_type = "" fixes = {} libraries = [] - legit_type = ['tool', 'daemon-direct', 'daemon-disrupt', 'daemon-cluster', 'reboot', 'reboot-cluster', 'library'] + legit_type = ['tool', 'daemon-direct', 'daemon-disrupt', 'daemon-cluster', 'reboot', + 'reboot-cluster', 'library'] downgrade = False - def __init__(self, updatespec, supported_distros): ''' Parse an update spec file. updatespec : Filename of the update spec file (string) - supported_distros : These are the distro codenames for which a fixed version can be provided (list of strings) + supported_distros : These are the distro codenames for which a fixed version can be provided + (list of strings) ''' try: @@ -33,22 +33,24 @@ updatefile = yaml.load(stream) except IOError: - print("Error: Could not open", updatespec) + print("Error: Could not open {}".format(updatespec)) sys.exit(1) - except yaml.scanner.ScannerError, e: + except yaml.scanner.ScannerError as e: print("Invalid YAML file:") print(e) sys.exit(1) - if not updatefile.has_key("source"): - print("Invalid YAML file, you need to specify the source package using the 'source' stanza, see the annotated example file for details") + if "source" not in updatefile: + print(("Invalid YAML file, you need to specify the source package using the 'source' " + "stanza, see the annotated example file for details")) sys.exit(1) else: self.source = updatefile["source"] - if not updatefile.has_key("update_type"): - print("Invalid YAML file, you need to specify the type of update using the 'update_type' stanza, see the annotated example file for details") + if "update_type" not in updatefile: + print(("Invalid YAML file, you need to specify the type of update using the " + "'update_type' stanza, see the annotated example file for details")) sys.exit(1) else: if updatefile["update_type"] not in self.legit_type: @@ -56,24 +58,26 @@ sys.exit(1) self.update_type = updatefile["update_type"] - if updatefile.has_key("comment"): + if "comment" in updatefile: self.comment = updatefile["comment"] - if updatefile.has_key("libraries"): + if "libraries" in updatefile: self.libraries = updatefile["libraries"] if "downgrade" in updatefile: self.downgrade = updatefile["downgrade"] - if not updatefile.has_key("fixes"): - print("Invalid YAML file, you need to specify at least one fixed version using the 'fixes' stanza, see the annotated example file for details") + if "fixes" not in updatefile: + print(("Invalid YAML file, you need to specify at least one fixed version using the " + "'fixes' stanza, see the annotated example file for details")) sys.exit(1) else: for i in updatefile["fixes"]: - if len(supported_distros.keys()) >= 1: + if len(list(supported_distros.keys())) >= 1: self.fixes[i] = updatefile["fixes"].get(i) else: - print("Invalid YAML file,", i, "is not a supported distribution. You need to activate it in /deb/debdeploy.conf") + print(("Invalid YAML file, {} is not a supported distribution. You need to " + "activate it in /deb/debdeploy.conf".format(i))) sys.exit(1) # Local variables: diff --git a/server/generate-debdeploy-spec b/server/generate-debdeploy-spec index e90a55a..3b3c300 100755 --- a/server/generate-debdeploy-spec +++ b/server/generate-debdeploy-spec @@ -1,11 +1,7 @@ -#! /usr/bin/python +#! /usr/bin/python3 # -*- coding: utf-8 -*- -from __future__ import print_function - import datetime -import logging -import optparse import os import readline import sys @@ -99,15 +95,10 @@ for i in libraries: yamlfile.write(" - " + i + "\n") except IOError, e: - print("Error:", e) + print("Error: {}".format(e)) sys.exit(1) # Local variables: # mode: python # End: - - - - - diff --git a/server/rollback-update b/server/rollback-update index d4bf3cd..6bd5695 100755 --- a/server/rollback-update +++ b/server/rollback-update @@ -1,7 +1,5 @@ -#! /usr/bin/python +#! /usr/bin/python3 # -*- coding: utf-8 -*- - -from __future__ import print_function import argparse import readline @@ -9,6 +7,7 @@ import debdeploy_conf import debdeploy_updatespec + def parse_args(): p = argparse.ArgumentParser( @@ -28,7 +27,7 @@ old_update = debdeploy_updatespec.DebDeployUpdateSpec(args.update_spec, conf.supported_distros) yamlfilename = "rollback-" + args.update_spec - print("Generating a spec file to rollback", args.update_spec, "\n") + print("Generating a spec file to rollback {}\n".format(args.update_spec)) for distro in conf.supported_distros: if old_update.fixes[distro]: @@ -42,8 +41,7 @@ comment: Rollback of update {spec} update_type: {updatetype} downgrade: true -fixes:\n'''.format(src=old_update.source, spec=args.update_spec, - updatetype=old_update.update_type) +fixes:\n'''.format(src=old_update.source, spec=args.update_spec, updatetype=old_update.update_type) yamlfile.write(yamltext) for i in downgrade_fixes: yamltext = " {distro}: {fix} \n".format(distro=i, fix=downgrade_fixes[i]) @@ -54,7 +52,7 @@ yamltext = " - {lib} \n".format(lib=i) yamlfile.write(yamltext) except IOError as e: - print("Error:", e) + print("Error: {}".format(e)) sys.exit(1) -- To view, visit https://gerrit.wikimedia.org/r/405879 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I9c330ad93aa93c2873f87f8127c5123b79e5db79 Gerrit-PatchSet: 1 Gerrit-Project: operations/debs/debdeploy Gerrit-Branch: master Gerrit-Owner: Volans <rcocci...@wikimedia.org> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits