Colin Watson has proposed merging ~cjwatson/lpbuildbot-worker:pre-commit into lpbuildbot-worker:main.
Commit message: Apply pre-commit, isort, and black Requested reviews: Launchpad code reviewers (launchpad-reviewers) For more details, see: https://code.launchpad.net/~cjwatson/lpbuildbot-worker/+git/lpbuildbot-worker/+merge/416333 -- Your team Launchpad code reviewers is requested to review the proposed merge of ~cjwatson/lpbuildbot-worker:pre-commit into lpbuildbot-worker:main.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..1d63864 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,25 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.1.0 + hooks: + - id: check-added-large-files + - id: check-ast + - id: check-merge-conflict + - id: check-yaml + - id: debug-statements + - id: end-of-file-fixer + - id: trailing-whitespace +- repo: https://github.com/PyCQA/isort + rev: 5.10.1 + hooks: + - id: isort +- repo: https://github.com/psf/black + rev: 22.1.0 + hooks: + - id: black +- repo: https://github.com/PyCQA/flake8 + rev: 4.0.1 + hooks: + - id: flake8 diff --git a/create-lp-tests-lxd b/create-lp-tests-lxd index 8a5822a..8e2e916 100755 --- a/create-lp-tests-lxd +++ b/create-lp-tests-lxd @@ -1,18 +1,17 @@ #!/usr/bin/env python3 import argparse -from datetime import datetime import os -from os.path import expanduser -from pathlib import Path -from pwd import getpwnam import shlex import subprocess import sys +from datetime import datetime +from os.path import expanduser +from pathlib import Path +from pwd import getpwnam from pylxd import Client - APT_REPOSITORIES = ( "deb http://archive.ubuntu.com/ubuntu {distro} multiverse", "deb http://archive.ubuntu.com/ubuntu {distro}-updates multiverse", @@ -34,19 +33,26 @@ LP_APACHE_ROOTS = [ "/var/tmp/ppa", ] -LP_APACHE_MODULES = ["proxy", "proxy_http", "rewrite", "ssl", "deflate headers"] +LP_APACHE_MODULES = [ + "proxy", + "proxy_http", + "rewrite", + "ssl", + "deflate headers", +] LXD_HOSTS_CONTENT = ( ( "127.0.0.88", "launchpad.test answers.launchpad.test archive.launchpad.test " - "api.launchpad.test bazaar-internal.launchpad.test beta.launchpad.test " - "blueprints.launchpad.test bugs.launchpad.test code.launchpad.test " - "feeds.launchpad.test id.launchpad.test keyserver.launchpad.test " - "lists.launchpad.test openid.launchpad.test " + "api.launchpad.test bazaar-internal.launchpad.test " + "beta.launchpad.test blueprints.launchpad.test bugs.launchpad.test " + "code.launchpad.test feeds.launchpad.test id.launchpad.test " + "keyserver.launchpad.test lists.launchpad.test openid.launchpad.test " "ubuntu-openid.launchpad.test ppa.launchpad.test " - "private-ppa.launchpad.test testopenid.test translations.launchpad.test " - "xmlrpc-private.launchpad.test xmlrpc.launchpad.test", + "private-ppa.launchpad.test testopenid.test " + "translations.launchpad.test xmlrpc-private.launchpad.test " + "xmlrpc.launchpad.test", ), ("127.0.0.99", "bazaar.launchpad.test"), ) @@ -58,7 +64,12 @@ def _put_file(container, source, destination): def _exec( - container, command, return_stdout=False, user=None, cwd=None, exit_on_error=True + container, + command, + return_stdout=False, + user=None, + cwd=None, + exit_on_error=True, ): container_name = container.name command = " ".join(shlex.quote(arg) for arg in command) @@ -130,7 +141,11 @@ def create_new_instance(client, image_name, series, code_directory): ) }, "devices": { - "build": {"path": code_directory, "source": code_directory, "type": "disk"} + "build": { + "path": code_directory, + "source": code_directory, + "type": "disk", + } }, } container = client.containers.create(instance_config, wait=True) @@ -147,7 +162,9 @@ def install_code(container, series, directory): print("Configuring apt") _exec(container, ["apt-get", "update"]) _exec(container, ["apt-get", "upgrade", "-y"]) - _exec(container, ["apt-get", "install", "-y", "software-properties-common"]) + _exec( + container, ["apt-get", "install", "-y", "software-properties-common"] + ) for apt_repository in APT_REPOSITORIES: repository = apt_repository.format(distro=series) @@ -157,7 +174,8 @@ def install_code(container, series, directory): print("Installing Launchpad apt dependencies") _exec( - container, ["apt-get", "install", "-y"] + LP_DEB_DEPENDENCIES, + container, + ["apt-get", "install", "-y"] + LP_DEB_DEPENDENCIES, ) print("User configuration") @@ -180,7 +198,9 @@ def install_code(container, series, directory): return_stdout=True, ) _exec( - container, ["addgroup", "--gid", gid.strip(), "buildbot"], exit_on_error=False + container, + ["addgroup", "--gid", gid.strip(), "buildbot"], + exit_on_error=False, ) _exec(container, ["chown", "-R", "buildbot:buildbot", directory]) @@ -203,7 +223,10 @@ def install_code(container, series, directory): _exec( container, - ["./utilities/link-external-sourcecode", "{}/dependencies".format(directory),], + [ + "./utilities/link-external-sourcecode", + "{}/dependencies".format(directory), + ], user="buildbot", cwd="{}/devel".format(directory), ) @@ -232,9 +255,14 @@ def install_code(container, series, directory): print("Building launchpad") _exec( - container, ["make", "clean"], user="buildbot", cwd="{}/devel".format(directory) + container, + ["make", "clean"], + user="buildbot", + cwd="{}/devel".format(directory), + ) + _exec( + container, ["make"], user="buildbot", cwd="{}/devel".format(directory) ) - _exec(container, ["make"], user="buildbot", cwd="{}/devel".format(directory)) print("Make install for launchpad") _exec(container, ["make", "install"], cwd="{}/devel".format(directory)) @@ -255,9 +283,15 @@ def remove_build_container(container): if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Build a LXD image for Buildbot") - parser.add_argument("series", type=str, help="Ubuntu series to base the image on.") - parser.add_argument("directory", type=str, help="Directory to mount code from.") + parser = argparse.ArgumentParser( + description="Build a LXD image for Buildbot" + ) + parser.add_argument( + "series", type=str, help="Ubuntu series to base the image on." + ) + parser.add_argument( + "directory", type=str, help="Directory to mount code from." + ) args = parser.parse_args() image_name = "lptests-{}".format(args.series) @@ -280,7 +314,9 @@ if __name__ == "__main__": client = Client() delete_old_image(client, image_name) - container = create_new_instance(client, image_name, args.series, code_directory) + container = create_new_instance( + client, image_name, args.series, code_directory + ) install_code(container, args.series, code_directory) publish_image(container, image_name) remove_build_container(container) diff --git a/lp-setup-lxd-build b/lp-setup-lxd-build index 4de2e4d..51bea4e 100755 --- a/lp-setup-lxd-build +++ b/lp-setup-lxd-build @@ -1,14 +1,14 @@ #!/usr/bin/env python3 import argparse -from datetime import datetime import os -from os.path import expanduser import pathlib -from pwd import getpwnam import shlex import subprocess import sys +from datetime import datetime +from os.path import expanduser +from pwd import getpwnam from pylxd import Client @@ -21,7 +21,12 @@ BUILD_STEPS = [ def _exec( - container, command, return_stdout=False, user=None, cwd=None, exit_on_error=True + container, + command, + return_stdout=False, + user=None, + cwd=None, + exit_on_error=True, ): container_name = container.name command = " ".join(shlex.quote(arg) for arg in command) @@ -63,7 +68,9 @@ def start_new_container(client, image_name, work_dir): getpwnam("buildbot").pw_uid, getpwnam("buildbot").pw_gid ) }, - "devices": {"build": {"path": work_dir, "source": work_dir, "type": "disk"}}, + "devices": { + "build": {"path": work_dir, "source": work_dir, "type": "disk"} + }, } container = client.containers.create(image_data, wait=True) print("Starting container") @@ -100,9 +107,8 @@ def copy_workdir_to_container(container, work_dir): "rsync", "-a", "-e", - "ssh -i {} -o 'UserKnownHostsFile=/dev/null' -o 'StrictHostKeyChecking=no'".format( - ssh_pub_file - ), + "ssh -i {} -o 'UserKnownHostsFile=/dev/null' " + "-o 'StrictHostKeyChecking=no'".format(ssh_pub_file), str(work_dir) + "/", "root@{}:{}".format(container_ip, str(work_dir) + "/"), ] @@ -118,15 +124,23 @@ def build_launchpad_in_container(container, work_dir, variables): for key, value in variables: command.append("{}={}".format(key, value)) _exec( - container, command, user="buildbot", cwd="{}/devel".format(work_dir), + container, + command, + user="buildbot", + cwd="{}/devel".format(work_dir), ) if __name__ == "__main__": parser = argparse.ArgumentParser( - description="Create a new LXD instance from a base image and update the LP installation" + description=( + "Create a new LXD instance from a base image and update the LP " + "installation" + ) + ) + parser.add_argument( + "name", type=str, help="Name of the image and container" ) - parser.add_argument("name", type=str, help="Name of the image and container") parser.add_argument( "work_dir", type=str, help="Base directory for code and dependencies" ) diff --git a/lp-setup-lxd-cleanup b/lp-setup-lxd-cleanup index 6650e0b..a9b809d 100755 --- a/lp-setup-lxd-cleanup +++ b/lp-setup-lxd-cleanup @@ -36,7 +36,9 @@ if __name__ == "__main__": parser = argparse.ArgumentParser( description="Clean up old LXD from a buildbot test run" ) - parser.add_argument("name", type=str, help="Base name of containers to clean up") + parser.add_argument( + "name", type=str, help="Base name of containers to clean up" + ) args = parser.parse_args() # Work around xenial's pylxd not understanding the lxd snap. diff --git a/lp-setup-lxd-test b/lp-setup-lxd-test index 360969a..ea901a0 100755 --- a/lp-setup-lxd-test +++ b/lp-setup-lxd-test @@ -1,22 +1,27 @@ #!/usr/bin/env python3 import argparse -from datetime import datetime import os -from os.path import expanduser import random import shlex import string import subprocess import sys import time +from datetime import datetime +from os.path import expanduser from pylxd import Client from pylxd.exceptions import LXDAPIException def _exec( - container, command, return_stdout=False, user=None, cwd=None, exit_on_error=True + container, + command, + return_stdout=False, + user=None, + cwd=None, + exit_on_error=True, ): container_name = container.name command = " ".join(shlex.quote(arg) for arg in command) @@ -49,7 +54,8 @@ def create_ephemeral_container( # At this point, the base instance should already be running client.containers.get(image_name) test_container_name = "{}-{}".format( - image_name, "".join(random.choice(string.ascii_lowercase) for i in range(8)) + image_name, + "".join(random.choice(string.ascii_lowercase) for i in range(8)), ) instance_config = { @@ -111,14 +117,18 @@ def create_ephemeral_container( "rsync", "-a", "-e", - "ssh -i {} -o 'UserKnownHostsFile=/dev/null' -o 'StrictHostKeyChecking=no'".format( - ssh_pub_file - ), + "ssh -i {} -o 'UserKnownHostsFile=/dev/null' " + "-o 'StrictHostKeyChecking=no'".format(ssh_pub_file), str(work_dir) + "/temp/", "root@{}:{}".format(container_ip, str(work_dir) + "/temp/"), ] ) - return_code, _ = _exec(test_container, command, cwd=work_dir, exit_on_error=False,) + return_code, _ = _exec( + test_container, + command, + cwd=work_dir, + exit_on_error=False, + ) if list_tests: print("rsync temp dir out of container {}".format(test_container_name)) subprocess.run( @@ -126,9 +136,8 @@ def create_ephemeral_container( "rsync", "-a", "-e", - "ssh -i {} -o 'UserKnownHostsFile=/dev/null' -o 'StrictHostKeyChecking=no'".format( - ssh_pub_file - ), + "ssh -i {} -o 'UserKnownHostsFile=/dev/null' " + "-o 'StrictHostKeyChecking=no'".format(ssh_pub_file), "root@{}:{}".format(container_ip, str(work_dir) + "/temp/"), str(work_dir) + "/temp/", ] @@ -141,11 +150,17 @@ def create_ephemeral_container( if __name__ == "__main__": parser = argparse.ArgumentParser( - description="Create ephemeral copies of a base container instance to run tests" + description=( + "Create ephemeral copies of a base container instance to run tests" + ) + ) + parser.add_argument( + "name", type=str, help="Name of the container to clone" ) - parser.add_argument("name", type=str, help="Name of the container to clone") parser.add_argument("work_dir") - parser.add_argument("--list", action="store_true", help="List available tests") + parser.add_argument( + "--list", action="store_true", help="List available tests" + ) parser.add_argument("--load-list", type=str, help="Run tests from file") args = parser.parse_args() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..a8f43fe --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,2 @@ +[tool.black] +line-length = 79 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..ea073a0 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,11 @@ +[flake8] +ignore = + # imcompatible with black + E203 + # W503 and W504 are mutally exclusive + W503 + W504 + +[isort] +line_length = 79 +profile = black
_______________________________________________ Mailing list: https://launchpad.net/~launchpad-reviewers Post to : launchpad-reviewers@lists.launchpad.net Unsubscribe : https://launchpad.net/~launchpad-reviewers More help : https://help.launchpad.net/ListHelp
