[gentoo-commits] proj/catalyst:wip/mattst88 commit in: catalyst/base/, catalyst/targets/, catalyst/
commit: ad54d2e7669326cb313cccf281544559463fbcad Author: Matt Turner gentoo org> AuthorDate: Wed Jun 9 06:17:31 2021 + Commit: Matt Turner gentoo org> CommitDate: Thu Jun 10 01:38:47 2021 + URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=ad54d2e7 catalyst: Replace snakeoil's locks with fasteners To no great surprise, the existing locking was broken. For example, clear_chroot() releases the lock. It is called by unpack(), which is part of prepare_sequence. The result is that the whole build could be done without holding the lock. Just lock around run(). It's not apparent that finer-grained locking does anything for us. Signed-off-by: Matt Turner gentoo.org> catalyst/base/clearbase.py | 2 -- catalyst/base/stagebase.py | 10 +++- catalyst/lock.py | 58 catalyst/targets/snapshot.py | 6 ++--- 4 files changed, 7 insertions(+), 69 deletions(-) diff --git a/catalyst/base/clearbase.py b/catalyst/base/clearbase.py index dcf6c523..6218330e 100644 --- a/catalyst/base/clearbase.py +++ b/catalyst/base/clearbase.py @@ -27,12 +27,10 @@ class ClearBase(): self.resume.clear_all(remove=True) def clear_chroot(self): -self.chroot_lock.unlock() log.notice('Clearing the chroot path ...') clear_dir(self.settings["chroot_path"], mode=0o755) def remove_chroot(self): -self.chroot_lock.unlock() log.notice('Removing the chroot path ...') clear_dir(self.settings["chroot_path"], mode=0o755, remove=True) diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py index 448d6265..10cffd4c 100644 --- a/catalyst/base/stagebase.py +++ b/catalyst/base/stagebase.py @@ -8,6 +8,7 @@ import sys from pathlib import Path +import fasteners import libmount import toml @@ -25,7 +26,6 @@ from catalyst.support import (CatalystError, file_locate, normpath, from catalyst.base.targetbase import TargetBase from catalyst.base.clearbase import ClearBase from catalyst.base.genbase import GenBase -from catalyst.lock import LockDir, LockInUse from catalyst.fileops import ensure_dirs, clear_dir, clear_path from catalyst.base.resume import AutoResume @@ -36,9 +36,6 @@ def run_sequence(sequence): sys.stdout.flush() try: func() -except LockInUse: -log.error('Unable to aquire the lock...') -return False except Exception: log.error('Exception running action sequence %s', func.__name__, exc_info=True) @@ -478,7 +475,6 @@ class StageBase(TargetBase, ClearBase, GenBase): """ self.settings["chroot_path"] = normpath(self.settings["storedir"] + "/tmp/" + self.settings["target_subpath"].rstrip('/')) -self.chroot_lock = LockDir(self.settings["chroot_path"]) def set_autoresume_path(self): self.settings["autoresume_path"] = normpath(pjoin( @@ -1366,8 +1362,10 @@ class StageBase(TargetBase, ClearBase, GenBase): pass def run(self): -self.chroot_lock.write_lock() +with fasteners.InterProcessLock(self.settings["chroot_path"] + '.lock'): +return self._run() +def _run(self): if "clear-autoresume" in self.settings["options"]: self.clear_autoresume() diff --git a/catalyst/lock.py b/catalyst/lock.py deleted file mode 100644 index e31745b2.. --- a/catalyst/lock.py +++ /dev/null @@ -1,58 +0,0 @@ - -import os - -from contextlib import contextmanager - -from snakeoil import fileutils -from snakeoil import osutils -from catalyst.fileops import ensure_dirs - - -LockInUse = osutils.LockException - -class Lock: -""" -A fnctl-based filesystem lock -""" -def __init__(self, lockfile): -fileutils.touch(lockfile, mode=0o664) -os.chown(lockfile, uid=-1, gid=250) -self.lock = osutils.FsLock(lockfile) - -def read_lock(self): -self.lock.acquire_read_lock() - -def write_lock(self): -self.lock.acquire_write_lock() - -def unlock(self): -# Releasing a write lock is the same as a read lock. -self.lock.release_write_lock() - -class LockDir(Lock): -""" -A fnctl-based filesystem lock in a directory -""" -def __init__(self, lockdir): -ensure_dirs(lockdir) -lockfile = os.path.join(lockdir, '.catalyst_lock') - -Lock.__init__(self, lockfile) - -@contextmanager -def read_lock(filename): -lock = Lock(filename) -lock.read_lock() -try: -yield -finally: -lock.unlock() - -@contextmanager -def write_lock(filename): -lock = Lock(filename) -lock.write_lock() -try: -yield -finally: -lock.unlock() diff --git a/catalyst/targets/snapshot.py b/catalyst/targets/snapshot.py index b494575a..ef68765d 100644 --- a/catalyst/targets/snapshot.p
[gentoo-commits] proj/catalyst:wip/mattst88 commit in: catalyst/base/, catalyst/targets/, catalyst/
commit: f535f7c03185442359ab1c5381205de736f547ea Author: Matt Turner gentoo org> AuthorDate: Wed Jun 9 06:17:31 2021 + Commit: Matt Turner gentoo org> CommitDate: Thu Jun 10 00:40:14 2021 + URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=f535f7c0 catalyst: Replace snakeoil's locks with fasteners To no great surprise, the existing locking was broken. For example, clear_chroot() releases the lock. It is called by unpack(), which is part of prepare_sequence. The result is that the whole build could be done without holding the lock. Just lock around run(). It's not apparent that finer-grained locking does anything for us. Signed-off-by: Matt Turner gentoo.org> catalyst/base/clearbase.py | 2 -- catalyst/base/stagebase.py | 8 +- catalyst/lock.py | 58 catalyst/targets/snapshot.py | 6 ++--- 4 files changed, 4 insertions(+), 70 deletions(-) diff --git a/catalyst/base/clearbase.py b/catalyst/base/clearbase.py index dcf6c523..6218330e 100644 --- a/catalyst/base/clearbase.py +++ b/catalyst/base/clearbase.py @@ -27,12 +27,10 @@ class ClearBase(): self.resume.clear_all(remove=True) def clear_chroot(self): -self.chroot_lock.unlock() log.notice('Clearing the chroot path ...') clear_dir(self.settings["chroot_path"], mode=0o755) def remove_chroot(self): -self.chroot_lock.unlock() log.notice('Removing the chroot path ...') clear_dir(self.settings["chroot_path"], mode=0o755, remove=True) diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py index 448d6265..fac5d790 100644 --- a/catalyst/base/stagebase.py +++ b/catalyst/base/stagebase.py @@ -25,7 +25,6 @@ from catalyst.support import (CatalystError, file_locate, normpath, from catalyst.base.targetbase import TargetBase from catalyst.base.clearbase import ClearBase from catalyst.base.genbase import GenBase -from catalyst.lock import LockDir, LockInUse from catalyst.fileops import ensure_dirs, clear_dir, clear_path from catalyst.base.resume import AutoResume @@ -36,9 +35,6 @@ def run_sequence(sequence): sys.stdout.flush() try: func() -except LockInUse: -log.error('Unable to aquire the lock...') -return False except Exception: log.error('Exception running action sequence %s', func.__name__, exc_info=True) @@ -478,7 +474,6 @@ class StageBase(TargetBase, ClearBase, GenBase): """ self.settings["chroot_path"] = normpath(self.settings["storedir"] + "/tmp/" + self.settings["target_subpath"].rstrip('/')) -self.chroot_lock = LockDir(self.settings["chroot_path"]) def set_autoresume_path(self): self.settings["autoresume_path"] = normpath(pjoin( @@ -1365,9 +1360,8 @@ class StageBase(TargetBase, ClearBase, GenBase): except CatalystError: pass +@fasteners.interprocess_locked(self.chroot_path + '.lock') def run(self): -self.chroot_lock.write_lock() - if "clear-autoresume" in self.settings["options"]: self.clear_autoresume() diff --git a/catalyst/lock.py b/catalyst/lock.py deleted file mode 100644 index e31745b2.. --- a/catalyst/lock.py +++ /dev/null @@ -1,58 +0,0 @@ - -import os - -from contextlib import contextmanager - -from snakeoil import fileutils -from snakeoil import osutils -from catalyst.fileops import ensure_dirs - - -LockInUse = osutils.LockException - -class Lock: -""" -A fnctl-based filesystem lock -""" -def __init__(self, lockfile): -fileutils.touch(lockfile, mode=0o664) -os.chown(lockfile, uid=-1, gid=250) -self.lock = osutils.FsLock(lockfile) - -def read_lock(self): -self.lock.acquire_read_lock() - -def write_lock(self): -self.lock.acquire_write_lock() - -def unlock(self): -# Releasing a write lock is the same as a read lock. -self.lock.release_write_lock() - -class LockDir(Lock): -""" -A fnctl-based filesystem lock in a directory -""" -def __init__(self, lockdir): -ensure_dirs(lockdir) -lockfile = os.path.join(lockdir, '.catalyst_lock') - -Lock.__init__(self, lockfile) - -@contextmanager -def read_lock(filename): -lock = Lock(filename) -lock.read_lock() -try: -yield -finally: -lock.unlock() - -@contextmanager -def write_lock(filename): -lock = Lock(filename) -lock.write_lock() -try: -yield -finally: -lock.unlock() diff --git a/catalyst/targets/snapshot.py b/catalyst/targets/snapshot.py index b494575a..ef68765d 100644 --- a/catalyst/targets/snapshot.py +++ b/catalyst/targets/snapshot.py @@ -5,11 +5,12 @@ Snapshot target import subprocess import sys +import fasteners + from pathlib import Path
[gentoo-commits] proj/catalyst:wip/mattst88 commit in: catalyst/base/, catalyst/targets/
commit: 047d9c1a2a968365b55e5b3b4b3418fd0ef24764 Author: Matt Turner gentoo org> AuthorDate: Mon Jan 18 03:15:02 2021 + Commit: Matt Turner gentoo org> CommitDate: Mon Jan 18 19:37:55 2021 + URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=047d9c1a catalyst: Deduplicate prepare_sequence assignments Signed-off-by: Matt Turner gentoo.org> catalyst/base/stagebase.py| 13 ++--- catalyst/targets/embedded.py | 6 -- catalyst/targets/livecd_stage1.py | 6 -- catalyst/targets/livecd_stage2.py | 6 -- catalyst/targets/netboot.py | 6 -- catalyst/targets/stage4.py| 6 -- 6 files changed, 6 insertions(+), 37 deletions(-) diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py index 53b0a224..3b8d3a6e 100644 --- a/catalyst/base/stagebase.py +++ b/catalyst/base/stagebase.py @@ -84,7 +84,12 @@ class StageBase(TargetBase, ClearBase, GenBase): "portage_overlay", "portage_prefix", ]) -self.prepare_sequence = [] +self.prepare_sequence = [ +self.unpack, +self.config_profile_link, +self.setup_confdir, +self.portage_overlay, +] self.build_sequence = [] self.finish_sequence = [] @@ -498,12 +503,6 @@ class StageBase(TargetBase, ClearBase, GenBase): def set_action_sequence(self): """Set basic stage1, 2, 3 action sequences""" -self.prepare_sequence.extend([ -self.unpack, -self.config_profile_link, -self.setup_confdir, -self.portage_overlay, -]) self.build_sequence.extend([ self.bind, self.chroot_setup, diff --git a/catalyst/targets/embedded.py b/catalyst/targets/embedded.py index 7d328808..83b17ad2 100644 --- a/catalyst/targets/embedded.py +++ b/catalyst/targets/embedded.py @@ -41,12 +41,6 @@ class embedded(StageBase): StageBase.__init__(self, spec, addlargs) def set_action_sequence(self): -self.prepare_sequence.extend([ -self.unpack, -self.config_profile_link, -self.setup_confdir, -self.portage_overlay, -]) self.build_sequence.extend([ self.bind, self.chroot_setup, diff --git a/catalyst/targets/livecd_stage1.py b/catalyst/targets/livecd_stage1.py index eb6bb0f0..9fe95a00 100644 --- a/catalyst/targets/livecd_stage1.py +++ b/catalyst/targets/livecd_stage1.py @@ -23,12 +23,6 @@ class livecd_stage1(StageBase): StageBase.__init__(self, spec, addlargs) def set_action_sequence(self): -self.prepare_sequence.extend([ -self.unpack, -self.config_profile_link, -self.setup_confdir, -self.portage_overlay, -]) self.build_sequence.extend([ self.bind, self.chroot_setup, diff --git a/catalyst/targets/livecd_stage2.py b/catalyst/targets/livecd_stage2.py index 5fecff85..ab8f4a0e 100644 --- a/catalyst/targets/livecd_stage2.py +++ b/catalyst/targets/livecd_stage2.py @@ -86,12 +86,6 @@ class livecd_stage2(StageBase): print_traceback=True) def set_action_sequence(self): -self.prepare_sequence.extend([ -self.unpack, -self.config_profile_link, -self.setup_confdir, -self.portage_overlay, -]) self.build_sequence.extend([ self.bind, self.chroot_setup, diff --git a/catalyst/targets/netboot.py b/catalyst/targets/netboot.py index 9a7e59e5..cb5e7ae4 100644 --- a/catalyst/targets/netboot.py +++ b/catalyst/targets/netboot.py @@ -158,12 +158,6 @@ class netboot(StageBase): self.resume.enable("empty") def set_action_sequence(self): -self.prepare_sequence.extend([ -self.unpack, -self.config_profile_link, -self.setup_confdir, -self.portage_overlay, -]) self.build_sequence.extend([ self.bind, self.chroot_setup, diff --git a/catalyst/targets/stage4.py b/catalyst/targets/stage4.py index 997139a5..9e53f070 100644 --- a/catalyst/targets/stage4.py +++ b/catalyst/targets/stage4.py @@ -39,12 +39,6 @@ class stage4(StageBase): self.settings["cleanables"].remove('/etc/resolv.conf') def set_action_sequence(self): -self.prepare_sequence.extend([ -self.unpack, -self.config_profile_link, -self.setup_confdir, -self.portage_overlay, -]) self.build_sequence.extend([ self.bind, self.chroot_setup,
[gentoo-commits] proj/catalyst:wip/mattst88 commit in: catalyst/base/, catalyst/targets/
commit: 9ad4ff837c440d92af2c35d1fe085b0fb6250e2d Author: Matt Turner gentoo org> AuthorDate: Mon Jan 18 03:15:02 2021 + Commit: Matt Turner gentoo org> CommitDate: Mon Jan 18 18:01:21 2021 + URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=9ad4ff83 catalyst: Deduplicate prepare_sequence assignments Signed-off-by: Matt Turner gentoo.org> catalyst/base/stagebase.py| 13 ++--- catalyst/targets/embedded.py | 6 -- catalyst/targets/livecd_stage1.py | 6 -- catalyst/targets/livecd_stage2.py | 6 -- catalyst/targets/netboot.py | 6 -- catalyst/targets/stage4.py| 6 -- 6 files changed, 6 insertions(+), 37 deletions(-) diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py index 9d72e765..8107ee7a 100644 --- a/catalyst/base/stagebase.py +++ b/catalyst/base/stagebase.py @@ -84,7 +84,12 @@ class StageBase(TargetBase, ClearBase, GenBase): "portage_overlay", "portage_prefix", ]) -self.prepare_sequence = [] +self.prepare_sequence = [ +self.unpack, +self.config_profile_link, +self.setup_confdir, +self.portage_overlay, +] self.build_sequence = [] self.finish_sequence = [] @@ -498,12 +503,6 @@ class StageBase(TargetBase, ClearBase, GenBase): def set_action_sequence(self): """Set basic stage1, 2, 3 action sequences""" -self.prepare_sequence.extend([ -self.unpack, -self.config_profile_link, -self.setup_confdir, -self.portage_overlay, -]) self.build_sequence.extend([ self.bind, self.chroot_setup, diff --git a/catalyst/targets/embedded.py b/catalyst/targets/embedded.py index 7d328808..83b17ad2 100644 --- a/catalyst/targets/embedded.py +++ b/catalyst/targets/embedded.py @@ -41,12 +41,6 @@ class embedded(StageBase): StageBase.__init__(self, spec, addlargs) def set_action_sequence(self): -self.prepare_sequence.extend([ -self.unpack, -self.config_profile_link, -self.setup_confdir, -self.portage_overlay, -]) self.build_sequence.extend([ self.bind, self.chroot_setup, diff --git a/catalyst/targets/livecd_stage1.py b/catalyst/targets/livecd_stage1.py index eb6bb0f0..9fe95a00 100644 --- a/catalyst/targets/livecd_stage1.py +++ b/catalyst/targets/livecd_stage1.py @@ -23,12 +23,6 @@ class livecd_stage1(StageBase): StageBase.__init__(self, spec, addlargs) def set_action_sequence(self): -self.prepare_sequence.extend([ -self.unpack, -self.config_profile_link, -self.setup_confdir, -self.portage_overlay, -]) self.build_sequence.extend([ self.bind, self.chroot_setup, diff --git a/catalyst/targets/livecd_stage2.py b/catalyst/targets/livecd_stage2.py index 5fecff85..ab8f4a0e 100644 --- a/catalyst/targets/livecd_stage2.py +++ b/catalyst/targets/livecd_stage2.py @@ -86,12 +86,6 @@ class livecd_stage2(StageBase): print_traceback=True) def set_action_sequence(self): -self.prepare_sequence.extend([ -self.unpack, -self.config_profile_link, -self.setup_confdir, -self.portage_overlay, -]) self.build_sequence.extend([ self.bind, self.chroot_setup, diff --git a/catalyst/targets/netboot.py b/catalyst/targets/netboot.py index 9a7e59e5..cb5e7ae4 100644 --- a/catalyst/targets/netboot.py +++ b/catalyst/targets/netboot.py @@ -158,12 +158,6 @@ class netboot(StageBase): self.resume.enable("empty") def set_action_sequence(self): -self.prepare_sequence.extend([ -self.unpack, -self.config_profile_link, -self.setup_confdir, -self.portage_overlay, -]) self.build_sequence.extend([ self.bind, self.chroot_setup, diff --git a/catalyst/targets/stage4.py b/catalyst/targets/stage4.py index 997139a5..9e53f070 100644 --- a/catalyst/targets/stage4.py +++ b/catalyst/targets/stage4.py @@ -39,12 +39,6 @@ class stage4(StageBase): self.settings["cleanables"].remove('/etc/resolv.conf') def set_action_sequence(self): -self.prepare_sequence.extend([ -self.unpack, -self.config_profile_link, -self.setup_confdir, -self.portage_overlay, -]) self.build_sequence.extend([ self.bind, self.chroot_setup,
[gentoo-commits] proj/catalyst:wip/mattst88 commit in: catalyst/base/, catalyst/targets/
commit: 9e6046dd34b6f225d544713ffabe36574cd17143 Author: Matt Turner gentoo org> AuthorDate: Mon Jan 18 03:20:41 2021 + Commit: Matt Turner gentoo org> CommitDate: Mon Jan 18 18:01:21 2021 + URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=9e6046dd catalyst: Deduplicate the common build_sequence steps This also provides a single location to add a function to all targets' build_sequence. Signed-off-by: Matt Turner gentoo.org> catalyst/base/stagebase.py| 11 ++- catalyst/targets/embedded.py | 3 --- catalyst/targets/livecd_stage1.py | 3 --- catalyst/targets/livecd_stage2.py | 3 --- catalyst/targets/netboot.py | 3 --- catalyst/targets/stage4.py| 3 --- 6 files changed, 6 insertions(+), 20 deletions(-) diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py index 8107ee7a..dfa4680d 100644 --- a/catalyst/base/stagebase.py +++ b/catalyst/base/stagebase.py @@ -90,7 +90,11 @@ class StageBase(TargetBase, ClearBase, GenBase): self.setup_confdir, self.portage_overlay, ] -self.build_sequence = [] +self.build_sequence = [ +self.bind, +self.chroot_setup, +self.setup_environment, +] self.finish_sequence = [] self.set_valid_build_kernel_vars(addlargs) @@ -497,16 +501,13 @@ class StageBase(TargetBase, ClearBase, GenBase): Or it calls the normal set_action_sequence() for the target stage. """ if "purgeonly" in self.settings["options"]: -self.build_sequence.append(self.remove_chroot) +self.build_sequence = [self.remove_chroot] return self.set_action_sequence() def set_action_sequence(self): """Set basic stage1, 2, 3 action sequences""" self.build_sequence.extend([ -self.bind, -self.chroot_setup, -self.setup_environment, self.run_local, self.preclean, ]) diff --git a/catalyst/targets/embedded.py b/catalyst/targets/embedded.py index 83b17ad2..b677b226 100644 --- a/catalyst/targets/embedded.py +++ b/catalyst/targets/embedded.py @@ -42,9 +42,6 @@ class embedded(StageBase): def set_action_sequence(self): self.build_sequence.extend([ -self.bind, -self.chroot_setup, -self.setup_environment, self.build_kernel, self.build_packages, self.root_overlay, diff --git a/catalyst/targets/livecd_stage1.py b/catalyst/targets/livecd_stage1.py index 9fe95a00..dbfa54ed 100644 --- a/catalyst/targets/livecd_stage1.py +++ b/catalyst/targets/livecd_stage1.py @@ -24,9 +24,6 @@ class livecd_stage1(StageBase): def set_action_sequence(self): self.build_sequence.extend([ -self.bind, -self.chroot_setup, -self.setup_environment, self.build_packages, ]) self.finish_sequence.extend([ diff --git a/catalyst/targets/livecd_stage2.py b/catalyst/targets/livecd_stage2.py index ab8f4a0e..e90e9f53 100644 --- a/catalyst/targets/livecd_stage2.py +++ b/catalyst/targets/livecd_stage2.py @@ -87,9 +87,6 @@ class livecd_stage2(StageBase): def set_action_sequence(self): self.build_sequence.extend([ -self.bind, -self.chroot_setup, -self.setup_environment, self.run_local, self.build_kernel ]) diff --git a/catalyst/targets/netboot.py b/catalyst/targets/netboot.py index cb5e7ae4..a2a9fcb3 100644 --- a/catalyst/targets/netboot.py +++ b/catalyst/targets/netboot.py @@ -159,9 +159,6 @@ class netboot(StageBase): def set_action_sequence(self): self.build_sequence.extend([ -self.bind, -self.chroot_setup, -self.setup_environment, self.build_packages, self.root_overlay, self.copy_files_to_image, diff --git a/catalyst/targets/stage4.py b/catalyst/targets/stage4.py index 9e53f070..f8383f75 100644 --- a/catalyst/targets/stage4.py +++ b/catalyst/targets/stage4.py @@ -40,9 +40,6 @@ class stage4(StageBase): def set_action_sequence(self): self.build_sequence.extend([ -self.bind, -self.chroot_setup, -self.setup_environment, self.build_packages, self.build_kernel, self.bootloader,
[gentoo-commits] proj/catalyst:wip/mattst88 commit in: catalyst/base/, catalyst/targets/
commit: 3bb5ead36cb748af2de00d2c526ac7af2128cc22 Author: Matt Turner gentoo org> AuthorDate: Sun Jan 17 23:53:21 2021 + Commit: Matt Turner gentoo org> CommitDate: Mon Jan 18 17:05:43 2021 + URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=3bb5ead3 catalyst: Store references to functions ... rather than their names. This makes it possible for tooling to understand the code structure better. Signed-off-by: Matt Turner gentoo.org> catalyst/base/stagebase.py| 69 --- catalyst/targets/embedded.py | 34 +-- catalyst/targets/livecd_stage1.py | 18 +- catalyst/targets/livecd_stage2.py | 46 +- catalyst/targets/netboot.py | 34 +-- catalyst/targets/stage1.py| 12 +++ catalyst/targets/stage4.py| 36 ++-- 7 files changed, 125 insertions(+), 124 deletions(-) diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py index ed4d1227..447e073d 100644 --- a/catalyst/base/stagebase.py +++ b/catalyst/base/stagebase.py @@ -30,6 +30,23 @@ from catalyst.fileops import ensure_dirs, clear_dir, clear_path from catalyst.base.resume import AutoResume +def run_sequence(sequence): +for func in sequence: +log.notice('--- Running action sequence: %s', func.__name__) +sys.stdout.flush() +try: +func() +except LockInUse: +log.error('Unable to aquire the lock...') +return False +except Exception: +log.error('Exception running action sequence %s', + func.__name__, exc_info=True) +return False + +return True + + class StageBase(TargetBase, ClearBase, GenBase): """ This class does all of the chroot setup, copying of files, etc. It is @@ -475,39 +492,39 @@ class StageBase(TargetBase, ClearBase, GenBase): Or it calls the normal set_action_sequence() for the target stage. """ if "purgeonly" in self.settings["options"]: -self.build_sequence.append("remove_chroot") +self.build_sequence.append(self.remove_chroot) return self.set_action_sequence() def set_action_sequence(self): """Set basic stage1, 2, 3 action sequences""" self.prepare_sequence.extend([ -"unpack", -"setup_confdir", -"portage_overlay", +self.unpack, +self.setup_confdir, +self.portage_overlay, ]) self.build_sequence.extend([ -"bind", -"chroot_setup", -"setup_environment", -"run_local", -"preclean", +self.bind, +self.chroot_setup, +self.setup_environment, +self.run_local, +self.preclean, ]) self.finish_sequence.extend([ -"clean", +self.clean, ]) self.set_completion_action_sequences() def set_completion_action_sequences(self): if "fetch" not in self.settings["options"]: -self.finish_sequence.append("capture") +self.finish_sequence.append(self.capture) if "keepwork" in self.settings["options"]: -self.finish_sequence.append("clear_autoresume") +self.finish_sequence.append(self.clear_autoresume) elif "seedcache" in self.settings["options"]: -self.finish_sequence.append("remove_autoresume") +self.finish_sequence.append(self.remove_autoresume) else: -self.finish_sequence.append("remove_autoresume") -self.finish_sequence.append("remove_chroot") +self.finish_sequence.append(self.remove_autoresume) +self.finish_sequence.append(self.remove_chroot) def set_use(self): use = self.settings["spec_prefix"] + "/use" @@ -1308,22 +1325,6 @@ class StageBase(TargetBase, ClearBase, GenBase): log.debug('setup_environment(); env = %r', self.env) -def run_sequence(self, sequence): -for func in sequence: -log.notice('--- Running action sequence: %s', func) -sys.stdout.flush() -try: -getattr(self, func)() -except LockInUse: -log.error('Unable to aquire the lock...') -return False -except Exception: -log.error('Exception running action sequence %s', - func, exc_info=True) -return False - -return True - def run(self): self.chroot_lock.write_lock() @@ -1342,14 +1343,14 @@ class StageBase(TargetBase, ClearBase, GenBase): log.info('StageBase: run() purge') self.purge() -if not self.run_sequence(self.prepare_sequence): +if not run_sequence(self.prepare_sequence):
[gentoo-commits] proj/catalyst:wip/mattst88 commit in: catalyst/base/, catalyst/targets/
commit: 402ac4495d25cbb91f9f81de8150e03428c94ae6 Author: Matt Turner gentoo org> AuthorDate: Mon Jan 18 03:15:02 2021 + Commit: Matt Turner gentoo org> CommitDate: Mon Jan 18 16:55:29 2021 + URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=402ac449 catalyst: Deduplicate prepare_sequence assignments Signed-off-by: Matt Turner gentoo.org> catalyst/base/stagebase.py| 13 ++--- catalyst/targets/embedded.py | 6 -- catalyst/targets/livecd_stage1.py | 6 -- catalyst/targets/livecd_stage2.py | 6 -- catalyst/targets/netboot.py | 6 -- catalyst/targets/stage4.py| 6 -- 6 files changed, 6 insertions(+), 37 deletions(-) diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py index bccba11c..85eb1f6d 100644 --- a/catalyst/base/stagebase.py +++ b/catalyst/base/stagebase.py @@ -67,7 +67,12 @@ class StageBase(TargetBase, ClearBase, GenBase): "portage_overlay", "portage_prefix", ]) -self.prepare_sequence = [] +self.prepare_sequence = [ +self.unpack, +self.config_profile_link, +self.setup_confdir, +self.portage_overlay, +] self.build_sequence = [] self.finish_sequence = [] @@ -481,12 +486,6 @@ class StageBase(TargetBase, ClearBase, GenBase): def set_action_sequence(self): """Set basic stage1, 2, 3 action sequences""" -self.prepare_sequence.extend([ -self.unpack, -self.config_profile_link, -self.setup_confdir, -self.portage_overlay, -]) self.build_sequence.extend([ self.bind, self.chroot_setup, diff --git a/catalyst/targets/embedded.py b/catalyst/targets/embedded.py index 7d328808..83b17ad2 100644 --- a/catalyst/targets/embedded.py +++ b/catalyst/targets/embedded.py @@ -41,12 +41,6 @@ class embedded(StageBase): StageBase.__init__(self, spec, addlargs) def set_action_sequence(self): -self.prepare_sequence.extend([ -self.unpack, -self.config_profile_link, -self.setup_confdir, -self.portage_overlay, -]) self.build_sequence.extend([ self.bind, self.chroot_setup, diff --git a/catalyst/targets/livecd_stage1.py b/catalyst/targets/livecd_stage1.py index eb6bb0f0..9fe95a00 100644 --- a/catalyst/targets/livecd_stage1.py +++ b/catalyst/targets/livecd_stage1.py @@ -23,12 +23,6 @@ class livecd_stage1(StageBase): StageBase.__init__(self, spec, addlargs) def set_action_sequence(self): -self.prepare_sequence.extend([ -self.unpack, -self.config_profile_link, -self.setup_confdir, -self.portage_overlay, -]) self.build_sequence.extend([ self.bind, self.chroot_setup, diff --git a/catalyst/targets/livecd_stage2.py b/catalyst/targets/livecd_stage2.py index 5fecff85..ab8f4a0e 100644 --- a/catalyst/targets/livecd_stage2.py +++ b/catalyst/targets/livecd_stage2.py @@ -86,12 +86,6 @@ class livecd_stage2(StageBase): print_traceback=True) def set_action_sequence(self): -self.prepare_sequence.extend([ -self.unpack, -self.config_profile_link, -self.setup_confdir, -self.portage_overlay, -]) self.build_sequence.extend([ self.bind, self.chroot_setup, diff --git a/catalyst/targets/netboot.py b/catalyst/targets/netboot.py index 9a7e59e5..cb5e7ae4 100644 --- a/catalyst/targets/netboot.py +++ b/catalyst/targets/netboot.py @@ -158,12 +158,6 @@ class netboot(StageBase): self.resume.enable("empty") def set_action_sequence(self): -self.prepare_sequence.extend([ -self.unpack, -self.config_profile_link, -self.setup_confdir, -self.portage_overlay, -]) self.build_sequence.extend([ self.bind, self.chroot_setup, diff --git a/catalyst/targets/stage4.py b/catalyst/targets/stage4.py index 997139a5..9e53f070 100644 --- a/catalyst/targets/stage4.py +++ b/catalyst/targets/stage4.py @@ -39,12 +39,6 @@ class stage4(StageBase): self.settings["cleanables"].remove('/etc/resolv.conf') def set_action_sequence(self): -self.prepare_sequence.extend([ -self.unpack, -self.config_profile_link, -self.setup_confdir, -self.portage_overlay, -]) self.build_sequence.extend([ self.bind, self.chroot_setup,
[gentoo-commits] proj/catalyst:wip/mattst88 commit in: catalyst/base/, catalyst/targets/
commit: a5f2e195d83f815e303f7bae3f0bb3591369d944 Author: Matt Turner gentoo org> AuthorDate: Sun Jan 17 23:53:21 2021 + Commit: Matt Turner gentoo org> CommitDate: Mon Jan 18 03:49:48 2021 + URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=a5f2e195 catalyst: Store references to functions ... rather than their names. This makes it possible for tooling to understand the code structure better. Signed-off-by: Matt Turner gentoo.org> catalyst/base/stagebase.py| 41 +- catalyst/targets/embedded.py | 34 ++--- catalyst/targets/livecd_stage1.py | 18 +++ catalyst/targets/livecd_stage2.py | 46 +++ catalyst/targets/netboot.py | 34 ++--- catalyst/targets/stage1.py| 12 +- catalyst/targets/stage4.py| 36 +++--- 7 files changed, 111 insertions(+), 110 deletions(-) diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py index ed4d1227..ae13e35f 100644 --- a/catalyst/base/stagebase.py +++ b/catalyst/base/stagebase.py @@ -475,39 +475,39 @@ class StageBase(TargetBase, ClearBase, GenBase): Or it calls the normal set_action_sequence() for the target stage. """ if "purgeonly" in self.settings["options"]: -self.build_sequence.append("remove_chroot") +self.build_sequence.append(self.remove_chroot) return self.set_action_sequence() def set_action_sequence(self): """Set basic stage1, 2, 3 action sequences""" self.prepare_sequence.extend([ -"unpack", -"setup_confdir", -"portage_overlay", +self.unpack, +self.setup_confdir, +self.portage_overlay, ]) self.build_sequence.extend([ -"bind", -"chroot_setup", -"setup_environment", -"run_local", -"preclean", +self.bind, +self.chroot_setup, +self.setup_environment, +self.run_local, +self.preclean, ]) self.finish_sequence.extend([ -"clean", +self.clean, ]) self.set_completion_action_sequences() def set_completion_action_sequences(self): if "fetch" not in self.settings["options"]: -self.finish_sequence.append("capture") +self.finish_sequence.append(self.capture) if "keepwork" in self.settings["options"]: -self.finish_sequence.append("clear_autoresume") +self.finish_sequence.append(self.clear_autoresume) elif "seedcache" in self.settings["options"]: -self.finish_sequence.append("remove_autoresume") +self.finish_sequence.append(self.remove_autoresume) else: -self.finish_sequence.append("remove_autoresume") -self.finish_sequence.append("remove_chroot") +self.finish_sequence.append(self.remove_autoresume) +self.finish_sequence.append(self.remove_chroot) def set_use(self): use = self.settings["spec_prefix"] + "/use" @@ -1308,12 +1308,13 @@ class StageBase(TargetBase, ClearBase, GenBase): log.debug('setup_environment(); env = %r', self.env) -def run_sequence(self, sequence): +@staticmethod +def run_sequence(sequence): for func in sequence: log.notice('--- Running action sequence: %s', func) sys.stdout.flush() try: -getattr(self, func)() +func() except LockInUse: log.error('Unable to aquire the lock...') return False @@ -1342,14 +1343,14 @@ class StageBase(TargetBase, ClearBase, GenBase): log.info('StageBase: run() purge') self.purge() -if not self.run_sequence(self.prepare_sequence): +if not run_sequence(self.prepare_sequence): return False with namespace(mount=True): -if not self.run_sequence(self.build_sequence): +if not run_sequence(self.build_sequence): return False -if not self.run_sequence(self.finish_sequence): +if not run_sequence(self.finish_sequence): return False return True diff --git a/catalyst/targets/embedded.py b/catalyst/targets/embedded.py index 918b36bf..7d328808 100644 --- a/catalyst/targets/embedded.py +++ b/catalyst/targets/embedded.py @@ -42,27 +42,27 @@ class embedded(StageBase): def set_action_sequence(self): self.prepare_sequence.extend([ -"unpack", -"config_profile_link", -"setup_confdir", -"portage_overlay", +self.unpack, +self.config_profile_link, +self.setup_confdir, +se
[gentoo-commits] proj/catalyst:wip/mattst88 commit in: catalyst/base/, catalyst/targets/
commit: 4362ffbf257f7d9a05ad81e46b522670e82ad78d Author: Matt Turner gentoo org> AuthorDate: Sun Jan 17 23:53:21 2021 + Commit: Matt Turner gentoo org> CommitDate: Mon Jan 18 03:27:41 2021 + URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=4362ffbf catalyst: Store references to functions ... rather than their names. This makes it possible for tooling to understand the code structure better. Signed-off-by: Matt Turner gentoo.org> catalyst/base/stagebase.py| 41 +- catalyst/targets/embedded.py | 34 ++--- catalyst/targets/livecd_stage1.py | 18 +++ catalyst/targets/livecd_stage2.py | 46 +++ catalyst/targets/netboot.py | 34 ++--- catalyst/targets/stage1.py| 12 +- catalyst/targets/stage4.py| 36 +++--- 7 files changed, 111 insertions(+), 110 deletions(-) diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py index 8d8d01d1..79f00149 100644 --- a/catalyst/base/stagebase.py +++ b/catalyst/base/stagebase.py @@ -475,39 +475,39 @@ class StageBase(TargetBase, ClearBase, GenBase): Or it calls the normal set_action_sequence() for the target stage. """ if "purgeonly" in self.settings["options"]: -self.build_sequence.append("remove_chroot") +self.build_sequence.append(self.remove_chroot) return self.set_action_sequence() def set_action_sequence(self): """Set basic stage1, 2, 3 action sequences""" self.prepare_sequence.extend([ -"unpack", -"setup_confdir", -"portage_overlay", +self.unpack, +self.setup_confdir, +self.portage_overlay, ]) self.build_sequence.extend([ -"bind", -"chroot_setup", -"setup_environment", -"run_local", -"preclean", +self.bind, +self.chroot_setup, +self.setup_environment, +self.run_local, +self.preclean, ]) self.finish_sequence.extend([ -"clean", +self.clean, ]) self.set_completion_action_sequences() def set_completion_action_sequences(self): if "fetch" not in self.settings["options"]: -self.finish_sequence.append("capture") +self.finish_sequence.append(self.capture) if "keepwork" in self.settings["options"]: -self.finish_sequence.append("clear_autoresume") +self.finish_sequence.append(self.clear_autoresume) elif "seedcache" in self.settings["options"]: -self.finish_sequence.append("remove_autoresume") +self.finish_sequence.append(self.remove_autoresume) else: -self.finish_sequence.append("remove_autoresume") -self.finish_sequence.append("remove_chroot") +self.finish_sequence.append(self.remove_autoresume) +self.finish_sequence.append(self.remove_chroot) def set_use(self): use = self.settings["spec_prefix"] + "/use" @@ -1308,12 +1308,13 @@ class StageBase(TargetBase, ClearBase, GenBase): log.debug('setup_environment(); env = %r', self.env) -def run_sequence(self, sequence): +@staticmethod +def run_sequence(sequence): for func in sequence: log.notice('--- Running action sequence: %s', func) sys.stdout.flush() try: -getattr(self, func)() +func() except LockInUse: log.error('Unable to aquire the lock...') return False @@ -1342,14 +1343,14 @@ class StageBase(TargetBase, ClearBase, GenBase): log.info('StageBase: run() purge') self.purge() -if not self.run_sequence(self.prepare_sequence): +if not run_sequence(self.prepare_sequence): return False with namespace(mount=True): -if not self.run_sequence(self.build_sequence): +if not run_sequence(self.build_sequence): return False -if not self.run_sequence(self.finish_sequence): +if not run_sequence(self.finish_sequence): return False return True diff --git a/catalyst/targets/embedded.py b/catalyst/targets/embedded.py index 918b36bf..7d328808 100644 --- a/catalyst/targets/embedded.py +++ b/catalyst/targets/embedded.py @@ -42,27 +42,27 @@ class embedded(StageBase): def set_action_sequence(self): self.prepare_sequence.extend([ -"unpack", -"config_profile_link", -"setup_confdir", -"portage_overlay", +self.unpack, +self.config_profile_link, +self.setup_confdir, +se
[gentoo-commits] proj/catalyst:wip/mattst88 commit in: catalyst/base/, catalyst/targets/
commit: 3d85d537b1a00d05234cea064f863602b83ffaab Author: Matt Turner gentoo org> AuthorDate: Thu Oct 29 03:08:04 2020 + Commit: Matt Turner gentoo org> CommitDate: Fri Oct 30 22:40:52 2020 + URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=3d85d537 catalyst: Split action_sequence into prepare/build/finish We want to run the "build" sequence in a different mount namespace from the "prepare" and "finish" sequences, so this splits action_sequence into those groupings. Signed-off-by: Matt Turner gentoo.org> catalyst/base/stagebase.py| 24 +++- catalyst/targets/embedded.py | 6 +- catalyst/targets/livecd_stage1.py | 6 +- catalyst/targets/livecd_stage2.py | 10 +++--- catalyst/targets/netboot.py | 6 +- catalyst/targets/stage1.py| 12 ++-- catalyst/targets/stage4.py| 6 +- 7 files changed, 48 insertions(+), 22 deletions(-) diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py index 46b7c59c..75c84baa 100644 --- a/catalyst/base/stagebase.py +++ b/catalyst/base/stagebase.py @@ -64,7 +64,9 @@ class StageBase(TargetBase, ClearBase, GenBase): "portage_overlay", "portage_prefix", ]) -self.action_sequence = [] +self.prepare_sequence = [] +self.build_sequence = [] +self.finish_sequence = [] self.set_valid_build_kernel_vars(addlargs) TargetBase.__init__(self, myspec, addlargs) @@ -478,36 +480,40 @@ class StageBase(TargetBase, ClearBase, GenBase): Or it calls the normal set_action_sequence() for the target stage. """ if "purgeonly" in self.settings["options"]: -self.action_sequence.append("remove_chroot") +self.build_sequence.append("remove_chroot") return self.set_action_sequence() def set_action_sequence(self): """Set basic stage1, 2, 3 action sequences""" -self.action_sequence.extend([ +self.prepare_sequence.extend([ "unpack", "setup_confdir", "portage_overlay", +]) +self.build_sequence.extend([ "bind", "chroot_setup", "setup_environment", "run_local", "preclean", "unbind", +]) +self.finish_sequence.extend([ "clean", ]) self.set_completion_action_sequences() def set_completion_action_sequences(self): if "fetch" not in self.settings["options"]: -self.action_sequence.append("capture") +self.finish_sequence.append("capture") if "keepwork" in self.settings["options"]: -self.action_sequence.append("clear_autoresume") +self.finish_sequence.append("clear_autoresume") elif "seedcache" in self.settings["options"]: -self.action_sequence.append("remove_autoresume") +self.finish_sequence.append("remove_autoresume") else: -self.action_sequence.append("remove_autoresume") -self.action_sequence.append("remove_chroot") +self.finish_sequence.append("remove_autoresume") +self.finish_sequence.append("remove_chroot") def set_use(self): use = self.settings["spec_prefix"] + "/use" @@ -1381,7 +1387,7 @@ class StageBase(TargetBase, ClearBase, GenBase): self.purge() failure = False -for x in self.action_sequence: +for x in self.prepare_sequence + self.build_sequence + self.finish_sequence: log.notice('--- Running action sequence: %s', x) sys.stdout.flush() try: diff --git a/catalyst/targets/embedded.py b/catalyst/targets/embedded.py index 75eb68e4..1314ce7c 100644 --- a/catalyst/targets/embedded.py +++ b/catalyst/targets/embedded.py @@ -41,11 +41,13 @@ class embedded(StageBase): StageBase.__init__(self, spec, addlargs) def set_action_sequence(self): -self.action_sequence.extend([ +self.prepare_sequence.extend([ "unpack", "config_profile_link", "setup_confdir", "portage_overlay", +]) +self.build_sequence.extend([ "bind", "chroot_setup", "setup_environment", @@ -55,6 +57,8 @@ class embedded(StageBase): "fsscript", "unmerge", "unbind", +]) +self.finish_sequence.extend([ "remove", "empty", "clean", diff --git a/catalyst/targets/livecd_stage1.py b/catalyst/targets/livecd_stage1.py index 9dbfa506..81367053 100644 --- a/catalyst/targets/livecd_stage1.py +++ b/catalyst/targets/livecd_stage1.py @@ -23,16 +23,20 @@ class livecd_stage1(StageBase): StageBase.__init__(self, spec, addlargs) def set_action_sequence(self): -self.action_
[gentoo-commits] proj/catalyst:wip/mattst88 commit in: catalyst/base/, catalyst/targets/, /
commit: d8e3b697d02d170179a87de66a46130c66ccb3b6 Author: Matt Turner gentoo org> AuthorDate: Fri Oct 30 00:31:10 2020 + Commit: Matt Turner gentoo org> CommitDate: Fri Oct 30 00:52:39 2020 + URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=d8e3b697 wip Signed-off-by: Matt Turner gentoo.org> catalyst/base/__init__.py| 0 catalyst/targets/__init__.py | 0 setup.py | 19 +-- 3 files changed, 5 insertions(+), 14 deletions(-) diff --git a/catalyst/base/__init__.py b/catalyst/base/__init__.py deleted file mode 100644 index e69de29b.. diff --git a/catalyst/targets/__init__.py b/catalyst/targets/__init__.py deleted file mode 100644 index e69de29b.. diff --git a/setup.py b/setup.py index d807ea02..d2fc3781 100755 --- a/setup.py +++ b/setup.py @@ -1,18 +1,9 @@ #!/usr/bin/env python -from glob import glob - from ppsetuptools import setup -data_files = [ -('/etc/catalyst', ['etc/catalyst.conf', 'etc/catalystrc']), -('/usr/share/man/man1', ['files/catalyst.1']), -('/usr/share/man/man5', - ['files/catalyst-config.5', 'files/catalyst-spec.5'] -), -('share/catalyst/arch', glob('arch/*')), -('share/catalyst/livecd', glob('livecd/**/*', recursive=True)), -('share/catalyst/targets', glob('targets/**', recursive=True)), -] - -setup(data_files=data_files) +setup( +packages=[ +'catalyst', 'catalyst/base', 'catalyst/targets' +], +)
[gentoo-commits] proj/catalyst:wip/mattst88 commit in: catalyst/base/, /, catalyst/targets/
commit: 1b9f81017f6653480f1e514d1d1c24ba3fa24fdd Author: Matt Turner gentoo org> AuthorDate: Thu Oct 29 23:33:27 2020 + Commit: Matt Turner gentoo org> CommitDate: Fri Oct 30 00:30:31 2020 + URL:https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=1b9f8101 wip Signed-off-by: Matt Turner gentoo.org> catalyst/base/__init__.py| 0 catalyst/targets/__init__.py | 0 setup.py | 19 --- 3 files changed, 4 insertions(+), 15 deletions(-) diff --git a/catalyst/base/__init__.py b/catalyst/base/__init__.py deleted file mode 100644 index e69de29b.. diff --git a/catalyst/targets/__init__.py b/catalyst/targets/__init__.py deleted file mode 100644 index e69de29b.. diff --git a/setup.py b/setup.py index d807ea02..02c96dcf 100755 --- a/setup.py +++ b/setup.py @@ -1,18 +1,7 @@ #!/usr/bin/env python -from glob import glob +from ppsetuptools import setup, find_namespace_packages -from ppsetuptools import setup - -data_files = [ -('/etc/catalyst', ['etc/catalyst.conf', 'etc/catalystrc']), -('/usr/share/man/man1', ['files/catalyst.1']), -('/usr/share/man/man5', - ['files/catalyst-config.5', 'files/catalyst-spec.5'] -), -('share/catalyst/arch', glob('arch/*')), -('share/catalyst/livecd', glob('livecd/**/*', recursive=True)), -('share/catalyst/targets', glob('targets/**', recursive=True)), -] - -setup(data_files=data_files) +setup( +packages=find_namespace_packages('catalyst'), +)