commit: 98cf07d2ca4da1f88e213a520095bfccc9c81ffc
Author: Matt Turner <mattst88 <AT> gentoo <DOT> org>
AuthorDate: Fri Apr 17 23:31:52 2020 +0000
Commit: Matt Turner <mattst88 <AT> gentoo <DOT> org>
CommitDate: Thu Apr 30 22:54:49 2020 +0000
URL: https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=98cf07d2
catalyst: Make and use squashfs snapshots
There were a number of problems with catalyst's snapshot system. It was
built around using the build system's portdir and had no control over
what was in that portdir or when it was updated.
As a result, when a stage build failed, it was difficult to tell what
the snapshot consistet of precisely or whether it contained a particular
recent fix.
With snapcache disabled, ebuild repo snapshots were tar'd and compressed
and then unpacked into the stage chroot which is an unnecessarily
expensive process. Moreover, a porttree has more than 100k small files,
which are stored extremely inefficiently on most file systems—a whole
porttree is usually around 700M on disk. Just removing all of those
files during the cleaning stage is an expensive operation.
Instead, we make a compressed squashfs image and mount it in the build
chroot. The porttree has many duplicate files, and squashfs deduplicates
the files and then compresses, so the result is very efficiently packed:
~38M with gzip -9 compression.
The snapshot target has been modified to generate a squashfs image from
a bare ebuild git repo. Piping git-archive to tar2sqfs generates the
squashfs image in less than 10 seconds on a modern system. The git repo
is fetched with --depth=1 to minize bandwidth and disk usage, and git gc
is run after fetch to minimize disk usage. Storage requirements for the
git ebuild repo with metadata are ~70M.
The squashfs snapshot is stored in /var/tmp/catalyst/snapshots/ by
default with a name <repo_name>-<git sha1>.sqfs. With this convention,
we know the exact point in history that the snapshot was taken. The
catalyst-auto script can use the sha1 to get a deterministic timestamp,
so that it is independent on when `catalyst -s` was run, but is instead
the timestamp of the commit date of the repo's git SHA1.
Signed-off-by: Matt Turner <mattst88 <AT> gentoo.org>
README | 3 +-
catalyst/base/stagebase.py | 88 ++++----------------
catalyst/base/targetbase.py | 16 +++-
catalyst/defaults.py | 2 +-
catalyst/main.py | 24 +++---
catalyst/targets/embedded.py | 1 -
catalyst/targets/livecd_stage1.py | 1 -
catalyst/targets/livecd_stage2.py | 1 -
catalyst/targets/netboot.py | 1 -
catalyst/targets/snapshot.py | 165 +++++++++++++++++++-------------------
catalyst/targets/stage4.py | 1 -
doc/catalyst-config.5.txt | 13 +--
doc/catalyst-spec.5.txt | 4 +-
13 files changed, 137 insertions(+), 183 deletions(-)
diff --git a/README b/README
index 1a039fca..1cceb63e 100644
--- a/README
+++ b/README
@@ -18,8 +18,9 @@ Requirements
=======================
- Python 3.6 or greater
-- An ebuild repository snapshot (or an ebuild tree to create one)
- A generic stage3 tarball for your architecture
+- A squashfs ebuild repository snapshot
+ - Or an ebuild git repo with sys-fs/squashfs-tools-ng and dev-vcs/git
What is catalyst?
========================
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 9aecf013..41da97b3 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -35,7 +35,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
self.required_values |= frozenset([
"profile",
"rel_type",
- "snapshot",
+ "snapshot_treeish",
"source_subpath",
"subarch",
"target",
@@ -149,7 +149,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
self.set_source_subpath()
# Set paths
- self.set_snapshot_path()
+ self.set_snapshot()
self.set_root_path()
self.set_source_path()
self.set_chroot_path()
@@ -191,9 +191,8 @@ class StageBase(TargetBase, ClearBase, GenBase):
# Setup our mount points.
self.mount = MOUNT_DEFAULTS.copy()
- # Always unpack snapshot tarball
- self.mount['portdir']['enable'] = False
-
+ self.mount['portdir']['source'] = self.snapshot
+ self.mount['portdir']['target'] = self.settings['repo_basedir'] + '/'
+ self.settings['repo_name']
self.mount['distdir']['source'] = self.settings['distdir']
self.mount["distdir"]['target'] = self.settings['target_distdir']
@@ -435,21 +434,11 @@ class StageBase(TargetBase, ClearBase, GenBase):
self.settings["destpath"] = normpath(self.settings["chroot_path"])
def set_cleanables(self):
- self.settings["cleanables"] = ["/etc/resolv.conf", "/var/tmp/*",
"/tmp/*",
- self.settings["repo_basedir"] + "/" +
- self.settings["repo_name"]]
-
- def set_snapshot_path(self):
- self.settings["snapshot_path"] = file_check(
- normpath(self.settings["storedir"] +
- "/snapshots/" + self.settings["snapshot_name"] +
- self.settings["snapshot"]),
- self.accepted_extensions,
- self.settings["source_matching"] == "strict"
- )
- log.info('SNAPSHOT_PATH set to: %s', self.settings['snapshot_path'])
- self.settings["snapshot_path_hash"] = \
- self.generate_hash(self.settings["snapshot_path"], "sha1")
+ self.settings['cleanables'] = [
+ "/etc/resolv.conf",
+ "/var/tmp/*",
+ "/tmp/*",
+ ]
def set_chroot_path(self):
"""
@@ -485,7 +474,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
"ISO volume ID must not exceed 32 characters.")
else:
self.settings["iso_volume_id"] = "catalyst " + \
- self.settings["snapshot"]
+ self.settings['snapshot_treeish']
def set_default_action_sequence(self):
""" Default action sequence for run method.
@@ -502,7 +491,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
"""Set basic stage1, 2, 3 action sequences"""
self.settings['action_sequence'] = [
"unpack",
- "unpack_snapshot",
"setup_confdir",
"portage_overlay",
"bind",
@@ -810,50 +798,6 @@ class StageBase(TargetBase, ClearBase, GenBase):
log.notice(
'Resume: Valid resume point detected, skipping seed unpack
operation...')
- def unpack_snapshot(self):
- unpack = True
- snapshot_hash = self.resume.get("unpack_repo")
-
- unpack_errmsg = "Error unpacking snapshot using mode %(mode)s"
-
- unpack_info = self.decompressor.create_infodict(
- source=self.settings["snapshot_path"],
- arch=self.settings["compressor_arch"],
- other_options=self.settings["compressor_options"],
- )
-
- target_portdir = normpath(self.settings["chroot_path"] +
- self.settings["repo_basedir"] + "/" +
self.settings["repo_name"])
- log.info('%s', self.settings['chroot_path'])
- log.info('unpack_snapshot(), target_portdir = %s', target_portdir)
- cleanup_msg = \
- 'Cleaning up existing portage tree (this can take a long time)...'
- unpack_info['destination'] = normpath(
- self.settings["chroot_path"] + self.settings["repo_basedir"])
- unpack_info['mode'] = self.decompressor.determine_mode(
- unpack_info['source'])
-
- if "autoresume" in self.settings["options"] \
- and os.path.exists(target_portdir) \
- and self.resume.is_enabled("unpack_repo") \
- and self.settings["snapshot_path_hash"] == snapshot_hash:
- log.notice(
- 'Valid Resume point detected, skipping unpack of portage
tree...')
- unpack = False
-
- if unpack:
- if os.path.exists(target_portdir):
- log.info('%s', cleanup_msg)
- clear_dir(target_portdir)
-
- log.notice('Unpacking portage tree (this can take a long time)
...')
- if not self.decompressor.extract(unpack_info):
- log.error('%s', unpack_errmsg % unpack_info)
-
- log.info('Setting snapshot autoresume point')
- self.resume.enable("unpack_repo",
- data=self.settings["snapshot_path_hash"])
-
def config_profile_link(self):
log.info('Configuring profile link...')
make_profile = Path(self.settings['chroot_path'] +
self.settings['port_conf'],
@@ -929,14 +873,16 @@ class StageBase(TargetBase, ClearBase, GenBase):
_cmd = ['mount', '-t', 'tmpfs', '-o', 'noexec,nosuid,nodev',
'shm', target]
else:
- _cmd = ['mount', '--bind', source, target]
+ _cmd = ['mount', source, target]
source = Path(self.mount[x]['source'])
+ if source.suffix != '.sqfs':
+ _cmd.insert(1, '--bind')
- # We may need to create the source of the bind mount. E.g., in
the
- # case of an empty package cache we must create the directory
that
- # the binary packages will be stored into.
- source.mkdir(mode=0o755, exist_ok=True)
+ # We may need to create the source of the bind mount.
E.g., in the
+ # case of an empty package cache we must create the
directory that
+ # the binary packages will be stored into.
+ source.mkdir(mode=0o755, exist_ok=True)
Path(target).mkdir(mode=0o755, parents=True, exist_ok=True)
diff --git a/catalyst/base/targetbase.py b/catalyst/base/targetbase.py
index fa15ec11..5bcea920 100644
--- a/catalyst/base/targetbase.py
+++ b/catalyst/base/targetbase.py
@@ -1,8 +1,9 @@
import os
from abc import ABC, abstractmethod
+from pathlib import Path
-from catalyst.support import addl_arg_parse
+from catalyst.support import addl_arg_parse, CatalystError
class TargetBase(ABC):
@@ -18,6 +19,19 @@ class TargetBase(ABC):
'PATH': '/bin:/sbin:/usr/bin:/usr/sbin',
'TERM': os.getenv('TERM', 'dumb'),
}
+ self.snapshot = None
+
+ def set_snapshot(self, treeish=None):
+ # Make snapshots directory
+ snapshot_dir = Path(self.settings['storedir'], 'snapshots')
+ snapshot_dir.mkdir(mode=0o755, exist_ok=True)
+
+ repo_name = self.settings['repo_name']
+ if treeish is None:
+ treeish = self.settings['snapshot_treeish']
+
+ self.snapshot = Path(snapshot_dir,
+ f'{repo_name}-{treeish}.sqfs')
@property
@classmethod
diff --git a/catalyst/defaults.py b/catalyst/defaults.py
index 787a13cc..33f06d34 100644
--- a/catalyst/defaults.py
+++ b/catalyst/defaults.py
@@ -68,8 +68,8 @@ confdefaults = {
"PythonDir": "./catalyst",
"repo_basedir": "/var/db/repos",
"repo_name": "gentoo",
+ "repos": "%(storedir)s/repos",
"sharedir": "/usr/share/catalyst",
- "snapshot_name": "%(repo_name)s-",
"shdir": "%(sharedir)s/targets",
"source_matching": "strict",
"storedir": "/var/tmp/catalyst",
diff --git a/catalyst/main.py b/catalyst/main.py
index 8ded4bd1..4417a295 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -3,6 +3,7 @@ import datetime
import hashlib
import os
import sys
+import textwrap
from snakeoil.process import namespaces
@@ -63,7 +64,7 @@ def parse_config(config_files):
log.info(option_messages[opt])
for key in ["digests", "envscript", "var_tmpfs_portage", "port_logdir",
- "local_overlay"]:
+ "local_overlay", "repos"]:
if key in myconf:
conf_values[key] = myconf[key]
@@ -121,16 +122,15 @@ class FilePath():
def get_parser():
"""Return an argument parser"""
- epilog = """Usage examples:
+ epilog = textwrap.dedent("""\
+ Usage examples:
-Using the commandline option (-C, --cli) to build a Portage snapshot:
-$ catalyst -C target=snapshot version_stamp=my_date
+ Using the snapshot option to make a snapshot of the ebuild repo:
+ $ catalyst --snapshot <git-treeish>
-Using the snapshot option (-s, --snapshot) to build a release snapshot:
-$ catalyst -s 20071121
-
-Using the specfile option (-f, --file) to build a stage target:
-$ catalyst -f stage1-specfile.spec"""
+ Using the specfile option (-f, --file) to build a stage target:
+ $ catalyst -f stage1-specfile.spec
+ """)
parser = argparse.ArgumentParser(
epilog=epilog, formatter_class=argparse.RawDescriptionHelpFormatter)
@@ -200,8 +200,8 @@ $ catalyst -f stage1-specfile.spec"""
group.add_argument('-f', '--file',
type=FilePath(),
help='read specfile')
- group.add_argument('-s', '--snapshot',
- help='generate a release snapshot')
+ group.add_argument('-s', '--snapshot', type=str,
+ help='Make an ebuild repo snapshot')
group.add_argument('-C', '--cli',
default=[], nargs=argparse.REMAINDER,
help='catalyst commandline (MUST BE LAST OPTION)')
@@ -298,7 +298,7 @@ def _main(parser, opts):
if opts.snapshot:
mycmdline.append('target=snapshot')
- mycmdline.append('version_stamp=' + opts.snapshot)
+ mycmdline.append('snapshot_treeish=' + opts.snapshot)
conf_values['DEBUG'] = opts.debug
conf_values['VERBOSE'] = opts.debug or opts.verbose
diff --git a/catalyst/targets/embedded.py b/catalyst/targets/embedded.py
index 189eb722..aa23f5b3 100644
--- a/catalyst/targets/embedded.py
+++ b/catalyst/targets/embedded.py
@@ -43,7 +43,6 @@ class embedded(StageBase):
self.settings['action_sequence'] = [
"dir_setup",
"unpack",
- "unpack_snapshot",
"config_profile_link",
"setup_confdir",
"portage_overlay",
diff --git a/catalyst/targets/livecd_stage1.py
b/catalyst/targets/livecd_stage1.py
index 727d95f2..f0b6be8b 100644
--- a/catalyst/targets/livecd_stage1.py
+++ b/catalyst/targets/livecd_stage1.py
@@ -25,7 +25,6 @@ class livecd_stage1(StageBase):
def set_action_sequence(self):
self.settings['action_sequence'] = [
"unpack",
- "unpack_snapshot",
"config_profile_link",
"setup_confdir",
"portage_overlay",
diff --git a/catalyst/targets/livecd_stage2.py
b/catalyst/targets/livecd_stage2.py
index 09de22fb..22450645 100644
--- a/catalyst/targets/livecd_stage2.py
+++ b/catalyst/targets/livecd_stage2.py
@@ -90,7 +90,6 @@ class livecd_stage2(StageBase):
def set_action_sequence(self):
self.settings['action_sequence'] = [
"unpack",
- "unpack_snapshot",
"config_profile_link",
"setup_confdir",
"portage_overlay",
diff --git a/catalyst/targets/netboot.py b/catalyst/targets/netboot.py
index 7c37bad5..5620e0d3 100644
--- a/catalyst/targets/netboot.py
+++ b/catalyst/targets/netboot.py
@@ -162,7 +162,6 @@ class netboot(StageBase):
def set_action_sequence(self):
self.settings['action_sequence'] = [
"unpack",
- "unpack_snapshot",
"config_profile_link",
"setup_confdir",
"portage_overlay",
diff --git a/catalyst/targets/snapshot.py b/catalyst/targets/snapshot.py
index 16563f14..b6c72c51 100644
--- a/catalyst/targets/snapshot.py
+++ b/catalyst/targets/snapshot.py
@@ -2,106 +2,103 @@
Snapshot target
"""
-from DeComp.compress import CompressMap
+import subprocess
+import sys
+
+from pathlib import Path
from catalyst import log
-from catalyst.support import normpath, cmd
from catalyst.base.targetbase import TargetBase
-from catalyst.base.genbase import GenBase
-from catalyst.fileops import (clear_dir, ensure_dirs)
-
+from catalyst.lock import write_lock
+from catalyst.support import command
-class snapshot(TargetBase, GenBase):
+class snapshot(TargetBase):
"""
Builder class for snapshots.
"""
required_values = frozenset([
- "target",
- "version_stamp",
+ 'target',
])
valid_values = required_values | frozenset([
- "compression_mode",
+ 'snapshot_treeish',
])
def __init__(self, myspec, addlargs):
TargetBase.__init__(self, myspec, addlargs)
- GenBase.__init__(self, myspec)
- self.settings["target_subpath"] = "repos"
- st = self.settings["storedir"]
- self.settings["snapshot_path"] = normpath(st + "/snapshots/"
- +
self.settings["snapshot_name"]
- +
self.settings["version_stamp"])
- self.settings["tmp_path"] = normpath(
- st+"/tmp/"+self.settings["target_subpath"])
+ self.git = command('git')
+ self.ebuild_repo = Path(self.settings['repos'],
+ self.settings['repo_name']).with_suffix('.git')
+ self.gitdir = str(self.ebuild_repo)
- def setup(self):
- x = normpath(self.settings["storedir"]+"/snapshots")
- ensure_dirs(x)
+ def update_ebuild_repo(self) -> str:
+ repouri = 'https://anongit.gentoo.org/git/repo/sync/gentoo.git'
+
+ if self.ebuild_repo.is_dir():
+ git_cmds = [
+ [self.git, '-C', self.gitdir, 'fetch', '--quiet', '--depth=1'],
+ [self.git, '-C', self.gitdir, 'update-ref', 'HEAD',
'FETCH_HEAD'],
+ [self.git, '-C', self.gitdir, 'gc', '--quiet'],
+ ]
+ else:
+ git_cmds = [
+ [self.git, 'clone', '--quiet', '--depth=1', '--bare',
+ '-c', 'gc.reflogExpire=0',
+ '-c', 'gc.reflogExpireUnreachable=0',
+ '-c', 'gc.rerereresolved=0',
+ '-c', 'gc.rerereunresolved=0',
+ '-c', 'gc.pruneExpire=now',
+ '--branch=stable',
+ repouri, self.gitdir],
+ ]
+
+ for cmd in git_cmds:
+ log.notice('>>> ' + ' '.join(cmd))
+ subprocess.run(cmd,
+ encoding='utf-8',
+ close_fds=False)
+
+ sp = subprocess.run([self.git, '-C', self.gitdir, 'rev-parse',
'stable'],
+ stdout=subprocess.PIPE,
+ encoding='utf-8',
+ close_fds=False)
+ return sp.stdout.rstrip()
def run(self):
- if "purgeonly" in self.settings["options"]:
- self.purge()
- return True
-
- if "purge" in self.settings["options"]:
- self.purge()
-
- success = True
- self.setup()
- log.notice('Creating %s tree snapshot %s from %s ...',
- self.settings["repo_name"], self.settings['version_stamp'],
- self.settings['portdir'])
-
- mytmp = self.settings["tmp_path"]
- ensure_dirs(mytmp)
-
- cmd(['rsync', '-a', '--no-o', '--no-g', '--delete',
- '--exclude=/packages/',
- '--exclude=/distfiles/',
- '--exclude=/local/',
- '--exclude=CVS/',
- '--exclude=.svn',
- '--exclude=.git/',
- '--filter=H_**/files/digest-*',
- self.settings['portdir'] + '/',
- mytmp + '/' + self.settings['repo_name'] + '/'],
- env=self.env)
-
- log.notice('Compressing %s snapshot tarball ...',
- self.settings["repo_name"])
- compressor = CompressMap(self.settings["compress_definitions"],
- env=self.env,
default_mode=self.settings['compression_mode'],
- comp_prog=self.settings["comp_prog"])
- infodict = compressor.create_infodict(
- source=self.settings["repo_name"],
- destination=self.settings["snapshot_path"],
- basedir=mytmp,
- filename=self.settings["snapshot_path"],
- mode=self.settings["compression_mode"],
- auto_extension=True
- )
- if not compressor.compress(infodict):
- success = False
- log.error('Snapshot compression failure')
+ if self.settings['snapshot_treeish'] == 'stable':
+ treeish = self.update_ebuild_repo()
+ else:
+ treeish = self.settings['snapshot_treeish']
+
+ self.set_snapshot(treeish)
+
+ git_cmd = [self.git, '-C', self.gitdir, 'archive', '--format=tar',
+ treeish]
+ tar2sqfs_cmd = [command('tar2sqfs'), str(self.snapshot), '-q', '-f',
+ '-j1', '-c', 'gzip']
+
+ log.notice('Creating %s tree snapshot %s from %s',
+ self.settings['repo_name'], treeish, self.gitdir)
+ log.notice('>>> ' + ' '.join([*git_cmd, '|']))
+ log.notice(' ' + ' '.join(tar2sqfs_cmd))
+
+ lockfile = self.snapshot.with_suffix('.lock')
+ with write_lock(lockfile):
+ git = subprocess.Popen(git_cmd,
+ stdout=subprocess.PIPE,
+ stderr=sys.stderr,
+ close_fds=False)
+ tar2sqfs = subprocess.Popen(tar2sqfs_cmd,
+ stdin=git.stdout,
+ stdout=sys.stdout,
+ stderr=sys.stderr,
+ close_fds=False)
+ git.stdout.close()
+ git.wait()
+ tar2sqfs.wait()
+
+ if tar2sqfs.returncode == 0:
+ log.notice('Wrote snapshot to %s', self.snapshot)
else:
- filename = '.'.join([self.settings["snapshot_path"],
-
compressor.extension(self.settings["compression_mode"])])
- log.notice('Snapshot successfully written to %s', filename)
- self.gen_contents_file(filename)
- self.gen_digest_file(filename)
- if "keepwork" not in self.settings["options"]:
- self.cleanup()
- if success:
- log.info('snapshot: complete!')
- return success
-
- def kill_chroot_pids(self):
- pass
-
- def cleanup(self):
- log.info('Cleaning up ...')
- self.purge()
-
- def purge(self):
- clear_dir(self.settings['tmp_path'])
+ log.error('Failed to create snapshot')
+ return tar2sqfs.returncode == 0
diff --git a/catalyst/targets/stage4.py b/catalyst/targets/stage4.py
index a3de2cae..17719f0e 100644
--- a/catalyst/targets/stage4.py
+++ b/catalyst/targets/stage4.py
@@ -38,7 +38,6 @@ class stage4(StageBase):
def set_action_sequence(self):
self.settings['action_sequence'] = [
"unpack",
- "unpack_snapshot",
"config_profile_link",
"setup_confdir",
"portage_overlay",
diff --git a/doc/catalyst-config.5.txt b/doc/catalyst-config.5.txt
index 44c905d7..925934ad 100644
--- a/doc/catalyst-config.5.txt
+++ b/doc/catalyst-config.5.txt
@@ -111,17 +111,18 @@ Defaults to the host's DISTDIR.
Source Gentoo tree location (primary repo). `/var/db/repos/gentoo/` should
work for most
default installations.
+*repos*::
+The directory in which git repositories exist for use by the snapshot target.
+Defaults to `${storedir}/repos`.
+
*repo_basedir*::
The target repository directory to contain the primary repo (gentoo repo) and
any overlays. The default location is `/var/db/repos`.
*repo_name*::
-The name of the main repository (ie: gentoo). This has had a directory name
-of `portage` in the past. But it has an internal name of `gentoo`, which is
-what its directory name should be. This name is used in the snapshot name
-generated and also the directory name of the repository created with the
-snapshot target. The new general rule is that the directory name and its
-internal repo_name value should be the same.
+The name of the main repository (e.g. gentoo). The git repository at
+`${repos}/${repo_name}.git` will be used to produce the portdir sqfs
+snapshot.
*target_distdir*::
This is the target distfiles directory location for the stage being created.
diff --git a/doc/catalyst-spec.5.txt b/doc/catalyst-spec.5.txt
index 58f0a9f0..f87bd69e 100644
--- a/doc/catalyst-spec.5.txt
+++ b/doc/catalyst-spec.5.txt
@@ -62,9 +62,9 @@ allowing multiple concurrent builds. Usually, `default` will
suffice.
*profile*::
This is the system profile to be used by catalyst to build this target
(example: `default/linux/x86/10.0/`). It is specified as a relative
-path from `profiles` in your portage snapshot
+path from `profiles` in your portdir snapshot
-*snapshot*::
+*snapshot_treeish*::
This specifies which snapshot to use for building this target
(example: `2006.1`).
