Ryan Harper has proposed merging ~raharper/cloud-init:feature/disk_setup_async into cloud-init:master.
Commit message: Implement async mount via fstab entry On systemd enabled systems, cloud-init can make use of systemd-makefs service via use of 'x-systemd.makefs' fs_mntop entry. This results in the creation of a mount unit and mkfs service including dependencies to ensure that the target is formatted and mounted by systemd prior to the start of cloud-config.service. This feature is enabled by specifying 'x-systemd.makefs' in a mount cloud-config entry. If enabled, cc_disk_setup will skip filesystem creation deferring to systemd-makefs, if the specified device in the mount entry matches an entry in fs_setup. Lastly, cc_mounts now supports specifying 'cidefaults' as a fs_mntop value which will replace the string 'cidefaults' with the default system mount options; either cloud-init defaults or via cloud-config when users specify 'default_mount_options'. Also included in this branch: - Add eventReportStack context managers for timing disk_setup and fs_setup separately - fixes to cloud-init analyze parsing for sub-stage events Requested reviews: cloud-init commiters (cloud-init-dev) For more details, see: https://code.launchpad.net/~raharper/cloud-init/+git/cloud-init/+merge/369362 -- Your team cloud-init commiters is requested to review the proposed merge of ~raharper/cloud-init:feature/disk_setup_async into cloud-init:master.
diff --git a/cloudinit/analyze/show.py b/cloudinit/analyze/show.py index 3e778b8..b15cd2c 100644 --- a/cloudinit/analyze/show.py +++ b/cloudinit/analyze/show.py @@ -94,6 +94,10 @@ def event_parent(event): return None +def event_is_stage(event): + return '/' not in event_name(event) + + def event_timestamp(event): return float(event.get('timestamp')) @@ -146,7 +150,9 @@ def generate_records(events, blame_sort=False, next_evt = None if event_type(event) == 'start': - if event.get('name') in stages_seen: + stage_name = event_parent(event) + if stage_name == event_name(event) and stage_name in stages_seen: + # new boot record records.append(total_time_record(total_time)) boot_records.append(records) records = [] @@ -166,19 +172,26 @@ def generate_records(events, blame_sort=False, event, next_evt))) else: - # This is a parent event - records.append("Starting stage: %s" % event.get('name')) - unprocessed.append(event) - stages_seen.append(event.get('name')) - continue + if event_is_stage(event): + records.append("Starting stage: %s" % event.get('name')) + unprocessed.append(event) + stages_seen.append(event.get('name')) + else: + # Start of a substage event + records.append(format_record(print_format, + event_record(start_time, + event, + next_evt))) + else: prev_evt = unprocessed.pop() if event_name(event) == event_name(prev_evt): - record = event_record(start_time, prev_evt, event) - records.append(format_record("Finished stage: " - "(%n) %d seconds ", - record) + "\n") - total_time += record.get('delta') + if event_is_stage(event): + record = event_record(start_time, prev_evt, event) + records.append(format_record("Finished stage: " + "(%n) %d seconds ", + record) + "\n") + total_time += record.get('delta') else: # not a match, put it back unprocessed.append(prev_evt) diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index 29e192e..00d9930 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -71,6 +71,11 @@ specified using ``filesystem``. .. note:: ``replace_fs`` is ignored unless ``partition`` is ``auto`` or ``any``. +.. note:: + Filesystem creation can be deferred to systemd-makefs if a specified + device is present in ``mounts`` cloud-config and includes the + ``x-systemd-makefs`` in the fs_mntopts field. + **Internal name:** ``cc_disk_setup`` **Module frequency:** per instance @@ -99,6 +104,7 @@ specified using ``filesystem``. from cloudinit.settings import PER_INSTANCE from cloudinit import util +from cloudinit.reporting import events import logging import os import shlex @@ -126,9 +132,23 @@ def handle(_name, cfg, cloud, log, _args): """ disk_setup = cfg.get("disk_setup") if isinstance(disk_setup, dict): - update_disk_setup_devices(disk_setup, cloud.device_name_to_device) - log.debug("Partitioning disks: %s", str(disk_setup)) - for disk, definition in disk_setup.items(): + _handle_disk_setup(disk_setup, cloud, log) + + fs_setup = cfg.get("fs_setup") + mounts = cfg.get("mounts", []) + if isinstance(fs_setup, list): + _handle_fs_setup(fs_setup, mounts, cloud, log) + + +def _handle_disk_setup(disk_setup, cloud, log): + update_disk_setup_devices(disk_setup, cloud.device_name_to_device) + log.debug("Partitioning disks: %s", str(disk_setup)) + for disk, definition in disk_setup.items(): + disk_str = disk.lstrip('/').replace('/', '-') + run_name = 'config-disk_setup/partition-' + disk_str + with events.ReportEventStack(name=run_name, + description='Partitioning ' + disk_str, + parent=cloud.reporter): if not isinstance(definition, dict): log.warning("Invalid disk definition for %s" % disk) continue @@ -141,18 +161,32 @@ def handle(_name, cfg, cloud, log, _args): except Exception as e: util.logexc(LOG, "Failed partitioning operation\n%s" % e) - fs_setup = cfg.get("fs_setup") - if isinstance(fs_setup, list): - log.debug("setting up filesystems: %s", str(fs_setup)) - update_fs_setup_devices(fs_setup, cloud.device_name_to_device) - for definition in fs_setup: + +def _handle_fs_setup(fs_setup, mounts, cloud, log): + log.debug("setting up filesystems: %s", str(fs_setup)) + update_fs_setup_devices(fs_setup, cloud.device_name_to_device) + + # list of devices which use systemd-makefs to format + skip_mkfs = [mnt[0] for mnt in mounts + if len(mnt) > 3 and 'x-systemd.makefs' in mnt[3]] + + for idx, definition in enumerate(fs_setup): + run_name = 'config-disk_setup/mkfs-entry-%s' % idx + desc = 'Create Filesystem for entry %s' % idx + with events.ReportEventStack(name=run_name, description=desc, + parent=cloud.reporter): if not isinstance(definition, dict): log.warning("Invalid file system definition: %s" % definition) continue try: - log.debug("Creating new filesystem.") device = definition.get('device') + og_dev = definition.get('_origname') + if set(skip_mkfs).intersection(set([device, og_dev])): + log.debug('Deferring mkfs of %s to systemd-make-fs', + device) + continue + log.debug("Creating new filesystem.") util.log_time(logfunc=LOG.debug, msg="Creating fs for %s" % device, func=mkfs, args=(definition,)) diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index 123ffb8..c227253 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -38,6 +38,10 @@ On a systemd booted system that default is the mostly equivalent:: Note that `nobootwait` is an upstart specific boot option that somewhat equates to the more standard `nofail`. +When specifying mount opts, using ``cidefaults`` will expand into the +the mount_default_fields values for fs_mntops. This is useful for +appending non-standard values without having to track the default values. + Swap files can be configured by setting the path to the swap file to create with ``filename``, the size of the swap file with ``size`` maximum size of the swap file if using an ``size: auto`` with ``maxsize``. By default no @@ -55,6 +59,7 @@ swap file is created. - [ /dev/ephemeral0, /mnt, auto, "defaults,noexec" ] - [ sdc, /opt/data ] - [ xvdh, /opt/data, "auto", "defaults,nofail", "0", "0" ] + - [ ephemeral0.1, ext4, cidefaults,x-systemd.makefs, none, none ] mount_default_fields: [None, None, "auto", "defaults,nofail", "0", "2"] swap: filename: <file> @@ -77,6 +82,9 @@ DEVICE_NAME_RE = re.compile(DEVICE_NAME_FILTER) WS = re.compile("[%s]+" % (whitespace)) FSTAB_PATH = "/etc/fstab" MNT_COMMENT = "comment=cloudconfig" +MNT_BEFORE_CC = "x-systemd.before=cloud-config.service" +DEF_MNT_OPTS = "defaults,nobootwait" +SYSTEMD_DEF_MNT_OPTS = "defaults,nofail,x-systemd.requires=cloud-init.service" LOG = logging.getLogger(__name__) @@ -310,16 +318,16 @@ def handle_swapcfg(swapcfg): def handle(_name, cfg, cloud, log, _args): # fs_spec, fs_file, fs_vfstype, fs_mntops, fs-freq, fs_passno - def_mnt_opts = "defaults,nobootwait" + def_mnt_opts = DEF_MNT_OPTS uses_systemd = cloud.distro.uses_systemd() if uses_systemd: - def_mnt_opts = "defaults,nofail,x-systemd.requires=cloud-init.service" + def_mnt_opts = SYSTEMD_DEF_MNT_OPTS defvals = [None, None, "auto", def_mnt_opts, "0", "2"] defvals = cfg.get("mount_default_fields", defvals) # these are our default set of mounts - defmnts = [["ephemeral0", "/mnt", "auto", defvals[3], "0", "2"], + defmnts = [["ephemeral0", "/mnt", defvals[2], defvals[3], "0", "2"], ["swap", "none", "swap", "sw", "0", "0"]] cfgmnt = [] @@ -390,6 +398,12 @@ def handle(_name, cfg, cloud, log, _args): if cfgmnt[j][0] == cfgmnt[i][0]: cfgmnt[j][1] = None + # expand fs_mntopts 'cidefaults' to defvals fs_mntopts + if 'cidefaults' in cfgmnt[i][3]: + log.debug("expanding fs_mntops keyword 'cidefaults' => %s", + defvals[3]) + cfgmnt[i][3] = cfgmnt[i][3].replace('cidefaults', defvals[3]) + # for each of the "default" mounts, add them only if no other # entry has the same device name for defmnt in defmnts: @@ -441,7 +455,15 @@ def handle(_name, cfg, cloud, log, _args): needswap = False need_mount_all = False dirs = [] + systemd_mounts = [] for line in actlist: + # if user specifies a x-systemd.makefs, extract the mount point + # which is the name of the systemd mount unit we need to enqueue + if 'x-systemd.makefs' in line[3]: + # transform mount point entry to systemd mount unit name + systemd_mounts.append(line[1].lstrip('/').replace('/', '-')) + # ensure this unit is mounted before cloud-config.target + line[3] = "%s,%s" % (line[3], MNT_BEFORE_CC) # write 'comment' in the fs_mntops, entry, claiming this line[3] = "%s,%s" % (line[3], MNT_COMMENT) if line[2] == "swap": @@ -474,19 +496,22 @@ def handle(_name, cfg, cloud, log, _args): util.write_file(FSTAB_PATH, contents) activate_cmds = [] - if needswap: - activate_cmds.append(["swapon", "-a"]) - if len(sops) == 0: log.debug("No changes to /etc/fstab made.") else: log.debug("Changes to fstab: %s", sops) need_mount_all = True + if needswap: + activate_cmds.append(["swapon", "-a"]) if need_mount_all: - activate_cmds.append(["mount", "-a"]) if uses_systemd: activate_cmds.append(["systemctl", "daemon-reload"]) + for sd_mount in systemd_mounts: + activate_cmds.append(['systemctl', 'start', '--no-block', + '%s.mount' % sd_mount]) + if len(systemd_mounts) == 0: + activate_cmds.append(["mount", "-a"]) fmt = "Activating swap and mounts with: %s" for cmd in activate_cmds: diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index d2fad9b..21dc0f0 100755 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -232,6 +232,9 @@ BUILTIN_CLOUD_CONFIG = { }, 'fs_setup': [{'filesystem': DEFAULT_FS, 'device': 'ephemeral0.1'}], + 'mounts': [ + ['ephemeral0.1', '/mnt', DEFAULT_FS, 'cidefaults,x-systemd.makefs'] + ], } DS_CFG_PATH = ['datasource', DS_NAME] diff --git a/cloudinit/tests/helpers.py b/cloudinit/tests/helpers.py index f41180f..f088bab 100644 --- a/cloudinit/tests/helpers.py +++ b/cloudinit/tests/helpers.py @@ -6,7 +6,9 @@ import functools import httpretty import logging import os +import random import shutil +import string import sys import tempfile import time @@ -122,6 +124,12 @@ class TestCase(unittest2.TestCase): parser.readfp(contents) return parser + @classmethod + def random_string(cls, length=8): + """ return a random lowercase string with default length of 8""" + return ''.join( + random.choice(string.ascii_lowercase) for _ in range(length)) + class CiTestCase(TestCase): """This is the preferred test case base class unless user diff --git a/tests/unittests/test_handler/test_handler_mounts.py b/tests/unittests/test_handler/test_handler_mounts.py index 0fb160b..4c7470d 100644 --- a/tests/unittests/test_handler/test_handler_mounts.py +++ b/tests/unittests/test_handler/test_handler_mounts.py @@ -255,7 +255,52 @@ class TestFstabHandling(test_helpers.FilesystemMockingTestCase): self.assertEqual(fstab_expected_content, fstab_new_content) cc_mounts.handle(None, cc, self.mock_cloud, self.mock_log, []) self.m_util_subp.assert_has_calls([ + mock.call(['swapon', '-a']), + mock.call(['systemctl', 'daemon-reload']), mock.call(['mount', '-a']), - mock.call(['systemctl', 'daemon-reload'])]) + ]) + + def test_ephemeral_systemd_makefs(self): + self.maxDiff = None + dev = '/dev/disk/cloud/azure_resource-part1' + def_opts = "defaults,nofail,x-systemd.requires=cloud-init.service" + make_opts = ',x-systemd.makefs' + before_opts = ',x-systemd.before=cloud-config.service' + comment_opts = ',comment=cloudconfig' + opts = def_opts + make_opts + before_opts + comment_opts + + fstab_original_content = ( + 'LABEL=cloudimg-rootfs / ext4 defaults 0 0\n' + 'LABEL=UEFI /boot/efi vfat defaults 0 0\n' + ) + fstab_expected_content = ( + fstab_original_content + + '%s /mnt ext4 %s 0 2\n' % (dev, opts) + ) + + with open(cc_mounts.FSTAB_PATH, 'w') as fd: + fd.write(fstab_original_content) + + # enable systemd + self.mock_cloud.distro.uses_systemd.return_value = True + + # return devname + name_mock = mock.Mock() + name_mock.return_value = dev + self.mock_cloud.device_name_to_device = name_mock + + cc_opts = def_opts + make_opts + cc = {'mount_default_fields': [None, None, 'ext4', cc_opts, "0", "2"]} + cc_mounts.handle(None, cc, self.mock_cloud, self.mock_log, []) + + self.m_util_subp.assert_has_calls([ + mock.call(['systemctl', 'daemon-reload']), + mock.call(['systemctl', 'start', '--no-block', 'mnt.mount']), + ]) + + with open(cc_mounts.FSTAB_PATH, 'r') as fd: + fstab_new_content = fd.read() + print('\n%s' % fstab_new_content) + self.assertEqual(fstab_expected_content, fstab_new_content) # vi: ts=4 expandtab
_______________________________________________ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp