Re: [Cloud-init-dev] [Merge] ~smoser/cloud-init:fix/1849640-adjust-yaml-usage into cloud-init:master
I'm happy with the way it is here... we got rid of all 'import yaml' from cloudinit.util at least. and down to a signle place that 'yaml.load' is called. -- https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/374679 Your team cloud-init Commiters is requested to review the proposed merge of ~smoser/cloud-init:fix/1849640-adjust-yaml-usage into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
Re: [Cloud-init-dev] [Merge] ~smoser/cloud-init:fix/1849640-adjust-yaml-usage into cloud-init:master
not a real advantage other than getting closer to getting all yaml usage into a single place. On Thu, Oct 24, 2019 at 12:56 PM Ryan Harper wrote: > > Is there an advantage to moving dumps into safeyaml? I didn't think dump had > any concerns. Could we just have replaced the yaml.load() calls with > util.load_yaml() ? > -- > https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/374679 > You are the owner of ~smoser/cloud-init:fix/1849640-adjust-yaml-usage. -- https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/374679 Your team cloud-init Commiters is requested to review the proposed merge of ~smoser/cloud-init:fix/1849640-adjust-yaml-usage into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
Re: [Cloud-init-dev] [Merge] ~smoser/cloud-init:fix/1849640-adjust-yaml-usage into cloud-init:master
just in better use of modules... putting yaml things into their own module rather than having them all in util. I'm willing to move load_yaml also if you'd like. which would mean util would have no yaml code in it. -- https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/374679 Your team cloud-init Commiters is requested to review the proposed merge of ~smoser/cloud-init:fix/1849640-adjust-yaml-usage into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
[Cloud-init-dev] [Merge] ~smoser/cloud-init:fix/1849640-adjust-yaml-usage into cloud-init:master
Scott Moser has proposed merging
~smoser/cloud-init:fix/1849640-adjust-yaml-usage into cloud-init:master.
Commit message:
Fix usages of yaml, and move yaml_dump to safeyaml.dumps.
Here we replace uses of the pyyaml module directly with functions
provided by cloudinit.safeyaml. Also, change/move
cloudinit.util.yaml_dumps
to
cloudinit.safeyaml.dumps
LP: #1849640
Requested reviews:
cloud-init Commiters (cloud-init-dev)
For more details, see:
https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/374679
see commit message
--
Your team cloud-init Commiters is requested to review the proposed merge of
~smoser/cloud-init:fix/1849640-adjust-yaml-usage into cloud-init:master.
diff --git a/cloudinit/cmd/devel/net_convert.py b/cloudinit/cmd/devel/net_convert.py
index 1ad7e0b..9b76830 100755
--- a/cloudinit/cmd/devel/net_convert.py
+++ b/cloudinit/cmd/devel/net_convert.py
@@ -5,13 +5,12 @@ import argparse
import json
import os
import sys
-import yaml
from cloudinit.sources.helpers import openstack
from cloudinit.sources import DataSourceAzure as azure
from cloudinit.sources import DataSourceOVF as ovf
-from cloudinit import distros
+from cloudinit import distros, safeyaml
from cloudinit.net import eni, netplan, network_state, sysconfig
from cloudinit import log
@@ -78,13 +77,12 @@ def handle_args(name, args):
if args.kind == "eni":
pre_ns = eni.convert_eni_data(net_data)
elif args.kind == "yaml":
-pre_ns = yaml.load(net_data)
+pre_ns = safeyaml.load(net_data)
if 'network' in pre_ns:
pre_ns = pre_ns.get('network')
if args.debug:
sys.stderr.write('\n'.join(
-["Input YAML",
- yaml.dump(pre_ns, default_flow_style=False, indent=4), ""]))
+["Input YAML", safeyaml.dumps(pre_ns), ""]))
elif args.kind == 'network_data.json':
pre_ns = openstack.convert_net_json(
json.loads(net_data), known_macs=known_macs)
@@ -100,9 +98,8 @@ def handle_args(name, args):
"input data")
if args.debug:
-sys.stderr.write('\n'.join([
-"", "Internal State",
-yaml.dump(ns, default_flow_style=False, indent=4), ""]))
+sys.stderr.write('\n'.join(
+["", "Internal State", safeyaml.dumps(ns), ""]))
distro_cls = distros.fetch(args.distro)
distro = distro_cls(args.distro, {}, None)
config = {}
diff --git a/cloudinit/cmd/tests/test_main.py b/cloudinit/cmd/tests/test_main.py
index a1e534f..57b8fdf 100644
--- a/cloudinit/cmd/tests/test_main.py
+++ b/cloudinit/cmd/tests/test_main.py
@@ -6,8 +6,9 @@ import os
from six import StringIO
from cloudinit.cmd import main
+from cloudinit import safeyaml
from cloudinit.util import (
-ensure_dir, load_file, write_file, yaml_dumps)
+ensure_dir, load_file, write_file)
from cloudinit.tests.helpers import (
FilesystemMockingTestCase, wrap_and_call)
@@ -39,7 +40,7 @@ class TestMain(FilesystemMockingTestCase):
],
'cloud_init_modules': ['write-files', 'runcmd'],
}
-cloud_cfg = yaml_dumps(self.cfg)
+cloud_cfg = safeyaml.dumps(self.cfg)
ensure_dir(os.path.join(self.new_root, 'etc', 'cloud'))
self.cloud_cfg_file = os.path.join(
self.new_root, 'etc', 'cloud', 'cloud.cfg')
@@ -113,7 +114,7 @@ class TestMain(FilesystemMockingTestCase):
"""When local-hostname metadata is present, call cc_set_hostname."""
self.cfg['datasource'] = {
'None': {'metadata': {'local-hostname': 'md-hostname'}}}
-cloud_cfg = yaml_dumps(self.cfg)
+cloud_cfg = safeyaml.dumps(self.cfg)
write_file(self.cloud_cfg_file, cloud_cfg)
cmdargs = myargs(
debug=False, files=None, force=False, local=False, reporter=None,
diff --git a/cloudinit/config/cc_debug.py b/cloudinit/config/cc_debug.py
index 0a039eb..610dbc8 100644
--- a/cloudinit/config/cc_debug.py
+++ b/cloudinit/config/cc_debug.py
@@ -33,6 +33,7 @@ from six import StringIO
from cloudinit import type_utils
from cloudinit import util
+from cloudinit import safeyaml
SKIP_KEYS = frozenset(['log_cfgs'])
@@ -49,7 +50,7 @@ def _make_header(text):
def _dumps(obj):
-text = util.yaml_dumps(obj, explicit_start=False, explicit_end=False)
+text = safeyaml.dumps(obj, explicit_start=False, explicit_end=False)
return text.rstrip()
diff --git a/cloudinit/config/cc_salt_minion.py b/cloudinit/config/cc_salt_minion.py
index d6a21d7..cd9cb0b 100644
--- a/cloudinit/config/cc_salt_minion.py
+++ b/cloudinit/config/cc_salt_minion.py
@@ -45,7 +45,7 @@ specify them with ``pkg_name``, ``service_name`` and ``config_dir``.
import os
-from cloudinit import util
+from cloudinit im
[Cloud-init-dev] [Merge] ~d-info-e/cloud-init:fix-typos into cloud-init:master
The proposal to merge ~d-info-e/cloud-init:fix-typos into cloud-init:master has been updated. Status: Needs review => Approved For more details, see: https://code.launchpad.net/~d-info-e/cloud-init/+git/cloud-init/+merge/374205 -- Your team cloud-init Commiters is requested to review the proposed merge of ~d-info-e/cloud-init:fix-typos into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
[Cloud-init-dev] [Merge] ~d-info-e/cloud-init:fix-typos into cloud-init:master
The proposal to merge ~d-info-e/cloud-init:fix-typos into cloud-init:master has been updated. Commit message changed to: Small typo fixes in code comments. For more details, see: https://code.launchpad.net/~d-info-e/cloud-init/+git/cloud-init/+merge/374205 -- Your team cloud-init Commiters is requested to review the proposed merge of ~d-info-e/cloud-init:fix-typos into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
Re: [Cloud-init-dev] [Merge] ~harald-jensas/cloud-init:bug/1847517 into cloud-init:master
thanks for the well done bug report and merge proposal!
Please add a test to the unfortunately long and difficult to follow
tests/unittests/test_net.py .
Also, as this is your first submission:
To contribute, you must sign the Canonical Contributor License Agreement
(CLA) [1].
If you have already signed it as an individual, your Launchpad user will
be listed in the contributor-agreement-canonical launchpad group [2].
Unfortunately there is no easy way to check if an organization or company
you are doing work for has signed. If you are unsure or have questions,
email josh.pow...@canonical.com or ping powersj in #cloud-init channel
via freenode.
For information on how to sign, please see the HACKING document [3].
Thanks again, and please feel free to reach out with any questions.
–
[1] http://www.canonical.com/contributors
[2] https://launchpad.net/~contributor-agreement-canonical/+members
[3] http://cloudinit.readthedocs.io/en/latest/topics/hacking.html
Diff comments:
> diff --git a/cloudinit/sources/helpers/openstack.py
> b/cloudinit/sources/helpers/openstack.py
> index 8f06911..d1c4601 100644
> --- a/cloudinit/sources/helpers/openstack.py
> +++ b/cloudinit/sources/helpers/openstack.py
> @@ -585,7 +585,8 @@ def convert_net_json(network_json=None, known_macs=None):
> subnet = dict((k, v) for k, v in network.items()
>if k in valid_keys['subnet'])
> if 'dhcp' in network['type']:
> -t = 'dhcp6' if network['type'].startswith('ipv6') else
> 'dhcp4'
> +t = (network['type'] if network['type'].startswith('ipv6')
> + else 'dhcp4')
I don't love that a unknown network type 'dhcp6wobbly' will just be passed
directly on here. I'd rather raise exception as unknown type than rely on it
just happening to work (which in this case would take the DHCPV6C path above).
> subnet.update({
> 'type': t,
> })
--
https://code.launchpad.net/~harald-jensas/cloud-init/+git/cloud-init/+merge/373932
Your team cloud-init Commiters is requested to review the proposed merge of
~harald-jensas/cloud-init:bug/1847517 into cloud-init:master.
___
Mailing list: https://launchpad.net/~cloud-init-dev
Post to : cloud-init-dev@lists.launchpad.net
Unsubscribe : https://launchpad.net/~cloud-init-dev
More help : https://help.launchpad.net/ListHelp
Re: [Cloud-init-dev] [Merge] ~chad.smith/cloud-init:bug/180134-openstack-random-seed-encoding into cloud-init:master
doing an import in an exception handling path doesn't seem right.
other than that, good work.
Diff comments:
> diff --git a/cloudinit/util.py b/cloudinit/util.py
> index aa23b3f..49433dc 100644
> --- a/cloudinit/util.py
> +++ b/cloudinit/util.py
> @@ -1599,10 +1599,36 @@ def json_serialize_default(_obj):
> return 'Warning: redacted unserializable type {0}'.format(type(_obj))
>
>
> +def json_preserialize_binary(data):
> +"""Preserialize any discovered binary values to avoid json.dumps issues.
> +
> +Used only on python 2.7 where default type handling is not honored for
> +failure to encode binary data. LP: #1801364.
> +TODO(Drop this function when py2.7 support is dropped from cloud-init)
> +"""
> +data = obj_copy.deepcopy(data)
> +for key, value in data.items():
> +if isinstance(value, (dict)):
> +data[key] = json_preserialize_binary(value)
> +if isinstance(value, bytes):
> +data[key] = 'ci-b64:{0}'.format(b64e(value))
> +return data
> +
> +
> def json_dumps(data):
> """Return data in nicely formatted json."""
> -return json.dumps(data, indent=1, sort_keys=True,
> - separators=(',', ': '), default=json_serialize_default)
> +try:
> +return json.dumps(
> +data, indent=1, sort_keys=True, separators=(',', ': '),
> +default=json_serialize_default)
> +except UnicodeDecodeError as e:
> +from cloudinit.sources import process_instance_metadata
i dont like the late import here.
> +if sys.version_info[:2] == (2, 7):
> +data = json_preserialize_binary(data)
> +data = process_instance_metadata(data)
> +return json.dumps(
can you just call json_dumps ? as you cleaded it, it should work now.
> +data, indent=1, sort_keys=True, separators=(',', ': '),
> +default=json_serialize_default)
don't you need to raise /re-raise if not?
>
>
> def yaml_dumps(obj, explicit_start=True, explicit_end=True, noalias=False):
--
https://code.launchpad.net/~chad.smith/cloud-init/+git/cloud-init/+merge/373291
Your team cloud-init Commiters is requested to review the proposed merge of
~chad.smith/cloud-init:bug/180134-openstack-random-seed-encoding into
cloud-init:master.
___
Mailing list: https://launchpad.net/~cloud-init-dev
Post to : cloud-init-dev@lists.launchpad.net
Unsubscribe : https://launchpad.net/~cloud-init-dev
More help : https://help.launchpad.net/ListHelp
[Cloud-init-dev] [Merge] ~chad.smith/cloud-init:bug/180134-openstack-random-seed-encoding into cloud-init:master
The proposal to merge ~chad.smith/cloud-init:bug/180134-openstack-random-seed-encoding into cloud-init:master has been updated. Commit message changed to: util: json.dumps on python 2.7 will handle UnicodeDecodeError on binary Since python 2.7 doesn't handle UnicodeDecodeErrors with the default handler LP: #1801364 For more details, see: https://code.launchpad.net/~chad.smith/cloud-init/+git/cloud-init/+merge/373291 -- Your team cloud-init Commiters is requested to review the proposed merge of ~chad.smith/cloud-init:bug/180134-openstack-random-seed-encoding into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
Re: [Cloud-init-dev] [Merge] ~raharper/cloud-init:fix/pylint-no-member into cloud-init:master
seems like filing an issue with astroid would be good thing to do. but other than that, i guess approve. -- https://code.launchpad.net/~raharper/cloud-init/+git/cloud-init/+merge/373177 Your team cloud-init Commiters is requested to review the proposed merge of ~raharper/cloud-init:fix/pylint-no-member into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
Re: [Cloud-init-dev] [Merge] ~raharper/cloud-init:fix/pylint-no-member into cloud-init:master
Did the bionic version of pylint fail? Really just curious what changed. -- https://code.launchpad.net/~raharper/cloud-init/+git/cloud-init/+merge/373177 Your team cloud-init Commiters is requested to review the proposed merge of ~raharper/cloud-init:fix/pylint-no-member into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
Re: [Cloud-init-dev] [Merge] ~raharper/cloud-init:feature/cloudinit-clean-from-write-log into cloud-init:master
I think that this will not really get you what you're after.
Having a list of the files that cloud-init created or appended to or truncated
isn't going to get you to your goal. The partial solution comes at the cost of
2 open and writes for every write. Additionally, we're already writing
messages with the same information to the log. You could just parse the log to
get the same information, possibly making it more machine friendly if you need
to.
I think I'd prefer a solution that could at least get us to the goal.
Diff comments:
> diff --git a/cloudinit/util.py b/cloudinit/util.py
> index aa23b3f..950921c 100644
> --- a/cloudinit/util.py
> +++ b/cloudinit/util.py
> @@ -1841,6 +1843,55 @@ def chmod(path, mode):
> os.chmod(path, real_mode)
>
>
> +def write_file_log_append(filename, omode, target=None):
> +""" Create an audit log of files that cloud-init has written.
> +
> +The log is located at: /var/lib/cloud/instance/write_file.log
> +
> +The format is JSON dict, one per line
> + {'timestamp': time.time(), 'path': filename, 'mode': omode}
> +
> +Example entries:
> + {'filename': '/etc/apt/sources.list', 'mode': 'wb', 'timestamp': ts}
> +
> +"""
> +global WRITE_FILE_LOG
> +global _ENABLE_WRITE_FILE_LOG
> +
> +if not _ENABLE_WRITE_FILE_LOG:
> +return
> +
> +log_file = target_path(target, path=WRITE_FILE_LOG)
> +if not os.path.exists(os.path.dirname(log_file)):
> +return
> +
> +record = {'timestamp': time.time(), 'filename': filename, 'mode': omode}
> +content = json.dumps(record, default=json_serialize_default)
> +try:
> +with open(log_file, "ab") as wfl:
> +wfl.write((content + '\n').encode('utf-8'))
> +wfl.flush()
> +except IOError as e:
> +LOG.debug('Failed to append to write file log (%s): %s', log_file, e)
> +
> +
> +def write_file_log_read(target=None):
today (for the second time) i read this function name and assumed that it was
'write'ing something.
like 'write_file' writes a file.
maybe: read_global_write_log() ?
> +""" Read the WRITE_FILE_LOG and yield the contents.
> +
> +:returns a list of record dicts
> +"""
> +global WRITE_FILE_LOG
> +
> +log_file = target_path(target, path=WRITE_FILE_LOG)
> +if os.path.exists(log_file):
> +with open(log_file, "rb") as wfl:
> +contents = wfl.read()
> +for line in contents.splitlines():
> +record = load_json(line)
> +if record:
> +yield record
> +
> +
> def write_file(filename, content, mode=0o644, omode="wb", copy_mode=False):
> """
> Writes a file with the given content and sets the file mode as specified.
--
https://code.launchpad.net/~raharper/cloud-init/+git/cloud-init/+merge/372946
Your team cloud-init Commiters is requested to review the proposed merge of
~raharper/cloud-init:feature/cloudinit-clean-from-write-log into
cloud-init:master.
___
Mailing list: https://launchpad.net/~cloud-init-dev
Post to : cloud-init-dev@lists.launchpad.net
Unsubscribe : https://launchpad.net/~cloud-init-dev
More help : https://help.launchpad.net/ListHelp
Re: [Cloud-init-dev] [Merge] ~raharper/cloud-init:feature/cloudinit-clean-from-write-log into cloud-init:master
For (a), I suggest maybe writing exactly one write for every write? Why do you need a log of files' you've written? Should we have a separate log for: - files opened? - files checked for existence? - commands run? - log files written? What is motivating you to have this list of files that cloud-init wrote? Realize that in regard to usefulness, it will likely be incomplete. For example, /var/log/cloud-init-output.log won't be in there, and neither would files written by commands cloud-init executes. We already have a log file which I find to be way too verbose, and a "reporting" mechanism. Do we really need more logging? I suspect that you're after something here, but what? -- https://code.launchpad.net/~raharper/cloud-init/+git/cloud-init/+merge/372946 Your team cloud-init Commiters is requested to review the proposed merge of ~raharper/cloud-init:feature/cloudinit-clean-from-write-log into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
Re: [Cloud-init-dev] [Merge] ~raharper/cloud-init:feature/cloudinit-clean-from-write-log into cloud-init:master
Some thoughts:
a.) Why? Why do two writes for every write?
b.) currently atomic write_file is not covered.
Diff comments:
> diff --git a/cloudinit/util.py b/cloudinit/util.py
> index aa23b3f..950921c 100644
> --- a/cloudinit/util.py
> +++ b/cloudinit/util.py
> @@ -1841,6 +1843,55 @@ def chmod(path, mode):
> os.chmod(path, real_mode)
>
>
> +def write_file_log_append(filename, omode, target=None):
> +""" Create an audit log of files that cloud-init has written.
> +
> +The log is located at: /var/lib/cloud/instance/write_file.log
> +
> +The format is JSON dict, one per line
> + {'timestamp': time.time(), 'path': filename, 'mode': omode}
> +
> +Example entries:
> + {'filename': '/etc/apt/sources.list', 'mode': 'wb', 'timestamp': ts}
> +
> +"""
> +global WRITE_FILE_LOG
> +global _ENABLE_WRITE_FILE_LOG
> +
> +if not _ENABLE_WRITE_FILE_LOG:
> +return
> +
> +log_file = target_path(target, path=WRITE_FILE_LOG)
> +if not os.path.exists(os.path.dirname(log_file)):
> +return
> +
> +record = {'timestamp': time.time(), 'filename': filename, 'mode': omode}
> +content = json.dumps(record, default=json_serialize_default)
> +try:
> +with open(log_file, "ab") as wfl:
> +wfl.write((content + '\n').encode('utf-8'))
why open this in binary mode so you can convert text to binary?
Just open in text mode and write the text?
> +wfl.flush()
unnecessary flush().
the close will cover that.
> +except IOError as e:
> +LOG.debug('Failed to append to write file log (%s): %s', log_file, e)
> +
> +
> +def write_file_log_read(target=None):
> +""" Read the WRITE_FILE_LOG and yield the contents.
> +
> +:returns a list of record dicts
> +"""
> +global WRITE_FILE_LOG
> +
> +log_file = target_path(target, path=WRITE_FILE_LOG)
> +if os.path.exists(log_file):
> +with open(log_file, "rb") as wfl:
> +contents = wfl.read()
seems like you're just doing more work than necessary here, and breaking the
builtin iterator on a file()
with open(log_file, "r") as fp:
for line in fp:
record = load_json(line)
if record:
yield record
> +for line in contents.splitlines():
> +record = load_json(line)
> +if record:
> +yield record
> +
> +
> def write_file(filename, content, mode=0o644, omode="wb", copy_mode=False):
> """
> Writes a file with the given content and sets the file mode as specified.
--
https://code.launchpad.net/~raharper/cloud-init/+git/cloud-init/+merge/372946
Your team cloud-init Commiters is requested to review the proposed merge of
~raharper/cloud-init:feature/cloudinit-clean-from-write-log into
cloud-init:master.
___
Mailing list: https://launchpad.net/~cloud-init-dev
Post to : cloud-init-dev@lists.launchpad.net
Unsubscribe : https://launchpad.net/~cloud-init-dev
More help : https://help.launchpad.net/ListHelp
[Cloud-init-dev] [Merge] ~smoser/cloud-init:feature/e24cloud into cloud-init:master
Scott Moser has proposed merging ~smoser/cloud-init:feature/e24cloud into
cloud-init:master.
Commit message:
Add Support for e24cloud to Ec2 datasource.
e24cloud provides an EC2 compatible datasource.
This just identifies their platform based on dmi 'system-vendor'
having 'e24cloud'. https://www.e24cloud.com/en/ .
LP: #1696476
Requested reviews:
cloud-init Commiters (cloud-init-dev)
Related bugs:
Bug #1696476 in cloud-init: "Identification of e24cloud platform as using Ec2
datasource"
https://bugs.launchpad.net/cloud-init/+bug/1696476
For more details, see:
https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/372948
see commit message
--
Your team cloud-init Commiters is requested to review the proposed merge of
~smoser/cloud-init:feature/e24cloud into cloud-init:master.
diff --git a/cloudinit/apport.py b/cloudinit/apport.py
index fde1f75..c6797f1 100644
--- a/cloudinit/apport.py
+++ b/cloudinit/apport.py
@@ -22,6 +22,7 @@ KNOWN_CLOUD_NAMES = [
'CloudSigma',
'CloudStack',
'DigitalOcean',
+'E24Cloud',
'GCE - Google Compute Engine',
'Exoscale',
'Hetzner Cloud',
diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py
index 6c72ace..1d88c9b 100644
--- a/cloudinit/sources/DataSourceEc2.py
+++ b/cloudinit/sources/DataSourceEc2.py
@@ -34,6 +34,7 @@ class CloudNames(object):
AWS = "aws"
BRIGHTBOX = "brightbox"
ZSTACK = "zstack"
+E24CLOUD = "e24cloud"
# UNKNOWN indicates no positive id. If strict_id is 'warn' or 'false',
# then an attempt at the Ec2 Metadata service will be made.
UNKNOWN = "unknown"
@@ -483,11 +484,16 @@ def identify_zstack(data):
return CloudNames.ZSTACK
+def identify_e24cloud(data):
+if data['vendor'] == 'e24cloud':
+return CloudNames.E24CLOUD
+
+
def identify_platform():
# identify the platform and return an entry in CloudNames.
data = _collect_platform_data()
checks = (identify_aws, identify_brightbox, identify_zstack,
- lambda x: CloudNames.UNKNOWN)
+ identify_e24cloud, lambda x: CloudNames.UNKNOWN)
for checker in checks:
try:
result = checker(data)
@@ -506,6 +512,7 @@ def _collect_platform_data():
uuid_source: 'hypervisor' (/sys/hypervisor/uuid) or 'dmi'
serial: dmi 'system-serial-number' (/sys/.../product_serial)
asset_tag: 'dmidecode -s chassis-asset-tag'
+ vendor: dmi 'system-manufacturer' (/sys/.../sys_vendor)
On Ec2 instances experimentation is that product_serial is upper case,
and product_uuid is lower case. This returns lower case values for both.
@@ -534,6 +541,9 @@ def _collect_platform_data():
data['asset_tag'] = asset_tag.lower()
+vendor = util.read_dmi_data('system-manufacturer')
+data['vendor'] = (vendor if vendor else '').lower()
+
return data
diff --git a/doc/rtd/topics/datasources.rst b/doc/rtd/topics/datasources.rst
index a337c08..70fbe07 100644
--- a/doc/rtd/topics/datasources.rst
+++ b/doc/rtd/topics/datasources.rst
@@ -29,8 +29,9 @@ The following is a list of documents for each supported datasource:
datasources/aliyun.rst
datasources/altcloud.rst
- datasources/ec2.rst
datasources/azure.rst
+ datasources/ec2.rst
+ datasources/e24cloud.rst
datasources/cloudsigma.rst
datasources/cloudstack.rst
datasources/configdrive.rst
diff --git a/doc/rtd/topics/datasources/e24cloud.rst b/doc/rtd/topics/datasources/e24cloud.rst
new file mode 100644
index 000..de9a412
--- /dev/null
+++ b/doc/rtd/topics/datasources/e24cloud.rst
@@ -0,0 +1,9 @@
+.. _datasource_e24cloud:
+
+E24Cloud
+
+`E24Cloud <https://www.e24cloud.com/en/>` platform provides an AWS Ec2 metadata
+service clone. It identifies itself to guests using the dmi
+system-manufacturer (/sys/class/dmi/id/sys_vendor).
+
+.. vi: textwidth=78
diff --git a/tests/unittests/test_datasource/test_ec2.py b/tests/unittests/test_datasource/test_ec2.py
index 6fabf25..e88d807 100644
--- a/tests/unittests/test_datasource/test_ec2.py
+++ b/tests/unittests/test_datasource/test_ec2.py
@@ -678,7 +678,7 @@ class TesIdentifyPlatform(test_helpers.CiTestCase):
@mock.patch('cloudinit.sources.DataSourceEc2._collect_platform_data')
def test_identify_zstack(self, m_collect):
-"""zstack should be identified if cassis-asset-tag ends in .zstack.io
+"""zstack should be identified if chassis-asset-tag ends in .zstack.io
"""
m_collect.return_value = self.collmock(asset_tag='123456.zstack.io')
self.assertEqual(ec2.CloudNames.ZSTACK, ec2.identify_platform())
@@ -690,4 +690,16 @@ class TesIdentifyPlatform(test_helpers.CiTestCase):
m_collect.return_value = self.collmock(asset_tag='123456.buzzstack.io')
self.assertEqual(ec2.CloudNames.UNKNOWN,
Re: [Cloud-init-dev] [Merge] ~raharper/cloud-init:fix/sysconfig-skip-resolvconf-no-dns-config into cloud-init:master
It can be argued that cloud-init *was* doing the right thing. It was told that there were no dns servers and no dns 'search' (by lack of that in network config). dhcp muddies that argument for sure. That said, based on the content in the description of the bug (the previously-existing /etc/resolv.conf), cloud-init should *never* be writing that file when 'netconfig' is involved. It seems rather that resolv.conf is generated based on contents of /etc/sysconfig/network/config . Would writing /etc/sysconfig/network/config with valid content for NETCONFIG_DNS_STATIC_SEARCHLIST and friends fix this? -- https://code.launchpad.net/~raharper/cloud-init/+git/cloud-init/+merge/372727 Your team cloud-init Commiters is requested to review the proposed merge of ~raharper/cloud-init:fix/sysconfig-skip-resolvconf-no-dns-config into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
Re: [Cloud-init-dev] [Merge] ~smoser/cloud-init:fix/brightbox-less-matchy into cloud-init:master
I happened to see this false positive when reviewing the add of zstack under Bug 1841181. https://code.launchpad.net/~ruansx/cloud-init/+git/cloud-init/+merge/372445 -- https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/372622 Your team cloud-init Commiters is requested to review the proposed merge of ~smoser/cloud-init:fix/brightbox-less-matchy into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
Re: [Cloud-init-dev] [Merge] ~smoser/cloud-init:fix/brightbox-less-matchy into cloud-init:master
Just for reference, bug 1661693 has some real collected output that shows: $ ( cd /sys/class/dmi/id && sudo grep -r . . ) 2>/dev/null ./product_serial:srv-fajvm.gb1.brightbox.com ./bios_vendor:Seabios ./product_version:RHEL 6.6.0 PC ./power/runtime_active_kids:0 ./power/runtime_suspended_time:0 ./power/runtime_enabled:disabled ./power/runtime_active_time:0 ./power/control:auto ./power/async:disabled ./power/runtime_usage:0 ./power/runtime_status:unsupported ./chassis_vendor:Red Hat ./modalias:dmi:bvnSeabios:bvr0.5.1:bd01/01/2007:svnRedHat:pnKVM:pvrRHEL6.6.0PC:cvnRedHat:ct1:cvr: ./product_uuid:D5BF3AE9-7D9C-2923-0B1E-B0ED967E7DF0 ./bios_version:0.5.1 ./sys_vendor:Red Hat ./chassis_type:1 ./uevent:MODALIAS=dmi:bvnSeabios:bvr0.5.1:bd01/01/2007:svnRedHat:pnKVM:pvrRHEL6.6.0PC:cvnRedHat:ct1:cvr: ./product_name:KVM ./bios_date:01/01/2007 -- https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/372622 Your team cloud-init Commiters is requested to review the proposed merge of ~smoser/cloud-init:fix/brightbox-less-matchy into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
[Cloud-init-dev] [Merge] ~smoser/cloud-init:fix/brightbox-less-matchy into cloud-init:master
Scott Moser has proposed merging ~smoser/cloud-init:fix/brightbox-less-matchy
into cloud-init:master.
Commit message:
Brightbox: restrict detection to require full domain match .brightbox.com
The detection for brightbox in both ds-identify and in
identify_brightbox would incorrectly match the domain 'bobrightbox',
which is not a brightbox platform. The fix here is to restrict
matching to '*.brightbox.com' rather than '*brightbox.com'
Requested reviews:
cloud-init Commiters (cloud-init-dev)
For more details, see:
https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/372622
see commit message
--
Your team cloud-init Commiters is requested to review the proposed merge of
~smoser/cloud-init:fix/brightbox-less-matchy into cloud-init:master.
diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py
index 5c017bf..1010745 100644
--- a/cloudinit/sources/DataSourceEc2.py
+++ b/cloudinit/sources/DataSourceEc2.py
@@ -473,7 +473,7 @@ def identify_aws(data):
def identify_brightbox(data):
-if data['serial'].endswith('brightbox.com'):
+if data['serial'].endswith('.brightbox.com'):
return CloudNames.BRIGHTBOX
diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py
index 587e699..2826497 100644
--- a/tests/unittests/test_ds_identify.py
+++ b/tests/unittests/test_ds_identify.py
@@ -195,6 +195,10 @@ class DsIdentifyBase(CiTestCase):
return self._check_via_dict(
data, RC_FOUND, dslist=[data.get('ds'), DS_NONE])
+def _test_ds_not_found(self, name):
+data = copy.deepcopy(VALID_CFG[name])
+return self._check_via_dict(data, RC_NOT_FOUND)
+
def _check_via_dict(self, data, rc, dslist=None, **kwargs):
ret = self._call_via_dict(data, **kwargs)
good = False
@@ -244,9 +248,13 @@ class TestDsIdentify(DsIdentifyBase):
self._test_ds_found('Ec2-xen')
def test_brightbox_is_ec2(self):
-"""EC2: product_serial ends with 'brightbox.com'"""
+"""EC2: product_serial ends with '.brightbox.com'"""
self._test_ds_found('Ec2-brightbox')
+def test_bobrightbox_is_ec2(self):
+"""EC2: bobrightbox.com in product_serial is not brightbox'"""
+self._test_ds_not_found('Ec2-brightbox-negative')
+
def test_gce_by_product_name(self):
"""GCE identifies itself with product_name."""
self._test_ds_found('GCE')
@@ -726,6 +734,10 @@ VALID_CFG = {
'ds': 'Ec2',
'files': {P_PRODUCT_SERIAL: 'facc6e2f.brightbox.com\n'},
},
+'Ec2-brightbox-negative': {
+'ds': 'Ec2',
+'files': {P_PRODUCT_SERIAL: 'facc6e2f.bobrightbox.com\n'},
+},
'GCE': {
'ds': 'GCE',
'files': {P_PRODUCT_NAME: 'Google Compute Engine\n'},
diff --git a/tools/ds-identify b/tools/ds-identify
index e0d4865..0f37401 100755
--- a/tools/ds-identify
+++ b/tools/ds-identify
@@ -893,7 +893,7 @@ ec2_identify_platform() {
# brightbox https://bugs.launchpad.net/cloud-init/+bug/1661693
case "$serial" in
-*brightbox.com) _RET="Brightbox"; return 0;;
+*.brightbox.com) _RET="Brightbox"; return 0;;
esac
# AWS http://docs.aws.amazon.com/AWSEC2/
___
Mailing list: https://launchpad.net/~cloud-init-dev
Post to : cloud-init-dev@lists.launchpad.net
Unsubscribe : https://launchpad.net/~cloud-init-dev
More help : https://help.launchpad.net/ListHelp
Re: [Cloud-init-dev] [Merge] ~ruansx/cloud-init:add-zstack-datasource into cloud-init:master
Opened https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/372622 to address the brightbox issue. -- https://code.launchpad.net/~ruansx/cloud-init/+git/cloud-init/+merge/372445 Your team cloud-init Commiters is requested to review the proposed merge of ~ruansx/cloud-init:add-zstack-datasource into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
Re: [Cloud-init-dev] [Merge] ~ruansx/cloud-init:add-zstack-datasource into cloud-init:master
Also, this request is still relevant. * Add doc/rtd/topics/datasources/zstack.rst and update doc/rtd/topics/datasources.rst to reference it. See Exoscale for a recently added datasource example. -- https://code.launchpad.net/~ruansx/cloud-init/+git/cloud-init/+merge/372445 Your team cloud-init Commiters is requested to review the proposed merge of ~ruansx/cloud-init:add-zstack-datasource into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
Re: [Cloud-init-dev] [Merge] ~ruansx/cloud-init:add-zstack-datasource into cloud-init:master
Some comments inline. I also updated your commit message a bit.
Diff comments:
> diff --git a/cloudinit/sources/DataSourceEc2.py
> b/cloudinit/sources/DataSourceEc2.py
> index 5c017bf..525121b 100644
> --- a/cloudinit/sources/DataSourceEc2.py
> +++ b/cloudinit/sources/DataSourceEc2.py
> @@ -477,10 +478,16 @@ def identify_brightbox(data):
> return CloudNames.BRIGHTBOX
>
>
> +def identify_zstack(data):
> +if data['asset_tag'].endswith('zstack.io'):
endswith .zstack.io (see below)
> +return CloudNames.ZStack
> +
> +
> def identify_platform():
> # identify the platform and return an entry in CloudNames.
> data = _collect_platform_data()
> -checks = (identify_aws, identify_brightbox, lambda x: CloudNames.UNKNOWN)
> +checks = (identify_aws, identify_brightbox, identify_zstack,
> + lambda x: CloudNames.UNKNOWN)
> for checker in checks:
> try:
> result = checker(data)
> diff --git a/tools/ds-identify b/tools/ds-identify
> index e0d4865..0a59416 100755
> --- a/tools/ds-identify
> +++ b/tools/ds-identify
> @@ -896,6 +896,12 @@ ec2_identify_platform() {
> *brightbox.com) _RET="Brightbox"; return 0;;
> esac
>
> +# ZStack https://bugs.launchpad.net/cloud-init/+bug/1841181
drop the bug reference here.
> +local asset_tag="${DI_DMI_CHASSIS_ASSET_TAG}"
> +case "$asset_tag" in
> +*zstack.io) _RET="ZStack"; return 0;;
if it works, i'd prefere '*.zstack.io' to '*zstack.io' as the latter
inadvertantly matches "buzzstack.io".
And yes, we should fix brightbox above also.
> +esac
> +
> # AWS http://docs.aws.amazon.com/AWSEC2/
> # latest/UserGuide/identify_ec2_instances.html
> local uuid="" hvuuid="${PATH_SYS_HYPERVISOR}/uuid"
--
https://code.launchpad.net/~ruansx/cloud-init/+git/cloud-init/+merge/372445
Your team cloud-init Commiters is requested to review the proposed merge of
~ruansx/cloud-init:add-zstack-datasource into cloud-init:master.
___
Mailing list: https://launchpad.net/~cloud-init-dev
Post to : cloud-init-dev@lists.launchpad.net
Unsubscribe : https://launchpad.net/~cloud-init-dev
More help : https://help.launchpad.net/ListHelp
[Cloud-init-dev] [Merge] ~ruansx/cloud-init:add-zstack-datasource into cloud-init:master
The proposal to merge ~ruansx/cloud-init:add-zstack-datasource into cloud-init:master has been updated. Commit message changed to: Add datastore for ZStack platfrom Zstack platform provides a AWS Ec2 metadata service, and identifies their platform to the guest by setting the 'chassis asset tag' to a string that ends with 'zstack.io'. For more details, see: https://code.launchpad.net/~ruansx/cloud-init/+git/cloud-init/+merge/372445 -- Your team cloud-init Commiters is requested to review the proposed merge of ~ruansx/cloud-init:add-zstack-datasource into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
Re: [Cloud-init-dev] [Merge] ~daniel-thewatkins/cloud-init/+git/cloud-init:lp1843276 into cloud-init:master
In the interest of making "cloud-init debug is way to loud" even worse, Could one log message be enough? "Atomic writing to %s via temporary %s", filename, tf.name If you see such a log and its not followed by a stack trace, then its good. -- https://code.launchpad.net/~daniel-thewatkins/cloud-init/+git/cloud-init/+merge/372491 Your team cloud-init Commiters is requested to review the proposed merge of ~daniel-thewatkins/cloud-init/+git/cloud-init:lp1843276 into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
Re: [Cloud-init-dev] [Merge] ~daniel-thewatkins/cloud-init/+git/cloud-init:lp1843276 into cloud-init:master
Your successful log path would not even include the final location for the file, which is ultimately the useful thing (except in the write-failed case, which is probably rare). I think I'd prefer logging with a message that included the target file (filename), not just the temporary file name. And hopefully a way to differentiate in the log message between atomic_helpers.write_file and write_file. -- https://code.launchpad.net/~daniel-thewatkins/cloud-init/+git/cloud-init/+merge/372491 Your team cloud-init Commiters is requested to review the proposed merge of ~daniel-thewatkins/cloud-init/+git/cloud-init:lp1843276 into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
Re: [Cloud-init-dev] [Merge] ~ruansx/cloud-init:add-zstack-datasource into cloud-init:master
Quick review things:
* make a better commit message (the 'Commit message` above will be be used
when this is ultimately squashed and merged)
* add 'LP: #1841181' to your commit message. See git log for examples. Thats
how MPs get associated with bugs.
* Tests needed.
* tests/unittests/test_ds_identify.py should be fairly to understand and
update for Zstack
* Probably add a tests/unittests/test_datasource/test_zstack.py
* Add doc/rtd/topics/datasources/zstack.rst and update
doc/rtd/topics/datasources.rst to reference it. See Exoscale for a recently
added datasource example.
* Did you check to see if simply adding Zstack in the same way as BrightBox
is added would work? That could be even less code.
Diff comments:
> diff --git a/cloudinit/sources/DataSourceEc2.py
> b/cloudinit/sources/DataSourceEc2.py
> index 5c017bf..c75bfc8 100644
> --- a/cloudinit/sources/DataSourceEc2.py
> +++ b/cloudinit/sources/DataSourceEc2.py
> @@ -33,6 +33,7 @@ class CloudNames(object):
> ALIYUN = "aliyun"
> AWS = "aws"
> BRIGHTBOX = "brightbox"
> +ZStack = "zstack"
seems like this should be all caps.
> # UNKNOWN indicates no positive id. If strict_id is 'warn' or 'false',
> # then an attempt at the Ec2 Metadata service will be made.
> UNKNOWN = "unknown"
> diff --git a/cloudinit/sources/DataSourceZStack.py
> b/cloudinit/sources/DataSourceZStack.py
> new file mode 100644
> index 000..e3b50c7
> --- /dev/null
> +++ b/cloudinit/sources/DataSourceZStack.py
> @@ -0,0 +1,56 @@
> +# This file is part of cloud-init. See LICENSE file for license information.
> +
> +from cloudinit import sources
> +from cloudinit.sources import DataSourceEc2 as EC2
> +from cloudinit import util
> +
> +CHASSIS_ASSET_TAG = "zstack.io"
> +
> +class DataSourceZStack(EC2.DataSourceEc2):
> +
> +dsname = 'ZStack'
> +
> +def get_hostname(self, fqdn=False, resolve_ip=False,
> metadata_only=False):
> +return self.metadata.get('hostname', 'localhost.localdomain')
> +
> +def get_public_ssh_keys(self):
> +return parse_public_keys(self.metadata.get('public-keys', {}))
> +
> +def _get_cloud_name(self):
> +if _is_zstack():
> +return EC2.CloudNames.ZStack
> +else:
> +return EC2.CloudNames.NO_EC2_METADATA
> +
> +def _is_zstack():
> +asset_tag = util.read_dmi_data('chassis-asset-tag')
> +return asset_tag == CHASSIS_ASSET_TAG
> +
> +
> +def parse_public_keys(public_keys):
Is this necessary? I suspect it is copied from Aliyun, which has small
differences in their public key meta-data compared to Ec2. So 2 possibilities:
a.) Zstack provides data just like Aliyun.
b.) Zstack provides data just like amazon.
In either option, lets avoid the copy paste.
> +keys = []
> +for _key_id, key_body in public_keys.items():
> +if isinstance(key_body, str):
> +keys.append(key_body.strip())
> +elif isinstance(key_body, list):
> +keys.extend(key_body)
> +elif isinstance(key_body, dict):
> +key = key_body.get('openssh-key', [])
> +if isinstance(key, str):
> +keys.append(key.strip())
> +elif isinstance(key, list):
> +keys.extend(key)
> +return keys
> +
> +
> +# Used to match classes to dependencies
> +datasources = [
> +(DataSourceZStack, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
> +]
> +
> +
> +# Return a list of data sources that match this set of dependencies
> +def get_datasource_list(depends):
> +return sources.list_from_depends(depends, datasources)
Does Zstack provide any network configuration information? EC2 provides some
and cloud-init is moving to have "local" datasources that hit the metadata
service to get information about how the network should be configured. So if
this info is available, it'd be great for the datasource to start its life as a
Local datasource and use that. Even if not... then we can still provide
default networking from the datasource and run at local time frame...
> +
> +# vi: ts=4 expandtab
--
https://code.launchpad.net/~ruansx/cloud-init/+git/cloud-init/+merge/372445
Your team cloud-init Commiters is requested to review the proposed merge of
~ruansx/cloud-init:add-zstack-datasource into cloud-init:master.
___
Mailing list: https://launchpad.net/~cloud-init-dev
Post to : cloud-init-dev@lists.launchpad.net
Unsubscribe : https://launchpad.net/~cloud-init-dev
More help : https://help.launchpad.net/ListHelp
Re: [Cloud-init-dev] [Merge] ~powersj/cloud-init:docs/config-tox into cloud-init:master
Diff comments: > diff --git a/doc/rtd/topics/docs.rst b/doc/rtd/topics/docs.rst > new file mode 100644 > index 000..890f2f5 > --- /dev/null > +++ b/doc/rtd/topics/docs.rst > @@ -0,0 +1,83 @@ > +.. _docs: > + > +Docs > + > + > +These docs are hosted on Read the Docs. The following will explain how to > +contribute to and build these docs locally. > + > +The documentation is primarily written in reStructuredText. > + > + > +Building > + > + > +There is a makefile target to build the documentation for you: > + > +.. code-block:: shell-session > + > +$ tox -e doc > + > +This will do two things: > + > +- Build the documentation using sphinx > +- Run doc8 against the documentation source code > + > +Once build the HTML files will be viewable in ``doc/rtd_html``. Use your > +web browser to open ``index.html`` to view and navigate the site. > + > +Style Guide > +=== > + > +Headings > + > +The headings used across the documentation use the following hierarchy: > + > +- ``*``: used once atop of a new page > +- ``=``: each sections on the page > +- ``-``: subsections > +- ``^``: sub-subsections > +- ``"``: paragraphs > + > +The top level header ``##`` is reserved for the first page. > + > +If under and overline are used, their length must be identical. The length of > +the underline must be at least as long as the title itself > + > +Line Length > +--- > +Please keep the line lengths to a maximum of **79** characters. This ensures > +that the pages and tables do not get too wide that side scrolling is > required. > + > +Header > +-- > +Adding a link at the top of the page allows for the page to be referenced by > +other pages. For example for the FAQ page this would be: > + > +.. code-block:: rst > + > +.. _faq: > + > +Footer > +-- > +The footer should include the textwidth > + > +.. code-block:: rst > + > +.. vi: textwidth=79 > + > +Vertical Whitespace > +--- > +One newline between each section helps ensure readability of the > documentation > +source code. > + > +Common Words > + > +There are some common words that should follow specific usage: > + > +- ``cloud-init``: always lower case with a hyphen unless starting a sentence Can you just clarify what the proper capitalization: Cloud-init or Cloud-Init > +- ``metadata``: one word > +- ``user data``: two words, not to be combined > +- ``vendor data``: like user data, it is two words > + > +.. vi: textwidth=80 -- https://code.launchpad.net/~powersj/cloud-init/+git/cloud-init/+merge/372102 Your team cloud-init commiters is requested to review the proposed merge of ~powersj/cloud-init:docs/config-tox into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
Re: [Cloud-init-dev] [Merge] ~powersj/cloud-init:docs/config-tox into cloud-init:master
Diff comments: > diff --git a/doc/rtd/conf.py b/doc/rtd/conf.py > index 4174477..9b27484 100644 > --- a/doc/rtd/conf.py > +++ b/doc/rtd/conf.py > @@ -17,7 +17,8 @@ from cloudinit.config.schema import get_schema_doc > # ] > > # General information about the project. > -project = 'Cloud-Init' > +project = 'cloud-init' More I was confused by the 'I' than the 'C'. I dont know what you intended in Common Words: A.) Cloud-init is awesome or B.) Cloud-Init is awesome. I had assumed it was 'A', but then you did 'B' here. > +copyright = '2019, Canonical Ltd.' > > # -- General configuration > > > diff --git a/doc/rtd/topics/docs.rst b/doc/rtd/topics/docs.rst > new file mode 100644 > index 000..af2a3fb > --- /dev/null > +++ b/doc/rtd/topics/docs.rst > @@ -0,0 +1,83 @@ > +.. _docs: > + > +Docs > + > + > +These docs are hosted on Read the Docs. The following will explain how to > +contribute to and build these docs locally. > + > +The documentation is primarily written in reStructuredText. > + > + > +Building > + > + > +There is a makefile target to build the documentation for you: > + > +.. code-block:: shell > + > +$ make doc > + > +This will do two things: > + > +- Build the documentation using sphinx > +- Run doc8 against the documentation source code > + > +Once build the HTML files will be viewable in ``doc/rtd_html``. Use your > +web browser to open ``index.html`` to view and navigate the site. > + > +Style Guide > +=== > + > +Headings > + > +The headings used across the documentation use the following hierarchy: > + > +- ``*``: used once atop of a new page > +- ``=``: each sections on the page > +- ``-``: subsections > +- ``^``: sub-subsections > +- ``"``: paragraphs > + > +The top level header ``##`` is reserved for the first page. > + > +If under and overline are used, their length must be identical. The length of > +the underline must be at least as long as the title itself > + > +Line Length > +--- > +Please keep the line lengths to a maximum of **79** characters. This ensures > +that the pages and tables do not get too wide that side scrolling is > required. > + > +Header > +-- > +Adding a link at the top of the page allows for the page to be referenced by > +other pages. For example for the FAQ page this would be: > + > +.. code-block:: rst > + > +.. _faq: > + > +Footer > +-- > +The footer should include the textwidth > + > +.. code-block:: rst > + > +.. vi: textwidth=80 > + > +Vertical Whitespace > +--- > +One newline between each section helps ensure readability of the > documentation > +source code. > + > +Common Words > + > +There are some common words that should follow specific usage: > + > +- ``cloud-init``: always lower case with a hyphen unless starting a sentence > +- ``metadata``: one word > +- ``user data``: two words, not to be combined > +- ``vendor data``: like user data, it is two words It just makes 'metadata' seem like an odd ball to me (should that be 'odd-ball' ? or 'oddball'. not sure. ). But oh well. > + > +.. vi: textwidth=80 -- https://code.launchpad.net/~powersj/cloud-init/+git/cloud-init/+merge/372102 Your team cloud-init commiters is requested to review the proposed merge of ~powersj/cloud-init:docs/config-tox into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
Re: [Cloud-init-dev] [Merge] ~powersj/cloud-init:docs/config-tox into cloud-init:master
I don't personally feel the need to have 'make doc' at all, but its not a problem to have it. In the commit message subject line you have 'create markefile' Diff comments: > diff --git a/doc/rtd/conf.py b/doc/rtd/conf.py > index 4174477..9b27484 100644 > --- a/doc/rtd/conf.py > +++ b/doc/rtd/conf.py > @@ -17,7 +17,8 @@ from cloudinit.config.schema import get_schema_doc > # ] > > # General information about the project. > -project = 'Cloud-Init' > +project = 'cloud-init' Odd that this does not fit the spelling guidelines (Common Words) below. > +copyright = '2019, Canonical Ltd.' > > # -- General configuration > > > diff --git a/doc/rtd/topics/docs.rst b/doc/rtd/topics/docs.rst > new file mode 100644 > index 000..af2a3fb > --- /dev/null > +++ b/doc/rtd/topics/docs.rst > @@ -0,0 +1,83 @@ > +.. _docs: > + > +Docs > + > + > +These docs are hosted on Read the Docs. The following will explain how to > +contribute to and build these docs locally. > + > +The documentation is primarily written in reStructuredText. > + > + > +Building > + > + > +There is a makefile target to build the documentation for you: > + > +.. code-block:: shell shell-session is probably what you want here. > + > +$ make doc We document tox as the way to run integration tests (doc/rtd/topics/tests.rst) and also unit tests (HACKING.rst) so it feels more consistent to document 'tox -e doc' here. Maybe there is some reason that you wanted 'make' ? > + > +This will do two things: > + > +- Build the documentation using sphinx > +- Run doc8 against the documentation source code > + > +Once build the HTML files will be viewable in ``doc/rtd_html``. Use your > +web browser to open ``index.html`` to view and navigate the site. > + > +Style Guide > +=== > + > +Headings > + > +The headings used across the documentation use the following hierarchy: > + > +- ``*``: used once atop of a new page > +- ``=``: each sections on the page > +- ``-``: subsections > +- ``^``: sub-subsections > +- ``"``: paragraphs Can you remove this from the comments doc/rtd/index.rst ? Just to avoid the duplication and inevitable divergence. > + > +The top level header ``##`` is reserved for the first page. > + > +If under and overline are used, their length must be identical. The length of > +the underline must be at least as long as the title itself > + > +Line Length > +--- > +Please keep the line lengths to a maximum of **79** characters. This ensures > +that the pages and tables do not get too wide that side scrolling is > required. > + > +Header > +-- > +Adding a link at the top of the page allows for the page to be referenced by > +other pages. For example for the FAQ page this would be: > + > +.. code-block:: rst > + > +.. _faq: > + > +Footer > +-- > +The footer should include the textwidth > + > +.. code-block:: rst > + > +.. vi: textwidth=80 > + > +Vertical Whitespace > +--- > +One newline between each section helps ensure readability of the > documentation > +source code. > + > +Common Words > + > +There are some common words that should follow specific usage: > + > +- ``cloud-init``: always lower case with a hyphen unless starting a sentence > +- ``metadata``: one word > +- ``user data``: two words, not to be combined > +- ``vendor data``: like user data, it is two words I'm guessing that Josh didn't decide on this, but personally 'user-data' 'vendor-data' and 'meta-data' seem more consistent. > + > +.. vi: textwidth=80 -- https://code.launchpad.net/~powersj/cloud-init/+git/cloud-init/+merge/372102 Your team cloud-init commiters is requested to review the proposed merge of ~powersj/cloud-init:docs/config-tox into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
Re: [Cloud-init-dev] [Merge] ~raharper/cloud-init:fix/debian-config-yaml-spaces into cloud-init:ubuntu/devel
Diff comments:
> diff --git a/debian/cloud-init.config b/debian/cloud-init.config
> index 6e9c6f7..adaabb8 100644
> --- a/debian/cloud-init.config
> +++ b/debian/cloud-init.config
> @@ -32,13 +32,13 @@ hasEc2Md() {
> get_yaml_list() {
> # get_yaml_list(file, key, def): return a comma delimited list with the
> value
> # for the yaml array defined in 'key' from 'file'. if not found ,
> return 'def'
> - # only really supports 'key: [en1, en2 ]' format.
> + # only really supports 'key: [ en1, en2 ]' or 'key: [en1, en2]' formats.
> local file="$1" key="$2" default="$3"
> [ -f "$file" ] || return 1
> - # any thing that didn't match the key is deleted so the final 'p' only
> - # prints things that matched.
> - RET=$(sed -n -e "/^$key:/"'!'d -e "s/$key:[ \[]*//"\
> - -e "s, \]$,," -e p "$file")
> + # strip all whitespace, delete lines not matching key:,
> + # strip key: and [] and replace ',' with ', '
> + RET=$(sed -e "s/\s//g" -e "/^$key:/"'!'d\
> + -e "s/$key:\[//;s/]//;s/,/, /g" $file)
the combination of multiple '-e' and ';' inside a single '-e' is odd.
> [ -n "$RET" ] || RET="$default"
> }
>
--
https://code.launchpad.net/~raharper/cloud-init/+git/cloud-init/+merge/371919
Your team cloud-init commiters is requested to review the proposed merge of
~raharper/cloud-init:fix/debian-config-yaml-spaces into cloud-init:ubuntu/devel.
___
Mailing list: https://launchpad.net/~cloud-init-dev
Post to : cloud-init-dev@lists.launchpad.net
Unsubscribe : https://launchpad.net/~cloud-init-dev
More help : https://help.launchpad.net/ListHelp
Re: [Cloud-init-dev] [Merge] ~raharper/cloud-init:fix/debian-config-yaml-spaces into cloud-init:ubuntu/devel
Diff comments:
> diff --git a/debian/cloud-init.config b/debian/cloud-init.config
> index 6e9c6f7..4c35e50 100644
> --- a/debian/cloud-init.config
> +++ b/debian/cloud-init.config
> @@ -32,13 +32,13 @@ hasEc2Md() {
> get_yaml_list() {
> # get_yaml_list(file, key, def): return a comma delimited list with the
> value
> # for the yaml array defined in 'key' from 'file'. if not found ,
> return 'def'
> - # only really supports 'key: [en1, en2 ]' format.
> + # only really supports 'key: [ en1, en2 ]' or 'key: [en1, en2]' formats.
> local file="$1" key="$2" default="$3"
> [ -f "$file" ] || return 1
> # any thing that didn't match the key is deleted so the final 'p' only
> # prints things that matched.
or i guess:
if RET=$(try_python_yaml "$file"); then
[ -n "$RET" ] || RET="$default"
return 0
fi
> - RET=$(sed -n -e "/^$key:/"'!'d -e "s/$key:[ \[]*//"\
> - -e "s, \]$,," -e p "$file")
> + RET=$(sed -n -e "/^$key:/"'!'d -e "s/$key:[[[:space:]]+\[]*//"\
> +-e "s,[[:space:]]+\]$,," -e p "$file")
> [ -n "$RET" ] || RET="$default"
> }
>
--
https://code.launchpad.net/~raharper/cloud-init/+git/cloud-init/+merge/371919
Your team cloud-init commiters is requested to review the proposed merge of
~raharper/cloud-init:fix/debian-config-yaml-spaces into cloud-init:ubuntu/devel.
___
Mailing list: https://launchpad.net/~cloud-init-dev
Post to : cloud-init-dev@lists.launchpad.net
Unsubscribe : https://launchpad.net/~cloud-init-dev
More help : https://help.launchpad.net/ListHelp
Re: [Cloud-init-dev] [Merge] ~raharper/cloud-init:fix/debian-config-yaml-spaces into cloud-init:ubuntu/devel
fwiw, there is 'parse_yaml_array', in tools/ds-identify, which does better (at
least did not have this bug). Unfortunately that uses functions 'trim' and
'unquote', so you can't just grab the single function.
Second option would be to first *try* to use a proper yaml parser (python).
There are some rules about not using dependencies in a config script I think
(only essential maybe?). Thats why we don't use cloudinit infra to load it.
So you can't *depend* on python and yaml but you could try.
Here is an example that could be added to your suggested fix:
try_python_yaml() {
# try to load something like 'datasource_list: [xxx]' and write space
delimeted
# values to stdout
local py out="" fname="$1"
command -v python3 >/dev/null || return
out=$(python3 -c '
import yaml, sys;
print(",".join(yaml.load(sys.stdin.read()).get("datasource_list")))' \
< "$fname"
) || return
echo "$out"
}
Diff comments:
> diff --git a/debian/cloud-init.config b/debian/cloud-init.config
> index 6e9c6f7..4c35e50 100644
> --- a/debian/cloud-init.config
> +++ b/debian/cloud-init.config
> @@ -32,13 +32,13 @@ hasEc2Md() {
> get_yaml_list() {
> # get_yaml_list(file, key, def): return a comma delimited list with the
> value
> # for the yaml array defined in 'key' from 'file'. if not found ,
> return 'def'
> - # only really supports 'key: [en1, en2 ]' format.
> + # only really supports 'key: [ en1, en2 ]' or 'key: [en1, en2]' formats.
> local file="$1" key="$2" default="$3"
> [ -f "$file" ] || return 1
> # any thing that didn't match the key is deleted so the final 'p' only
> # prints things that matched.
then just add right here:
RET=$(try_python_yaml "$file") && return 0
> - RET=$(sed -n -e "/^$key:/"'!'d -e "s/$key:[ \[]*//"\
> - -e "s, \]$,," -e p "$file")
> + RET=$(sed -n -e "/^$key:/"'!'d -e "s/$key:[[[:space:]]+\[]*//"\
> +-e "s,[[:space:]]+\]$,," -e p "$file")
> [ -n "$RET" ] || RET="$default"
> }
>
--
https://code.launchpad.net/~raharper/cloud-init/+git/cloud-init/+merge/371919
Your team cloud-init commiters is requested to review the proposed merge of
~raharper/cloud-init:fix/debian-config-yaml-spaces into cloud-init:ubuntu/devel.
___
Mailing list: https://launchpad.net/~cloud-init-dev
Post to : cloud-init-dev@lists.launchpad.net
Unsubscribe : https://launchpad.net/~cloud-init-dev
More help : https://help.launchpad.net/ListHelp
Re: [Cloud-init-dev] [Merge] ~smoser/cloud-init:fix/1836949-mtu-lost-in-translation into cloud-init:master
Diff comments:
> diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py
> index de4e7f4..e2bbb84 100644
> --- a/tests/unittests/test_net.py
> +++ b/tests/unittests/test_net.py
> @@ -2856,6 +2856,97 @@ USERCTL=no
> self._compare_files_to_expected(entry['expected_sysconfig'], found)
> self._assert_headers(found)
>
> +def test_from_v2_vlan_mtu(self):
> +"""verify mtu gets rendered on bond when source is netplan."""
> +v2data = {
> +'version': 2,
> +'ethernets': {'eno1': {}},
> +'vlans': {
> +'eno1.1000': {
> +'addresses': ["192.6.1.9/24"],
> +'id': 1000, 'link': 'eno1', 'mtu': 1495}}}
> +expected = {
> +'ifcfg-eno1': textwrap.dedent("""\
> +BOOTPROTO=none
> +DEVICE=eno1
> +NM_CONTROLLED=no
> +ONBOOT=yes
> +STARTMODE=auto
> +TYPE=Ethernet
> +USERCTL=no
> +"""),
> +'ifcfg-eno1.1000': textwrap.dedent("""\
> +BOOTPROTO=none
> +DEVICE=eno1.1000
> +IPADDR=192.6.1.9
> +MTU=1495
> +NETMASK=255.255.255.0
> +NM_CONTROLLED=no
> +ONBOOT=yes
> +PHYSDEV=eno1
> +STARTMODE=auto
> +TYPE=Ethernet
> +USERCTL=no
> +VLAN=yes
> +""")
> +}
> +self._compare_files_to_expected(
> +expected, self._render_and_read(network_config=v2data))
> +
> +def test_from_v2_bond_mtu(self):
> +"""verify mtu gets rendered on bond when source is netplan."""
> +v2data = {
> +'version': 2,
> +'bonds': {
> +'bond0': {'addresses': ['10.101.8.65/26'],
> + 'interfaces': ['enp0s0', 'enp0s1'],
> + 'mtu': 1334,
> + 'parameters': {}}}
> +}
> +expected = {
> +'ifcfg-bond0': textwrap.dedent("""\
> +BONDING_MASTER=yes
> +BONDING_SLAVE0=enp0s0
> +BONDING_SLAVE1=enp0s1
> +BOOTPROTO=none
> +DEVICE=bond0
> +IPADDR=10.101.8.65
> +MTU=1334
The answer is .. I don't know. My particular case was interested in vlan < 1500
on a bond.
I kind of think that cloud-init can reasonably say "garbage in garbage out" on
this. It sucks, and I'll look to see what works on centos, but reality is that
if we warn or fail on something because we thought it was not supported by
sysconfig on centos 7 (or centos8 or fedora) then that may well not be true on
suse or another version. So the easiest answer is for cloud-init to do what
the user told it to do.
> +NETMASK=255.255.255.192
> +NM_CONTROLLED=no
> +ONBOOT=yes
> +STARTMODE=auto
> +TYPE=Bond
> +USERCTL=no
> +"""),
> +'ifcfg-enp0s0': textwrap.dedent("""\
> +BONDING_MASTER=yes
> +BOOTPROTO=none
> +DEVICE=enp0s0
> +MASTER=bond0
> +NM_CONTROLLED=no
> +ONBOOT=yes
> +SLAVE=yes
> +STARTMODE=auto
> +TYPE=Bond
> +USERCTL=no
> +"""),
> +'ifcfg-enp0s1': textwrap.dedent("""\
> +BONDING_MASTER=yes
> +BOOTPROTO=none
> +DEVICE=enp0s1
> +MASTER=bond0
> +NM_CONTROLLED=no
> +ONBOOT=yes
> +SLAVE=yes
> +STARTMODE=auto
> +TYPE=Bond
> +USERCTL=no
> +""")
> +}
> +self._compare_files_to_expected(
> +expected, self._render_and_read(network_config=v2data))
> +
>
> class TestOpenSuseSysConfigRendering(CiTestCase):
>
--
https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/370280
Your team cloud-init commiters is requested to review the proposed merge of
~smoser/cloud-init:fix/1836949-mtu-lost-in-translation into cloud-init:master.
___
Mailing list: https://launchpad.net/~cloud-init-dev
Post to : cloud-init-dev@lists.launchpad.net
Unsubscribe : https://launchpad.net/~cloud-init-dev
More help : https://help.launchpad.net/ListHelp
[Cloud-init-dev] [Merge] ~smoser/cloud-init:fix/1836949-mtu-lost-in-translation into cloud-init:master
Scott Moser has proposed merging
~smoser/cloud-init:fix/1836949-mtu-lost-in-translation into cloud-init:master.
Commit message:
Fix bug rendering MTU on bond or vlan when input was netplan.
If input to network_state.parse_net_config_data was netplan (v2 yaml)
then the network state would lose the mtu information on bond or vlan.
LP: #1836949
Requested reviews:
Server Team CI bot (server-team-bot): continuous-integration
cloud-init commiters (cloud-init-dev)
Related bugs:
Bug #1836949 in cloud-init: "mtu gets lost in translation from netplan (v2)
yaml"
https://bugs.launchpad.net/cloud-init/+bug/1836949
For more details, see:
https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/370280
see commit message
--
Your team cloud-init commiters is requested to review the proposed merge of
~smoser/cloud-init:fix/1836949-mtu-lost-in-translation into cloud-init:master.
diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py
index 3702130..0ca576b 100644
--- a/cloudinit/net/network_state.py
+++ b/cloudinit/net/network_state.py
@@ -673,6 +673,8 @@ class NetworkStateInterpreter(object):
'vlan_id': cfg.get('id'),
'vlan_link': cfg.get('link'),
}
+if 'mtu' in cfg:
+vlan_cmd['mtu'] = cfg['mtu']
subnets = self._v2_to_v1_ipcfg(cfg)
if len(subnets) > 0:
vlan_cmd.update({'subnets': subnets})
@@ -722,6 +724,8 @@ class NetworkStateInterpreter(object):
'params': dict((v2key_to_v1[k], v) for k, v in
item_params.get('parameters', {}).items())
}
+if 'mtu' in item_cfg:
+v1_cmd['mtu'] = item_cfg['mtu']
subnets = self._v2_to_v1_ipcfg(item_cfg)
if len(subnets) > 0:
v1_cmd.update({'subnets': subnets})
diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py
index de4e7f4..e2bbb84 100644
--- a/tests/unittests/test_net.py
+++ b/tests/unittests/test_net.py
@@ -2856,6 +2856,97 @@ USERCTL=no
self._compare_files_to_expected(entry['expected_sysconfig'], found)
self._assert_headers(found)
+def test_from_v2_vlan_mtu(self):
+"""verify mtu gets rendered on bond when source is netplan."""
+v2data = {
+'version': 2,
+'ethernets': {'eno1': {}},
+'vlans': {
+'eno1.1000': {
+'addresses': ["192.6.1.9/24"],
+'id': 1000, 'link': 'eno1', 'mtu': 1495}}}
+expected = {
+'ifcfg-eno1': textwrap.dedent("""\
+BOOTPROTO=none
+DEVICE=eno1
+NM_CONTROLLED=no
+ONBOOT=yes
+STARTMODE=auto
+TYPE=Ethernet
+USERCTL=no
+"""),
+'ifcfg-eno1.1000': textwrap.dedent("""\
+BOOTPROTO=none
+DEVICE=eno1.1000
+IPADDR=192.6.1.9
+MTU=1495
+NETMASK=255.255.255.0
+NM_CONTROLLED=no
+ONBOOT=yes
+PHYSDEV=eno1
+STARTMODE=auto
+TYPE=Ethernet
+USERCTL=no
+VLAN=yes
+""")
+}
+self._compare_files_to_expected(
+expected, self._render_and_read(network_config=v2data))
+
+def test_from_v2_bond_mtu(self):
+"""verify mtu gets rendered on bond when source is netplan."""
+v2data = {
+'version': 2,
+'bonds': {
+'bond0': {'addresses': ['10.101.8.65/26'],
+ 'interfaces': ['enp0s0', 'enp0s1'],
+ 'mtu': 1334,
+ 'parameters': {}}}
+}
+expected = {
+'ifcfg-bond0': textwrap.dedent("""\
+BONDING_MASTER=yes
+BONDING_SLAVE0=enp0s0
+BONDING_SLAVE1=enp0s1
+BOOTPROTO=none
+DEVICE=bond0
+IPADDR=10.101.8.65
+MTU=1334
+NETMASK=255.255.255.192
+NM_CONTROLLED=no
+ONBOOT=yes
+STARTMODE=auto
+TYPE=Bond
+USERCTL=no
+"""),
+'ifcfg-enp0s0': textwrap.dedent("""\
+BONDING_MASTER=yes
+BOOTPROTO=none
+DEVICE=enp0s0
+MASTER=bond0
+NM_CONTROLLED=no
+ONBOOT=yes
+SLAVE=yes
+STARTMODE=auto
+TYPE=Bond
+USERCTL=no
+"""),
+
Re: [Cloud-init-dev] [Merge] ~smoser/cloud-init:fix/1788915-vlan-sysconfig-rendering into cloud-init:master
The error shows:
ERROR: pylint: could not install deps [pylint==2.3.1,
-r/var/lib/jenkins/slaves/torkoal/workspace/cloud-init-ci@2/test-requirements.txt];
v =
InvocationError('/var/lib/jenkins/slaves/torkoal/workspace/cloud-init-ci@2/.tox/pylint/bin/pip
install pylint==2.3.1
-r/var/lib/jenkins/slaves/torkoal/workspace/cloud-init-ci@2/test-requirements.txt
(see
/var/lib/jenkins/slaves/torkoal/workspace/cloud-init-ci@2/.tox/pylint/log/pylint-1.log)',
1)
I don't think that is related to my changes.
Help?
--
https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/366602
Your team cloud-init commiters is requested to review the proposed merge of
~smoser/cloud-init:fix/1788915-vlan-sysconfig-rendering into cloud-init:master.
___
Mailing list: https://launchpad.net/~cloud-init-dev
Post to : cloud-init-dev@lists.launchpad.net
Unsubscribe : https://launchpad.net/~cloud-init-dev
More help : https://help.launchpad.net/ListHelp
[Cloud-init-dev] [Merge] ~smoser/cloud-init:fix/1788915-vlan-sysconfig-rendering into cloud-init:master
The proposal to merge ~smoser/cloud-init:fix/1788915-vlan-sysconfig-rendering into cloud-init:master has been updated. Description changed to: For reference: [1] documentes that no TYPE is present in the vlan examples. [2] shows that random names are officially not supported. [1] https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/7/html/networking_guide/sec-configure_802_1q_vlan_tagging_using_the_command_line [2] https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/7/html/networking_guide/sec-naming_scheme_for_vlan_interfaces For more details, see: https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/366602 -- Your team cloud-init commiters is requested to review the proposed merge of ~smoser/cloud-init:fix/1788915-vlan-sysconfig-rendering into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
[Cloud-init-dev] [Merge] ~smoser/cloud-init:fix/no-tox-yaml-warnings into cloud-init:master
Scott Moser has proposed merging ~smoser/cloud-init:fix/no-tox-yaml-warnings into cloud-init:master. Commit message: git tests: no longer show warning about safe yaml. Currently on 18.04, running tox -e py27 will spew errors like: .tests/unittests/test_net.py:2649: YAMLLoadWarning: calling yaml.load() without Loader=... is deprecated, as the default Loader is unsafe. Please read https://msg.pyyaml.org/load for full details. The change here just uses cloud-init's yaml, which does safeloading by default. Requested reviews: cloud-init commiters (cloud-init-dev) For more details, see: https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/366603 see commit message -- Your team cloud-init commiters is requested to review the proposed merge of ~smoser/cloud-init:fix/no-tox-yaml-warnings into cloud-init:master. diff --git a/cloudinit/net/tests/test_init.py b/cloudinit/net/tests/test_init.py index f55c31e..6d2affe 100644 --- a/cloudinit/net/tests/test_init.py +++ b/cloudinit/net/tests/test_init.py @@ -7,11 +7,11 @@ import mock import os import requests import textwrap -import yaml import cloudinit.net as net from cloudinit.util import ensure_file, write_file, ProcessExecutionError from cloudinit.tests.helpers import CiTestCase, HttprettyTestCase +from cloudinit import safeyaml as yaml class TestSysDevPath(CiTestCase): diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 9db0156..e85e964 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -9,6 +9,7 @@ from cloudinit.net import ( from cloudinit.sources.helpers import openstack from cloudinit import temp_utils from cloudinit import util +from cloudinit import safeyaml as yaml from cloudinit.tests.helpers import ( CiTestCase, FilesystemMockingTestCase, dir2dict, mock, populate_dir) @@ -21,7 +22,7 @@ import json import os import re import textwrap -import yaml +from yaml.serializer import Serializer DHCP_CONTENT_1 = """ @@ -3575,7 +3576,7 @@ class TestNetplanRoundTrip(CiTestCase): # now look for any alias, avoid rendering them entirely # generate the first anchor string using the template # as of this writing, looks like "" -anchor = r'&' + yaml.serializer.Serializer.ANCHOR_TEMPLATE % 1 +anchor = r'&' + Serializer.ANCHOR_TEMPLATE % 1 found_alias = re.search(anchor, content, re.MULTILINE) if found_alias: msg = "Error at: %s\nContent:\n%s" % (found_alias, content) ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
[Cloud-init-dev] [Merge] ~smoser/cloud-init:fix/1788915-vlan-sysconfig-rendering into cloud-init:master
Scott Moser has proposed merging
~smoser/cloud-init:fix/1788915-vlan-sysconfig-rendering into cloud-init:master.
Commit message:
network: Fix type and respect name when rendering vlan in sysconfig.
Prior to this change, vlans were rendered in sysconfig with
'TYPE=Ethernet', and incorrectly renders the PHYSDEV based on
the name of the vlan device rather than the 'link' provided
in the network config.
The change here fixes:
* rendering of TYPE=Ethernet for a vlan
* adds a warning if the configured device name is not supported
per the RHEL 7 docs "11.5. Naming Scheme for VLAN Interfaces"
LP: #1788915
LP: #1826608
Requested reviews:
cloud-init commiters (cloud-init-dev)
Related bugs:
Bug #1788915 in cloud-init: "sysconfig renders vlan with TYPE=Ethernet"
https://bugs.launchpad.net/cloud-init/+bug/1788915
Bug #1826608 in cloud-init: "sysconfig rendering ignores vlan name"
https://bugs.launchpad.net/cloud-init/+bug/1826608
For more details, see:
https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/366602
see commit message
--
Your team cloud-init commiters is requested to review the proposed merge of
~smoser/cloud-init:fix/1788915-vlan-sysconfig-rendering into cloud-init:master.
diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py
index a47da0a..6118552 100644
--- a/cloudinit/net/sysconfig.py
+++ b/cloudinit/net/sysconfig.py
@@ -96,6 +96,9 @@ class ConfigMap(object):
def __len__(self):
return len(self._conf)
+def str_skipped(self, key, val):
+return False
+
def to_string(self):
buf = six.StringIO()
buf.write(_make_header())
@@ -103,6 +106,8 @@ class ConfigMap(object):
buf.write("\n")
for key in sorted(self._conf.keys()):
value = self._conf[key]
+if self.str_skipped(key, value):
+continue
if isinstance(value, bool):
value = self._bool_map[value]
if not isinstance(value, six.string_types):
@@ -208,6 +213,7 @@ class NetInterface(ConfigMap):
'bond': 'Bond',
'bridge': 'Bridge',
'infiniband': 'InfiniBand',
+'vlan': 'Vlan',
}
def __init__(self, iface_name, base_sysconf_dir, templates,
@@ -261,6 +267,11 @@ class NetInterface(ConfigMap):
c.routes = self.routes.copy()
return c
+def str_skipped(self, key, val):
+if key == 'TYPE' and val == 'Vlan':
+return True
+return False
+
class Renderer(renderer.Renderer):
"""Renders network information in a /etc/sysconfig format."""
@@ -555,7 +566,16 @@ class Renderer(renderer.Renderer):
iface_name = iface['name']
iface_cfg = iface_contents[iface_name]
iface_cfg['VLAN'] = True
-iface_cfg['PHYSDEV'] = iface_name[:iface_name.rfind('.')]
+iface_cfg.kind = 'vlan'
+
+rdev = iface['vlan-raw-device']
+supported = _supported_vlan_names(rdev, iface['vlan_id'])
+if iface_name not in supported:
+LOG.warning(
+"Name '%s' not officially supported for vlan "
+"device backed by '%s'. Supported: %s",
+iface_name, rdev, ' '.join(supported))
+iface_cfg['PHYSDEV'] = rdev
iface_subnets = iface.get("subnets", [])
route_cfg = iface_cfg.routes
@@ -716,6 +736,15 @@ class Renderer(renderer.Renderer):
"\n".join(netcfg) + "\n", file_mode)
+def _supported_vlan_names(rdev, vid):
+"""Return list of supported names for vlan devices per RHEL doc
+11.5. Naming Scheme for VLAN Interfaces."""
+return [
+v.format(rdev=rdev, vid=int(vid))
+for v in ("{rdev}{vid:04}", "{rdev}{vid}",
+ "{rdev}.{vid:04}", "{rdev}.{vid}")]
+
+
def available(target=None):
sysconfig = available_sysconfig(target=target)
nm = available_nm(target=target)
diff --git a/tests/unittests/test_distros/test_netconfig.py b/tests/unittests/test_distros/test_netconfig.py
index c3c0c8c..b0ef4aa 100644
--- a/tests/unittests/test_distros/test_netconfig.py
+++ b/tests/unittests/test_distros/test_netconfig.py
@@ -526,6 +526,91 @@ class TestNetCfgDistroRedhat(TestNetCfgDistroBase):
V1_NET_CFG_IPV6,
expected_cfgs=expected_cfgs.copy())
+def test_vlan_render_unsupported(self):
+"""Render officially unsupported vlan names."""
+cfg = {
+'version': 2,
+'ethernets': {
+'eth0': {'addresses': ["192.10.1.2/24"],
+ 'match': {'macaddress': "00:16:3e:6
[Cloud-init-dev] [Merge] ~smoser/cloud-init:cleanup/openssl-manager-no-cleanup into cloud-init:master
Scott Moser has proposed merging
~smoser/cloud-init:cleanup/openssl-manager-no-cleanup into cloud-init:master.
Commit message:
Azure: Make the openssl manager object not need cleanup.
The 'clean_up' method on the OpenSSLManager object was annoying
as it had to be called or it would leave temp files around.
The change here makes it not need a persistent temporary storage
but rather make a temp dir and clean up as it needs.
Requested reviews:
cloud-init commiters (cloud-init-dev)
For more details, see:
https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/363757
Azure: Make the openssl manager object not need cleanup.
The 'clean_up' method on the OpenSSLManager object was annoying
as it had to be called or it would leave temp files around.
The change here makes it not need a persistent temporary storage
but rather make a temp dir and clean up as it needs.
--
Your team cloud-init commiters is requested to review the proposed merge of
~smoser/cloud-init:cleanup/openssl-manager-no-cleanup into cloud-init:master.
diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py
index 2829dd2..1053f69 100644
--- a/cloudinit/sources/helpers/azure.py
+++ b/cloudinit/sources/helpers/azure.py
@@ -4,6 +4,7 @@ import json
import logging
import os
import re
+import shutil
import socket
import struct
import time
@@ -20,16 +21,6 @@ from cloudinit import util
LOG = logging.getLogger(__name__)
-@contextmanager
-def cd(newdir):
-prevdir = os.getcwd()
-os.chdir(os.path.expanduser(newdir))
-try:
-yield
-finally:
-os.chdir(prevdir)
-
-
def _get_dhcp_endpoint_option_name():
if util.is_FreeBSD():
azure_endpoint = "option-245"
@@ -105,43 +96,34 @@ class GoalState(object):
class OpenSSLManager(object):
-
-certificate_names = {
-'private_key': 'TransportPrivate.pem',
-'certificate': 'TransportCert.pem',
-}
-
def __init__(self):
-self.tmpdir = temp_utils.mkdtemp()
-self.certificate = None
-self.generate_certificate()
-
-def clean_up(self):
-util.del_dir(self.tmpdir)
-
-def generate_certificate(self):
-LOG.debug('Generating certificate for communication with fabric...')
-if self.certificate is not None:
-LOG.debug('Certificate already generated.')
-return
-with cd(self.tmpdir):
-util.subp([
-'openssl', 'req', '-x509', '-nodes', '-subj',
-'/CN=LinuxTransport', '-days', '32768', '-newkey', 'rsa:2048',
-'-keyout', self.certificate_names['private_key'],
-'-out', self.certificate_names['certificate'],
-])
-certificate = ''
-for line in open(self.certificate_names['certificate']):
-if "CERTIFICATE" not in line:
-certificate += line.rstrip()
-self.certificate = certificate
-LOG.debug('New certificate generated.')
+self.certificate, self.private_key = self._generate_certificate()
+self.certificate_data = ''.join(
+[l for l in self.certificate.splitlines()
+ if not l.startswith("---")])
+
+@staticmethod
+def _generate_certificate():
+tmpd = temp_utils.mkdtemp(suffix="sslmanager.d")
+key_out = os.path.join(tmpd, 'private.key')
+cert_out = os.path.join(tmpd, 'out.cert')
+cmd = ['openssl', 'req', '-x509', '-nodes', '-subj',
+ '/CN=LinuxTransport', '-days', '32768', '-newkey', 'rsa:2048',
+ '-keyout', key_out, '-out', cert_out]
+try:
+util.subp(cmd)
+with open(key_out) as fp:
+private = fp.read()
+with open(cert_out) as fp:
+cert = fp.read()
+finally:
+shutil.rmtree(tmpd)
+return cert, private
@staticmethod
def _run_x509_action(action, cert):
cmd = ['openssl', 'x509', '-noout', action]
-result, _ = util.subp(cmd, data=cert)
+result, _ = util.subp(cmd, data=cert.encode('utf-8'))
return result
def _get_ssh_key_from_cert(self, certificate):
@@ -170,19 +152,29 @@ class OpenSSLManager(object):
tag = ElementTree.fromstring(certificates_xml).find('.//Data')
certificates_content = tag.text
lines = [
-b'MIME-Version: 1.0',
-b'Content-Disposition: attachment; filename="Certificates.p7m"',
-b'Content-Type: application/x-pkcs7-mime; name="Certificates.p7m"',
-b'Content-Transfer-Encoding: base64',
-b'',
-certificates_content.encode('utf-8'),
+'MIME-Version: 1.0',
+'Content-Disposition: attachment; filename="Certificates.p7m"',
+'Content-Type: application/x-pkcs7-mime; nam
Re: [Cloud-init-dev] [Merge] ~raharper/cloud-init:fix/merge-how-docs into cloud-init:master
A doc reference to the tools/ccfg-merge-debug might be useful (probably that could use some improvements too) Diff comments: > diff --git a/doc/rtd/topics/merging.rst b/doc/rtd/topics/merging.rst > index c75ca59..3761305 100644 > --- a/doc/rtd/topics/merging.rst > +++ b/doc/rtd/topics/merging.rst > @@ -201,4 +201,43 @@ Note, however, that merge algorithms are not used > *across* types of > configuration. As was the case before merging was implemented, > user-data will overwrite conf.d configuration without merging. > > +Example cloud-config > + > + > +A common request is to include multiple ``runcmd`` directives in different > +files and merge all of the commands together. To achieve this, we must > modify > +the default merging to allow for dictionaries to join list values. > + > + > +The first config > + > +.. code-block:: yaml > + > + #cloud-config > + merge_how: > +- name: list > + settings: [append] > +- name: dict > + settings: [no_replace, recurse_list] Does the first config need to specify this? I don't think so, and if not then this just leads to confusion. The second config *should* indicate how it is to be merged independent of any settings in the first. > + > + runcmd: > + - bash1 > + - bash2 > + > +The second config > + > +.. code-block:: yaml > + > + #cloud-config > + merge_how: > +- name: list > + settings: [append] > +- name: dict > + settings: [no_replace, recurse_list] > + > + runcmd: > + - bash3 > + - bash4 > + > + > .. vi: textwidth=78 -- https://code.launchpad.net/~raharper/cloud-init/+git/cloud-init/+merge/362083 Your team cloud-init commiters is requested to review the proposed merge of ~raharper/cloud-init:fix/merge-how-docs into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
Re: [Cloud-init-dev] [Merge] ~johnsonshi/cloud-init:rhel-centos-dhclient-lease-path into cloud-init:master
Hi, This is kind of expected to work on rhel via the code added in 648dbbf6b090c81e989f1ab70bf99f4de16a6a70 . That provides dhclient hooks in tools/hook-rhel.sh that should write /run/cloud-init/dhclient.hooks/.json that should be read from 'find_endpoint' use of _load_dhclient_json. I suspect that we just need to make sure that the dhclient hook is getting installed correctly and then this may "just work". -- https://code.launchpad.net/~johnsonshi/cloud-init/+git/cloud-init/+merge/361108 Your team cloud-init commiters is requested to review the proposed merge of ~johnsonshi/cloud-init:rhel-centos-dhclient-lease-path into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
[Cloud-init-dev] [Merge] ~smoser/cloud-init:doc/cleanup-doc-errors into cloud-init:master
Scott Moser has proposed merging ~smoser/cloud-init:doc/cleanup-doc-errors into cloud-init:master. Commit message: doc: clean up some datasource documentation. The change to datasources.rst here is obvious typo fix. The change to azure is to reduce the two 'Customization' sections to a single and clean up some other duplicate text. Requested reviews: cloud-init commiters (cloud-init-dev) For more details, see: https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/361467 see commit message -- Your team cloud-init commiters is requested to review the proposed merge of ~smoser/cloud-init:doc/cleanup-doc-errors into cloud-init:master. diff --git a/doc/rtd/topics/datasources.rst b/doc/rtd/topics/datasources.rst index e34f145..5abbaef 100644 --- a/doc/rtd/topics/datasources.rst +++ b/doc/rtd/topics/datasources.rst @@ -18,7 +18,7 @@ single way to access the different cloud systems methods to provide this data through the typical usage of subclasses. Any metadata processed by cloud-init's datasources is persisted as -``/run/cloud0-init/instance-data.json``. Cloud-init provides tooling +``/run/cloud-init/instance-data.json``. Cloud-init provides tooling to quickly introspect some of that data. See :ref:`instance_metadata` for more information. diff --git a/doc/rtd/topics/datasources/azure.rst b/doc/rtd/topics/datasources/azure.rst index f73c369..720a475 100644 --- a/doc/rtd/topics/datasources/azure.rst +++ b/doc/rtd/topics/datasources/azure.rst @@ -23,18 +23,18 @@ information in json format to /run/cloud-init/dhclient.hook/.json. In order for cloud-init to leverage this method to find the endpoint, the cloud.cfg file must contain: -datasource: - Azure: -set_hostname: False -agent_command: __builtin__ +.. sourcecode:: yaml + + datasource: +Azure: + set_hostname: False + agent_command: __builtin__ If those files are not available, the fallback is to check the leases file for the endpoint server (again option 245). You can define the path to the lease file with the 'dhclient_lease_file' -configuration. The default value is /var/lib/dhcp/dhclient.eth0.leases. - -dhclient_lease_file: /var/lib/dhcp/dhclient.eth0.leases +configuration. walinuxagent @@ -60,7 +60,7 @@ in order to use waagent.conf with cloud-init, the following settings are recomme Configuration - The following configuration can be set for the datasource in system -configuration (in `/etc/cloud/cloud.cfg` or `/etc/cloud/cloud.cfg.d/`). +configuration (in ``/etc/cloud/cloud.cfg`` or ``/etc/cloud/cloud.cfg.d/``). The settings that may be configured are: @@ -76,13 +76,25 @@ The settings that may be configured are: * **disk_aliases**: A dictionary defining which device paths should be interpreted as ephemeral images. See cc_disk_setup module for more info. * **hostname_bounce**: A dictionary Azure hostname bounce behavior to react to - metadata changes. + metadata changes. The '``hostname_bounce: command``' entry can be either + the literal string 'builtin' or a command to execute. The command will be + invoked after the hostname is set, and will have the 'interface' in its + environment. If ``set_hostname`` is not true, then ``hostname_bounce`` + will be ignored. An example might be: + + ``command: ["sh", "-c", "killall dhclient; dhclient $interface"]`` + * **hostname_bounce**: A dictionary Azure hostname bounce behavior to react to metadata changes. Azure will throttle ifup/down in some cases after metadata has been updated to inform dhcp server about updated hostnames. * **set_hostname**: Boolean set to True when we want Azure to set the hostname based on metadata. +Configuration for the datasource can also be read from a +``dscfg`` entry in the ``LinuxProvisioningConfigurationSet``. Content in +dscfg node is expected to be base64 encoded yaml content, and it will be +merged into the 'datasource: Azure' entry. + An example configuration with the default values is provided below: .. sourcecode:: yaml @@ -143,37 +155,6 @@ Example: -Configuration -- -Configuration for the datasource can be read from the system config's or set -via the `dscfg` entry in the `LinuxProvisioningConfigurationSet`. Content in -dscfg node is expected to be base64 encoded yaml content, and it will be -merged into the 'datasource: Azure' entry. - -The '``hostname_bounce: command``' entry can be either the literal string -'builtin' or a command to execute. The command will be invoked after the -hostname is set, and will have the 'interface' in its environment. If -``set_hostname`` is not true, then ``hostname_bounce`` will be ignored. - -An example might be: - command: ["sh", "-c", "killall dhclient; dhclient $interface"] - -.. code:: yaml - - datasource: - agent_command - Azure: -agent_command: [service, walinuxagent, start] -
[Cloud-init-dev] [Merge] ~smoser/cloud-init:doc/add-new-datasource-doc into cloud-init:master
Scott Moser has proposed merging ~smoser/cloud-init:doc/add-new-datasource-doc into cloud-init:master. Commit message: Add documentation on adding a datasource. This adds documentation intended for a developer on how to add a new datasource to cloud-init. Requested reviews: cloud-init commiters (cloud-init-dev) For more details, see: https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/361466 see commit message -- Your team cloud-init commiters is requested to review the proposed merge of ~smoser/cloud-init:doc/add-new-datasource-doc into cloud-init:master. diff --git a/doc/rtd/topics/datasources.rst b/doc/rtd/topics/datasources.rst index e34f145..7fa82f1 100644 --- a/doc/rtd/topics/datasources.rst +++ b/doc/rtd/topics/datasources.rst @@ -18,7 +18,7 @@ single way to access the different cloud systems methods to provide this data through the typical usage of subclasses. Any metadata processed by cloud-init's datasources is persisted as -``/run/cloud0-init/instance-data.json``. Cloud-init provides tooling +``/run/cloud-init/instance-data.json``. Cloud-init provides tooling to quickly introspect some of that data. See :ref:`instance_metadata` for more information. @@ -80,6 +80,46 @@ The current interface that a datasource object must provide is the following: def get_package_mirror_info(self) +Adding a new Datasource +--- +The datasource objects have a few touch points with cloud-init. If you +are interested in adding a new datasource for your cloud platform you'll +need to take care of the following items: + +* **Add datasource module ``cloudinit/sources/DataSource.py``**: + It is suggested that you start by copying one of the simpler datasources + such as DataSourceHetzner. + +* **Add tests for datasource module**: + Add a new file with some tests for the module to + ``cloudinit/sources/test_.py``. For example see + ``cloudinit/sources/tests/test_oracle.py`` + +* **Update ds-identify**: In systemd systems, ds-identify is used to detect + which datasources should be enabled or if cloud-init should run at all. + +* **Add tests for ds-identify**: Add relevant tests in a new class to + ``tests/unittests/test_ds_identify.py``. You can use ``TestOracle`` as an + example. + +* **Add your datasource name to the builtin list of datasources:** Add + your datasource module name to the end of the ``datasource_list`` + entry in ``cloudinit/settings.py``. + +* **Add your your cloud platform to apport collection prompts:** Update the + list of cloud platforms in ``cloudinit/apport.py``. This list will be + provided to the user who invokes ``ubuntu-bug cloud-init``. + +* **Enable datasource by default in ubuntu packaging branches:** + Ubuntu packaging branches contain a template file + ``debian/cloud-init.templates`` that ultimately sets the default + datasource_list when installed via package. This file needs updating when + the commit gets into a package. + +* **Add documentation for your datasource**: You should add a new + file in ``doc/datasources/.rst`` + + Datasource Documentation The following is a list of the implemented datasources. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
[Cloud-init-dev] [Merge] ~smoser/cloud-init:fix/ds-identify-vmware-detection into cloud-init:master
Scott Moser has proposed merging
~smoser/cloud-init:fix/ds-identify-vmware-detection into cloud-init:master.
Commit message:
ds-identify: fix wrong variable name in ovf_vmware_transport_guestinfo.
ovf_vmware_transport_guestinfo is not currently tested.
It used '$1' instead of '$out' when checking for xml content in
the output of vmware-rpctool.
Requested reviews:
cloud-init commiters (cloud-init-dev)
For more details, see:
https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/361218
see commit message
--
Your team cloud-init commiters is requested to review the proposed merge of
~smoser/cloud-init:fix/ds-identify-vmware-detection into cloud-init:master.
diff --git a/tools/ds-identify b/tools/ds-identify
index c61f18a..b78b273 100755
--- a/tools/ds-identify
+++ b/tools/ds-identify
@@ -736,7 +736,7 @@ ovf_vmware_transport_guestinfo() {
debug 1 "Running on vmware but rpctool query returned $ret: $out"
return 1
fi
-case "$1" in
+case "$out" in
"___
Mailing list: https://launchpad.net/~cloud-init-dev
Post to : cloud-init-dev@lists.launchpad.net
Unsubscribe : https://launchpad.net/~cloud-init-dev
More help : https://help.launchpad.net/ListHelp
[Cloud-init-dev] [Merge] ~smoser/cloud-init:cleanup/ovf-transport-signatures into cloud-init:master
Scott Moser has proposed merging
~smoser/cloud-init:cleanup/ovf-transport-signatures into cloud-init:master.
Commit message:
OVF: simplify expected return values of transport functions.
Transport functions (transport_iso9660 and transport_vmware_guestinfo)
would return a tuple of 3 values, but only the first was ever used
outside of test. The other values (device and filename) were just ignored.
This just simplifies the transport functions to now return content
(in string format) or None indicating that the transport was not found.
Requested reviews:
Server Team CI bot (server-team-bot): continuous-integration
cloud-init commiters (cloud-init-dev)
For more details, see:
https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/361204
see commit message
--
Your team cloud-init commiters is requested to review the proposed merge of
~smoser/cloud-init:cleanup/ovf-transport-signatures into cloud-init:master.
diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py
index 891d654..3ca8ea1 100644
--- a/cloudinit/sources/DataSourceOVF.py
+++ b/cloudinit/sources/DataSourceOVF.py
@@ -236,7 +236,7 @@ class DataSourceOVF(sources.DataSource):
('iso', transport_iso9660)]
name = None
for name, transfunc in np:
-(contents, _dev, _fname) = transfunc()
+contents = transfunc()
if contents:
break
if contents:
@@ -464,8 +464,8 @@ def maybe_cdrom_device(devname):
return cdmatch.match(devname) is not None
-# Transport functions take no input and return
-# a 3 tuple of content, path, filename
+# Transport functions are called with no arguments and return
+# either None (indicating not present) or string content of an ovf-env.xml
def transport_iso9660(require_iso=True):
# Go through mounts to see if it was already mounted
@@ -479,7 +479,7 @@ def transport_iso9660(require_iso=True):
mp = info['mountpoint']
(fname, contents) = get_ovf_env(mp)
if contents is not False:
-return (contents, dev, fname)
+return contents
if require_iso:
mtype = "iso9660"
@@ -498,21 +498,21 @@ def transport_iso9660(require_iso=True):
continue
if contents is not False:
-return (contents, dev, fname)
+return contents
-return (False, None, None)
+return None
def transport_vmware_guestinfo():
rpctool = "vmware-rpctool"
-not_found = (False, None, None)
+not_found = None
if not util.which(rpctool):
return not_found
cmd = [rpctool, "info-get guestinfo.ovfEnv"]
try:
out, _err = util.subp(cmd)
if out:
-return (out, rpctool, "guestinfo.ovfEnv")
+return out
LOG.debug("cmd %s exited 0 with empty stdout: %s", cmd, out)
except util.ProcessExecutionError as e:
if e.exit_code != 1:
diff --git a/tests/unittests/test_datasource/test_ovf.py b/tests/unittests/test_datasource/test_ovf.py
index e4af0fa..89b26b9 100644
--- a/tests/unittests/test_datasource/test_ovf.py
+++ b/tests/unittests/test_datasource/test_ovf.py
@@ -19,6 +19,8 @@ from cloudinit.sources.helpers.vmware.imc.config_custom_script import (
MPATH = 'cloudinit.sources.DataSourceOVF.'
+NOT_FOUND = None
+
OVF_ENV_CONTENT = """
http://schemas.dmtf.org/ovf/environment/1;
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
@@ -127,8 +129,8 @@ class TestDatasourceOVF(CiTestCase):
retcode = wrap_and_call(
'cloudinit.sources.DataSourceOVF',
{'util.read_dmi_data': None,
- 'transport_iso9660': (False, None, None),
- 'transport_vmware_guestinfo': (False, None, None)},
+ 'transport_iso9660': NOT_FOUND,
+ 'transport_vmware_guestinfo': NOT_FOUND},
ds.get_data)
self.assertFalse(retcode, 'Expected False return from ds.get_data')
self.assertIn(
@@ -143,8 +145,8 @@ class TestDatasourceOVF(CiTestCase):
retcode = wrap_and_call(
'cloudinit.sources.DataSourceOVF',
{'util.read_dmi_data': 'vmware',
- 'transport_iso9660': (False, None, None),
- 'transport_vmware_guestinfo': (False, None, None)},
+ 'transport_iso9660': NOT_FOUND,
+ 'transport_vmware_guestinfo': NOT_FOUND},
ds.get_data)
self.assertFalse(retcode, 'Expected False return from ds.get_data')
self.assertIn(
@@ -194,8 +196,8 @@ class TestDatasourceOVF(CiTestCase):
with mock.patch(MPATH + 'util.read_dmi_data', return_value='!VMware'):
with mock.patch(MPATH + 'transport_vmware_guestinfo') as m_guestd:
with mock.patch(MPATH + 'transport_iso9660') as m_iso9660:
-m_iso9660.return_value = (None, 'i
[Cloud-init-dev] [Merge] ~smoser/cloud-init:feature/1807466-ovf-guestinfo-transport into cloud-init:master
Scott Moser has proposed merging
~smoser/cloud-init:feature/1807466-ovf-guestinfo-transport into
cloud-init:master.
Commit message:
Vmware: Add support for the com.vmware.guestInfo OVF transport.
This adds support for reading OVF information over the
'com.vmware.guestInfo' tranport. The current implementation requires
vmware-rpctool be installed in the system.
LP: #1807466
Requested reviews:
cloud-init commiters (cloud-init-dev)
For more details, see:
https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/361140
--
Your team cloud-init commiters is requested to review the proposed merge of
~smoser/cloud-init:feature/1807466-ovf-guestinfo-transport into
cloud-init:master.
diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py
index 045291e..ca57469 100644
--- a/cloudinit/sources/DataSourceOVF.py
+++ b/cloudinit/sources/DataSourceOVF.py
@@ -233,7 +233,7 @@ class DataSourceOVF(sources.DataSource):
else:
np = {'iso': transport_iso9660,
- 'vmware-guestd': transport_vmware_guestd, }
+ 'com.vmware.guestInfo': transport_vmware_guestinfo}
name = None
for (name, transfunc) in np.items():
(contents, _dev, _fname) = transfunc()
@@ -503,17 +503,20 @@ def transport_iso9660(require_iso=True):
return (False, None, None)
-def transport_vmware_guestd():
-# http://blogs.vmware.com/vapp/2009/07/ \
-#selfconfiguration-and-the-ovf-environment.html
-# try:
-# cmd = ['vmware-guestd', '--cmd', 'info-get guestinfo.ovfEnv']
-# (out, err) = subp(cmd)
-# return(out, 'guestinfo.ovfEnv', 'vmware-guestd')
-# except:
-# # would need to error check here and see why this failed
-# # to know if log/error should be raised
-# return(False, None, None)
+def transport_vmware_guestinfo():
+rpctool = "vmware-rpctool"
+if not util.which(rpctool):
+return None
+cmd = [rpctool, "info-get guestinfo.ovfEnv"]
+try:
+out, _err = util.subp(cmd)
+if out:
+return (out, rpctool, "guestinfo.ovfEnv")
+LOG.debug("cmd %s exited 0 with empty stdout: %s", cmd, out)
+except util.ProcessExecutionError as e:
+if e.exit_code != 1:
+LOG.warn("%s exited with code %d", rpctool, e.exit_code)
+LOG.debug(e)
return (False, None, None)
diff --git a/tests/unittests/test_datasource/test_ovf.py b/tests/unittests/test_datasource/test_ovf.py
index a226c03..d1e6640 100644
--- a/tests/unittests/test_datasource/test_ovf.py
+++ b/tests/unittests/test_datasource/test_ovf.py
@@ -126,7 +126,7 @@ class TestDatasourceOVF(CiTestCase):
'cloudinit.sources.DataSourceOVF',
{'util.read_dmi_data': None,
'transport_iso9660': (False, None, None),
- 'transport_vmware_guestd': (False, None, None)},
+ 'transport_vmware_guestinfo': (False, None, None)},
ds.get_data)
self.assertFalse(retcode, 'Expected False return from ds.get_data')
self.assertIn(
@@ -142,7 +142,7 @@ class TestDatasourceOVF(CiTestCase):
'cloudinit.sources.DataSourceOVF',
{'util.read_dmi_data': 'vmware',
'transport_iso9660': (False, None, None),
- 'transport_vmware_guestd': (False, None, None)},
+ 'transport_vmware_guestinfo': (False, None, None)},
ds.get_data)
self.assertFalse(retcode, 'Expected False return from ds.get_data')
self.assertIn(
@@ -191,7 +191,7 @@ class TestDatasourceOVF(CiTestCase):
self.assertEqual('ovf', ds.platform_type)
MPATH = 'cloudinit.sources.DataSourceOVF.'
with mock.patch(MPATH + 'util.read_dmi_data', return_value='!VMware'):
-with mock.patch(MPATH + 'transport_vmware_guestd') as m_guestd:
+with mock.patch(MPATH + 'transport_vmware_guestinfo') as m_guestd:
with mock.patch(MPATH + 'transport_iso9660') as m_iso9660:
m_iso9660.return_value = (None, 'ignored', 'ignored')
m_guestd.return_value = (None, 'ignored', 'ignored')
@@ -213,7 +213,7 @@ class TestDatasourceOVF(CiTestCase):
self.assertEqual('ovf', ds.platform_type)
MPATH = 'cloudinit.sources.DataSourceOVF.'
with mock.patch(MPATH + 'util.read_dmi_data', return_value='VMWare'):
-with mock.patch(MPATH + 'transport_vmware_guestd') as m_guestd:
+with mock.patch(MPATH + 'transport_vmware_guestinfo') as m_guestd:
with mock.patch(MPATH + 'transport_iso9660') as m_iso9660:
m_iso9660.return_value = (None, 'ignored', 'ignored')
m_guestd.return_value = (None, 'ignored', 'ignored')
diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py
i
[Cloud-init-dev] [Merge] ~smoser/cloud-init:feature/1807466-ovf-guestinfo-transport into cloud-init:master
The proposal to merge ~smoser/cloud-init:feature/1807466-ovf-guestinfo-transport into cloud-init:master has been updated. Commit message changed to: Vmware: Add support for the com.vmware.guestInfo OVF transport. This adds support for reading OVF information over the 'com.vmware.guestInfo' tranport. The current implementation requires vmware-rpctool be installed in the system. LP: #1807466 For more details, see: https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/361140 -- Your team cloud-init commiters is requested to review the proposed merge of ~smoser/cloud-init:feature/1807466-ovf-guestinfo-transport into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
Re: [Cloud-init-dev] [Merge] ~chad.smith/cloud-init:tests/cii-summary-fallback-to-traceback into cloud-init:master
@Chad. the first commit is in. So rebase on mater, and push for c-i. can you give an example of the end result of this chagne? -- https://code.launchpad.net/~chad.smith/cloud-init/+git/cloud-init/+merge/356427 Your team cloud-init commiters is requested to review the proposed merge of ~chad.smith/cloud-init:tests/cii-summary-fallback-to-traceback into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
[Cloud-init-dev] [Merge] ~smoser/cloud-init:fix/tip-pylint-20181210 into cloud-init:master
The proposal to merge ~smoser/cloud-init:fix/tip-pylint-20181210 into cloud-init:master has been updated. Commit message changed to: Update to pylint 2.2.2. The tip-pylint tox target correctly reported the invalid use of string formatting. The change here is to: a.) Fix the error that was caught. b.) move to pylint 2.2.2 for the default 'pylint' target. For more details, see: https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/360632 -- Your team cloud-init commiters is requested to review the proposed merge of ~smoser/cloud-init:fix/tip-pylint-20181210 into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
[Cloud-init-dev] [Merge] ~smoser/cloud-init:fix/tip-pylint-20181210 into cloud-init:master
The proposal to merge ~smoser/cloud-init:fix/tip-pylint-20181210 into cloud-init:master has been updated. Commit message changed to: Azure: Fix incorrect string format reported by pylint 2.2.2. The tip-pylint tox target correctly reported the invalid use of string formatting. The change here is to: a.) Fix the error that was caught. b.) move to pylint 2.2.2 for the default 'pylint' target. For more details, see: https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/360632 -- Your team cloud-init commiters is requested to review the proposed merge of ~smoser/cloud-init:fix/tip-pylint-20181210 into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
[Cloud-init-dev] [Merge] ~smoser/cloud-init:fix/tip-pylint-20181210 into cloud-init:master
Scott Moser has proposed merging ~smoser/cloud-init:fix/tip-pylint-20181210
into cloud-init:master.
Commit message:
Azure: Fix incorrect string format.
Fix incorrect order in a format string. If the old code would have
been hit, a stacktrace would have been raised.
Requested reviews:
cloud-init commiters (cloud-init-dev)
For more details, see:
https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/360632
see commit message
--
Your team cloud-init commiters is requested to review the proposed merge of
~smoser/cloud-init:fix/tip-pylint-20181210 into cloud-init:master.
diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
index e076d5d..9132176 100644
--- a/cloudinit/sources/DataSourceAzure.py
+++ b/cloudinit/sources/DataSourceAzure.py
@@ -980,8 +980,8 @@ def read_azure_ovf(contents):
raise NonAzureDataSource("No LinuxProvisioningConfigurationSet")
if len(lpcs_nodes) > 1:
raise BrokenAzureDataSource("found '%d' %ss" %
-("LinuxProvisioningConfigurationSet",
- len(lpcs_nodes)))
+(len(lpcs_nodes),
+"LinuxProvisioningConfigurationSet"))
lpcs = lpcs_nodes[0]
if not lpcs.hasChildNodes():
___
Mailing list: https://launchpad.net/~cloud-init-dev
Post to : cloud-init-dev@lists.launchpad.net
Unsubscribe : https://launchpad.net/~cloud-init-dev
More help : https://help.launchpad.net/ListHelp
[Cloud-init-dev] [Merge] ~raharper/cloud-init:fix/ovf-imc-network-config-gateway-routes into cloud-init:master
The proposal to merge ~raharper/cloud-init:fix/ovf-imc-network-config-gateway-routes into cloud-init:master has been updated. Status: Needs review => Approved For more details, see: https://code.launchpad.net/~raharper/cloud-init/+git/cloud-init/+merge/359946 -- Your team cloud-init commiters is requested to review the proposed merge of ~raharper/cloud-init:fix/ovf-imc-network-config-gateway-routes into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
Re: [Cloud-init-dev] [Merge] ~raharper/cloud-init:fix/ovf-imc-network-config-gateway-routes into cloud-init:master
Could you just add a unit test with exactly: http://paste.ubuntu.com/p/5sGWRHB2d8/ I realize we mostly have it covered, but the genuine content is nice. Other than that, I approve. -- https://code.launchpad.net/~raharper/cloud-init/+git/cloud-init/+merge/359946 Your team cloud-init commiters is requested to review the proposed merge of ~raharper/cloud-init:fix/ovf-imc-network-config-gateway-routes into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
[Cloud-init-dev] [Merge] ~raharper/cloud-init:fix/ovf-imc-network-config-gateway-routes into cloud-init:master
The proposal to merge
~raharper/cloud-init:fix/ovf-imc-network-config-gateway-routes into
cloud-init:master has been updated.
Commit message changed to:
ovf: Fix ovf network config generation gateway/routes
Move routes under the nic's subnet rather than use top-level
("global") route config ensuring all net renderers will provide the
configured route.
Also updated cloudinit/cmd/devel/net_convert.py:
- Add input type 'vmware-imc' for OVF customization config files
- Fix bug when output-type was netplan which invoked netplan
generate/apply and attempted to write to
/etc/netplan/50-cloud-init.yaml instead of joining with the
output directory.
LP: #1806103
For more details, see:
https://code.launchpad.net/~raharper/cloud-init/+git/cloud-init/+merge/359946
--
Your team cloud-init commiters is requested to review the proposed merge of
~raharper/cloud-init:fix/ovf-imc-network-config-gateway-routes into
cloud-init:master.
___
Mailing list: https://launchpad.net/~cloud-init-dev
Post to : cloud-init-dev@lists.launchpad.net
Unsubscribe : https://launchpad.net/~cloud-init-dev
More help : https://help.launchpad.net/ListHelp
[Cloud-init-dev] [Merge] ~smoser/cloud-init:feature/1805854-add-non-x86-mirrors into cloud-init:master
Scott Moser has proposed merging ~smoser/cloud-init:feature/1805854-add-non-x86-mirrors into cloud-init:master. Commit message: config: On ubuntu select cloud archive mirrors for armel, armhf, arm64. Infrastructure is now set up for Ubuntu to handle Amazon instances hitting a ports archive at: - http://%(ec2_region)s.ec2.ports.ubuntu.com/ubuntu-ports/ And additionally, generic mirrors at *.clouds.ports.ubuntu.com/ubuntu-ports The change here will utilize those mirrors for the arm64, armel and armhf arches. We've decided to limit the auto-selection of those mirrors to arm, where we know a use case. That way new instances of ppc64el or other arches will not select them. Such a behavior change could be problematic for a user in a firewalled environment. LP: #1805854 Requested reviews: cloud-init commiters (cloud-init-dev) Related bugs: Bug #1805854 in cloud-init: "[feature-request] Add non-x86 Ubuntu EC2 mirrors in to default cloud-init configuration" https://bugs.launchpad.net/cloud-init/+bug/1805854 For more details, see: https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/359948 see commit message -- Your team cloud-init commiters is requested to review the proposed merge of ~smoser/cloud-init:feature/1805854-add-non-x86-mirrors into cloud-init:master. diff --git a/config/cloud.cfg.tmpl b/config/cloud.cfg.tmpl index 1fef133..7513176 100644 --- a/config/cloud.cfg.tmpl +++ b/config/cloud.cfg.tmpl @@ -167,7 +167,17 @@ system_info: - http://%(availability_zone)s.clouds.archive.ubuntu.com/ubuntu/ - http://%(region)s.clouds.archive.ubuntu.com/ubuntu/ security: [] - - arches: [armhf, armel, default] + - arches: [arm64, armel, armhf] + failsafe: + primary: http://ports.ubuntu.com/ubuntu-ports + security: http://ports.ubuntu.com/ubuntu-ports + search: + primary: + - http://%(ec2_region)s.ec2.ports.ubuntu.com/ubuntu-ports/ + - http://%(availability_zone)s.clouds.ports.ubuntu.com/ubuntu-ports/ + - http://%(region)s.clouds.ports.ubuntu.com/ubuntu-ports/ + security: [] + - arches: [default] failsafe: primary: http://ports.ubuntu.com/ubuntu-ports security: http://ports.ubuntu.com/ubuntu-ports ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
[Cloud-init-dev] [Merge] ~smoser/cloud-init:fix/test-mock-order into cloud-init:master
Scott Moser has proposed merging ~smoser/cloud-init:fix/test-mock-order into
cloud-init:master.
Commit message:
tests: fix incorrect order of mocks in test_handle_zfs_root.
The order of parameters to test_handle_zfs_root did not match
the order of the mocks applied.
Thanks to Jason Zions for pointing this out.
Requested reviews:
Jason Zions (jasonzio)
cloud-init commiters (cloud-init-dev)
For more details, see:
https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/359752
see commit message
--
Your team cloud-init commiters is requested to review the proposed merge of
~smoser/cloud-init:fix/test-mock-order into cloud-init:master.
diff --git a/tests/unittests/test_handler/test_handler_resizefs.py b/tests/unittests/test_handler/test_handler_resizefs.py
index 6ebacb1..3518784 100644
--- a/tests/unittests/test_handler/test_handler_resizefs.py
+++ b/tests/unittests/test_handler/test_handler_resizefs.py
@@ -151,9 +151,9 @@ class TestResizefs(CiTestCase):
_resize_ufs(mount_point, devpth))
@mock.patch('cloudinit.util.is_container', return_value=False)
-@mock.patch('cloudinit.util.get_mount_info')
-@mock.patch('cloudinit.util.get_device_info_from_zpool')
@mock.patch('cloudinit.util.parse_mount')
+@mock.patch('cloudinit.util.get_device_info_from_zpool')
+@mock.patch('cloudinit.util.get_mount_info')
def test_handle_zfs_root(self, mount_info, zpool_info, parse_mount,
is_container):
devpth = 'vmzroot/ROOT/freebsd'
___
Mailing list: https://launchpad.net/~cloud-init-dev
Post to : cloud-init-dev@lists.launchpad.net
Unsubscribe : https://launchpad.net/~cloud-init-dev
More help : https://help.launchpad.net/ListHelp
[Cloud-init-dev] [Merge] ~smoser/cloud-init:fix/ovf-id-env-vmware into cloud-init:master
Scott Moser has proposed merging ~smoser/cloud-init:fix/ovf-id-env-vmware into
cloud-init:master.
Commit message:
OVF: identify label iso9660 filesystems with label 'OVF ENV'.
When deploying an OVA, at least some versions of vmware
attach a cdrom with an ISO9660 filesystem label of 'OVF ENV'.
This was seen on Vmware vCenter Server, 6.0.0, 2776510.
In order to accomplish this we had to change the content of
the DI_ISO9660_DEVS variable to be comma delimited rather
than space delimited.
Requested reviews:
cloud-init commiters (cloud-init-dev)
For more details, see:
https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/359646
see commit message
--
Your team cloud-init commiters is requested to review the proposed merge of
~smoser/cloud-init:fix/ovf-id-env-vmware into cloud-init:master.
diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py
index 46778e9..80640f1 100644
--- a/tests/unittests/test_ds_identify.py
+++ b/tests/unittests/test_ds_identify.py
@@ -499,7 +499,7 @@ class TestDsIdentify(DsIdentifyBase):
# Add recognized labels
valid_ovf_labels = ['ovf-transport', 'OVF-TRANSPORT',
-"OVFENV", "ovfenv"]
+"OVFENV", "ovfenv", "OVF ENV", "ovf env"]
for valid_ovf_label in valid_ovf_labels:
ovf_cdrom_by_label['mocks'][0]['out'] = blkid_out([
{'DEVNAME': 'sda1', 'TYPE': 'ext4', 'LABEL': 'rootfs'},
diff --git a/tools/ds-identify b/tools/ds-identify
index 5afe5aa..1acfeeb 100755
--- a/tools/ds-identify
+++ b/tools/ds-identify
@@ -237,7 +237,7 @@ read_fs_info() {
case "${line}" in
DEVNAME=*)
[ -n "$dev" -a "$ftype" = "iso9660" ] &&
-isodevs="${isodevs} ${dev}=$label"
+isodevs="${isodevs},${dev}=$label"
ftype=""; dev=""; label="";
dev=${line#DEVNAME=};;
LABEL=*) label="${line#LABEL=}";
@@ -247,11 +247,11 @@ read_fs_info() {
esac
done
[ -n "$dev" -a "$ftype" = "iso9660" ] &&
-isodevs="${isodevs} ${dev}=$label"
+isodevs="${isodevs},${dev}=$label"
DI_FS_LABELS="${labels%${delim}}"
DI_FS_UUIDS="${uuids%${delim}}"
-DI_ISO9660_DEVS="${isodevs# }"
+DI_ISO9660_DEVS="${isodevs#,}"
}
cached() {
@@ -735,9 +735,10 @@ is_cdrom_ovf() {
return 1;;
esac
+debug 1 "got label=$label"
# fast path known 'OVF' labels
case "$label" in
-OVF-TRANSPORT|ovf-transport|OVFENV|ovfenv) return 0;;
+OVF-TRANSPORT|ovf-transport|OVFENV|ovfenv|OVF\ ENV|ovf\ env) return 0;;
esac
# explicitly skip known labels of other types. rd_rdfe is azure.
@@ -757,9 +758,13 @@ dscheck_OVF() {
# Azure provides ovf. Skip false positive by dis-allowing.
is_azure_chassis && return $DS_NOT_FOUND
-# DI_ISO9660_DEVS is =label, like /dev/sr0=OVF-TRANSPORT
+# DI_ISO9660_DEVS is =label,=label2
+# like /dev/sr0=OVF-TRANSPORT,/dev/other=with spaces
if [ "${DI_ISO9660_DEVS#${UNAVAILABLE}:}" = "${DI_ISO9660_DEVS}" ]; then
-for tok in ${DI_ISO9660_DEVS}; do
+local oifs="$IFS"
+# shellcheck disable=2086
+{ IFS=","; set -- ${DI_ISO9660_DEVS}; IFS="$oifs"; }
+for tok in "$@"; do
is_cdrom_ovf "${tok%%=*}" "${tok#*=}" && return $DS_FOUND
done
fi
___
Mailing list: https://launchpad.net/~cloud-init-dev
Post to : cloud-init-dev@lists.launchpad.net
Unsubscribe : https://launchpad.net/~cloud-init-dev
More help : https://help.launchpad.net/ListHelp
[Cloud-init-dev] [Merge] ~smoser/cloud-init:fix/ovf-id-env-vmware into cloud-init:master
The proposal to merge ~smoser/cloud-init:fix/ovf-id-env-vmware into cloud-init:master has been updated. Commit message changed to: OVF: identify label iso9660 filesystems with label 'OVF ENV'. When deploying an OVA, at least some versions of vmware attach a cdrom with an ISO9660 filesystem label of 'OVF ENV'. This was seen on Vmware vCenter Server, 6.0.0, 2776510. In order to accomplish this we had to change the content of the DI_ISO9660_DEVS variable to be comma delimited rather than space delimited. For more details, see: https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/359646 -- Your team cloud-init commiters is requested to review the proposed merge of ~smoser/cloud-init:fix/ovf-id-env-vmware into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
[Cloud-init-dev] [Merge] ~adepue/cloud-init:master into cloud-init:master
The proposal to merge ~adepue/cloud-init:master into cloud-init:master has been updated. Commit message changed to: Azure: fix copy/paste error in error handling when reading azure ovf. Check the appropriate variables based on code review. Correcting what seems to be a copy/paste mistake for the error handling from a few lines above. For more details, see: https://code.launchpad.net/~adepue/cloud-init/+git/cloud-init/+merge/359630 -- Your team cloud-init commiters is requested to review the proposed merge of ~adepue/cloud-init:master into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
[Cloud-init-dev] [Merge] ~chad.smith/cloud-init:bug/1805201-non-root-collect-logs into cloud-init:master
The proposal to merge ~chad.smith/cloud-init:bug/1805201-non-root-collect-logs into cloud-init:master has been updated. Commit message changed to: logs: collect-logs ignore instance-data-sensitive.json on non-root user Since /run/cloud-init/instance-data-sensitive.json is root read-only, ignore this file if non-root user runs collect-logs. If --include-userdata is provided on the command line, exit in error if non-root user attempts this operation. Lastly, update the __main__ to exit based on return value of main. LP: #1805201 For more details, see: https://code.launchpad.net/~chad.smith/cloud-init/+git/cloud-init/+merge/359557 -- Your team cloud-init commiters is requested to review the proposed merge of ~chad.smith/cloud-init:bug/1805201-non-root-collect-logs into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
[Cloud-init-dev] [Merge] ~chad.smith/cloud-init:bug/1805201-non-root-collect-logs into cloud-init:master
The proposal to merge ~chad.smith/cloud-init:bug/1805201-non-root-collect-logs into cloud-init:master has been updated. Status: Needs review => Approved For more details, see: https://code.launchpad.net/~chad.smith/cloud-init/+git/cloud-init/+merge/359557 -- Your team cloud-init commiters is requested to review the proposed merge of ~chad.smith/cloud-init:bug/1805201-non-root-collect-logs into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
Re: [Cloud-init-dev] [Merge] ~chad.smith/cloud-init:bug/1805201-non-root-collect-logs into cloud-init:master
one commit message fix. other than that, i approve.
Diff comments:
> diff --git a/cloudinit/cmd/devel/logs.py b/cloudinit/cmd/devel/logs.py
> index df72520..4c086b5 100644
> --- a/cloudinit/cmd/devel/logs.py
> +++ b/cloudinit/cmd/devel/logs.py
> @@ -118,21 +133,21 @@ def collect_logs(tarfile, include_userdata,
> verbosity=0):
> with chdir(tmp_dir):
> subp(['tar', 'czvf', tarfile, log_dir.replace(tmp_dir + '/',
> '')])
> sys.stderr.write("Wrote %s\n" % tarfile)
> +return 0
>
>
> def handle_collect_logs_args(name, args):
> """Handle calls to 'cloud-init collect-logs' as a subcommand."""
> -collect_logs(args.tarfile, args.userdata, args.verbosity)
> +return collect_logs(args.tarfile, args.userdata, args.verbosity)
>
>
> def main():
> """Tool to collect and tar all cloud-init related logs."""
> parser = get_parser()
> -handle_collect_logs_args('collect-logs', parser.parse_args())
> -return 0
> +return handle_collect_logs_args('collect-logs', parser.parse_args())
>
>
> if __name__ == '__main__':
> -main()
> +sys.exit(main())
mention this fix in commit message.
I think its only valid when called as:
python cloudinit.commands.collect_logs
>
> # vi: ts=4 expandtab
--
https://code.launchpad.net/~chad.smith/cloud-init/+git/cloud-init/+merge/359557
Your team cloud-init commiters is requested to review the proposed merge of
~chad.smith/cloud-init:bug/1805201-non-root-collect-logs into cloud-init:master.
___
Mailing list: https://launchpad.net/~cloud-init-dev
Post to : cloud-init-dev@lists.launchpad.net
Unsubscribe : https://launchpad.net/~cloud-init-dev
More help : https://help.launchpad.net/ListHelp
Re: [Cloud-init-dev] [Merge] ~chad.smith/cloud-init:bug/1800223-retry-imds-on-timeout into cloud-init:master
Generally agree with rharper that the code is duplicate. lets just put it in a single function? def should_retry_on_url_exc(exc): if not isinstance(exc, UrlError): return False if exc.code == url_helper.NOT_FOUND return True if exc.cause and isinstance(exc.cause, requests.Timeout): return True Also using 'exc' in that function compared to 'exception' means we dont have sunc long lines. i only looked at it here though, so maybe i missed some context that i should have seen if i'd have branched and looked in the full file. scott -- https://code.launchpad.net/~chad.smith/cloud-init/+git/cloud-init/+merge/358112 Your team cloud-init commiters is requested to review the proposed merge of ~chad.smith/cloud-init:bug/1800223-retry-imds-on-timeout into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
Re: [Cloud-init-dev] [Merge] ~chad.smith/cloud-init:cleanup/report-unreadable-instance-data into cloud-init:master
seems sane. one request.
Diff comments:
> diff --git a/cloudinit/cmd/query.py b/cloudinit/cmd/query.py
> index ff03de9..650bc0a 100644
> --- a/cloudinit/cmd/query.py
> +++ b/cloudinit/cmd/query.py
> @@ -106,8 +107,11 @@ def handle_args(name, args):
>
> try:
> instance_json = util.load_file(instance_data_fn)
> -except IOError:
> -LOG.error('Missing instance-data.json file: %s', instance_data_fn)
> +except (IOError, OSError) as e:
> +if e.errno == EACCES:
> +LOG.error('No read permission on %s. Try sudo', instance_data_fn)
no read permission on '%s'
add the single quote here and below i think.
> +else:
> +LOG.error('Missing instance-data file: %s', instance_data_fn)
> return 1
>
> instance_data = util.load_json(instance_json)
--
https://code.launchpad.net/~chad.smith/cloud-init/+git/cloud-init/+merge/358041
Your team cloud-init commiters is requested to review the proposed merge of
~chad.smith/cloud-init:cleanup/report-unreadable-instance-data into
cloud-init:master.
___
Mailing list: https://launchpad.net/~cloud-init-dev
Post to : cloud-init-dev@lists.launchpad.net
Unsubscribe : https://launchpad.net/~cloud-init-dev
More help : https://help.launchpad.net/ListHelp
[Cloud-init-dev] [Merge] ~tomerc/cloud-init:patch-replace-nameserver-with-nameservers into cloud-init:master
The proposal to merge ~tomerc/cloud-init:patch-replace-nameserver-with-nameservers into cloud-init:master has been updated. Commit message changed to: doc: Change dns_nameserver property to dns_nameservers. According to the examples in the page, v1 network config DNS should be defined using the dns_nameservers. The singular dns_nameserver is undefined. For more details, see: https://code.launchpad.net/~tomerc/cloud-init/+git/cloud-init/+merge/357919 -- Your team cloud-init commiters is requested to review the proposed merge of ~tomerc/cloud-init:patch-replace-nameserver-with-nameservers into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
Re: [Cloud-init-dev] [Merge] ~i.galic/cloud-init:fix/fbsd-resizefs into cloud-init:master
Hi, Thank you for contributing to cloud-init. To contribute, you must sign the Canonical Contributor License Agreement (CLA) [1]. If you have already signed it as an individual, your Launchpad user will be listed in the contributor-agreement-canonical launchpad group [2]. Unfortunately there is no easy way to check if an organization or company you are doing work for has signed. If you are unsure or have questions, email scott.mo...@canonical.com or ping smoser in #cloud-init channel via freenode. For information on how to sign, please see the HACKING document [3]. Thanks again, and please feel free to reach out with any questions. – [1] http://www.canonical.com/contributors [2] https://launchpad.net/~contributor-agreement-canonical/+members [3] http://cloudinit.readthedocs.io/en/latest/topics/hacking.html -- https://code.launchpad.net/~i.galic/cloud-init/+git/cloud-init/+merge/357723 Your team cloud-init commiters is requested to review the proposed merge of ~i.galic/cloud-init:fix/fbsd-resizefs into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
Re: [Cloud-init-dev] [Merge] ~jasonzio/cloud-init:fixcloudid into cloud-init:master
Hi, Thank you for contributing to cloud-init. To contribute, you must sign the Canonical Contributor License Agreement (CLA) [1]. If you have already signed it as an individual, your Launchpad user will be listed in the contributor-agreement-canonical launchpad group [2]. Unfortunately there is no easy way to check if an organization or company you are doing work for has signed. If you are unsure or have questions, email scott.mo...@canonical.com or ping smoser in #cloud-init channel via freenode. For information on how to sign, please see the HACKING document [3]. Thanks again, and please feel free to reach out with any questions. – [1] http://www.canonical.com/contributors [2] https://launchpad.net/~contributor-agreement-canonical/+members [3] http://cloudinit.readthedocs.io/en/latest/topics/hacking.html -- https://code.launchpad.net/~jasonzio/cloud-init/+git/cloud-init/+merge/357668 Your team cloud-init commiters is requested to review the proposed merge of ~jasonzio/cloud-init:fixcloudid into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
Re: [Cloud-init-dev] [Merge] ~jasonzio/cloud-init:fixcloudid into cloud-init:master
@Jason, Thank you for finding this. You're probably right about the redhat also, and we would like that fixed. Could you fix packages/redhat/cloud-init.spec.in also ? Thanks again. -- https://code.launchpad.net/~jasonzio/cloud-init/+git/cloud-init/+merge/357668 Your team cloud-init commiters is requested to review the proposed merge of ~jasonzio/cloud-init:fixcloudid into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
Re: [Cloud-init-dev] [Merge] ~tomerc/cloud-init:remove-colon into cloud-init:master
I pointed https://jenkins.ubuntu.com/server/job/cloud-init-ci/414/ at this MP. If it succeeds, I'll grab. Thanks, Tom! -- https://code.launchpad.net/~tomerc/cloud-init/+git/cloud-init/+merge/357103 Your team cloud-init commiters is requested to review the proposed merge of ~tomerc/cloud-init:remove-colon into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
[Cloud-init-dev] [Merge] ~tomerc/cloud-init:remove-colon into cloud-init:master
The proposal to merge ~tomerc/cloud-init:remove-colon into cloud-init:master has been updated. Commit message changed to: docs: remove colon from network v1 config example. The docs for network v1 config contained a errant ':'. Simply drop it. For more details, see: https://code.launchpad.net/~tomerc/cloud-init/+git/cloud-init/+merge/357103 -- Your team cloud-init commiters is requested to review the proposed merge of ~tomerc/cloud-init:remove-colon into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
Re: [Cloud-init-dev] [Merge] ~mskalka/cloud-init:zfs-package-fix into cloud-init:master
looks like you will have to fix a test. so please do that, and then i'll approve. -- https://code.launchpad.net/~mskalka/cloud-init/+git/cloud-init/+merge/357775 Your team cloud-init commiters is requested to review the proposed merge of ~mskalka/cloud-init:zfs-package-fix into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
Re: [Cloud-init-dev] [Merge] ~mskalka/cloud-init:zfs-package-fix into cloud-init:master
Pointed jenkins c-i at this https://jenkins.ubuntu.com/server/job/cloud-init-ci/413/console it should report shortly. -- https://code.launchpad.net/~mskalka/cloud-init/+git/cloud-init/+merge/357775 Your team cloud-init commiters is requested to review the proposed merge of ~mskalka/cloud-init:zfs-package-fix into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
[Cloud-init-dev] [Merge] ~mskalka/cloud-init:zfs-package-fix into cloud-init:master
The proposal to merge ~mskalka/cloud-init:zfs-package-fix into cloud-init:master has been updated. Commit message changed to: Fix the ZFS package name in the LXD module to point to a real package. On 16.04 and earlier, 'apt-get install zfs' would select the zfsutils-linux package correctly. After that (currently 18.04+) it will just error. The change here will still work on 16.04 as the zfsutils-linux package is available there. LP: #1799779 For more details, see: https://code.launchpad.net/~mskalka/cloud-init/+git/cloud-init/+merge/357775 -- Your team cloud-init commiters is requested to review the proposed merge of ~mskalka/cloud-init:zfs-package-fix into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
Re: [Cloud-init-dev] [Merge] ~chad.smith/cloud-init:feature/azure-disable-imds-networking into cloud-init:master
Diff comments:
> diff --git a/cloudinit/sources/DataSourceAzure.py
> b/cloudinit/sources/DataSourceAzure.py
> index 39391d0..2a6d5b1 100644
> --- a/cloudinit/sources/DataSourceAzure.py
> +++ b/cloudinit/sources/DataSourceAzure.py
> @@ -619,7 +622,14 @@ class DataSourceAzure(sources.DataSource):
>the blacklisted devices.
> """
> if not self._network_config:
> -self._network_config = parse_network_config(self._metadata_imds)
> +if self.ds_cfg.get('apply_network_config'):
> +nc_src = self._metadata_imds
> +else:
the comment you have here is confusing. "or when disabled" implies that
"Xenial and earlier" are not disabled.
I think enough comment is:
Ubuntu 16.04 packages carry patches to take this path for backward compat.
given the RELEASE_BLOCKER comment above, I dont even really think you need this
at all.
> +# Xenial and earlier, or when disabled use fallback network
> +# config instead of IMDS because the image contains the
> +# necessary ifup/down hotplug hooks
> +nc_src = None
> +self._network_config = parse_network_config(nc_src)
> return self._network_config
>
>
--
https://code.launchpad.net/~chad.smith/cloud-init/+git/cloud-init/+merge/356989
Your team cloud-init commiters is requested to review the proposed merge of
~chad.smith/cloud-init:feature/azure-disable-imds-networking into
cloud-init:master.
___
Mailing list: https://launchpad.net/~cloud-init-dev
Post to : cloud-init-dev@lists.launchpad.net
Unsubscribe : https://launchpad.net/~cloud-init-dev
More help : https://help.launchpad.net/ListHelp
Re: [Cloud-init-dev] [Merge] ~chad.smith/cloud-init:feature/azure-disable-imds-networking into cloud-init:master
inline comments.
Diff comments:
> diff --git a/cloudinit/sources/DataSourceAzure.py
> b/cloudinit/sources/DataSourceAzure.py
> index 39391d0..11b4ba5 100644
> --- a/cloudinit/sources/DataSourceAzure.py
> +++ b/cloudinit/sources/DataSourceAzure.py
> @@ -207,7 +207,9 @@ BUILTIN_DS_CONFIG = {
> },
> 'disk_aliases': {'ephemeral0': RESOURCE_DISK_PATH},
> 'dhclient_lease_file': LEASE_FILE,
> +'apply_network_config': True, # Use IMDS published network configuration
> }
> +# RELEASE_BLOCKER: Xenial-only apply_network_config default is False
why the RELEASE_BLOCKER comment here ? and below?
>
> BUILTIN_CLOUD_CONFIG = {
> 'disk_setup': {
> diff --git a/doc/rtd/topics/datasources/azure.rst
> b/doc/rtd/topics/datasources/azure.rst
> index 559011e..bdf1659 100644
> --- a/doc/rtd/topics/datasources/azure.rst
> +++ b/doc/rtd/topics/datasources/azure.rst
> @@ -57,6 +57,52 @@ in order to use waagent.conf with cloud-init, the
> following settings are recomme
> ResourceDisk.MountPoint=/mnt
>
>
> +Configuration
> +-
> +The following configuration can be set for the datasource in system
> +configuration (in `/etc/cloud/cloud.cfg` or `/etc/cloud/cloud.cfg.d/`).
> +
> +The settings that may be configured are:
> +
> + * **agent_command**: Either __builtin__ (default) or a command to run to
> getcw
> + metadata. If __builtin__, get metadata from walinuxagent. Otherwise run
> the
> + provided command to obtain metadata.
> + * **apply_network_config**: Boolean set to True to use network configuration
> + described by Azure's IMDS endpoint instead of fallback network config of
> + dhcp on eth0. Default is True on netplan-enabled platforms.
on netplan-enabled platforms?
i think the default is 'true'. you can mention that it is set to false for
16.04 ubuntu, but i dont think it has specifically to do with netplan
> + * **data_dir**: Path used to read metadata files and write crawled data.
> + * **dhclient_lease_file**: The fallback lease file to source when looking
> for
> + custom DHCP option 245 from Azure fabric.
> + * **disk_aliases**: A dictionary defining which device paths should be
> + interpreted as ephemeral images. See cc_disk_setup module for more info.
> + * **hostname_bounce**: A dictionary Azure hostname bounce behavior to react
> to
> + metadata changes.
> + * **hostname_bounce**: A dictionary Azure hostname bounce behavior to react
> to
> + metadata changes. Azure will throttle ifup/down in some cases after
> metadata
> + has been updated to inform dhcp server about updated hostnames.
> + * **set_hostname**: Boolean set to True when we want Azure to set the
> hostname
> + based on metadata.
> +
> +An example configuration with the default values is provided below:
> +
> +.. sourcecode:: yaml
> +
> + datasource:
> + Azure:
> +agent_command: __builtin__
> +apply_network_config: true
> +data_dir: /var/lib/waagent
> +dhclient_lease_file: /var/lib/dhcp/dhclient.eth0.leases
> +disk_aliases:
> +ephemeral0: /dev/disk/cloud/azure_resource
> +hostname_bounce:
> +interface: eth0
> +command: builtin
> +policy: true
> +hostname_command: hostname
> +set_hostname: true
> +
> +
> Userdata
>
> Userdata is provided to cloud-init inside the ovf-env.xml file. Cloud-init
--
https://code.launchpad.net/~chad.smith/cloud-init/+git/cloud-init/+merge/356989
Your team cloud-init commiters is requested to review the proposed merge of
~chad.smith/cloud-init:feature/azure-disable-imds-networking into
cloud-init:master.
___
Mailing list: https://launchpad.net/~cloud-init-dev
Post to : cloud-init-dev@lists.launchpad.net
Unsubscribe : https://launchpad.net/~cloud-init-dev
More help : https://help.launchpad.net/ListHelp
Re: [Cloud-init-dev] [Merge] ~chad.smith/cloud-init:ubuntu/bionic into cloud-init:ubuntu/bionic
thoughts?
Diff comments:
> diff --git a/debian/changelog b/debian/changelog
> index 2bb9520..d83e08b 100644
> --- a/debian/changelog
> +++ b/debian/changelog
> @@ -1,3 +1,10 @@
> +cloud-init (18.4-0ubuntu1~18.04.2) bionic-proposed; urgency=medium
can you change that to be 'bionic' not bionic-proposed.
then we can upload straight away to a ppa with no hackery or fighting dput-ng.
> +
> + * debian/cloud-init.postinst: Rewrite /run/cloud-init/instance-data.json
> +on upgrade. (LP: #1798189)
> +
> + -- Chad Smith Wed, 16 Oct 2018 21:30:45 -0600
> +
> cloud-init (18.4-0ubuntu1~18.04.1) bionic-proposed; urgency=medium
>
>* drop the following cherry-picks now included:
> diff --git a/debian/cloud-init.postinst b/debian/cloud-init.postinst
> index f88d1c5..11ae75b 100644
> --- a/debian/cloud-init.postinst
> +++ b/debian/cloud-init.postinst
> @@ -206,6 +206,20 @@ cleanup_lp1552999() {
> "$edir/cloud-init-local.service" "$edir/cloud-init.service"
> }
>
> +# Old instance-data.json was root read-only, new is redacted world-readable
> +# Also add instance-data-sensitive.json that is root read-only
> +regenerate_instance_data_json_on_upgrade() {
> +if [ -f /run/cloud-init/instance-data.json -a
> + ! -f /run/cloud-init/instance-data-sensitive.json ]; then
> +# this is an upgraded system with old instance-data.json file
> +echo "Updating /run/cloud-init/instance-data.json"
> +python3 -c '
> +from cloudinit.stages import _pkl_load
> +pickled_ds = _pkl_load("/var/lib/cloud/instance/obj.pkl")
> +pickled_ds.persist_instance_data()'
my only issue is that how are we ever going to remember in upstream that ubuntu
does this set of function calls?
we can/should also check that the version being upgraded from is older than one
that should have written this stuff.
if dpkg --compare-versions "$oldver" ...
> +fi
> +}
> +
> disable_network_config_on_upgrade() {
> local oldver="$1" last_without_net="0.7.7~bzr1182-0ubuntu1"
> if [ ! -f /var/lib/cloud/instance/obj.pkl ]; then
--
https://code.launchpad.net/~chad.smith/cloud-init/+git/cloud-init/+merge/356897
Your team cloud-init commiters is requested to review the proposed merge of
~chad.smith/cloud-init:ubuntu/bionic into cloud-init:ubuntu/bionic.
___
Mailing list: https://launchpad.net/~cloud-init-dev
Post to : cloud-init-dev@lists.launchpad.net
Unsubscribe : https://launchpad.net/~cloud-init-dev
More help : https://help.launchpad.net/ListHelp
[Cloud-init-dev] [Merge] ~smoser/cloud-init:fix/1798117-allow-toplevel-network-in-network-config into cloud-init:master
Scott Moser has proposed merging
~smoser/cloud-init:fix/1798117-allow-toplevel-network-in-network-config into
cloud-init:master.
Commit message:
NoCloud: Allow top level 'network' key in network-config.
NoCloud's 'network-config' file was originally expected to contain
network configuration without the top level 'network' key. This was
because the file was named 'network-config' so specifying 'network'
seemed redundant.
However, JuJu is currently providing a top level 'network' config when
it tries to disable networking ({"network": {"config": "disabled"}).
Other users have also been surprised/confused by the fact that
a network config in /etc/cloud/cloud.cfg.d/network.cfg differed from
what was expected in 'network-config'.
LP: #1798117
Requested reviews:
cloud-init commiters (cloud-init-dev)
Related bugs:
Bug #1798117 in cloud-init: "juju sends "network" top level key to
user.network-config in lxd containers"
https://bugs.launchpad.net/cloud-init/+bug/1798117
For more details, see:
https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/356850
see commit message
--
Your team cloud-init commiters is requested to review the proposed merge of
~smoser/cloud-init:fix/1798117-allow-toplevel-network-in-network-config into
cloud-init:master.
diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py
index 9010f06..8b4895b 100644
--- a/cloudinit/sources/DataSourceNoCloud.py
+++ b/cloudinit/sources/DataSourceNoCloud.py
@@ -311,6 +311,33 @@ def parse_cmdline_data(ds_id, fill, cmdline=None):
return True
+def _maybe_remove_top_network(cfg):
+"""If network-config contains top level 'network' key, then remove it.
+
+Some providers of network configuration may provide a top level
+'network' key (LP: #1798117) even though it is not necessary.
+
+Be friendly and remove it if it really seems so.
+
+Return the original value if no change or the updated value if changed."""
+nullval = object()
+network_val = cfg.get('network', nullval)
+if network_val is nullval:
+return cfg
+bmsg = 'Top level network key in network-config %s: %s'
+if not isinstance(network_val, dict):
+LOG.debug(bmsg, "was not a dict", cfg)
+return cfg
+if len(list(cfg.keys())) != 1:
+LOG.debug(bmsg, "had multiple top level keys", cfg)
+return cfg
+if not ('config' in network_val or 'version' in network_val):
+LOG.debug(bmsg, "but missing 'config' and 'version'", cfg)
+return cfg
+LOG.debug(bmsg, "fixed by removing shifting network.", cfg)
+return network_val
+
+
def _merge_new_seed(cur, seeded):
ret = cur.copy()
@@ -320,7 +347,8 @@ def _merge_new_seed(cur, seeded):
ret['meta-data'] = util.mergemanydict([cur['meta-data'], newmd])
if seeded.get('network-config'):
-ret['network-config'] = util.load_yaml(seeded['network-config'])
+ret['network-config'] = _maybe_remove_top_network(
+util.load_yaml(seeded.get('network-config')))
if 'user-data' in seeded:
ret['user-data'] = seeded['user-data']
diff --git a/tests/unittests/test_datasource/test_nocloud.py b/tests/unittests/test_datasource/test_nocloud.py
index b6468b6..848876b 100644
--- a/tests/unittests/test_datasource/test_nocloud.py
+++ b/tests/unittests/test_datasource/test_nocloud.py
@@ -1,7 +1,10 @@
# This file is part of cloud-init. See LICENSE file for license information.
from cloudinit import helpers
-from cloudinit.sources import DataSourceNoCloud
+from cloudinit.sources.DataSourceNoCloud import (
+DataSourceNoCloud as dsNoCloud,
+_maybe_remove_top_network,
+parse_cmdline_data)
from cloudinit import util
from cloudinit.tests.helpers import CiTestCase, populate_dir, mock, ExitStack
@@ -40,9 +43,7 @@ class TestNoCloudDataSource(CiTestCase):
'datasource': {'NoCloud': {'fs_label': None}}
}
-ds = DataSourceNoCloud.DataSourceNoCloud
-
-dsrc = ds(sys_cfg=sys_cfg, distro=None, paths=self.paths)
+dsrc = dsNoCloud(sys_cfg=sys_cfg, distro=None, paths=self.paths)
ret = dsrc.get_data()
self.assertEqual(dsrc.userdata_raw, ud)
self.assertEqual(dsrc.metadata, md)
@@ -63,9 +64,7 @@ class TestNoCloudDataSource(CiTestCase):
'datasource': {'NoCloud': {'fs_label': None}}
}
-ds = DataSourceNoCloud.DataSourceNoCloud
-
-dsrc = ds(sys_cfg=sys_cfg, distro=None, paths=self.paths)
+dsrc = dsNoCloud(sys_cfg=sys_cfg, distro=None, paths=self.paths)
self.assertTrue(dsrc.get_data())
self.assertEqual(dsrc.platform_type, 'nocloud')
self.assertEqual(
@@ -73,8 +72,6 @@ class TestNoCloudDataSource(CiTestCase):
def test_fs_label(self, m_is_lxd):
# find_devs_with should not be
[Cloud-init-dev] [Merge] ~smoser/cloud-init:fix/fix-dhclient-hook-down into cloud-init:master
The proposal to merge ~smoser/cloud-init:fix/fix-dhclient-hook-down into cloud-init:master has been updated. Commit message changed to: dhclient-hook: cleanups, tests and fix a bug on 'down' event. I noticed a bug in dhclient_hook on the 'down' event, using 'is' operator rather than '==' (if self.net_action is 'down'). This refactors/simplifies the code a bit for easier testing and adds tests. The reason for the rename of 'action' to 'event' is to just be internally consistent. The word and Namespace 'action' is used by cloud-init main, so it was not really usable here. Also adds a main which can easily be debugged with: CI_DHCP_HOOK_DATA_D=./my.d python -m cloudinit.dhclient_hook up eth0 For more details, see: https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/356428 -- Your team cloud-init commiters is requested to review the proposed merge of ~smoser/cloud-init:fix/fix-dhclient-hook-down into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
[Cloud-init-dev] [Merge] ~chad.smith/cloud-init:cleanup/metadata-cloud-platform into cloud-init:master
The proposal to merge ~chad.smith/cloud-init:cleanup/metadata-cloud-platform into cloud-init:master has been updated. Status: Needs review => Approved For more details, see: https://code.launchpad.net/~chad.smith/cloud-init/+git/cloud-init/+merge/355999 -- Your team cloud-init commiters is requested to review the proposed merge of ~chad.smith/cloud-init:cleanup/metadata-cloud-platform into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
Re: [Cloud-init-dev] [Merge] ~rjschwei/cloud-init:systemdGenTmpl into cloud-init:master
How about this: http://paste.ubuntu.com/p/bbkSPcv7TP/ the primary difference being that setup.py "knows" that the systemd files should be executable. versus that being stored in filesystem. -- https://code.launchpad.net/~rjschwei/cloud-init/+git/cloud-init/+merge/356098 Your team cloud-init commiters is requested to review the proposed merge of ~rjschwei/cloud-init:systemdGenTmpl into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
Re: [Cloud-init-dev] [Merge] ~chad.smith/cloud-init:cleanup/metadata-cloud-platform into cloud-init:master
if manual_cache_clean is set, then we should be careful not to re-crawl
anything on upgrade. This is most likely a case when there was a "config disk"
in the generic sense. Ie, where a datasource provided a disk (cdrom or block
device) and then later pulled it or user formatted it.
Diff comments:
> diff --git a/cloudinit/sources/DataSourceEc2.py
> b/cloudinit/sources/DataSourceEc2.py
> index 968ab3f..28f08e4 100644
> --- a/cloudinit/sources/DataSourceEc2.py
> +++ b/cloudinit/sources/DataSourceEc2.py
> @@ -306,13 +303,11 @@ class DataSourceEc2(sources.DataSource):
> result = None
> no_network_metadata_on_aws = bool(
> 'network' not in self.metadata and
> -self.cloud_platform == Platforms.AWS)
> +self.cloud_name == CloudNames.AWS)
> if no_network_metadata_on_aws:
> LOG.debug("Metadata 'network' not present:"
>" Refreshing stale metadata from prior to upgrade.")
> -util.log_time(
> -logfunc=LOG.debug, msg='Re-crawl of metadata service',
> -func=self._crawl_metadata)
> +self.get_data()
any reason you dropped the time on that?
probably not important, but curious.
>
> # Limit network configuration to only the primary/fallback nic
> iface = self.fallback_interface
> @@ -340,28 +335,32 @@ class DataSourceEc2(sources.DataSource):
> return super(DataSourceEc2, self).fallback_interface
> return self._fallback_interface
>
> -def _crawl_metadata(self):
> +def crawl_metadata(self):
> """Crawl metadata service when available.
>
> -@returns: True on success, False otherwise.
> +@returns: Dictionary of craweled metadata content containing the
> keys:
craweled -> crawled
> + meta-data, user-data and dynamic.
> """
> if not self.wait_for_metadata_service():
> -return False
> +return {}
> api_version = self.get_metadata_api_version()
> +crawled_metadata = {}
> try:
> -self.userdata_raw = ec2.get_instance_userdata(
> +crawled_metadata['user-data'] = ec2.get_instance_userdata(
> api_version, self.metadata_address)
> -self.metadata = ec2.get_instance_metadata(
> +crawled_metadata['meta-data'] = ec2.get_instance_metadata(
> api_version, self.metadata_address)
> -if self.cloud_platform == Platforms.AWS:
> -self.identity = ec2.get_instance_identity(
> -api_version, self.metadata_address).get('document', {})
> +if self.cloud_name == CloudNames.AWS:
> +identity = ec2.get_instance_identity(
> +api_version, self.metadata_address)
> +crawled_metadata['dynamic'] = {'instance-identity': identity}
> except Exception:
> util.logexc(
> LOG, "Failed reading from metadata address %s",
> self.metadata_address)
> -return False
> -return True
> +return {}
> +crawled_metadata['_metadata_api_version'] = api_version
> +return crawled_metadata
>
>
> class DataSourceEc2Local(DataSourceEc2):
> diff --git a/doc/rtd/topics/instancedata.rst b/doc/rtd/topics/instancedata.rst
> index 634e180..9c30bc1 100644
> --- a/doc/rtd/topics/instancedata.rst
> +++ b/doc/rtd/topics/instancedata.rst
> @@ -90,24 +90,39 @@ There are three basic top-level keys:
>
> The standardized keys present:
>
> -+--+---+---+
> -| Key path| Description |
> Examples |
> -+==+===+===+
> -| v1.cloud_name| The name of the cloud provided by metadata|
> aws, openstack, azure,|
> -| | key 'cloud-name' or the cloud-init datasource |
> configdrive, nocloud, |
> -| | name which was discovered.|
> ovf, etc. |
> -+--+---+---+
> -| v1.instance_id | Unique instance_id allocated by the cloud |
> i- |
> -+--+---+---+
> -| v1.local_hostname| The internal or local hostname of the system |
> ip-10-41-41-70, |
> -| | |
> |
> -+--+---+---+
> -| v1.region| The physical region/datacenter in which the |
> us-east-2 |
> -| |
[Cloud-init-dev] [Merge] ~rjschwei/cloud-init:systemdGenTmpl into cloud-init:master
The proposal to merge ~rjschwei/cloud-init:systemdGenTmpl into cloud-init:master has been updated. Commit message changed to: systemd: Render generator from template to account for system differences. The systemd generator used had a hard coded path for the location target file to create. This path does not apply to all distributions. Make the generator and template to have the path set during build time. For more details, see: https://code.launchpad.net/~rjschwei/cloud-init/+git/cloud-init/+merge/356098 -- Your team cloud-init commiters is requested to review the proposed merge of ~rjschwei/cloud-init:systemdGenTmpl into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
[Cloud-init-dev] [Merge] ~chad.smith/cloud-init:cleanup/metadata-cloud-platform into cloud-init:master
The proposal to merge ~chad.smith/cloud-init:cleanup/metadata-cloud-platform into cloud-init:master has been updated. Commit message changed to: instance-data: Add standardized keys platform and subplatform. Refactor ec2. Add the following instance-data.json standardized keys: * v1._beta_keys: List any v1 keys in beta development, e.g. ['subplatform']. * v1.public_ssh_keys: List of any cloud-provided ssh keys for the instance. * v1.platform: String representing the cloud platform api supporting the datasource. For example: 'ec2' for aws, aliyun and brightbox cloud names. * v1.subplatform: String with more details about the source of the metadata consumed. For example, metadata uri, config drive device path or seed directory. To support the new platform and subplatform standardized instance-data, DataSource and its subclasses grew platform and subplatform attributes. The platform attribute defaults to the lowercase string datasource name at self.dsname. This method is overridden in NoCloud, Ec2 and ConfigDrive datasources. The subplatform attribute calls a _get_subplatform method which will return a string containing a simple slug for subplatform type such as metadata, seed-dir or config-drive followed by a detailed uri, device or directory path where the datasource consumed its configuration. As part of this work, DatasourceEC2 methods _get_data and _crawl_metadata have been refactored for a few reasons: - crawl_metadata is now a read-only operation, persisting no attributes on the datasource instance and returns a dictionary of consumed metadata. - crawl_metadata now closely represents the raw stucture of the ec2 metadata consumed, so that end-users can leverage public ec2 metadata documentation where possible. - crawl_metadata adds a '_metadata_api_version' key to the crawled ds.metadata to advertise what version of EC2's api was consumed by cloud-init. - _get_data now does all the processing of crawl_metadata and saves datasource instance attributes userdata_raw, metadata etc. Additional drive-bys: * unit test rework for test_altcloud and test_azure to simplify mocks and make use of existing util and test_helpers functions. For more details, see: https://code.launchpad.net/~chad.smith/cloud-init/+git/cloud-init/+merge/355999 -- Your team cloud-init commiters is requested to review the proposed merge of ~chad.smith/cloud-init:cleanup/metadata-cloud-platform into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
[Cloud-init-dev] [Merge] ~smoser/cloud-init:fix/1796917-ignore-all-zero-macs into cloud-init:master
Scott Moser has proposed merging
~smoser/cloud-init:fix/1796917-ignore-all-zero-macs into cloud-init:master.
Commit message:
net: ignore nics that have "zero" mac address.
Previously we explicitly excluded mac address '00:00:00:00:00:00'.
But then some nics (tunl0 and sit0) ended up having a mac address like
'00:00:00:00'.
The change here just ignores all 00[:00[:00...]].
LP: #1796917
Requested reviews:
cloud-init commiters (cloud-init-dev)
Related bugs:
Bug #1796917 in cloud-images: "cloud-init fails to run on latest cosmic
minimal image"
https://bugs.launchpad.net/cloud-images/+bug/1796917
For more details, see:
https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/356342
see commit message
--
Your team cloud-init commiters is requested to review the proposed merge of
~smoser/cloud-init:fix/1796917-ignore-all-zero-macs into cloud-init:master.
diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py
index f83d368..e7aad35 100644
--- a/cloudinit/net/__init__.py
+++ b/cloudinit/net/__init__.py
@@ -612,7 +612,6 @@ def get_interfaces():
Bridges and any devices that have a 'stolen' mac are excluded."""
ret = []
devs = get_devicelist()
-empty_mac = '00:00:00:00:00:00'
for name in devs:
if not interface_has_own_mac(name):
continue
@@ -624,7 +623,8 @@ def get_interfaces():
# some devices may not have a mac (tun0)
if not mac:
continue
-if mac == empty_mac and name != 'lo':
+# skip nics that have no mac (00:00)
+if name != 'lo' and mac == ':'.join(('00',) * len(mac.split(":"))):
continue
ret.append((name, mac, device_driver(name), device_devid(name)))
return ret
diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py
index 5d9c7d9..8e38373 100644
--- a/tests/unittests/test_net.py
+++ b/tests/unittests/test_net.py
@@ -3339,9 +3339,23 @@ class TestGetInterfacesByMac(CiTestCase):
addnics = ('greptap1', 'lo', 'greptap2')
self.data['macs'].update(dict((k, empty_mac) for k in addnics))
self.data['devices'].update(set(addnics))
+self.data['own_macs'].extend(list(addnics))
ret = net.get_interfaces_by_mac()
self.assertEqual('lo', ret[empty_mac])
+def test_skip_all_zeros(self):
+"""Any mac of 00:... should be skipped."""
+self._mock_setup()
+emac1, emac2, emac4, emac6 = (
+'00', '00:00', '00:00:00:00', '00:00:00:00:00:00')
+addnics = {'empty1': emac1, 'emac2a': emac2, 'emac2b': emac2,
+ 'emac4': emac4, 'emac6': emac6}
+self.data['macs'].update(addnics)
+self.data['devices'].update(set(addnics))
+self.data['own_macs'].extend(addnics.keys())
+ret = net.get_interfaces_by_mac()
+self.assertEqual('lo', ret['00:00:00:00:00:00'])
+
def test_ib(self):
ib_addr = '80:00:00:28:fe:80:00:00:00:00:00:00:00:11:22:03:00:33:44:56'
ib_addr_eth_format = '00:11:22:33:44:56'
___
Mailing list: https://launchpad.net/~cloud-init-dev
Post to : cloud-init-dev@lists.launchpad.net
Unsubscribe : https://launchpad.net/~cloud-init-dev
More help : https://help.launchpad.net/ListHelp
Re: [Cloud-init-dev] [Merge] ~chad.smith/cloud-init:cleanup/metadata-cloud-platform into cloud-init:master
document beta_keys somwhere ?
Diff comments:
> diff --git a/cloudinit/sources/DataSourceAliYun.py
> b/cloudinit/sources/DataSourceAliYun.py
> index 858e082..16abd09 100644
> --- a/cloudinit/sources/DataSourceAliYun.py
> +++ b/cloudinit/sources/DataSourceAliYun.py
> @@ -18,25 +18,17 @@ class DataSourceAliYun(EC2.DataSourceEc2):
> min_metadata_version = '2016-01-01'
> extended_metadata_versions = []
>
> -def __init__(self, sys_cfg, distro, paths):
> -super(DataSourceAliYun, self).__init__(sys_cfg, distro, paths)
> -self.seed_dir = os.path.join(paths.seed_dir, "AliYun")
> -
> def get_hostname(self, fqdn=False, resolve_ip=False,
> metadata_only=False):
> return self.metadata.get('hostname', 'localhost.localdomain')
>
> def get_public_ssh_keys(self):
> return parse_public_keys(self.metadata.get('public-keys', {}))
>
> -@property
> -def cloud_platform(self):
> -if self._cloud_platform is None:
> -if _is_aliyun():
> -self._cloud_platform = EC2.Platforms.ALIYUN
> -else:
> -self._cloud_platform = EC2.Platforms.NO_EC2_METADATA
> -
> -return self._cloud_platform
> +def _get_cloud_name(self):
> +if _is_aliyun():
you dropped the caching of _cloud_platform. was that intended ?
> +return EC2.CloudNames.ALIYUN
> +else:
> +return EC2.CloudNames.NO_EC2_METADATA
>
>
> def _is_aliyun():
> diff --git a/cloudinit/sources/DataSourceAltCloud.py
> b/cloudinit/sources/DataSourceAltCloud.py
> index 8cd312d..5270fda 100644
> --- a/cloudinit/sources/DataSourceAltCloud.py
> +++ b/cloudinit/sources/DataSourceAltCloud.py
> @@ -99,7 +101,14 @@ class DataSourceAltCloud(sources.DataSource):
> 'RHEV', 'VSPHERE' or 'UNKNOWN'
>
> '''
> -
> +if os.path.exists(CLOUD_INFO_FILE):
these changes are hard... as can't test this anywhere.
I think at one poitn ovirt was using this, but I think it might be completely
gone from current use (ie, we could/should drop).
not sure how to figure that out.
> +try:
> +cloud_type = util.load_file(CLOUD_INFO_FILE).strip().upper()
> +except IOError:
> +util.logexc(LOG, 'Unable to access cloud info file at %s.',
> +CLOUD_INFO_FILE)
> +return 'UNKNOWN'
> +return cloud_type
> system_name = util.read_dmi_data("system-product-name")
> if not system_name:
> return 'UNKNOWN'
> diff --git a/cloudinit/sources/DataSourceAzure.py
> b/cloudinit/sources/DataSourceAzure.py
> index 783445e..39391d0 100644
> --- a/cloudinit/sources/DataSourceAzure.py
> +++ b/cloudinit/sources/DataSourceAzure.py
> @@ -351,6 +351,14 @@ class DataSourceAzure(sources.DataSource):
> metadata['public-keys'] = key_value or
> pubkeys_from_crt_files(fp_files)
> return metadata
>
> +def _get_subplatform(self):
> +"""Return the subplatform metadata source details."""
> +if self.seed.startswith('/dev'):
> +subplatform_type = 'config-disk'
> +else:
> +subplatform_type = 'seed-dir'
in reality this willi always say 'config-disk' now.
right?
> +return '%s (%s)' % (subplatform_type, self.seed)
> +
> def crawl_metadata(self):
> """Walk all instance metadata sources returning a dict on success.
>
> diff --git a/cloudinit/sources/DataSourceEc2.py
> b/cloudinit/sources/DataSourceEc2.py
> index 968ab3f..618714b 100644
> --- a/cloudinit/sources/DataSourceEc2.py
> +++ b/cloudinit/sources/DataSourceEc2.py
> @@ -28,18 +28,16 @@ STRICT_ID_PATH = ("datasource", "Ec2", "strict_id")
> STRICT_ID_DEFAULT = "warn"
>
>
> -class Platforms(object):
> -# TODO Rename and move to cloudinit.cloud.CloudNames
> -ALIYUN = "AliYun"
> -AWS = "AWS"
> -BRIGHTBOX = "Brightbox"
> -SEEDED = "Seeded"
I'm wondering about 'aliyun' and 'brightbox' as platforms (and as cloudnames).
they're *not* ec2 api clones (just the metadata service portion). at least
brightbox is not.
dont worry too much about this.
> +class CloudNames(object):
> +ALIYUN = "aliyun"
> +AWS = "aws"
> +BRIGHTBOX = "brightbox"
> # UNKNOWN indicates no positive id. If strict_id is 'warn' or 'false',
> # then an attempt at the Ec2 Metadata service will be made.
> -UNKNOWN = "Unknown"
> +UNKNOWN = "unknown"
> # NO_EC2_METADATA indicates this platform does not have a Ec2 metadata
> # service available. No attempt at the Ec2 Metadata service will be made.
> -NO_EC2_METADATA = "No-EC2-Metadata"
> +NO_EC2_METADATA = "no-ec2-metadata"
>
>
> class DataSourceEc2(sources.DataSource):
> diff --git a/cloudinit/sources/DataSourceOVF.py
> b/cloudinit/sources/DataSourceOVF.py
> index 178ccb0..045291e 100644
> --- a/cloudinit/sources/DataSourceOVF.py
> +++ b/cloudinit/sources/DataSourceOVF.py
>
Re: [Cloud-init-dev] [Merge] ~raharper/cloud-init:feature/cloud-init-hotplug-handler into cloud-init:master
Diff comments:
> diff --git a/cloudinit/event.py b/cloudinit/event.py
> index f7b311f..77ce631 100644
> --- a/cloudinit/event.py
> +++ b/cloudinit/event.py
> @@ -2,16 +2,68 @@
>
> """Classes and functions related to event handling."""
>
> +from cloudinit import log as logging
> +from cloudinit import util
> +
> +
> +LOG = logging.getLogger(__name__)
> +
>
> # Event types which can generate maintenance requests for cloud-init.
> class EventType(object):
> BOOT = "System boot"
> BOOT_NEW_INSTANCE = "New instance first boot"
> +UDEV = "Udev add|change event on net|storage"
>
> # TODO: Cloud-init will grow support for the follow event types:
> -# UDEV
> # METADATA_CHANGE
> # USER_REQUEST
>
> +EventTypeMap = {
> +'boot': EventType.BOOT,
> +'boot-new-instance': EventType.BOOT_NEW_INSTANCE,
> +'udev': EventType.UDEV,
i don't know how i feel about the name 'udev'.
is there a reason you didn't use 'hotplug' ?
> +}
> +
> +# inverted mapping
> +EventNameMap = {v: k for k, v in EventTypeMap.items()}
> +
> +
> +def get_allowed_events(sys_events, ds_events):
> +'''Merge datasource capabilties with system config to determine which
> + update events are allowed.'''
> +
> +# updates:
> +# policy-version: 1
> +# network:
> +# when: [boot-new-instance, boot, udev]
> +# storage:
> +# when: [boot-new-instance, udev]
> +# watch: http://169.254.169.254/metadata/storage_config/
> +
> +LOG.debug('updates: system cfg: %s', sys_events)
> +LOG.debug('updates: datasrc caps: %s', ds_events)
> +
> +updates = util.mergemanydict([sys_events, ds_events])
> +LOG.debug('updates: merged cfg: %s', updates)
> +
> +events = {}
> +for etype in ['network', 'storage']:
> +events[etype] = (
> +set([EventTypeMap.get(evt)
> + for evt in updates.get(etype, {}).get('when', [])
> + if evt in EventTypeMap]))
> +
> +LOG.debug('updates: allowed events: %s', events)
> +return events
> +
> +
> +def get_update_events_config(update_events):
> +'''Return a dictionary of updates config'''
> +evt_cfg = {'policy-version': 1}
> +for scope, events in update_events.items():
> +evt_cfg[scope] = {'when': [EventNameMap[evt] for evt in events]}
> +
> +return evt_cfg
>
> # vi: ts=4 expandtab
> diff --git a/systemd/cloud-init-hotplugd.service
> b/systemd/cloud-init-hotplugd.service
> new file mode 100644
> index 000..6f231cd
> --- /dev/null
> +++ b/systemd/cloud-init-hotplugd.service
> @@ -0,0 +1,11 @@
https://www.linux.com/blog/end-road-systemds-socket-units
"In most cases, the service will have the same name the socket unit, except
with an @ and the service suffix. As your socket unit was echo.socket, your
service will be echo@.service."
that implies this should be cloud-init-hotplugd@.service.
should it be?
> +[Unit]
> +Description=cloud-init hotplug hook daemon
> +After=cloud-init-hotplugd.socket
> +
> +[Service]
> +Type=simple
> +ExecStart=/bin/bash -c 'read args <&3; echo "args=$args"; \
> +exec /usr/bin/cloud-init devel hotplug-hook $args; \
> +exit 0'
> +SyslogIdentifier=cloud-init-hotplugd
> +TimeoutStopSec=5
> diff --git a/tools/hook-hotplug b/tools/hook-hotplug
> new file mode 100755
> index 000..697d3ad
> --- /dev/null
> +++ b/tools/hook-hotplug
> @@ -0,0 +1,26 @@
> +#!/bin/bash
> +# This file is part of cloud-init. See LICENSE file for license information.
> +
> +# This script checks if cloud-init has hotplug hooked and if
> +# cloud-init has finished; if so invoke cloud-init hotplug-hook
> +
> +is_finished() {
> +[ -e /run/cloud-init/result.json ] || return 1
you don't need the 'return 1'
> +}
> +
> +if is_finished; then
> +# only hook pci devices at this time
> +case ${DEVPATH} in
> +/devices/pci*)
> +# open cloud-init's hotplug-hook fifo rw
> +exec 3<>/run/cloud-init/hook-hotplug-cmd
probably dont need to *read* from that file. (isn't that what <> does?)
> +env_params=( \
don't need trailing \ here.
> +--devpath=${DEVPATH}
quote these things.
even though they're in all reality going to not include space or odd chars,
they could.
ie, DEVPATH="/dev/disk/by-name/this is my n ame" would mess it up.
> +--subsystem=${SUBSYSTEM}
> +--udevaction=${ACTION}
> +)
> +# write params to cloud-init's hotplug-hook fifo
> +echo "--hotplug-debug ${env_params[@]}" >&3
> +;;
> +esac
> +fi
--
https://code.launchpad.net/~raharper/cloud-init/+git/cloud-init/+merge/356152
Your team cloud-init commiters is requested to review the proposed merge of
~raharper/cloud-init:feature/cloud-init-hotplug-handler into cloud-init:master.
___
Mailing list: https://launchpad.net/~cloud-init-dev
Re: [Cloud-init-dev] [Merge] ~raharper/cloud-init:feature/cloud-init-hotplug-handler into cloud-init:master
i'm not all the way through, but here are some comments.
Diff comments:
> diff --git a/cloudinit/cmd/devel/hotplug_hook.py
> b/cloudinit/cmd/devel/hotplug_hook.py
> new file mode 100644
> index 000..c24b1ff
> --- /dev/null
> +++ b/cloudinit/cmd/devel/hotplug_hook.py
> @@ -0,0 +1,195 @@
> +# This file is part of cloud-init. See LICENSE file for license information.
> +
> +"""Handle reconfiguration on hotplug events"""
> +import argparse
> +import os
> +import sys
> +
> +from cloudinit.event import EventType
> +from cloudinit import log
> +from cloudinit import reporting
> +from cloudinit.reporting import events
> +from cloudinit import sources
> +from cloudinit.stages import Init
> +from cloudinit.net import read_sys_net_safe
> +from cloudinit.net.network_state import parse_net_config_data
> +
> +
> +LOG = log.getLogger(__name__)
> +NAME = 'hotplug-hook'
> +
> +
> +def get_parser(parser=None):
> +"""Build or extend and arg parser for hotplug-hook utility.
> +
> +@param parser: Optional existing ArgumentParser instance representing the
> +subcommand which will be extended to support the args of this
> utility.
> +
> +@returns: ArgumentParser with proper argument configuration.
> +"""
> +if not parser:
> +parser = argparse.ArgumentParser(prog=NAME, description=__doc__)
> +
> +parser.add_argument("-d", "--devpath",
> +metavar="PATH",
> +help="sysfs path to hotplugged device")
> +parser.add_argument("--hotplug-debug", action='store_true',
> +help='enable debug logging to stderr.')
> +parser.add_argument("-s", "--subsystem",
> +choices=['net', 'block'])
> +parser.add_argument("-u", "--udevaction",
> +choices=['add', 'change', 'remove'])
> +
> +return parser
> +
> +
> +def log_console(msg):
lets figure out how to do this right and get verbosity levels handled.
ie, currently we have lots of debug level things going to go to stderr
"Checking if %s in netconfig" isn't something you'd expect to see
by default.
> +"""Log messages to stderr console and configured logging."""
> +sys.stderr.write(msg + '\n')
> +sys.stderr.flush()
> +LOG.debug(msg)
> +
> +
> +def devpath_to_macaddr(devpath):
> +macaddr = read_sys_net_safe(os.path.basename(devpath), 'address')
> +log_console('Checking if %s in netconfig' % macaddr)
i'm pretty sure this message is not correct.
at least it doesnt match the function name, and even when called from
NetHandler below it doesnt really make sense.
> +return macaddr
> +
> +
> +def in_netconfig(unique_id, netconfig):
> +netstate = parse_net_config_data(netconfig)
> +found = [iface
> + for iface in netstate.iter_interfaces()
> + if iface.get('mac_address') == unique_id]
> +log_console('Ifaces with ID=%s : %s' % (unique_id, found))
> +return len(found) > 0
> +
> +
> +class UeventHandler(object):
> +def __init__(self, ds, devpath, success_fn):
> +self.datasource = ds
> +self.devpath = devpath
> +self.success_fn = success_fn
> +
> +def apply(self):
@abc.abstractmethod
> +raise NotImplemented()
> +
> +@property
> +def config(self):
> +raise NotImplemented()
> +
> +def detect(self, action):
> +raise NotImplemented()
> +
> +def success(self):
> +return self.success_fn()
> +
> +def update(self):
> +self.datasource.update_metadata([EventType.UDEV])
you dont ever read the return value.
as implemented, if it returns False it is always going to return false so there
is no reason to retry (i think).
at least it says "Datasource %s not updated for events: %s".
> +
> +
> +class NetHandler(UeventHandler):
> +def __init__(self, ds, devpath, success_fn):
> +super(NetHandler, self).__init__(ds, devpath, success_fn)
> +self.id = devpath_to_macaddr(self.devpath)
> +
> +def apply(self):
> +return self.datasource.distro.apply_network_config(self.config,
> + bring_up=True)
> +
> +@property
> +def config(self):
> +return self.datasource.network_config
> +
> +def detect(self, action):
> +detect_presence = None
> +if action == 'add':
> +detect_presence = True
> +elif action == 'remove':
> +detect_presence = False
> +else:
> +raise ValueError('Cannot detect unknown action: %s' % action)
> +
> +return detect_presence == in_netconfig(self.id, self.config)
> +
> +
> +UEVENT_HANDLERS = {
> +'net': NetHandler,
> +}
> +
> +SUBSYSTEM_TO_EVENT = {
> +'net': 'network',
> +'block': 'storage',
> +}
> +
> +
> +def handle_args(name, args):
> +log_console('%s called with args=%s' % (NAME, args))
> +hotplug_reporter = events.ReportEventStack(NAME, __doc__,
> +
[Cloud-init-dev] [Merge] ~raharper/cloud-init:feature/cloud-init-hotplug-handler into cloud-init:master
Scott Moser has proposed merging
~raharper/cloud-init:feature/cloud-init-hotplug-handler into cloud-init:master.
Commit message:
wip fixme
Requested reviews:
cloud-init commiters (cloud-init-dev): review-wip
For more details, see:
https://code.launchpad.net/~raharper/cloud-init/+git/cloud-init/+merge/356152
--
Your team cloud-init commiters is requested to review the proposed merge of
~raharper/cloud-init:feature/cloud-init-hotplug-handler into cloud-init:master.
diff --git a/bash_completion/cloud-init b/bash_completion/cloud-init
index 8c25032..0eab40c 100644
--- a/bash_completion/cloud-init
+++ b/bash_completion/cloud-init
@@ -28,7 +28,7 @@ _cloudinit_complete()
COMPREPLY=($(compgen -W "--help --tarfile --include-userdata" -- $cur_word))
;;
devel)
-COMPREPLY=($(compgen -W "--help schema net-convert" -- $cur_word))
+COMPREPLY=($(compgen -W "--help hotplug-hook net-convert schema" -- $cur_word))
;;
dhclient-hook|features)
COMPREPLY=($(compgen -W "--help" -- $cur_word))
@@ -61,6 +61,9 @@ _cloudinit_complete()
--frequency)
COMPREPLY=($(compgen -W "--help instance always once" -- $cur_word))
;;
+hotplug-hook)
+COMPREPLY=($(compgen -W "--help" -- $cur_word))
+;;
net-convert)
COMPREPLY=($(compgen -W "--help --network-data --kind --directory --output-kind" -- $cur_word))
;;
diff --git a/cloudinit/cmd/devel/hotplug_hook.py b/cloudinit/cmd/devel/hotplug_hook.py
new file mode 100644
index 000..c24b1ff
--- /dev/null
+++ b/cloudinit/cmd/devel/hotplug_hook.py
@@ -0,0 +1,195 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+"""Handle reconfiguration on hotplug events"""
+import argparse
+import os
+import sys
+
+from cloudinit.event import EventType
+from cloudinit import log
+from cloudinit import reporting
+from cloudinit.reporting import events
+from cloudinit import sources
+from cloudinit.stages import Init
+from cloudinit.net import read_sys_net_safe
+from cloudinit.net.network_state import parse_net_config_data
+
+
+LOG = log.getLogger(__name__)
+NAME = 'hotplug-hook'
+
+
+def get_parser(parser=None):
+"""Build or extend and arg parser for hotplug-hook utility.
+
+@param parser: Optional existing ArgumentParser instance representing the
+subcommand which will be extended to support the args of this utility.
+
+@returns: ArgumentParser with proper argument configuration.
+"""
+if not parser:
+parser = argparse.ArgumentParser(prog=NAME, description=__doc__)
+
+parser.add_argument("-d", "--devpath",
+metavar="PATH",
+help="sysfs path to hotplugged device")
+parser.add_argument("--hotplug-debug", action='store_true',
+help='enable debug logging to stderr.')
+parser.add_argument("-s", "--subsystem",
+choices=['net', 'block'])
+parser.add_argument("-u", "--udevaction",
+choices=['add', 'change', 'remove'])
+
+return parser
+
+
+def log_console(msg):
+"""Log messages to stderr console and configured logging."""
+sys.stderr.write(msg + '\n')
+sys.stderr.flush()
+LOG.debug(msg)
+
+
+def devpath_to_macaddr(devpath):
+macaddr = read_sys_net_safe(os.path.basename(devpath), 'address')
+log_console('Checking if %s in netconfig' % macaddr)
+return macaddr
+
+
+def in_netconfig(unique_id, netconfig):
+netstate = parse_net_config_data(netconfig)
+found = [iface
+ for iface in netstate.iter_interfaces()
+ if iface.get('mac_address') == unique_id]
+log_console('Ifaces with ID=%s : %s' % (unique_id, found))
+return len(found) > 0
+
+
+class UeventHandler(object):
+def __init__(self, ds, devpath, success_fn):
+self.datasource = ds
+self.devpath = devpath
+self.success_fn = success_fn
+
+def apply(self):
+raise NotImplemented()
+
+@property
+def config(self):
+raise NotImplemented()
+
+def detect(self, action):
+raise NotImplemented()
+
+def success(self):
+return self.success_fn()
+
+def update(self):
+self.datasource.update_metadata([EventType.UDEV])
+
+
+class NetHandler(UeventHandler):
+def __init__(self, ds, devpath, success_fn):
+super(NetHandler, self).__init__(ds, devpath, success_fn)
+self.id = devpath_to_macaddr(self.devpath)
+
+def apply(self):
+
[Cloud-init-dev] [Merge] ~raharper/cloud-init:feature/cloud-init-hotplug-handler into cloud-init:master
The proposal to merge ~raharper/cloud-init:feature/cloud-init-hotplug-handler into cloud-init:master has been updated. Status: Needs review => Work in progress For more details, see: https://code.launchpad.net/~raharper/cloud-init/+git/cloud-init/+merge/356152 -- Your team cloud-init commiters is requested to review the proposed merge of ~raharper/cloud-init:feature/cloud-init-hotplug-handler into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
[Cloud-init-dev] [Merge] ~smoser/cloud-init:fix/cent-6-jinja2 into cloud-init:master
The proposal to merge ~smoser/cloud-init:fix/cent-6-jinja2 into cloud-init:master has been updated. Commit message changed to: Centos 6: Fix jinja rendering on older jinja versions. jinja2.runtime.implements_to_string is not available in older versions of jinja2. This just catches that error separately from other jinja support allowing the code to work. The testcase to recreate was: ./tools/run-container --source-package --package \ --artifacts=./rpm/ centos/6 LP: #1795933 For more details, see: https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/356079 -- Your team cloud-init commiters is requested to review the proposed merge of ~smoser/cloud-init:fix/cent-6-jinja2 into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
Re: [Cloud-init-dev] [Merge] ~smoser/cloud-init:fix/cent-6-jinja2 into cloud-init:master
we have unit tests that test rendering of jinja2, but because jinja2 is not required dep, it will skip the tests if it is not available. the code here is what determined "do we have jinja2". And the tests skip if not JINJA_AVAILABLE -- https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/356079 Your team cloud-init commiters is requested to review the proposed merge of ~smoser/cloud-init:fix/cent-6-jinja2 into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
[Cloud-init-dev] [Merge] ~smoser/cloud-init:fix/cent-6-jinja2 into cloud-init:master
Scott Moser has proposed merging ~smoser/cloud-init:fix/cent-6-jinja2 into cloud-init:master. Commit message: Centos 6: Fix jinja rendering on older jinja versions. jinja2.runtime.implements_to_string is not available in older versions of jinja2. This just catches that error separately from other jinja support allowing the code to work. The testcase to recreate was: ./tools/run-container --source-package --package \ --artifacts=./rpm/ centos/6 Requested reviews: cloud-init commiters (cloud-init-dev) For more details, see: https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/356079 see commit message -- Your team cloud-init commiters is requested to review the proposed merge of ~smoser/cloud-init:fix/cent-6-jinja2 into cloud-init:master. diff --git a/cloudinit/templater.py b/cloudinit/templater.py index b668674..f1034aa 100644 --- a/cloudinit/templater.py +++ b/cloudinit/templater.py @@ -21,16 +21,19 @@ except (ImportError, AttributeError): CHEETAH_AVAILABLE = False try: -from jinja2.runtime import implements_to_string from jinja2 import Template as JTemplate from jinja2 import DebugUndefined as JUndefined JINJA_AVAILABLE = True except (ImportError, AttributeError): -from cloudinit.helpers import identity -implements_to_string = identity JINJA_AVAILABLE = False JUndefined = object +try: +from jinja2.runtime import implements_to_string +except (ImportError, AttributeError): +from cloudinit.helpers import identity +implements_to_string = identity + from cloudinit import log as logging from cloudinit import type_utils as tu from cloudinit import util ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
[Cloud-init-dev] [Merge] ~smoser/cloud-init:release/18.4 into cloud-init:master
Scott Moser has proposed merging ~smoser/cloud-init:release/18.4 into cloud-init:master. Commit message: release 18.4 Bump the version in cloudinit/version.py to be 18.4 and update ChangeLog. LP: #1795741 Requested reviews: cloud-init commiters (cloud-init-dev) Related bugs: Bug #1795741 in cloud-init: "Release 18.4" https://bugs.launchpad.net/cloud-init/+bug/1795741 For more details, see: https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/356019 see commit message -- Your team cloud-init commiters is requested to review the proposed merge of ~smoser/cloud-init:release/18.4 into cloud-init:master. diff --git a/ChangeLog b/ChangeLog index 72c5287..9c043b0 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,86 @@ +18.4: + - add rtd example docs about new standardized keys + - use ds._crawled_metadata instance attribute if set when writing + instance-data.json + - ec2: update crawled metadata. add standardized keys + - tests: allow skipping an entire cloud_test without running. + - tests: disable lxd tests on cosmic + - cii-tests: use unittest2.SkipTest in ntp_chrony due to new deps + - lxd: adjust to snap installed lxd. + - docs: surface experimental doc in instance-data.json + - tests: fix ec2 integration tests. process meta_data instead of meta-data + - Add support for Infiniband network interfaces (IPoIB). [Mark Goddard] + - cli: add cloud-init query subcommand to query instance metadata + - tools/tox-venv: update for new features. + - pylint: ignore warning assignment-from-no-return for _write_network + - stages: Fix bug causing datasource to have incorrect sys_cfg. + (LP: #1787459) + - Remove dead-code _write_network distro implementations. + - net_util: ensure static configs have netmask in translate_network result + [Thomas Berger] (LP: #1792454) + - Fall back to root:root on syslog permissions if other options fail. + [Robert Schweikert] + - tests: Add mock for util.get_hostname. [Robert Schweikert] (LP: #1792799) + - ds-identify: doc string cleanup. + - OpenStack: Support setting mac address on bond. + [Fabian Wiesel] (LP: #1682064) + - bash_completion/cloud-init: fix shell syntax error. + - EphemeralIPv4Network: Be more explicit when adding default route. + (LP: #1792415) + - OpenStack: support reading of newer versions of metdata. + - OpenStack: fix bug causing 'latest' version to be used from network. + (LP: #1792157) + - user-data: jinja template to render instance-data.json in cloud-config + (LP: #1791781) + - config: disable ssh access to a configured user account + - tests: print failed testname instead of docstring upon failure + - tests: Disallow use of util.subp except for where needed. + - sysconfig: refactor sysconfig to accept distro specific templates paths + - Add unit tests for config/cc_ssh.py [Francis Ginther] + - Fix the built-in cloudinit/tests/helpers:skipIf + - read-version: enhance error message [Joshua Powers] + - hyperv_reporting_handler: simplify threaded publisher + - VMWare: Fix a network config bug in vm with static IPv4 and no gateway. + [Pengpeng Sun] (LP: #1766538) + - logging: Add logging config type hyperv for reporting via Azure KVP + [Andy Liu] + - tests: disable other snap test as well [Joshua Powers] + - tests: disable snap, fix write_files binary [Joshua Powers] + - Add datasource Oracle Compute Infrastructure (OCI). + - azure: allow azure to generate network configuration from IMDS per boot. + - Scaleway: Add network configuration to the DataSource [Louis Bouchard] + - docs: Fix example cloud-init analyze command to match output. + [Wesley Gao] + - netplan: Correctly render macaddress on a bonds and bridges when + provided. (LP: #1784699) + - tools: Add 'net-convert' subcommand command to 'cloud-init devel'. + - redhat: remove ssh keys on new instance. (LP: #1781094) + - Use typeset or local in profile.d scripts. (LP: #1784713) + - OpenNebula: Fix null gateway6 [Akihiko Ota] (LP: #1768547) + - oracle: fix detect_openstack to report True on OracleCloud.com DMI data + (LP: #1784685) + - tests: improve LXDInstance trying to workaround or catch bug. + - update_metadata re-config on every boot comments and tests not quite + right [Mike Gerdts] + - tests: Collect build_info from system if available. + - pylint: Fix pylint warnings reported in pylint 2.0.0. + - get_linux_distro: add support for rhel via redhat-release. + - get_linux_distro: add support for centos6 and rawhide flavors of redhat + (LP: #1781229) + - tools: add '--debug' to tools/net-convert.py + - tests: bump the version of paramiko to 2.4.1. + - docs: note in rtd about avoiding /tmp when writing files (LP: #1727876) + - ubuntu,centos,debian: get_linux_distro to align with platform.dist + (LP: #1780481) + - Fix boothook docs on environment variable name (INSTANCE_I -> + INSTANCE_ID) [Marc Tamsky] + - update_metadata: a datasource can support network re-config every boot + - tests: drop salt-minion integration test
[Cloud-init-dev] [Merge] ~smoser/cloud-init:bug/74747-no-deb-src-in-sources-list into cloud-init:master
Scott Moser has proposed merging
~smoser/cloud-init:bug/74747-no-deb-src-in-sources-list into cloud-init:master.
Commit message:
Ubuntu: update sources.list to comment out deb-src entries.
Other installation modes began to comment out the deb-src lines in
/etc/apt/sources.list sometime in 16.04 time frame.
This makes the cloud-init rendered sources.list the same as that
currently present in the lxd images.
The changes here are:
a.) comment out all 'deb-src' lines.
b.) move security to the bottom of the file.
c.) trim trailing white space from 3 comment lines.
LP: #74747
Requested reviews:
cloud-init commiters (cloud-init-dev)
Related bugs:
Bug #74747 in cloud-init: "Default sources.list file has source packages
enabled by default"
https://bugs.launchpad.net/cloud-init/+bug/74747
For more details, see:
https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/356000
see commit message
--
Your team cloud-init commiters is requested to review the proposed merge of
~smoser/cloud-init:bug/74747-no-deb-src-in-sources-list into cloud-init:master.
diff --git a/templates/sources.list.ubuntu.tmpl b/templates/sources.list.ubuntu.tmpl
index d879972..edb92f1 100644
--- a/templates/sources.list.ubuntu.tmpl
+++ b/templates/sources.list.ubuntu.tmpl
@@ -10,30 +10,30 @@
# See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to
# newer versions of the distribution.
deb {{mirror}} {{codename}} main restricted
-deb-src {{mirror}} {{codename}} main restricted
+# deb-src {{mirror}} {{codename}} main restricted
## Major bug fix updates produced after the final release of the
## distribution.
deb {{mirror}} {{codename}}-updates main restricted
-deb-src {{mirror}} {{codename}}-updates main restricted
+# deb-src {{mirror}} {{codename}}-updates main restricted
## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu
## team. Also, please note that software in universe WILL NOT receive any
## review or updates from the Ubuntu security team.
deb {{mirror}} {{codename}} universe
-deb-src {{mirror}} {{codename}} universe
+# deb-src {{mirror}} {{codename}} universe
deb {{mirror}} {{codename}}-updates universe
-deb-src {{mirror}} {{codename}}-updates universe
+# deb-src {{mirror}} {{codename}}-updates universe
-## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu
-## team, and may not be under a free licence. Please satisfy yourself as to
-## your rights to use the software. Also, please note that software in
+## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu
+## team, and may not be under a free licence. Please satisfy yourself as to
+## your rights to use the software. Also, please note that software in
## multiverse WILL NOT receive any review or updates from the Ubuntu
## security team.
deb {{mirror}} {{codename}} multiverse
-deb-src {{mirror}} {{codename}} multiverse
+# deb-src {{mirror}} {{codename}} multiverse
deb {{mirror}} {{codename}}-updates multiverse
-deb-src {{mirror}} {{codename}}-updates multiverse
+# deb-src {{mirror}} {{codename}}-updates multiverse
## N.B. software from this repository may not have been tested as
## extensively as that contained in the main release, although it includes
@@ -41,14 +41,7 @@ deb-src {{mirror}} {{codename}}-updates multiverse
## Also, please note that software in backports WILL NOT receive any review
## or updates from the Ubuntu security team.
deb {{mirror}} {{codename}}-backports main restricted universe multiverse
-deb-src {{mirror}} {{codename}}-backports main restricted universe multiverse
-
-deb {{security}} {{codename}}-security main restricted
-deb-src {{security}} {{codename}}-security main restricted
-deb {{security}} {{codename}}-security universe
-deb-src {{security}} {{codename}}-security universe
-deb {{security}} {{codename}}-security multiverse
-deb-src {{security}} {{codename}}-security multiverse
+# deb-src {{mirror}} {{codename}}-backports main restricted universe multiverse
## Uncomment the following two lines to add software from Canonical's
## 'partner' repository.
@@ -56,3 +49,10 @@ deb-src {{security}} {{codename}}-security multiverse
## respective vendors as a service to Ubuntu users.
# deb http://archive.canonical.com/ubuntu {{codename}} partner
# deb-src http://archive.canonical.com/ubuntu {{codename}} partner
+
+deb {{security}} {{codename}}-security main restricted
+# deb-src {{security}} {{codename}}-security main restricted
+deb {{security}} {{codename}}-security universe
+# deb-src {{security}} {{codename}}-security universe
+deb {{security}} {{codename}}-security multiverse
+# deb-src {{security}} {{codename}}-security multiverse
___
Mailing list: https://launchpad.net/~cloud-init-dev
Post to : cloud-init-dev@lists.launchpad.net
Unsubscribe : https://launchpad.net/~cloud-init-dev
More help : https://help.launchpad.net/ListHelp
[Cloud-init-dev] [Merge] ~smoser/cloud-init:feature/skip-better into cloud-init:master
The proposal to merge ~smoser/cloud-init:feature/skip-better into cloud-init:master has been updated. Commit message changed to: tests: allow skipping an entire cloud_test without running. Individual skipTest or setUp SkipTest will still launch the instance. This allows us to stop the running of the instance so we don't waste cycles or boot systems that are known to fail. Also replace remaining unittest usage in tests/cloud_tests/ with unittest2. For more details, see: https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/355870 -- Your team cloud-init commiters is requested to review the proposed merge of ~smoser/cloud-init:feature/skip-better into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
[Cloud-init-dev] [Merge] ~smoser/cloud-init:feature/skip-better into cloud-init:master
Scott Moser has proposed merging ~smoser/cloud-init:feature/skip-better into
cloud-init:master.
Commit message:
tests: allow skipping an entire cloud_test without running.
Individual skipTest or setUp SkipTest will still launch the instance.
This allows us to stop the running of the instance so we don't
waste cycles or boot systems that are known to fail.
Requested reviews:
cloud-init commiters (cloud-init-dev)
For more details, see:
https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/355870
see commit message
--
Your team cloud-init commiters is requested to review the proposed merge of
~smoser/cloud-init:feature/skip-better into cloud-init:master.
diff --git a/tests/cloud_tests/collect.py b/tests/cloud_tests/collect.py
index 75b5061..642745d 100644
--- a/tests/cloud_tests/collect.py
+++ b/tests/cloud_tests/collect.py
@@ -9,6 +9,7 @@ from cloudinit import util as c_util
from tests.cloud_tests import (config, LOG, setup_image, util)
from tests.cloud_tests.stage import (PlatformComponent, run_stage, run_single)
from tests.cloud_tests import platforms
+from tests.cloud_tests.testcases import base, get_test_class
def collect_script(instance, base_dir, script, script_name):
@@ -63,6 +64,7 @@ def collect_test_data(args, snapshot, os_name, test_name):
res = ({}, 1)
# load test config
+test_name_in = test_name
test_name = config.path_to_name(test_name)
test_config = config.load_test_config(test_name)
user_data = test_config['cloud_config']
@@ -75,6 +77,16 @@ def collect_test_data(args, snapshot, os_name, test_name):
LOG.warning('test config %s is not enabled, skipping', test_name)
return ({}, 0)
+test_class = get_test_class(
+config.name_to_module(test_name_in),
+test_data={'platform': snapshot.platform_name, 'os_name': os_name},
+test_conf=test_config['cloud_config'])
+try:
+test_class.maybeSkipTest()
+except base.SkipTest as s:
+LOG.warning('skipping test config %s: %s', test_name, s)
+return ({}, 0)
+
# if testcase requires a feature flag that the image does not support,
# skip the testcase with a warning
req_features = test_config.get('required_features', [])
diff --git a/tests/cloud_tests/testcases/__init__.py b/tests/cloud_tests/testcases/__init__.py
index bd548f5..d300b12 100644
--- a/tests/cloud_tests/testcases/__init__.py
+++ b/tests/cloud_tests/testcases/__init__.py
@@ -4,7 +4,7 @@
import importlib
import inspect
-import unittest
+import unittest2
from unittest.util import strclass
from cloudinit.util import read_conf
@@ -25,35 +25,48 @@ def discover_tests(test_name):
except NameError:
raise ValueError('no test verifier found at: {}'.format(testmod_name))
-return [mod for name, mod in inspect.getmembers(testmod)
-if inspect.isclass(mod) and base_test in inspect.getmro(mod) and
-getattr(mod, '__test__', True)]
+found = [mod for name, mod in inspect.getmembers(testmod)
+ if (inspect.isclass(mod)
+ and base_test in inspect.getmro(mod)
+ and getattr(mod, '__test__', True))]
+if len(found) != 1:
+raise RuntimeError(
+"Unexpected situation, multiple tests for %s: %s" % (
+test_name, found))
+return found
-def get_suite(test_name, data, conf):
-"""Get test suite with all tests for 'testname'.
-@return_value: a test suite
-"""
-suite = unittest.TestSuite()
-for test_class in discover_tests(test_name):
+def get_test_class(test_name, test_data, test_conf):
+test_class = discover_tests(test_name)[0]
+
+class DynamicTestSubclass(test_class):
-class tmp(test_class):
+_realclass = test_class
+data = test_data
+conf = test_conf
+release_conf = read_conf(config.RELEASES_CONF)['releases']
-_realclass = test_class
+def __str__(self):
+return "%s (%s)" % (self._testMethodName,
+strclass(self._realclass))
-def __str__(self):
-return "%s (%s)" % (self._testMethodName,
-strclass(self._realclass))
+@classmethod
+def setUpClass(cls):
+cls.maybeSkipTest()
-@classmethod
-def setUpClass(cls):
-cls.data = data
-cls.conf = conf
-cls.release_conf = read_conf(config.RELEASES_CONF)['releases']
+return DynamicTestSubclass
-suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(tmp))
+def get_suite(test_name, data, conf):
+"""Get test suite with all tests for 'testname'.
+
+@return_value: a test suite
+"""
+suite = unittest2.TestSuite()
+suite.addTest(
+unittest2.defaultTestLoader.loadTestsFr
[Cloud-init-dev] [Merge] ~smoser/cloud-init:fix/disable-lxd-cosmic-tests-1795036 into cloud-init:master
Scott Moser has proposed merging
~smoser/cloud-init:fix/disable-lxd-cosmic-tests-1795036 into cloud-init:master.
Commit message:
tests: disable lxd tests on cosmic
Skip lxd tests on cosmic for two reasons:
a.) LP: #1795036 - 'lxd init' fails on cosmic kernel.
b.) apt install lxd installs via snap which can be slow
as that will download core snap and lxd.
Requested reviews:
cloud-init commiters (cloud-init-dev)
Related bugs:
Bug #1795036 in linux (Ubuntu): "iptables --list --numeric fails on -virtual
kernel"
https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1795036
For more details, see:
https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/355867
see commit message
--
Your team cloud-init commiters is requested to review the proposed merge of
~smoser/cloud-init:fix/disable-lxd-cosmic-tests-1795036 into cloud-init:master.
diff --git a/tests/cloud_tests/testcases/modules/lxd_bridge.py b/tests/cloud_tests/testcases/modules/lxd_bridge.py
index f6011de..8697ae9 100644
--- a/tests/cloud_tests/testcases/modules/lxd_bridge.py
+++ b/tests/cloud_tests/testcases/modules/lxd_bridge.py
@@ -7,6 +7,15 @@ from tests.cloud_tests.testcases import base
class TestLxdBridge(base.CloudTestCase):
"""Test LXD module."""
+def setUp(self):
+"""Skip on cosmic for two reasons:
+a.) LP: #1795036 - 'lxd init' fails on cosmic kernel.
+b.) apt install lxd installs via snap which can be slow
+as that will download core snap and lxd."""
+if self.os_name == "cosmic":
+raise self.skipTest('Skipping test on cosmic (LP: #1795036).')
+return base.CloudTestCase.setUp(self)
+
def test_lxd(self):
"""Test lxd installed."""
out = self.get_data_file('lxd')
diff --git a/tests/cloud_tests/testcases/modules/lxd_dir.py b/tests/cloud_tests/testcases/modules/lxd_dir.py
index 26a3db3..8bd8c6e 100644
--- a/tests/cloud_tests/testcases/modules/lxd_dir.py
+++ b/tests/cloud_tests/testcases/modules/lxd_dir.py
@@ -7,6 +7,15 @@ from tests.cloud_tests.testcases import base
class TestLxdDir(base.CloudTestCase):
"""Test LXD module."""
+def setUp(self):
+"""Skip on cosmic for two reasons:
+a.) LP: #1795036 - 'lxd init' fails on cosmic kernel.
+b.) apt install lxd installs via snap which can be slow
+as that will download core snap and lxd."""
+if self.os_name == "cosmic":
+raise self.skipTest('Skipping test on cosmic (LP: #1795036).')
+return base.CloudTestCase.setUp(self)
+
def test_lxd(self):
"""Test lxd installed."""
out = self.get_data_file('lxd')
___
Mailing list: https://launchpad.net/~cloud-init-dev
Post to : cloud-init-dev@lists.launchpad.net
Unsubscribe : https://launchpad.net/~cloud-init-dev
More help : https://help.launchpad.net/ListHelp
Re: [Cloud-init-dev] [Merge] ~chad.smith/cloud-init:cleanup/integration-chrony-unittest2-skiptest into cloud-init:master
assuming it passes c-i and the failing test passes then looksk good to me. -- https://code.launchpad.net/~chad.smith/cloud-init/+git/cloud-init/+merge/355803 Your team cloud-init commiters is requested to review the proposed merge of ~chad.smith/cloud-init:cleanup/integration-chrony-unittest2-skiptest into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
Re: [Cloud-init-dev] [Merge] ~glasnt/cloud-init:topic/udevadm-sp into cloud-init:master
Hi, Thank you for contributing to cloud-init. To contribute, you must sign the Canonical Contributor License Agreement (CLA) [1]. If you have already signed it as an individual, your Launchpad user will be listed in the contributor-agreement-canonical launchpad group [2]. Unfortunately there is no easy way to check if an organization or company you are doing work for has signed. If you are unsure or have questions, email scott.mo...@canonical.com or ping smoser in #cloud-init channel via freenode. For information on how to sign, please see the HACKING document [3]. Thanks again, and please feel free to reach out with any questions. – [1] http://www.canonical.com/contributors [2] https://launchpad.net/~contributor-agreement-canonical/+members [3] http://cloudinit.readthedocs.io/en/latest/topics/hacking.html -- https://code.launchpad.net/~glasnt/cloud-init/+git/cloud-init/+merge/355287 Your team cloud-init commiters is requested to review the proposed merge of ~glasnt/cloud-init:topic/udevadm-sp into cloud-init:master. ___ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp
[Cloud-init-dev] [Merge] ~smoser/cloud-init:fix/lxd-may-come-in-snap into cloud-init:master
Scott Moser has proposed merging ~smoser/cloud-init:fix/lxd-may-come-in-snap
into cloud-init:master.
Commit message:
lxd: adjust to snap installed lxd.
Relax expectation on path to lxc and lxd. The deb path still does
install them in /usr/bin/ but that is overlay pedantic.
Add a 'lxd waitready' (present since lxd 0.5) to wait until lxd
is ready before operating on it.
Requested reviews:
cloud-init commiters (cloud-init-dev)
For more details, see:
https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/355730
see commit message
--
Your team cloud-init commiters is requested to review the proposed merge of
~smoser/cloud-init:fix/lxd-may-come-in-snap into cloud-init:master.
diff --git a/cloudinit/config/cc_lxd.py b/cloudinit/config/cc_lxd.py
index a604825..f9f5aab 100644
--- a/cloudinit/config/cc_lxd.py
+++ b/cloudinit/config/cc_lxd.py
@@ -104,6 +104,7 @@ def handle(name, cfg, cloud, log, args):
'network_address', 'network_port', 'storage_backend',
'storage_create_device', 'storage_create_loop',
'storage_pool', 'trust_password')
+util.subp(['lxd', 'waitready'])
cmd = ['lxd', 'init', '--auto']
for k in init_keys:
if init_cfg.get(k):
diff --git a/tests/cloud_tests/testcases/modules/lxd_bridge.py b/tests/cloud_tests/testcases/modules/lxd_bridge.py
index c0262ba..f6011de 100644
--- a/tests/cloud_tests/testcases/modules/lxd_bridge.py
+++ b/tests/cloud_tests/testcases/modules/lxd_bridge.py
@@ -10,12 +10,12 @@ class TestLxdBridge(base.CloudTestCase):
def test_lxd(self):
"""Test lxd installed."""
out = self.get_data_file('lxd')
-self.assertIn('/usr/bin/lxd', out)
+self.assertIn('/lxd', out)
def test_lxc(self):
"""Test lxc installed."""
out = self.get_data_file('lxc')
-self.assertIn('/usr/bin/lxc', out)
+self.assertIn('/lxc', out)
def test_bridge(self):
"""Test bridge config."""
diff --git a/tests/cloud_tests/testcases/modules/lxd_dir.py b/tests/cloud_tests/testcases/modules/lxd_dir.py
index 1495674..26a3db3 100644
--- a/tests/cloud_tests/testcases/modules/lxd_dir.py
+++ b/tests/cloud_tests/testcases/modules/lxd_dir.py
@@ -10,11 +10,11 @@ class TestLxdDir(base.CloudTestCase):
def test_lxd(self):
"""Test lxd installed."""
out = self.get_data_file('lxd')
-self.assertIn('/usr/bin/lxd', out)
+self.assertIn('/lxd', out)
def test_lxc(self):
"""Test lxc installed."""
out = self.get_data_file('lxc')
-self.assertIn('/usr/bin/lxc', out)
+self.assertIn('/lxc', out)
# vi: ts=4 expandtab
___
Mailing list: https://launchpad.net/~cloud-init-dev
Post to : cloud-init-dev@lists.launchpad.net
Unsubscribe : https://launchpad.net/~cloud-init-dev
More help : https://help.launchpad.net/ListHelp
