Ryan Harper has proposed merging ~raharper/cloud-init:fix/fedora-build into cloud-init:master.
Commit message: Add a fedora packaging specfile and adjust tools Fedora has a python3 version of cloud-init downstream available. This branch takes that as a starting point and adjusts it to work with our jinja template format and building from master. This branch also updates the systemd/cloud-init-generator.tmpl file to specify the path to ds-identify with the libexec prefix used on redhat based systems. For non-systemd versions of redhat el. al, this is a non issue, however for python3 systems, it means that the systemd generator does not invoke ds-identify and will keep cloud-init disabled. Requested reviews: Server Team CI bot (server-team-bot): continuous-integration cloud-init commiters (cloud-init-dev) For more details, see: https://code.launchpad.net/~raharper/cloud-init/+git/cloud-init/+merge/368845 -- Your team cloud-init commiters is requested to review the proposed merge of ~raharper/cloud-init:fix/fedora-build into cloud-init:master.
diff --git a/packages/brpm b/packages/brpm index a154ef2..502cfb4 100755 --- a/packages/brpm +++ b/packages/brpm @@ -42,13 +42,14 @@ def run_helper(helper, args=None, strip=True): return stdout -def read_dependencies(distro, requirements_file='requirements.txt'): +def read_dependencies(distro, pyver, requirements_file='requirements.txt'): """Returns the Python package depedencies from requirements.txt files. @returns a tuple of (requirements, test_requirements) """ pkg_deps = run_helper( - 'read-dependencies', args=['--distro', distro]).splitlines() + 'read-dependencies', args=['--distro', distro, + '--python-version', pyver]).splitlines() test_deps = run_helper( 'read-dependencies', args=[ '--requirements-file', 'test-requirements.txt', @@ -83,9 +84,10 @@ def generate_spec_contents(args, version_data, tmpl_fn, top_dir, arc_fn): rpm_upstream_version = version_data['version'] subs['rpm_upstream_version'] = rpm_upstream_version - deps, test_deps = read_dependencies(distro=args.distro) - subs['buildrequires'] = deps + test_deps - subs['requires'] = deps + for pyver in ['2', '3']: + deps, test_deps = read_dependencies(distro=args.distro, pyver=pyver) + subs['buildrequires_py%s' % pyver] = deps + test_deps + subs['requires_py%s' % pyver] = deps if args.boot == 'sysvinit': subs['sysvinit'] = True @@ -108,7 +110,7 @@ def main(): parser.add_argument("-d", "--distro", dest="distro", help="select distro (default: %(default)s)", metavar="DISTRO", default='redhat', - choices=('redhat', 'suse')) + choices=('fedora', 'redhat', 'suse')) parser.add_argument('--srpm', help='Produce a source rpm', action='store_true') @@ -155,6 +157,12 @@ def main(): 'make-tarball', ['--long', '--output=' + real_archive_fn]) print("Archived the code in %r" % (real_archive_fn)) + if args.distro == "fedora": + args.distro = "redhat" + args.variant = "fedora" + else: + args.variant = args.distro + # Form the spec file to be used tmpl_fn = util.abs_join(find_root(), 'packages', args.distro, 'cloud-init.spec.in') @@ -189,7 +197,7 @@ def main(): for rpm_fn in globs: tgt_fn = util.abs_join(os.getcwd(), os.path.basename(rpm_fn)) shutil.move(rpm_fn, tgt_fn) - print("Wrote out %s package %r" % (args.distro, tgt_fn)) + print("Wrote out %s package %r" % (args.variant, tgt_fn)) finally: if workdir is not None: shutil.rmtree(workdir) diff --git a/packages/fedora/cloud-init.spec.in b/packages/fedora/cloud-init.spec.in new file mode 100644 index 0000000..b04b46d --- /dev/null +++ b/packages/fedora/cloud-init.spec.in @@ -0,0 +1,166 @@ +## template: jinja +%{!?python_sitelib: %global python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print get_python_lib()")} +%{!?python_major: %global python_major %(%{__python} -c "import sys; sys.stdout.write('%s' % (sys.version_info.major))")} +%{!?python_minor: %global python_minor %(%{__python} -c "import sys; sys.stdout.write('%s' % (sys.version_info.minor))")} + +%define use_systemd (0%{?fedora} && 0%{?fedora} >= 18) || (0%{?rhel} && 0%{?rhel} >= 7) + +%if %{use_systemd} +%define init_system systemd +%else +%define init_system sysvinit +%endif + +# See: http://www.zarb.org/~jasonc/macros.php +# Or: http://fedoraproject.org/wiki/Packaging:ScriptletSnippets +# Or: http://www.rpm.org/max-rpm/ch-rpm-inside.html + +Name: cloud-init +Version: {{rpm_upstream_version}} +Release: 1{{subrelease}}%{?dist} +Summary: Cloud instance init scripts + +Group: System Environment/Base +License: Dual-licensed GPLv3 or Apache 2.0 +URL: http://launchpad.net/cloud-init + +Source0: {{archive_name}} +BuildArch: noarch +BuildRoot: %{_tmppath} + +BuildRequires: pkgconfig(systemd) +BuildRequires: python3-devel +BuildRequires: python3-setuptools +BuildRequires: systemd + +# For tests +BuildRequires: iproute +BuildRequires: python3-configobj + +# https://bugzilla.redhat.com/show_bug.cgi?id=1695953 +BuildRequires: python3-distro +# https://bugzilla.redhat.com/show_bug.cgi?id=1417029 +BuildRequires: python3-httpretty >= 0.8.14-2 +BuildRequires: python3-jinja2 +BuildRequires: python3-jsonpatch +BuildRequires: python3-jsonschema +BuildRequires: python3-mock +BuildRequires: python3-nose +BuildRequires: python3-oauthlib +BuildRequires: python3-prettytable +BuildRequires: python3-pyserial +BuildRequires: python3-PyYAML +BuildRequires: python3-requests +BuildRequires: python3-six + +Requires: e2fsprogs +Requires: iproute +Requires: libselinux-python3 +Requires: net-tools +Requires: policycoreutils-python3 +Requires: procps +Requires: python3-configobj +# https://bugzilla.redhat.com/show_bug.cgi?id=1695953 +Requires: python3-distro +Requires: python3-jinja2 +Requires: python3-jsonpatch +Requires: python3-jsonschema +Requires: python3-oauthlib +Requires: python3-prettytable +Requires: python3-pyserial +Requires: python3-PyYAML +Requires: python3-requests +Requires: python3-six +Requires: shadow-utils +Requires: util-linux +Requires: xfsprogs + +%{?systemd_requires} + +# System util packages needed +%ifarch %{?ix86} x86_64 ia64 +Requires: dmidecode +%endif + + +%description +Cloud-init is a set of init scripts for cloud instances. Cloud instances +need special scripts to run during initialization to retrieve and install +ssh keys and to let the user run various scripts. + + +%prep +%setup -q -n {{source_name}} + +# Change shebangs +sed -i -e 's|#!/usr/bin/env python|#!/usr/bin/env python3|' \ + -e 's|#!/usr/bin/python|#!/usr/bin/python3|' tools/* cloudinit/ssh_util.py + + +%build +%py3_build + + +%install +%py3_install -- --init-system=systemd + +python3 tools/render-cloudcfg --variant fedora > $RPM_BUILD_ROOT/%{_sysconfdir}/cloud/cloud.cfg + +mkdir -p $RPM_BUILD_ROOT/var/lib/cloud + +mkdir -p $RPM_BUILD_ROOT/%{_sysconfdir}/rsyslog.d +cp -p tools/21-cloudinit.conf $RPM_BUILD_ROOT/%{_sysconfdir}/rsyslog.d/21-cloudinit.conf + +# patch in the full version to version.py +version_pys=$(cd "$RPM_BUILD_ROOT" && find . -name version.py -type f) +[ -n "$version_pys" ] || + { echo "failed to find 'version.py' to patch with version." 1>&2; exit 1; } +( cd "$RPM_BUILD_ROOT" && + sed -i "s,@@PACKAGED_VERSION@@,%{version}-%{release}," $version_pys ) + + +%post +%systemd_post cloud-config.service cloud-config.target cloud-final.service cloud-init.service cloud-init.target cloud-init-local.service + + +%preun +%systemd_preun cloud-config.service cloud-config.target cloud-final.service cloud-init.service cloud-init.target cloud-init-local.service + + +%postun +%systemd_postun cloud-config.service cloud-config.target cloud-final.service cloud-init.service cloud-init.target cloud-init-local.service + + +%files +%license LICENSE LICENSE-Apache2.0 LICENSE-GPLv3 +%doc ChangeLog +%doc doc/* +%config(noreplace) %{_sysconfdir}/cloud/cloud.cfg +%dir %{_sysconfdir}/cloud/cloud.cfg.d +%config(noreplace) %{_sysconfdir}/cloud/cloud.cfg.d/*.cfg +%doc %{_sysconfdir}/cloud/cloud.cfg.d/README +%dir %{_sysconfdir}/cloud/templates +%config(noreplace) %{_sysconfdir}/cloud/templates/* +%dir %{_sysconfdir}/rsyslog.d +%config(noreplace) %{_sysconfdir}/rsyslog.d/21-cloudinit.conf +%{_sysconfdir}/NetworkManager/dispatcher.d/hook-network-manager +%{_sysconfdir}/dhcp/dhclient-exit-hooks.d/hook-dhclient +/lib/udev/rules.d/66-azure-ephemeral.rules +%{_unitdir}/cloud-config.service +%{_unitdir}/cloud-final.service +%{_unitdir}/cloud-init.service +%{_unitdir}/cloud-init-local.service +%{_unitdir}/cloud-config.target +%{_unitdir}/cloud-init.target +/usr/lib/systemd/system-generators/cloud-init-generator +%{python3_sitelib}/* +%{_libexecdir}/%{name} +%{_bindir}/cloud-init* +%{_bindir}/cloud-id +%{_datadir}/bash-completion/completions/cloud-init +%dir /var/lib/cloud + + +%changelog +* Fri Jun 14 2019 Ryan Harper <ryan.har...@canonical.com> +- Adapt Fedora cloud-init.spec for upstream diff --git a/packages/pkg-deps.json b/packages/pkg-deps.json index 72409dd..0470187 100644 --- a/packages/pkg-deps.json +++ b/packages/pkg-deps.json @@ -24,32 +24,43 @@ }, "redhat" : { "build-requires" : [ - "python-devel", - "python-setuptools" + "xfsprogs", + "rpm-build" ], "renames" : { + "httpretty" : { + "3" : "python3-httpretty" + }, "jinja2" : { - "3" : "python34-jinja2" + "3" : "python3-jinja2" }, "jsonschema" : { - "3" : "python34-jsonschema" + "3" : "python3-jsonschema" + }, + "prettytable" : { + "3": "python3-prettytable" }, "pyflakes" : { "2" : "pyflakes", - "3" : "python34-pyflakes" + "3" : "python3-pyflakes" }, "pyyaml" : { "2" : "PyYAML", - "3" : "python34-PyYAML" + "3" : "python3-PyYAML" }, "pyserial" : { - "2" : "pyserial" + "2" : "pyserial", + "3" : "python3-pyserial" }, "requests" : { - "3" : "python34-requests" + "3" : "python3-requests" }, + "setuptools" : { + "2" : "python-setuptools", + "3" : "python3-setuptools" + }, "six" : { - "3" : "python34-six" + "3" : "python3-six" } }, "requires" : [ @@ -59,7 +70,9 @@ "procps", "rsyslog", "shadow-utils", - "sudo" + "sudo", + "util-linux", + "xfsprogs" ] }, "suse" : { diff --git a/packages/redhat/cloud-init.spec.in b/packages/redhat/cloud-init.spec.in index 057a578..aa5287f 100644 --- a/packages/redhat/cloud-init.spec.in +++ b/packages/redhat/cloud-init.spec.in @@ -2,6 +2,8 @@ %{!?python_sitelib: %global python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print get_python_lib()")} %define use_systemd (0%{?fedora} && 0%{?fedora} >= 18) || (0%{?rhel} && 0%{?rhel} >= 7) +%define use_python3 (0%{?fedora} && 0%{?fedora} >= 29) || (0%{?rhel} && 0%{?rhel} >= 8) +%define os_variant %(source /etc/os-release; echo $ID ;) %if %{use_systemd} %define init_system systemd @@ -9,6 +11,11 @@ %define init_system sysvinit %endif +%if %{use_python3} +%define python_major 3 +%define python_minor %(%{__python}3 -c "import sys; %sys.stdout.write(str(sys.version_info.minor))") +%endif + # See: http://www.zarb.org/~jasonc/macros.php # Or: http://fedoraproject.org/wiki/Packaging:ScriptletSnippets # Or: http://www.rpm.org/max-rpm/ch-rpm-inside.html @@ -41,12 +48,29 @@ Requires(post): chkconfig Requires(preun): chkconfig %endif +%if "%{python_major}" == "3" +BuildRequires: python3-devel +BuildRequires: python3-setuptools +%if "%{python_minor}" >= "8" +BuildRequires: python3-distro +%endif +%else +BuildRequires: python-devel +BuildRequires: python-setuptools +%endif + # These are runtime dependencies, but declared as BuildRequires so that # - tests can be run here. # - parts of cloud-init such (setup.py) use these dependencies. -{% for r in requires %} +%if %{use_python3} +{% for r in requires_py3 %} BuildRequires: {{r}} {% endfor %} +%else +{% for r in requires_py2 %} +BuildRequires: {{r}} +{% endfor %} +%endif # System util packages needed %ifarch %{?ix86} x86_64 ia64 @@ -58,11 +82,16 @@ Requires: dmidecode Requires: python-argparse %endif - # Install 'dynamic' runtime reqs from *requirements.txt and pkg-deps.json -{% for r in requires %} +%if %{use_python3} +{% for r in requires_py3 %} +Requires: {{r}} +{% endfor %} +%else +{% for r in requires_py2 %} Requires: {{r}} {% endfor %} +%endif # Custom patches {% for p in patches %} @@ -93,14 +122,35 @@ ssh keys and to let the user run various scripts. %patch{{loop.index0}} -p1 {% endfor %} +%if "%{python_major}" == "3" +# Change shebangs +sed -i -e 's|#!/usr/bin/env python|#!/usr/bin/env python3|' \ + -e 's|#!/usr/bin/python|#!/usr/bin/python3|' tools/* cloudinit/ssh_util.py +%endif + + %build +%if "%{python_major}" == "3" +%py3_build +%else %{__python} setup.py build +%endif + %install +%if "%{python_major}" == "3" +%py3_install -- --init-system=systemd + + +%define variant "--variant=%{os_variant}" +python3 tools/render-cloudcfg %{variant} > $RPM_BUILD_ROOT/%{_sysconfdir}/cloud/cloud.cfg +mkdir -p $RPM_BUILD_ROOT/var/lib/cloud +%else %{__python} setup.py install -O1 \ --skip-build --root $RPM_BUILD_ROOT \ --init-system=%{init_system} +%endif # Note that /etc/rsyslog.d didn't exist by default until F15. # el6 request: https://bugzilla.redhat.com/show_bug.cgi?id=740420 @@ -109,7 +159,11 @@ cp -p tools/21-cloudinit.conf \ $RPM_BUILD_ROOT/%{_sysconfdir}/rsyslog.d/21-cloudinit.conf # Remove the tests +%if "%{python_major}" == "3" +rm -rf $RPM_BUILD_ROOT%{python3_sitelib}/tests +%else rm -rf $RPM_BUILD_ROOT%{python_sitelib}/tests +%endif # Required dirs... mkdir -p $RPM_BUILD_ROOT/%{_sharedstatedir}/cloud @@ -122,11 +176,12 @@ version_pys=$(cd "$RPM_BUILD_ROOT" && find . -name version.py -type f) ( cd "$RPM_BUILD_ROOT" && sed -i "s,@@PACKAGED_VERSION@@,%{version}-%{release}," $version_pys ) + %clean rm -rf $RPM_BUILD_ROOT -%post +%post %if "%{init_system}" == "systemd" if [ $1 -eq 1 ] then @@ -142,8 +197,8 @@ fi /sbin/chkconfig --add %{_initrddir}/cloud-final %endif -%preun +%preun %if "%{init_system}" == "systemd" if [ $1 -eq 0 ] then @@ -166,14 +221,14 @@ then fi %endif -%postun +%postun %if "%{init_system}" == "systemd" /bin/systemctl daemon-reload >/dev/null 2>&1 || : %endif -%files +%files /lib/udev/rules.d/66-azure-ephemeral.rules %if "%{init_system}" == "systemd" @@ -188,7 +243,6 @@ fi %{_sysconfdir}/NetworkManager/dispatcher.d/hook-network-manager %{_sysconfdir}/dhcp/dhclient-exit-hooks.d/hook-dhclient - # Program binaries %{_bindir}/cloud-init* %{_bindir}/cloud-id* @@ -213,4 +267,8 @@ fi %dir %{_sharedstatedir}/cloud # Python code is here... +%if "%{python_major}" == "3" +%{python3_sitelib}/* +%else %{python_sitelib}/* +%endif diff --git a/systemd/cloud-init-generator.tmpl b/systemd/cloud-init-generator.tmpl index cfa5eb5..45efa24 100755 --- a/systemd/cloud-init-generator.tmpl +++ b/systemd/cloud-init-generator.tmpl @@ -82,7 +82,12 @@ default() { } check_for_datasource() { - local ds_rc="" dsidentify="/usr/lib/cloud-init/ds-identify" + local ds_rc="" +{% if variant in ["redhat", "fedora", "centos"] %} + local dsidentify="/usr/libexec/cloud-init/ds-identify" +{% else %} + local dsidentify="/usr/lib/cloud-init/ds-identify" +{% endif %} if [ ! -x "$dsidentify" ]; then debug 1 "no ds-identify in $dsidentify. _RET=$FOUND" return 0 diff --git a/tools/read-dependencies b/tools/read-dependencies index b4656e6..77c2ec5 100755 --- a/tools/read-dependencies +++ b/tools/read-dependencies @@ -23,6 +23,7 @@ DEFAULT_REQUIREMENTS = 'requirements.txt' # Map the appropriate package dir needed for each distro choice DISTRO_PKG_TYPE_MAP = { 'centos': 'redhat', + 'fedora': 'fedora', 'redhat': 'redhat', 'debian': 'debian', 'ubuntu': 'debian', @@ -51,17 +52,41 @@ MAYBE_RELIABLE_YUM_INSTALL = [ """, 'reliable-yum-install'] +MAYBE_RELIABLE_DNF_INSTALL = [ + 'sh', '-c', + """ + error() { echo "$@" 1>&2; } + n=0; max=10; + bcmd="dnf install --downloadonly --assumeyes --setopt=keepcache=1" + while n=$(($n+1)); do + error ":: running $bcmd $* [$n/$max]" + $bcmd "$@" + r=$? + [ $r -eq 0 ] && break + [ $n -ge $max ] && { error "gave up on $bcmd"; exit $r; } + nap=$(($n*5)) + error ":: failed [$r] ($n/$max). sleeping $nap." + sleep $nap + done + error ":: running dnf install --cacheonly --assumeyes $*" + dnf install --cacheonly --assumeyes "$@" + """, + 'reliable-dnf-install'] + + ZYPPER_INSTALL = [ 'zypper', '--non-interactive', '--gpg-auto-import-keys', 'install', '--auto-agree-with-licenses'] DRY_DISTRO_INSTALL_PKG_CMD = { 'centos': ['yum', 'install', '--assumeyes'], + 'fedora': ['dnf', 'install', '--assumeyes'], 'redhat': ['yum', 'install', '--assumeyes'], } DISTRO_INSTALL_PKG_CMD = { 'centos': MAYBE_RELIABLE_YUM_INSTALL, + 'fedora': MAYBE_RELIABLE_DNF_INSTALL, 'redhat': MAYBE_RELIABLE_YUM_INSTALL, 'debian': ['apt', 'install', '-y'], 'ubuntu': ['apt', 'install', '-y'], @@ -73,6 +98,7 @@ DISTRO_INSTALL_PKG_CMD = { # List of base system packages required to enable ci automation CI_SYSTEM_BASE_PKGS = { 'common': ['make', 'sudo', 'tar'], + 'fedora': ['python3-tox'], 'redhat': ['python-tox'], 'centos': ['python-tox'], 'ubuntu': ['devscripts', 'python3-dev', 'libssl-dev', 'tox', 'sbuild'], @@ -192,6 +218,13 @@ def main(distro): else: topd = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + # allow fedora to use redhat, mark it as variant + if args.distro in ['fedora']: + args.variant = args.distro + args.distro = 'redhat' + else: + args.variant = args.distro + if args.test_distro: # Give us all the system deps we need for continuous integration if args.req_files: @@ -234,7 +267,7 @@ def main(distro): else: all_deps = pip_pkg_names if args.install: - pkg_install(all_deps, args.distro, args.test_distro, args.dry_run) + pkg_install(all_deps, args.variant, args.test_distro, args.dry_run) else: print('\n'.join(all_deps)) diff --git a/tools/run-container b/tools/run-container index 1d24e15..114803c 100755 --- a/tools/run-container +++ b/tools/run-container @@ -218,9 +218,10 @@ get_os_info() { { error "Unable to determine OS_NAME/OS_VERSION"; return 1; } } -yum_install() { +_rpm_install() { + install_cmd="$1"; shift; local n=0 max=10 ret - bcmd="yum install --downloadonly --assumeyes --setopt=keepcache=1" + bcmd="$install_cmd install --downloadonly --assumeyes --setopt=keepcache=1" while n=$((n+1)); do error ":: running $bcmd $* [$n/$max]" $bcmd "$@" @@ -231,8 +232,16 @@ yum_install() { error ":: failed [$ret] ($n/$max). sleeping $nap." sleep $nap done - error ":: running yum install --cacheonly --assumeyes $*" - yum install --cacheonly --assumeyes "$@" + error ":: running $install_cmd install --cacheonly --assumeyes $*" + $install_cmd install --cacheonly --assumeyes "$@" +} + +yum_install() { + _rpm_install "yum" "$@" +} + +dnf_install() { + _rpm_install "dnf" "$@" } zypper_install() { @@ -250,6 +259,7 @@ apt_install() { install_packages() { get_os_info || return case "$OS_NAME" in + fedora) dnf_install "$@";; centos) yum_install "$@";; opensuse) zypper_install "$@";; debian|ubuntu) apt_install "$@";; @@ -492,8 +502,15 @@ main() { return } + local distflag="" + case "$OS_NAME" in + centos) distflag="--distro=redhat";; + fedora) distflag="--distro=fedora";; + opensuse) distflag="--distro=suse";; + esac + inside_as_cd "$name" root "$cdir" \ - $pyexe ./tools/read-dependencies "--distro=${OS_NAME}" \ + $pyexe ./tools/read-dependencies "$distflag" \ --test-distro || { errorrc "FAIL: failed to install dependencies with read-dependencies" return @@ -514,18 +531,13 @@ main() { } fi - local build_pkg="" build_srcpkg="" pkg_ext="" distflag="" - case "$OS_NAME" in - centos) distflag="--distro=redhat";; - opensuse) distflag="--distro=suse";; - esac - + local build_pkg="" build_srcpkg="" pkg_ext="" pyver="" case "$OS_NAME" in debian|ubuntu) - build_pkg="./packages/bddeb -d" + build_pkg="./packages/bddeb -d" build_srcpkg="./packages/bddeb -S -d" pkg_ext=".deb";; - centos|opensuse) + centos|fedora|opensuse|redhat) build_pkg="./packages/brpm $distflag" build_srcpkg="./packages/brpm $distflag --srpm" pkg_ext=".rpm";;
_______________________________________________ Mailing list: https://launchpad.net/~cloud-init-dev Post to : cloud-init-dev@lists.launchpad.net Unsubscribe : https://launchpad.net/~cloud-init-dev More help : https://help.launchpad.net/ListHelp