URL: https://github.com/freeipa/freeipa/pull/397 Author: tiran Title: #397: Improve wheel building and provide ipaserver wheel for local testing Action: synchronized
To pull the PR as Git branch: git remote add ghfreeipa https://github.com/freeipa/freeipa git fetch ghfreeipa pull/397/head:pr397 git checkout pr397
From 96f614ea132f61c1d052d831568c8b5e2aa3fa34 Mon Sep 17 00:00:00 2001 From: Christian Heimes <chei...@redhat.com> Date: Tue, 17 Jan 2017 08:49:54 +0100 Subject: [PATCH 1/5] Conditionally import pyhbac The pyhbac module is part of SSSD. It's not available as stand-alone PyPI package. It would take a lot of effort to package it because the code is deeply tight into SSSD. Let's follow the example of other SSSD Python packages and make the import of pyhbac conditionally. It's only necessary for caacl and hbactest plugins. I renamed convert_to_ipa_rule() to _convert_to_ipa_rule() because it does not check for presence of pyhbac package itself. The check is performed earlier in execute(). The prefix indicates that it is an internal function and developers have to think twice before using it in another place. This makes it much easier to install ipaserver with instrumented build of Python with a different ABI or in isolated virtual envs to profile and debug the server. Signed-off-by: Christian Heimes <chei...@redhat.com> --- ipaserver/plugins/caacl.py | 11 ++++++++++- ipaserver/plugins/hbactest.py | 19 ++++++++++++++++--- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/ipaserver/plugins/caacl.py b/ipaserver/plugins/caacl.py index a7817c4..691f4e9 100644 --- a/ipaserver/plugins/caacl.py +++ b/ipaserver/plugins/caacl.py @@ -2,7 +2,6 @@ # Copyright (C) 2015 FreeIPA Contributors see COPYING for license # -import pyhbac import six from ipalib import api, errors, output @@ -17,6 +16,11 @@ from ipalib import _, ngettext from ipapython.dn import DN +try: + import pyhbac +except ImportError: + pyhbac = None + if six.PY3: unicode = str @@ -152,6 +156,11 @@ def _acl_make_rule(principal_type, obj): def acl_evaluate(principal_type, principal, ca_id, profile_id): + if pyhbac is None: + raise errors.ValidationError( + name=_('missing pyhbac'), + error=_('pyhbac is not available on the server.') + ) req = _acl_make_request(principal_type, principal, ca_id, profile_id) acls = api.Command.caacl_find(no_members=False)['result'] rules = [_acl_make_rule(principal_type, obj) for obj in acls] diff --git a/ipaserver/plugins/hbactest.py b/ipaserver/plugins/hbactest.py index 626e894..e156568 100644 --- a/ipaserver/plugins/hbactest.py +++ b/ipaserver/plugins/hbactest.py @@ -29,9 +29,14 @@ except ImportError: _dcerpc_bindings_installed = False -import pyhbac import six +try: + import pyhbac +except ImportError: + pyhbac = None + + if six.PY3: unicode = str @@ -210,7 +215,7 @@ register = Registry() -def convert_to_ipa_rule(rule): +def _convert_to_ipa_rule(rule): # convert a dict with a rule to an pyhbac rule ipa_rule = pyhbac.HbacRule(rule['cn'][0]) ipa_rule.enabled = rule['ipaenabledflag'][0] @@ -309,6 +314,14 @@ def canonicalize(self, host): return host def execute(self, *args, **options): + if pyhbac is None: + raise errors.ValidationError( + name=_('missing pyhbac'), + error=_( + 'pyhbac is not available on the server.' + ) + ) + # First receive all needed information: # 1. HBAC rules (whether enabled or disabled) # 2. Required options are (user, target host, service) @@ -356,7 +369,7 @@ def execute(self, *args, **options): # --disabled will import all disabled rules # --rules will implicitly add the rules from a rule list for rule in hbacset: - ipa_rule = convert_to_ipa_rule(rule) + ipa_rule = _convert_to_ipa_rule(rule) if ipa_rule.name in testrules: ipa_rule.enabled = True rules.append(ipa_rule) From 6d73ae828d7a18b813338cc5a5770a53d0fd29b8 Mon Sep 17 00:00:00 2001 From: Christian Heimes <chei...@redhat.com> Date: Tue, 17 Jan 2017 08:57:33 +0100 Subject: [PATCH 2/5] Add extra_requires for additional dependencies ipaserver did not have extra_requires to state additional dependencies. Signed-off-by: Christian Heimes <chei...@redhat.com> --- ipaserver/setup.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/ipaserver/setup.py b/ipaserver/setup.py index 95ba5eb..acdae5a 100755 --- a/ipaserver/setup.py +++ b/ipaserver/setup.py @@ -60,12 +60,6 @@ "pyldap", "python-nss", "six", - # not available on PyPI - # "python-libipa_hbac", - # "python-sss", - # "python-sss-murmur", - # "python-SSSDConfig", - # "samba-python", ], entry_points={ 'custodia.authorizers': [ @@ -75,4 +69,12 @@ 'IPASecStore = ipaserver.secrets.store:IPASecStore', ], }, + extras_require={ + # These packages are currently not available on PyPI. + "caacl": ["pyhbac"], + "dcerpc": ["samba", "pysss", "pysss_nss_idmap"], + "hbactest": ["pyhbac"], + "install": ["SSSDConfig"], + "trust": ["pysss_murmur", "pysss_nss_idmap"], + } ) From 58785cf53e11a8b1c2cf2a74ca7d168151bb2077 Mon Sep 17 00:00:00 2001 From: Christian Heimes <chei...@redhat.com> Date: Tue, 17 Jan 2017 12:16:25 +0100 Subject: [PATCH 3/5] Add an option to build ipaserver wheels To create a wheel bundle with ipaserver and its dependencies: make wheel_bundle IPA_SERVER_WHEELS=1 To include additional dependencies: make wheel_bundle IPA_EXTRA_WHEELS=ipatests[webui] Signed-off-by: Christian Heimes <chei...@redhat.com> --- Makefile.am | 27 +++++++++++++++++++++++---- configure.ac | 6 ++++++ ipasetup.py.in | 3 ++- 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/Makefile.am b/Makefile.am index 30ad9bb..c3ac252 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,7 +1,11 @@ ACLOCAL_AMFLAGS = -I m4 -IPACLIENT_SUBDIRS = ipaclient ipalib ipapython -SUBDIRS = asn1 util client contrib daemons init install $(IPACLIENT_SUBDIRS) ipaplatform ipaserver ipatests po +IPA_CLIENT_PACKAGES = ipaclient ipalib ipapython +IPA_SERVER_PACKAGES = ipaplatform ipaserver +IPA_TEST_PACKAGES = ipatests +SUBDIRS = asn1 util client contrib daemons init install \ + $(IPA_CLIENT_PACKAGES) $(IPA_SERVER_PACKAGES) $(IPA_TEST_PACKAGES) \ + po MOSTLYCLEANFILES = ipasetup.pyc ipasetup.pyo \ ignore_import_errors.pyc ignore_import_errors.pyo \ @@ -199,6 +203,14 @@ jslint-html: WHEELDISTDIR = $(top_builddir)/dist/wheels WHEELBUNDLEDIR = $(top_builddir)/dist/bundle +@MK_IFEQ@ ($(IPA_SERVER_WHEELS),1) + IPA_WHEEL_PACKAGES @MK_ASSIGN@ $(IPA_CLIENT_PACKAGES) $(IPA_SERVER_PACKAGES) + IPA_OMIT_INSTALL @MK_ASSIGN@ 0 +@MK_ELSE@ + IPA_WHEEL_PACKAGES @MK_ASSIGN@ $(IPA_CLIENT_PACKAGES) + IPA_OMIT_INSTALL @MK_ASSIGN@ 1 +@MK_ENDIF@ + $(WHEELDISTDIR): mkdir -p $(WHEELDISTDIR) @@ -206,12 +218,19 @@ $(WHEELBUNDLEDIR): mkdir -p $(WHEELBUNDLEDIR) bdist_wheel: $(WHEELDISTDIR) - for dir in $(IPACLIENT_SUBDIRS); do \ + rm -f $(foreach item,$(IPA_WHEEL_PACKAGES),$(WHEELDISTDIR)/$(item)*.whl) + export IPA_OMIT_INSTALL=$(IPA_OMIT_INSTALL); \ + for dir in $(IPA_WHEEL_PACKAGES) $(IPA_TEST_PACKAGES); do \ $(MAKE) $(AM_MAKEFLAGS) -C $${dir} $@ || exit 1; \ done wheel_bundle: $(WHEELBUNDLEDIR) bdist_wheel - $(PYTHON) -m pip wheel --wheel-dir $(WHEELBUNDLEDIR) $(WHEELDISTDIR)/*.whl + rm -f $(foreach item,$(IPA_WHEEL_PACKAGES),$(WHEELBUNDLEDIR)/$(item)*.whl) + $(PYTHON) -m pip wheel \ + --find-links $(WHEELDISTDIR) \ + --find-links $(WHEELBUNDLEDIR) \ + --wheel-dir $(WHEELBUNDLEDIR) \ + $(IPA_WHEEL_PACKAGES) $(IPA_EXTRA_WHEELS) .PHONY: strip-po: diff --git a/configure.ac b/configure.ac index 44dc11b..17a7dfe 100644 --- a/configure.ac +++ b/configure.ac @@ -404,6 +404,12 @@ AC_SUBST([GIT_VERSION], [IPA_GIT_VERSION]) # used by Makefile.am for files depending on templates AC_SUBST([CONFIG_STATUS]) +# workaround for syntax clash between make and automake +AC_SUBST([MK_IFEQ], [ifeq]) +AC_SUBST([MK_ELSE], [else]) +AC_SUBST([MK_ENDIF], [endif]) +AC_SUBST([MK_ASSIGN], [=]) + dnl --------------------------------------------------------------------------- dnl Finish dnl --------------------------------------------------------------------------- diff --git a/ipasetup.py.in b/ipasetup.py.in index 915f0ed..8564a72 100644 --- a/ipasetup.py.in +++ b/ipasetup.py.in @@ -29,7 +29,8 @@ class build_py(setuptools_build_py): def finalize_options(self): setuptools_build_py.finalize_options(self) - if 'bdist_wheel' in self.distribution.commands: + omit = os.environ.get('IPA_OMIT_INSTALL', '0') + if omit == '1': distname = self.distribution.metadata.name self.skip_package = '{}.install'.format(distname) log.warn("bdist_wheel: Ignore package: %s", From 612122a86fb764a05b8e332677835e470984a6f3 Mon Sep 17 00:00:00 2001 From: Christian Heimes <chei...@redhat.com> Date: Tue, 17 Jan 2017 12:24:19 +0100 Subject: [PATCH 4/5] Workaround to build python-nss as wheel python-nss 1.0.0 has a packaging bug that binary wheel builds. I have forked the package and releases a private copy of python-nss with a build fix. https://bugzilla.redhat.com/show_bug.cgi?id=1389739 Building wheels for collected packages: python-nss Running setup.py bdist_wheel for python-nss ... error Complete output from command /usr/bin/python -u -c "import setuptools, tokenize;__file__='/tmp/pip-build-GWIC6o/python-nss/setup.py';exec(compile(getattr(tokenize, 'open', open)(__file__).read().replace('\r\n', '\n'), __file__, 'exec'))" bdist_wheel -d /tmp/tmpnbByaMpip-wheel-: compiling with debug invalid command name '/tmp/tmpnbByaMpip-wheel-' ---------------------------------------- Failed building wheel for python-nss Running setup.py clean for python-nss Failed to build python-nss Signed-off-by: Christian Heimes <chei...@redhat.com> --- Makefile.am | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Makefile.am b/Makefile.am index c3ac252..2da752a 100644 --- a/Makefile.am +++ b/Makefile.am @@ -226,10 +226,15 @@ bdist_wheel: $(WHEELDISTDIR) wheel_bundle: $(WHEELBUNDLEDIR) bdist_wheel rm -f $(foreach item,$(IPA_WHEEL_PACKAGES),$(WHEELBUNDLEDIR)/$(item)*.whl) + @ # TODO: python-nss 1.0.0 does not build as wheel, remove the workaround + @ # https://bugzilla.redhat.com/show_bug.cgi?id=1389739 has been + @ # resolved. $(PYTHON) -m pip wheel \ --find-links $(WHEELDISTDIR) \ --find-links $(WHEELBUNDLEDIR) \ --wheel-dir $(WHEELBUNDLEDIR) \ + --find-links https://github.com/tiran/python-nss/releases/tag/v1.0.1 \ + python-nss\>=1.0.1 \ $(IPA_WHEEL_PACKAGES) $(IPA_EXTRA_WHEELS) .PHONY: From c505a319fc0dd24d28a66fdd49d4b277150c0d63 Mon Sep 17 00:00:00 2001 From: Christian Heimes <chei...@redhat.com> Date: Tue, 21 Feb 2017 17:13:12 +0100 Subject: [PATCH 5/5] caacl does not depend on pyhbac Signed-off-by: Christian Heimes <chei...@redhat.com> --- ipaserver/setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ipaserver/setup.py b/ipaserver/setup.py index acdae5a..ddc86d8 100755 --- a/ipaserver/setup.py +++ b/ipaserver/setup.py @@ -71,7 +71,6 @@ }, extras_require={ # These packages are currently not available on PyPI. - "caacl": ["pyhbac"], "dcerpc": ["samba", "pysss", "pysss_nss_idmap"], "hbactest": ["pyhbac"], "install": ["SSSDConfig"],
-- Manage your subscription for the Freeipa-devel mailing list: https://www.redhat.com/mailman/listinfo/freeipa-devel Contribute to FreeIPA: http://www.freeipa.org/page/Contribute/Code