URL: https://github.com/freeipa/freeipa/pull/393 Author: MartinBasti Title: #393: [Py3] allow to run wsgi - part1 Action: synchronized
To pull the PR as Git branch: git remote add ghfreeipa https://github.com/freeipa/freeipa git fetch ghfreeipa pull/393/head:pr393 git checkout pr393
From 7916e9756da15bbeb06256101b8316c5e8dc9f80 Mon Sep 17 00:00:00 2001 From: Martin Basti <mba...@redhat.com> Date: Wed, 11 Jan 2017 16:54:25 +0100 Subject: [PATCH 01/15] py3: session.py decode server name to str This fix is temporal because Memcache will be removed soon, so it is more workaround than fix https://fedorahosted.org/freeipa/ticket/4985 --- ipaserver/session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ipaserver/session.py b/ipaserver/session.py index 85deb15..020dcc1 100644 --- a/ipaserver/session.py +++ b/ipaserver/session.py @@ -828,7 +828,7 @@ def get_server_statistics(self): result = {} stats = self.mc.get_stats() for server in stats: - match = self.mc_server_stat_name_re.search(server[0]) + match = self.mc_server_stat_name_re.search(server[0].decode()) if match: name = match.group(1) result[name] = server[1] From be4ab4f89262f33d71b4bf29937deef09e2527e8 Mon Sep 17 00:00:00 2001 From: Martin Basti <mba...@redhat.com> Date: Wed, 11 Jan 2017 17:13:52 +0100 Subject: [PATCH 02/15] py3: rpcserver: decode input because json requires string json library parses string so input must be decoded https://fedorahosted.org/freeipa/ticket/4985 --- ipaserver/rpcserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ipaserver/rpcserver.py b/ipaserver/rpcserver.py index 1da4ec4..7f800ac 100644 --- a/ipaserver/rpcserver.py +++ b/ipaserver/rpcserver.py @@ -195,7 +195,7 @@ def read_input(environ): length = int(environ.get('CONTENT_LENGTH')) except (ValueError, TypeError): return - return environ['wsgi.input'].read(length) + return environ['wsgi.input'].read(length).decode('utf-8') def params_2_args_options(params): From 11c15490e6ee911d386b94282300a2435b60e822 Mon Sep 17 00:00:00 2001 From: Martin Basti <mba...@redhat.com> Date: Wed, 11 Jan 2017 17:15:49 +0100 Subject: [PATCH 03/15] Py3: Fix undefined variable Variable 'e' has only local scope in except block in Py3 https://fedorahosted.org/freeipa/ticket/4985 --- ipaserver/rpcserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ipaserver/rpcserver.py b/ipaserver/rpcserver.py index 7f800ac..306d085 100644 --- a/ipaserver/rpcserver.py +++ b/ipaserver/rpcserver.py @@ -404,7 +404,7 @@ def wsgi_execute(self, environ): type(self).__name__, principal, name, - type(e).__name__) + type(error).__name__) version = options.get('version', VERSION_WITHOUT_CAPABILITIES) return self.marshal(result, error, _id, version) From 2a8cd79a2291300d26470af2cd27dd197271a632 Mon Sep 17 00:00:00 2001 From: Martin Basti <mba...@redhat.com> Date: Wed, 11 Jan 2017 17:24:16 +0100 Subject: [PATCH 04/15] py3: session: fix r/w ccache data ccache contains binary data, so it should be read and write in binary mode https://fedorahosted.org/freeipa/ticket/4985 --- ipaserver/session.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/ipaserver/session.py b/ipaserver/session.py index 020dcc1..0f3a9ad 100644 --- a/ipaserver/session.py +++ b/ipaserver/session.py @@ -21,6 +21,7 @@ import os import re import time +import io # pylint: disable=import-error from six.moves.urllib.parse import urlparse @@ -1228,9 +1229,8 @@ def load_ccache_data(ccache_name): scheme, name = krb5_parse_ccache(ccache_name) if scheme == 'FILE': root_logger.debug('reading ccache data from file "%s"', name) - src = open(name) - ccache_data = src.read() - src.close() + with io.open(name, "rb") as src: + ccache_data = src.read() return ccache_data else: raise ValueError('ccache scheme "%s" unsupported (%s)', scheme, ccache_name) @@ -1239,9 +1239,8 @@ def bind_ipa_ccache(ccache_data, scheme='FILE'): if scheme == 'FILE': name = _get_krbccache_pathname() root_logger.debug('storing ccache data into file "%s"', name) - dst = open(name, 'w') - dst.write(ccache_data) - dst.close() + with io.open(name, 'wb') as dst: + dst.write(ccache_data) else: raise ValueError('ccache scheme "%s" unsupported', scheme) From 36f849e2c876c6418db99d8cf72b54026d32a18f Mon Sep 17 00:00:00 2001 From: Martin Basti <mba...@redhat.com> Date: Thu, 12 Jan 2017 18:50:56 +0100 Subject: [PATCH 05/15] py3: WSGI executioners must return bytes in list WSGI prints TypeError into error log when IPA doesn't return bytes in list as result https://fedorahosted.org/freeipa/ticket/4985 --- ipaserver/rpcserver.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/ipaserver/rpcserver.py b/ipaserver/rpcserver.py index 306d085..8a18f94 100644 --- a/ipaserver/rpcserver.py +++ b/ipaserver/rpcserver.py @@ -144,7 +144,7 @@ def not_found(self, environ, start_response, url, message): self.info('%s: URL="%s", %s', status, url, message) start_response(status, response_headers) output = _not_found_template % dict(url=escape(url)) - return [output] + return [output.encode('utf-8')] def bad_request(self, environ, start_response, message): """ @@ -157,7 +157,7 @@ def bad_request(self, environ, start_response, message): start_response(status, response_headers) output = _bad_request_template % dict(message=escape(message)) - return [output] + return [output.encode('utf-8')] def internal_error(self, environ, start_response, message): """ @@ -170,7 +170,7 @@ def internal_error(self, environ, start_response, message): start_response(status, response_headers) output = _internal_error_template % dict(message=escape(message)) - return [output] + return [output.encode('utf-8')] def unauthorized(self, environ, start_response, message, reason): """ @@ -185,7 +185,7 @@ def unauthorized(self, environ, start_response, message, reason): start_response(status, response_headers) output = _unauthorized_template % dict(message=escape(message)) - return [output] + return [output.encode('utf-8')] def read_input(environ): """ @@ -427,7 +427,7 @@ def __call__(self, environ, start_response): except Exception: self.exception('WSGI %s.__call__():', self.name) status = HTTP_STATUS_SERVER_ERROR - response = status + response = status.encode('utf-8') headers = [('Content-Type', 'text/plain; charset=utf-8')] session_data = getattr(context, 'session_data', None) @@ -489,7 +489,8 @@ def marshal(self, result, error, _id=None, version=unicode(VERSION), ) response = json_encode_binary(response, version) - return json.dumps(response, sort_keys=True, indent=4) + dump = json.dumps(response, sort_keys=True, indent=4) + return dump.encode('utf-8') def unmarshal(self, data): try: @@ -672,7 +673,7 @@ def __call__(self, environ, start_response): 'xmlserver', user_ccache, environ, start_response, headers) except PublicError as e: status = HTTP_STATUS_SUCCESS - response = status + response = status.encode('utf-8') start_response(status, headers) return self.marshal(None, e) finally: @@ -758,7 +759,8 @@ def marshal(self, result, error, _id=None, if isinstance(result, dict): self.debug('response: entries returned %d', result.get('count', 1)) response = (result,) - return xml_dumps(response, version, methodresponse=True) + dump = xml_dumps(response, version, methodresponse=True) + return dump.encode('utf-8') class jsonserver_session(jsonserver, KerberosSession): @@ -782,7 +784,7 @@ def _on_finalize(self): def need_login(self, start_response): status = '401 Unauthorized' headers = [] - response = '' + response = b'' self.debug('jsonserver_session: %s need login', status) @@ -1252,7 +1254,7 @@ def _on_finalize(self): def need_login(self, start_response): status = '401 Unauthorized' headers = [] - response = '' + response = b'' self.debug('xmlserver_session: %s need login', status) From d19f16d6bbd05360920806d9a82c35dd4639fa83 Mon Sep 17 00:00:00 2001 From: Martin Basti <mba...@redhat.com> Date: Fri, 13 Jan 2017 12:11:19 +0100 Subject: [PATCH 06/15] py3: rpcserver fix undefined variable variable 'e' is valid only in except block in py3, so it must be assigned to different variable for further usage https://fedorahosted.org/freeipa/ticket/4985 --- ipaserver/rpcserver.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ipaserver/rpcserver.py b/ipaserver/rpcserver.py index 8a18f94..45550fb 100644 --- a/ipaserver/rpcserver.py +++ b/ipaserver/rpcserver.py @@ -389,8 +389,9 @@ def wsgi_execute(self, environ): ) # get at least some context of what is going on params = options + error = e if error: - result_string = type(e).__name__ + result_string = type(error).__name__ else: result_string = 'SUCCESS' self.info('[%s] %s: %s(%s): %s', From bd05cfd4855fd24476c7b826fbd114fe03d72980 Mon Sep 17 00:00:00 2001 From: Martin Basti <mba...@redhat.com> Date: Fri, 13 Jan 2017 12:16:28 +0100 Subject: [PATCH 07/15] py3: ipaldap: update encode/decode methods Update encoding/decoding accordingly to work under Py3 Removing functions that were used only once in code and give no real improvements https://fedorahosted.org/freeipa/ticket/4985 --- ipapython/ipaldap.py | 41 +++++++---------------------------------- 1 file changed, 7 insertions(+), 34 deletions(-) diff --git a/ipapython/ipaldap.py b/ipapython/ipaldap.py index 81d8c8d..bb14b45 100644 --- a/ipapython/ipaldap.py +++ b/ipapython/ipaldap.py @@ -69,36 +69,6 @@ DIRMAN_DN = DN(('cn', 'directory manager')) -def unicode_from_utf8(val): - ''' - val is a UTF-8 encoded string, return a unicode object. - ''' - return val.decode('utf-8') - - -def value_to_utf8(val): - ''' - Coerce the val parameter to a UTF-8 encoded string representation - of the val. - ''' - - # If val is not a string we need to convert it to a string - # (specifically a unicode string). Naively we might think we need to - # call str(val) to convert to a string. This is incorrect because if - # val is already a unicode object then str() will call - # encode(default_encoding) returning a str object encoded with - # default_encoding. But we don't want to apply the default_encoding! - # Rather we want to guarantee the val object has been converted to a - # unicode string because from a unicode string we want to explicitly - # encode to a str using our desired encoding (utf-8 in this case). - # - # Note: calling unicode on a unicode object simply returns the exact - # same object (with it's ref count incremented). This means calling - # unicode on a unicode object is effectively a no-op, thus it's not - # inefficient. - - return unicode(val).encode('utf-8') - class _ServerSchema(object): ''' Properties of a schema retrieved from an LDAP server. @@ -877,7 +847,7 @@ def encode(self, val): return 'FALSE' elif isinstance(val, (unicode, six.integer_types, Decimal, DN, Principal)): - return value_to_utf8(val) + return six.text_type(val).encode('utf-8') elif isinstance(val, DNSName): return val.to_text() elif isinstance(val, bytes): @@ -909,9 +879,12 @@ def decode(self, val, attr): elif target_type is unicode: return val.decode('utf-8') elif target_type is datetime.datetime: - return datetime.datetime.strptime(val, LDAP_GENERALIZED_TIME_FORMAT) + return datetime.datetime.strptime( + val.decode('utf-8'), LDAP_GENERALIZED_TIME_FORMAT) elif target_type is DNSName: - return DNSName.from_text(val) + return DNSName.from_text(val.decode('utf-8')) + elif target_type in (DN, Principal): + return target_type(val.decode('utf-8')) else: return target_type(val) except Exception: @@ -923,7 +896,7 @@ def decode(self, val, attr): elif isinstance(val, tuple): return tuple(self.decode(m, attr) for m in val) elif isinstance(val, dict): - dct = dict((unicode_from_utf8(k), self.decode(v, k)) for k, v in val.items()) + dct = dict((k.decode('utf-8'), self.decode(v, k)) for k, v in val.items()) return dct elif val is None: return None From c53d4c0bea967242b5bd4f9128dd2d65d13e1738 Mon Sep 17 00:00:00 2001 From: Martin Basti <mba...@redhat.com> Date: Fri, 13 Jan 2017 12:37:47 +0100 Subject: [PATCH 08/15] py3: get_effective_rights: values passed to ldap must be bytes Values passed to LDAP must be bytes https://fedorahosted.org/freeipa/ticket/4985 --- ipaserver/plugins/ldap2.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ipaserver/plugins/ldap2.py b/ipaserver/plugins/ldap2.py index 25fbfb8..c4b7580 100644 --- a/ipaserver/plugins/ldap2.py +++ b/ipaserver/plugins/ldap2.py @@ -289,7 +289,10 @@ def get_effective_rights(self, dn, attrs_list): principal = getattr(context, 'principal') entry = self.find_entry_by_attr("krbprincipalname", principal, "krbPrincipalAux", base_dn=self.api.env.basedn) - sctrl = [GetEffectiveRightsControl(True, "dn: " + str(entry.dn))] + sctrl = [ + GetEffectiveRightsControl( + True, "dn: {0}".format(entry.dn).encode('utf-8')) + ] self.conn.set_option(_ldap.OPT_SERVER_CONTROLS, sctrl) try: entry = self.get_entry(dn, attrs_list) From 339af52060f9aeefd52184167478fe952d7f7171 Mon Sep 17 00:00:00 2001 From: Martin Basti <mba...@redhat.com> Date: Fri, 13 Jan 2017 12:40:20 +0100 Subject: [PATCH 09/15] py3: can_read: attributelevelrights is already string Remove decode() as it causes error in py3 because the attribute is already string not bytes https://fedorahosted.org/freeipa/ticket/4985 --- ipaserver/plugins/ldap2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ipaserver/plugins/ldap2.py b/ipaserver/plugins/ldap2.py index c4b7580..71c095d 100644 --- a/ipaserver/plugins/ldap2.py +++ b/ipaserver/plugins/ldap2.py @@ -310,7 +310,7 @@ def can_write(self, dn, attr): attrs = self.get_effective_rights(dn, [attr]) if 'attributelevelrights' in attrs: - attr_rights = attrs.get('attributelevelrights')[0].decode('UTF-8') + attr_rights = attrs.get('attributelevelrights')[0] (attr, rights) = attr_rights.split(':') if 'w' in rights: return True From a89cd037e5b2338151b61d0be2a083e0dd17592b Mon Sep 17 00:00:00 2001 From: Martin Basti <mba...@redhat.com> Date: Fri, 13 Jan 2017 13:04:07 +0100 Subject: [PATCH 10/15] Use dict comprehension --- ipapython/ipaldap.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ipapython/ipaldap.py b/ipapython/ipaldap.py index bb14b45..497b947 100644 --- a/ipapython/ipaldap.py +++ b/ipapython/ipaldap.py @@ -896,7 +896,9 @@ def decode(self, val, attr): elif isinstance(val, tuple): return tuple(self.decode(m, attr) for m in val) elif isinstance(val, dict): - dct = dict((k.decode('utf-8'), self.decode(v, k)) for k, v in val.items()) + dct = { + k.decode('utf-8'): self.decode(v, k) for k, v in val.items() + } return dct elif val is None: return None From fc63cb684ecb3700640cfb2064d5e9768f183fd1 Mon Sep 17 00:00:00 2001 From: Martin Basti <mba...@redhat.com> Date: Fri, 13 Jan 2017 14:50:11 +0100 Subject: [PATCH 11/15] Principal: validate type of input parameter Bytes are unsupported and we should raise a TypeError from Principal __init__ method otherwise we get hard to debug result --- ipapython/kerberos.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ipapython/kerberos.py b/ipapython/kerberos.py index 3d3530c..9b02790 100644 --- a/ipapython/kerberos.py +++ b/ipapython/kerberos.py @@ -66,7 +66,12 @@ class Principal(object): Container for the principal name and realm according to RFC 1510 """ def __init__(self, components, realm=None): - if isinstance(components, six.string_types): + if isinstance(components, six.binary_type): + raise TypeError( + "Cannot create a principal object from bytes: {!r}".format( + components) + ) + elif isinstance(components, six.string_types): # parse principal components from realm self.components, self.realm = self._parse_from_text( components, realm) From 86e5442030070b20df50310792f946eab02dd649 Mon Sep 17 00:00:00 2001 From: Martin Basti <mba...@redhat.com> Date: Fri, 13 Jan 2017 18:54:34 +0100 Subject: [PATCH 12/15] py3: fix CSR encoding inside framework csr must be in string because framework excpects only strings, so we have to decode it back https://fedorahosted.org/freeipa/ticket/4985 --- ipaserver/plugins/cert.py | 4 +++- ipaserver/plugins/dogtag.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ipaserver/plugins/cert.py b/ipaserver/plugins/cert.py index d8bfc1c..4d3f1cf 100644 --- a/ipaserver/plugins/cert.py +++ b/ipaserver/plugins/cert.py @@ -804,7 +804,9 @@ def execute(self, csr, all=False, raw=False, **kw): try: # re-serialise to PEM, in case the user-supplied data has # extraneous material that will cause Dogtag to freak out - csr_pem = csr_obj.public_bytes(serialization.Encoding.PEM) + # keep it as string not bytes, it is required later + csr_pem = csr_obj.public_bytes( + serialization.Encoding.PEM).decode('utf-8') result = self.Backend.ra.request_certificate( csr_pem, profile_id, ca_id, request_type=request_type) except errors.HTTPRequestError as e: diff --git a/ipaserver/plugins/dogtag.py b/ipaserver/plugins/dogtag.py index fbfe608..7ea1b56 100644 --- a/ipaserver/plugins/dogtag.py +++ b/ipaserver/plugins/dogtag.py @@ -1634,7 +1634,7 @@ def request_certificate( self.debug('%s.request_certificate()', type(self).__name__) # Call CMS - template = '''<?xml version="1.0" encoding="UTF-8" standalone="yes"?> + template = u'''<?xml version="1.0" encoding="UTF-8" standalone="yes"?> <CertEnrollmentRequest> <ProfileID>{profile}</ProfileID> <Input id="i1"> From 4ea0339b93f0e2f08a3abcd8ecefe08024b9b9c4 Mon Sep 17 00:00:00 2001 From: Martin Basti <mba...@redhat.com> Date: Thu, 19 Jan 2017 14:30:23 +0100 Subject: [PATCH 13/15] py3: fingerprint_hex_sha256: fix encoding/decoding https://fedorahosted.org/freeipa/ticket/4985 --- ipapython/ssh.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ipapython/ssh.py b/ipapython/ssh.py index 57752ae..2edfa8a 100644 --- a/ipapython/ssh.py +++ b/ipapython/ssh.py @@ -192,9 +192,8 @@ def openssh(self): def fingerprint_hex_sha256(self): # OpenSSH trims the trailing '=' of base64 sha256 FP representation - # Using unicode argument converts the result to unicode object - fp = base64.b64encode(sha256(self._key).digest()).rstrip(u'=') - return 'SHA256:{fp}'.format(fp=fp) + fp = base64.b64encode(sha256(self._key).digest()).rstrip(b'=') + return u'SHA256:{fp}'.format(fp=fp.decode('utf-8')) def _fingerprint_dns(self, fpfunc, fptype): if self._keytype == 'ssh-rsa': From c00a509ad803fb694de3bcbf71b840c75d81a53d Mon Sep 17 00:00:00 2001 From: Martin Basti <mba...@redhat.com> Date: Thu, 19 Jan 2017 15:28:15 +0100 Subject: [PATCH 14/15] py3: strip_header: support both bytes and unicode Various method passed various bytes or unicode as parameter https://fedorahosted.org/freeipa/ticket/4985 --- ipalib/x509.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/ipalib/x509.py b/ipalib/x509.py index 13327c1..bc25390 100644 --- a/ipalib/x509.py +++ b/ipalib/x509.py @@ -85,9 +85,16 @@ def strip_header(pem): """ Remove the header and footer from a certificate. """ - s = pem.find("-----BEGIN CERTIFICATE-----") + if isinstance(pem, bytes): + re_begin = b"-----BEGIN CERTIFICATE-----" + re_end = b"-----END CERTIFICATE-----" + else: + re_begin = u"-----BEGIN CERTIFICATE-----" + re_end = u"-----END CERTIFICATE-----" + + s = pem.find(re_begin) if s >= 0: - e = pem.find("-----END CERTIFICATE-----") + e = pem.find(re_end) pem = pem[s+27:e] return pem From 8e8ee0952651a58c7e9cd5aef4b4bc0c7120007e Mon Sep 17 00:00:00 2001 From: Martin Basti <mba...@redhat.com> Date: Thu, 19 Jan 2017 16:11:08 +0100 Subject: [PATCH 15/15] py3: normalize_certificate: support both bytes and unicode https://fedorahosted.org/freeipa/ticket/4985 --- ipalib/x509.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/ipalib/x509.py b/ipalib/x509.py index bc25390..83b9330 100644 --- a/ipalib/x509.py +++ b/ipalib/x509.py @@ -254,13 +254,22 @@ def normalize_certificate(rawcert): rawcert = strip_header(rawcert) - if util.isvalid_base64(rawcert): - try: - dercert = base64.b64decode(rawcert) - except Exception as e: - raise errors.Base64DecodeError(reason=str(e)) - else: + try: + if isinstance(rawcert, bytes): + # base64 must work with utf-8, otherwise it is raw bin certificate + decoded_cert = rawcert.decode('utf-8') + else: + decoded_cert = rawcert + except UnicodeDecodeError: dercert = rawcert + else: + if util.isvalid_base64(decoded_cert): + try: + dercert = base64.b64decode(decoded_cert) + except Exception as e: + raise errors.Base64DecodeError(reason=str(e)) + else: + dercert = rawcert # At this point we should have a DER certificate. # Attempt to decode it.
-- Manage your subscription for the Freeipa-devel mailing list: https://www.redhat.com/mailman/listinfo/freeipa-devel Contribute to FreeIPA: http://www.freeipa.org/page/Contribute/Code
