[Freeipa-devel] [PATCH] Don't load the LDAP schema during startup
Loading of the schema is now performed in the first request that requires it. https://fedorahosted.org/freeipa/ticket/583 Jan From 0b1368442254cb738a95e766539fa030fe2504c8 Mon Sep 17 00:00:00 2001 From: Jan Zeleny Date: Tue, 15 Feb 2011 09:37:58 +0100 Subject: [PATCH] Don't load the LDAP schema during startup https://fedorahosted.org/freeipa/ticket/583 --- ipalib/plugins/baseldap.py |6 - ipaserver/plugins/ldap2.py | 49 --- 2 files changed, 32 insertions(+), 23 deletions(-) diff --git a/ipalib/plugins/baseldap.py b/ipalib/plugins/baseldap.py index 6817af413c9c4e4ebf951e933e66449343a7d50a..a3e341172a4cf9c1e861b96b26c2151bcfe93ac0 100644 --- a/ipalib/plugins/baseldap.py +++ b/ipalib/plugins/baseldap.py @@ -379,7 +379,11 @@ class LDAPObject(Object): objectclasses += self.possible_objectclasses # Get list of available attributes for this object for use # in the ACI UI. -attrs = self.api.Backend.ldap2.schema.attribute_types(objectclasses) +schema = self.api.Backend.ldap2.get_schema() +if not schema: +attrs = [] +else: +attrs = schema.attribute_types(objectclasses) attrlist = [] # Go through the MUST first for (oid, attr) in attrs[0].iteritems(): diff --git a/ipaserver/plugins/ldap2.py b/ipaserver/plugins/ldap2.py index b03c8def7416780a6dedf2a0d48358ec09ef9da3..9c689633b879072824dbb4729a89dbc5da3575cb 100644 --- a/ipaserver/plugins/ldap2.py +++ b/ipaserver/plugins/ldap2.py @@ -183,12 +183,6 @@ def get_schema(url, conn=None): return _ldap.schema.SubSchema(schema_entry[1]) -# cache schema when importing module -try: -_schema = get_schema(api.env.ldap_uri) -except AttributeError: -_schema = None - # The UPG setting will be cached the first time a module checks it _upg = None @@ -228,7 +222,6 @@ class ldap2(CrudBackend, Encoder): def __init__(self, shared_instance=True, ldap_uri=None, base_dn=None, schema=None): -global _schema CrudBackend.__init__(self, shared_instance=shared_instance) Encoder.__init__(self) self.encoder_settings.encode_dict_keys = True @@ -248,7 +241,7 @@ class ldap2(CrudBackend, Encoder): self.base_dn = api.env.basedn except AttributeError: self.base_dn = '' -self.schema = schema or _schema +self.schema = schema def __del__(self): if self.isconnected(): @@ -259,7 +252,9 @@ class ldap2(CrudBackend, Encoder): def get_syntax(self, attr, value): if not self.schema: -return None +self.schema = get_schema(self.ldap_uri, self.conn) +if not self.schema: +return None obj = self.schema.get_obj(_ldap.schema.AttributeType, attr) if obj is not None: return obj.syntax @@ -268,7 +263,9 @@ class ldap2(CrudBackend, Encoder): def get_allowed_attributes(self, objectclasses): if not self.schema: -return [] +self.schema = get_schema(self.ldap_uri, self.conn) +if not self.schema: +return [] allowed_attributes = [] for oc in objectclasses: obj = self.schema.get_obj(_ldap.schema.ObjectClass, oc) @@ -285,10 +282,13 @@ class ldap2(CrudBackend, Encoder): If there is a problem loading the schema or the attribute is not in the schema return None """ -if self.schema: -obj = self.schema.get_obj(_ldap.schema.AttributeType, attr) -return obj and obj.single_value -return None +if not self.schema: +self.schema = get_schema(self.ldap_uri, self.conn) +if not self.schema: +return None + +obj = self.schema.get_obj(_ldap.schema.AttributeType, attr) +return obj and obj.single_value @encode_args(2, 3, 'bind_dn', 'bind_pw') def create_connection(self, ccache=None, bind_dn='', bind_pw='', @@ -309,7 +309,6 @@ class ldap2(CrudBackend, Encoder): Extends backend.Connectible.create_connection. """ -global _schema if tls_cacertfile is not None: _ldap.set_option(_ldap.OPT_X_TLS_CACERTFILE, tls_cacertfile) if tls_certfile is not None: @@ -334,10 +333,10 @@ class ldap2(CrudBackend, Encoder): except _ldap.LDAPError, e: _handle_errors(e, **{}) -if self.schema is None and _schema is None: -# explicitly use setattr here so the schema can be set after -# the object is finalized. -object.__setattr__(self, 'schema', get_schema(self.ldap_uri, conn)) +# For now let's say the schema is None (will be loaded later) +# - explicitly use setattr here so the schema can be set after +# the object is finalized. +object.__setattr__(self, 'schema', None)
Re: [Freeipa-devel] [PATCH] Don't load the LDAP schema during startup
Rob Crittenden wrote: > Jan Zelený wrote: > > Rob Crittenden wrote: > >> Jan Zelený wrote: > >>> Loading of the schema is now performed in the first request that > >>> requires it. > >>> > >>> https://fedorahosted.org/freeipa/ticket/583 > >>> > >>> Jan > >> > >> We still need to enforce that we get the schema, some low-level > >> functions depend on it. Also, if the UI doesn't get its aciattrs (which > >> are derived from the schema) then nothing will be editable. > > > >> I'm getting this backtrace if I force no schema by disabling get_schema: > > Ok, I'm sending new version, it should handle these exceptions better and > > the operation should fail if it needs the schema and the schema is not > > available for some reason. > > This breaks the XML-RPC server. I fixed one problem: > --- a/ipaserver/plugins/ldap2.py > +++ b/ipaserver/plugins/ldap2.py > @@ -253,9 +253,10 @@ class ldap2(CrudBackend, Encoder): > > def get_syntax(self, attr, value): > if not self.schema: > -self.schema = get_schema(self.ldap_uri, self.conn) > -if not self.schema: > +schema = get_schema(self.ldap_uri, self.conn) > +if not schema: > return None > +object.__setattr__(self, 'schema', schema) > obj = self.schema.get_obj(_ldap.schema.AttributeType, attr) > if obj is not None: > return obj.syntax > > But simply things like get_entry() return an InternalError now. I'm not > sure where you were going by adding this. > > rob Ok, no problem. It's possible that I simply did a mistake thinking I can do something in Python what is not really possible. About that InternalError: I think raising InternalError when we cannot load the schema to do the decoding is the right thing to do. Do you have a better solution? I thought about returning empty result, but that would mean we have to check the result in every funtction that is calling them and raising InternalError there. -- Thank you Jan Zeleny Red Hat Software Engineer Brno, Czech Republic ___ Freeipa-devel mailing list Freeipa-devel@redhat.com https://www.redhat.com/mailman/listinfo/freeipa-devel
Re: [Freeipa-devel] [PATCH] Don't load the LDAP schema during startup
Jan Zelený wrote: > Rob Crittenden wrote: > > Jan Zelený wrote: > > > Rob Crittenden wrote: > > >> Jan Zelený wrote: > > >>> Loading of the schema is now performed in the first request that > > >>> requires it. > > >>> > > >>> https://fedorahosted.org/freeipa/ticket/583 > > >>> > > >>> Jan > > >> > > >> We still need to enforce that we get the schema, some low-level > > >> functions depend on it. Also, if the UI doesn't get its aciattrs > > >> (which are derived from the schema) then nothing will be editable. > > > > > >> I'm getting this backtrace if I force no schema by disabling get_schema: > > > Ok, I'm sending new version, it should handle these exceptions better > > > and the operation should fail if it needs the schema and the schema is > > > not available for some reason. > > > > This breaks the XML-RPC server. I fixed one problem: > > --- a/ipaserver/plugins/ldap2.py > > +++ b/ipaserver/plugins/ldap2.py > > > > @@ -253,9 +253,10 @@ class ldap2(CrudBackend, Encoder): > > def get_syntax(self, attr, value): > > if not self.schema: > > -self.schema = get_schema(self.ldap_uri, self.conn) > > -if not self.schema: > > +schema = get_schema(self.ldap_uri, self.conn) > > > > +if not schema: > > return None > > > > +object.__setattr__(self, 'schema', schema) > > > > obj = self.schema.get_obj(_ldap.schema.AttributeType, attr) > > > > if obj is not None: > > return obj.syntax > > > > But simply things like get_entry() return an InternalError now. I'm not > > sure where you were going by adding this. > > > > rob > > Ok, no problem. It's possible that I simply did a mistake thinking I can do > something in Python what is not really possible. > > About that InternalError: I think raising InternalError when we cannot load > the schema to do the decoding is the right thing to do. Do you have a > better solution? I thought about returning empty result, but that would > mean we have to check the result in every funtction that is calling them > and raising InternalError there. I'm sending updated patch. I modified the get_syntax() as you suggested and I slightly modified raising that InternalError - currently it isn't raised when results from get_entry() are not required by calling method. Currently I'm running some tests, preliminary results looked ok. -- Thank you Jan Zeleny Red Hat Software Engineer Brno, Czech Republic From bdab2d3b9b8a7397e13acdad9d2809ae355cd5a7 Mon Sep 17 00:00:00 2001 From: Jan Zeleny Date: Tue, 15 Feb 2011 09:37:58 +0100 Subject: [PATCH] Don't load the LDAP schema during startup https://fedorahosted.org/freeipa/ticket/583 --- ipalib/encoder.py | 11 +++-- ipalib/plugins/baseldap.py | 21 - ipalib/plugins/dns.py |2 +- ipalib/plugins/host.py |2 +- ipalib/plugins/permission.py|4 +- ipalib/plugins/sudocmd.py |2 +- ipaserver/install/dsinstance.py |2 +- ipaserver/plugins/ldap2.py | 90 --- 8 files changed, 96 insertions(+), 38 deletions(-) diff --git a/ipalib/encoder.py b/ipalib/encoder.py index f23e5659e848d37db1072ff59aa7e11796b0836c..fbcae1739ee6541bdb989d6d01f8a4c9fb614b62 100644 --- a/ipalib/encoder.py +++ b/ipalib/encoder.py @@ -56,11 +56,12 @@ class Encoder(object): self.encoder_settings = EncoderSettings() def _decode_dict_val(self, key, val): -f = self.encoder_settings.decode_dict_vals_table.get( -self.encoder_settings.decode_dict_vals_table_keygen(key, val) -) +k = self.encoder_settings.decode_dict_vals_table_keygen(key, val) +if k is False: +return False +f = self.encoder_settings.decode_dict_vals_table.get(key) if f: -return val +return f(val) return self.decode(val) def encode(self, var): @@ -155,6 +156,8 @@ class Encoder(object): self.encoder_settings.decode_postprocessor = lambda x: x for (k, v) in dct.iteritems(): dct[k] = self._decode_dict_val(k, v) +if dct[k] is False: +return False if not self.encoder_settings.decode_dict_vals_postprocess: self.encoder_settings.decode_postprocessor = tmp return dct diff --git a/ipalib/plugins/baseldap.py b/ipalib/plugins/baseldap.py index 3cb72d7b09cc8c8a77bd4e594660ee376d668013..adc19fec8ff96eef2750ad08715e9324c1c536c6 100644 --- a/ipalib/plugins/baseldap.py +++ b/ipalib/plugins/baseldap.py @@ -189,6 +189,9 @@ def get_effective_rights(ldap, dn, attrs=None): if attrs is None: attrs = ['*', 'nsaccountlock', 'cospriority'] rights = ldap.get_effective_rights(dn, attrs) +if rights[1] in None: +return None + rdict = {} if 'attributelevelrights'
Re: [Freeipa-devel] [PATCH] Don't load the LDAP schema during startup
Jan Zelený wrote: > Jan Zelený wrote: > > Rob Crittenden wrote: > > > Jan Zelený wrote: > > > > Rob Crittenden wrote: > > > >> Jan Zelený wrote: > > > >>> Loading of the schema is now performed in the first request that > > > >>> requires it. > > > >>> > > > >>> https://fedorahosted.org/freeipa/ticket/583 > > > >>> > > > >>> Jan > > > >> > > > >> We still need to enforce that we get the schema, some low-level > > > >> functions depend on it. Also, if the UI doesn't get its aciattrs > > > >> (which are derived from the schema) then nothing will be editable. > > > >> > > > >> I'm getting this backtrace if I force no schema by disabling > > get_schema: > > > > Ok, I'm sending new version, it should handle these exceptions better > > > > and the operation should fail if it needs the schema and the schema > > > > is not available for some reason. > > > > > > This breaks the XML-RPC server. I fixed one problem: > > > --- a/ipaserver/plugins/ldap2.py > > > +++ b/ipaserver/plugins/ldap2.py > > > > > > @@ -253,9 +253,10 @@ class ldap2(CrudBackend, Encoder): > > > def get_syntax(self, attr, value): > > > if not self.schema: > > > -self.schema = get_schema(self.ldap_uri, self.conn) > > > -if not self.schema: > > > +schema = get_schema(self.ldap_uri, self.conn) > > > > > > +if not schema: > > > return None > > > > > > +object.__setattr__(self, 'schema', schema) > > > > > > obj = self.schema.get_obj(_ldap.schema.AttributeType, attr) > > > > > > if obj is not None: > > > return obj.syntax > > > > > > But simply things like get_entry() return an InternalError now. I'm not > > > sure where you were going by adding this. > > > > > > rob > > > > Ok, no problem. It's possible that I simply did a mistake thinking I can > > do something in Python what is not really possible. > > > > About that InternalError: I think raising InternalError when we cannot > > load the schema to do the decoding is the right thing to do. Do you have > > a better solution? I thought about returning empty result, but that > > would mean we have to check the result in every funtction that is > > calling them and raising InternalError there. > > I'm sending updated patch. I modified the get_syntax() as you suggested and > I slightly modified raising that InternalError - currently it isn't raised > when results from get_entry() are not required by calling method. > Currently I'm running some tests, preliminary results looked ok. self-nack I discovered some issues discovered by internal test suite, I'm working on them Jan ___ Freeipa-devel mailing list Freeipa-devel@redhat.com https://www.redhat.com/mailman/listinfo/freeipa-devel
Re: [Freeipa-devel] [PATCH] Don't load the LDAP schema during startup
Jan Zeleny wrote: > Jan Zelený wrote: > > Jan Zelený wrote: > > > Rob Crittenden wrote: > > > > Jan Zelený wrote: > > > > > Rob Crittenden wrote: > > > > >> Jan Zelený wrote: > > > > >>> Loading of the schema is now performed in the first request that > > > > >>> requires it. > > > > >>> > > > > >>> https://fedorahosted.org/freeipa/ticket/583 > > > > >>> > > > > >>> Jan > > > > >> > > > > >> We still need to enforce that we get the schema, some low-level > > > > >> functions depend on it. Also, if the UI doesn't get its aciattrs > > > > >> (which are derived from the schema) then nothing will be editable. > > > > >> > > > > >> I'm getting this backtrace if I force no schema by disabling > > > > get_schema: > > > > > Ok, I'm sending new version, it should handle these exceptions > > > > > better and the operation should fail if it needs the schema and > > > > > the schema is not available for some reason. > > > > > > > > This breaks the XML-RPC server. I fixed one problem: > > > > --- a/ipaserver/plugins/ldap2.py > > > > +++ b/ipaserver/plugins/ldap2.py > > > > > > > > @@ -253,9 +253,10 @@ class ldap2(CrudBackend, Encoder): > > > > def get_syntax(self, attr, value): > > > > if not self.schema: > > > > -self.schema = get_schema(self.ldap_uri, self.conn) > > > > -if not self.schema: > > > > +schema = get_schema(self.ldap_uri, self.conn) > > > > > > > > +if not schema: > > > > return None > > > > > > > > +object.__setattr__(self, 'schema', schema) > > > > > > > > obj = self.schema.get_obj(_ldap.schema.AttributeType, attr) > > > > > > > > if obj is not None: > > > > return obj.syntax > > > > > > > > But simply things like get_entry() return an InternalError now. I'm > > > > not sure where you were going by adding this. > > > > > > > > rob > > > > > > Ok, no problem. It's possible that I simply did a mistake thinking I > > > can do something in Python what is not really possible. > > > > > > About that InternalError: I think raising InternalError when we cannot > > > load the schema to do the decoding is the right thing to do. Do you > > > have a better solution? I thought about returning empty result, but > > > that would mean we have to check the result in every funtction that is > > > calling them and raising InternalError there. > > > > I'm sending updated patch. I modified the get_syntax() as you suggested > > and I slightly modified raising that InternalError - currently it isn't > > raised when results from get_entry() are not required by calling method. > > Currently I'm running some tests, preliminary results looked ok. > > self-nack > > I discovered some issues discovered by internal test suite, I'm working on > them > > Jan Ok, everything is solved, I'm sending final version of the patch in the attachment. But I still think this should go to 2.1, since it's quite extensive patch in the core of IPA server and it has potential to break many things. Jan From 8ce877994e9122d89f842e358f38005eb980b632 Mon Sep 17 00:00:00 2001 From: Jan Zeleny Date: Tue, 15 Feb 2011 09:37:58 +0100 Subject: [PATCH] Don't load the LDAP schema during startup https://fedorahosted.org/freeipa/ticket/583 --- ipalib/encoder.py | 12 +++-- ipalib/plugins/baseldap.py | 21 - ipalib/plugins/dns.py |2 +- ipalib/plugins/host.py |2 +- ipalib/plugins/permission.py|4 +- ipalib/plugins/sudocmd.py |2 +- ipaserver/install/dsinstance.py |2 +- ipaserver/plugins/ldap2.py | 92 +++--- 8 files changed, 97 insertions(+), 40 deletions(-) diff --git a/ipalib/encoder.py b/ipalib/encoder.py index f23e5659e848d37db1072ff59aa7e11796b0836c..762be9c096a4b7cbea03ecfcc82d03789951c0b5 100644 --- a/ipalib/encoder.py +++ b/ipalib/encoder.py @@ -56,9 +56,10 @@ class Encoder(object): self.encoder_settings = EncoderSettings() def _decode_dict_val(self, key, val): -f = self.encoder_settings.decode_dict_vals_table.get( -self.encoder_settings.decode_dict_vals_table_keygen(key, val) -) +k = self.encoder_settings.decode_dict_vals_table_keygen(key, val) +if k is False: +return False +f = self.encoder_settings.decode_dict_vals_table.get(k) if f: return val return self.decode(val) @@ -154,7 +155,10 @@ class Encoder(object): tmp = self.encoder_settings.decode_postprocessor self.encoder_settings.decode_postprocessor = lambda x: x for (k, v) in dct.iteritems(): -dct[k] = self._decode_dict_val(k, v) +decoded_val = self._decode_dict_val(k, v) +if decoded_val is False: +return False +dct[k] = decoded_val
Re: [Freeipa-devel] [PATCH] Don't load the LDAP schema during startup
Jan Zelený wrote: > Jan Zeleny wrote: > > Jan Zelený wrote: > > > Jan Zelený wrote: > > > > Rob Crittenden wrote: > > > > > Jan Zelený wrote: > > > > > > Rob Crittenden wrote: > > > > > >> Jan Zelený wrote: > > > > > >>> Loading of the schema is now performed in the first request > > > > > >>> that requires it. > > > > > >>> > > > > > >>> https://fedorahosted.org/freeipa/ticket/583 > > > > > >>> > > > > > >>> Jan > > > > > >> > > > > > >> We still need to enforce that we get the schema, some low-level > > > > > >> functions depend on it. Also, if the UI doesn't get its aciattrs > > > > > >> (which are derived from the schema) then nothing will be > > > > > >> editable. > > > > > >> > > > > > >> I'm getting this backtrace if I force no schema by disabling > > > > > > get_schema: > > > > > > Ok, I'm sending new version, it should handle these exceptions > > > > > > better and the operation should fail if it needs the schema and > > > > > > the schema is not available for some reason. > > > > > > > > > > This breaks the XML-RPC server. I fixed one problem: > > > > > --- a/ipaserver/plugins/ldap2.py > > > > > +++ b/ipaserver/plugins/ldap2.py > > > > > > > > > > @@ -253,9 +253,10 @@ class ldap2(CrudBackend, Encoder): > > > > > def get_syntax(self, attr, value): > > > > > if not self.schema: > > > > > -self.schema = get_schema(self.ldap_uri, self.conn) > > > > > -if not self.schema: > > > > > +schema = get_schema(self.ldap_uri, self.conn) > > > > > > > > > > +if not schema: > > > > > return None > > > > > > > > > > +object.__setattr__(self, 'schema', schema) > > > > > > > > > > obj = self.schema.get_obj(_ldap.schema.AttributeType, > > > > > attr) > > > > > > > > > > if obj is not None: > > > > > return obj.syntax > > > > > > > > > > But simply things like get_entry() return an InternalError now. I'm > > > > > not sure where you were going by adding this. > > > > > > > > > > rob > > > > > > > > Ok, no problem. It's possible that I simply did a mistake thinking I > > > > can do something in Python what is not really possible. > > > > > > > > About that InternalError: I think raising InternalError when we > > > > cannot load the schema to do the decoding is the right thing to do. > > > > Do you have a better solution? I thought about returning empty > > > > result, but that would mean we have to check the result in every > > > > funtction that is calling them and raising InternalError there. > > > > > > I'm sending updated patch. I modified the get_syntax() as you suggested > > > and I slightly modified raising that InternalError - currently it isn't > > > raised when results from get_entry() are not required by calling > > > method. Currently I'm running some tests, preliminary results looked > > > ok. > > > > self-nack > > > > I discovered some issues discovered by internal test suite, I'm working > > on them > > > > Jan > > Ok, everything is solved, I'm sending final version of the patch in the > attachment. But I still think this should go to 2.1, since it's quite > extensive patch in the core of IPA server and it has potential to break > many things. > > Jan Rebased against master Jan From 7ff1e505395c401a163ac479220910cb39e4182d Mon Sep 17 00:00:00 2001 From: Jan Zeleny Date: Tue, 15 Feb 2011 09:37:58 +0100 Subject: [PATCH] Don't load the LDAP schema during startup https://fedorahosted.org/freeipa/ticket/583 --- ipalib/encoder.py | 12 +++-- ipalib/plugins/baseldap.py | 21 - ipalib/plugins/dns.py |2 +- ipalib/plugins/host.py |2 +- ipalib/plugins/permission.py|4 +- ipalib/plugins/sudocmd.py |2 +- ipaserver/install/dsinstance.py |2 +- ipaserver/plugins/ldap2.py | 92 +++ 8 files changed, 96 insertions(+), 41 deletions(-) diff --git a/ipalib/encoder.py b/ipalib/encoder.py index f23e5659e848d37db1072ff59aa7e11796b0836c..762be9c096a4b7cbea03ecfcc82d03789951c0b5 100644 --- a/ipalib/encoder.py +++ b/ipalib/encoder.py @@ -56,9 +56,10 @@ class Encoder(object): self.encoder_settings = EncoderSettings() def _decode_dict_val(self, key, val): -f = self.encoder_settings.decode_dict_vals_table.get( -self.encoder_settings.decode_dict_vals_table_keygen(key, val) -) +k = self.encoder_settings.decode_dict_vals_table_keygen(key, val) +if k is False: +return False +f = self.encoder_settings.decode_dict_vals_table.get(k) if f: return val return self.decode(val) @@ -154,7 +155,10 @@ class Encoder(object): tmp = self.encoder_settings.decode_postprocessor self.encoder_settings.decode_postprocessor = lambda x: x for (k, v) in dct.iteritems(): -
Re: [Freeipa-devel] [PATCH] Don't load the LDAP schema during startup
Jan Zelený wrote: Jan Zelený wrote: Jan Zeleny wrote: Jan Zelený wrote: Jan Zelený wrote: Rob Crittenden wrote: Jan Zelený wrote: Rob Crittenden wrote: Jan Zelený wrote: Loading of the schema is now performed in the first request that requires it. https://fedorahosted.org/freeipa/ticket/583 Jan We still need to enforce that we get the schema, some low-level functions depend on it. Also, if the UI doesn't get its aciattrs (which are derived from the schema) then nothing will be editable. I'm getting this backtrace if I force no schema by disabling get_schema: Ok, I'm sending new version, it should handle these exceptions better and the operation should fail if it needs the schema and the schema is not available for some reason. This breaks the XML-RPC server. I fixed one problem: --- a/ipaserver/plugins/ldap2.py +++ b/ipaserver/plugins/ldap2.py @@ -253,9 +253,10 @@ class ldap2(CrudBackend, Encoder): def get_syntax(self, attr, value): if not self.schema: -self.schema = get_schema(self.ldap_uri, self.conn) -if not self.schema: +schema = get_schema(self.ldap_uri, self.conn) +if not schema: return None +object.__setattr__(self, 'schema', schema) obj = self.schema.get_obj(_ldap.schema.AttributeType, attr) if obj is not None: return obj.syntax But simply things like get_entry() return an InternalError now. I'm not sure where you were going by adding this. rob Ok, no problem. It's possible that I simply did a mistake thinking I can do something in Python what is not really possible. About that InternalError: I think raising InternalError when we cannot load the schema to do the decoding is the right thing to do. Do you have a better solution? I thought about returning empty result, but that would mean we have to check the result in every funtction that is calling them and raising InternalError there. I'm sending updated patch. I modified the get_syntax() as you suggested and I slightly modified raising that InternalError - currently it isn't raised when results from get_entry() are not required by calling method. Currently I'm running some tests, preliminary results looked ok. self-nack I discovered some issues discovered by internal test suite, I'm working on them Jan Ok, everything is solved, I'm sending final version of the patch in the attachment. But I still think this should go to 2.1, since it's quite extensive patch in the core of IPA server and it has potential to break many things. Jan Rebased against master Jan I came up with an alternate patch that just does lazy retrieval of the schema rather than getting it at start up. Without the schema we can do very little required enforcement so we shouldn't proceed. See my patch 787 Don't load LDAP schema at startup rob ___ Freeipa-devel mailing list Freeipa-devel@redhat.com https://www.redhat.com/mailman/listinfo/freeipa-devel
Re: [Freeipa-devel] [PATCH] Don't load the LDAP schema during startup
Jan Zelený wrote: Loading of the schema is now performed in the first request that requires it. https://fedorahosted.org/freeipa/ticket/583 Jan We still need to enforce that we get the schema, some low-level functions depend on it. Also, if the UI doesn't get its aciattrs (which are derived from the schema) then nothing will be editable. I'm getting this backtrace if I force no schema by disabling get_schema: [Mon Feb 21 13:57:33 2011] [error] ipa: ERROR: non-public: UnicodeDecodeError: 'utf8' codec can't decode byte 0xb3 in position 3: invalid start byte [Mon Feb 21 13:57:33 2011] [error] Traceback (most recent call last): [Mon Feb 21 13:57:33 2011] [error] File "/usr/lib/python2.7/site-packages/ipaserver/rpcserver.py", line 211, in wsgi_execute [Mon Feb 21 13:57:33 2011] [error] result = self.Command[name](*args, **options) [Mon Feb 21 13:57:33 2011] [error] File "/usr/lib/python2.7/site-packages/ipalib/frontend.py", line 422, in __call__ [Mon Feb 21 13:57:33 2011] [error] ret = self.run(*args, **options) [Mon Feb 21 13:57:33 2011] [error] File "/usr/lib/python2.7/site-packages/ipalib/frontend.py", line 728, in run [Mon Feb 21 13:57:33 2011] [error] return self.execute(*args, **options) [Mon Feb 21 13:57:33 2011] [error] File "/usr/lib/python2.7/site-packages/ipalib/plugins/baseldap.py", line 720, in execute [Mon Feb 21 13:57:33 2011] [error] dn, attrs_list, normalize=self.obj.normalize_dn [Mon Feb 21 13:57:33 2011] [error] File "/usr/lib/python2.7/site-packages/ipaserver/plugins/ldap2.py", line 629, in get_entry [Mon Feb 21 13:57:33 2011] [error] size_limit=size_limit, normalize=normalize [Mon Feb 21 13:57:33 2011] [error] File "/usr/lib/python2.7/site-packages/ipalib/encoder.py", line 188, in new_f [Mon Feb 21 13:57:33 2011] [error] return f(*new_args, **kwargs) [Mon Feb 21 13:57:33 2011] [error] File "/usr/lib/python2.7/site-packages/ipalib/encoder.py", line 199, in new_f [Mon Feb 21 13:57:33 2011] [error] return args[0].decode(f(*args, **kwargs)) [Mon Feb 21 13:57:33 2011] [error] File "/usr/lib/python2.7/site-packages/ipalib/encoder.py", line 139, in decode [Mon Feb 21 13:57:33 2011] [error] return tuple(self.decode(m) for m in var) [Mon Feb 21 13:57:33 2011] [error] File "/usr/lib/python2.7/site-packages/ipalib/encoder.py", line 139, in [Mon Feb 21 13:57:33 2011] [error] return tuple(self.decode(m) for m in var) [Mon Feb 21 13:57:33 2011] [error] File "/usr/lib/python2.7/site-packages/ipalib/encoder.py", line 137, in decode [Mon Feb 21 13:57:33 2011] [error] return [self.decode(m) for m in var] [Mon Feb 21 13:57:33 2011] [error] File "/usr/lib/python2.7/site-packages/ipalib/encoder.py", line 139, in decode [Mon Feb 21 13:57:33 2011] [error] return tuple(self.decode(m) for m in var) [Mon Feb 21 13:57:33 2011] [error] File "/usr/lib/python2.7/site-packages/ipalib/encoder.py", line 139, in [Mon Feb 21 13:57:33 2011] [error] return tuple(self.decode(m) for m in var) [Mon Feb 21 13:57:33 2011] [error] File "/usr/lib/python2.7/site-packages/ipalib/encoder.py", line 157, in decode [Mon Feb 21 13:57:33 2011] [error] dct[k] = self._decode_dict_val(k, v) [Mon Feb 21 13:57:33 2011] [error] File "/usr/lib/python2.7/site-packages/ipalib/encoder.py", line 64, in _decode_dict_val [Mon Feb 21 13:57:33 2011] [error] return self.decode(val) [Mon Feb 21 13:57:33 2011] [error] File "/usr/lib/python2.7/site-packages/ipalib/encoder.py", line 137, in decode [Mon Feb 21 13:57:33 2011] [error] return [self.decode(m) for m in var] [Mon Feb 21 13:57:33 2011] [error] File "/usr/lib/python2.7/site-packages/ipalib/encoder.py", line 132, in decode [Mon Feb 21 13:57:33 2011] [error] var.decode(self.encoder_settings.decode_from) [Mon Feb 21 13:57:33 2011] [error] File "/usr/lib64/python2.7/encodings/utf_8.py", line 16, in decode [Mon Feb 21 13:57:33 2011] [error] return codecs.utf_8_decode(input, errors, True) [Mon Feb 21 13:57:33 2011] [error] UnicodeDecodeError: 'utf8' codec can't decode byte 0xb3 in position 3: invalid start byte [Mon Feb 21 13:57:33 2011] [error] ipa: INFO: ad...@greyoak.com: user_show(u'admin', rights=True, all=True, raw=False, version=u'2.0'): UnicodeDecodeError ___ Freeipa-devel mailing list Freeipa-devel@redhat.com https://www.redhat.com/mailman/listinfo/freeipa-devel
Re: [Freeipa-devel] [PATCH] Don't load the LDAP schema during startup
Rob Crittenden wrote: > Jan Zelený wrote: > > Loading of the schema is now performed in the first request that requires > > it. > > > > https://fedorahosted.org/freeipa/ticket/583 > > > > Jan > > We still need to enforce that we get the schema, some low-level > functions depend on it. Also, if the UI doesn't get its aciattrs (which > are derived from the schema) then nothing will be editable. > > I'm getting this backtrace if I force no schema by disabling get_schema: Ok, I'm sending new version, it should handle these exceptions better and the operation should fail if it needs the schema and the schema is not available for some reason. -- Thank you Jan Zeleny Red Hat Software Engineer Brno, Czech Republic From 5ef34748ad1b2d055c86e6674f060d78ad2f8f5f Mon Sep 17 00:00:00 2001 From: Jan Zeleny Date: Tue, 15 Feb 2011 09:37:58 +0100 Subject: [PATCH] Don't load the LDAP schema during startup https://fedorahosted.org/freeipa/ticket/583 --- ipalib/encoder.py | 11 +++-- ipalib/plugins/baseldap.py |6 ++- ipaserver/install/dsinstance.py |2 +- ipaserver/plugins/ldap2.py | 79 ++ 4 files changed, 67 insertions(+), 31 deletions(-) diff --git a/ipalib/encoder.py b/ipalib/encoder.py index f23e5659e848d37db1072ff59aa7e11796b0836c..1874d903aa0dc2a8c9ee1497164b9d418457c82a 100644 --- a/ipalib/encoder.py +++ b/ipalib/encoder.py @@ -56,11 +56,12 @@ class Encoder(object): self.encoder_settings = EncoderSettings() def _decode_dict_val(self, key, val): -f = self.encoder_settings.decode_dict_vals_table.get( -self.encoder_settings.decode_dict_vals_table_keygen(key, val) -) +key = self.encoder_settings.decode_dict_vals_table_keygen(key, val) +if key is None: +return None +f = self.encoder_settings.decode_dict_vals_table.get(key) if f: -return val +return f(val) return self.decode(val) def encode(self, var): @@ -155,6 +156,8 @@ class Encoder(object): self.encoder_settings.decode_postprocessor = lambda x: x for (k, v) in dct.iteritems(): dct[k] = self._decode_dict_val(k, v) +if dct[k] is None: +return None if not self.encoder_settings.decode_dict_vals_postprocess: self.encoder_settings.decode_postprocessor = tmp return dct diff --git a/ipalib/plugins/baseldap.py b/ipalib/plugins/baseldap.py index 3cb72d7b09cc8c8a77bd4e594660ee376d668013..8866c2f6f1c84b9b5b785562db34b82481a52fb9 100644 --- a/ipalib/plugins/baseldap.py +++ b/ipalib/plugins/baseldap.py @@ -394,7 +394,11 @@ class LDAPObject(Object): objectclasses += self.possible_objectclasses # Get list of available attributes for this object for use # in the ACI UI. -attrs = self.api.Backend.ldap2.schema.attribute_types(objectclasses) +schema = self.api.Backend.ldap2.get_schema() +if not schema: +attrs = [] +else: +attrs = schema.attribute_types(objectclasses) attrlist = [] # Go through the MUST first for (oid, attr) in attrs[0].iteritems(): diff --git a/ipaserver/install/dsinstance.py b/ipaserver/install/dsinstance.py index 2544e167bdff28c13201c5371070ab729ca84b67..c2081700bb7348f4db7e3467f64d040effd07cbe 100644 --- a/ipaserver/install/dsinstance.py +++ b/ipaserver/install/dsinstance.py @@ -133,7 +133,7 @@ def has_managed_entries(host_name, dm_password): conn = ldap2(shared_instance=False, ldap_uri=ldapuri, base_dn='cn=config') conn.connect(bind_dn='cn=Directory Manager', bind_pw=dm_password) (dn, attrs) = conn.get_entry('cn=Managed Entries,cn=plugins', - ['*'], time_limit=2, size_limit=3000) + ['*'], time_limit=2, size_limit=3000, override=True) return True except errors.NotFound: return False diff --git a/ipaserver/plugins/ldap2.py b/ipaserver/plugins/ldap2.py index d1e31f5e6eff20cd162c0a11eb4e4404b43ae4b2..3ef076a3dd0c92422bea8fdd29b01e8e311be8ae 100644 --- a/ipaserver/plugins/ldap2.py +++ b/ipaserver/plugins/ldap2.py @@ -184,12 +184,6 @@ def get_schema(url, conn=None): return _ldap.schema.SubSchema(schema_entry[1]) -# cache schema when importing module -try: -_schema = get_schema(api.env.ldap_uri) -except AttributeError: -_schema = None - # The UPG setting will be cached the first time a module checks it _upg = None @@ -229,7 +223,6 @@ class ldap2(CrudBackend, Encoder): def __init__(self, shared_instance=True, ldap_uri=None, base_dn=None, schema=None): -global _schema CrudBackend.__init__(self, shared_instance=shared_instance) Encoder.__init__(self) self.encoder_settings.encode_dict_keys = True @@ -249,7 +242,7 @@ class ldap2(CrudBacken
Re: [Freeipa-devel] [PATCH] Don't load the LDAP schema during startup
Jan Zelený wrote: Rob Crittenden wrote: Jan Zelený wrote: Loading of the schema is now performed in the first request that requires it. https://fedorahosted.org/freeipa/ticket/583 Jan We still need to enforce that we get the schema, some low-level functions depend on it. Also, if the UI doesn't get its aciattrs (which are derived from the schema) then nothing will be editable. I'm getting this backtrace if I force no schema by disabling get_schema: Ok, I'm sending new version, it should handle these exceptions better and the operation should fail if it needs the schema and the schema is not available for some reason. This breaks the XML-RPC server. I fixed one problem: --- a/ipaserver/plugins/ldap2.py +++ b/ipaserver/plugins/ldap2.py @@ -253,9 +253,10 @@ class ldap2(CrudBackend, Encoder): def get_syntax(self, attr, value): if not self.schema: -self.schema = get_schema(self.ldap_uri, self.conn) -if not self.schema: +schema = get_schema(self.ldap_uri, self.conn) +if not schema: return None +object.__setattr__(self, 'schema', schema) obj = self.schema.get_obj(_ldap.schema.AttributeType, attr) if obj is not None: return obj.syntax But simply things like get_entry() return an InternalError now. I'm not sure where you were going by adding this. rob ___ Freeipa-devel mailing list Freeipa-devel@redhat.com https://www.redhat.com/mailman/listinfo/freeipa-devel