[ https://issues.apache.org/jira/browse/AMBARI-25624?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Brahma Reddy Battula updated AMBARI-25624: ------------------------------------------ Resolution: Fixed Status: Resolved (was: Patch Available) [~eub] thanks for your contribution. It's committed for branch-2.7 > "Creating Kerberos keytabs" takes too long > ------------------------------------------ > > Key: AMBARI-25624 > URL: https://issues.apache.org/jira/browse/AMBARI-25624 > Project: Ambari > Issue Type: Bug > Components: ambari-server > Affects Versions: 2.7.1 > Reporter: Jeongin Ju > Assignee: YUBI LEE > Priority: Major > Fix For: 2.7.7 > > Time Spent: 2.5h > Remaining Estimate: 0h > > In kerberized cluster, Ambari tried to find out which keytabs are required to > be created when adding components. > During this process, ambari instantiates serviceMapping variable in > KerberosKeytabPrincipalEntity class which is not used on creating keytab > process. > More seriously, serviceMapping variable can invoke nested variable > instantiation and internal lock acquisitions which takes so much time on > large cluster as shown in below callstack. > h6. {{Thread 129281: (state = IN_JAVA) > - > org.eclipse.persistence.descriptors.FetchGroupManager.isPartialObject(java.lang.Object) > @bci=8, line=466 (Compiled frame; information may be imprecise) > - > org.eclipse.persistence.internal.helper.WriteLockManager.traverseRelatedLocks(java.lang.Object, > java.util.Map, java.util.Map, > org.eclipse.persistence.descriptors.ClassDescriptor, > org.eclipse.persistence.internal.sessions.AbstractSession) @bci=23, line=185 > (Compiled frame) > - > org.eclipse.persistence.internal.helper.WriteLockManager.acquireLockAndRelatedLocks(java.lang.Object, > java.util.Map, java.util.Map, > org.eclipse.persistence.internal.identitymaps.CacheKey, > org.eclipse.persistence.descriptors.ClassDescriptor, > org.eclipse.persistence.internal.sessions.AbstractSession) @bci=96, line=144 > (Compiled frame) > - > org.eclipse.persistence.internal.helper.WriteLockManager.checkAndLockObject(java.lang.Object, > java.util.Map, java.util.Map, > org.eclipse.persistence.mappings.DatabaseMapping, > org.eclipse.persistence.internal.sessions.AbstractSession) @bci=148, line=450 > (Compiled frame) > - > org.eclipse.persistence.internal.helper.WriteLockManager.traverseRelatedLocks(java.lang.Object, > java.util.Map, java.util.Map, > org.eclipse.persistence.descriptors.ClassDescriptor, > org.eclipse.persistence.internal.sessions.AbstractSession) @bci=237, line=213 > (Compiled frame) > - > org.eclipse.persistence.internal.helper.WriteLockManager.acquireLockAndRelatedLocks(java.lang.Object, > java.util.Map, java.util.Map, > org.eclipse.persistence.internal.identitymaps.CacheKey, > org.eclipse.persistence.descriptors.ClassDescriptor, > org.eclipse.persistence.internal.sessions.AbstractSession) @bci=96, line=144 > (Compiled frame) > - > org.eclipse.persistence.internal.helper.WriteLockManager.checkAndLockObject(java.lang.Object, > java.util.Map, java.util.Map, > org.eclipse.persistence.mappings.DatabaseMapping, > org.eclipse.persistence.internal.sessions.AbstractSession) @bci=148, line=450 > (Compiled frame) > - > org.eclipse.persistence.internal.helper.WriteLockManager.traverseRelatedLocks(java.lang.Object, > java.util.Map, java.util.Map, > org.eclipse.persistence.descriptors.ClassDescriptor, > org.eclipse.persistence.internal.sessions.AbstractSession) @bci=237, line=213 > (Compiled frame) > - > org.eclipse.persistence.internal.helper.WriteLockManager.acquireLockAndRelatedLocks(java.lang.Object, > java.util.Map, java.util.Map, > org.eclipse.persistence.internal.identitymaps.CacheKey, > org.eclipse.persistence.descriptors.ClassDescriptor, > org.eclipse.persistence.internal.sessions.AbstractSession) @bci=96, line=144 > (Compiled frame) > - > org.eclipse.persistence.internal.helper.WriteLockManager.checkAndLockObject(java.lang.Object, > java.util.Map, java.util.Map, > org.eclipse.persistence.mappings.DatabaseMapping, > org.eclipse.persistence.internal.sessions.AbstractSession) @bci=148, line=450 > (Compiled frame) > - > org.eclipse.persistence.internal.helper.WriteLockManager.traverseRelatedLocks(java.lang.Object, > java.util.Map, java.util.Map, > org.eclipse.persistence.descriptors.ClassDescriptor, > org.eclipse.persistence.internal.sessions.AbstractSession) @bci=172, line=203 > (Compiled frame) > - > org.eclipse.persistence.internal.helper.WriteLockManager.acquireLockAndRelatedLocks(java.lang.Object, > java.util.Map, java.util.Map, > org.eclipse.persistence.internal.identitymaps.CacheKey, > org.eclipse.persistence.descriptors.ClassDescriptor, > org.eclipse.persistence.internal.sessions.AbstractSession) @bci=96, line=144 > (Compiled frame) > - > org.eclipse.persistence.internal.helper.WriteLockManager.checkAndLockObject(java.lang.Object, > java.util.Map, java.util.Map, > org.eclipse.persistence.mappings.DatabaseMapping, > org.eclipse.persistence.internal.sessions.AbstractSession) @bci=148, line=450 > (Compiled frame) > - > org.eclipse.persistence.internal.helper.WriteLockManager.traverseRelatedLocks(java.lang.Object, > java.util.Map, java.util.Map, > org.eclipse.persistence.descriptors.ClassDescriptor, > org.eclipse.persistence.internal.sessions.AbstractSession) @bci=237, line=213 > (Compiled frame) > - > org.eclipse.persistence.internal.helper.WriteLockManager.acquireLockAndRelatedLocks(java.lang.Object, > java.util.Map, java.util.Map, > org.eclipse.persistence.internal.identitymaps.CacheKey, > org.eclipse.persistence.descriptors.ClassDescriptor, > org.eclipse.persistence.internal.sessions.AbstractSession) @bci=96, line=144 > (Compiled frame) > - > org.eclipse.persistence.internal.helper.WriteLockManager.checkAndLockObject(java.lang.Object, > java.util.Map, java.util.Map, > org.eclipse.persistence.mappings.DatabaseMapping, > org.eclipse.persistence.internal.sessions.AbstractSession) @bci=148, line=450 > (Compiled frame) > - > org.eclipse.persistence.internal.helper.WriteLockManager.traverseRelatedLocks(java.lang.Object, > java.util.Map, java.util.Map, > org.eclipse.persistence.descriptors.ClassDescriptor, > org.eclipse.persistence.internal.sessions.AbstractSession) @bci=237, line=213 > (Compiled frame) > - > org.eclipse.persistence.internal.helper.WriteLockManager.acquireLockAndRelatedLocks(java.lang.Object, > java.util.Map, java.util.Map, > org.eclipse.persistence.internal.identitymaps.CacheKey, > org.eclipse.persistence.descriptors.ClassDescriptor, > org.eclipse.persistence.internal.sessions.AbstractSession) @bci=96, line=144 > (Compiled frame) > - > org.eclipse.persistence.internal.helper.WriteLockManager.acquireLocksForClone(java.lang.Object, > org.eclipse.persistence.descriptors.ClassDescriptor, > org.eclipse.persistence.internal.identitymaps.CacheKey, > org.eclipse.persistence.internal.sessions.AbstractSession) @bci=31, line=75 > (Compiled frame) > - > org.eclipse.persistence.internal.sessions.UnitOfWorkImpl.cloneAndRegisterObject(java.lang.Object, > org.eclipse.persistence.internal.identitymaps.CacheKey, > org.eclipse.persistence.internal.identitymaps.CacheKey, > org.eclipse.persistence.descriptors.ClassDescriptor) @bci=140, line=997 > (Compiled frame) > - > org.eclipse.persistence.internal.sessions.UnitOfWorkImpl.cloneAndRegisterObject(java.lang.Object, > org.eclipse.persistence.internal.identitymaps.CacheKey, > org.eclipse.persistence.descriptors.ClassDescriptor) @bci=59, line=955 > (Compiled frame) > - > org.eclipse.persistence.internal.sessions.UnitOfWorkIdentityMapAccessor.getAndCloneCacheKeyFromParent(java.lang.Object, > java.lang.Object, java.lang.Class, boolean, > org.eclipse.persistence.descriptors.ClassDescriptor) @bci=288, line=209 > (Compiled frame) > - > org.eclipse.persistence.internal.sessions.UnitOfWorkIdentityMapAccessor.getFromIdentityMap(java.lang.Object, > java.lang.Object, java.lang.Class, boolean, > org.eclipse.persistence.descriptors.ClassDescriptor) @bci=28, line=137 > (Compiled frame) > - > org.eclipse.persistence.internal.sessions.UnitOfWorkImpl.registerExistingObject(java.lang.Object, > org.eclipse.persistence.descriptors.ClassDescriptor, java.lang.Object, > boolean) @bci=139, line=3940 (Compiled frame) > - > org.eclipse.persistence.internal.sessions.UnitOfWorkImpl.registerExistingObject(java.lang.Object, > boolean) @bci=62, line=3894 (Compiled frame) > - > org.eclipse.persistence.mappings.CollectionMapping.buildElementUnitOfWorkClone(java.lang.Object, > java.lang.Object, java.lang.Integer, > org.eclipse.persistence.internal.sessions.UnitOfWorkImpl, boolean, boolean) > @bci=90, line=308 (Compiled frame) > - > org.eclipse.persistence.mappings.CollectionMapping.buildElementClone(java.lang.Object, > java.lang.Object, org.eclipse.persistence.internal.identitymaps.CacheKey, > java.lang.Integer, org.eclipse.persistence.internal.sessions.AbstractSession, > boolean, boolean) @bci=22, line=321 (Compiled frame) > - > org.eclipse.persistence.internal.queries.ContainerPolicy.addNextValueFromIteratorInto(java.lang.Object, > java.lang.Object, org.eclipse.persistence.internal.identitymaps.CacheKey, > java.lang.Object, org.eclipse.persistence.mappings.CollectionMapping, > java.lang.Integer, org.eclipse.persistence.internal.sessions.AbstractSession, > boolean, boolean) @bci=19, line=215 (Compiled frame) > - > org.eclipse.persistence.mappings.CollectionMapping.buildCloneForPartObject(java.lang.Object, > java.lang.Object, org.eclipse.persistence.internal.identitymaps.CacheKey, > java.lang.Object, org.eclipse.persistence.internal.sessions.AbstractSession, > java.lang.Integer, boolean, boolean) @bci=182, line=223 (Compiled frame) > - > org.eclipse.persistence.internal.indirection.UnitOfWorkQueryValueHolder.buildCloneFor(java.lang.Object) > @bci=41, line=60 (Compiled frame) > - > org.eclipse.persistence.internal.indirection.UnitOfWorkValueHolder.instantiateImpl() > @bci=154, line=173 (Compiled frame) > - > org.eclipse.persistence.internal.indirection.UnitOfWorkValueHolder.instantiate() > @bci=33, line=234 (Compiled frame) > - > org.eclipse.persistence.internal.indirection.DatabaseValueHolder.getValue() > @bci=20, line=89 (Compiled frame) > - org.eclipse.persistence.indirection.IndirectList.buildDelegate() @bci=4, > line=271 (Compiled frame) > - org.eclipse.persistence.indirection.IndirectList.getDelegate() @bci=20, > line=455 (Compiled frame) > - > org.eclipse.persistence.internal.indirection.jdk8.IndirectList.access$000(org.eclipse.persistence.internal.indirection.jdk8.IndirectList) > @bci=1, line=32 (Compiled frame) > - > org.eclipse.persistence.internal.indirection.jdk8.IndirectList$1.<init>(org.eclipse.persistence.internal.indirection.jdk8.IndirectList, > int) @bci=19, line=57 (Compiled frame) > - > org.eclipse.persistence.internal.indirection.jdk8.IndirectList.listIterator(int) > @bci=6, line=56 (Compiled frame) > - org.eclipse.persistence.indirection.IndirectList.iterator() @bci=2, > line=555 (Compiled frame) > - > org.apache.ambari.server.orm.entities.KerberosKeytabPrincipalEntity.getServiceMappingAsMultimap() > @bci=8, line=213 (Compiled frame) > - > org.apache.ambari.server.serveraction.kerberos.stageutils.KerberosKeytabController.fromPrincipalEntities(java.util.Collection) > @bci=79, line=221 (Compiled frame) > - > org.apache.ambari.server.serveraction.kerberos.stageutils.KerberosKeytabController.fromKeytabEntity(org.apache.ambari.server.orm.entities.KerberosKeytabEntity, > boolean) @bci=9, line=184 (Compiled frame) > - > org.apache.ambari.server.serveraction.kerberos.stageutils.KerberosKeytabController.fromKeytabEntity(org.apache.ambari.server.orm.entities.KerberosKeytabEntity) > @bci=3, line=198 (Compiled frame) > - > org.apache.ambari.server.serveraction.kerberos.stageutils.KerberosKeytabController.fromKeytabEntities(java.util.Collection) > @bci=35, line=204 (Compiled frame) > - > org.apache.ambari.server.serveraction.kerberos.stageutils.KerberosKeytabController.getFromPrincipal(org.apache.ambari.server.serveraction.kerberos.stageutils.ResolvedKerberosPrincipal) > @bci=16, line=94 (Compiled frame) > - > org.apache.ambari.server.serveraction.kerberos.CreateKeytabFilesServerAction.processIdentity(org.apache.ambari.server.serveraction.kerberos.stageutils.ResolvedKerberosPrincipal, > org.apache.ambari.server.serveraction.kerberos.KerberosOperationHandler, > java.util.Map, boolean, java.util.Map) @bci=86, line=161 (Compiled frame) > - > org.apache.ambari.server.serveraction.kerberos.KerberosServerAction.processIdentities(java.util.Map, > boolean) @bci=427, line=478 (Interpreted frame) > - > org.apache.ambari.server.serveraction.kerberos.KerberosServerAction.processIdentities(java.util.Map) > @bci=3, line=427 (Interpreted frame) > - > org.apache.ambari.server.serveraction.kerberos.CreateKeytabFilesServerAction.execute(java.util.concurrent.ConcurrentMap) > @bci=2, line=105 (Interpreted frame) > - > org.apache.ambari.server.serveraction.ServerActionExecutor$Worker.execute(org.apache.ambari.server.actionmanager.HostRoleCommand, > org.apache.ambari.server.agent.ExecutionCommand) @bci=424, line=550 > (Interpreted frame) > - org.apache.ambari.server.serveraction.ServerActionExecutor$Worker.run() > @bci=48, line=466 (Interpreted frame) > - java.lang.Thread.run() @bci=11, line=745 (Compiled frame)}} > > In our clusters, which are composed of 4000+ physical machines, adding new 5 > hosts (with 7 components, 11 keytabs to be created each) spend about 2.5 > hours. > 2 hours out of 2.5 hours are spended on the above callstack. > After this patch is applied, adding new 5 hosts with the same requirements > spend only half hour as expected. > -- This message was sent by Atlassian Jira (v8.20.10#820010) --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@ambari.apache.org For additional commands, e-mail: issues-h...@ambari.apache.org