[
https://issues.apache.org/jira/browse/AMBARI-25624?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Jeongin Ju updated AMBARI-25624:
--------------------------------
Description:
In kerberized cluster, Ambari tried to find out which keytabs are required to
be created when adding components.
During this process, ambari instantiates serviceMapping variable in
KerberosKeytabPrincipalEntity class which is not used on creating keytab
process.
More seriously, serviceMapping variable can invoke nested variable
instantiation and internal lock acquisitions which takes so much time on large
cluster as shown in below callstack.
h6. {{Thread 129281: (state = IN_JAVA)
-
org.eclipse.persistence.descriptors.FetchGroupManager.isPartialObject(java.lang.Object)
@bci=8, line=466 (Compiled frame; information may be imprecise)
-
org.eclipse.persistence.internal.helper.WriteLockManager.traverseRelatedLocks(java.lang.Object,
java.util.Map, java.util.Map,
org.eclipse.persistence.descriptors.ClassDescriptor,
org.eclipse.persistence.internal.sessions.AbstractSession) @bci=23, line=185
(Compiled frame)
-
org.eclipse.persistence.internal.helper.WriteLockManager.acquireLockAndRelatedLocks(java.lang.Object,
java.util.Map, java.util.Map,
org.eclipse.persistence.internal.identitymaps.CacheKey,
org.eclipse.persistence.descriptors.ClassDescriptor,
org.eclipse.persistence.internal.sessions.AbstractSession) @bci=96, line=144
(Compiled frame)
-
org.eclipse.persistence.internal.helper.WriteLockManager.checkAndLockObject(java.lang.Object,
java.util.Map, java.util.Map,
org.eclipse.persistence.mappings.DatabaseMapping,
org.eclipse.persistence.internal.sessions.AbstractSession) @bci=148, line=450
(Compiled frame)
-
org.eclipse.persistence.internal.helper.WriteLockManager.traverseRelatedLocks(java.lang.Object,
java.util.Map, java.util.Map,
org.eclipse.persistence.descriptors.ClassDescriptor,
org.eclipse.persistence.internal.sessions.AbstractSession) @bci=237, line=213
(Compiled frame)
-
org.eclipse.persistence.internal.helper.WriteLockManager.acquireLockAndRelatedLocks(java.lang.Object,
java.util.Map, java.util.Map,
org.eclipse.persistence.internal.identitymaps.CacheKey,
org.eclipse.persistence.descriptors.ClassDescriptor,
org.eclipse.persistence.internal.sessions.AbstractSession) @bci=96, line=144
(Compiled frame)
-
org.eclipse.persistence.internal.helper.WriteLockManager.checkAndLockObject(java.lang.Object,
java.util.Map, java.util.Map,
org.eclipse.persistence.mappings.DatabaseMapping,
org.eclipse.persistence.internal.sessions.AbstractSession) @bci=148, line=450
(Compiled frame)
-
org.eclipse.persistence.internal.helper.WriteLockManager.traverseRelatedLocks(java.lang.Object,
java.util.Map, java.util.Map,
org.eclipse.persistence.descriptors.ClassDescriptor,
org.eclipse.persistence.internal.sessions.AbstractSession) @bci=237, line=213
(Compiled frame)
-
org.eclipse.persistence.internal.helper.WriteLockManager.acquireLockAndRelatedLocks(java.lang.Object,
java.util.Map, java.util.Map,
org.eclipse.persistence.internal.identitymaps.CacheKey,
org.eclipse.persistence.descriptors.ClassDescriptor,
org.eclipse.persistence.internal.sessions.AbstractSession) @bci=96, line=144
(Compiled frame)
-
org.eclipse.persistence.internal.helper.WriteLockManager.checkAndLockObject(java.lang.Object,
java.util.Map, java.util.Map,
org.eclipse.persistence.mappings.DatabaseMapping,
org.eclipse.persistence.internal.sessions.AbstractSession) @bci=148, line=450
(Compiled frame)
-
org.eclipse.persistence.internal.helper.WriteLockManager.traverseRelatedLocks(java.lang.Object,
java.util.Map, java.util.Map,
org.eclipse.persistence.descriptors.ClassDescriptor,
org.eclipse.persistence.internal.sessions.AbstractSession) @bci=172, line=203
(Compiled frame)
-
org.eclipse.persistence.internal.helper.WriteLockManager.acquireLockAndRelatedLocks(java.lang.Object,
java.util.Map, java.util.Map,
org.eclipse.persistence.internal.identitymaps.CacheKey,
org.eclipse.persistence.descriptors.ClassDescriptor,
org.eclipse.persistence.internal.sessions.AbstractSession) @bci=96, line=144
(Compiled frame)
-
org.eclipse.persistence.internal.helper.WriteLockManager.checkAndLockObject(java.lang.Object,
java.util.Map, java.util.Map,
org.eclipse.persistence.mappings.DatabaseMapping,
org.eclipse.persistence.internal.sessions.AbstractSession) @bci=148, line=450
(Compiled frame)
-
org.eclipse.persistence.internal.helper.WriteLockManager.traverseRelatedLocks(java.lang.Object,
java.util.Map, java.util.Map,
org.eclipse.persistence.descriptors.ClassDescriptor,
org.eclipse.persistence.internal.sessions.AbstractSession) @bci=237, line=213
(Compiled frame)
-
org.eclipse.persistence.internal.helper.WriteLockManager.acquireLockAndRelatedLocks(java.lang.Object,
java.util.Map, java.util.Map,
org.eclipse.persistence.internal.identitymaps.CacheKey,
org.eclipse.persistence.descriptors.ClassDescriptor,
org.eclipse.persistence.internal.sessions.AbstractSession) @bci=96, line=144
(Compiled frame)
-
org.eclipse.persistence.internal.helper.WriteLockManager.checkAndLockObject(java.lang.Object,
java.util.Map, java.util.Map,
org.eclipse.persistence.mappings.DatabaseMapping,
org.eclipse.persistence.internal.sessions.AbstractSession) @bci=148, line=450
(Compiled frame)
-
org.eclipse.persistence.internal.helper.WriteLockManager.traverseRelatedLocks(java.lang.Object,
java.util.Map, java.util.Map,
org.eclipse.persistence.descriptors.ClassDescriptor,
org.eclipse.persistence.internal.sessions.AbstractSession) @bci=237, line=213
(Compiled frame)
-
org.eclipse.persistence.internal.helper.WriteLockManager.acquireLockAndRelatedLocks(java.lang.Object,
java.util.Map, java.util.Map,
org.eclipse.persistence.internal.identitymaps.CacheKey,
org.eclipse.persistence.descriptors.ClassDescriptor,
org.eclipse.persistence.internal.sessions.AbstractSession) @bci=96, line=144
(Compiled frame)
-
org.eclipse.persistence.internal.helper.WriteLockManager.acquireLocksForClone(java.lang.Object,
org.eclipse.persistence.descriptors.ClassDescriptor,
org.eclipse.persistence.internal.identitymaps.CacheKey,
org.eclipse.persistence.internal.sessions.AbstractSession) @bci=31, line=75
(Compiled frame)
-
org.eclipse.persistence.internal.sessions.UnitOfWorkImpl.cloneAndRegisterObject(java.lang.Object,
org.eclipse.persistence.internal.identitymaps.CacheKey,
org.eclipse.persistence.internal.identitymaps.CacheKey,
org.eclipse.persistence.descriptors.ClassDescriptor) @bci=140, line=997
(Compiled frame)
-
org.eclipse.persistence.internal.sessions.UnitOfWorkImpl.cloneAndRegisterObject(java.lang.Object,
org.eclipse.persistence.internal.identitymaps.CacheKey,
org.eclipse.persistence.descriptors.ClassDescriptor) @bci=59, line=955
(Compiled frame)
-
org.eclipse.persistence.internal.sessions.UnitOfWorkIdentityMapAccessor.getAndCloneCacheKeyFromParent(java.lang.Object,
java.lang.Object, java.lang.Class, boolean,
org.eclipse.persistence.descriptors.ClassDescriptor) @bci=288, line=209
(Compiled frame)
-
org.eclipse.persistence.internal.sessions.UnitOfWorkIdentityMapAccessor.getFromIdentityMap(java.lang.Object,
java.lang.Object, java.lang.Class, boolean,
org.eclipse.persistence.descriptors.ClassDescriptor) @bci=28, line=137
(Compiled frame)
-
org.eclipse.persistence.internal.sessions.UnitOfWorkImpl.registerExistingObject(java.lang.Object,
org.eclipse.persistence.descriptors.ClassDescriptor, java.lang.Object,
boolean) @bci=139, line=3940 (Compiled frame)
-
org.eclipse.persistence.internal.sessions.UnitOfWorkImpl.registerExistingObject(java.lang.Object,
boolean) @bci=62, line=3894 (Compiled frame)
-
org.eclipse.persistence.mappings.CollectionMapping.buildElementUnitOfWorkClone(java.lang.Object,
java.lang.Object, java.lang.Integer,
org.eclipse.persistence.internal.sessions.UnitOfWorkImpl, boolean, boolean)
@bci=90, line=308 (Compiled frame)
-
org.eclipse.persistence.mappings.CollectionMapping.buildElementClone(java.lang.Object,
java.lang.Object, org.eclipse.persistence.internal.identitymaps.CacheKey,
java.lang.Integer, org.eclipse.persistence.internal.sessions.AbstractSession,
boolean, boolean) @bci=22, line=321 (Compiled frame)
-
org.eclipse.persistence.internal.queries.ContainerPolicy.addNextValueFromIteratorInto(java.lang.Object,
java.lang.Object, org.eclipse.persistence.internal.identitymaps.CacheKey,
java.lang.Object, org.eclipse.persistence.mappings.CollectionMapping,
java.lang.Integer, org.eclipse.persistence.internal.sessions.AbstractSession,
boolean, boolean) @bci=19, line=215 (Compiled frame)
-
org.eclipse.persistence.mappings.CollectionMapping.buildCloneForPartObject(java.lang.Object,
java.lang.Object, org.eclipse.persistence.internal.identitymaps.CacheKey,
java.lang.Object, org.eclipse.persistence.internal.sessions.AbstractSession,
java.lang.Integer, boolean, boolean) @bci=182, line=223 (Compiled frame)
-
org.eclipse.persistence.internal.indirection.UnitOfWorkQueryValueHolder.buildCloneFor(java.lang.Object)
@bci=41, line=60 (Compiled frame)
-
org.eclipse.persistence.internal.indirection.UnitOfWorkValueHolder.instantiateImpl()
@bci=154, line=173 (Compiled frame)
-
org.eclipse.persistence.internal.indirection.UnitOfWorkValueHolder.instantiate()
@bci=33, line=234 (Compiled frame)
- org.eclipse.persistence.internal.indirection.DatabaseValueHolder.getValue()
@bci=20, line=89 (Compiled frame)
- org.eclipse.persistence.indirection.IndirectList.buildDelegate() @bci=4,
line=271 (Compiled frame)
- org.eclipse.persistence.indirection.IndirectList.getDelegate() @bci=20,
line=455 (Compiled frame)
-
org.eclipse.persistence.internal.indirection.jdk8.IndirectList.access$000(org.eclipse.persistence.internal.indirection.jdk8.IndirectList)
@bci=1, line=32 (Compiled frame)
-
org.eclipse.persistence.internal.indirection.jdk8.IndirectList$1.<init>(org.eclipse.persistence.internal.indirection.jdk8.IndirectList,
int) @bci=19, line=57 (Compiled frame)
-
org.eclipse.persistence.internal.indirection.jdk8.IndirectList.listIterator(int)
@bci=6, line=56 (Compiled frame)
- org.eclipse.persistence.indirection.IndirectList.iterator() @bci=2, line=555
(Compiled frame)
-
org.apache.ambari.server.orm.entities.KerberosKeytabPrincipalEntity.getServiceMappingAsMultimap()
@bci=8, line=213 (Compiled frame)
-
org.apache.ambari.server.serveraction.kerberos.stageutils.KerberosKeytabController.fromPrincipalEntities(java.util.Collection)
@bci=79, line=221 (Compiled frame)
-
org.apache.ambari.server.serveraction.kerberos.stageutils.KerberosKeytabController.fromKeytabEntity(org.apache.ambari.server.orm.entities.KerberosKeytabEntity,
boolean) @bci=9, line=184 (Compiled frame)
-
org.apache.ambari.server.serveraction.kerberos.stageutils.KerberosKeytabController.fromKeytabEntity(org.apache.ambari.server.orm.entities.KerberosKeytabEntity)
@bci=3, line=198 (Compiled frame)
-
org.apache.ambari.server.serveraction.kerberos.stageutils.KerberosKeytabController.fromKeytabEntities(java.util.Collection)
@bci=35, line=204 (Compiled frame)
-
org.apache.ambari.server.serveraction.kerberos.stageutils.KerberosKeytabController.getFromPrincipal(org.apache.ambari.server.serveraction.kerberos.stageutils.ResolvedKerberosPrincipal)
@bci=16, line=94 (Compiled frame)
-
org.apache.ambari.server.serveraction.kerberos.CreateKeytabFilesServerAction.processIdentity(org.apache.ambari.server.serveraction.kerberos.stageutils.ResolvedKerberosPrincipal,
org.apache.ambari.server.serveraction.kerberos.KerberosOperationHandler,
java.util.Map, boolean, java.util.Map) @bci=86, line=161 (Compiled frame)
-
org.apache.ambari.server.serveraction.kerberos.KerberosServerAction.processIdentities(java.util.Map,
boolean) @bci=427, line=478 (Interpreted frame)
-
org.apache.ambari.server.serveraction.kerberos.KerberosServerAction.processIdentities(java.util.Map)
@bci=3, line=427 (Interpreted frame)
-
org.apache.ambari.server.serveraction.kerberos.CreateKeytabFilesServerAction.execute(java.util.concurrent.ConcurrentMap)
@bci=2, line=105 (Interpreted frame)
-
org.apache.ambari.server.serveraction.ServerActionExecutor$Worker.execute(org.apache.ambari.server.actionmanager.HostRoleCommand,
org.apache.ambari.server.agent.ExecutionCommand) @bci=424, line=550
(Interpreted frame)
- org.apache.ambari.server.serveraction.ServerActionExecutor$Worker.run()
@bci=48, line=466 (Interpreted frame)
- java.lang.Thread.run() @bci=11, line=745 (Compiled frame)}}
In our clusters, which are composed of 4000+ physical machines, adding new 5
hosts (with 7 components, 11 keytabs to be created each) spend about 2.5 hours.
2 hours out of 2.5 hours are spended on the above callstack.
After this patch is applied, adding new 5 hosts with the same requirements
spend only half hour as expected.
was:
In kerberized cluster, Ambari tried to find out which keytabs are required to
be created when adding components.
During this process, ambari instantiates serviceMapping variable in
KerberosKeytabPrincipalEntity class which is not used on creating keytab
process.
More seriously, serviceMapping variable can invoke nested variable
instantiation and internal lock acquisitions which takes so much time on large
cluster as shown in below callstack.
h6. {{Thread 129281: (state = IN_JAVA)
-
org.eclipse.persistence.descriptors.FetchGroupManager.isPartialObject(java.lang.Object)
@bci=8, line=466 (Compiled frame; information may be imprecise)
-
org.eclipse.persistence.internal.helper.WriteLockManager.traverseRelatedLocks(java.lang.Object,
java.util.Map, java.util.Map,
org.eclipse.persistence.descriptors.ClassDescriptor,
org.eclipse.persistence.internal.sessions.AbstractSession) @bci=23, line=185
(Compiled frame)
-
org.eclipse.persistence.internal.helper.WriteLockManager.acquireLockAndRelatedLocks(java.lang.Object,
java.util.Map, java.util.Map,
org.eclipse.persistence.internal.identitymaps.CacheKey,
org.eclipse.persistence.descriptors.ClassDescriptor,
org.eclipse.persistence.internal.sessions.AbstractSession) @bci=96, line=144
(Compiled frame)
-
org.eclipse.persistence.internal.helper.WriteLockManager.checkAndLockObject(java.lang.Object,
java.util.Map, java.util.Map,
org.eclipse.persistence.mappings.DatabaseMapping,
org.eclipse.persistence.internal.sessions.AbstractSession) @bci=148, line=450
(Compiled frame)
-
org.eclipse.persistence.internal.helper.WriteLockManager.traverseRelatedLocks(java.lang.Object,
java.util.Map, java.util.Map,
org.eclipse.persistence.descriptors.ClassDescriptor,
org.eclipse.persistence.internal.sessions.AbstractSession) @bci=237, line=213
(Compiled frame)
-
org.eclipse.persistence.internal.helper.WriteLockManager.acquireLockAndRelatedLocks(java.lang.Object,
java.util.Map, java.util.Map,
org.eclipse.persistence.internal.identitymaps.CacheKey,
org.eclipse.persistence.descriptors.ClassDescriptor,
org.eclipse.persistence.internal.sessions.AbstractSession) @bci=96, line=144
(Compiled frame)
-
org.eclipse.persistence.internal.helper.WriteLockManager.checkAndLockObject(java.lang.Object,
java.util.Map, java.util.Map,
org.eclipse.persistence.mappings.DatabaseMapping,
org.eclipse.persistence.internal.sessions.AbstractSession) @bci=148, line=450
(Compiled frame)
-
org.eclipse.persistence.internal.helper.WriteLockManager.traverseRelatedLocks(java.lang.Object,
java.util.Map, java.util.Map,
org.eclipse.persistence.descriptors.ClassDescriptor,
org.eclipse.persistence.internal.sessions.AbstractSession) @bci=237, line=213
(Compiled frame)
-
org.eclipse.persistence.internal.helper.WriteLockManager.acquireLockAndRelatedLocks(java.lang.Object,
java.util.Map, java.util.Map,
org.eclipse.persistence.internal.identitymaps.CacheKey,
org.eclipse.persistence.descriptors.ClassDescriptor,
org.eclipse.persistence.internal.sessions.AbstractSession) @bci=96, line=144
(Compiled frame)
-
org.eclipse.persistence.internal.helper.WriteLockManager.checkAndLockObject(java.lang.Object,
java.util.Map, java.util.Map,
org.eclipse.persistence.mappings.DatabaseMapping,
org.eclipse.persistence.internal.sessions.AbstractSession) @bci=148, line=450
(Compiled frame)
-
org.eclipse.persistence.internal.helper.WriteLockManager.traverseRelatedLocks(java.lang.Object,
java.util.Map, java.util.Map,
org.eclipse.persistence.descriptors.ClassDescriptor,
org.eclipse.persistence.internal.sessions.AbstractSession) @bci=172, line=203
(Compiled frame)
-
org.eclipse.persistence.internal.helper.WriteLockManager.acquireLockAndRelatedLocks(java.lang.Object,
java.util.Map, java.util.Map,
org.eclipse.persistence.internal.identitymaps.CacheKey,
org.eclipse.persistence.descriptors.ClassDescriptor,
org.eclipse.persistence.internal.sessions.AbstractSession) @bci=96, line=144
(Compiled frame)
-
org.eclipse.persistence.internal.helper.WriteLockManager.checkAndLockObject(java.lang.Object,
java.util.Map, java.util.Map,
org.eclipse.persistence.mappings.DatabaseMapping,
org.eclipse.persistence.internal.sessions.AbstractSession) @bci=148, line=450
(Compiled frame)
-
org.eclipse.persistence.internal.helper.WriteLockManager.traverseRelatedLocks(java.lang.Object,
java.util.Map, java.util.Map,
org.eclipse.persistence.descriptors.ClassDescriptor,
org.eclipse.persistence.internal.sessions.AbstractSession) @bci=237, line=213
(Compiled frame)
-
org.eclipse.persistence.internal.helper.WriteLockManager.acquireLockAndRelatedLocks(java.lang.Object,
java.util.Map, java.util.Map,
org.eclipse.persistence.internal.identitymaps.CacheKey,
org.eclipse.persistence.descriptors.ClassDescriptor,
org.eclipse.persistence.internal.sessions.AbstractSession) @bci=96, line=144
(Compiled frame)
-
org.eclipse.persistence.internal.helper.WriteLockManager.checkAndLockObject(java.lang.Object,
java.util.Map, java.util.Map,
org.eclipse.persistence.mappings.DatabaseMapping,
org.eclipse.persistence.internal.sessions.AbstractSession) @bci=148, line=450
(Compiled frame)
-
org.eclipse.persistence.internal.helper.WriteLockManager.traverseRelatedLocks(java.lang.Object,
java.util.Map, java.util.Map,
org.eclipse.persistence.descriptors.ClassDescriptor,
org.eclipse.persistence.internal.sessions.AbstractSession) @bci=237, line=213
(Compiled frame)
-
org.eclipse.persistence.internal.helper.WriteLockManager.acquireLockAndRelatedLocks(java.lang.Object,
java.util.Map, java.util.Map,
org.eclipse.persistence.internal.identitymaps.CacheKey,
org.eclipse.persistence.descriptors.ClassDescriptor,
org.eclipse.persistence.internal.sessions.AbstractSession) @bci=96, line=144
(Compiled frame)
-
org.eclipse.persistence.internal.helper.WriteLockManager.acquireLocksForClone(java.lang.Object,
org.eclipse.persistence.descriptors.ClassDescriptor,
org.eclipse.persistence.internal.identitymaps.CacheKey,
org.eclipse.persistence.internal.sessions.AbstractSession) @bci=31, line=75
(Compiled frame)
-
org.eclipse.persistence.internal.sessions.UnitOfWorkImpl.cloneAndRegisterObject(java.lang.Object,
org.eclipse.persistence.internal.identitymaps.CacheKey,
org.eclipse.persistence.internal.identitymaps.CacheKey,
org.eclipse.persistence.descriptors.ClassDescriptor) @bci=140, line=997
(Compiled frame)
-
org.eclipse.persistence.internal.sessions.UnitOfWorkImpl.cloneAndRegisterObject(java.lang.Object,
org.eclipse.persistence.internal.identitymaps.CacheKey,
org.eclipse.persistence.descriptors.ClassDescriptor) @bci=59, line=955
(Compiled frame)
-
org.eclipse.persistence.internal.sessions.UnitOfWorkIdentityMapAccessor.getAndCloneCacheKeyFromParent(java.lang.Object,
java.lang.Object, java.lang.Class, boolean,
org.eclipse.persistence.descriptors.ClassDescriptor) @bci=288, line=209
(Compiled frame)
-
org.eclipse.persistence.internal.sessions.UnitOfWorkIdentityMapAccessor.getFromIdentityMap(java.lang.Object,
java.lang.Object, java.lang.Class, boolean,
org.eclipse.persistence.descriptors.ClassDescriptor) @bci=28, line=137
(Compiled frame)
-
org.eclipse.persistence.internal.sessions.UnitOfWorkImpl.registerExistingObject(java.lang.Object,
org.eclipse.persistence.descriptors.ClassDescriptor, java.lang.Object,
boolean) @bci=139, line=3940 (Compiled frame)
-
org.eclipse.persistence.internal.sessions.UnitOfWorkImpl.registerExistingObject(java.lang.Object,
boolean) @bci=62, line=3894 (Compiled frame)
-
org.eclipse.persistence.mappings.CollectionMapping.buildElementUnitOfWorkClone(java.lang.Object,
java.lang.Object, java.lang.Integer,
org.eclipse.persistence.internal.sessions.UnitOfWorkImpl, boolean, boolean)
@bci=90, line=308 (Compiled frame)
-
org.eclipse.persistence.mappings.CollectionMapping.buildElementClone(java.lang.Object,
java.lang.Object, org.eclipse.persistence.internal.identitymaps.CacheKey,
java.lang.Integer, org.eclipse.persistence.internal.sessions.AbstractSession,
boolean, boolean) @bci=22, line=321 (Compiled frame)
-
org.eclipse.persistence.internal.queries.ContainerPolicy.addNextValueFromIteratorInto(java.lang.Object,
java.lang.Object, org.eclipse.persistence.internal.identitymaps.CacheKey,
java.lang.Object, org.eclipse.persistence.mappings.CollectionMapping,
java.lang.Integer, org.eclipse.persistence.internal.sessions.AbstractSession,
boolean, boolean) @bci=19, line=215 (Compiled frame)
-
org.eclipse.persistence.mappings.CollectionMapping.buildCloneForPartObject(java.lang.Object,
java.lang.Object, org.eclipse.persistence.internal.identitymaps.CacheKey,
java.lang.Object, org.eclipse.persistence.internal.sessions.AbstractSession,
java.lang.Integer, boolean, boolean) @bci=182, line=223 (Compiled frame)
-
org.eclipse.persistence.internal.indirection.UnitOfWorkQueryValueHolder.buildCloneFor(java.lang.Object)
@bci=41, line=60 (Compiled frame)
-
org.eclipse.persistence.internal.indirection.UnitOfWorkValueHolder.instantiateImpl()
@bci=154, line=173 (Compiled frame)
-
org.eclipse.persistence.internal.indirection.UnitOfWorkValueHolder.instantiate()
@bci=33, line=234 (Compiled frame)
- org.eclipse.persistence.internal.indirection.DatabaseValueHolder.getValue()
@bci=20, line=89 (Compiled frame)
- org.eclipse.persistence.indirection.IndirectList.buildDelegate() @bci=4,
line=271 (Compiled frame)
- org.eclipse.persistence.indirection.IndirectList.getDelegate() @bci=20,
line=455 (Compiled frame)
-
org.eclipse.persistence.internal.indirection.jdk8.IndirectList.access$000(org.eclipse.persistence.internal.indirection.jdk8.IndirectList)
@bci=1, line=32 (Compiled frame)
-
org.eclipse.persistence.internal.indirection.jdk8.IndirectList$1.<init>(org.eclipse.persistence.internal.indirection.jdk8.IndirectList,
int) @bci=19, line=57 (Compiled frame)
-
org.eclipse.persistence.internal.indirection.jdk8.IndirectList.listIterator(int)
@bci=6, line=56 (Compiled frame)
- org.eclipse.persistence.indirection.IndirectList.iterator() @bci=2, line=555
(Compiled frame)
-
org.apache.ambari.server.orm.entities.KerberosKeytabPrincipalEntity.getServiceMappingAsMultimap()
@bci=8, line=213 (Compiled frame)
-
org.apache.ambari.server.serveraction.kerberos.stageutils.KerberosKeytabController.fromPrincipalEntities(java.util.Collection)
@bci=79, line=221 (Compiled frame)
-
org.apache.ambari.server.serveraction.kerberos.stageutils.KerberosKeytabController.fromKeytabEntity(org.apache.ambari.server.orm.entities.KerberosKeytabEntity,
boolean) @bci=9, line=184 (Compiled frame)
-
org.apache.ambari.server.serveraction.kerberos.stageutils.KerberosKeytabController.fromKeytabEntity(org.apache.ambari.server.orm.entities.KerberosKeytabEntity)
@bci=3, line=198 (Compiled frame)
-
org.apache.ambari.server.serveraction.kerberos.stageutils.KerberosKeytabController.fromKeytabEntities(java.util.Collection)
@bci=35, line=204 (Compiled frame)
-
org.apache.ambari.server.serveraction.kerberos.stageutils.KerberosKeytabController.getFromPrincipal(org.apache.ambari.server.serveraction.kerberos.stageutils.ResolvedKerberosPrincipal)
@bci=16, line=94 (Compiled frame)
-
org.apache.ambari.server.serveraction.kerberos.CreateKeytabFilesServerAction.processIdentity(org.apache.ambari.server.serveraction.kerberos.stageutils.ResolvedKerberosPrincipal,
org.apache.ambari.server.serveraction.kerberos.KerberosOperationHandler,
java.util.Map, boolean, java.util.Map) @bci=86, line=161 (Compiled frame)
-
org.apache.ambari.server.serveraction.kerberos.KerberosServerAction.processIdentities(java.util.Map,
boolean) @bci=427, line=478 (Interpreted frame)
-
org.apache.ambari.server.serveraction.kerberos.KerberosServerAction.processIdentities(java.util.Map)
@bci=3, line=427 (Interpreted frame)
-
org.apache.ambari.server.serveraction.kerberos.CreateKeytabFilesServerAction.execute(java.util.concurrent.ConcurrentMap)
@bci=2, line=105 (Interpreted frame)
-
org.apache.ambari.server.serveraction.ServerActionExecutor$Worker.execute(org.apache.ambari.server.actionmanager.HostRoleCommand,
org.apache.ambari.server.agent.ExecutionCommand) @bci=424, line=550
(Interpreted frame)
- org.apache.ambari.server.serveraction.ServerActionExecutor$Worker.run()
@bci=48, line=466 (Interpreted frame)
- java.lang.Thread.run() @bci=11, line=745 (Compiled frame)}}
In our clusters, which are composed of 4000+ physical machines, adding new 5
hosts with 7 components, 11 keytabs to be created each spend about 2.5 hours.
2 hours out of 2.5 hours are spended on the above callstack.
After this patch is applied, adding new 5 hosts with the same requirements
spend only half hour as expected.
> "Creating Kerberos keytabs" takes too long
> ------------------------------------------
>
> Key: AMBARI-25624
> URL: https://issues.apache.org/jira/browse/AMBARI-25624
> Project: Ambari
> Issue Type: Bug
> Components: ambari-server
> Affects Versions: 2.7.1
> Reporter: Jeongin Ju
> Priority: Major
> Time Spent: 10m
>
> In kerberized cluster, Ambari tried to find out which keytabs are required to
> be created when adding components.
> During this process, ambari instantiates serviceMapping variable in
> KerberosKeytabPrincipalEntity class which is not used on creating keytab
> process.
> More seriously, serviceMapping variable can invoke nested variable
> instantiation and internal lock acquisitions which takes so much time on
> large cluster as shown in below callstack.
> h6. {{Thread 129281: (state = IN_JAVA)
> -
> org.eclipse.persistence.descriptors.FetchGroupManager.isPartialObject(java.lang.Object)
> @bci=8, line=466 (Compiled frame; information may be imprecise)
> -
> org.eclipse.persistence.internal.helper.WriteLockManager.traverseRelatedLocks(java.lang.Object,
> java.util.Map, java.util.Map,
> org.eclipse.persistence.descriptors.ClassDescriptor,
> org.eclipse.persistence.internal.sessions.AbstractSession) @bci=23, line=185
> (Compiled frame)
> -
> org.eclipse.persistence.internal.helper.WriteLockManager.acquireLockAndRelatedLocks(java.lang.Object,
> java.util.Map, java.util.Map,
> org.eclipse.persistence.internal.identitymaps.CacheKey,
> org.eclipse.persistence.descriptors.ClassDescriptor,
> org.eclipse.persistence.internal.sessions.AbstractSession) @bci=96, line=144
> (Compiled frame)
> -
> org.eclipse.persistence.internal.helper.WriteLockManager.checkAndLockObject(java.lang.Object,
> java.util.Map, java.util.Map,
> org.eclipse.persistence.mappings.DatabaseMapping,
> org.eclipse.persistence.internal.sessions.AbstractSession) @bci=148, line=450
> (Compiled frame)
> -
> org.eclipse.persistence.internal.helper.WriteLockManager.traverseRelatedLocks(java.lang.Object,
> java.util.Map, java.util.Map,
> org.eclipse.persistence.descriptors.ClassDescriptor,
> org.eclipse.persistence.internal.sessions.AbstractSession) @bci=237, line=213
> (Compiled frame)
> -
> org.eclipse.persistence.internal.helper.WriteLockManager.acquireLockAndRelatedLocks(java.lang.Object,
> java.util.Map, java.util.Map,
> org.eclipse.persistence.internal.identitymaps.CacheKey,
> org.eclipse.persistence.descriptors.ClassDescriptor,
> org.eclipse.persistence.internal.sessions.AbstractSession) @bci=96, line=144
> (Compiled frame)
> -
> org.eclipse.persistence.internal.helper.WriteLockManager.checkAndLockObject(java.lang.Object,
> java.util.Map, java.util.Map,
> org.eclipse.persistence.mappings.DatabaseMapping,
> org.eclipse.persistence.internal.sessions.AbstractSession) @bci=148, line=450
> (Compiled frame)
> -
> org.eclipse.persistence.internal.helper.WriteLockManager.traverseRelatedLocks(java.lang.Object,
> java.util.Map, java.util.Map,
> org.eclipse.persistence.descriptors.ClassDescriptor,
> org.eclipse.persistence.internal.sessions.AbstractSession) @bci=237, line=213
> (Compiled frame)
> -
> org.eclipse.persistence.internal.helper.WriteLockManager.acquireLockAndRelatedLocks(java.lang.Object,
> java.util.Map, java.util.Map,
> org.eclipse.persistence.internal.identitymaps.CacheKey,
> org.eclipse.persistence.descriptors.ClassDescriptor,
> org.eclipse.persistence.internal.sessions.AbstractSession) @bci=96, line=144
> (Compiled frame)
> -
> org.eclipse.persistence.internal.helper.WriteLockManager.checkAndLockObject(java.lang.Object,
> java.util.Map, java.util.Map,
> org.eclipse.persistence.mappings.DatabaseMapping,
> org.eclipse.persistence.internal.sessions.AbstractSession) @bci=148, line=450
> (Compiled frame)
> -
> org.eclipse.persistence.internal.helper.WriteLockManager.traverseRelatedLocks(java.lang.Object,
> java.util.Map, java.util.Map,
> org.eclipse.persistence.descriptors.ClassDescriptor,
> org.eclipse.persistence.internal.sessions.AbstractSession) @bci=172, line=203
> (Compiled frame)
> -
> org.eclipse.persistence.internal.helper.WriteLockManager.acquireLockAndRelatedLocks(java.lang.Object,
> java.util.Map, java.util.Map,
> org.eclipse.persistence.internal.identitymaps.CacheKey,
> org.eclipse.persistence.descriptors.ClassDescriptor,
> org.eclipse.persistence.internal.sessions.AbstractSession) @bci=96, line=144
> (Compiled frame)
> -
> org.eclipse.persistence.internal.helper.WriteLockManager.checkAndLockObject(java.lang.Object,
> java.util.Map, java.util.Map,
> org.eclipse.persistence.mappings.DatabaseMapping,
> org.eclipse.persistence.internal.sessions.AbstractSession) @bci=148, line=450
> (Compiled frame)
> -
> org.eclipse.persistence.internal.helper.WriteLockManager.traverseRelatedLocks(java.lang.Object,
> java.util.Map, java.util.Map,
> org.eclipse.persistence.descriptors.ClassDescriptor,
> org.eclipse.persistence.internal.sessions.AbstractSession) @bci=237, line=213
> (Compiled frame)
> -
> org.eclipse.persistence.internal.helper.WriteLockManager.acquireLockAndRelatedLocks(java.lang.Object,
> java.util.Map, java.util.Map,
> org.eclipse.persistence.internal.identitymaps.CacheKey,
> org.eclipse.persistence.descriptors.ClassDescriptor,
> org.eclipse.persistence.internal.sessions.AbstractSession) @bci=96, line=144
> (Compiled frame)
> -
> org.eclipse.persistence.internal.helper.WriteLockManager.checkAndLockObject(java.lang.Object,
> java.util.Map, java.util.Map,
> org.eclipse.persistence.mappings.DatabaseMapping,
> org.eclipse.persistence.internal.sessions.AbstractSession) @bci=148, line=450
> (Compiled frame)
> -
> org.eclipse.persistence.internal.helper.WriteLockManager.traverseRelatedLocks(java.lang.Object,
> java.util.Map, java.util.Map,
> org.eclipse.persistence.descriptors.ClassDescriptor,
> org.eclipse.persistence.internal.sessions.AbstractSession) @bci=237, line=213
> (Compiled frame)
> -
> org.eclipse.persistence.internal.helper.WriteLockManager.acquireLockAndRelatedLocks(java.lang.Object,
> java.util.Map, java.util.Map,
> org.eclipse.persistence.internal.identitymaps.CacheKey,
> org.eclipse.persistence.descriptors.ClassDescriptor,
> org.eclipse.persistence.internal.sessions.AbstractSession) @bci=96, line=144
> (Compiled frame)
> -
> org.eclipse.persistence.internal.helper.WriteLockManager.acquireLocksForClone(java.lang.Object,
> org.eclipse.persistence.descriptors.ClassDescriptor,
> org.eclipse.persistence.internal.identitymaps.CacheKey,
> org.eclipse.persistence.internal.sessions.AbstractSession) @bci=31, line=75
> (Compiled frame)
> -
> org.eclipse.persistence.internal.sessions.UnitOfWorkImpl.cloneAndRegisterObject(java.lang.Object,
> org.eclipse.persistence.internal.identitymaps.CacheKey,
> org.eclipse.persistence.internal.identitymaps.CacheKey,
> org.eclipse.persistence.descriptors.ClassDescriptor) @bci=140, line=997
> (Compiled frame)
> -
> org.eclipse.persistence.internal.sessions.UnitOfWorkImpl.cloneAndRegisterObject(java.lang.Object,
> org.eclipse.persistence.internal.identitymaps.CacheKey,
> org.eclipse.persistence.descriptors.ClassDescriptor) @bci=59, line=955
> (Compiled frame)
> -
> org.eclipse.persistence.internal.sessions.UnitOfWorkIdentityMapAccessor.getAndCloneCacheKeyFromParent(java.lang.Object,
> java.lang.Object, java.lang.Class, boolean,
> org.eclipse.persistence.descriptors.ClassDescriptor) @bci=288, line=209
> (Compiled frame)
> -
> org.eclipse.persistence.internal.sessions.UnitOfWorkIdentityMapAccessor.getFromIdentityMap(java.lang.Object,
> java.lang.Object, java.lang.Class, boolean,
> org.eclipse.persistence.descriptors.ClassDescriptor) @bci=28, line=137
> (Compiled frame)
> -
> org.eclipse.persistence.internal.sessions.UnitOfWorkImpl.registerExistingObject(java.lang.Object,
> org.eclipse.persistence.descriptors.ClassDescriptor, java.lang.Object,
> boolean) @bci=139, line=3940 (Compiled frame)
> -
> org.eclipse.persistence.internal.sessions.UnitOfWorkImpl.registerExistingObject(java.lang.Object,
> boolean) @bci=62, line=3894 (Compiled frame)
> -
> org.eclipse.persistence.mappings.CollectionMapping.buildElementUnitOfWorkClone(java.lang.Object,
> java.lang.Object, java.lang.Integer,
> org.eclipse.persistence.internal.sessions.UnitOfWorkImpl, boolean, boolean)
> @bci=90, line=308 (Compiled frame)
> -
> org.eclipse.persistence.mappings.CollectionMapping.buildElementClone(java.lang.Object,
> java.lang.Object, org.eclipse.persistence.internal.identitymaps.CacheKey,
> java.lang.Integer, org.eclipse.persistence.internal.sessions.AbstractSession,
> boolean, boolean) @bci=22, line=321 (Compiled frame)
> -
> org.eclipse.persistence.internal.queries.ContainerPolicy.addNextValueFromIteratorInto(java.lang.Object,
> java.lang.Object, org.eclipse.persistence.internal.identitymaps.CacheKey,
> java.lang.Object, org.eclipse.persistence.mappings.CollectionMapping,
> java.lang.Integer, org.eclipse.persistence.internal.sessions.AbstractSession,
> boolean, boolean) @bci=19, line=215 (Compiled frame)
> -
> org.eclipse.persistence.mappings.CollectionMapping.buildCloneForPartObject(java.lang.Object,
> java.lang.Object, org.eclipse.persistence.internal.identitymaps.CacheKey,
> java.lang.Object, org.eclipse.persistence.internal.sessions.AbstractSession,
> java.lang.Integer, boolean, boolean) @bci=182, line=223 (Compiled frame)
> -
> org.eclipse.persistence.internal.indirection.UnitOfWorkQueryValueHolder.buildCloneFor(java.lang.Object)
> @bci=41, line=60 (Compiled frame)
> -
> org.eclipse.persistence.internal.indirection.UnitOfWorkValueHolder.instantiateImpl()
> @bci=154, line=173 (Compiled frame)
> -
> org.eclipse.persistence.internal.indirection.UnitOfWorkValueHolder.instantiate()
> @bci=33, line=234 (Compiled frame)
> -
> org.eclipse.persistence.internal.indirection.DatabaseValueHolder.getValue()
> @bci=20, line=89 (Compiled frame)
> - org.eclipse.persistence.indirection.IndirectList.buildDelegate() @bci=4,
> line=271 (Compiled frame)
> - org.eclipse.persistence.indirection.IndirectList.getDelegate() @bci=20,
> line=455 (Compiled frame)
> -
> org.eclipse.persistence.internal.indirection.jdk8.IndirectList.access$000(org.eclipse.persistence.internal.indirection.jdk8.IndirectList)
> @bci=1, line=32 (Compiled frame)
> -
> org.eclipse.persistence.internal.indirection.jdk8.IndirectList$1.<init>(org.eclipse.persistence.internal.indirection.jdk8.IndirectList,
> int) @bci=19, line=57 (Compiled frame)
> -
> org.eclipse.persistence.internal.indirection.jdk8.IndirectList.listIterator(int)
> @bci=6, line=56 (Compiled frame)
> - org.eclipse.persistence.indirection.IndirectList.iterator() @bci=2,
> line=555 (Compiled frame)
> -
> org.apache.ambari.server.orm.entities.KerberosKeytabPrincipalEntity.getServiceMappingAsMultimap()
> @bci=8, line=213 (Compiled frame)
> -
> org.apache.ambari.server.serveraction.kerberos.stageutils.KerberosKeytabController.fromPrincipalEntities(java.util.Collection)
> @bci=79, line=221 (Compiled frame)
> -
> org.apache.ambari.server.serveraction.kerberos.stageutils.KerberosKeytabController.fromKeytabEntity(org.apache.ambari.server.orm.entities.KerberosKeytabEntity,
> boolean) @bci=9, line=184 (Compiled frame)
> -
> org.apache.ambari.server.serveraction.kerberos.stageutils.KerberosKeytabController.fromKeytabEntity(org.apache.ambari.server.orm.entities.KerberosKeytabEntity)
> @bci=3, line=198 (Compiled frame)
> -
> org.apache.ambari.server.serveraction.kerberos.stageutils.KerberosKeytabController.fromKeytabEntities(java.util.Collection)
> @bci=35, line=204 (Compiled frame)
> -
> org.apache.ambari.server.serveraction.kerberos.stageutils.KerberosKeytabController.getFromPrincipal(org.apache.ambari.server.serveraction.kerberos.stageutils.ResolvedKerberosPrincipal)
> @bci=16, line=94 (Compiled frame)
> -
> org.apache.ambari.server.serveraction.kerberos.CreateKeytabFilesServerAction.processIdentity(org.apache.ambari.server.serveraction.kerberos.stageutils.ResolvedKerberosPrincipal,
> org.apache.ambari.server.serveraction.kerberos.KerberosOperationHandler,
> java.util.Map, boolean, java.util.Map) @bci=86, line=161 (Compiled frame)
> -
> org.apache.ambari.server.serveraction.kerberos.KerberosServerAction.processIdentities(java.util.Map,
> boolean) @bci=427, line=478 (Interpreted frame)
> -
> org.apache.ambari.server.serveraction.kerberos.KerberosServerAction.processIdentities(java.util.Map)
> @bci=3, line=427 (Interpreted frame)
> -
> org.apache.ambari.server.serveraction.kerberos.CreateKeytabFilesServerAction.execute(java.util.concurrent.ConcurrentMap)
> @bci=2, line=105 (Interpreted frame)
> -
> org.apache.ambari.server.serveraction.ServerActionExecutor$Worker.execute(org.apache.ambari.server.actionmanager.HostRoleCommand,
> org.apache.ambari.server.agent.ExecutionCommand) @bci=424, line=550
> (Interpreted frame)
> - org.apache.ambari.server.serveraction.ServerActionExecutor$Worker.run()
> @bci=48, line=466 (Interpreted frame)
> - java.lang.Thread.run() @bci=11, line=745 (Compiled frame)}}
>
> In our clusters, which are composed of 4000+ physical machines, adding new 5
> hosts (with 7 components, 11 keytabs to be created each) spend about 2.5
> hours.
> 2 hours out of 2.5 hours are spended on the above callstack.
> After this patch is applied, adding new 5 hosts with the same requirements
> spend only half hour as expected.
>
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
