This is an automated email from the ASF dual-hosted git repository. mehul pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/ranger.git
commit 8121961018350014860e29951cbd5b106e8b7923 Author: Andrew <andrewluotechnolog...@outlook.com> AuthorDate: Fri Nov 15 23:11:00 2019 -0800 RANGER-2643: Use JPA/JDBC batching during policy creation Signed-off-by: Mehul Parikh <me...@apache.org> --- .../org/apache/ranger/biz/PolicyRefUpdater.java | 28 ++++++++++++++++------ .../java/org/apache/ranger/biz/RangerBizUtil.java | 3 ++- .../java/org/apache/ranger/biz/ServiceDBStore.java | 4 ++-- .../java/org/apache/ranger/common/db/BaseDao.java | 17 +++++++++++++ .../java/org/apache/ranger/rest/ServiceREST.java | 8 +++---- .../src/main/resources/META-INF/persistence.xml | 2 ++ .../conf.dist/ranger-admin-default-site.xml | 8 ++++++- 7 files changed, 55 insertions(+), 15 deletions(-) diff --git a/security-admin/src/main/java/org/apache/ranger/biz/PolicyRefUpdater.java b/security-admin/src/main/java/org/apache/ranger/biz/PolicyRefUpdater.java index 7b2356b..3846d00 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/PolicyRefUpdater.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/PolicyRefUpdater.java @@ -113,6 +113,7 @@ public class PolicyRefUpdater { } } + List<XXPolicyRefResource> xPolResources = new ArrayList<>(); for (String resource : resourceNames) { XXResourceDef xResDef = daoMgr.getXXResourceDef().findByNameAndPolicyId(resource, policy.getId()); @@ -126,9 +127,11 @@ public class PolicyRefUpdater { xPolRes.setResourceDefId(xResDef.getId()); xPolRes.setResourceName(resource); - daoMgr.getXXPolicyRefResource().create(xPolRes); + xPolResources.add(xPolRes); } + daoMgr.getXXPolicyRefResource().batchCreate(xPolResources); + List<XXPolicyRefRole> xPolRoles = new ArrayList<>(); for (String role : roleNames) { if (StringUtils.isBlank(role)) { continue; @@ -146,9 +149,11 @@ public class PolicyRefUpdater { xPolRole.setRoleId(xRole.getId()); xPolRole.setRoleName(role); - daoMgr.getXXPolicyRefRole().create(xPolRole); + xPolRoles.add(xPolRole); } + daoMgr.getXXPolicyRefRole().batchCreate(xPolRoles); + List<XXPolicyRefGroup> xPolGroups = new ArrayList<>(); for (String group : groupNames) { if (StringUtils.isBlank(group)) { continue; @@ -166,9 +171,11 @@ public class PolicyRefUpdater { xPolGroup.setGroupId(xGroup.getId()); xPolGroup.setGroupName(group); - daoMgr.getXXPolicyRefGroup().create(xPolGroup); + xPolGroups.add(xPolGroup); } + daoMgr.getXXPolicyRefGroup().batchCreate(xPolGroups); + List<XXPolicyRefUser> xPolUsers = new ArrayList<>(); for (String user : userNames) { if (StringUtils.isBlank(user)) { continue; @@ -186,9 +193,11 @@ public class PolicyRefUpdater { xPolUser.setUserId(xUser.getId()); xPolUser.setUserName(user); - daoMgr.getXXPolicyRefUser().create(xPolUser); + xPolUsers.add(xPolUser); } + daoMgr.getXXPolicyRefUser().batchCreate(xPolUsers); + List<XXPolicyRefAccessType> xPolAccesses = new ArrayList<>(); for (String accessType : accessTypes) { XXAccessTypeDef xAccTypeDef = daoMgr.getXXAccessTypeDef().findByNameAndServiceId(accessType, xPolicy.getService()); @@ -202,9 +211,11 @@ public class PolicyRefUpdater { xPolAccess.setAccessDefId(xAccTypeDef.getId()); xPolAccess.setAccessTypeName(accessType); - daoMgr.getXXPolicyRefAccessType().create(xPolAccess); + xPolAccesses.add(xPolAccess); } + daoMgr.getXXPolicyRefAccessType().batchCreate(xPolAccesses); + List<XXPolicyRefCondition> xPolConds = new ArrayList<>(); for (String condition : conditionTypes) { XXPolicyConditionDef xPolCondDef = daoMgr.getXXPolicyConditionDef().findByServiceDefIdAndName(xServiceDef.getId(), condition); @@ -218,9 +229,11 @@ public class PolicyRefUpdater { xPolCond.setConditionDefId(xPolCondDef.getId()); xPolCond.setConditionName(condition); - daoMgr.getXXPolicyRefCondition().create(xPolCond); + xPolConds.add(xPolCond); } + daoMgr.getXXPolicyRefCondition().batchCreate(xPolConds); + List<XXPolicyRefDataMaskType> xxDataMaskInfos = new ArrayList<>(); for (String dataMaskType : dataMaskTypes ) { XXDataMaskTypeDef dataMaskDef = daoMgr.getXXDataMaskTypeDef().findByNameAndServiceId(dataMaskType, xPolicy.getService()); @@ -234,8 +247,9 @@ public class PolicyRefUpdater { xxDataMaskInfo.setDataMaskDefId(dataMaskDef.getId()); xxDataMaskInfo.setDataMaskTypeName(dataMaskType); - daoMgr.getXXPolicyRefDataMaskType().create(xxDataMaskInfo); + xxDataMaskInfos.add(xxDataMaskInfo); } + daoMgr.getXXPolicyRefDataMaskType().batchCreate(xxDataMaskInfos); } public Boolean cleanupRefTables(RangerPolicy policy) { diff --git a/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java b/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java index 206f6af..e8b58d1 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java @@ -105,7 +105,8 @@ public class RangerBizUtil { public static final String AUDIT_STORE_RDBMS = "DB"; public static final String AUDIT_STORE_SOLR = "solr"; public static final boolean batchClearEnabled = PropertiesUtil.getBooleanProperty("ranger.jpa.jdbc.batch-clear.enable", true); - public static final int batchSize = PropertiesUtil.getIntProperty("ranger.jpa.jdbc.batch-clear.size", 10); + public static final int policyBatchSize = PropertiesUtil.getIntProperty("ranger.jpa.jdbc.batch-clear.size", 10); + public static final int batchPersistSize = PropertiesUtil.getIntProperty("ranger.jpa.jdbc.batch-persist.size", 500); String auditDBType = AUDIT_STORE_RDBMS; private final boolean allowUnauthenticatedAccessInSecureEnvironment; diff --git a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java index 0fd5093..ecb8d11 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java @@ -1768,7 +1768,7 @@ public class ServiceDBStore extends AbstractServiceStore { deletePolicy(rangerPolicy, service); totalDeletedPolicies = totalDeletedPolicies + 1; // its a bulk policy delete call flush and clear - if (totalDeletedPolicies % RangerBizUtil.batchSize == 0) { + if (totalDeletedPolicies % RangerBizUtil.policyBatchSize == 0) { bizUtil.bulkModeOnlyFlushAndClear(); } } @@ -3151,7 +3151,7 @@ public class ServiceDBStore extends AbstractServiceStore { for (RangerPolicy rangerPolicy : rangerPolicyList) { deletePolicy(rangerPolicy, service); totalDeletedPolicies = totalDeletedPolicies + 1; - if (totalDeletedPolicies % RangerBizUtil.batchSize == 0) { + if (totalDeletedPolicies % RangerBizUtil.policyBatchSize == 0) { bizUtil.bulkModeOnlyFlushAndClear(); } } diff --git a/security-admin/src/main/java/org/apache/ranger/common/db/BaseDao.java b/security-admin/src/main/java/org/apache/ranger/common/db/BaseDao.java index bdd8fbb..98837fe 100644 --- a/security-admin/src/main/java/org/apache/ranger/common/db/BaseDao.java +++ b/security-admin/src/main/java/org/apache/ranger/common/db/BaseDao.java @@ -93,6 +93,23 @@ public abstract class BaseDao<T> { return ret; } + public List<T> batchCreate(List<T> obj) { + List<T> ret = null; + + for (int n = 0; n < obj.size(); ++n) { + em.persist(obj.get(n)); + if (!RangerBizUtil.isBulkMode() && (n % RangerBizUtil.policyBatchSize == 0)) { + em.flush(); + } + } + if (!RangerBizUtil.isBulkMode()) { + em.flush(); + } + + ret = obj; + return ret; + } + public T update(T obj) { em.merge(obj); if (!RangerBizUtil.isBulkMode()) { diff --git a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java index f2bbd3c..62ffee4 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java @@ -2477,7 +2477,7 @@ public class ServiceREST { } } } - if(totalPolicyCreate % RangerBizUtil.batchSize == 0) { + if(totalPolicyCreate % RangerBizUtil.policyBatchSize == 0) { bizUtil.bulkModeOnlyFlushAndClear(); } } @@ -2748,7 +2748,7 @@ public class ServiceREST { bizUtil.blockAuditorRoleUser(); svcStore.deletePolicy(rangerPolicy, service); totalDeletedPolicies = totalDeletedPolicies + 1; - if (totalDeletedPolicies % RangerBizUtil.batchSize == 0) { + if (totalDeletedPolicies % RangerBizUtil.policyBatchSize == 0) { bizUtil.bulkModeOnlyFlushAndClear(); } if (LOG.isDebugEnabled()) { @@ -2821,7 +2821,7 @@ public class ServiceREST { LOG.debug("Policy " + rangerPolicy.getName() + " deleted successfully."); } totalDeletedPolicies = totalDeletedPolicies + 1; - if (totalDeletedPolicies % RangerBizUtil.batchSize == 0) { + if (totalDeletedPolicies % RangerBizUtil.policyBatchSize == 0) { bizUtil.bulkModeOnlyFlushAndClear(); } } @@ -4267,7 +4267,7 @@ public class ServiceREST { if (existingPolicy != null) { svcStore.deletePolicy(existingPolicy, null); totalDeletedPolicies = totalDeletedPolicies + 1; - if (totalDeletedPolicies % RangerBizUtil.batchSize == 0) { + if (totalDeletedPolicies % RangerBizUtil.policyBatchSize == 0) { bizUtil.bulkModeOnlyFlushAndClear(); } if (LOG.isDebugEnabled()) { diff --git a/security-admin/src/main/resources/META-INF/persistence.xml b/security-admin/src/main/resources/META-INF/persistence.xml index 2117159..20f08d2 100644 --- a/security-admin/src/main/resources/META-INF/persistence.xml +++ b/security-admin/src/main/resources/META-INF/persistence.xml @@ -83,6 +83,7 @@ <properties> <property name="eclipselink.logging.level" value="WARNING"/> + <property name="eclipselink.jdbc.batch-writing" value="jdbc"/> </properties> </persistence-unit> <persistence-unit name="loggingPU"> @@ -92,6 +93,7 @@ <properties> <property name="eclipselink.logging.level" value="WARNING"/> + <property name="eclipselink.jdbc.batch-writing" value="jdbc"/> </properties> </persistence-unit> diff --git a/security-admin/src/main/resources/conf.dist/ranger-admin-default-site.xml b/security-admin/src/main/resources/conf.dist/ranger-admin-default-site.xml index 9916297..3eb47f9 100644 --- a/security-admin/src/main/resources/conf.dist/ranger-admin-default-site.xml +++ b/security-admin/src/main/resources/conf.dist/ranger-admin-default-site.xml @@ -374,7 +374,13 @@ <property> <name>ranger.jpa.jdbc.batch-clear.size</name> <value>10</value> - <description>batch size to flush and clear jdbc statements</description> + <description>batch size (in number of policies) to flush and clear jdbc statements during batch policy import/delete</description> + </property> + + <property> + <name>ranger.jpa.jdbc.batch-persist.size</name> + <value>500</value> + <description>batch size (in number of objects) to flush and clear jdbc statements during jpa persistence</description> </property> <property>