This is an automated email from the ASF dual-hosted git repository.
mehul pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git
commit 8121961018350014860e29951cbd5b106e8b7923
Author: Andrew <andrewluotechnolog...@outlook.com>
AuthorDate: Fri Nov 15 23:11:00 2019 -0800
RANGER-2643: Use JPA/JDBC batching during policy creation
Signed-off-by: Mehul Parikh <me...@apache.org>
---
.../org/apache/ranger/biz/PolicyRefUpdater.java | 28 ++++++++++++++++------
.../java/org/apache/ranger/biz/RangerBizUtil.java | 3 ++-
.../java/org/apache/ranger/biz/ServiceDBStore.java | 4 ++--
.../java/org/apache/ranger/common/db/BaseDao.java | 17 +++++++++++++
.../java/org/apache/ranger/rest/ServiceREST.java | 8 +++----
.../src/main/resources/META-INF/persistence.xml | 2 ++
.../conf.dist/ranger-admin-default-site.xml | 8 ++++++-
7 files changed, 55 insertions(+), 15 deletions(-)
diff --git
a/security-admin/src/main/java/org/apache/ranger/biz/PolicyRefUpdater.java
b/security-admin/src/main/java/org/apache/ranger/biz/PolicyRefUpdater.java
index 7b2356b..3846d00 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/PolicyRefUpdater.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/PolicyRefUpdater.java
@@ -113,6 +113,7 @@ public class PolicyRefUpdater {
}
}
+ List<XXPolicyRefResource> xPolResources = new ArrayList<>();
for (String resource : resourceNames) {
XXResourceDef xResDef =
daoMgr.getXXResourceDef().findByNameAndPolicyId(resource, policy.getId());
@@ -126,9 +127,11 @@ public class PolicyRefUpdater {
xPolRes.setResourceDefId(xResDef.getId());
xPolRes.setResourceName(resource);
- daoMgr.getXXPolicyRefResource().create(xPolRes);
+ xPolResources.add(xPolRes);
}
+ daoMgr.getXXPolicyRefResource().batchCreate(xPolResources);
+ List<XXPolicyRefRole> xPolRoles = new ArrayList<>();
for (String role : roleNames) {
if (StringUtils.isBlank(role)) {
continue;
@@ -146,9 +149,11 @@ public class PolicyRefUpdater {
xPolRole.setRoleId(xRole.getId());
xPolRole.setRoleName(role);
- daoMgr.getXXPolicyRefRole().create(xPolRole);
+ xPolRoles.add(xPolRole);
}
+ daoMgr.getXXPolicyRefRole().batchCreate(xPolRoles);
+ List<XXPolicyRefGroup> xPolGroups = new ArrayList<>();
for (String group : groupNames) {
if (StringUtils.isBlank(group)) {
continue;
@@ -166,9 +171,11 @@ public class PolicyRefUpdater {
xPolGroup.setGroupId(xGroup.getId());
xPolGroup.setGroupName(group);
- daoMgr.getXXPolicyRefGroup().create(xPolGroup);
+ xPolGroups.add(xPolGroup);
}
+ daoMgr.getXXPolicyRefGroup().batchCreate(xPolGroups);
+ List<XXPolicyRefUser> xPolUsers = new ArrayList<>();
for (String user : userNames) {
if (StringUtils.isBlank(user)) {
continue;
@@ -186,9 +193,11 @@ public class PolicyRefUpdater {
xPolUser.setUserId(xUser.getId());
xPolUser.setUserName(user);
- daoMgr.getXXPolicyRefUser().create(xPolUser);
+ xPolUsers.add(xPolUser);
}
+ daoMgr.getXXPolicyRefUser().batchCreate(xPolUsers);
+ List<XXPolicyRefAccessType> xPolAccesses = new ArrayList<>();
for (String accessType : accessTypes) {
XXAccessTypeDef xAccTypeDef =
daoMgr.getXXAccessTypeDef().findByNameAndServiceId(accessType,
xPolicy.getService());
@@ -202,9 +211,11 @@ public class PolicyRefUpdater {
xPolAccess.setAccessDefId(xAccTypeDef.getId());
xPolAccess.setAccessTypeName(accessType);
- daoMgr.getXXPolicyRefAccessType().create(xPolAccess);
+ xPolAccesses.add(xPolAccess);
}
+ daoMgr.getXXPolicyRefAccessType().batchCreate(xPolAccesses);
+ List<XXPolicyRefCondition> xPolConds = new ArrayList<>();
for (String condition : conditionTypes) {
XXPolicyConditionDef xPolCondDef =
daoMgr.getXXPolicyConditionDef().findByServiceDefIdAndName(xServiceDef.getId(),
condition);
@@ -218,9 +229,11 @@ public class PolicyRefUpdater {
xPolCond.setConditionDefId(xPolCondDef.getId());
xPolCond.setConditionName(condition);
- daoMgr.getXXPolicyRefCondition().create(xPolCond);
+ xPolConds.add(xPolCond);
}
+ daoMgr.getXXPolicyRefCondition().batchCreate(xPolConds);
+ List<XXPolicyRefDataMaskType> xxDataMaskInfos = new
ArrayList<>();
for (String dataMaskType : dataMaskTypes ) {
XXDataMaskTypeDef dataMaskDef =
daoMgr.getXXDataMaskTypeDef().findByNameAndServiceId(dataMaskType,
xPolicy.getService());
@@ -234,8 +247,9 @@ public class PolicyRefUpdater {
xxDataMaskInfo.setDataMaskDefId(dataMaskDef.getId());
xxDataMaskInfo.setDataMaskTypeName(dataMaskType);
-
daoMgr.getXXPolicyRefDataMaskType().create(xxDataMaskInfo);
+ xxDataMaskInfos.add(xxDataMaskInfo);
}
+
daoMgr.getXXPolicyRefDataMaskType().batchCreate(xxDataMaskInfos);
}
public Boolean cleanupRefTables(RangerPolicy policy) {
diff --git
a/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java
b/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java
index 206f6af..e8b58d1 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java
@@ -105,7 +105,8 @@ public class RangerBizUtil {
public static final String AUDIT_STORE_RDBMS = "DB";
public static final String AUDIT_STORE_SOLR = "solr";
public static final boolean batchClearEnabled =
PropertiesUtil.getBooleanProperty("ranger.jpa.jdbc.batch-clear.enable", true);
- public static final int batchSize =
PropertiesUtil.getIntProperty("ranger.jpa.jdbc.batch-clear.size", 10);
+ public static final int policyBatchSize =
PropertiesUtil.getIntProperty("ranger.jpa.jdbc.batch-clear.size", 10);
+ public static final int batchPersistSize =
PropertiesUtil.getIntProperty("ranger.jpa.jdbc.batch-persist.size", 500);
String auditDBType = AUDIT_STORE_RDBMS;
private final boolean allowUnauthenticatedAccessInSecureEnvironment;
diff --git
a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
index 0fd5093..ecb8d11 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
@@ -1768,7 +1768,7 @@ public class ServiceDBStore extends AbstractServiceStore {
deletePolicy(rangerPolicy, service);
totalDeletedPolicies = totalDeletedPolicies + 1;
// its a bulk policy delete call flush and clear
- if (totalDeletedPolicies %
RangerBizUtil.batchSize == 0) {
+ if (totalDeletedPolicies %
RangerBizUtil.policyBatchSize == 0) {
bizUtil.bulkModeOnlyFlushAndClear();
}
}
@@ -3151,7 +3151,7 @@ public class ServiceDBStore extends AbstractServiceStore {
for (RangerPolicy rangerPolicy :
rangerPolicyList) {
deletePolicy(rangerPolicy,
service);
totalDeletedPolicies =
totalDeletedPolicies + 1;
- if (totalDeletedPolicies %
RangerBizUtil.batchSize == 0) {
+ if (totalDeletedPolicies %
RangerBizUtil.policyBatchSize == 0) {
bizUtil.bulkModeOnlyFlushAndClear();
}
}
diff --git
a/security-admin/src/main/java/org/apache/ranger/common/db/BaseDao.java
b/security-admin/src/main/java/org/apache/ranger/common/db/BaseDao.java
index bdd8fbb..98837fe 100644
--- a/security-admin/src/main/java/org/apache/ranger/common/db/BaseDao.java
+++ b/security-admin/src/main/java/org/apache/ranger/common/db/BaseDao.java
@@ -93,6 +93,23 @@ public abstract class BaseDao<T> {
return ret;
}
+ public List<T> batchCreate(List<T> obj) {
+ List<T> ret = null;
+
+ for (int n = 0; n < obj.size(); ++n) {
+ em.persist(obj.get(n));
+ if (!RangerBizUtil.isBulkMode() && (n %
RangerBizUtil.policyBatchSize == 0)) {
+ em.flush();
+ }
+ }
+ if (!RangerBizUtil.isBulkMode()) {
+ em.flush();
+ }
+
+ ret = obj;
+ return ret;
+ }
+
public T update(T obj) {
em.merge(obj);
if (!RangerBizUtil.isBulkMode()) {
diff --git
a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java
b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java
index f2bbd3c..62ffee4 100644
--- a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java
+++ b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java
@@ -2477,7 +2477,7 @@ public class ServiceREST {
}
}
}
- if(totalPolicyCreate % RangerBizUtil.batchSize
== 0) {
+ if(totalPolicyCreate %
RangerBizUtil.policyBatchSize == 0) {
bizUtil.bulkModeOnlyFlushAndClear();
}
}
@@ -2748,7 +2748,7 @@ public class ServiceREST {
bizUtil.blockAuditorRoleUser();
svcStore.deletePolicy(rangerPolicy, service);
totalDeletedPolicies = totalDeletedPolicies + 1;
- if
(totalDeletedPolicies % RangerBizUtil.batchSize == 0) {
+ if
(totalDeletedPolicies % RangerBizUtil.policyBatchSize == 0) {
bizUtil.bulkModeOnlyFlushAndClear();
}
if
(LOG.isDebugEnabled()) {
@@ -2821,7 +2821,7 @@ public class ServiceREST {
LOG.debug("Policy " + rangerPolicy.getName() + " deleted successfully.");
}
totalDeletedPolicies = totalDeletedPolicies + 1;
- if
(totalDeletedPolicies % RangerBizUtil.batchSize == 0) {
+ if
(totalDeletedPolicies % RangerBizUtil.policyBatchSize == 0) {
bizUtil.bulkModeOnlyFlushAndClear();
}
}
@@ -4267,7 +4267,7 @@ public class ServiceREST {
if (existingPolicy != null) {
svcStore.deletePolicy(existingPolicy,
null);
totalDeletedPolicies =
totalDeletedPolicies + 1;
- if (totalDeletedPolicies %
RangerBizUtil.batchSize == 0) {
+ if (totalDeletedPolicies %
RangerBizUtil.policyBatchSize == 0) {
bizUtil.bulkModeOnlyFlushAndClear();
}
if (LOG.isDebugEnabled()) {
diff --git a/security-admin/src/main/resources/META-INF/persistence.xml
b/security-admin/src/main/resources/META-INF/persistence.xml
index 2117159..20f08d2 100644
--- a/security-admin/src/main/resources/META-INF/persistence.xml
+++ b/security-admin/src/main/resources/META-INF/persistence.xml
@@ -83,6 +83,7 @@
<properties>
<property name="eclipselink.logging.level"
value="WARNING"/>
+ <property name="eclipselink.jdbc.batch-writing"
value="jdbc"/>
</properties>
</persistence-unit>
<persistence-unit name="loggingPU">
@@ -92,6 +93,7 @@
<properties>
<property name="eclipselink.logging.level"
value="WARNING"/>
+ <property name="eclipselink.jdbc.batch-writing"
value="jdbc"/>
</properties>
</persistence-unit>
diff --git
a/security-admin/src/main/resources/conf.dist/ranger-admin-default-site.xml
b/security-admin/src/main/resources/conf.dist/ranger-admin-default-site.xml
index 9916297..3eb47f9 100644
--- a/security-admin/src/main/resources/conf.dist/ranger-admin-default-site.xml
+++ b/security-admin/src/main/resources/conf.dist/ranger-admin-default-site.xml
@@ -374,7 +374,13 @@
<property>
<name>ranger.jpa.jdbc.batch-clear.size</name>
<value>10</value>
- <description>batch size to flush and clear jdbc
statements</description>
+ <description>batch size (in number of policies) to flush and
clear jdbc statements during batch policy import/delete</description>
+ </property>
+
+ <property>
+ <name>ranger.jpa.jdbc.batch-persist.size</name>
+ <value>500</value>
+ <description>batch size (in number of objects) to flush and
clear jdbc statements during jpa persistence</description>
</property>
<property>
